{ pkgs, serverPkg, clientPkg }: { name = "lightscale-lab-router"; nodes = { # Control plane server node1 = { ... }: { networking.hostName = "node1"; networking.usePredictableInterfaceNames = false; virtualisation.vlans = [ 1 ]; networking.interfaces.eth1.useDHCP = false; networking.interfaces.eth1.ipv4.addresses = [ { address = "10.0.0.1"; prefixLength = 24; } ]; networking.firewall.enable = false; boot.kernelModules = [ "wireguard" ]; environment.systemPackages = [ serverPkg clientPkg pkgs.wireguard-tools pkgs.iproute2 pkgs.iputils pkgs.netcat-openbsd pkgs.curl pkgs.nftables ]; }; # Subnet router node router = { ... }: { networking.hostName = "router"; networking.usePredictableInterfaceNames = false; virtualisation.vlans = [ 1 2 ]; # eth1: control plane, eth2: LAN networking.interfaces.eth1.useDHCP = false; networking.interfaces.eth1.ipv4.addresses = [ { address = "10.0.0.10"; prefixLength = 24; } ]; networking.interfaces.eth2.useDHCP = false; networking.interfaces.eth2.ipv4.addresses = [ { address = "192.168.100.1"; prefixLength = 24; } ]; networking.firewall.enable = false; boot.kernelModules = [ "wireguard" ]; environment.systemPackages = [ clientPkg pkgs.wireguard-tools pkgs.iproute2 pkgs.iputils pkgs.netcat-openbsd pkgs.curl pkgs.nftables ]; }; # Overlay client client = { ... }: { networking.hostName = "client"; networking.usePredictableInterfaceNames = false; virtualisation.vlans = [ 1 ]; networking.interfaces.eth1.useDHCP = false; networking.interfaces.eth1.ipv4.addresses = [ { address = "10.0.0.20"; prefixLength = 24; } ]; networking.firewall.enable = false; boot.kernelModules = [ "wireguard" ]; environment.systemPackages = [ clientPkg pkgs.wireguard-tools pkgs.iproute2 pkgs.iputils pkgs.netcat-openbsd pkgs.curl ]; }; }; testScript = '' start_all() node1.wait_for_unit("multi-user.target") router.wait_for_unit("multi-user.target") client.wait_for_unit("multi-user.target") # Verify network connectivity node1.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.1/24'") router.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.10/24'") router.wait_until_succeeds("ip -4 addr show dev eth2 | grep -q '192.168.100.1/24'") client.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.20/24'") # Start control plane node1.succeed("touch /tmp/lightscale-server.log") node1.execute("sh -c 'tail -n +1 -f /tmp/lightscale-server.log >/dev/console 2>&1 &'") node1.succeed( "systemd-run --no-block --unit=lightscale-server --service-type=simple " "--property=Restart=on-failure --property=RestartSec=1 " "--property=TimeoutStartSec=30 " "--property=StandardOutput=append:/tmp/lightscale-server.log " "--property=StandardError=append:/tmp/lightscale-server.log " "--setenv=RUST_LOG=info -- " "lightscale-server --listen 10.0.0.1:8080 --state /tmp/lightscale-state.json --admin-token test-token-12345" ) node1.wait_for_unit("lightscale-server.service") node1.wait_for_open_port(8080, addr="10.0.0.1", timeout=120) import json import time # Create network net = json.loads(node1.succeed( "curl -sSf -X POST http://10.0.0.1:8080/v1/networks " "-H 'authorization: Bearer test-token-12345' " "-H 'content-type: application/json' " "-d '{\"name\":\"lab\",\"bootstrap_token_ttl_seconds\":600," "\"bootstrap_token_uses\":10,\"bootstrap_token_tags\":[\"lab\"]}'" )) token = net["bootstrap_token"]["token"] def enroll(node, name, ip, routes=None): node.succeed( "lightscale-client --profile test --config /tmp/ls-config.json " "init http://10.0.0.1:8080" ) cmd = ( f"lightscale-client --profile test --config /tmp/ls-config.json " f"--state-dir /tmp/ls-state register --node-name {name} -- {token}" ) node.succeed(cmd) # Build heartbeat command hb_cmd = ( f"lightscale-client --profile test --config /tmp/ls-config.json " f"--state-dir /tmp/ls-state heartbeat --endpoint {ip}:51820" ) if routes: for route in routes: hb_cmd += f" --route {route}" node.succeed(hb_cmd) # Enroll router with subnet route (SNAT enabled) enroll(router, "router", "10.0.0.10", routes=["192.168.100.0/24"]) # Enroll client enroll(client, "client", "10.0.0.20") # Start agents def start_agent(node, endpoints): node.succeed("touch /tmp/lightscale-agent.log") cmd = ( "lightscale-client --profile test --config /tmp/ls-config.json " "--state-dir /tmp/ls-state agent --listen-port 51820 " "--heartbeat-interval 5 --longpoll-timeout 5" ) for endpoint in endpoints: cmd += f" --endpoint {endpoint}" node.succeed( "systemd-run --no-block --unit=lightscale-agent --service-type=simple " "--property=Restart=on-failure --property=RestartSec=1 " "--property=TimeoutStartSec=30 " "--property=StandardOutput=append:/tmp/lightscale-agent.log " "--property=StandardError=append:/tmp/lightscale-agent.log -- " + cmd ) node.wait_for_unit("lightscale-agent.service") node.wait_until_succeeds("ip link show ls-test", timeout=60) start_agent(router, ["10.0.0.10:51820"]) start_agent(client, ["10.0.0.20:51820"]) time.sleep(2) # Get overlay IPs def overlay_ipv4(node): data = json.loads(node.succeed("cat /tmp/ls-state/state.json")) return data["ipv4"] router_overlay_ip = overlay_ipv4(router) client_overlay_ip = overlay_ipv4(client) # Verify direct overlay connectivity router.succeed(f"ping -c 3 {client_overlay_ip}") client.succeed(f"ping -c 3 {router_overlay_ip}") # ===== Test 1: Enable subnet router with SNAT ===== router.succeed("touch /tmp/router-enable.log") router.succeed( "lightscale-client --profile test --config /tmp/ls-config.json " "--state-dir /tmp/ls-state router enable " "--interface ls-test --out-interface eth2 2>&1 | tee /tmp/router-enable.log" ) # Verify nftables rules were created via libnftnl (not via nft CLI) # Check that the lightscale table exists router.succeed("nft list table inet lightscale") router.succeed("nft list table ip lightscale-nat") # Show the rules for debugging router.succeed("nft list ruleset > /tmp/nft-ruleset.txt") router.succeed("cat /tmp/nft-ruleset.txt") # Verify forwarding chain exists with our rules router.succeed("nft list chain inet lightscale ls-forward") router.succeed("nft list chain ip lightscale-nat ls-postrouting") # Check that masquerade rule exists (SNAT enabled) router.succeed("nft list chain ip lightscale-nat ls-postrouting | grep -q masquerade") # Verify forwarding rules are correct # Rule: iifname ls-test oifname eth2 accept router.succeed("nft list chain inet lightscale ls-forward | grep -q 'iifname.*ls-test'") router.succeed("nft list chain inet lightscale ls-forward | grep -q 'oifname.*eth2'") # Rule: iifname eth2 oifname ls-test ct state established,related accept router.succeed("nft list chain inet lightscale ls-forward | grep -q 'iifname.*eth2'") router.succeed("nft list chain inet lightscale ls-forward | grep -q 'oifname.*ls-test'") # Verify sysctl settings for forwarding router.succeed("cat /proc/sys/net/ipv4/ip_forward | grep -q 1") router.succeed("cat /proc/sys/net/ipv6/conf/all/forwarding | grep -q 1") # ===== Test 2: Disable router and verify cleanup ===== router.succeed( "lightscale-client --profile test --config /tmp/ls-config.json " "--state-dir /tmp/ls-state router disable " "--interface ls-test --out-interface eth2" ) # Verify tables are cleaned up router.fail("nft list table inet lightscale 2>/dev/null") router.fail("nft list table ip lightscale-nat 2>/dev/null") # ===== Test 3: Re-enable with --no-snat ===== router.succeed( "lightscale-client --profile test --config /tmp/ls-config.json " "--state-dir /tmp/ls-state router enable " "--interface ls-test --out-interface eth2 --no-snat" ) # Verify the filter table exists router.succeed("nft list table inet lightscale") # Show ruleset for debugging (no NAT table or no masquerade) router.succeed("nft list ruleset > /tmp/nft-ruleset-no-snat.txt") router.succeed("cat /tmp/nft-ruleset-no-snat.txt") # Verify no masquerade rule (NAT table should not exist with --no-snat) router.fail("nft list table ip lightscale-nat 2>/dev/null") # ===== Test 4: Re-disable and final cleanup verification ===== router.succeed( "lightscale-client --profile test --config /tmp/ls-config.json " "--state-dir /tmp/ls-state router disable " "--interface ls-test --out-interface eth2" ) # Final cleanup verification router.fail("nft list table inet lightscale 2>/dev/null") router.fail("nft list table ip lightscale-nat 2>/dev/null") print("SUCCESS: libnftnl-based nftables operations work correctly!") print("- SNAT mode: inet lightscale + ip lightscale-nat tables created with masquerade") print("- No-SNAT mode: only inet lightscale table created") print("- Cleanup: all tables properly removed on disable") ''; }