{ pkgs, serverPkg, clientPkg }: { name = "lightscale-lab"; nodes = { node1 = { ... }: { networking.hostName = "node1"; networking.usePredictableInterfaceNames = false; virtualisation.vlans = [ 1 ]; networking.interfaces.eth1.useDHCP = false; networking.interfaces.eth1.ipv4.addresses = [ { address = "10.0.0.1"; prefixLength = 24; } ]; networking.firewall.enable = false; boot.kernelModules = [ "wireguard" ]; environment.systemPackages = [ serverPkg clientPkg pkgs.wireguard-tools pkgs.iproute2 pkgs.iputils pkgs.netcat-openbsd pkgs.curl ]; }; node2 = { ... }: { networking.hostName = "node2"; networking.usePredictableInterfaceNames = false; virtualisation.vlans = [ 1 ]; networking.interfaces.eth1.useDHCP = false; networking.interfaces.eth1.ipv4.addresses = [ { address = "10.0.0.2"; prefixLength = 24; } ]; networking.firewall.enable = false; boot.kernelModules = [ "wireguard" ]; environment.systemPackages = [ clientPkg pkgs.wireguard-tools pkgs.iproute2 pkgs.iputils pkgs.netcat-openbsd pkgs.curl ]; }; node3 = { ... }: { networking.hostName = "node3"; networking.usePredictableInterfaceNames = false; virtualisation.vlans = [ 1 ]; networking.interfaces.eth1.useDHCP = false; networking.interfaces.eth1.ipv4.addresses = [ { address = "10.0.0.3"; prefixLength = 24; } ]; networking.firewall.enable = false; boot.kernelModules = [ "wireguard" ]; environment.systemPackages = [ clientPkg pkgs.wireguard-tools pkgs.iproute2 pkgs.iputils pkgs.netcat-openbsd pkgs.curl ]; }; node4 = { ... }: { networking.hostName = "node4"; networking.usePredictableInterfaceNames = false; virtualisation.vlans = [ 1 ]; networking.interfaces.eth1.useDHCP = false; networking.interfaces.eth1.ipv4.addresses = [ { address = "10.0.0.4"; prefixLength = 24; } ]; networking.firewall.enable = false; boot.kernelModules = [ "wireguard" ]; environment.systemPackages = [ clientPkg pkgs.wireguard-tools pkgs.iproute2 pkgs.iputils pkgs.netcat-openbsd pkgs.curl ]; }; node5 = { ... }: { networking.hostName = "node5"; networking.usePredictableInterfaceNames = false; virtualisation.vlans = [ 1 ]; networking.interfaces.eth1.useDHCP = false; networking.interfaces.eth1.ipv4.addresses = [ { address = "10.0.0.5"; prefixLength = 24; } ]; networking.firewall.enable = false; boot.kernelModules = [ "wireguard" ]; environment.systemPackages = [ clientPkg pkgs.wireguard-tools pkgs.iproute2 pkgs.iputils pkgs.netcat-openbsd pkgs.curl ]; }; }; testScript = '' start_all() node1.wait_for_unit("multi-user.target") node2.wait_for_unit("multi-user.target") node3.wait_for_unit("multi-user.target") node4.wait_for_unit("multi-user.target") node5.wait_for_unit("multi-user.target") node1.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.1/24'") node2.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.2/24'") node3.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.3/24'") node4.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.4/24'") node5.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.5/24'") node1.succeed("touch /tmp/lightscale-server.log") node1.execute("sh -c 'tail -n +1 -f /tmp/lightscale-server.log >/dev/console 2>&1 &'") node1.succeed( "systemd-run --no-block --unit=lightscale-server --service-type=simple " "--property=Restart=on-failure --property=RestartSec=1 " "--property=TimeoutStartSec=30 " "--property=StandardOutput=append:/tmp/lightscale-server.log " "--property=StandardError=append:/tmp/lightscale-server.log " "--setenv=RUST_LOG=info -- " "lightscale-server --listen 10.0.0.1:8080 --state /tmp/lightscale-state.json " "--stream-relay 10.0.0.1:8443 --stream-relay-listen 10.0.0.1:8443" ) node1.wait_for_unit("lightscale-server.service") node1.wait_for_open_port(8080, addr="10.0.0.1", timeout=120) node1.wait_for_open_port(8443, addr="10.0.0.1", timeout=120) node1.succeed("tail -n 50 /tmp/lightscale-server.log || true") import json import time net = json.loads(node1.succeed( "curl -sSf -X POST http://10.0.0.1:8080/v1/networks " "-H 'content-type: application/json' " "-d '{\"name\":\"lab\",\"bootstrap_token_ttl_seconds\":600," \ "\"bootstrap_token_uses\":10,\"bootstrap_token_tags\":[\"lab\"]}'" )) token = net["bootstrap_token"]["token"] def enroll(node, name, ip): node.succeed( "lightscale-client --profile test --config /tmp/ls-config.json " "init http://10.0.0.1:8080" ) node.succeed( f"lightscale-client --profile test --config /tmp/ls-config.json " f"--state-dir /tmp/ls-state register --node-name {name} -- {token}" ) node.succeed( f"lightscale-client --profile test --config /tmp/ls-config.json " f"--state-dir /tmp/ls-state heartbeat --endpoint {ip}:51820" ) enroll(node1, "node1", "10.0.0.1") enroll(node2, "node2", "10.0.0.2") enroll(node3, "node3", "10.0.0.3") enroll(node4, "node4", "10.0.0.4") enroll(node5, "node5", "10.0.0.5") def start_agent(node, endpoints, stream_relay): node.succeed("touch /tmp/lightscale-agent.log") cmd = ( "lightscale-client --profile test --config /tmp/ls-config.json " "--state-dir /tmp/ls-state agent --listen-port 51820 " "--heartbeat-interval 5 --longpoll-timeout 5" ) for endpoint in endpoints: cmd += f" --endpoint {endpoint}" if stream_relay: cmd += " --stream-relay" node.succeed( "systemd-run --no-block --unit=lightscale-agent --service-type=simple " "--property=Restart=on-failure --property=RestartSec=1 " "--property=TimeoutStartSec=30 " "--property=StandardOutput=append:/tmp/lightscale-agent.log " "--property=StandardError=append:/tmp/lightscale-agent.log -- " + cmd ) node.wait_for_unit("lightscale-agent.service") node.wait_until_succeeds("ip link show ls-test", timeout=60) # Phase 1: direct endpoint rotation (invalid -> valid) start_agent(node1, ["203.0.113.1:51820", "10.0.0.1:51820"], False) start_agent(node2, ["203.0.113.2:51820", "10.0.0.2:51820"], False) start_agent(node3, ["203.0.113.3:51820", "10.0.0.3:51820"], False) start_agent(node4, ["203.0.113.4:51820", "10.0.0.4:51820"], False) start_agent(node5, ["203.0.113.5:51820", "10.0.0.5:51820"], False) time.sleep(1) def overlay_ipv4(node): data = json.loads(node.succeed("cat /tmp/ls-state/state.json")) return data["ipv4"] ip1 = overlay_ipv4(node1) ip2 = overlay_ipv4(node2) ip3 = overlay_ipv4(node3) ip4 = overlay_ipv4(node4) ip5 = overlay_ipv4(node5) nodes = [node1, node2, node3, node4, node5] ips = [ip1, ip2, ip3, ip4, ip5] def full_mesh_ping(nodes, ips): for i, src in enumerate(nodes): for j, dst in enumerate(nodes): if i == j: continue src.wait_until_succeeds(f"ping -c 3 {ips[j]}", timeout=120) full_mesh_ping(nodes, ips) # Phase 2: stream relay fallback (invalid only) for node in (node1, node2, node3, node4, node5): node.succeed("systemctl stop lightscale-agent.service") start_agent(node1, ["203.0.113.1:51820"], True) start_agent(node2, ["203.0.113.2:51820"], True) start_agent(node3, ["203.0.113.3:51820"], True) start_agent(node4, ["203.0.113.4:51820"], True) start_agent(node5, ["203.0.113.5:51820"], True) full_mesh_ping(nodes, ips) ''; }