lightscale/lab/test-scale.nix

{ pkgs, serverPkg, clientPkg }:
{
  name = "lightscale-lab-scale";
  nodes = {
    node1 = { ... }: {
      networking.hostName = "node1";
      networking.usePredictableInterfaceNames = false;
      virtualisation.vlans = [ 1 ];
      networking.interfaces.eth1.useDHCP = false;
      networking.interfaces.eth1.ipv4.addresses = [
        { address = "10.0.0.1"; prefixLength = 24; }
      ];
      networking.firewall.enable = false;
      boot.kernelModules = [ "wireguard" ];
      environment.systemPackages = [
        serverPkg
        clientPkg
        pkgs.wireguard-tools
        pkgs.iproute2
        pkgs.iputils
        pkgs.netcat-openbsd
        pkgs.curl
      ];
    };
    node2 = { ... }: {
      networking.hostName = "node2";
      networking.usePredictableInterfaceNames = false;
      virtualisation.vlans = [ 1 ];
      networking.interfaces.eth1.useDHCP = false;
      networking.interfaces.eth1.ipv4.addresses = [
        { address = "10.0.0.2"; prefixLength = 24; }
      ];
      networking.firewall.enable = false;
      boot.kernelModules = [ "wireguard" ];
      environment.systemPackages = [
        clientPkg
        pkgs.wireguard-tools
        pkgs.iproute2
        pkgs.iputils
        pkgs.netcat-openbsd
        pkgs.curl
      ];
    };
    node3 = { ... }: {
      networking.hostName = "node3";
      networking.usePredictableInterfaceNames = false;
      virtualisation.vlans = [ 1 ];
      networking.interfaces.eth1.useDHCP = false;
      networking.interfaces.eth1.ipv4.addresses = [
        { address = "10.0.0.3"; prefixLength = 24; }
      ];
      networking.firewall.enable = false;
      boot.kernelModules = [ "wireguard" ];
      environment.systemPackages = [
        clientPkg
        pkgs.wireguard-tools
        pkgs.iproute2
        pkgs.iputils
        pkgs.netcat-openbsd
        pkgs.curl
      ];
    };
    node4 = { ... }: {
      networking.hostName = "node4";
      networking.usePredictableInterfaceNames = false;
      virtualisation.vlans = [ 1 ];
      networking.interfaces.eth1.useDHCP = false;
      networking.interfaces.eth1.ipv4.addresses = [
        { address = "10.0.0.4"; prefixLength = 24; }
      ];
      networking.firewall.enable = false;
      boot.kernelModules = [ "wireguard" ];
      environment.systemPackages = [
        clientPkg
        pkgs.wireguard-tools
        pkgs.iproute2
        pkgs.iputils
        pkgs.netcat-openbsd
        pkgs.curl
      ];
    };
    node5 = { ... }: {
      networking.hostName = "node5";
      networking.usePredictableInterfaceNames = false;
      virtualisation.vlans = [ 1 ];
      networking.interfaces.eth1.useDHCP = false;
      networking.interfaces.eth1.ipv4.addresses = [
        { address = "10.0.0.5"; prefixLength = 24; }
      ];
      networking.firewall.enable = false;
      boot.kernelModules = [ "wireguard" ];
      environment.systemPackages = [
        clientPkg
        pkgs.wireguard-tools
        pkgs.iproute2
        pkgs.iputils
        pkgs.netcat-openbsd
        pkgs.curl
      ];
    };
    node6 = { ... }: {
      networking.hostName = "node6";
      networking.usePredictableInterfaceNames = false;
      virtualisation.vlans = [ 1 ];
      networking.interfaces.eth1.useDHCP = false;
      networking.interfaces.eth1.ipv4.addresses = [
        { address = "10.0.0.6"; prefixLength = 24; }
      ];
      networking.firewall.enable = false;
      boot.kernelModules = [ "wireguard" ];
      environment.systemPackages = [
        clientPkg
        pkgs.wireguard-tools
        pkgs.iproute2
        pkgs.iputils
        pkgs.netcat-openbsd
        pkgs.curl
      ];
    };
    node7 = { ... }: {
      networking.hostName = "node7";
      networking.usePredictableInterfaceNames = false;
      virtualisation.vlans = [ 1 ];
      networking.interfaces.eth1.useDHCP = false;
      networking.interfaces.eth1.ipv4.addresses = [
        { address = "10.0.0.7"; prefixLength = 24; }
      ];
      networking.firewall.enable = false;
      boot.kernelModules = [ "wireguard" ];
      environment.systemPackages = [
        clientPkg
        pkgs.wireguard-tools
        pkgs.iproute2
        pkgs.iputils
        pkgs.netcat-openbsd
        pkgs.curl
      ];
    };
    node8 = { ... }: {
      networking.hostName = "node8";
      networking.usePredictableInterfaceNames = false;
      virtualisation.vlans = [ 1 ];
      networking.interfaces.eth1.useDHCP = false;
      networking.interfaces.eth1.ipv4.addresses = [
        { address = "10.0.0.8"; prefixLength = 24; }
      ];
      networking.firewall.enable = false;
      boot.kernelModules = [ "wireguard" ];
      environment.systemPackages = [
        clientPkg
        pkgs.wireguard-tools
        pkgs.iproute2
        pkgs.iputils
        pkgs.netcat-openbsd
        pkgs.curl
      ];
    };
  };

  testScript = ''
    start_all()

    nodes = [node1, node2, node3, node4, node5, node6, node7, node8]
    for idx, node in enumerate(nodes, start=1):
        node.wait_for_unit("multi-user.target")
        node.wait_until_succeeds(
            f"ip -4 addr show dev eth1 | grep -q '10.0.0.{idx}/24'"
        )

    node1.succeed("touch /tmp/lightscale-server.log")
    node1.execute("sh -c 'tail -n +1 -f /tmp/lightscale-server.log >/dev/console 2>&1 &'")
    node1.succeed(
        "systemd-run --no-block --unit=lightscale-server --service-type=simple "
        "--property=Restart=on-failure --property=RestartSec=1 "
        "--property=TimeoutStartSec=30 "
        "--property=StandardOutput=append:/tmp/lightscale-server.log "
        "--property=StandardError=append:/tmp/lightscale-server.log "
        "--setenv=RUST_LOG=info -- "
        "lightscale-server --listen 10.0.0.1:8080 --state /tmp/lightscale-state.json "
        "--stream-relay 10.0.0.1:8443 --stream-relay-listen 10.0.0.1:8443"
    )
    node1.wait_for_unit("lightscale-server.service")
    node1.wait_for_open_port(8080, addr="10.0.0.1", timeout=120)
    node1.wait_for_open_port(8443, addr="10.0.0.1", timeout=120)
    node1.succeed("tail -n 50 /tmp/lightscale-server.log || true")

    import json
    import time

    net = json.loads(node1.succeed(
        "curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
        "-H 'content-type: application/json' "
        "-d '{\"name\":\"lab\",\"bootstrap_token_ttl_seconds\":600," \
        "\"bootstrap_token_uses\":20,\"bootstrap_token_tags\":[\"lab\"]}'"
    ))
    token = net["bootstrap_token"]["token"]

    def enroll(node, name, ip):
        node.succeed(
            "lightscale-client --profile test --config /tmp/ls-config.json "
            "init http://10.0.0.1:8080"
        )
        node.succeed(
            f"lightscale-client --profile test --config /tmp/ls-config.json "
            f"--state-dir /tmp/ls-state register --node-name {name} -- {token}"
        )
        node.succeed(
            f"lightscale-client --profile test --config /tmp/ls-config.json "
            f"--state-dir /tmp/ls-state heartbeat --endpoint {ip}:51820"
        )

    for idx, node in enumerate(nodes, start=1):
        enroll(node, f"node{idx}", f"10.0.0.{idx}")

    def start_agent(node, endpoints, stream_relay):
        node.succeed("touch /tmp/lightscale-agent.log")
        cmd = (
            "lightscale-client --profile test --config /tmp/ls-config.json "
            "--state-dir /tmp/ls-state agent --listen-port 51820 "
            "--heartbeat-interval 5 --longpoll-timeout 5"
        )
        for endpoint in endpoints:
            cmd += f" --endpoint {endpoint}"
        if stream_relay:
            cmd += " --stream-relay"
        node.succeed(
            "systemd-run --no-block --unit=lightscale-agent --service-type=simple "
            "--property=Restart=on-failure --property=RestartSec=1 "
            "--property=TimeoutStartSec=30 "
            "--property=StandardOutput=append:/tmp/lightscale-agent.log "
            "--property=StandardError=append:/tmp/lightscale-agent.log -- "
            + cmd
        )
        node.wait_for_unit("lightscale-agent.service")
        node.wait_until_succeeds("ip link show ls-test", timeout=60)

    for idx, node in enumerate(nodes, start=1):
        start_agent(
            node,
            [f"203.0.113.{idx}:51820", f"10.0.0.{idx}:51820"],
            False,
        )

    time.sleep(1)

    def overlay_ipv4(node):
        data = json.loads(node.succeed("cat /tmp/ls-state/state.json"))
        return data["ipv4"]

    ips = [overlay_ipv4(node) for node in nodes]

    def full_mesh_ping(nodes, ips):
        for i, src in enumerate(nodes):
            for j, dst in enumerate(nodes):
                if i == j:
                    continue
                src.wait_until_succeeds(f"ping -c 3 {ips[j]}", timeout=120)

    full_mesh_ping(nodes, ips)

    for node in nodes:
        node.succeed("systemctl stop lightscale-agent.service")

    for idx, node in enumerate(nodes, start=1):
        start_agent(node, [f"203.0.113.{idx}:51820"], True)

    full_mesh_ping(nodes, ips)
  '';
}