215 lines
8.2 KiB
Nix
215 lines
8.2 KiB
Nix
{ pkgs, serverPkg, clientPkg }:
|
|
{
|
|
name = "lightscale-lab-multi";
|
|
nodes = {
|
|
node1 = { ... }: {
|
|
networking.hostName = "node1";
|
|
networking.usePredictableInterfaceNames = false;
|
|
virtualisation.vlans = [ 1 ];
|
|
networking.interfaces.eth1.useDHCP = false;
|
|
networking.interfaces.eth1.ipv4.addresses = [
|
|
{ address = "10.0.0.1"; prefixLength = 24; }
|
|
];
|
|
networking.firewall.enable = false;
|
|
boot.kernelModules = [ "wireguard" ];
|
|
environment.systemPackages = [
|
|
serverPkg
|
|
clientPkg
|
|
pkgs.wireguard-tools
|
|
pkgs.iproute2
|
|
pkgs.iputils
|
|
pkgs.netcat-openbsd
|
|
pkgs.curl
|
|
];
|
|
};
|
|
node2 = { ... }: {
|
|
networking.hostName = "node2";
|
|
networking.usePredictableInterfaceNames = false;
|
|
virtualisation.vlans = [ 1 2 ];
|
|
networking.interfaces.eth1.useDHCP = false;
|
|
networking.interfaces.eth1.ipv4.addresses = [
|
|
{ address = "10.0.0.2"; prefixLength = 24; }
|
|
];
|
|
networking.interfaces.eth2.useDHCP = false;
|
|
networking.interfaces.eth2.ipv4.addresses = [
|
|
{ address = "192.168.100.1"; prefixLength = 24; }
|
|
];
|
|
networking.firewall.enable = false;
|
|
boot.kernelModules = [ "wireguard" ];
|
|
environment.systemPackages = [
|
|
clientPkg
|
|
pkgs.wireguard-tools
|
|
pkgs.iproute2
|
|
pkgs.iputils
|
|
pkgs.curl
|
|
pkgs.nftables
|
|
];
|
|
};
|
|
node3 = { ... }: {
|
|
networking.hostName = "node3";
|
|
networking.usePredictableInterfaceNames = false;
|
|
virtualisation.vlans = [ 1 3 ];
|
|
networking.interfaces.eth1.useDHCP = false;
|
|
networking.interfaces.eth1.ipv4.addresses = [
|
|
{ address = "10.0.0.3"; prefixLength = 24; }
|
|
];
|
|
networking.interfaces.eth2.useDHCP = false;
|
|
networking.interfaces.eth2.ipv4.addresses = [
|
|
{ address = "192.168.100.1"; prefixLength = 24; }
|
|
];
|
|
networking.firewall.enable = false;
|
|
boot.kernelModules = [ "wireguard" ];
|
|
environment.systemPackages = [
|
|
clientPkg
|
|
pkgs.wireguard-tools
|
|
pkgs.iproute2
|
|
pkgs.iputils
|
|
pkgs.curl
|
|
pkgs.nftables
|
|
];
|
|
};
|
|
node4 = { ... }: {
|
|
networking.hostName = "node4";
|
|
networking.usePredictableInterfaceNames = false;
|
|
virtualisation.vlans = [ 1 ];
|
|
networking.interfaces.eth1.useDHCP = false;
|
|
networking.interfaces.eth1.ipv4.addresses = [
|
|
{ address = "10.0.0.4"; prefixLength = 24; }
|
|
];
|
|
networking.firewall.enable = false;
|
|
boot.kernelModules = [ "wireguard" ];
|
|
environment.systemPackages = [
|
|
clientPkg
|
|
pkgs.wireguard-tools
|
|
pkgs.iproute2
|
|
pkgs.iputils
|
|
pkgs.curl
|
|
];
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
start_all()
|
|
node1.wait_for_unit("multi-user.target")
|
|
node2.wait_for_unit("multi-user.target")
|
|
node3.wait_for_unit("multi-user.target")
|
|
node4.wait_for_unit("multi-user.target")
|
|
|
|
node1.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.1/24'")
|
|
node2.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.2/24'")
|
|
node2.wait_until_succeeds("ip -4 addr show dev eth2 | grep -q '192.168.100.1/24'")
|
|
node3.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.3/24'")
|
|
node3.wait_until_succeeds("ip -4 addr show dev eth2 | grep -q '192.168.100.1/24'")
|
|
node4.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.4/24'")
|
|
|
|
node2.succeed("ip addr add 192.168.100.10/24 dev eth2 || true")
|
|
node3.succeed("ip addr add 192.168.100.10/24 dev eth2 || true")
|
|
|
|
node1.succeed("touch /tmp/lightscale-server.log")
|
|
node1.execute("sh -c 'tail -n +1 -f /tmp/lightscale-server.log >/dev/console 2>&1 &'")
|
|
node1.succeed(
|
|
"systemd-run --no-block --unit=lightscale-server --service-type=simple "
|
|
"--property=Restart=on-failure --property=RestartSec=1 "
|
|
"--property=TimeoutStartSec=30 "
|
|
"--property=StandardOutput=append:/tmp/lightscale-server.log "
|
|
"--property=StandardError=append:/tmp/lightscale-server.log "
|
|
"--setenv=RUST_LOG=info -- "
|
|
"lightscale-server --listen 10.0.0.1:8080 --state /tmp/lightscale-state.json"
|
|
)
|
|
node1.wait_for_unit("lightscale-server.service")
|
|
node1.wait_for_open_port(8080, addr="10.0.0.1", timeout=120)
|
|
|
|
import json
|
|
|
|
net_a = json.loads(node1.succeed(
|
|
"curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
|
|
"-H 'content-type: application/json' "
|
|
"-d '{\"name\":\"net-a\",\"bootstrap_token_ttl_seconds\":600," \
|
|
"\"bootstrap_token_uses\":10,\"bootstrap_token_tags\":[\"a\"]}'"
|
|
))
|
|
net_b = json.loads(node1.succeed(
|
|
"curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
|
|
"-H 'content-type: application/json' "
|
|
"-d '{\"name\":\"net-b\",\"bootstrap_token_ttl_seconds\":600," \
|
|
"\"bootstrap_token_uses\":10,\"bootstrap_token_tags\":[\"b\"]}'"
|
|
))
|
|
token_a = net_a["bootstrap_token"]["token"]
|
|
token_b = net_b["bootstrap_token"]["token"]
|
|
|
|
def enroll(node, profile, token, name, ip, state_dir):
|
|
node.succeed(
|
|
f"lightscale-client --profile {profile} --config /tmp/ls-config.json "
|
|
"init http://10.0.0.1:8080"
|
|
)
|
|
node.succeed(
|
|
f"lightscale-client --profile {profile} --config /tmp/ls-config.json "
|
|
f"--state-dir {state_dir} register --node-name {name} -- {token}"
|
|
)
|
|
node.succeed(
|
|
f"lightscale-client --profile {profile} --config /tmp/ls-config.json "
|
|
f"--state-dir {state_dir} heartbeat --endpoint {ip}:51820"
|
|
)
|
|
|
|
enroll(node2, "neta", token_a, "router-a", "10.0.0.2", "/tmp/ls-state-a")
|
|
enroll(node3, "netb", token_b, "router-b", "10.0.0.3", "/tmp/ls-state-b")
|
|
enroll(node4, "neta", token_a, "client-a", "10.0.0.4", "/tmp/ls-state-a")
|
|
enroll(node4, "netb", token_b, "client-b", "10.0.0.4", "/tmp/ls-state-b")
|
|
|
|
def start_agent(node, profile, state_dir, listen_port, extra_args):
|
|
node.succeed("touch /tmp/lightscale-agent.log")
|
|
cmd = (
|
|
f"lightscale-client --profile {profile} --config /tmp/ls-config.json "
|
|
f"--state-dir {state_dir} agent --listen-port {listen_port} "
|
|
"--heartbeat-interval 5 --longpoll-timeout 5 "
|
|
"--endpoint-stale-after 5 --endpoint-max-rotations 1"
|
|
)
|
|
for arg in extra_args:
|
|
cmd += f" {arg}"
|
|
node.succeed(
|
|
f"systemd-run --no-block --unit=lightscale-agent-{profile} "
|
|
"--service-type=simple "
|
|
"--property=Restart=on-failure --property=RestartSec=1 "
|
|
"--property=TimeoutStartSec=30 "
|
|
"--property=StandardOutput=append:/tmp/lightscale-agent.log "
|
|
"--property=StandardError=append:/tmp/lightscale-agent.log -- "
|
|
+ cmd
|
|
)
|
|
node.wait_for_unit(f"lightscale-agent-{profile}.service")
|
|
node.wait_until_succeeds(f"ip link show ls-{profile}", timeout=60)
|
|
|
|
start_agent(node2, "neta", "/tmp/ls-state-a", 51820, [
|
|
"--endpoint 10.0.0.2:51820",
|
|
"--advertise-route 192.168.100.0/24",
|
|
"--advertise-map 192.168.100.0/24=10.250.0.0/24",
|
|
])
|
|
start_agent(node3, "netb", "/tmp/ls-state-b", 51820, [
|
|
"--endpoint 10.0.0.3:51820",
|
|
"--advertise-route 192.168.100.0/24",
|
|
"--advertise-map 192.168.100.0/24=10.251.0.0/24",
|
|
])
|
|
start_agent(node4, "neta", "/tmp/ls-state-a", 51820, [
|
|
"--endpoint 10.0.0.4:51820",
|
|
"--apply-routes",
|
|
])
|
|
start_agent(node4, "netb", "/tmp/ls-state-b", 51821, [
|
|
"--endpoint 10.0.0.4:51821",
|
|
"--apply-routes",
|
|
])
|
|
|
|
node2.succeed(
|
|
"lightscale-client --profile neta --config /tmp/ls-config.json "
|
|
"--state-dir /tmp/ls-state-a router enable --interface ls-neta "
|
|
"--out-interface eth2 --no-snat "
|
|
"--map 192.168.100.0/24=10.250.0.0/24"
|
|
)
|
|
node3.succeed(
|
|
"lightscale-client --profile netb --config /tmp/ls-config.json "
|
|
"--state-dir /tmp/ls-state-b router enable --interface ls-netb "
|
|
"--out-interface eth2 --no-snat "
|
|
"--map 192.168.100.0/24=10.251.0.0/24"
|
|
)
|
|
|
|
node4.wait_until_succeeds("ping -c 3 10.250.0.10", timeout=180)
|
|
node4.wait_until_succeeds("ping -c 3 10.251.0.10", timeout=180)
|
|
'';
|
|
}
|