262 lines
9.8 KiB
Nix
262 lines
9.8 KiB
Nix
{ pkgs, serverPkg, clientPkg }:
|
|
{
|
|
name = "lightscale-lab-router";
|
|
nodes = {
|
|
# Control plane server
|
|
node1 = { ... }: {
|
|
networking.hostName = "node1";
|
|
networking.usePredictableInterfaceNames = false;
|
|
virtualisation.vlans = [ 1 ];
|
|
networking.interfaces.eth1.useDHCP = false;
|
|
networking.interfaces.eth1.ipv4.addresses = [
|
|
{ address = "10.0.0.1"; prefixLength = 24; }
|
|
];
|
|
networking.firewall.enable = false;
|
|
boot.kernelModules = [ "wireguard" ];
|
|
environment.systemPackages = [
|
|
serverPkg
|
|
clientPkg
|
|
pkgs.wireguard-tools
|
|
pkgs.iproute2
|
|
pkgs.iputils
|
|
pkgs.netcat-openbsd
|
|
pkgs.curl
|
|
pkgs.nftables
|
|
];
|
|
};
|
|
|
|
# Subnet router node
|
|
router = { ... }: {
|
|
networking.hostName = "router";
|
|
networking.usePredictableInterfaceNames = false;
|
|
virtualisation.vlans = [ 1 2 ]; # eth1: control plane, eth2: LAN
|
|
networking.interfaces.eth1.useDHCP = false;
|
|
networking.interfaces.eth1.ipv4.addresses = [
|
|
{ address = "10.0.0.10"; prefixLength = 24; }
|
|
];
|
|
networking.interfaces.eth2.useDHCP = false;
|
|
networking.interfaces.eth2.ipv4.addresses = [
|
|
{ address = "192.168.100.1"; prefixLength = 24; }
|
|
];
|
|
networking.firewall.enable = false;
|
|
boot.kernelModules = [ "wireguard" ];
|
|
environment.systemPackages = [
|
|
clientPkg
|
|
pkgs.wireguard-tools
|
|
pkgs.iproute2
|
|
pkgs.iputils
|
|
pkgs.netcat-openbsd
|
|
pkgs.curl
|
|
pkgs.nftables
|
|
];
|
|
};
|
|
|
|
# Overlay client
|
|
client = { ... }: {
|
|
networking.hostName = "client";
|
|
networking.usePredictableInterfaceNames = false;
|
|
virtualisation.vlans = [ 1 ];
|
|
networking.interfaces.eth1.useDHCP = false;
|
|
networking.interfaces.eth1.ipv4.addresses = [
|
|
{ address = "10.0.0.20"; prefixLength = 24; }
|
|
];
|
|
networking.firewall.enable = false;
|
|
boot.kernelModules = [ "wireguard" ];
|
|
environment.systemPackages = [
|
|
clientPkg
|
|
pkgs.wireguard-tools
|
|
pkgs.iproute2
|
|
pkgs.iputils
|
|
pkgs.netcat-openbsd
|
|
pkgs.curl
|
|
];
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
start_all()
|
|
node1.wait_for_unit("multi-user.target")
|
|
router.wait_for_unit("multi-user.target")
|
|
client.wait_for_unit("multi-user.target")
|
|
|
|
# Verify network connectivity
|
|
node1.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.1/24'")
|
|
router.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.10/24'")
|
|
router.wait_until_succeeds("ip -4 addr show dev eth2 | grep -q '192.168.100.1/24'")
|
|
client.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.20/24'")
|
|
|
|
# Start control plane
|
|
node1.succeed("touch /tmp/lightscale-server.log")
|
|
node1.execute("sh -c 'tail -n +1 -f /tmp/lightscale-server.log >/dev/console 2>&1 &'")
|
|
node1.succeed(
|
|
"systemd-run --no-block --unit=lightscale-server --service-type=simple "
|
|
"--property=Restart=on-failure --property=RestartSec=1 "
|
|
"--property=TimeoutStartSec=30 "
|
|
"--property=StandardOutput=append:/tmp/lightscale-server.log "
|
|
"--property=StandardError=append:/tmp/lightscale-server.log "
|
|
"--setenv=RUST_LOG=info -- "
|
|
"lightscale-server --listen 10.0.0.1:8080 --state /tmp/lightscale-state.json --admin-token test-token-12345"
|
|
)
|
|
node1.wait_for_unit("lightscale-server.service")
|
|
node1.wait_for_open_port(8080, addr="10.0.0.1", timeout=120)
|
|
|
|
import json
|
|
import time
|
|
|
|
# Create network
|
|
net = json.loads(node1.succeed(
|
|
"curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
|
|
"-H 'authorization: Bearer test-token-12345' "
|
|
"-H 'content-type: application/json' "
|
|
"-d '{\"name\":\"lab\",\"bootstrap_token_ttl_seconds\":600,"
|
|
"\"bootstrap_token_uses\":10,\"bootstrap_token_tags\":[\"lab\"]}'"
|
|
))
|
|
token = net["bootstrap_token"]["token"]
|
|
|
|
def enroll(node, name, ip, routes=None):
|
|
node.succeed(
|
|
"lightscale-client --profile test --config /tmp/ls-config.json "
|
|
"init http://10.0.0.1:8080"
|
|
)
|
|
cmd = (
|
|
f"lightscale-client --profile test --config /tmp/ls-config.json "
|
|
f"--state-dir /tmp/ls-state register --node-name {name} -- {token}"
|
|
)
|
|
node.succeed(cmd)
|
|
|
|
# Build heartbeat command
|
|
hb_cmd = (
|
|
f"lightscale-client --profile test --config /tmp/ls-config.json "
|
|
f"--state-dir /tmp/ls-state heartbeat --endpoint {ip}:51820"
|
|
)
|
|
if routes:
|
|
for route in routes:
|
|
hb_cmd += f" --route {route}"
|
|
node.succeed(hb_cmd)
|
|
|
|
# Enroll router with subnet route (SNAT enabled)
|
|
enroll(router, "router", "10.0.0.10", routes=["192.168.100.0/24"])
|
|
|
|
# Enroll client
|
|
enroll(client, "client", "10.0.0.20")
|
|
|
|
# Start agents
|
|
def start_agent(node, endpoints):
|
|
node.succeed("touch /tmp/lightscale-agent.log")
|
|
cmd = (
|
|
"lightscale-client --profile test --config /tmp/ls-config.json "
|
|
"--state-dir /tmp/ls-state agent --listen-port 51820 "
|
|
"--heartbeat-interval 5 --longpoll-timeout 5"
|
|
)
|
|
for endpoint in endpoints:
|
|
cmd += f" --endpoint {endpoint}"
|
|
node.succeed(
|
|
"systemd-run --no-block --unit=lightscale-agent --service-type=simple "
|
|
"--property=Restart=on-failure --property=RestartSec=1 "
|
|
"--property=TimeoutStartSec=30 "
|
|
"--property=StandardOutput=append:/tmp/lightscale-agent.log "
|
|
"--property=StandardError=append:/tmp/lightscale-agent.log -- "
|
|
+ cmd
|
|
)
|
|
node.wait_for_unit("lightscale-agent.service")
|
|
node.wait_until_succeeds("ip link show ls-test", timeout=60)
|
|
|
|
start_agent(router, ["10.0.0.10:51820"])
|
|
start_agent(client, ["10.0.0.20:51820"])
|
|
|
|
time.sleep(2)
|
|
|
|
# Get overlay IPs
|
|
def overlay_ipv4(node):
|
|
data = json.loads(node.succeed("cat /tmp/ls-state/state.json"))
|
|
return data["ipv4"]
|
|
|
|
router_overlay_ip = overlay_ipv4(router)
|
|
client_overlay_ip = overlay_ipv4(client)
|
|
|
|
# Verify direct overlay connectivity
|
|
router.succeed(f"ping -c 3 {client_overlay_ip}")
|
|
client.succeed(f"ping -c 3 {router_overlay_ip}")
|
|
|
|
# ===== Test 1: Enable subnet router with SNAT =====
|
|
router.succeed("touch /tmp/router-enable.log")
|
|
router.succeed(
|
|
"lightscale-client --profile test --config /tmp/ls-config.json "
|
|
"--state-dir /tmp/ls-state router enable "
|
|
"--interface ls-test --out-interface eth2 2>&1 | tee /tmp/router-enable.log"
|
|
)
|
|
|
|
# Verify nftables rules were created via libnftnl (not via nft CLI)
|
|
# Check that the lightscale table exists
|
|
router.succeed("nft list table inet lightscale")
|
|
router.succeed("nft list table ip lightscale-nat")
|
|
|
|
# Show the rules for debugging
|
|
router.succeed("nft list ruleset > /tmp/nft-ruleset.txt")
|
|
router.succeed("cat /tmp/nft-ruleset.txt")
|
|
|
|
# Verify forwarding chain exists with our rules
|
|
router.succeed("nft list chain inet lightscale ls-forward")
|
|
router.succeed("nft list chain ip lightscale-nat ls-postrouting")
|
|
|
|
# Check that masquerade rule exists (SNAT enabled)
|
|
router.succeed("nft list chain ip lightscale-nat ls-postrouting | grep -q masquerade")
|
|
|
|
# Verify forwarding rules are correct
|
|
# Rule: iifname ls-test oifname eth2 accept
|
|
router.succeed("nft list chain inet lightscale ls-forward | grep -q 'iifname.*ls-test'")
|
|
router.succeed("nft list chain inet lightscale ls-forward | grep -q 'oifname.*eth2'")
|
|
|
|
# Rule: iifname eth2 oifname ls-test ct state established,related accept
|
|
router.succeed("nft list chain inet lightscale ls-forward | grep -q 'iifname.*eth2'")
|
|
router.succeed("nft list chain inet lightscale ls-forward | grep -q 'oifname.*ls-test'")
|
|
|
|
# Verify sysctl settings for forwarding
|
|
router.succeed("cat /proc/sys/net/ipv4/ip_forward | grep -q 1")
|
|
router.succeed("cat /proc/sys/net/ipv6/conf/all/forwarding | grep -q 1")
|
|
|
|
# ===== Test 2: Disable router and verify cleanup =====
|
|
router.succeed(
|
|
"lightscale-client --profile test --config /tmp/ls-config.json "
|
|
"--state-dir /tmp/ls-state router disable "
|
|
"--interface ls-test --out-interface eth2"
|
|
)
|
|
|
|
# Verify tables are cleaned up
|
|
router.fail("nft list table inet lightscale 2>/dev/null")
|
|
router.fail("nft list table ip lightscale-nat 2>/dev/null")
|
|
|
|
# ===== Test 3: Re-enable with --no-snat =====
|
|
router.succeed(
|
|
"lightscale-client --profile test --config /tmp/ls-config.json "
|
|
"--state-dir /tmp/ls-state router enable "
|
|
"--interface ls-test --out-interface eth2 --no-snat"
|
|
)
|
|
|
|
# Verify the filter table exists
|
|
router.succeed("nft list table inet lightscale")
|
|
|
|
# Show ruleset for debugging (no NAT table or no masquerade)
|
|
router.succeed("nft list ruleset > /tmp/nft-ruleset-no-snat.txt")
|
|
router.succeed("cat /tmp/nft-ruleset-no-snat.txt")
|
|
|
|
# Verify no masquerade rule (NAT table should not exist with --no-snat)
|
|
router.fail("nft list table ip lightscale-nat 2>/dev/null")
|
|
|
|
# ===== Test 4: Re-disable and final cleanup verification =====
|
|
router.succeed(
|
|
"lightscale-client --profile test --config /tmp/ls-config.json "
|
|
"--state-dir /tmp/ls-state router disable "
|
|
"--interface ls-test --out-interface eth2"
|
|
)
|
|
|
|
# Final cleanup verification
|
|
router.fail("nft list table inet lightscale 2>/dev/null")
|
|
router.fail("nft list table ip lightscale-nat 2>/dev/null")
|
|
|
|
print("SUCCESS: libnftnl-based nftables operations work correctly!")
|
|
print("- SNAT mode: inet lightscale + ip lightscale-nat tables created with masquerade")
|
|
print("- No-SNAT mode: only inet lightscale table created")
|
|
print("- Cleanup: all tables properly removed on disable")
|
|
'';
|
|
}
|