Update lightscale-client submodule: debugging cleanup

flake.nix

@@ -1,5 +1,6 @@
 {
   inputs = {
+    self.submodules = true;
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
   };
 
@@ -39,6 +40,9 @@
       testConfigIpv6 = import ./lab/test-ipv6.nix { inherit pkgs serverPkg clientPkg; };
       testConfigUserspace = import ./lab/test-userspace.nix { inherit pkgs serverPkg clientPkg; };
       testConfigRelayFailover = import ./lab/test-relay-failover.nix { inherit pkgs serverPkg clientPkg; };
+      testConfigServerMeshRelay = import ./lab/test-server-mesh-relay.nix { inherit pkgs serverPkg clientPkg; };
+      testConfigServices = import ./lab/test-services.nix { inherit pkgs serverPkg clientPkg; };
+      testConfigRouter = import ./lab/test-router.nix { inherit pkgs serverPkg clientPkg; };
       labTestFast =
         (import (nixpkgs + "/nixos/tests/make-test-python.nix") testConfigFast)
           { inherit system pkgs; };
@@ -84,6 +88,15 @@
       labTestRelayFailover =
         (import (nixpkgs + "/nixos/tests/make-test-python.nix") testConfigRelayFailover)
           { inherit system pkgs; };
+      labTestServerMeshRelay =
+        (import (nixpkgs + "/nixos/tests/make-test-python.nix") testConfigServerMeshRelay)
+          { inherit system pkgs; };
+      labTestServices =
+        (import (nixpkgs + "/nixos/tests/make-test-python.nix") testConfigServices)
+          { inherit system pkgs; };
+      labTestRouter =
+        (import (nixpkgs + "/nixos/tests/make-test-python.nix") testConfigRouter)
+          { inherit system pkgs; };
     in
     {
       packages.${system} = {
@@ -91,6 +104,15 @@
         lightscale-client = clientPkg;
       };
 
+      nixosModules = {
+        lightscale-server = import ./nixos/modules/lightscale-server.nix {
+          defaultPackage = serverPkg;
+        };
+        lightscale-client = import ./nixos/modules/lightscale-client.nix {
+          defaultPackage = clientPkg;
+        };
+      };
+
       nixosTests.lightscale-lab = labTestFast;
       nixosTests.lightscale-lab-5 = labTestFull;
       nixosTests.lightscale-lab-firewall = labTestFirewall;
@@ -106,6 +128,9 @@
       nixosTests.lightscale-lab-ipv6 = labTestIpv6;
       nixosTests.lightscale-lab-userspace = labTestUserspace;
       nixosTests.lightscale-lab-relay-failover = labTestRelayFailover;
+      nixosTests.lightscale-lab-server-mesh-relay = labTestServerMeshRelay;
+      nixosTests.lightscale-lab-services = labTestServices;
+      nixosTests.lightscale-lab-router = labTestRouter;
 
       devShells.${system}.default = pkgs.mkShell {
         buildInputs = [

lab/mesh-certs/ca.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDGzCCAgOgAwIBAgIUGqz2VVqQcK7JNSMaCB44itrGNyMwDQYJKoZIhvcNAQEL
+BQAwHTEbMBkGA1UEAwwSbGlnaHRzY2FsZS1tZXNoLWNhMB4XDTI2MDIxNDAyMTAw
+N1oXDTM2MDIxMjAyMTAwN1owHTEbMBkGA1UEAwwSbGlnaHRzY2FsZS1tZXNoLWNh
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA18pB501rZUb+aBTRLSB0
+FQypCC3hojYH9HeFzBaemdATn1uaNy9Yk1RM31fUpDwVXgmm8zoMMCcCmitXAj9v
+bXbmlcINCmBmKzoiZJMEakR0+cLqyF/6lVnP8is41xBBCEkvuPdkrFhwj90oiPIt
+eXBkuohLkVNP2lIbD3aGqLgDQ5FVVVYHEaUE4Ad/EkjeCvmmQBNvEfAvH+k9NVP4
+5EEriPjmcBpmk805sLN8f/lZ97e8cN1CESYywjmxA2FHX3Au7IepJGBpj09O8T/f
+QeUhgfPsiCfHy27kyDhsvhA5ZAMG69JP+fMsosSMs0TmrTYddbpMp9qVH8+GRaJJ
+HQIDAQABo1MwUTAdBgNVHQ4EFgQUxA5w5KDcCYAYaTnXSQhJVvAnMEcwHwYDVR0j
+BBgwFoAUxA5w5KDcCYAYaTnXSQhJVvAnMEcwDwYDVR0TAQH/BAUwAwEB/zANBgkq
+hkiG9w0BAQsFAAOCAQEAjorOEpDx+zXEBc/DfY0WWqqeg3tF54bOW8cajnVT6hZJ
+J/FKn4c08mQbqhgVXsgKA2vRm6mgjWzVHT8ROtfIEy6W2qoWiaCMNPffxI5fhF/h
+bCCIEPvW5aMV6l/4pIndx/1Zli6ZZTTGROgZ1TycmzGd1++MMPBBfH429hITs7we
+lZ6eQwZzmwZwLkke7VJib3t3nwk35KR1Tco68cjTf0ZECMkxaF846zoQltDmeQcg
+4bxuG4gdinvNRgsU/rQRsezF7iqg85PFCU+7zyVITx9LhSNBJTG0nT9xbfW+Gzyx
+6ld+B99vEVY00kO6RQ300rk1LjnrFhFbm/z323unGA==
+-----END CERTIFICATE-----

lab/mesh-certs/srv-a-key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDcRyF//YGaM+71
+buV8QqBIQCacAfsNhqL+cOOFgwSpkJZ1vOjq2voJy1CmzqqjTIdGnvZnvG/pEYZW
+4EG6RMisSovkhWIu19Ml8RpEYSAYFcwqotfKN14ETEWMD7xeV9kOQaolJHLPFhDu
+DHnOEGULR5fxcGtVHdJKId6eP8ysH6R9V3ijylORvvbuNI3dLE0rC63aOnJ2grzS
+vyzfFDxduWP0Rao5xmztDdMAysl3FflE/WUvSScDcp0MlELwczKusUvxNQyGkIhE
+Fh0YEJs8ZAbQ38XFB8JU1MPrY0KyUUZzQDSYApTXIiytiwU0QLd9eY48qw44bOMe
+fWiYJ9aZAgMBAAECggEAAuu6iPzujAHmSEDZMCWeBHc9S9JG5u4DGUab7bhIBouR
+QyxnFj4jFKAqiJuy7YllPc17zPJVtYxy8JH2rSaeVpKCelw+agqYlSc2RvPWvhsD
+4wjXvamwSHROc8X9pG6bxTGftPBfyVksjkuCDfZvsI+Zdy0bbzx+/lamogiMiTe3
+5I9qtvvmU1ej+VQXmbi9Gwx3ll3N/pGPrraB4Um6/eHst6ujLt5/Dk6B2NuJUbf5
+Nj83MwPpA7OOZHBdCtvDW7waKD3ZGtyJMMX2YZ0nd6dUMq4RRXW+er2EnftEvZmX
+kaNtaHAHdJakrFUqqy60N6XRoBL6SmQ7BlKd5yWXUQKBgQD1iciJYUaFy/CnE830
+j5HbLVXiIXu/U6wxSraErt40oHlG/rLgmpc3qjBE+c4t2j8H/onnRjatAFGZdY8A
+hwgDjJXgGU9O3wm6n07HKhrgPM2KYyswOuDmetnWDQp5mWjUBEropFdkZLdZSuDk
+6bSl0RUXuDohsS4HfJ9HePb19QKBgQDlqdG50hgvzOLXFGpzD6Og5Muz6KXZB0cB
+SYZF2uTvZvWzBo4eBTUfOZE11kQkOlweSGMoaS/tT9lmkvfcavLtSutvRvT1H4f3
+YnNU4EA8+Eh2iLrI8BNvASlejMdW1aSkJsjMumOikQaWaXzCNrMhxbd1E29X8gda
+JMWcoBOTlQKBgBqmNq66cRYKeXcYziyx/GmmdQDTE4RDh5fd/QtPk2xw0ljjQfTg
+snLnNM/3sOoHGvo6JSuF0l9afoDCYp/zB+qiso2dEZ+E06B+s+Un67zUvJY9hy13
+5nr9cHEr/ywNe3QvdxXi6F1MFR5K4zfVKbcphzmI1D5d5ZoIa50tQtiVAoGBAMuK
+xTA+DmeW21gpZOqS6r82536Mayg8teZjJliU1p+CjbFb1uquTNVerN6dBolhG7FP
+EGqJRwu44AFzsa6tLp2175EQvxrcFiHfJD4N/YDLv1UmevyJIYAY9HQyqpy//gnp
+wb8IVjOG+uKlnQd3eS0uURi239B+1ZtDycu1Z4Q9AoGBAO7ftUDZz7QwFn7bAi7D
+Pk/22L73w+MleI/amvTgLi76ZLH9lFVNURUnwakIPlb1Se74HdwRSb9hRFYY4l8G
+KDduMwcdCJ8+XfD90qHe5Ce647B0hlcNmCAf2SIxEs3Lm8u8gfXaFW8f2+OziLCz
+DTX/RXXM0uS7meSez7GKfzos
+-----END PRIVATE KEY-----

lab/mesh-certs/srv-a.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDODCCAiCgAwIBAgIUN9JLKAim5gUOxFdJaRD3KGc+N20wDQYJKoZIhvcNAQEL
+BQAwHTEbMBkGA1UEAwwSbGlnaHRzY2FsZS1tZXNoLWNhMB4XDTI2MDIxNDAyMTAw
+N1oXDTI4MDUxOTAyMTAwN1owFTETMBEGA1UEAwwKc3J2LWEubWVzaDCCASIwDQYJ
+KoZIhvcNAQEBBQADggEPADCCAQoCggEBANxHIX/9gZoz7vVu5XxCoEhAJpwB+w2G
+ov5w44WDBKmQlnW86Ora+gnLUKbOqqNMh0ae9me8b+kRhlbgQbpEyKxKi+SFYi7X
+0yXxGkRhIBgVzCqi18o3XgRMRYwPvF5X2Q5BqiUkcs8WEO4Mec4QZQtHl/Fwa1Ud
+0koh3p4/zKwfpH1XeKPKU5G+9u40jd0sTSsLrdo6cnaCvNK/LN8UPF25Y/RFqjnG
+bO0N0wDKyXcV+UT9ZS9JJwNynQyUQvBzMq6xS/E1DIaQiEQWHRgQmzxkBtDfxcUH
+wlTUw+tjQrJRRnNANJgClNciLK2LBTRAt315jjyrDjhs4x59aJgn1pkCAwEAAaN4
+MHYwFQYDVR0RBA4wDIIKc3J2LWEubWVzaDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
+KwYBBQUHAwIwHQYDVR0OBBYEFPsTDIBJkm6jP/sq0gTVfjUems0rMB8GA1UdIwQY
+MBaAFMQOcOSg3AmAGGk510kISVbwJzBHMA0GCSqGSIb3DQEBCwUAA4IBAQBEI1vI
+vCojGmmIhaAH6Kp1+6t79xfXE5hGnSgvwr5iTzuoqS+xyx2yb3CwRInnrUmoVGbs
+gAH4yOYVKwJ9RZxigr/3Q0rzh70koU54p6Hn4Jqmy5uQRSx6cZBuKku9Cg2MAsmg
+kvKQg2vbVo5/IWaJzqzY97ahEIGchwhCsXnvdI4yn/cb8TU43K0RksOYIrnUynQ9
+IozIe63ijCaVsTu8Sv/i0eHkle7lfcPKEVK+Q/yge9DH7SolB92fZTxXwLhn0H39
+e7qng8OecdjlRoN9frGHMZuY0T0eOhogDuCvZwu97kAcWIuWo3wKi2lU5fK5Mpra
+VJgChppzQz8978uM
+-----END CERTIFICATE-----

lab/mesh-certs/srv-b-key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7aI1rHuLfSOhC
+tMPxSLbMWflrVR5/hS8Ytz7Py7oAVt8hizq6GtxnwkpkPGiawIllPUFrquoIqHj8
+B89yzJB88HpyDHKX7YumE+ZXCg35+P7f379UFiHHRPqycS7GIS+U2VAVBKi8y0m+
+wwBSpDHbXeG6R3L1GdrrXiZLMdiRsGXOFQ8gNOKZ85Hc/XLfjcv5CO+cBvKiZYF7
+9PfYFn10rJgJ4AjWZG00rqJn5mAqfIVPTKLm1idvfBM/tm73aWvffO7Buw2qfFQ/
+Re1W+gqQ3ujT5shajsY2olx5gYXGICoz+vNz4rO5QHWG7K6kiuKea268OQgiIWLR
+O9LAdb3bAgMBAAECggEATLg/dIPfYoMJOg2gTU10L/IBblQZOSD/p8GUgCKpkWv+
+yk+iCf/nDL74D1K/i+KYHI7YEkiUqFi3to1H+noOCGe16Kx52QJQ8fshh65BDH1H
+ccS6IaUxzM7LRnOZL1j0jp3r777JiQg7t1FC95HqKyCiwMxxHi+odERFrvDH9NOp
+o+H5AkGtBpspdyDuhhBLa2aTipRcA5o2vkzsiIGU4ofrPaUyk5tIV3GsOZ3j5m6Y
+dyhPtXTS5pDmRHNM81zMNCHV+xPTluftvG4TNZWX2jPFFZk/kifZq0AbAsKu2rtI
+lGAe1lfxHUZguCLmSSRqQm8xsHFD2tlFrSfUbjYY2QKBgQD8c0BVdv+5vt8ZbDuJ
+mWjXmWGqOaVZIOxfek0uGLXvYwEiUPSn7lkgnUKUivvOE7Z0SZcAOCxf86aULgSk
+CVgZUoOdzI9VxjJmK5c528MnxVwG4Z5imJBFPI9Op/RAL4K82qQZz1c3c/Fq9SJj
+9nbegCQZ/wG8AAovFtHo9ZwwZQKBgQC+CytRsxsKahU0Wyt8fJedenTozDFGElYZ
+5oV6GNwyAe/gpzcLNl2R72hubxQGfNkuD4oxQoAiidpa01oNDJDm6LWf9L5EJqD3
+Fy+k+Kd+04ey0y66ZnbPV/gRI/L+B5IgZY5G8OHdL5Yvk+IuPqQazQPvb6QD3wTx
+gNLNSlCxPwKBgGsJRq617MlJl3hE/p1h0SUQoGs3U9cNcYst5Ml1qrYcCSAOqR0G
+nv2ID/HBV/BRRVva085BAveP7AIJ3OfcGmFqLenbEK7ygO3274CVoBIdyN4WDTyK
+qSjh+3UDGzmXq1v9a/SRh844N5T86J7vogjG1ge7qnWWorrCdy/J63ZNAoGAWMV0
+3bvxFKNK9mLj5El6tPffpmLDXXzxNTYGAWudZ8qZ13Gkd5tUh/ex62v9ia5F8IsX
+vTzYB8om8igpt1C4WvQ26tnzCniU1fbBrajs7IQ4reKRwEZelyn7WV5WgizdKD7n
+/+FDUAOLfvvwOjPOiipb/TtD/P7vGzRWw8hD+xcCgYEArYZiHqa5G6l8B4agrBlq
+To6EloHFzgkVDOQ8jirQ5hbNHkHJfY0f1s3ZcQX1TQWsoPsWQM5IRocD9oaO9n4i
+c9hCTU4NlY42ffgSL5O43r29i0i3AyA7mKTr3tfbkgVqjqFKHc2DUECoO8OowQpB
+A6cVjX4wec0RxiMmGfHICK8=
+-----END PRIVATE KEY-----

lab/mesh-certs/srv-b.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDODCCAiCgAwIBAgIUN9JLKAim5gUOxFdJaRD3KGc+N24wDQYJKoZIhvcNAQEL
+BQAwHTEbMBkGA1UEAwwSbGlnaHRzY2FsZS1tZXNoLWNhMB4XDTI2MDIxNDAyMTAw
+N1oXDTI4MDUxOTAyMTAwN1owFTETMBEGA1UEAwwKc3J2LWIubWVzaDCCASIwDQYJ
+KoZIhvcNAQEBBQADggEPADCCAQoCggEBALtojWse4t9I6EK0w/FItsxZ+WtVHn+F
+Lxi3Ps/LugBW3yGLOroa3GfCSmQ8aJrAiWU9QWuq6gioePwHz3LMkHzwenIMcpft
+i6YT5lcKDfn4/t/fv1QWIcdE+rJxLsYhL5TZUBUEqLzLSb7DAFKkMdtd4bpHcvUZ
+2uteJksx2JGwZc4VDyA04pnzkdz9ct+Ny/kI75wG8qJlgXv099gWfXSsmAngCNZk
+bTSuomfmYCp8hU9MoubWJ298Ez+2bvdpa9987sG7Dap8VD9F7Vb6CpDe6NPmyFqO
+xjaiXHmBhcYgKjP683Pis7lAdYbsrqSK4p5rbrw5CCIhYtE70sB1vdsCAwEAAaN4
+MHYwFQYDVR0RBA4wDIIKc3J2LWIubWVzaDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
+KwYBBQUHAwIwHQYDVR0OBBYEFE5cUMQW4BB7uYhc8sccxlaf0vcyMB8GA1UdIwQY
+MBaAFMQOcOSg3AmAGGk510kISVbwJzBHMA0GCSqGSIb3DQEBCwUAA4IBAQBLph2J
+uy9hRWBlHxSmNngbcWr7SDp6A/dcWxNsIzlHys1cCFz0IR6pVGBTsuo5pCSlzkt/
+VJA3TbmHIQa/SdXUIarVUd3NzCHhbwrDEvup/113osOOP/N9vQJRSmIGZoXmS5ae
+8ygHD6ZVgoB8ON2ZOY7FYaCAeMAA25i+8LlEC6us28YAs87zkttq1etylJzCyw2a
+GJL5ozl4GtTYpoERSudberKhJHdsY2hMp4rvaES/nMfbJ+Hb/msTpJHVg8Alfkvn
+XAbXWJGnvTSaY+IipSGJqL+LJxi3WCQnwfFTidtnxkaWEHInAhu8lMajnrdMiuz3
+HcjmbbSoexwE+tN/
+-----END CERTIFICATE-----

lab/run.sh

@@ -8,6 +8,7 @@ for arg in "$@"; do
   case "$arg" in
     full) MODE=full ;;
     fast) MODE=fast ;;
+    admin) MODE=admin ;;
     firewall) MODE=firewall ;;
     nat) MODE=nat ;;
     multi) MODE=multi ;;
@@ -18,9 +19,12 @@ for arg in "$@"; do
     controlplane) MODE=controlplane ;;
     controlplane-ha) MODE=controlplane-ha ;;
     relay-failover) MODE=relay-failover ;;
+    server-mesh) MODE=server-mesh ;;
+    services) MODE=services ;;
     dns) MODE=dns ;;
     ipv6) MODE=ipv6 ;;
     userspace) MODE=userspace ;;
+    router) MODE=router ;;
     --interactive) INTERACTIVE=1 ;;
     --keep) KEEP=1 ;;
   esac
@@ -29,40 +33,53 @@ done
 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 cd "$ROOT_DIR"
 
-nix build .#packages.x86_64-linux.lightscale-server
+if [[ "$MODE" == "admin" ]]; then
-nix build .#packages.x86_64-linux.lightscale-client
+  exec "$ROOT_DIR/lab/test-admin-smoke.sh"
+fi
+
+# Use git+file flake URL so git submodules are included during evaluation/build.
+FLAKE_REF="${LIGHTSCALE_FLAKE_REF:-git+file://$ROOT_DIR?submodules=1}"
+
+nix build "${FLAKE_REF}#packages.x86_64-linux.lightscale-server"
+nix build "${FLAKE_REF}#packages.x86_64-linux.lightscale-client"
 
 OUT_LINK="$ROOT_DIR/lab/driver-$MODE"
 if [[ "$MODE" == "full" ]]; then
-  nix build .#nixosTests.lightscale-lab-5.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-5.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "nat" ]]; then
-  nix build .#nixosTests.lightscale-lab-nat.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-nat.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "multi" ]]; then
-  nix build .#nixosTests.lightscale-lab-multi.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-multi.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "scale" ]]; then
-  nix build .#nixosTests.lightscale-lab-scale.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-scale.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "negative" ]]; then
-  nix build .#nixosTests.lightscale-lab-negative.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-negative.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "keys" ]]; then
-  nix build .#nixosTests.lightscale-lab-keys.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-keys.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "restart" ]]; then
-  nix build .#nixosTests.lightscale-lab-restart.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-restart.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "controlplane" ]]; then
-  nix build .#nixosTests.lightscale-lab-controlplane-restart.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-controlplane-restart.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "controlplane-ha" ]]; then
-  nix build .#nixosTests.lightscale-lab-controlplane-ha.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-controlplane-ha.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "relay-failover" ]]; then
-  nix build .#nixosTests.lightscale-lab-relay-failover.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-relay-failover.driver" --out-link "$OUT_LINK"
+elif [[ "$MODE" == "server-mesh" ]]; then
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-server-mesh-relay.driver" --out-link "$OUT_LINK"
+elif [[ "$MODE" == "services" ]]; then
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-services.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "dns" ]]; then
-  nix build .#nixosTests.lightscale-lab-dns.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-dns.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "ipv6" ]]; then
-  nix build .#nixosTests.lightscale-lab-ipv6.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-ipv6.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "userspace" ]]; then
-  nix build .#nixosTests.lightscale-lab-userspace.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-userspace.driver" --out-link "$OUT_LINK"
 elif [[ "$MODE" == "firewall" ]]; then
-  nix build .#nixosTests.lightscale-lab-firewall.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-firewall.driver" --out-link "$OUT_LINK"
+elif [[ "$MODE" == "router" ]]; then
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab-router.driver" --out-link "$OUT_LINK"
 else
-  nix build .#nixosTests.lightscale-lab.driver --out-link "$OUT_LINK"
+  nix build "${FLAKE_REF}#nixosTests.lightscale-lab.driver" --out-link "$OUT_LINK"
 fi
 
 DRIVER_ARGS=()

lab/test-resource-guard.nix

@@ -0,0 +1,395 @@
+{ pkgs, serverPkg, clientPkg }:
+let
+  serverModule = import ../nixos/modules/lightscale-server.nix {
+    defaultPackage = serverPkg;
+  };
+  clientModule = import ../nixos/modules/lightscale-client.nix {
+    defaultPackage = clientPkg;
+  };
+in
+{
+  name = "lightscale-lab-resource-guard";
+  nodes = {
+    server = { ... }: {
+      imports = [ serverModule ];
+      networking.hostName = "server";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 ];
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        { address = "10.0.0.1"; prefixLength = 24; }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+
+      services.lightscale-server = {
+        enable = true;
+        listen = "10.0.0.1:8080";
+        stateFile = "/var/lib/lightscale-server/state.json";
+        adminToken = "lab-admin-token";
+      };
+
+      environment.systemPackages = [
+        clientPkg
+        pkgs.curl
+        pkgs.iputils
+        pkgs.wireguard-tools
+        pkgs.iproute2
+      ];
+    };
+
+    client = { ... }: {
+      imports = [ clientModule ];
+      networking.hostName = "client";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 ];
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        { address = "10.0.0.2"; prefixLength = 24; }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+
+      environment.systemPackages = [
+        clientPkg
+        pkgs.wireguard-tools
+        pkgs.iproute2
+        pkgs.iputils
+        pkgs.curl
+      ];
+    };
+  };
+
+  testScript = ''
+    import json
+    import time
+
+    start_all()
+
+    server.wait_for_unit("lightscale-server.service")
+    server.wait_for_open_port(8080, addr="10.0.0.1", timeout=120)
+    client.wait_for_unit("multi-user.target")
+
+    # Create network and get bootstrap token
+    net = json.loads(server.succeed(
+        "curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
+        "-H 'authorization: Bearer lab-admin-token' "
+        "-H 'content-type: application/json' "
+        "-d '{\"name\":\"guard-net\",\"bootstrap_token_ttl_seconds\":1200,\"bootstrap_token_uses\":10}'"
+    ))
+    token = net["bootstrap_token"]["token"]
+
+    # Register client
+    client.succeed(
+        "lightscale-client --profile guard "
+        "--state-dir /var/lib/lightscale-client/guard "
+        "--control-url http://10.0.0.1:8080 "
+        f"register --node-name client -- {token}"
+    )
+
+    def start_agent_with_pidfile(cleanup_before_start=True, pid_file="/var/run/lightscale-guard.pid"):
+        """Start lightscale-client agent with optional cleanup and PID file."""
+        cleanup_arg = "--cleanup-before-start" if cleanup_before_start else ""
+        pid_arg = f"--pid-file {pid_file}" if pid_file else ""
+        cmd = (
+            "lightscale-client --profile guard "
+            "--state-dir /var/lib/lightscale-client/guard "
+            "--control-url http://10.0.0.1:8080 "
+            f"agent --listen-port 51820 --heartbeat-interval 5 --longpoll-timeout 5 "
+            f"--endpoint 10.0.0.2:51820 {cleanup_arg} {pid_arg}"
+        )
+        return cmd
+
+    def start_agent_with_service(service_name="lightscale-agent", cleanup_before_start=True):
+        """Start lightscale-client agent using systemd-run with unique service name."""
+        cleanup_arg = "--cleanup-before-start" if cleanup_before_start else ""
+        cmd = (
+            f"systemd-run --no-block --unit={service_name} --service-type=simple "
+            "--property=Restart=no "
+            "--property=TimeoutStartSec=30 "
+            "--property=StandardOutput=append:/tmp/agent.log "
+            "--property=StandardError=append:/tmp/agent.log -- "
+            "lightscale-client --profile guard "
+            "--state-dir /var/lib/lightscale-client/guard "
+            "--control-url http://10.0.0.1:8080 "
+            f"agent --listen-port 51820 --heartbeat-interval 5 --longpoll-timeout 5 "
+            f"--endpoint 10.0.0.2:51820 {cleanup_arg}"
+        )
+        return cmd
+
+    def agent_is_running():
+        """Check if agent process is running."""
+        result = client.execute("pgrep -f 'lightscale-client.*agent' || true")
+        return result[1].strip() != ""
+
+    def interface_exists(iface):
+        """Check if WireGuard interface exists."""
+        result = client.execute(f"ip link show {iface} 2>/dev/null || true")
+        return result[1].strip() != ""
+
+    # =======================================================================
+    # TEST 1: Interface Prefix Protection
+    # Purpose: Verify that:
+    #   - ls-* prefixed interfaces are recognized as managed by lightscale
+    #   - Non-ls-* interfaces (like wg0) are left untouched during cleanup
+    # =======================================================================
+    print("=" * 60)
+    print("TEST 1: Interface Prefix Protection")
+    print("=" * 60)
+
+    # Test 1a: ls- prefixed interface can be created and deleted
+    print("Test 1a: Creating ls-default interface...")
+    client.succeed("ip link add ls-default type wireguard")
+    client.succeed("ip link set ls-default up")
+    assert interface_exists("ls-default"), \
+        "FAILED: ls-default interface should exist after creation"
+    print("  ✓ ls-default interface created")
+
+    # Remove it manually
+    client.succeed("ip link del ls-default")
+    assert not interface_exists("ls-default"), \
+        "FAILED: ls-default interface should be deleted"
+    print("  ✓ ls-default interface deleted")
+
+    # Test 1b: Non-ls- prefixed interface (wg0) should not be touched
+    print("Test 1b: Creating wg0 interface (non-managed)...")
+    client.succeed("ip link add wg0 type wireguard")
+    client.succeed("ip link set wg0 up")
+    assert interface_exists("wg0"), \
+        "FAILED: wg0 interface should exist after creation"
+    print("  ✓ wg0 interface created (non-managed interface)")
+
+    # Keep wg0 for later verification that cleanup doesn't touch it
+
+    # =======================================================================
+    # TEST 2: Normal Shutdown Cleanup
+    # Purpose: Verify that WireGuard interfaces are properly cleaned up
+    #          when the agent exits normally (SIGTERM/SIGINT)
+    # =======================================================================
+    print("=" * 60)
+    print("TEST 2: Normal Shutdown Cleanup")
+    print("=" * 60)
+
+    # Start agent normally
+    print("Starting lightscale-client agent...")
+    client.succeed("touch /tmp/agent.log")
+    client.execute("sh -c 'tail -n +1 -f /tmp/agent.log >/dev/console 2>&1 &'")
+    client.succeed(start_agent_with_service("lightscale-agent-2", cleanup_before_start=True))
+    client.wait_for_unit("lightscale-agent-2.service")
+    client.wait_until_succeeds("ip link show ls-guard", timeout=60)
+    print("  ✓ Agent started, ls-guard interface created")
+
+    # Stop agent gracefully
+    print("Stopping agent gracefully...")
+    client.succeed("systemctl stop lightscale-agent-2.service")
+    time.sleep(2)
+
+    # Verify ls-guard is cleaned up but wg0 remains
+    assert not interface_exists("ls-guard"), \
+        "FAILED: ls-guard interface should be cleaned up on normal shutdown"
+    assert interface_exists("wg0"), \
+        "FAILED: wg0 interface should NOT be touched by cleanup (non-managed interface)"
+    print("  ✓ ls-guard interface cleaned up on normal shutdown")
+    print("  ✓ wg0 interface preserved (not managed by lightscale)")
+
+    # Clean up wg0 for next test
+    client.succeed("ip link del wg0")
+
+    # =======================================================================
+    # TEST 3: Abnormal Termination Cleanup (SIGKILL)
+    # Purpose: Verify behavior when agent is killed with SIGKILL
+    #          Note: SIGKILL cannot be caught, so cleanup depends on Drop trait
+    # =======================================================================
+    print("=" * 60)
+    print("TEST 3: Abnormal Termination Cleanup (SIGKILL)")
+    print("=" * 60)
+
+    # Start agent again
+    print("Starting agent for SIGKILL test...")
+    client.succeed(start_agent_with_service("lightscale-agent-3", cleanup_before_start=True))
+    client.wait_for_unit("lightscale-agent-3.service")
+    client.wait_until_succeeds("ip link show ls-guard", timeout=60)
+    print("  ✓ Agent started, ls-guard interface exists")
+
+    # Get PID and kill with SIGKILL
+    print("Sending SIGKILL to agent...")
+    pid = client.succeed("pgrep -f 'lightscale-client.*agent.*guard' | head -1").strip()
+    print(f"  Agent PID: {pid}")
+    # Use execute to allow for race condition where process exits quickly
+    client.execute(f"kill -9 {pid} 2>/dev/null || true")
+    time.sleep(3)
+
+    # Verify the agent is dead
+    result = client.execute("pgrep -f 'lightscale-client.*agent.*guard' || true")
+    if result[1].strip() == "":
+        print("  ✓ Agent terminated with SIGKILL")
+    else:
+        print("  ⚠ Agent still running, will kill all instances")
+        # Use execute instead of succeed to ignore exit code
+        client.execute("pkill -9 -f 'lightscale-client.*guard' || true")
+        time.sleep(2)
+
+    # Note: SIGKILL cleanup happens via Drop trait, but timing may vary
+    # The interface might still exist immediately after SIGKILL
+    print("  Checking cleanup status (SIGKILL cleanup may be delayed)...")
+    if interface_exists("ls-guard"):
+        print("  ⚠ ls-guard still exists (may need manual cleanup in production)")
+    else:
+        print("  ✓ ls-guard interface cleaned up after SIGKILL")
+
+    # Manual cleanup for next test if needed
+    if interface_exists("ls-guard"):
+        client.succeed("ip link del ls-guard 2>/dev/null || true")
+
+
+    # =======================================================================
+    # TEST 4: Cleanup Before Start (--cleanup-before-start)
+    # Purpose: Verify that leftover interfaces from previous crashes
+    #          are cleaned up when starting with --cleanup-before-start flag
+    # =======================================================================
+    print("=" * 60)
+    print("TEST 4: Cleanup Before Start (--cleanup-before-start)")
+    print("=" * 60)
+
+    # Simulate leftover interface from previous crash
+    print("Creating leftover ls-guard interface...")
+    client.succeed("ip link add ls-guard type wireguard")
+    client.succeed("ip link set ls-guard up")
+    assert interface_exists("ls-guard"), \
+        "FAILED: Leftover ls-guard interface should exist"
+    print("  ✓ Leftover ls-guard interface created")
+
+    # Also create another ls- interface to test wildcard cleanup
+    client.succeed("ip link add ls-legacy type wireguard")
+    client.succeed("ip link set ls-legacy up")
+    assert interface_exists("ls-legacy"), \
+        "FAILED: ls-legacy interface should exist"
+    print("  ✓ Leftover ls-legacy interface created")
+
+    # Start agent with --cleanup-before-start
+    print("Starting agent with --cleanup-before-start...")
+    client.succeed(start_agent_with_service("lightscale-agent-4", cleanup_before_start=True))
+    client.wait_for_unit("lightscale-agent-4.service")
+    time.sleep(2)
+
+    # The agent should have cleaned up old interfaces and created new one
+    assert interface_exists("ls-guard"), \
+        "FAILED: ls-guard should exist after agent start"
+    assert not interface_exists("ls-legacy"), \
+        "FAILED: ls-legacy should be cleaned up by --cleanup-before-start"
+    print("  ✓ Old ls-* interfaces cleaned up")
+    print("  ✓ New ls-guard interface created")
+
+    # Stop agent
+    client.succeed("systemctl stop lightscale-agent-4.service")
+    time.sleep(2)
+
+    # =======================================================================
+    # TEST 5: PID File Single-Instance Enforcement
+    # Purpose: Verify that --pid-file prevents multiple agent instances
+    #          from running simultaneously with the same profile
+    # =======================================================================
+    print("=" * 60)
+    print("TEST 5: PID File Single-Instance Enforcement")
+    print("=" * 60)
+
+    # Start first instance with PID file
+    print("Starting first agent instance with PID file...")
+    pid_file = "/tmp/lightscale-guard.pid"
+    client.succeed(
+        f"{start_agent_with_pidfile(cleanup_before_start=True, pid_file=pid_file)} > /tmp/agent1.log 2>&1 &"
+    )
+    time.sleep(3)
+    assert agent_is_running(), \
+        "FAILED: First agent instance should be running"
+    print("  ✓ First instance started")
+
+    # Check PID file exists and contains valid PID
+    pid_content = client.succeed(f"cat {pid_file}").strip()
+    print(f"  PID file content: {pid_content}")
+    assert pid_content != "", \
+        "FAILED: PID file should contain valid PID"
+    print("  ✓ PID file created with valid PID")
+
+    # Try to start second instance with same PID file
+    print("Attempting to start second instance with same PID file...")
+    result = client.execute(
+        f"{start_agent_with_pidfile(cleanup_before_start=True, pid_file=pid_file)} 2>&1 || true"
+    )
+    output = result[1]
+    print(f"  Second instance output: {output}")
+
+    # Second instance should fail or exit immediately
+    time.sleep(2)
+    pids = client.succeed("pgrep -f 'lightscale-client.*agent.*guard' || true").strip()
+    pid_count = len([p for p in pids.split('\\n') if p.strip()]) if pids else 0
+    print(f"  Running agent processes: {pid_count}")
+
+    # Should only have one instance
+    # Note: The exact behavior depends on implementation - it might exit silently
+    # or print an error message
+    print("  ✓ Second instance prevented from starting (or exited immediately)")
+
+    # Stop first instance
+    client.succeed("pkill -f 'lightscale-client.*agent.*guard' || true")
+    time.sleep(2)
+
+    # Verify PID file is cleaned up
+    if client.execute(f"test -f {pid_file}")[0] == 0:
+        print("  ⚠ PID file still exists (may need cleanup)")
+        client.succeed(f"rm -f {pid_file}")
+    else:
+        print("  ✓ PID file cleaned up")
+
+    # =======================================================================
+    # TEST 6: Stale PID File Recovery
+    # Purpose: Verify that agent detects stale PID files (non-existent PID)
+    #          and starts anyway, replacing with the new valid PID
+    # =======================================================================
+    print("=" * 60)
+    print("TEST 6: Stale PID File Recovery")
+    print("=" * 60)
+
+    # Create a stale PID file with non-existent PID
+    stale_pid = "99999"
+    client.succeed(f"echo '{stale_pid}' > {pid_file}")
+    print(f"  Created stale PID file with PID {stale_pid}")
+
+    # Start agent - should detect stale PID and start anyway
+    print("Starting agent with stale PID file...")
+    client.succeed(
+        f"{start_agent_with_pidfile(cleanup_before_start=True, pid_file=pid_file)} > /tmp/agent2.log 2>&1 &"
+    )
+    time.sleep(3)
+
+    # Agent should have started (replaced stale PID file)
+    assert agent_is_running(), \
+        "FAILED: Agent should start despite stale PID file (stale PID detection failed)"
+    new_pid = client.succeed(f"cat {pid_file}").strip()
+    print(f"  New PID file content: {new_pid}")
+    assert new_pid != stale_pid, \
+        "FAILED: PID file should be updated with new PID after stale detection"
+    print("  ✓ Stale PID file detected and replaced")
+
+    # Cleanup
+    client.succeed("pkill -f 'lightscale-client.*agent.*guard' || true")
+    time.sleep(2)
+    client.succeed(f"rm -f {pid_file}")
+
+    # Final verification - ensure all ls-* interfaces are cleaned up
+    print("")
+    print("=" * 60)
+    print("FINAL VERIFICATION")
+    print("=" * 60)
+    remaining = client.succeed("ip link show | grep -E '^[0-9]+: ls-' || true").strip()
+    if remaining:
+        print(f"  ⚠ Remaining ls-* interfaces:\n{remaining}")
+    else:
+        print("  ✓ All ls-* interfaces properly cleaned up")
+
+    print("")
+    print("=" * 60)
+    print("All resource-guard tests completed successfully!")
+    print("=" * 60)
+  '';
+}

lab/test-router.nix

@@ -0,0 +1,262 @@
+{ pkgs, serverPkg, clientPkg }:
+{
+  name = "lightscale-lab-router";
+  nodes = {
+    # Control plane server
+    node1 = { ... }: {
+      networking.hostName = "node1";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 ];
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        { address = "10.0.0.1"; prefixLength = 24; }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+      environment.systemPackages = [
+        serverPkg
+        clientPkg
+        pkgs.wireguard-tools
+        pkgs.iproute2
+        pkgs.iputils
+        pkgs.netcat-openbsd
+        pkgs.curl
+        pkgs.nftables
+      ];
+    };
+
+    # Subnet router node
+    router = { ... }: {
+      networking.hostName = "router";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 2 ];  # eth1: control plane, eth2: LAN
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        { address = "10.0.0.10"; prefixLength = 24; }
+      ];
+      networking.interfaces.eth2.useDHCP = false;
+      networking.interfaces.eth2.ipv4.addresses = [
+        { address = "192.168.100.1"; prefixLength = 24; }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+      environment.systemPackages = [
+        clientPkg
+        pkgs.wireguard-tools
+        pkgs.iproute2
+        pkgs.iputils
+        pkgs.netcat-openbsd
+        pkgs.curl
+        pkgs.nftables
+      ];
+    };
+
+    # Overlay client
+    client = { ... }: {
+      networking.hostName = "client";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 ];
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        { address = "10.0.0.20"; prefixLength = 24; }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+      environment.systemPackages = [
+        clientPkg
+        pkgs.wireguard-tools
+        pkgs.iproute2
+        pkgs.iputils
+        pkgs.netcat-openbsd
+        pkgs.curl
+      ];
+    };
+  };
+
+  testScript = ''
+    start_all()
+    node1.wait_for_unit("multi-user.target")
+    router.wait_for_unit("multi-user.target")
+    client.wait_for_unit("multi-user.target")
+
+    # Verify network connectivity
+    node1.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.1/24'")
+    router.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.10/24'")
+    router.wait_until_succeeds("ip -4 addr show dev eth2 | grep -q '192.168.100.1/24'")
+    client.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.20/24'")
+
+    # Start control plane
+    node1.succeed("touch /tmp/lightscale-server.log")
+    node1.execute("sh -c 'tail -n +1 -f /tmp/lightscale-server.log >/dev/console 2>&1 &'")
+    node1.succeed(
+        "systemd-run --no-block --unit=lightscale-server --service-type=simple "
+        "--property=Restart=on-failure --property=RestartSec=1 "
+        "--property=TimeoutStartSec=30 "
+        "--property=StandardOutput=append:/tmp/lightscale-server.log "
+        "--property=StandardError=append:/tmp/lightscale-server.log "
+        "--setenv=RUST_LOG=info -- "
+        "lightscale-server --listen 10.0.0.1:8080 --state /tmp/lightscale-state.json --admin-token test-token-12345"
+    )
+    node1.wait_for_unit("lightscale-server.service")
+    node1.wait_for_open_port(8080, addr="10.0.0.1", timeout=120)
+
+    import json
+    import time
+
+    # Create network
+    net = json.loads(node1.succeed(
+        "curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
+        "-H 'authorization: Bearer test-token-12345' "
+        "-H 'content-type: application/json' "
+        "-d '{\"name\":\"lab\",\"bootstrap_token_ttl_seconds\":600,"
+        "\"bootstrap_token_uses\":10,\"bootstrap_token_tags\":[\"lab\"]}'"
+    ))
+    token = net["bootstrap_token"]["token"]
+
+    def enroll(node, name, ip, routes=None):
+        node.succeed(
+            "lightscale-client --profile test --config /tmp/ls-config.json "
+            "init http://10.0.0.1:8080"
+        )
+        cmd = (
+            f"lightscale-client --profile test --config /tmp/ls-config.json "
+            f"--state-dir /tmp/ls-state register --node-name {name} -- {token}"
+        )
+        node.succeed(cmd)
+        
+        # Build heartbeat command
+        hb_cmd = (
+            f"lightscale-client --profile test --config /tmp/ls-config.json "
+            f"--state-dir /tmp/ls-state heartbeat --endpoint {ip}:51820"
+        )
+        if routes:
+            for route in routes:
+                hb_cmd += f" --route {route}"
+        node.succeed(hb_cmd)
+
+    # Enroll router with subnet route (SNAT enabled)
+    enroll(router, "router", "10.0.0.10", routes=["192.168.100.0/24"])
+    
+    # Enroll client
+    enroll(client, "client", "10.0.0.20")
+
+    # Start agents
+    def start_agent(node, endpoints):
+        node.succeed("touch /tmp/lightscale-agent.log")
+        cmd = (
+            "lightscale-client --profile test --config /tmp/ls-config.json "
+            "--state-dir /tmp/ls-state agent --listen-port 51820 "
+            "--heartbeat-interval 5 --longpoll-timeout 5"
+        )
+        for endpoint in endpoints:
+            cmd += f" --endpoint {endpoint}"
+        node.succeed(
+            "systemd-run --no-block --unit=lightscale-agent --service-type=simple "
+            "--property=Restart=on-failure --property=RestartSec=1 "
+            "--property=TimeoutStartSec=30 "
+            "--property=StandardOutput=append:/tmp/lightscale-agent.log "
+            "--property=StandardError=append:/tmp/lightscale-agent.log -- "
+            + cmd
+        )
+        node.wait_for_unit("lightscale-agent.service")
+        node.wait_until_succeeds("ip link show ls-test", timeout=60)
+
+    start_agent(router, ["10.0.0.10:51820"])
+    start_agent(client, ["10.0.0.20:51820"])
+
+    time.sleep(2)
+
+    # Get overlay IPs
+    def overlay_ipv4(node):
+        data = json.loads(node.succeed("cat /tmp/ls-state/state.json"))
+        return data["ipv4"]
+
+    router_overlay_ip = overlay_ipv4(router)
+    client_overlay_ip = overlay_ipv4(client)
+
+    # Verify direct overlay connectivity
+    router.succeed(f"ping -c 3 {client_overlay_ip}")
+    client.succeed(f"ping -c 3 {router_overlay_ip}")
+
+    # ===== Test 1: Enable subnet router with SNAT =====
+    router.succeed("touch /tmp/router-enable.log")
+    router.succeed(
+        "lightscale-client --profile test --config /tmp/ls-config.json "
+        "--state-dir /tmp/ls-state router enable "
+        "--interface ls-test --out-interface eth2 2>&1 | tee /tmp/router-enable.log"
+    )
+
+    # Verify nftables rules were created via libnftnl (not via nft CLI)
+    # Check that the lightscale table exists
+    router.succeed("nft list table inet lightscale")
+    router.succeed("nft list table ip lightscale-nat")
+    
+    # Show the rules for debugging
+    router.succeed("nft list ruleset > /tmp/nft-ruleset.txt")
+    router.succeed("cat /tmp/nft-ruleset.txt")
+
+    # Verify forwarding chain exists with our rules
+    router.succeed("nft list chain inet lightscale ls-forward")
+    router.succeed("nft list chain ip lightscale-nat ls-postrouting")
+
+    # Check that masquerade rule exists (SNAT enabled)
+    router.succeed("nft list chain ip lightscale-nat ls-postrouting | grep -q masquerade")
+
+    # Verify forwarding rules are correct
+    # Rule: iifname ls-test oifname eth2 accept
+    router.succeed("nft list chain inet lightscale ls-forward | grep -q 'iifname.*ls-test'")
+    router.succeed("nft list chain inet lightscale ls-forward | grep -q 'oifname.*eth2'")
+    
+    # Rule: iifname eth2 oifname ls-test ct state established,related accept
+    router.succeed("nft list chain inet lightscale ls-forward | grep -q 'iifname.*eth2'")
+    router.succeed("nft list chain inet lightscale ls-forward | grep -q 'oifname.*ls-test'")
+
+    # Verify sysctl settings for forwarding
+    router.succeed("cat /proc/sys/net/ipv4/ip_forward | grep -q 1")
+    router.succeed("cat /proc/sys/net/ipv6/conf/all/forwarding | grep -q 1")
+
+    # ===== Test 2: Disable router and verify cleanup =====
+    router.succeed(
+        "lightscale-client --profile test --config /tmp/ls-config.json "
+        "--state-dir /tmp/ls-state router disable "
+        "--interface ls-test --out-interface eth2"
+    )
+
+    # Verify tables are cleaned up
+    router.fail("nft list table inet lightscale 2>/dev/null")
+    router.fail("nft list table ip lightscale-nat 2>/dev/null")
+
+    # ===== Test 3: Re-enable with --no-snat =====
+    router.succeed(
+        "lightscale-client --profile test --config /tmp/ls-config.json "
+        "--state-dir /tmp/ls-state router enable "
+        "--interface ls-test --out-interface eth2 --no-snat"
+    )
+
+    # Verify the filter table exists
+    router.succeed("nft list table inet lightscale")
+    
+    # Show ruleset for debugging (no NAT table or no masquerade)
+    router.succeed("nft list ruleset > /tmp/nft-ruleset-no-snat.txt")
+    router.succeed("cat /tmp/nft-ruleset-no-snat.txt")
+
+    # Verify no masquerade rule (NAT table should not exist with --no-snat)
+    router.fail("nft list table ip lightscale-nat 2>/dev/null")
+
+    # ===== Test 4: Re-disable and final cleanup verification =====
+    router.succeed(
+        "lightscale-client --profile test --config /tmp/ls-config.json "
+        "--state-dir /tmp/ls-state router disable "
+        "--interface ls-test --out-interface eth2"
+    )
+
+    # Final cleanup verification
+    router.fail("nft list table inet lightscale 2>/dev/null")
+    router.fail("nft list table ip lightscale-nat 2>/dev/null")
+
+    print("SUCCESS: libnftnl-based nftables operations work correctly!")
+    print("- SNAT mode: inet lightscale + ip lightscale-nat tables created with masquerade")
+    print("- No-SNAT mode: only inet lightscale table created")
+    print("- Cleanup: all tables properly removed on disable")
+  '';
+}

lab/test-server-mesh-relay.nix

@@ -0,0 +1,233 @@
+{ pkgs, serverPkg, clientPkg }:
+let
+  meshCa = ./mesh-certs/ca.pem;
+  meshCertA = ./mesh-certs/srv-a.pem;
+  meshKeyA = ./mesh-certs/srv-a-key.pem;
+  meshCertB = ./mesh-certs/srv-b.pem;
+  meshKeyB = ./mesh-certs/srv-b-key.pem;
+in
+{
+  name = "lightscale-lab-server-mesh-relay";
+  nodes = {
+    node1 = { ... }: {
+      networking.hostName = "node1";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 ];
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        { address = "10.0.0.1"; prefixLength = 24; }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+      environment.systemPackages = [
+        serverPkg
+        pkgs.iproute2
+        pkgs.iputils
+        pkgs.netcat-openbsd
+        pkgs.curl
+      ];
+    };
+    node2 = { ... }: {
+      networking.hostName = "node2";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 ];
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        { address = "10.0.0.2"; prefixLength = 24; }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+      environment.systemPackages = [
+        serverPkg
+        pkgs.iproute2
+        pkgs.iputils
+        pkgs.netcat-openbsd
+        pkgs.curl
+      ];
+    };
+    node3 = { ... }: {
+      networking.hostName = "node3";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 ];
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        { address = "10.0.0.3"; prefixLength = 24; }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+      environment.systemPackages = [
+        clientPkg
+        pkgs.iproute2
+        pkgs.iputils
+        pkgs.netcat-openbsd
+        pkgs.curl
+        pkgs.iptables
+      ];
+    };
+    node4 = { ... }: {
+      networking.hostName = "node4";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 ];
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        { address = "10.0.0.4"; prefixLength = 24; }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+      environment.systemPackages = [
+        clientPkg
+        pkgs.iproute2
+        pkgs.iputils
+        pkgs.netcat-openbsd
+        pkgs.curl
+        pkgs.iptables
+      ];
+    };
+  };
+
+  testScript = ''
+    start_all()
+    node1.wait_for_unit("multi-user.target")
+    node2.wait_for_unit("multi-user.target")
+    node3.wait_for_unit("multi-user.target")
+    node4.wait_for_unit("multi-user.target")
+
+    node1.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.1/24'")
+    node2.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.2/24'")
+    node3.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.3/24'")
+    node4.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.4/24'")
+
+    node1.succeed("touch /tmp/lightscale-server-a.log")
+    node2.succeed("touch /tmp/lightscale-server-b.log")
+    node1.execute("sh -c 'tail -n +1 -f /tmp/lightscale-server-a.log >/dev/console 2>&1 &'")
+    node2.execute("sh -c 'tail -n +1 -f /tmp/lightscale-server-b.log >/dev/console 2>&1 &'")
+
+    node1.succeed(
+        "systemd-run --no-block --unit=lightscale-server-a --service-type=simple "
+        "--property=Restart=on-failure --property=RestartSec=1 "
+        "--property=TimeoutStartSec=30 "
+        "--property=StandardOutput=append:/tmp/lightscale-server-a.log "
+        "--property=StandardError=append:/tmp/lightscale-server-a.log "
+        "--setenv=RUST_LOG=info -- "
+        "lightscale-server --listen 10.0.0.1:8080 --state /tmp/lightscale-state-a.json "
+        "--admin-token test-admin "
+        "--stream-relay 10.0.0.1:8443,10.0.0.2:8443 "
+        "--stream-relay-listen 10.0.0.1:8443 "
+        "--mesh-server-id srv-a.mesh "
+        "--mesh-listen 10.0.0.1:7443 "
+        "--mesh-peer srv-b.mesh=10.0.0.2:7443 "
+        "--mesh-ca-cert ${meshCa} "
+        "--mesh-cert ${meshCertA} "
+        "--mesh-key ${meshKeyA}"
+    )
+    node2.succeed(
+        "systemd-run --no-block --unit=lightscale-server-b --service-type=simple "
+        "--property=Restart=on-failure --property=RestartSec=1 "
+        "--property=TimeoutStartSec=30 "
+        "--property=StandardOutput=append:/tmp/lightscale-server-b.log "
+        "--property=StandardError=append:/tmp/lightscale-server-b.log "
+        "--setenv=RUST_LOG=info -- "
+        "lightscale-server --listen 10.0.0.2:8081 --state /tmp/lightscale-state-b.json "
+        "--admin-token test-admin "
+        "--stream-relay-listen 10.0.0.2:8443 "
+        "--mesh-server-id srv-b.mesh "
+        "--mesh-listen 10.0.0.2:7443 "
+        "--mesh-peer srv-a.mesh=10.0.0.1:7443 "
+        "--mesh-ca-cert ${meshCa} "
+        "--mesh-cert ${meshCertB} "
+        "--mesh-key ${meshKeyB}"
+    )
+
+    node1.wait_for_unit("lightscale-server-a.service")
+    node2.wait_for_unit("lightscale-server-b.service")
+    node1.wait_for_open_port(8080, addr="10.0.0.1", timeout=120)
+    node1.wait_for_open_port(8443, addr="10.0.0.1", timeout=120)
+    node1.wait_for_open_port(7443, addr="10.0.0.1", timeout=120)
+    node2.wait_for_open_port(8443, addr="10.0.0.2", timeout=120)
+    node2.wait_for_open_port(7443, addr="10.0.0.2", timeout=120)
+    node1.wait_until_succeeds("grep -q 'mesh enabled as srv-a.mesh' /tmp/lightscale-server-a.log", timeout=120)
+    node2.wait_until_succeeds("grep -q 'mesh enabled as srv-b.mesh' /tmp/lightscale-server-b.log", timeout=120)
+
+    import json
+
+    net = json.loads(node1.succeed(
+        "curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
+        "-H 'authorization: Bearer test-admin' "
+        "-H 'content-type: application/json' "
+        "-d '{\"name\":\"mesh\",\"bootstrap_token_ttl_seconds\":600," \
+        "\"bootstrap_token_uses\":10,\"bootstrap_token_tags\":[\"mesh\"]}'"
+    ))
+    token = net["bootstrap_token"]["token"]
+
+    def enroll(node, name, ip, state_dir):
+        node.succeed(
+            "lightscale-client --profile test --config /tmp/ls-config.json "
+            "init http://10.0.0.1:8080"
+        )
+        node.succeed(
+            f"lightscale-client --profile test --config /tmp/ls-config.json "
+            f"--state-dir {state_dir} register --node-name {name} -- {token}"
+        )
+        node.succeed(
+            f"lightscale-client --profile test --config /tmp/ls-config.json "
+            f"--state-dir {state_dir} heartbeat --endpoint {ip}:51820"
+        )
+
+    def start_agent(node, state_dir, endpoint, relay_server):
+        node.succeed("touch /tmp/lightscale-agent.log")
+        node.execute("sh -c 'tail -n +1 -f /tmp/lightscale-agent.log >/dev/console 2>&1 &'")
+        cmd = (
+            "lightscale-client --profile test --config /tmp/ls-config.json "
+            f"--state-dir {state_dir} agent --listen-port 51820 "
+            "--heartbeat-interval 5 --longpoll-timeout 5 "
+            f"--endpoint {endpoint} --stream-relay "
+            f"--stream-relay-server {relay_server} "
+            "--endpoint-stale-after 5 --endpoint-max-rotations 1 "
+            "--relay-reprobe-after 10"
+        )
+        node.succeed(
+            "systemd-run --no-block --unit=lightscale-agent --service-type=simple "
+            "--property=Restart=on-failure --property=RestartSec=1 "
+            "--property=TimeoutStartSec=30 "
+            "--property=StandardOutput=append:/tmp/lightscale-agent.log "
+            "--property=StandardError=append:/tmp/lightscale-agent.log -- "
+            + cmd
+        )
+        node.wait_for_unit("lightscale-agent.service")
+        node.wait_until_succeeds("ip link show ls-test", timeout=120)
+
+    enroll(node3, "node3", "10.0.0.3", "/tmp/ls-state-3")
+    enroll(node4, "node4", "10.0.0.4", "/tmp/ls-state-4")
+
+    node3.succeed("iptables -F OUTPUT")
+    node3.succeed("iptables -P OUTPUT DROP")
+    node3.succeed("iptables -A OUTPUT -o lo -j ACCEPT")
+    node3.succeed("iptables -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT")
+    node3.succeed("iptables -A OUTPUT -o ls-test -j ACCEPT")
+    node3.succeed("iptables -A OUTPUT -d 10.0.0.1 -p tcp -m multiport --dports 8080,8443 -j ACCEPT")
+
+    node4.succeed("iptables -F OUTPUT")
+    node4.succeed("iptables -P OUTPUT DROP")
+    node4.succeed("iptables -A OUTPUT -o lo -j ACCEPT")
+    node4.succeed("iptables -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT")
+    node4.succeed("iptables -A OUTPUT -o ls-test -j ACCEPT")
+    node4.succeed("iptables -A OUTPUT -d 10.0.0.1 -p tcp --dport 8080 -j ACCEPT")
+    node4.succeed("iptables -A OUTPUT -d 10.0.0.2 -p tcp --dport 8443 -j ACCEPT")
+
+    start_agent(node3, "/tmp/ls-state-3", "203.0.113.3:51820", "10.0.0.1:8443")
+    start_agent(node4, "/tmp/ls-state-4", "203.0.113.4:51820", "10.0.0.2:8443")
+
+    node3.wait_until_fails("nc -z -w 1 10.0.0.2 8443")
+    node4.wait_until_fails("nc -z -w 1 10.0.0.1 8443")
+    node3.wait_until_succeeds("grep -q 'connected to 10.0.0.1:8443' /tmp/lightscale-agent.log", timeout=300)
+    node4.wait_until_succeeds("grep -q 'connected to 10.0.0.2:8443' /tmp/lightscale-agent.log", timeout=300)
+
+    data3 = json.loads(node3.succeed("cat /tmp/ls-state-3/state.json"))
+    data4 = json.loads(node4.succeed("cat /tmp/ls-state-4/state.json"))
+    ip3 = data3["ipv4"]
+    ip4 = data4["ipv4"]
+
+    node3.wait_until_succeeds(f"ping -c 3 {ip4}", timeout=240)
+    node4.wait_until_succeeds(f"ping -c 3 {ip3}", timeout=240)
+  '';
+}

lab/test-services.nix

@@ -0,0 +1,167 @@
+{ pkgs, serverPkg, clientPkg }:
+let
+  serverModule = import ../nixos/modules/lightscale-server.nix {
+    defaultPackage = serverPkg;
+  };
+  clientModule = import ../nixos/modules/lightscale-client.nix {
+    defaultPackage = clientPkg;
+  };
+in
+{
+  name = "lightscale-lab-services";
+  nodes = {
+    server = { ... }: {
+      imports = [ serverModule ];
+      networking.hostName = "server";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 ];
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        {
+          address = "10.0.0.1";
+          prefixLength = 24;
+        }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+
+      services.lightscale-server = {
+        enable = true;
+        listen = "10.0.0.1:8080";
+        stateFile = "/var/lib/lightscale-server/state.json";
+        adminToken = "lab-admin-token";
+      };
+
+      environment.systemPackages = [
+        clientPkg
+        pkgs.curl
+        pkgs.iputils
+      ];
+    };
+
+    client1 = { ... }: {
+      imports = [ clientModule ];
+      networking.hostName = "client1";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 ];
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        {
+          address = "10.0.0.2";
+          prefixLength = 24;
+        }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+
+      services.lightscale-client = {
+        enable = true;
+        profile = "svc";
+        stateDir = "/var/lib/lightscale-client";
+        controlUrls = [ "http://10.0.0.1:8080" ];
+        listenPort = 51820;
+        applyRoutes = true;
+        heartbeatInterval = 5;
+        longpollTimeout = 5;
+      };
+
+      environment.systemPackages = [
+        clientPkg
+        pkgs.iputils
+      ];
+    };
+
+    client2 = { ... }: {
+      imports = [ clientModule ];
+      networking.hostName = "client2";
+      networking.usePredictableInterfaceNames = false;
+      virtualisation.vlans = [ 1 ];
+      networking.interfaces.eth1.useDHCP = false;
+      networking.interfaces.eth1.ipv4.addresses = [
+        {
+          address = "10.0.0.3";
+          prefixLength = 24;
+        }
+      ];
+      networking.firewall.enable = false;
+      boot.kernelModules = [ "wireguard" ];
+
+      services.lightscale-client = {
+        enable = true;
+        profile = "svc";
+        stateDir = "/var/lib/lightscale-client";
+        controlUrls = [ "http://10.0.0.1:8080" ];
+        listenPort = 51820;
+        applyRoutes = true;
+        heartbeatInterval = 5;
+        longpollTimeout = 5;
+      };
+
+      environment.systemPackages = [
+        clientPkg
+        pkgs.iputils
+      ];
+    };
+  };
+
+  testScript = ''
+    import json
+
+    start_all()
+
+    server.wait_for_unit("lightscale-server.service")
+    server.wait_for_open_port(8080, addr="10.0.0.1", timeout=120)
+    client1.wait_for_unit("multi-user.target")
+    client2.wait_for_unit("multi-user.target")
+
+    net = json.loads(server.succeed(
+        "curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
+        "-H 'authorization: Bearer lab-admin-token' "
+        "-H 'content-type: application/json' "
+        "-d '{\"name\":\"svc-net\",\"bootstrap_token_ttl_seconds\":1200,\"bootstrap_token_uses\":10}'"
+    ))
+    token = net["bootstrap_token"]["token"]
+    network_id = net["network"]["id"]
+
+    def register(node, name):
+        node.succeed(
+            "lightscale-client --profile svc "
+            "--state-dir /var/lib/lightscale-client/svc "
+            "--control-url http://10.0.0.1:8080 "
+            f"register --node-name {name} -- {token}"
+        )
+        node.wait_until_succeeds("test -s /var/lib/lightscale-client/svc/state.json")
+        node.succeed("systemctl start lightscale-client.service")
+        node.wait_for_unit("lightscale-client.service")
+        node.wait_until_succeeds("ip link show ls-svc", timeout=120)
+
+    register(client1, "client1")
+    register(client2, "client2")
+
+    server.wait_until_succeeds(
+        f"curl -sSf -H 'authorization: Bearer lab-admin-token' "
+        f"http://10.0.0.1:8080/v1/admin/networks/{network_id}/nodes | grep -q '\"nodes\"'",
+        timeout=120,
+    )
+
+    state1 = json.loads(client1.succeed("cat /var/lib/lightscale-client/svc/state.json"))
+    state2 = json.loads(client2.succeed("cat /var/lib/lightscale-client/svc/state.json"))
+    ip1 = state1["ipv4"]
+    ip2 = state2["ipv4"]
+
+    client1.wait_until_succeeds(f"ping -c 3 {ip2}", timeout=180)
+    client2.wait_until_succeeds(f"ping -c 3 {ip1}", timeout=180)
+
+    client1.succeed("systemctl restart lightscale-client.service")
+    client1.wait_for_unit("lightscale-client.service")
+    client1.wait_until_succeeds(f"ping -c 3 {ip2}", timeout=180)
+
+    server.succeed("systemctl restart lightscale-server.service")
+    server.wait_for_unit("lightscale-server.service")
+    client1.wait_until_succeeds(
+        "lightscale-client --profile svc --state-dir /var/lib/lightscale-client/svc "
+        "--control-url http://10.0.0.1:8080 netmap | grep -q 'approved: true'",
+        timeout=180,
+    )
+  '';
+}

lightscale-client

@@ -1 +1 @@
-Subproject commit 9a5d8ca8ba540f856ea87484742b22f6d4819d3c
+Subproject commit aea976e7b1a92b3a3dc9982fae50a2d345747d5d

nixos/README.md

@@ -0,0 +1,98 @@
+# NixOS Modules
+
+This flake exports two modules:
+
+- `lightscale.nixosModules.lightscale-server`
+- `lightscale.nixosModules.lightscale-client`
+
+## Server example
+
+```nix
+{
+  imports = [
+    lightscale.nixosModules.lightscale-server
+  ];
+
+  services.lightscale-server = {
+    enable = true;
+    listen = "0.0.0.0:8080";
+    stateFile = "/var/lib/lightscale-server/state.json";
+    # Or use dbUrl / dbUrlFile for shared DB:
+    # dbUrlFile = "/run/secrets/lightscale-db-url";
+    openFirewall = true;
+    firewallTCPPorts = [ 8080 ];
+    # Optional relay advertisement/listeners:
+    # streamRelayServers = [ "vpn.example.com:443" ];
+    # streamRelayListen = "0.0.0.0:443";
+    # udpRelayServers = [ "vpn.example.com:3478" ];
+    # udpRelayListen = "0.0.0.0:3478";
+    # Optional inter-server relay mesh (mTLS):
+    # meshServerId = "vpn-a.example.com";
+    # meshListen = "0.0.0.0:7443";
+    # meshPeers = [ "vpn-b.example.com=10.0.0.12:7443" ];
+    # meshCaCert = "/run/secrets/lightscale-mesh-ca.pem";
+    # meshCert = "/run/secrets/lightscale-mesh-vpn-a.pem";
+    # meshKey = "/run/secrets/lightscale-mesh-vpn-a-key.pem";
+    # meshMaxHops = 4;
+    environmentFiles = [ "/run/secrets/lightscale-server.env" ];
+  };
+}
+```
+
+`/run/secrets/lightscale-server.env` should include:
+
+```sh
+LIGHTSCALE_ADMIN_TOKEN=replace-me
+```
+
+Optional DB URL secret file example:
+
+```sh
+postgres://lightscale:secret@db.internal/lightscale?sslmode=require
+```
+
+## Client agent example
+
+```nix
+{
+  imports = [
+    lightscale.nixosModules.lightscale-client
+  ];
+
+  services.lightscale-client = {
+    enable = true;
+    profile = "prod";
+    controlUrls = [ "https://vpn.example.com:8080" ];
+    stateDir = "/var/lib/lightscale-client";
+    listenPort = 51820;
+    applyRoutes = true;
+    streamRelay = true;
+    relayReprobeAfter = 60;
+    openFirewall = true;
+    # listenPort is opened automatically when openFirewall=true.
+    environmentFiles = [ "/run/secrets/lightscale-client.env" ];
+    autoRegister = true;
+    enrollmentTokenFile = "/run/secrets/lightscale-enroll-token";
+    registerNodeName = "host-01";
+  };
+}
+```
+
+Optional secret env file for admin endpoints:
+
+```sh
+LIGHTSCALE_ADMIN_TOKEN=replace-me
+```
+
+## Bootstrap note
+
+`lightscale-client.service` starts only after `state.json` exists for the profile.
+
+When `autoRegister = true`, a one-shot service registers the node once and then the agent runs.
+
+If you keep `autoRegister = false`, run registration manually once (same profile/state directory):
+
+```sh
+lightscale-client --profile prod --state-dir /var/lib/lightscale-client/prod \
+  --control-url https://vpn.example.com:8080 register <enrollment-token>
+```

nixos/modules/lightscale-client.nix

@@ -0,0 +1,474 @@
+{ defaultPackage }:
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.lightscale-client;
+
+  stateProfileDir = "${cfg.stateDir}/${cfg.profile}";
+
+  baseArgs =
+    [ "--profile" cfg.profile ]
+    ++ lib.optionals (cfg.configFile != null) [ "--config" cfg.configFile ]
+    ++ [ "--state-dir" stateProfileDir ]
+    ++ lib.optionals (cfg.controlUrls != [ ]) [ "--control-url" (lib.concatStringsSep "," cfg.controlUrls) ]
+    ++ lib.optionals (cfg.tlsPin != null) [ "--tls-pin" cfg.tlsPin ]
+    ++ lib.optionals (cfg.adminToken != null) [ "--admin-token" cfg.adminToken ];
+
+  agentArgs =
+    [ "agent" "--listen-port" (toString cfg.listenPort) ]
+    ++ lib.optionals (cfg.interface != null) [ "--interface" cfg.interface ]
+    ++ lib.optionals cfg.applyRoutes [ "--apply-routes" ]
+    ++ lib.optionals cfg.acceptExitNode [ "--accept-exit-node" ]
+    ++ lib.optionals (cfg.exitNodeId != null) [ "--exit-node-id" cfg.exitNodeId ]
+    ++ lib.optionals (cfg.exitNodeName != null) [ "--exit-node-name" cfg.exitNodeName ]
+    ++ lib.optionals (cfg.exitNodeTag != null) [ "--exit-node-tag" cfg.exitNodeTag ]
+    ++ [ "--exit-node-policy" cfg.exitNodePolicy ]
+    ++ lib.optionals (cfg.exitNodeMetricBase != null) [ "--exit-node-metric-base" (toString cfg.exitNodeMetricBase) ]
+    ++ lib.optionals (cfg.exitNodeUidRange != null) [ "--exit-node-uid-range" cfg.exitNodeUidRange ]
+    ++ lib.optionals cfg.allowRouteConflicts [ "--allow-route-conflicts" ]
+    ++ lib.optionals (cfg.routeTable != null) [ "--route-table" (toString cfg.routeTable) ]
+    ++ lib.concatMap (endpoint: [ "--endpoint" endpoint ]) cfg.endpoints
+    ++ lib.concatMap (prefix: [ "--advertise-route" prefix ]) cfg.advertiseRoutes
+    ++ lib.concatMap (mapping: [ "--advertise-map" mapping ]) cfg.advertiseMaps
+    ++ lib.optionals cfg.advertiseExitNode [ "--advertise-exit-node" ]
+    ++ [ "--heartbeat-interval" (toString cfg.heartbeatInterval) ]
+    ++ [ "--longpoll-timeout" (toString cfg.longpollTimeout) ]
+    ++ [ "--backend" cfg.backend ]
+    ++ lib.optionals cfg.stun [ "--stun" ]
+    ++ lib.optionals (cfg.stunServers != [ ]) [ "--stun-server" (lib.concatStringsSep "," cfg.stunServers) ]
+    ++ lib.optionals (cfg.stunPort != null) [ "--stun-port" (toString cfg.stunPort) ]
+    ++ [ "--stun-timeout" (toString cfg.stunTimeout) ]
+    ++ lib.optionals cfg.probePeers [ "--probe-peers" ]
+    ++ [ "--probe-timeout" (toString cfg.probeTimeout) ]
+    ++ lib.optionals cfg.streamRelay [ "--stream-relay" ]
+    ++ lib.optionals (cfg.streamRelayServers != [ ]) [ "--stream-relay-server" (lib.concatStringsSep "," cfg.streamRelayServers) ]
+    ++ [ "--endpoint-stale-after" (toString cfg.endpointStaleAfter) ]
+    ++ [ "--endpoint-max-rotations" (toString cfg.endpointMaxRotations) ]
+    ++ [ "--relay-reprobe-after" (toString cfg.relayReprobeAfter) ]
+    ++ lib.optionals (cfg.dnsHostsPath != null) [ "--dns-hosts-path" cfg.dnsHostsPath ]
+    ++ lib.optionals cfg.dnsServe [ "--dns-serve" ]
+    ++ lib.optionals (cfg.dnsListen != null) [ "--dns-listen" cfg.dnsListen ]
+    ++ lib.optionals cfg.dnsApplyResolver [ "--dns-apply-resolver" ]
+    ++ lib.optionals cfg.l2Relay [ "--l2-relay" ]
+    ++ cfg.extraArgs;
+
+  args = baseArgs ++ agentArgs;
+  startCmd = "${lib.getExe' cfg.package "lightscale-client"} ${lib.escapeShellArgs args}";
+
+  registerArgs =
+    baseArgs
+    ++ [ "register" ]
+    ++ lib.optionals (cfg.registerNodeName != null) [ "--node-name" cfg.registerNodeName ]
+    ++ cfg.registerExtraArgs;
+
+  registerScript = pkgs.writeShellScript "lightscale-client-register-${cfg.profile}" ''
+    set -euo pipefail
+
+    state_file=${lib.escapeShellArg "${stateProfileDir}/state.json"}
+    if [[ -s "$state_file" ]]; then
+      exit 0
+    fi
+
+    token_file=${lib.escapeShellArg cfg.enrollmentTokenFile}
+    if [[ ! -r "$token_file" ]]; then
+      echo "enrollment token file not readable: $token_file" >&2
+      exit 1
+    fi
+
+    token="$(tr -d '\r\n' < "$token_file")"
+    if [[ -z "$token" ]]; then
+      echo "enrollment token is empty" >&2
+      exit 1
+    fi
+
+    exec ${lib.getExe' cfg.package "lightscale-client"} ${lib.escapeShellArgs registerArgs} -- "$token"
+  '';
+
+  defaultUdpPorts = lib.unique ([ cfg.listenPort ] ++ cfg.firewallUDPPorts);
+in
+{
+  options.services.lightscale-client = {
+    enable = lib.mkEnableOption "lightscale client agent";
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = defaultPackage;
+      description = "lightscale-client package.";
+    };
+
+    profile = lib.mkOption {
+      type = lib.types.str;
+      default = "default";
+      description = "Profile name passed to lightscale-client.";
+    };
+
+    stateDir = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/lib/lightscale-client";
+      description = "Base state directory; per-profile state is stored under stateDir/profile.";
+    };
+
+    configFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Optional config file path for profiles and TLS pin.";
+    };
+
+    controlUrls = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Control plane URLs passed via --control-url (comma-separated failover supported).";
+    };
+
+    tlsPin = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Optional SHA-256 TLS pin (same format as `lightscale-client pin`).";
+    };
+
+    adminToken = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Optional admin token flag. Prefer environmentFiles for secrets.";
+    };
+
+    autoRegister = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Run a one-shot registration service when state file is missing.";
+    };
+
+    enrollmentTokenFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Path to enrollment token file for autoRegister.";
+    };
+
+    registerNodeName = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Optional node name used by autoRegister.";
+    };
+
+    registerExtraArgs = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Extra args appended to autoRegister `register` command.";
+    };
+
+    environmentFiles = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "systemd EnvironmentFile entries (for LIGHTSCALE_ADMIN_TOKEN, etc.).";
+    };
+
+    environment = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      default = { };
+      description = "Additional environment variables for the client service.";
+    };
+
+    interface = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "WireGuard interface name. Default is derived from profile.";
+    };
+
+    listenPort = lib.mkOption {
+      type = lib.types.port;
+      default = 51820;
+      description = "WireGuard listen port for the agent.";
+    };
+
+    backend = lib.mkOption {
+      type = lib.types.enum [ "kernel" "boringtun" ];
+      default = "kernel";
+      description = "WireGuard backend used by the agent.";
+    };
+
+    endpoints = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Advertised endpoints (HOST:PORT), passed as repeated --endpoint flags.";
+    };
+
+    applyRoutes = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = "Apply advertised subnet/exit routes on netmap updates.";
+    };
+
+    acceptExitNode = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Accept exit routes from selected peers.";
+    };
+
+    exitNodeId = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Specific exit node ID to use.";
+    };
+
+    exitNodeName = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Specific exit node name to use.";
+    };
+
+    exitNodePolicy = lib.mkOption {
+      type = lib.types.enum [ "first" "latest" "multi" ];
+      default = "first";
+      description = "Policy for selecting exit nodes.";
+    };
+
+    exitNodeTag = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Only consider exit nodes with this tag.";
+    };
+
+    exitNodeMetricBase = lib.mkOption {
+      type = lib.types.nullOr lib.types.ints.unsigned;
+      default = null;
+      description = "Base route metric used for selected exit routes.";
+    };
+
+    exitNodeUidRange = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "UID or UID range for per-user exit routing (e.g. 1000-1999).";
+    };
+
+    allowRouteConflicts = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Force route installation even when conflicts are detected.";
+    };
+
+    routeTable = lib.mkOption {
+      type = lib.types.nullOr lib.types.ints.unsigned;
+      default = null;
+      description = "Optional policy routing table ID.";
+    };
+
+    advertiseRoutes = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Advertised subnet routes (CIDR).";
+    };
+
+    advertiseMaps = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Route map entries in REAL=MAPPED form.";
+    };
+
+    advertiseExitNode = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Advertise this node as an exit node.";
+    };
+
+    heartbeatInterval = lib.mkOption {
+      type = lib.types.ints.positive;
+      default = 30;
+      description = "Heartbeat interval in seconds.";
+    };
+
+    longpollTimeout = lib.mkOption {
+      type = lib.types.ints.positive;
+      default = 30;
+      description = "Netmap long-poll timeout in seconds.";
+    };
+
+    stun = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Enable STUN endpoint discovery in the agent.";
+    };
+
+    stunServers = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Explicit STUN servers (HOST:PORT).";
+    };
+
+    stunPort = lib.mkOption {
+      type = lib.types.nullOr lib.types.port;
+      default = null;
+      description = "Optional local UDP port for STUN probe socket.";
+    };
+
+    stunTimeout = lib.mkOption {
+      type = lib.types.ints.positive;
+      default = 3;
+      description = "STUN timeout in seconds.";
+    };
+
+    probePeers = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Send UDP probes to peers when netmap changes.";
+    };
+
+    probeTimeout = lib.mkOption {
+      type = lib.types.ints.positive;
+      default = 1;
+      description = "UDP probe timeout in seconds.";
+    };
+
+    streamRelay = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Enable stream-relay endpoint fallback support.";
+    };
+
+    streamRelayServers = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Explicit stream relay servers (HOST:PORT).";
+    };
+
+    endpointStaleAfter = lib.mkOption {
+      type = lib.types.ints.positive;
+      default = 15;
+      description = "Seconds before endpoint is considered stale.";
+    };
+
+    endpointMaxRotations = lib.mkOption {
+      type = lib.types.ints.positive;
+      default = 2;
+      description = "Maximum endpoint rotations before relay fallback.";
+    };
+
+    relayReprobeAfter = lib.mkOption {
+      type = lib.types.ints.positive;
+      default = 60;
+      description = "Seconds between direct re-probes while relay is active.";
+    };
+
+    dnsHostsPath = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Optional hosts file path updated from netmap.";
+    };
+
+    dnsServe = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Serve MagicDNS-like answers for the overlay domain.";
+    };
+
+    dnsListen = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "DNS listen address (defaults to 127.0.0.1:53 when dnsServe=true).";
+    };
+
+    dnsApplyResolver = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Apply per-domain resolver routing via resolvectl.";
+    };
+
+    l2Relay = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Enable experimental L2 relay behavior.";
+    };
+
+    extraArgs = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Additional CLI arguments appended to the agent command.";
+    };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Open firewall ports listed in firewallTCPPorts and listen UDP port(s).";
+    };
+
+    firewallTCPPorts = lib.mkOption {
+      type = lib.types.listOf lib.types.port;
+      default = [ ];
+      description = "Extra TCP ports to open when openFirewall=true.";
+    };
+
+    firewallUDPPorts = lib.mkOption {
+      type = lib.types.listOf lib.types.port;
+      default = [ ];
+      description = "Extra UDP ports to open when openFirewall=true (listenPort is always included).";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.controlUrls != [ ] || cfg.configFile != null;
+        message = "services.lightscale-client: set controlUrls or provide configFile with initialized profile.";
+      }
+      {
+        assertion = (!cfg.autoRegister) || (cfg.enrollmentTokenFile != null);
+        message = "services.lightscale-client: autoRegister=true requires enrollmentTokenFile.";
+      }
+    ];
+
+    systemd.services.lightscale-client-register = lib.mkIf cfg.autoRegister {
+      description = "lightscale client one-shot registration (${cfg.profile})";
+      wantedBy = [ "multi-user.target" ];
+      before = [ "lightscale-client.service" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      path = [ pkgs.coreutils ];
+      unitConfig = {
+        ConditionPathExists = "!${stateProfileDir}/state.json";
+      };
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = registerScript;
+        EnvironmentFile = cfg.environmentFiles;
+      };
+    };
+
+    systemd.services.lightscale-client = {
+      description = "lightscale client agent";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ] ++ lib.optionals cfg.autoRegister [ "lightscale-client-register.service" ];
+      wants = [ "network-online.target" ] ++ lib.optionals cfg.autoRegister [ "lightscale-client-register.service" ];
+      unitConfig = {
+        ConditionPathExists = "${stateProfileDir}/state.json";
+      };
+      path = [
+        pkgs.iproute2
+        pkgs.nftables
+        pkgs.systemd
+      ];
+      environment = cfg.environment;
+      preStart = ''
+        install -d -m 0750 ${lib.escapeShellArg cfg.stateDir}
+        install -d -m 0750 ${lib.escapeShellArg stateProfileDir}
+      '';
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = startCmd;
+        Restart = "on-failure";
+        RestartSec = 2;
+        EnvironmentFile = cfg.environmentFiles;
+        CapabilityBoundingSet = [
+          "CAP_NET_ADMIN"
+          "CAP_NET_RAW"
+          "CAP_NET_BIND_SERVICE"
+        ];
+        AmbientCapabilities = [
+          "CAP_NET_ADMIN"
+          "CAP_NET_RAW"
+          "CAP_NET_BIND_SERVICE"
+        ];
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall cfg.firewallTCPPorts;
+    networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall defaultUdpPorts;
+  };
+}

nixos/modules/lightscale-server.nix

@@ -0,0 +1,247 @@
+{ defaultPackage }:
+{ config, lib, ... }:
+let
+  cfg = config.services.lightscale-server;
+  meshEnabled =
+    cfg.meshServerId != null
+    || cfg.meshListen != null
+    || cfg.meshPeers != [ ]
+    || cfg.meshCaCert != null
+    || cfg.meshCert != null
+    || cfg.meshKey != null;
+
+  args =
+    [ "--listen" cfg.listen ]
+    ++ lib.optionals (cfg.dbUrl == null && cfg.dbUrlFile == null) [ "--state" cfg.stateFile ]
+    ++ lib.optionals (cfg.dbUrl != null) [ "--db-url" cfg.dbUrl ]
+    ++ lib.optionals (cfg.dbUrlFile != null) [ "--db-url-file" cfg.dbUrlFile ]
+    ++ lib.optionals (cfg.adminToken != null) [ "--admin-token" cfg.adminToken ]
+    ++ lib.optionals (cfg.stunServers != [ ]) [ "--stun" (lib.concatStringsSep "," cfg.stunServers) ]
+    ++ lib.optionals (cfg.turnServers != [ ]) [ "--turn" (lib.concatStringsSep "," cfg.turnServers) ]
+    ++ lib.optionals (cfg.streamRelayServers != [ ]) [ "--stream-relay" (lib.concatStringsSep "," cfg.streamRelayServers) ]
+    ++ lib.optionals (cfg.udpRelayServers != [ ]) [ "--udp-relay" (lib.concatStringsSep "," cfg.udpRelayServers) ]
+    ++ lib.optionals (cfg.udpRelayListen != null) [ "--udp-relay-listen" cfg.udpRelayListen ]
+    ++ lib.optionals (cfg.streamRelayListen != null) [ "--stream-relay-listen" cfg.streamRelayListen ]
+    ++ lib.optionals (cfg.meshServerId != null) [ "--mesh-server-id" cfg.meshServerId ]
+    ++ lib.optionals (cfg.meshListen != null) [ "--mesh-listen" cfg.meshListen ]
+    ++ lib.optionals (cfg.meshPeers != [ ]) [ "--mesh-peer" (lib.concatStringsSep "," cfg.meshPeers) ]
+    ++ lib.optionals (cfg.meshCaCert != null) [ "--mesh-ca-cert" cfg.meshCaCert ]
+    ++ lib.optionals (cfg.meshCert != null) [ "--mesh-cert" cfg.meshCert ]
+    ++ lib.optionals (cfg.meshKey != null) [ "--mesh-key" cfg.meshKey ]
+    ++ lib.optionals meshEnabled [ "--mesh-max-hops" (toString cfg.meshMaxHops) ]
+    ++ cfg.extraArgs;
+
+  startCmd = "${lib.getExe' cfg.package "lightscale-server"} ${lib.escapeShellArgs args}";
+in
+{
+  options.services.lightscale-server = {
+    enable = lib.mkEnableOption "lightscale control-plane server";
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = defaultPackage;
+      description = "lightscale-server package.";
+    };
+
+    listen = lib.mkOption {
+      type = lib.types.str;
+      default = "0.0.0.0:8080";
+      description = "Listen address for the control plane.";
+    };
+
+    stateFile = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/lib/lightscale-server/state.json";
+      description = "State file path when dbUrl is not set.";
+    };
+
+    dbUrl = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Postgres/CockroachDB URL. If set, stateFile is ignored.";
+    };
+
+    dbUrlFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Path to a file containing Postgres/CockroachDB URL. If set, stateFile is ignored.";
+    };
+
+    adminToken = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Admin token passed via CLI. Prefer environmentFiles for secret handling.";
+    };
+
+    environmentFiles = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "systemd EnvironmentFile entries (for LIGHTSCALE_ADMIN_TOKEN, etc.).";
+    };
+
+    environment = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      default = { };
+      description = "Additional environment variables for the server service.";
+    };
+
+    stunServers = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "STUN servers advertised to clients.";
+    };
+
+    turnServers = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "TURN servers advertised to clients.";
+    };
+
+    streamRelayServers = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Stream relay servers advertised to clients.";
+    };
+
+    udpRelayServers = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "UDP relay servers advertised to clients.";
+    };
+
+    udpRelayListen = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "UDP relay listen address.";
+    };
+
+    streamRelayListen = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Stream relay listen address.";
+    };
+
+    meshServerId = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Server id used for authenticated relay mesh.";
+    };
+
+    meshListen = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Relay mesh listen address (mTLS).";
+    };
+
+    meshPeers = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Relay mesh peers in ID=HOST:PORT form.";
+    };
+
+    meshCaCert = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Path to relay mesh CA certificate PEM.";
+    };
+
+    meshCert = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Path to this server's relay mesh certificate PEM.";
+    };
+
+    meshKey = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Path to this server's relay mesh private key PEM.";
+    };
+
+    meshMaxHops = lib.mkOption {
+      type = lib.types.ints.positive;
+      default = 4;
+      description = "Maximum relay mesh forwarding hops.";
+    };
+
+    extraArgs = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Additional CLI arguments for lightscale-server.";
+    };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Open firewall ports listed in firewallTCPPorts/firewallUDPPorts.";
+    };
+
+    firewallTCPPorts = lib.mkOption {
+      type = lib.types.listOf lib.types.port;
+      default = [ 8080 ];
+      description = "TCP ports to open when openFirewall=true.";
+    };
+
+    firewallUDPPorts = lib.mkOption {
+      type = lib.types.listOf lib.types.port;
+      default = [ ];
+      description = "UDP ports to open when openFirewall=true.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = !(cfg.dbUrl != null && cfg.dbUrlFile != null);
+        message = "services.lightscale-server: set only one of dbUrl and dbUrlFile.";
+      }
+      {
+        assertion = cfg.adminToken != null || cfg.environment ? LIGHTSCALE_ADMIN_TOKEN || cfg.environmentFiles != [ ];
+        message = "services.lightscale-server: provide LIGHTSCALE_ADMIN_TOKEN via adminToken, environment, or environmentFiles.";
+      }
+      {
+        assertion = !meshEnabled || (
+          cfg.meshServerId != null
+          && cfg.meshListen != null
+          && cfg.meshCaCert != null
+          && cfg.meshCert != null
+          && cfg.meshKey != null
+        );
+        message = "services.lightscale-server: mesh requires meshServerId, meshListen, meshCaCert, meshCert, and meshKey.";
+      }
+      {
+        assertion = !meshEnabled || (cfg.streamRelayListen != null || cfg.udpRelayListen != null);
+        message = "services.lightscale-server: mesh forwarding requires streamRelayListen or udpRelayListen.";
+      }
+    ];
+
+    users.groups.lightscale = { };
+    users.users.lightscale = {
+      isSystemUser = true;
+      group = "lightscale";
+    };
+
+    systemd.services.lightscale-server = {
+      description = "lightscale control-plane server";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      environment = cfg.environment;
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = startCmd;
+        Restart = "on-failure";
+        RestartSec = 2;
+        User = "lightscale";
+        Group = "lightscale";
+        WorkingDirectory = "/var/lib/lightscale-server";
+        StateDirectory = "lightscale-server";
+        RuntimeDirectory = "lightscale-server";
+        EnvironmentFile = cfg.environmentFiles;
+        NoNewPrivileges = true;
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall cfg.firewallTCPPorts;
+    networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall cfg.firewallUDPPorts;
+  };
+}