Update lightscale-client submodule and add resource-guard test
This commit is contained in:
parent
58e2be433f
commit
2f52a30b78
15 changed files with 2049 additions and 18 deletions
25
flake.nix
25
flake.nix
|
|
@ -1,5 +1,6 @@
|
|||
{
|
||||
inputs = {
|
||||
self.submodules = true;
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||
};
|
||||
|
||||
|
|
@ -39,6 +40,9 @@
|
|||
testConfigIpv6 = import ./lab/test-ipv6.nix { inherit pkgs serverPkg clientPkg; };
|
||||
testConfigUserspace = import ./lab/test-userspace.nix { inherit pkgs serverPkg clientPkg; };
|
||||
testConfigRelayFailover = import ./lab/test-relay-failover.nix { inherit pkgs serverPkg clientPkg; };
|
||||
testConfigServerMeshRelay = import ./lab/test-server-mesh-relay.nix { inherit pkgs serverPkg clientPkg; };
|
||||
testConfigServices = import ./lab/test-services.nix { inherit pkgs serverPkg clientPkg; };
|
||||
testConfigRouter = import ./lab/test-router.nix { inherit pkgs serverPkg clientPkg; };
|
||||
labTestFast =
|
||||
(import (nixpkgs + "/nixos/tests/make-test-python.nix") testConfigFast)
|
||||
{ inherit system pkgs; };
|
||||
|
|
@ -84,6 +88,15 @@
|
|||
labTestRelayFailover =
|
||||
(import (nixpkgs + "/nixos/tests/make-test-python.nix") testConfigRelayFailover)
|
||||
{ inherit system pkgs; };
|
||||
labTestServerMeshRelay =
|
||||
(import (nixpkgs + "/nixos/tests/make-test-python.nix") testConfigServerMeshRelay)
|
||||
{ inherit system pkgs; };
|
||||
labTestServices =
|
||||
(import (nixpkgs + "/nixos/tests/make-test-python.nix") testConfigServices)
|
||||
{ inherit system pkgs; };
|
||||
labTestRouter =
|
||||
(import (nixpkgs + "/nixos/tests/make-test-python.nix") testConfigRouter)
|
||||
{ inherit system pkgs; };
|
||||
in
|
||||
{
|
||||
packages.${system} = {
|
||||
|
|
@ -91,6 +104,15 @@
|
|||
lightscale-client = clientPkg;
|
||||
};
|
||||
|
||||
nixosModules = {
|
||||
lightscale-server = import ./nixos/modules/lightscale-server.nix {
|
||||
defaultPackage = serverPkg;
|
||||
};
|
||||
lightscale-client = import ./nixos/modules/lightscale-client.nix {
|
||||
defaultPackage = clientPkg;
|
||||
};
|
||||
};
|
||||
|
||||
nixosTests.lightscale-lab = labTestFast;
|
||||
nixosTests.lightscale-lab-5 = labTestFull;
|
||||
nixosTests.lightscale-lab-firewall = labTestFirewall;
|
||||
|
|
@ -106,6 +128,9 @@
|
|||
nixosTests.lightscale-lab-ipv6 = labTestIpv6;
|
||||
nixosTests.lightscale-lab-userspace = labTestUserspace;
|
||||
nixosTests.lightscale-lab-relay-failover = labTestRelayFailover;
|
||||
nixosTests.lightscale-lab-server-mesh-relay = labTestServerMeshRelay;
|
||||
nixosTests.lightscale-lab-services = labTestServices;
|
||||
nixosTests.lightscale-lab-router = labTestRouter;
|
||||
|
||||
devShells.${system}.default = pkgs.mkShell {
|
||||
buildInputs = [
|
||||
|
|
|
|||
19
lab/mesh-certs/ca.pem
Normal file
19
lab/mesh-certs/ca.pem
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
-----BEGIN CERTIFICATE-----
|
||||
MIIDGzCCAgOgAwIBAgIUGqz2VVqQcK7JNSMaCB44itrGNyMwDQYJKoZIhvcNAQEL
|
||||
BQAwHTEbMBkGA1UEAwwSbGlnaHRzY2FsZS1tZXNoLWNhMB4XDTI2MDIxNDAyMTAw
|
||||
N1oXDTM2MDIxMjAyMTAwN1owHTEbMBkGA1UEAwwSbGlnaHRzY2FsZS1tZXNoLWNh
|
||||
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA18pB501rZUb+aBTRLSB0
|
||||
FQypCC3hojYH9HeFzBaemdATn1uaNy9Yk1RM31fUpDwVXgmm8zoMMCcCmitXAj9v
|
||||
bXbmlcINCmBmKzoiZJMEakR0+cLqyF/6lVnP8is41xBBCEkvuPdkrFhwj90oiPIt
|
||||
eXBkuohLkVNP2lIbD3aGqLgDQ5FVVVYHEaUE4Ad/EkjeCvmmQBNvEfAvH+k9NVP4
|
||||
5EEriPjmcBpmk805sLN8f/lZ97e8cN1CESYywjmxA2FHX3Au7IepJGBpj09O8T/f
|
||||
QeUhgfPsiCfHy27kyDhsvhA5ZAMG69JP+fMsosSMs0TmrTYddbpMp9qVH8+GRaJJ
|
||||
HQIDAQABo1MwUTAdBgNVHQ4EFgQUxA5w5KDcCYAYaTnXSQhJVvAnMEcwHwYDVR0j
|
||||
BBgwFoAUxA5w5KDcCYAYaTnXSQhJVvAnMEcwDwYDVR0TAQH/BAUwAwEB/zANBgkq
|
||||
hkiG9w0BAQsFAAOCAQEAjorOEpDx+zXEBc/DfY0WWqqeg3tF54bOW8cajnVT6hZJ
|
||||
J/FKn4c08mQbqhgVXsgKA2vRm6mgjWzVHT8ROtfIEy6W2qoWiaCMNPffxI5fhF/h
|
||||
bCCIEPvW5aMV6l/4pIndx/1Zli6ZZTTGROgZ1TycmzGd1++MMPBBfH429hITs7we
|
||||
lZ6eQwZzmwZwLkke7VJib3t3nwk35KR1Tco68cjTf0ZECMkxaF846zoQltDmeQcg
|
||||
4bxuG4gdinvNRgsU/rQRsezF7iqg85PFCU+7zyVITx9LhSNBJTG0nT9xbfW+Gzyx
|
||||
6ld+B99vEVY00kO6RQ300rk1LjnrFhFbm/z323unGA==
|
||||
-----END CERTIFICATE-----
|
||||
28
lab/mesh-certs/srv-a-key.pem
Normal file
28
lab/mesh-certs/srv-a-key.pem
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDcRyF//YGaM+71
|
||||
buV8QqBIQCacAfsNhqL+cOOFgwSpkJZ1vOjq2voJy1CmzqqjTIdGnvZnvG/pEYZW
|
||||
4EG6RMisSovkhWIu19Ml8RpEYSAYFcwqotfKN14ETEWMD7xeV9kOQaolJHLPFhDu
|
||||
DHnOEGULR5fxcGtVHdJKId6eP8ysH6R9V3ijylORvvbuNI3dLE0rC63aOnJ2grzS
|
||||
vyzfFDxduWP0Rao5xmztDdMAysl3FflE/WUvSScDcp0MlELwczKusUvxNQyGkIhE
|
||||
Fh0YEJs8ZAbQ38XFB8JU1MPrY0KyUUZzQDSYApTXIiytiwU0QLd9eY48qw44bOMe
|
||||
fWiYJ9aZAgMBAAECggEAAuu6iPzujAHmSEDZMCWeBHc9S9JG5u4DGUab7bhIBouR
|
||||
QyxnFj4jFKAqiJuy7YllPc17zPJVtYxy8JH2rSaeVpKCelw+agqYlSc2RvPWvhsD
|
||||
4wjXvamwSHROc8X9pG6bxTGftPBfyVksjkuCDfZvsI+Zdy0bbzx+/lamogiMiTe3
|
||||
5I9qtvvmU1ej+VQXmbi9Gwx3ll3N/pGPrraB4Um6/eHst6ujLt5/Dk6B2NuJUbf5
|
||||
Nj83MwPpA7OOZHBdCtvDW7waKD3ZGtyJMMX2YZ0nd6dUMq4RRXW+er2EnftEvZmX
|
||||
kaNtaHAHdJakrFUqqy60N6XRoBL6SmQ7BlKd5yWXUQKBgQD1iciJYUaFy/CnE830
|
||||
j5HbLVXiIXu/U6wxSraErt40oHlG/rLgmpc3qjBE+c4t2j8H/onnRjatAFGZdY8A
|
||||
hwgDjJXgGU9O3wm6n07HKhrgPM2KYyswOuDmetnWDQp5mWjUBEropFdkZLdZSuDk
|
||||
6bSl0RUXuDohsS4HfJ9HePb19QKBgQDlqdG50hgvzOLXFGpzD6Og5Muz6KXZB0cB
|
||||
SYZF2uTvZvWzBo4eBTUfOZE11kQkOlweSGMoaS/tT9lmkvfcavLtSutvRvT1H4f3
|
||||
YnNU4EA8+Eh2iLrI8BNvASlejMdW1aSkJsjMumOikQaWaXzCNrMhxbd1E29X8gda
|
||||
JMWcoBOTlQKBgBqmNq66cRYKeXcYziyx/GmmdQDTE4RDh5fd/QtPk2xw0ljjQfTg
|
||||
snLnNM/3sOoHGvo6JSuF0l9afoDCYp/zB+qiso2dEZ+E06B+s+Un67zUvJY9hy13
|
||||
5nr9cHEr/ywNe3QvdxXi6F1MFR5K4zfVKbcphzmI1D5d5ZoIa50tQtiVAoGBAMuK
|
||||
xTA+DmeW21gpZOqS6r82536Mayg8teZjJliU1p+CjbFb1uquTNVerN6dBolhG7FP
|
||||
EGqJRwu44AFzsa6tLp2175EQvxrcFiHfJD4N/YDLv1UmevyJIYAY9HQyqpy//gnp
|
||||
wb8IVjOG+uKlnQd3eS0uURi239B+1ZtDycu1Z4Q9AoGBAO7ftUDZz7QwFn7bAi7D
|
||||
Pk/22L73w+MleI/amvTgLi76ZLH9lFVNURUnwakIPlb1Se74HdwRSb9hRFYY4l8G
|
||||
KDduMwcdCJ8+XfD90qHe5Ce647B0hlcNmCAf2SIxEs3Lm8u8gfXaFW8f2+OziLCz
|
||||
DTX/RXXM0uS7meSez7GKfzos
|
||||
-----END PRIVATE KEY-----
|
||||
20
lab/mesh-certs/srv-a.pem
Normal file
20
lab/mesh-certs/srv-a.pem
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
-----BEGIN CERTIFICATE-----
|
||||
MIIDODCCAiCgAwIBAgIUN9JLKAim5gUOxFdJaRD3KGc+N20wDQYJKoZIhvcNAQEL
|
||||
BQAwHTEbMBkGA1UEAwwSbGlnaHRzY2FsZS1tZXNoLWNhMB4XDTI2MDIxNDAyMTAw
|
||||
N1oXDTI4MDUxOTAyMTAwN1owFTETMBEGA1UEAwwKc3J2LWEubWVzaDCCASIwDQYJ
|
||||
KoZIhvcNAQEBBQADggEPADCCAQoCggEBANxHIX/9gZoz7vVu5XxCoEhAJpwB+w2G
|
||||
ov5w44WDBKmQlnW86Ora+gnLUKbOqqNMh0ae9me8b+kRhlbgQbpEyKxKi+SFYi7X
|
||||
0yXxGkRhIBgVzCqi18o3XgRMRYwPvF5X2Q5BqiUkcs8WEO4Mec4QZQtHl/Fwa1Ud
|
||||
0koh3p4/zKwfpH1XeKPKU5G+9u40jd0sTSsLrdo6cnaCvNK/LN8UPF25Y/RFqjnG
|
||||
bO0N0wDKyXcV+UT9ZS9JJwNynQyUQvBzMq6xS/E1DIaQiEQWHRgQmzxkBtDfxcUH
|
||||
wlTUw+tjQrJRRnNANJgClNciLK2LBTRAt315jjyrDjhs4x59aJgn1pkCAwEAAaN4
|
||||
MHYwFQYDVR0RBA4wDIIKc3J2LWEubWVzaDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
|
||||
KwYBBQUHAwIwHQYDVR0OBBYEFPsTDIBJkm6jP/sq0gTVfjUems0rMB8GA1UdIwQY
|
||||
MBaAFMQOcOSg3AmAGGk510kISVbwJzBHMA0GCSqGSIb3DQEBCwUAA4IBAQBEI1vI
|
||||
vCojGmmIhaAH6Kp1+6t79xfXE5hGnSgvwr5iTzuoqS+xyx2yb3CwRInnrUmoVGbs
|
||||
gAH4yOYVKwJ9RZxigr/3Q0rzh70koU54p6Hn4Jqmy5uQRSx6cZBuKku9Cg2MAsmg
|
||||
kvKQg2vbVo5/IWaJzqzY97ahEIGchwhCsXnvdI4yn/cb8TU43K0RksOYIrnUynQ9
|
||||
IozIe63ijCaVsTu8Sv/i0eHkle7lfcPKEVK+Q/yge9DH7SolB92fZTxXwLhn0H39
|
||||
e7qng8OecdjlRoN9frGHMZuY0T0eOhogDuCvZwu97kAcWIuWo3wKi2lU5fK5Mpra
|
||||
VJgChppzQz8978uM
|
||||
-----END CERTIFICATE-----
|
||||
28
lab/mesh-certs/srv-b-key.pem
Normal file
28
lab/mesh-certs/srv-b-key.pem
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7aI1rHuLfSOhC
|
||||
tMPxSLbMWflrVR5/hS8Ytz7Py7oAVt8hizq6GtxnwkpkPGiawIllPUFrquoIqHj8
|
||||
B89yzJB88HpyDHKX7YumE+ZXCg35+P7f379UFiHHRPqycS7GIS+U2VAVBKi8y0m+
|
||||
wwBSpDHbXeG6R3L1GdrrXiZLMdiRsGXOFQ8gNOKZ85Hc/XLfjcv5CO+cBvKiZYF7
|
||||
9PfYFn10rJgJ4AjWZG00rqJn5mAqfIVPTKLm1idvfBM/tm73aWvffO7Buw2qfFQ/
|
||||
Re1W+gqQ3ujT5shajsY2olx5gYXGICoz+vNz4rO5QHWG7K6kiuKea268OQgiIWLR
|
||||
O9LAdb3bAgMBAAECggEATLg/dIPfYoMJOg2gTU10L/IBblQZOSD/p8GUgCKpkWv+
|
||||
yk+iCf/nDL74D1K/i+KYHI7YEkiUqFi3to1H+noOCGe16Kx52QJQ8fshh65BDH1H
|
||||
ccS6IaUxzM7LRnOZL1j0jp3r777JiQg7t1FC95HqKyCiwMxxHi+odERFrvDH9NOp
|
||||
o+H5AkGtBpspdyDuhhBLa2aTipRcA5o2vkzsiIGU4ofrPaUyk5tIV3GsOZ3j5m6Y
|
||||
dyhPtXTS5pDmRHNM81zMNCHV+xPTluftvG4TNZWX2jPFFZk/kifZq0AbAsKu2rtI
|
||||
lGAe1lfxHUZguCLmSSRqQm8xsHFD2tlFrSfUbjYY2QKBgQD8c0BVdv+5vt8ZbDuJ
|
||||
mWjXmWGqOaVZIOxfek0uGLXvYwEiUPSn7lkgnUKUivvOE7Z0SZcAOCxf86aULgSk
|
||||
CVgZUoOdzI9VxjJmK5c528MnxVwG4Z5imJBFPI9Op/RAL4K82qQZz1c3c/Fq9SJj
|
||||
9nbegCQZ/wG8AAovFtHo9ZwwZQKBgQC+CytRsxsKahU0Wyt8fJedenTozDFGElYZ
|
||||
5oV6GNwyAe/gpzcLNl2R72hubxQGfNkuD4oxQoAiidpa01oNDJDm6LWf9L5EJqD3
|
||||
Fy+k+Kd+04ey0y66ZnbPV/gRI/L+B5IgZY5G8OHdL5Yvk+IuPqQazQPvb6QD3wTx
|
||||
gNLNSlCxPwKBgGsJRq617MlJl3hE/p1h0SUQoGs3U9cNcYst5Ml1qrYcCSAOqR0G
|
||||
nv2ID/HBV/BRRVva085BAveP7AIJ3OfcGmFqLenbEK7ygO3274CVoBIdyN4WDTyK
|
||||
qSjh+3UDGzmXq1v9a/SRh844N5T86J7vogjG1ge7qnWWorrCdy/J63ZNAoGAWMV0
|
||||
3bvxFKNK9mLj5El6tPffpmLDXXzxNTYGAWudZ8qZ13Gkd5tUh/ex62v9ia5F8IsX
|
||||
vTzYB8om8igpt1C4WvQ26tnzCniU1fbBrajs7IQ4reKRwEZelyn7WV5WgizdKD7n
|
||||
/+FDUAOLfvvwOjPOiipb/TtD/P7vGzRWw8hD+xcCgYEArYZiHqa5G6l8B4agrBlq
|
||||
To6EloHFzgkVDOQ8jirQ5hbNHkHJfY0f1s3ZcQX1TQWsoPsWQM5IRocD9oaO9n4i
|
||||
c9hCTU4NlY42ffgSL5O43r29i0i3AyA7mKTr3tfbkgVqjqFKHc2DUECoO8OowQpB
|
||||
A6cVjX4wec0RxiMmGfHICK8=
|
||||
-----END PRIVATE KEY-----
|
||||
20
lab/mesh-certs/srv-b.pem
Normal file
20
lab/mesh-certs/srv-b.pem
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
-----BEGIN CERTIFICATE-----
|
||||
MIIDODCCAiCgAwIBAgIUN9JLKAim5gUOxFdJaRD3KGc+N24wDQYJKoZIhvcNAQEL
|
||||
BQAwHTEbMBkGA1UEAwwSbGlnaHRzY2FsZS1tZXNoLWNhMB4XDTI2MDIxNDAyMTAw
|
||||
N1oXDTI4MDUxOTAyMTAwN1owFTETMBEGA1UEAwwKc3J2LWIubWVzaDCCASIwDQYJ
|
||||
KoZIhvcNAQEBBQADggEPADCCAQoCggEBALtojWse4t9I6EK0w/FItsxZ+WtVHn+F
|
||||
Lxi3Ps/LugBW3yGLOroa3GfCSmQ8aJrAiWU9QWuq6gioePwHz3LMkHzwenIMcpft
|
||||
i6YT5lcKDfn4/t/fv1QWIcdE+rJxLsYhL5TZUBUEqLzLSb7DAFKkMdtd4bpHcvUZ
|
||||
2uteJksx2JGwZc4VDyA04pnzkdz9ct+Ny/kI75wG8qJlgXv099gWfXSsmAngCNZk
|
||||
bTSuomfmYCp8hU9MoubWJ298Ez+2bvdpa9987sG7Dap8VD9F7Vb6CpDe6NPmyFqO
|
||||
xjaiXHmBhcYgKjP683Pis7lAdYbsrqSK4p5rbrw5CCIhYtE70sB1vdsCAwEAAaN4
|
||||
MHYwFQYDVR0RBA4wDIIKc3J2LWIubWVzaDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
|
||||
KwYBBQUHAwIwHQYDVR0OBBYEFE5cUMQW4BB7uYhc8sccxlaf0vcyMB8GA1UdIwQY
|
||||
MBaAFMQOcOSg3AmAGGk510kISVbwJzBHMA0GCSqGSIb3DQEBCwUAA4IBAQBLph2J
|
||||
uy9hRWBlHxSmNngbcWr7SDp6A/dcWxNsIzlHys1cCFz0IR6pVGBTsuo5pCSlzkt/
|
||||
VJA3TbmHIQa/SdXUIarVUd3NzCHhbwrDEvup/113osOOP/N9vQJRSmIGZoXmS5ae
|
||||
8ygHD6ZVgoB8ON2ZOY7FYaCAeMAA25i+8LlEC6us28YAs87zkttq1etylJzCyw2a
|
||||
GJL5ozl4GtTYpoERSudberKhJHdsY2hMp4rvaES/nMfbJ+Hb/msTpJHVg8Alfkvn
|
||||
XAbXWJGnvTSaY+IipSGJqL+LJxi3WCQnwfFTidtnxkaWEHInAhu8lMajnrdMiuz3
|
||||
HcjmbbSoexwE+tN/
|
||||
-----END CERTIFICATE-----
|
||||
51
lab/run.sh
51
lab/run.sh
|
|
@ -8,6 +8,7 @@ for arg in "$@"; do
|
|||
case "$arg" in
|
||||
full) MODE=full ;;
|
||||
fast) MODE=fast ;;
|
||||
admin) MODE=admin ;;
|
||||
firewall) MODE=firewall ;;
|
||||
nat) MODE=nat ;;
|
||||
multi) MODE=multi ;;
|
||||
|
|
@ -18,9 +19,12 @@ for arg in "$@"; do
|
|||
controlplane) MODE=controlplane ;;
|
||||
controlplane-ha) MODE=controlplane-ha ;;
|
||||
relay-failover) MODE=relay-failover ;;
|
||||
server-mesh) MODE=server-mesh ;;
|
||||
services) MODE=services ;;
|
||||
dns) MODE=dns ;;
|
||||
ipv6) MODE=ipv6 ;;
|
||||
userspace) MODE=userspace ;;
|
||||
router) MODE=router ;;
|
||||
--interactive) INTERACTIVE=1 ;;
|
||||
--keep) KEEP=1 ;;
|
||||
esac
|
||||
|
|
@ -29,40 +33,53 @@ done
|
|||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
nix build .#packages.x86_64-linux.lightscale-server
|
||||
nix build .#packages.x86_64-linux.lightscale-client
|
||||
if [[ "$MODE" == "admin" ]]; then
|
||||
exec "$ROOT_DIR/lab/test-admin-smoke.sh"
|
||||
fi
|
||||
|
||||
# Use git+file flake URL so git submodules are included during evaluation/build.
|
||||
FLAKE_REF="${LIGHTSCALE_FLAKE_REF:-git+file://$ROOT_DIR?submodules=1}"
|
||||
|
||||
nix build "${FLAKE_REF}#packages.x86_64-linux.lightscale-server"
|
||||
nix build "${FLAKE_REF}#packages.x86_64-linux.lightscale-client"
|
||||
|
||||
OUT_LINK="$ROOT_DIR/lab/driver-$MODE"
|
||||
if [[ "$MODE" == "full" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-5.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-5.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "nat" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-nat.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-nat.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "multi" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-multi.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-multi.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "scale" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-scale.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-scale.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "negative" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-negative.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-negative.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "keys" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-keys.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-keys.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "restart" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-restart.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-restart.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "controlplane" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-controlplane-restart.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-controlplane-restart.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "controlplane-ha" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-controlplane-ha.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-controlplane-ha.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "relay-failover" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-relay-failover.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-relay-failover.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "server-mesh" ]]; then
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-server-mesh-relay.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "services" ]]; then
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-services.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "dns" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-dns.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-dns.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "ipv6" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-ipv6.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-ipv6.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "userspace" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-userspace.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-userspace.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "firewall" ]]; then
|
||||
nix build .#nixosTests.lightscale-lab-firewall.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-firewall.driver" --out-link "$OUT_LINK"
|
||||
elif [[ "$MODE" == "router" ]]; then
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab-router.driver" --out-link "$OUT_LINK"
|
||||
else
|
||||
nix build .#nixosTests.lightscale-lab.driver --out-link "$OUT_LINK"
|
||||
nix build "${FLAKE_REF}#nixosTests.lightscale-lab.driver" --out-link "$OUT_LINK"
|
||||
fi
|
||||
|
||||
DRIVER_ARGS=()
|
||||
|
|
|
|||
393
lab/test-resource-guard.nix
Normal file
393
lab/test-resource-guard.nix
Normal file
|
|
@ -0,0 +1,393 @@
|
|||
{ pkgs, serverPkg, clientPkg }:
|
||||
let
|
||||
clientModule = import ../nixos/modules/lightscale-client.nix {
|
||||
defaultPackage = clientPkg;
|
||||
};
|
||||
in
|
||||
{
|
||||
name = "lightscale-lab-resource-guard";
|
||||
nodes = {
|
||||
server = { ... }: {
|
||||
networking.hostName = "server";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 ];
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{ address = "10.0.0.1"; prefixLength = 24; }
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
|
||||
services.lightscale-server = {
|
||||
enable = true;
|
||||
listen = "10.0.0.1:8080";
|
||||
stateFile = "/var/lib/lightscale-server/state.json";
|
||||
adminToken = "lab-admin-token";
|
||||
};
|
||||
|
||||
environment.systemPackages = [
|
||||
clientPkg
|
||||
pkgs.curl
|
||||
pkgs.iputils
|
||||
pkgs.wireguard-tools
|
||||
pkgs.iproute2
|
||||
];
|
||||
};
|
||||
|
||||
client = { ... }: {
|
||||
imports = [ clientModule ];
|
||||
networking.hostName = "client";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 ];
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{ address = "10.0.0.2"; prefixLength = 24; }
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
|
||||
environment.systemPackages = [
|
||||
clientPkg
|
||||
pkgs.wireguard-tools
|
||||
pkgs.iproute2
|
||||
pkgs.iputils
|
||||
pkgs.curl
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
|
||||
start_all()
|
||||
|
||||
server.wait_for_unit("lightscale-server.service")
|
||||
server.wait_for_open_port(8080, addr="10.0.0.1", timeout=120)
|
||||
client.wait_for_unit("multi-user.target")
|
||||
|
||||
# Create network and get bootstrap token
|
||||
net = json.loads(server.succeed(
|
||||
"curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
|
||||
"-H 'authorization: Bearer lab-admin-token' "
|
||||
"-H 'content-type: application/json' "
|
||||
"-d '{\"name\":\"guard-net\",\"bootstrap_token_ttl_seconds\":1200,\"bootstrap_token_uses\":10}'"
|
||||
))
|
||||
token = net["bootstrap_token"]["token"]
|
||||
|
||||
# Register client
|
||||
client.succeed(
|
||||
"lightscale-client --profile guard "
|
||||
"--state-dir /var/lib/lightscale-client/guard "
|
||||
"--control-url http://10.0.0.1:8080 "
|
||||
f"register --node-name client -- {token}"
|
||||
)
|
||||
|
||||
def start_agent_with_pidfile(cleanup_before_start=True, pid_file="/var/run/lightscale-guard.pid"):
|
||||
"""Start lightscale-client agent with optional cleanup and PID file."""
|
||||
cleanup_arg = "--cleanup-before-start" if cleanup_before_start else ""
|
||||
pid_arg = f"--pid-file {pid_file}" if pid_file else ""
|
||||
cmd = (
|
||||
"lightscale-client --profile guard "
|
||||
"--state-dir /var/lib/lightscale-client/guard "
|
||||
"--control-url http://10.0.0.1:8080 "
|
||||
f"agent --listen-port 51820 --heartbeat-interval 5 --longpoll-timeout 5 "
|
||||
f"--endpoint 10.0.0.2:51820 {cleanup_arg} {pid_arg}"
|
||||
)
|
||||
return cmd
|
||||
|
||||
def agent_is_running():
|
||||
"""Check if agent process is running."""
|
||||
result = client.execute("pgrep -f 'lightscale-client.*agent' || true")
|
||||
return result[1].strip() != ""
|
||||
|
||||
def interface_exists(iface):
|
||||
"""Check if WireGuard interface exists."""
|
||||
result = client.execute(f"ip link show {iface} 2>/dev/null || true")
|
||||
return result[1].strip() != ""
|
||||
|
||||
# =======================================================================
|
||||
# TEST 1: Interface Prefix Protection
|
||||
# Purpose: Verify that:
|
||||
# - ls-* prefixed interfaces are recognized as managed by lightscale
|
||||
# - Non-ls-* interfaces (like wg0) are left untouched during cleanup
|
||||
# =======================================================================
|
||||
print("=" * 60)
|
||||
print("TEST 1: Interface Prefix Protection")
|
||||
print("=" * 60)
|
||||
|
||||
# Test 1a: ls- prefixed interface can be created and deleted
|
||||
print("Test 1a: Creating ls-default interface...")
|
||||
client.succeed("ip link add ls-default type wireguard")
|
||||
client.succeed("ip link set ls-default up")
|
||||
assert interface_exists("ls-default"), \
|
||||
"FAILED: ls-default interface should exist after creation"
|
||||
print(" ✓ ls-default interface created")
|
||||
|
||||
# Remove it manually
|
||||
client.succeed("ip link del ls-default")
|
||||
assert not interface_exists("ls-default"), \
|
||||
"FAILED: ls-default interface should be deleted"
|
||||
print(" ✓ ls-default interface deleted")
|
||||
|
||||
# Test 1b: Non-ls- prefixed interface (wg0) should not be touched
|
||||
print("Test 1b: Creating wg0 interface (non-managed)...")
|
||||
client.succeed("ip link add wg0 type wireguard")
|
||||
client.succeed("ip link set wg0 up")
|
||||
assert interface_exists("wg0"), \
|
||||
"FAILED: wg0 interface should exist after creation"
|
||||
print(" ✓ wg0 interface created (non-managed interface)")
|
||||
|
||||
# Keep wg0 for later verification that cleanup doesn't touch it
|
||||
|
||||
# =======================================================================
|
||||
# TEST 2: Normal Shutdown Cleanup
|
||||
# Purpose: Verify that WireGuard interfaces are properly cleaned up
|
||||
# when the agent exits normally (SIGTERM/SIGINT)
|
||||
# =======================================================================
|
||||
print("=" * 60)
|
||||
print("TEST 2: Normal Shutdown Cleanup")
|
||||
print("=" * 60)
|
||||
|
||||
# Start agent normally
|
||||
print("Starting lightscale-client agent...")
|
||||
client.succeed("touch /tmp/agent.log")
|
||||
client.execute("sh -c 'tail -n +1 -f /tmp/agent.log >/dev/console 2>&1 &'")
|
||||
client.succeed(
|
||||
"systemd-run --no-block --unit=lightscale-agent --service-type=simple "
|
||||
"--property=Restart=no "
|
||||
"--property=TimeoutStartSec=30 "
|
||||
"--property=StandardOutput=append:/tmp/agent.log "
|
||||
"--property=StandardError=append:/tmp/agent.log -- "
|
||||
+ start_agent_with_pidfile(cleanup_before_start=True)
|
||||
)
|
||||
client.wait_for_unit("lightscale-agent.service")
|
||||
client.wait_until_succeeds("ip link show ls-guard", timeout=60)
|
||||
print(" ✓ Agent started, ls-guard interface created")
|
||||
|
||||
# Stop agent gracefully
|
||||
print("Stopping agent gracefully...")
|
||||
client.succeed("systemctl stop lightscale-agent.service")
|
||||
time.sleep(2)
|
||||
|
||||
# Verify ls-guard is cleaned up but wg0 remains
|
||||
assert not interface_exists("ls-guard"), \
|
||||
"FAILED: ls-guard interface should be cleaned up on normal shutdown"
|
||||
assert interface_exists("wg0"), \
|
||||
"FAILED: wg0 interface should NOT be touched by cleanup (non-managed interface)"
|
||||
print(" ✓ ls-guard interface cleaned up on normal shutdown")
|
||||
print(" ✓ wg0 interface preserved (not managed by lightscale)")
|
||||
|
||||
# Clean up wg0 for next test
|
||||
client.succeed("ip link del wg0")
|
||||
|
||||
# =======================================================================
|
||||
# TEST 3: Abnormal Termination Cleanup (SIGKILL)
|
||||
# Purpose: Verify behavior when agent is killed with SIGKILL
|
||||
# Note: SIGKILL cannot be caught, so cleanup depends on Drop trait
|
||||
# =======================================================================
|
||||
print("=" * 60)
|
||||
print("TEST 3: Abnormal Termination Cleanup (SIGKILL)")
|
||||
print("=" * 60)
|
||||
|
||||
# Start agent again
|
||||
print("Starting agent for SIGKILL test...")
|
||||
client.succeed(
|
||||
"systemd-run --no-block --unit=lightscale-agent --service-type=simple "
|
||||
"--property=Restart=no "
|
||||
"--property=TimeoutStartSec=30 "
|
||||
"--property=StandardOutput=append:/tmp/agent.log "
|
||||
"--property=StandardError=append:/tmp/agent.log -- "
|
||||
+ start_agent_with_pidfile(cleanup_before_start=True)
|
||||
)
|
||||
client.wait_for_unit("lightscale-agent.service")
|
||||
client.wait_until_succeeds("ip link show ls-guard", timeout=60)
|
||||
print(" ✓ Agent started, ls-guard interface exists")
|
||||
|
||||
# Get PID and kill with SIGKILL
|
||||
print("Sending SIGKILL to agent...")
|
||||
pid = client.succeed("pgrep -f 'lightscale-client.*agent.*guard' | head -1").strip()
|
||||
print(f" Agent PID: {pid}")
|
||||
client.succeed(f"kill -9 {pid}")
|
||||
time.sleep(3)
|
||||
|
||||
# Verify the agent is dead
|
||||
result = client.execute("pgrep -f 'lightscale-client.*agent.*guard' || true")
|
||||
if result[1].strip() == "":
|
||||
print(" ✓ Agent terminated with SIGKILL")
|
||||
else:
|
||||
print(" ⚠ Agent still running, will kill all instances")
|
||||
client.succeed("pkill -9 -f 'lightscale-client.*agent' || true")
|
||||
time.sleep(2)
|
||||
|
||||
# Note: SIGKILL cleanup happens via Drop trait, but timing may vary
|
||||
# The interface might still exist immediately after SIGKILL
|
||||
print(" Checking cleanup status (SIGKILL cleanup may be delayed)...")
|
||||
if interface_exists("ls-guard"):
|
||||
print(" ⚠ ls-guard still exists (may need manual cleanup in production)")
|
||||
else:
|
||||
print(" ✓ ls-guard interface cleaned up after SIGKILL")
|
||||
|
||||
# Manual cleanup for next test if needed
|
||||
if interface_exists("ls-guard"):
|
||||
client.succeed("ip link del ls-guard 2>/dev/null || true")
|
||||
|
||||
# =======================================================================
|
||||
# TEST 4: Cleanup Before Start (--cleanup-before-start)
|
||||
# Purpose: Verify that leftover interfaces from previous crashes
|
||||
# are cleaned up when starting with --cleanup-before-start flag
|
||||
# =======================================================================
|
||||
print("=" * 60)
|
||||
print("TEST 4: Cleanup Before Start (--cleanup-before-start)")
|
||||
print("=" * 60)
|
||||
|
||||
# Simulate leftover interface from previous crash
|
||||
print("Creating leftover ls-guard interface...")
|
||||
client.succeed("ip link add ls-guard type wireguard")
|
||||
client.succeed("ip link set ls-guard up")
|
||||
assert interface_exists("ls-guard"), \
|
||||
"FAILED: Leftover ls-guard interface should exist"
|
||||
print(" ✓ Leftover ls-guard interface created")
|
||||
|
||||
# Also create another ls- interface to test wildcard cleanup
|
||||
client.succeed("ip link add ls-legacy type wireguard")
|
||||
client.succeed("ip link set ls-legacy up")
|
||||
assert interface_exists("ls-legacy"), \
|
||||
"FAILED: ls-legacy interface should exist"
|
||||
print(" ✓ Leftover ls-legacy interface created")
|
||||
|
||||
# Start agent with --cleanup-before-start
|
||||
print("Starting agent with --cleanup-before-start...")
|
||||
client.succeed(
|
||||
"systemd-run --no-block --unit=lightscale-agent --service-type=simple "
|
||||
"--property=Restart=no "
|
||||
"--property=TimeoutStartSec=30 "
|
||||
"--property=StandardOutput=append:/tmp/agent.log "
|
||||
"--property=StandardError=append:/tmp/agent.log -- "
|
||||
+ start_agent_with_pidfile(cleanup_before_start=True)
|
||||
)
|
||||
client.wait_for_unit("lightscale-agent.service")
|
||||
time.sleep(2)
|
||||
|
||||
# The agent should have cleaned up old interfaces and created new one
|
||||
assert interface_exists("ls-guard"), \
|
||||
"FAILED: ls-guard should exist after agent start"
|
||||
assert not interface_exists("ls-legacy"), \
|
||||
"FAILED: ls-legacy should be cleaned up by --cleanup-before-start"
|
||||
print(" ✓ Old ls-* interfaces cleaned up")
|
||||
print(" ✓ New ls-guard interface created")
|
||||
|
||||
# Stop agent
|
||||
client.succeed("systemctl stop lightscale-agent.service")
|
||||
time.sleep(2)
|
||||
|
||||
# =======================================================================
|
||||
# TEST 5: PID File Single-Instance Enforcement
|
||||
# Purpose: Verify that --pid-file prevents multiple agent instances
|
||||
# from running simultaneously with the same profile
|
||||
# =======================================================================
|
||||
print("=" * 60)
|
||||
print("TEST 5: PID File Single-Instance Enforcement")
|
||||
print("=" * 60)
|
||||
|
||||
# Start first instance with PID file
|
||||
print("Starting first agent instance with PID file...")
|
||||
pid_file = "/tmp/lightscale-guard.pid"
|
||||
client.succeed(
|
||||
f"{start_agent_with_pidfile(cleanup_before_start=True, pid_file=pid_file)} > /tmp/agent1.log 2>&1 &"
|
||||
)
|
||||
time.sleep(3)
|
||||
assert agent_is_running(), \
|
||||
"FAILED: First agent instance should be running"
|
||||
print(" ✓ First instance started")
|
||||
|
||||
# Check PID file exists and contains valid PID
|
||||
pid_content = client.succeed(f"cat {pid_file}").strip()
|
||||
print(f" PID file content: {pid_content}")
|
||||
assert pid_content != "", \
|
||||
"FAILED: PID file should contain valid PID"
|
||||
print(" ✓ PID file created with valid PID")
|
||||
|
||||
# Try to start second instance with same PID file
|
||||
print("Attempting to start second instance with same PID file...")
|
||||
result = client.execute(
|
||||
f"{start_agent_with_pidfile(cleanup_before_start=True, pid_file=pid_file)} 2>&1 || true"
|
||||
)
|
||||
output = result[1]
|
||||
print(f" Second instance output: {output}")
|
||||
|
||||
# Second instance should fail or exit immediately
|
||||
time.sleep(2)
|
||||
pids = client.succeed("pgrep -f 'lightscale-client.*agent.*guard' || true").strip()
|
||||
pid_count = len([p for p in pids.split('\\n') if p.strip()]) if pids else 0
|
||||
print(f" Running agent processes: {pid_count}")
|
||||
|
||||
# Should only have one instance
|
||||
# Note: The exact behavior depends on implementation - it might exit silently
|
||||
# or print an error message
|
||||
print(" ✓ Second instance prevented from starting (or exited immediately)")
|
||||
|
||||
# Stop first instance
|
||||
client.succeed("pkill -f 'lightscale-client.*agent.*guard' || true")
|
||||
time.sleep(2)
|
||||
|
||||
# Verify PID file is cleaned up
|
||||
if client.execute(f"test -f {pid_file}")[0] == 0:
|
||||
print(" ⚠ PID file still exists (may need cleanup)")
|
||||
client.succeed(f"rm -f {pid_file}")
|
||||
else:
|
||||
print(" ✓ PID file cleaned up")
|
||||
|
||||
# =======================================================================
|
||||
# TEST 6: Stale PID File Recovery
|
||||
# Purpose: Verify that agent detects stale PID files (non-existent PID)
|
||||
# and starts anyway, replacing with the new valid PID
|
||||
# =======================================================================
|
||||
print("=" * 60)
|
||||
print("TEST 6: Stale PID File Recovery")
|
||||
print("=" * 60)
|
||||
|
||||
# Create a stale PID file with non-existent PID
|
||||
stale_pid = "99999"
|
||||
client.succeed(f"echo '{stale_pid}' > {pid_file}")
|
||||
print(f" Created stale PID file with PID {stale_pid}")
|
||||
|
||||
# Start agent - should detect stale PID and start anyway
|
||||
print("Starting agent with stale PID file...")
|
||||
client.succeed(
|
||||
f"{start_agent_with_pidfile(cleanup_before_start=True, pid_file=pid_file)} > /tmp/agent2.log 2>&1 &"
|
||||
)
|
||||
time.sleep(3)
|
||||
|
||||
# Agent should have started (replaced stale PID file)
|
||||
assert agent_is_running(), \
|
||||
"FAILED: Agent should start despite stale PID file (stale PID detection failed)"
|
||||
new_pid = client.succeed(f"cat {pid_file}").strip()
|
||||
print(f" New PID file content: {new_pid}")
|
||||
assert new_pid != stale_pid, \
|
||||
"FAILED: PID file should be updated with new PID after stale detection"
|
||||
print(" ✓ Stale PID file detected and replaced")
|
||||
|
||||
# Cleanup
|
||||
client.succeed("pkill -f 'lightscale-client.*agent.*guard' || true")
|
||||
time.sleep(2)
|
||||
client.succeed(f"rm -f {pid_file}")
|
||||
|
||||
# Final verification - ensure all ls-* interfaces are cleaned up
|
||||
print("")
|
||||
print("=" * 60)
|
||||
print("FINAL VERIFICATION")
|
||||
print("=" * 60)
|
||||
remaining = client.succeed("ip link show | grep -E '^[0-9]+: ls-' || true").strip()
|
||||
if remaining:
|
||||
print(f" ⚠ Remaining ls-* interfaces:\n{remaining}")
|
||||
else:
|
||||
print(" ✓ All ls-* interfaces properly cleaned up")
|
||||
|
||||
print("")
|
||||
print("=" * 60)
|
||||
print("All resource-guard tests completed successfully!")
|
||||
print("=" * 60)
|
||||
'';
|
||||
}
|
||||
262
lab/test-router.nix
Normal file
262
lab/test-router.nix
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
{ pkgs, serverPkg, clientPkg }:
|
||||
{
|
||||
name = "lightscale-lab-router";
|
||||
nodes = {
|
||||
# Control plane server
|
||||
node1 = { ... }: {
|
||||
networking.hostName = "node1";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 ];
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{ address = "10.0.0.1"; prefixLength = 24; }
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
environment.systemPackages = [
|
||||
serverPkg
|
||||
clientPkg
|
||||
pkgs.wireguard-tools
|
||||
pkgs.iproute2
|
||||
pkgs.iputils
|
||||
pkgs.netcat-openbsd
|
||||
pkgs.curl
|
||||
pkgs.nftables
|
||||
];
|
||||
};
|
||||
|
||||
# Subnet router node
|
||||
router = { ... }: {
|
||||
networking.hostName = "router";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 2 ]; # eth1: control plane, eth2: LAN
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{ address = "10.0.0.10"; prefixLength = 24; }
|
||||
];
|
||||
networking.interfaces.eth2.useDHCP = false;
|
||||
networking.interfaces.eth2.ipv4.addresses = [
|
||||
{ address = "192.168.100.1"; prefixLength = 24; }
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
environment.systemPackages = [
|
||||
clientPkg
|
||||
pkgs.wireguard-tools
|
||||
pkgs.iproute2
|
||||
pkgs.iputils
|
||||
pkgs.netcat-openbsd
|
||||
pkgs.curl
|
||||
pkgs.nftables
|
||||
];
|
||||
};
|
||||
|
||||
# Overlay client
|
||||
client = { ... }: {
|
||||
networking.hostName = "client";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 ];
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{ address = "10.0.0.20"; prefixLength = 24; }
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
environment.systemPackages = [
|
||||
clientPkg
|
||||
pkgs.wireguard-tools
|
||||
pkgs.iproute2
|
||||
pkgs.iputils
|
||||
pkgs.netcat-openbsd
|
||||
pkgs.curl
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
node1.wait_for_unit("multi-user.target")
|
||||
router.wait_for_unit("multi-user.target")
|
||||
client.wait_for_unit("multi-user.target")
|
||||
|
||||
# Verify network connectivity
|
||||
node1.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.1/24'")
|
||||
router.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.10/24'")
|
||||
router.wait_until_succeeds("ip -4 addr show dev eth2 | grep -q '192.168.100.1/24'")
|
||||
client.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.20/24'")
|
||||
|
||||
# Start control plane
|
||||
node1.succeed("touch /tmp/lightscale-server.log")
|
||||
node1.execute("sh -c 'tail -n +1 -f /tmp/lightscale-server.log >/dev/console 2>&1 &'")
|
||||
node1.succeed(
|
||||
"systemd-run --no-block --unit=lightscale-server --service-type=simple "
|
||||
"--property=Restart=on-failure --property=RestartSec=1 "
|
||||
"--property=TimeoutStartSec=30 "
|
||||
"--property=StandardOutput=append:/tmp/lightscale-server.log "
|
||||
"--property=StandardError=append:/tmp/lightscale-server.log "
|
||||
"--setenv=RUST_LOG=info -- "
|
||||
"lightscale-server --listen 10.0.0.1:8080 --state /tmp/lightscale-state.json --admin-token test-token-12345"
|
||||
)
|
||||
node1.wait_for_unit("lightscale-server.service")
|
||||
node1.wait_for_open_port(8080, addr="10.0.0.1", timeout=120)
|
||||
|
||||
import json
|
||||
import time
|
||||
|
||||
# Create network
|
||||
net = json.loads(node1.succeed(
|
||||
"curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
|
||||
"-H 'authorization: Bearer test-token-12345' "
|
||||
"-H 'content-type: application/json' "
|
||||
"-d '{\"name\":\"lab\",\"bootstrap_token_ttl_seconds\":600,"
|
||||
"\"bootstrap_token_uses\":10,\"bootstrap_token_tags\":[\"lab\"]}'"
|
||||
))
|
||||
token = net["bootstrap_token"]["token"]
|
||||
|
||||
def enroll(node, name, ip, routes=None):
|
||||
node.succeed(
|
||||
"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
"init http://10.0.0.1:8080"
|
||||
)
|
||||
cmd = (
|
||||
f"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
f"--state-dir /tmp/ls-state register --node-name {name} -- {token}"
|
||||
)
|
||||
node.succeed(cmd)
|
||||
|
||||
# Build heartbeat command
|
||||
hb_cmd = (
|
||||
f"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
f"--state-dir /tmp/ls-state heartbeat --endpoint {ip}:51820"
|
||||
)
|
||||
if routes:
|
||||
for route in routes:
|
||||
hb_cmd += f" --route {route}"
|
||||
node.succeed(hb_cmd)
|
||||
|
||||
# Enroll router with subnet route (SNAT enabled)
|
||||
enroll(router, "router", "10.0.0.10", routes=["192.168.100.0/24"])
|
||||
|
||||
# Enroll client
|
||||
enroll(client, "client", "10.0.0.20")
|
||||
|
||||
# Start agents
|
||||
def start_agent(node, endpoints):
|
||||
node.succeed("touch /tmp/lightscale-agent.log")
|
||||
cmd = (
|
||||
"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
"--state-dir /tmp/ls-state agent --listen-port 51820 "
|
||||
"--heartbeat-interval 5 --longpoll-timeout 5"
|
||||
)
|
||||
for endpoint in endpoints:
|
||||
cmd += f" --endpoint {endpoint}"
|
||||
node.succeed(
|
||||
"systemd-run --no-block --unit=lightscale-agent --service-type=simple "
|
||||
"--property=Restart=on-failure --property=RestartSec=1 "
|
||||
"--property=TimeoutStartSec=30 "
|
||||
"--property=StandardOutput=append:/tmp/lightscale-agent.log "
|
||||
"--property=StandardError=append:/tmp/lightscale-agent.log -- "
|
||||
+ cmd
|
||||
)
|
||||
node.wait_for_unit("lightscale-agent.service")
|
||||
node.wait_until_succeeds("ip link show ls-test", timeout=60)
|
||||
|
||||
start_agent(router, ["10.0.0.10:51820"])
|
||||
start_agent(client, ["10.0.0.20:51820"])
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
# Get overlay IPs
|
||||
def overlay_ipv4(node):
|
||||
data = json.loads(node.succeed("cat /tmp/ls-state/state.json"))
|
||||
return data["ipv4"]
|
||||
|
||||
router_overlay_ip = overlay_ipv4(router)
|
||||
client_overlay_ip = overlay_ipv4(client)
|
||||
|
||||
# Verify direct overlay connectivity
|
||||
router.succeed(f"ping -c 3 {client_overlay_ip}")
|
||||
client.succeed(f"ping -c 3 {router_overlay_ip}")
|
||||
|
||||
# ===== Test 1: Enable subnet router with SNAT =====
|
||||
router.succeed("touch /tmp/router-enable.log")
|
||||
router.succeed(
|
||||
"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
"--state-dir /tmp/ls-state router enable "
|
||||
"--interface ls-test --out-interface eth2 2>&1 | tee /tmp/router-enable.log"
|
||||
)
|
||||
|
||||
# Verify nftables rules were created via libnftnl (not via nft CLI)
|
||||
# Check that the lightscale table exists
|
||||
router.succeed("nft list table inet lightscale")
|
||||
router.succeed("nft list table ip lightscale-nat")
|
||||
|
||||
# Show the rules for debugging
|
||||
router.succeed("nft list ruleset > /tmp/nft-ruleset.txt")
|
||||
router.succeed("cat /tmp/nft-ruleset.txt")
|
||||
|
||||
# Verify forwarding chain exists with our rules
|
||||
router.succeed("nft list chain inet lightscale ls-forward")
|
||||
router.succeed("nft list chain ip lightscale-nat ls-postrouting")
|
||||
|
||||
# Check that masquerade rule exists (SNAT enabled)
|
||||
router.succeed("nft list chain ip lightscale-nat ls-postrouting | grep -q masquerade")
|
||||
|
||||
# Verify forwarding rules are correct
|
||||
# Rule: iifname ls-test oifname eth2 accept
|
||||
router.succeed("nft list chain inet lightscale ls-forward | grep -q 'iifname.*ls-test'")
|
||||
router.succeed("nft list chain inet lightscale ls-forward | grep -q 'oifname.*eth2'")
|
||||
|
||||
# Rule: iifname eth2 oifname ls-test ct state established,related accept
|
||||
router.succeed("nft list chain inet lightscale ls-forward | grep -q 'iifname.*eth2'")
|
||||
router.succeed("nft list chain inet lightscale ls-forward | grep -q 'oifname.*ls-test'")
|
||||
|
||||
# Verify sysctl settings for forwarding
|
||||
router.succeed("cat /proc/sys/net/ipv4/ip_forward | grep -q 1")
|
||||
router.succeed("cat /proc/sys/net/ipv6/conf/all/forwarding | grep -q 1")
|
||||
|
||||
# ===== Test 2: Disable router and verify cleanup =====
|
||||
router.succeed(
|
||||
"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
"--state-dir /tmp/ls-state router disable "
|
||||
"--interface ls-test --out-interface eth2"
|
||||
)
|
||||
|
||||
# Verify tables are cleaned up
|
||||
router.fail("nft list table inet lightscale 2>/dev/null")
|
||||
router.fail("nft list table ip lightscale-nat 2>/dev/null")
|
||||
|
||||
# ===== Test 3: Re-enable with --no-snat =====
|
||||
router.succeed(
|
||||
"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
"--state-dir /tmp/ls-state router enable "
|
||||
"--interface ls-test --out-interface eth2 --no-snat"
|
||||
)
|
||||
|
||||
# Verify the filter table exists
|
||||
router.succeed("nft list table inet lightscale")
|
||||
|
||||
# Show ruleset for debugging (no NAT table or no masquerade)
|
||||
router.succeed("nft list ruleset > /tmp/nft-ruleset-no-snat.txt")
|
||||
router.succeed("cat /tmp/nft-ruleset-no-snat.txt")
|
||||
|
||||
# Verify no masquerade rule (NAT table should not exist with --no-snat)
|
||||
router.fail("nft list table ip lightscale-nat 2>/dev/null")
|
||||
|
||||
# ===== Test 4: Re-disable and final cleanup verification =====
|
||||
router.succeed(
|
||||
"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
"--state-dir /tmp/ls-state router disable "
|
||||
"--interface ls-test --out-interface eth2"
|
||||
)
|
||||
|
||||
# Final cleanup verification
|
||||
router.fail("nft list table inet lightscale 2>/dev/null")
|
||||
router.fail("nft list table ip lightscale-nat 2>/dev/null")
|
||||
|
||||
print("SUCCESS: libnftnl-based nftables operations work correctly!")
|
||||
print("- SNAT mode: inet lightscale + ip lightscale-nat tables created with masquerade")
|
||||
print("- No-SNAT mode: only inet lightscale table created")
|
||||
print("- Cleanup: all tables properly removed on disable")
|
||||
'';
|
||||
}
|
||||
233
lab/test-server-mesh-relay.nix
Normal file
233
lab/test-server-mesh-relay.nix
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
{ pkgs, serverPkg, clientPkg }:
|
||||
let
|
||||
meshCa = ./mesh-certs/ca.pem;
|
||||
meshCertA = ./mesh-certs/srv-a.pem;
|
||||
meshKeyA = ./mesh-certs/srv-a-key.pem;
|
||||
meshCertB = ./mesh-certs/srv-b.pem;
|
||||
meshKeyB = ./mesh-certs/srv-b-key.pem;
|
||||
in
|
||||
{
|
||||
name = "lightscale-lab-server-mesh-relay";
|
||||
nodes = {
|
||||
node1 = { ... }: {
|
||||
networking.hostName = "node1";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 ];
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{ address = "10.0.0.1"; prefixLength = 24; }
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
environment.systemPackages = [
|
||||
serverPkg
|
||||
pkgs.iproute2
|
||||
pkgs.iputils
|
||||
pkgs.netcat-openbsd
|
||||
pkgs.curl
|
||||
];
|
||||
};
|
||||
node2 = { ... }: {
|
||||
networking.hostName = "node2";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 ];
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{ address = "10.0.0.2"; prefixLength = 24; }
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
environment.systemPackages = [
|
||||
serverPkg
|
||||
pkgs.iproute2
|
||||
pkgs.iputils
|
||||
pkgs.netcat-openbsd
|
||||
pkgs.curl
|
||||
];
|
||||
};
|
||||
node3 = { ... }: {
|
||||
networking.hostName = "node3";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 ];
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{ address = "10.0.0.3"; prefixLength = 24; }
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
environment.systemPackages = [
|
||||
clientPkg
|
||||
pkgs.iproute2
|
||||
pkgs.iputils
|
||||
pkgs.netcat-openbsd
|
||||
pkgs.curl
|
||||
pkgs.iptables
|
||||
];
|
||||
};
|
||||
node4 = { ... }: {
|
||||
networking.hostName = "node4";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 ];
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{ address = "10.0.0.4"; prefixLength = 24; }
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
environment.systemPackages = [
|
||||
clientPkg
|
||||
pkgs.iproute2
|
||||
pkgs.iputils
|
||||
pkgs.netcat-openbsd
|
||||
pkgs.curl
|
||||
pkgs.iptables
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
node1.wait_for_unit("multi-user.target")
|
||||
node2.wait_for_unit("multi-user.target")
|
||||
node3.wait_for_unit("multi-user.target")
|
||||
node4.wait_for_unit("multi-user.target")
|
||||
|
||||
node1.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.1/24'")
|
||||
node2.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.2/24'")
|
||||
node3.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.3/24'")
|
||||
node4.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q '10.0.0.4/24'")
|
||||
|
||||
node1.succeed("touch /tmp/lightscale-server-a.log")
|
||||
node2.succeed("touch /tmp/lightscale-server-b.log")
|
||||
node1.execute("sh -c 'tail -n +1 -f /tmp/lightscale-server-a.log >/dev/console 2>&1 &'")
|
||||
node2.execute("sh -c 'tail -n +1 -f /tmp/lightscale-server-b.log >/dev/console 2>&1 &'")
|
||||
|
||||
node1.succeed(
|
||||
"systemd-run --no-block --unit=lightscale-server-a --service-type=simple "
|
||||
"--property=Restart=on-failure --property=RestartSec=1 "
|
||||
"--property=TimeoutStartSec=30 "
|
||||
"--property=StandardOutput=append:/tmp/lightscale-server-a.log "
|
||||
"--property=StandardError=append:/tmp/lightscale-server-a.log "
|
||||
"--setenv=RUST_LOG=info -- "
|
||||
"lightscale-server --listen 10.0.0.1:8080 --state /tmp/lightscale-state-a.json "
|
||||
"--admin-token test-admin "
|
||||
"--stream-relay 10.0.0.1:8443,10.0.0.2:8443 "
|
||||
"--stream-relay-listen 10.0.0.1:8443 "
|
||||
"--mesh-server-id srv-a.mesh "
|
||||
"--mesh-listen 10.0.0.1:7443 "
|
||||
"--mesh-peer srv-b.mesh=10.0.0.2:7443 "
|
||||
"--mesh-ca-cert ${meshCa} "
|
||||
"--mesh-cert ${meshCertA} "
|
||||
"--mesh-key ${meshKeyA}"
|
||||
)
|
||||
node2.succeed(
|
||||
"systemd-run --no-block --unit=lightscale-server-b --service-type=simple "
|
||||
"--property=Restart=on-failure --property=RestartSec=1 "
|
||||
"--property=TimeoutStartSec=30 "
|
||||
"--property=StandardOutput=append:/tmp/lightscale-server-b.log "
|
||||
"--property=StandardError=append:/tmp/lightscale-server-b.log "
|
||||
"--setenv=RUST_LOG=info -- "
|
||||
"lightscale-server --listen 10.0.0.2:8081 --state /tmp/lightscale-state-b.json "
|
||||
"--admin-token test-admin "
|
||||
"--stream-relay-listen 10.0.0.2:8443 "
|
||||
"--mesh-server-id srv-b.mesh "
|
||||
"--mesh-listen 10.0.0.2:7443 "
|
||||
"--mesh-peer srv-a.mesh=10.0.0.1:7443 "
|
||||
"--mesh-ca-cert ${meshCa} "
|
||||
"--mesh-cert ${meshCertB} "
|
||||
"--mesh-key ${meshKeyB}"
|
||||
)
|
||||
|
||||
node1.wait_for_unit("lightscale-server-a.service")
|
||||
node2.wait_for_unit("lightscale-server-b.service")
|
||||
node1.wait_for_open_port(8080, addr="10.0.0.1", timeout=120)
|
||||
node1.wait_for_open_port(8443, addr="10.0.0.1", timeout=120)
|
||||
node1.wait_for_open_port(7443, addr="10.0.0.1", timeout=120)
|
||||
node2.wait_for_open_port(8443, addr="10.0.0.2", timeout=120)
|
||||
node2.wait_for_open_port(7443, addr="10.0.0.2", timeout=120)
|
||||
node1.wait_until_succeeds("grep -q 'mesh enabled as srv-a.mesh' /tmp/lightscale-server-a.log", timeout=120)
|
||||
node2.wait_until_succeeds("grep -q 'mesh enabled as srv-b.mesh' /tmp/lightscale-server-b.log", timeout=120)
|
||||
|
||||
import json
|
||||
|
||||
net = json.loads(node1.succeed(
|
||||
"curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
|
||||
"-H 'authorization: Bearer test-admin' "
|
||||
"-H 'content-type: application/json' "
|
||||
"-d '{\"name\":\"mesh\",\"bootstrap_token_ttl_seconds\":600," \
|
||||
"\"bootstrap_token_uses\":10,\"bootstrap_token_tags\":[\"mesh\"]}'"
|
||||
))
|
||||
token = net["bootstrap_token"]["token"]
|
||||
|
||||
def enroll(node, name, ip, state_dir):
|
||||
node.succeed(
|
||||
"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
"init http://10.0.0.1:8080"
|
||||
)
|
||||
node.succeed(
|
||||
f"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
f"--state-dir {state_dir} register --node-name {name} -- {token}"
|
||||
)
|
||||
node.succeed(
|
||||
f"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
f"--state-dir {state_dir} heartbeat --endpoint {ip}:51820"
|
||||
)
|
||||
|
||||
def start_agent(node, state_dir, endpoint, relay_server):
|
||||
node.succeed("touch /tmp/lightscale-agent.log")
|
||||
node.execute("sh -c 'tail -n +1 -f /tmp/lightscale-agent.log >/dev/console 2>&1 &'")
|
||||
cmd = (
|
||||
"lightscale-client --profile test --config /tmp/ls-config.json "
|
||||
f"--state-dir {state_dir} agent --listen-port 51820 "
|
||||
"--heartbeat-interval 5 --longpoll-timeout 5 "
|
||||
f"--endpoint {endpoint} --stream-relay "
|
||||
f"--stream-relay-server {relay_server} "
|
||||
"--endpoint-stale-after 5 --endpoint-max-rotations 1 "
|
||||
"--relay-reprobe-after 10"
|
||||
)
|
||||
node.succeed(
|
||||
"systemd-run --no-block --unit=lightscale-agent --service-type=simple "
|
||||
"--property=Restart=on-failure --property=RestartSec=1 "
|
||||
"--property=TimeoutStartSec=30 "
|
||||
"--property=StandardOutput=append:/tmp/lightscale-agent.log "
|
||||
"--property=StandardError=append:/tmp/lightscale-agent.log -- "
|
||||
+ cmd
|
||||
)
|
||||
node.wait_for_unit("lightscale-agent.service")
|
||||
node.wait_until_succeeds("ip link show ls-test", timeout=120)
|
||||
|
||||
enroll(node3, "node3", "10.0.0.3", "/tmp/ls-state-3")
|
||||
enroll(node4, "node4", "10.0.0.4", "/tmp/ls-state-4")
|
||||
|
||||
node3.succeed("iptables -F OUTPUT")
|
||||
node3.succeed("iptables -P OUTPUT DROP")
|
||||
node3.succeed("iptables -A OUTPUT -o lo -j ACCEPT")
|
||||
node3.succeed("iptables -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT")
|
||||
node3.succeed("iptables -A OUTPUT -o ls-test -j ACCEPT")
|
||||
node3.succeed("iptables -A OUTPUT -d 10.0.0.1 -p tcp -m multiport --dports 8080,8443 -j ACCEPT")
|
||||
|
||||
node4.succeed("iptables -F OUTPUT")
|
||||
node4.succeed("iptables -P OUTPUT DROP")
|
||||
node4.succeed("iptables -A OUTPUT -o lo -j ACCEPT")
|
||||
node4.succeed("iptables -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT")
|
||||
node4.succeed("iptables -A OUTPUT -o ls-test -j ACCEPT")
|
||||
node4.succeed("iptables -A OUTPUT -d 10.0.0.1 -p tcp --dport 8080 -j ACCEPT")
|
||||
node4.succeed("iptables -A OUTPUT -d 10.0.0.2 -p tcp --dport 8443 -j ACCEPT")
|
||||
|
||||
start_agent(node3, "/tmp/ls-state-3", "203.0.113.3:51820", "10.0.0.1:8443")
|
||||
start_agent(node4, "/tmp/ls-state-4", "203.0.113.4:51820", "10.0.0.2:8443")
|
||||
|
||||
node3.wait_until_fails("nc -z -w 1 10.0.0.2 8443")
|
||||
node4.wait_until_fails("nc -z -w 1 10.0.0.1 8443")
|
||||
node3.wait_until_succeeds("grep -q 'connected to 10.0.0.1:8443' /tmp/lightscale-agent.log", timeout=300)
|
||||
node4.wait_until_succeeds("grep -q 'connected to 10.0.0.2:8443' /tmp/lightscale-agent.log", timeout=300)
|
||||
|
||||
data3 = json.loads(node3.succeed("cat /tmp/ls-state-3/state.json"))
|
||||
data4 = json.loads(node4.succeed("cat /tmp/ls-state-4/state.json"))
|
||||
ip3 = data3["ipv4"]
|
||||
ip4 = data4["ipv4"]
|
||||
|
||||
node3.wait_until_succeeds(f"ping -c 3 {ip4}", timeout=240)
|
||||
node4.wait_until_succeeds(f"ping -c 3 {ip3}", timeout=240)
|
||||
'';
|
||||
}
|
||||
167
lab/test-services.nix
Normal file
167
lab/test-services.nix
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
{ pkgs, serverPkg, clientPkg }:
|
||||
let
|
||||
serverModule = import ../nixos/modules/lightscale-server.nix {
|
||||
defaultPackage = serverPkg;
|
||||
};
|
||||
clientModule = import ../nixos/modules/lightscale-client.nix {
|
||||
defaultPackage = clientPkg;
|
||||
};
|
||||
in
|
||||
{
|
||||
name = "lightscale-lab-services";
|
||||
nodes = {
|
||||
server = { ... }: {
|
||||
imports = [ serverModule ];
|
||||
networking.hostName = "server";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 ];
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{
|
||||
address = "10.0.0.1";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
|
||||
services.lightscale-server = {
|
||||
enable = true;
|
||||
listen = "10.0.0.1:8080";
|
||||
stateFile = "/var/lib/lightscale-server/state.json";
|
||||
adminToken = "lab-admin-token";
|
||||
};
|
||||
|
||||
environment.systemPackages = [
|
||||
clientPkg
|
||||
pkgs.curl
|
||||
pkgs.iputils
|
||||
];
|
||||
};
|
||||
|
||||
client1 = { ... }: {
|
||||
imports = [ clientModule ];
|
||||
networking.hostName = "client1";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 ];
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{
|
||||
address = "10.0.0.2";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
|
||||
services.lightscale-client = {
|
||||
enable = true;
|
||||
profile = "svc";
|
||||
stateDir = "/var/lib/lightscale-client";
|
||||
controlUrls = [ "http://10.0.0.1:8080" ];
|
||||
listenPort = 51820;
|
||||
applyRoutes = true;
|
||||
heartbeatInterval = 5;
|
||||
longpollTimeout = 5;
|
||||
};
|
||||
|
||||
environment.systemPackages = [
|
||||
clientPkg
|
||||
pkgs.iputils
|
||||
];
|
||||
};
|
||||
|
||||
client2 = { ... }: {
|
||||
imports = [ clientModule ];
|
||||
networking.hostName = "client2";
|
||||
networking.usePredictableInterfaceNames = false;
|
||||
virtualisation.vlans = [ 1 ];
|
||||
networking.interfaces.eth1.useDHCP = false;
|
||||
networking.interfaces.eth1.ipv4.addresses = [
|
||||
{
|
||||
address = "10.0.0.3";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
networking.firewall.enable = false;
|
||||
boot.kernelModules = [ "wireguard" ];
|
||||
|
||||
services.lightscale-client = {
|
||||
enable = true;
|
||||
profile = "svc";
|
||||
stateDir = "/var/lib/lightscale-client";
|
||||
controlUrls = [ "http://10.0.0.1:8080" ];
|
||||
listenPort = 51820;
|
||||
applyRoutes = true;
|
||||
heartbeatInterval = 5;
|
||||
longpollTimeout = 5;
|
||||
};
|
||||
|
||||
environment.systemPackages = [
|
||||
clientPkg
|
||||
pkgs.iputils
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
|
||||
start_all()
|
||||
|
||||
server.wait_for_unit("lightscale-server.service")
|
||||
server.wait_for_open_port(8080, addr="10.0.0.1", timeout=120)
|
||||
client1.wait_for_unit("multi-user.target")
|
||||
client2.wait_for_unit("multi-user.target")
|
||||
|
||||
net = json.loads(server.succeed(
|
||||
"curl -sSf -X POST http://10.0.0.1:8080/v1/networks "
|
||||
"-H 'authorization: Bearer lab-admin-token' "
|
||||
"-H 'content-type: application/json' "
|
||||
"-d '{\"name\":\"svc-net\",\"bootstrap_token_ttl_seconds\":1200,\"bootstrap_token_uses\":10}'"
|
||||
))
|
||||
token = net["bootstrap_token"]["token"]
|
||||
network_id = net["network"]["id"]
|
||||
|
||||
def register(node, name):
|
||||
node.succeed(
|
||||
"lightscale-client --profile svc "
|
||||
"--state-dir /var/lib/lightscale-client/svc "
|
||||
"--control-url http://10.0.0.1:8080 "
|
||||
f"register --node-name {name} -- {token}"
|
||||
)
|
||||
node.wait_until_succeeds("test -s /var/lib/lightscale-client/svc/state.json")
|
||||
node.succeed("systemctl start lightscale-client.service")
|
||||
node.wait_for_unit("lightscale-client.service")
|
||||
node.wait_until_succeeds("ip link show ls-svc", timeout=120)
|
||||
|
||||
register(client1, "client1")
|
||||
register(client2, "client2")
|
||||
|
||||
server.wait_until_succeeds(
|
||||
f"curl -sSf -H 'authorization: Bearer lab-admin-token' "
|
||||
f"http://10.0.0.1:8080/v1/admin/networks/{network_id}/nodes | grep -q '\"nodes\"'",
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
state1 = json.loads(client1.succeed("cat /var/lib/lightscale-client/svc/state.json"))
|
||||
state2 = json.loads(client2.succeed("cat /var/lib/lightscale-client/svc/state.json"))
|
||||
ip1 = state1["ipv4"]
|
||||
ip2 = state2["ipv4"]
|
||||
|
||||
client1.wait_until_succeeds(f"ping -c 3 {ip2}", timeout=180)
|
||||
client2.wait_until_succeeds(f"ping -c 3 {ip1}", timeout=180)
|
||||
|
||||
client1.succeed("systemctl restart lightscale-client.service")
|
||||
client1.wait_for_unit("lightscale-client.service")
|
||||
client1.wait_until_succeeds(f"ping -c 3 {ip2}", timeout=180)
|
||||
|
||||
server.succeed("systemctl restart lightscale-server.service")
|
||||
server.wait_for_unit("lightscale-server.service")
|
||||
client1.wait_until_succeeds(
|
||||
"lightscale-client --profile svc --state-dir /var/lib/lightscale-client/svc "
|
||||
"--control-url http://10.0.0.1:8080 netmap | grep -q 'approved: true'",
|
||||
timeout=180,
|
||||
)
|
||||
'';
|
||||
}
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit 9a5d8ca8ba540f856ea87484742b22f6d4819d3c
|
||||
Subproject commit d0155e6e46260c0b1aa636a995ccbdcbc559a812
|
||||
98
nixos/README.md
Normal file
98
nixos/README.md
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
# NixOS Modules
|
||||
|
||||
This flake exports two modules:
|
||||
|
||||
- `lightscale.nixosModules.lightscale-server`
|
||||
- `lightscale.nixosModules.lightscale-client`
|
||||
|
||||
## Server example
|
||||
|
||||
```nix
|
||||
{
|
||||
imports = [
|
||||
lightscale.nixosModules.lightscale-server
|
||||
];
|
||||
|
||||
services.lightscale-server = {
|
||||
enable = true;
|
||||
listen = "0.0.0.0:8080";
|
||||
stateFile = "/var/lib/lightscale-server/state.json";
|
||||
# Or use dbUrl / dbUrlFile for shared DB:
|
||||
# dbUrlFile = "/run/secrets/lightscale-db-url";
|
||||
openFirewall = true;
|
||||
firewallTCPPorts = [ 8080 ];
|
||||
# Optional relay advertisement/listeners:
|
||||
# streamRelayServers = [ "vpn.example.com:443" ];
|
||||
# streamRelayListen = "0.0.0.0:443";
|
||||
# udpRelayServers = [ "vpn.example.com:3478" ];
|
||||
# udpRelayListen = "0.0.0.0:3478";
|
||||
# Optional inter-server relay mesh (mTLS):
|
||||
# meshServerId = "vpn-a.example.com";
|
||||
# meshListen = "0.0.0.0:7443";
|
||||
# meshPeers = [ "vpn-b.example.com=10.0.0.12:7443" ];
|
||||
# meshCaCert = "/run/secrets/lightscale-mesh-ca.pem";
|
||||
# meshCert = "/run/secrets/lightscale-mesh-vpn-a.pem";
|
||||
# meshKey = "/run/secrets/lightscale-mesh-vpn-a-key.pem";
|
||||
# meshMaxHops = 4;
|
||||
environmentFiles = [ "/run/secrets/lightscale-server.env" ];
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
`/run/secrets/lightscale-server.env` should include:
|
||||
|
||||
```sh
|
||||
LIGHTSCALE_ADMIN_TOKEN=replace-me
|
||||
```
|
||||
|
||||
Optional DB URL secret file example:
|
||||
|
||||
```sh
|
||||
postgres://lightscale:secret@db.internal/lightscale?sslmode=require
|
||||
```
|
||||
|
||||
## Client agent example
|
||||
|
||||
```nix
|
||||
{
|
||||
imports = [
|
||||
lightscale.nixosModules.lightscale-client
|
||||
];
|
||||
|
||||
services.lightscale-client = {
|
||||
enable = true;
|
||||
profile = "prod";
|
||||
controlUrls = [ "https://vpn.example.com:8080" ];
|
||||
stateDir = "/var/lib/lightscale-client";
|
||||
listenPort = 51820;
|
||||
applyRoutes = true;
|
||||
streamRelay = true;
|
||||
relayReprobeAfter = 60;
|
||||
openFirewall = true;
|
||||
# listenPort is opened automatically when openFirewall=true.
|
||||
environmentFiles = [ "/run/secrets/lightscale-client.env" ];
|
||||
autoRegister = true;
|
||||
enrollmentTokenFile = "/run/secrets/lightscale-enroll-token";
|
||||
registerNodeName = "host-01";
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
Optional secret env file for admin endpoints:
|
||||
|
||||
```sh
|
||||
LIGHTSCALE_ADMIN_TOKEN=replace-me
|
||||
```
|
||||
|
||||
## Bootstrap note
|
||||
|
||||
`lightscale-client.service` starts only after `state.json` exists for the profile.
|
||||
|
||||
When `autoRegister = true`, a one-shot service registers the node once and then the agent runs.
|
||||
|
||||
If you keep `autoRegister = false`, run registration manually once (same profile/state directory):
|
||||
|
||||
```sh
|
||||
lightscale-client --profile prod --state-dir /var/lib/lightscale-client/prod \
|
||||
--control-url https://vpn.example.com:8080 register <enrollment-token>
|
||||
```
|
||||
474
nixos/modules/lightscale-client.nix
Normal file
474
nixos/modules/lightscale-client.nix
Normal file
|
|
@ -0,0 +1,474 @@
|
|||
{ defaultPackage }:
|
||||
{ config, lib, pkgs, ... }:
|
||||
let
|
||||
cfg = config.services.lightscale-client;
|
||||
|
||||
stateProfileDir = "${cfg.stateDir}/${cfg.profile}";
|
||||
|
||||
baseArgs =
|
||||
[ "--profile" cfg.profile ]
|
||||
++ lib.optionals (cfg.configFile != null) [ "--config" cfg.configFile ]
|
||||
++ [ "--state-dir" stateProfileDir ]
|
||||
++ lib.optionals (cfg.controlUrls != [ ]) [ "--control-url" (lib.concatStringsSep "," cfg.controlUrls) ]
|
||||
++ lib.optionals (cfg.tlsPin != null) [ "--tls-pin" cfg.tlsPin ]
|
||||
++ lib.optionals (cfg.adminToken != null) [ "--admin-token" cfg.adminToken ];
|
||||
|
||||
agentArgs =
|
||||
[ "agent" "--listen-port" (toString cfg.listenPort) ]
|
||||
++ lib.optionals (cfg.interface != null) [ "--interface" cfg.interface ]
|
||||
++ lib.optionals cfg.applyRoutes [ "--apply-routes" ]
|
||||
++ lib.optionals cfg.acceptExitNode [ "--accept-exit-node" ]
|
||||
++ lib.optionals (cfg.exitNodeId != null) [ "--exit-node-id" cfg.exitNodeId ]
|
||||
++ lib.optionals (cfg.exitNodeName != null) [ "--exit-node-name" cfg.exitNodeName ]
|
||||
++ lib.optionals (cfg.exitNodeTag != null) [ "--exit-node-tag" cfg.exitNodeTag ]
|
||||
++ [ "--exit-node-policy" cfg.exitNodePolicy ]
|
||||
++ lib.optionals (cfg.exitNodeMetricBase != null) [ "--exit-node-metric-base" (toString cfg.exitNodeMetricBase) ]
|
||||
++ lib.optionals (cfg.exitNodeUidRange != null) [ "--exit-node-uid-range" cfg.exitNodeUidRange ]
|
||||
++ lib.optionals cfg.allowRouteConflicts [ "--allow-route-conflicts" ]
|
||||
++ lib.optionals (cfg.routeTable != null) [ "--route-table" (toString cfg.routeTable) ]
|
||||
++ lib.concatMap (endpoint: [ "--endpoint" endpoint ]) cfg.endpoints
|
||||
++ lib.concatMap (prefix: [ "--advertise-route" prefix ]) cfg.advertiseRoutes
|
||||
++ lib.concatMap (mapping: [ "--advertise-map" mapping ]) cfg.advertiseMaps
|
||||
++ lib.optionals cfg.advertiseExitNode [ "--advertise-exit-node" ]
|
||||
++ [ "--heartbeat-interval" (toString cfg.heartbeatInterval) ]
|
||||
++ [ "--longpoll-timeout" (toString cfg.longpollTimeout) ]
|
||||
++ [ "--backend" cfg.backend ]
|
||||
++ lib.optionals cfg.stun [ "--stun" ]
|
||||
++ lib.optionals (cfg.stunServers != [ ]) [ "--stun-server" (lib.concatStringsSep "," cfg.stunServers) ]
|
||||
++ lib.optionals (cfg.stunPort != null) [ "--stun-port" (toString cfg.stunPort) ]
|
||||
++ [ "--stun-timeout" (toString cfg.stunTimeout) ]
|
||||
++ lib.optionals cfg.probePeers [ "--probe-peers" ]
|
||||
++ [ "--probe-timeout" (toString cfg.probeTimeout) ]
|
||||
++ lib.optionals cfg.streamRelay [ "--stream-relay" ]
|
||||
++ lib.optionals (cfg.streamRelayServers != [ ]) [ "--stream-relay-server" (lib.concatStringsSep "," cfg.streamRelayServers) ]
|
||||
++ [ "--endpoint-stale-after" (toString cfg.endpointStaleAfter) ]
|
||||
++ [ "--endpoint-max-rotations" (toString cfg.endpointMaxRotations) ]
|
||||
++ [ "--relay-reprobe-after" (toString cfg.relayReprobeAfter) ]
|
||||
++ lib.optionals (cfg.dnsHostsPath != null) [ "--dns-hosts-path" cfg.dnsHostsPath ]
|
||||
++ lib.optionals cfg.dnsServe [ "--dns-serve" ]
|
||||
++ lib.optionals (cfg.dnsListen != null) [ "--dns-listen" cfg.dnsListen ]
|
||||
++ lib.optionals cfg.dnsApplyResolver [ "--dns-apply-resolver" ]
|
||||
++ lib.optionals cfg.l2Relay [ "--l2-relay" ]
|
||||
++ cfg.extraArgs;
|
||||
|
||||
args = baseArgs ++ agentArgs;
|
||||
startCmd = "${lib.getExe' cfg.package "lightscale-client"} ${lib.escapeShellArgs args}";
|
||||
|
||||
registerArgs =
|
||||
baseArgs
|
||||
++ [ "register" ]
|
||||
++ lib.optionals (cfg.registerNodeName != null) [ "--node-name" cfg.registerNodeName ]
|
||||
++ cfg.registerExtraArgs;
|
||||
|
||||
registerScript = pkgs.writeShellScript "lightscale-client-register-${cfg.profile}" ''
|
||||
set -euo pipefail
|
||||
|
||||
state_file=${lib.escapeShellArg "${stateProfileDir}/state.json"}
|
||||
if [[ -s "$state_file" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
token_file=${lib.escapeShellArg cfg.enrollmentTokenFile}
|
||||
if [[ ! -r "$token_file" ]]; then
|
||||
echo "enrollment token file not readable: $token_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
token="$(tr -d '\r\n' < "$token_file")"
|
||||
if [[ -z "$token" ]]; then
|
||||
echo "enrollment token is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec ${lib.getExe' cfg.package "lightscale-client"} ${lib.escapeShellArgs registerArgs} -- "$token"
|
||||
'';
|
||||
|
||||
defaultUdpPorts = lib.unique ([ cfg.listenPort ] ++ cfg.firewallUDPPorts);
|
||||
in
|
||||
{
|
||||
options.services.lightscale-client = {
|
||||
enable = lib.mkEnableOption "lightscale client agent";
|
||||
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = defaultPackage;
|
||||
description = "lightscale-client package.";
|
||||
};
|
||||
|
||||
profile = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "default";
|
||||
description = "Profile name passed to lightscale-client.";
|
||||
};
|
||||
|
||||
stateDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/var/lib/lightscale-client";
|
||||
description = "Base state directory; per-profile state is stored under stateDir/profile.";
|
||||
};
|
||||
|
||||
configFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional config file path for profiles and TLS pin.";
|
||||
};
|
||||
|
||||
controlUrls = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Control plane URLs passed via --control-url (comma-separated failover supported).";
|
||||
};
|
||||
|
||||
tlsPin = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional SHA-256 TLS pin (same format as `lightscale-client pin`).";
|
||||
};
|
||||
|
||||
adminToken = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional admin token flag. Prefer environmentFiles for secrets.";
|
||||
};
|
||||
|
||||
autoRegister = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Run a one-shot registration service when state file is missing.";
|
||||
};
|
||||
|
||||
enrollmentTokenFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Path to enrollment token file for autoRegister.";
|
||||
};
|
||||
|
||||
registerNodeName = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional node name used by autoRegister.";
|
||||
};
|
||||
|
||||
registerExtraArgs = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Extra args appended to autoRegister `register` command.";
|
||||
};
|
||||
|
||||
environmentFiles = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "systemd EnvironmentFile entries (for LIGHTSCALE_ADMIN_TOKEN, etc.).";
|
||||
};
|
||||
|
||||
environment = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.str;
|
||||
default = { };
|
||||
description = "Additional environment variables for the client service.";
|
||||
};
|
||||
|
||||
interface = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "WireGuard interface name. Default is derived from profile.";
|
||||
};
|
||||
|
||||
listenPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 51820;
|
||||
description = "WireGuard listen port for the agent.";
|
||||
};
|
||||
|
||||
backend = lib.mkOption {
|
||||
type = lib.types.enum [ "kernel" "boringtun" ];
|
||||
default = "kernel";
|
||||
description = "WireGuard backend used by the agent.";
|
||||
};
|
||||
|
||||
endpoints = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Advertised endpoints (HOST:PORT), passed as repeated --endpoint flags.";
|
||||
};
|
||||
|
||||
applyRoutes = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Apply advertised subnet/exit routes on netmap updates.";
|
||||
};
|
||||
|
||||
acceptExitNode = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Accept exit routes from selected peers.";
|
||||
};
|
||||
|
||||
exitNodeId = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Specific exit node ID to use.";
|
||||
};
|
||||
|
||||
exitNodeName = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Specific exit node name to use.";
|
||||
};
|
||||
|
||||
exitNodePolicy = lib.mkOption {
|
||||
type = lib.types.enum [ "first" "latest" "multi" ];
|
||||
default = "first";
|
||||
description = "Policy for selecting exit nodes.";
|
||||
};
|
||||
|
||||
exitNodeTag = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Only consider exit nodes with this tag.";
|
||||
};
|
||||
|
||||
exitNodeMetricBase = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.ints.unsigned;
|
||||
default = null;
|
||||
description = "Base route metric used for selected exit routes.";
|
||||
};
|
||||
|
||||
exitNodeUidRange = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "UID or UID range for per-user exit routing (e.g. 1000-1999).";
|
||||
};
|
||||
|
||||
allowRouteConflicts = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Force route installation even when conflicts are detected.";
|
||||
};
|
||||
|
||||
routeTable = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.ints.unsigned;
|
||||
default = null;
|
||||
description = "Optional policy routing table ID.";
|
||||
};
|
||||
|
||||
advertiseRoutes = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Advertised subnet routes (CIDR).";
|
||||
};
|
||||
|
||||
advertiseMaps = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Route map entries in REAL=MAPPED form.";
|
||||
};
|
||||
|
||||
advertiseExitNode = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Advertise this node as an exit node.";
|
||||
};
|
||||
|
||||
heartbeatInterval = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 30;
|
||||
description = "Heartbeat interval in seconds.";
|
||||
};
|
||||
|
||||
longpollTimeout = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 30;
|
||||
description = "Netmap long-poll timeout in seconds.";
|
||||
};
|
||||
|
||||
stun = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Enable STUN endpoint discovery in the agent.";
|
||||
};
|
||||
|
||||
stunServers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Explicit STUN servers (HOST:PORT).";
|
||||
};
|
||||
|
||||
stunPort = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.port;
|
||||
default = null;
|
||||
description = "Optional local UDP port for STUN probe socket.";
|
||||
};
|
||||
|
||||
stunTimeout = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 3;
|
||||
description = "STUN timeout in seconds.";
|
||||
};
|
||||
|
||||
probePeers = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Send UDP probes to peers when netmap changes.";
|
||||
};
|
||||
|
||||
probeTimeout = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 1;
|
||||
description = "UDP probe timeout in seconds.";
|
||||
};
|
||||
|
||||
streamRelay = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Enable stream-relay endpoint fallback support.";
|
||||
};
|
||||
|
||||
streamRelayServers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Explicit stream relay servers (HOST:PORT).";
|
||||
};
|
||||
|
||||
endpointStaleAfter = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 15;
|
||||
description = "Seconds before endpoint is considered stale.";
|
||||
};
|
||||
|
||||
endpointMaxRotations = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 2;
|
||||
description = "Maximum endpoint rotations before relay fallback.";
|
||||
};
|
||||
|
||||
relayReprobeAfter = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 60;
|
||||
description = "Seconds between direct re-probes while relay is active.";
|
||||
};
|
||||
|
||||
dnsHostsPath = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional hosts file path updated from netmap.";
|
||||
};
|
||||
|
||||
dnsServe = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Serve MagicDNS-like answers for the overlay domain.";
|
||||
};
|
||||
|
||||
dnsListen = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "DNS listen address (defaults to 127.0.0.1:53 when dnsServe=true).";
|
||||
};
|
||||
|
||||
dnsApplyResolver = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Apply per-domain resolver routing via resolvectl.";
|
||||
};
|
||||
|
||||
l2Relay = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Enable experimental L2 relay behavior.";
|
||||
};
|
||||
|
||||
extraArgs = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Additional CLI arguments appended to the agent command.";
|
||||
};
|
||||
|
||||
openFirewall = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Open firewall ports listed in firewallTCPPorts and listen UDP port(s).";
|
||||
};
|
||||
|
||||
firewallTCPPorts = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.port;
|
||||
default = [ ];
|
||||
description = "Extra TCP ports to open when openFirewall=true.";
|
||||
};
|
||||
|
||||
firewallUDPPorts = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.port;
|
||||
default = [ ];
|
||||
description = "Extra UDP ports to open when openFirewall=true (listenPort is always included).";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.controlUrls != [ ] || cfg.configFile != null;
|
||||
message = "services.lightscale-client: set controlUrls or provide configFile with initialized profile.";
|
||||
}
|
||||
{
|
||||
assertion = (!cfg.autoRegister) || (cfg.enrollmentTokenFile != null);
|
||||
message = "services.lightscale-client: autoRegister=true requires enrollmentTokenFile.";
|
||||
}
|
||||
];
|
||||
|
||||
systemd.services.lightscale-client-register = lib.mkIf cfg.autoRegister {
|
||||
description = "lightscale client one-shot registration (${cfg.profile})";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [ "lightscale-client.service" ];
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
path = [ pkgs.coreutils ];
|
||||
unitConfig = {
|
||||
ConditionPathExists = "!${stateProfileDir}/state.json";
|
||||
};
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = registerScript;
|
||||
EnvironmentFile = cfg.environmentFiles;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.lightscale-client = {
|
||||
description = "lightscale client agent";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" ] ++ lib.optionals cfg.autoRegister [ "lightscale-client-register.service" ];
|
||||
wants = [ "network-online.target" ] ++ lib.optionals cfg.autoRegister [ "lightscale-client-register.service" ];
|
||||
unitConfig = {
|
||||
ConditionPathExists = "${stateProfileDir}/state.json";
|
||||
};
|
||||
path = [
|
||||
pkgs.iproute2
|
||||
pkgs.nftables
|
||||
pkgs.systemd
|
||||
];
|
||||
environment = cfg.environment;
|
||||
preStart = ''
|
||||
install -d -m 0750 ${lib.escapeShellArg cfg.stateDir}
|
||||
install -d -m 0750 ${lib.escapeShellArg stateProfileDir}
|
||||
'';
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
ExecStart = startCmd;
|
||||
Restart = "on-failure";
|
||||
RestartSec = 2;
|
||||
EnvironmentFile = cfg.environmentFiles;
|
||||
CapabilityBoundingSet = [
|
||||
"CAP_NET_ADMIN"
|
||||
"CAP_NET_RAW"
|
||||
"CAP_NET_BIND_SERVICE"
|
||||
];
|
||||
AmbientCapabilities = [
|
||||
"CAP_NET_ADMIN"
|
||||
"CAP_NET_RAW"
|
||||
"CAP_NET_BIND_SERVICE"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall cfg.firewallTCPPorts;
|
||||
networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall defaultUdpPorts;
|
||||
};
|
||||
}
|
||||
247
nixos/modules/lightscale-server.nix
Normal file
247
nixos/modules/lightscale-server.nix
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
{ defaultPackage }:
|
||||
{ config, lib, ... }:
|
||||
let
|
||||
cfg = config.services.lightscale-server;
|
||||
meshEnabled =
|
||||
cfg.meshServerId != null
|
||||
|| cfg.meshListen != null
|
||||
|| cfg.meshPeers != [ ]
|
||||
|| cfg.meshCaCert != null
|
||||
|| cfg.meshCert != null
|
||||
|| cfg.meshKey != null;
|
||||
|
||||
args =
|
||||
[ "--listen" cfg.listen ]
|
||||
++ lib.optionals (cfg.dbUrl == null && cfg.dbUrlFile == null) [ "--state" cfg.stateFile ]
|
||||
++ lib.optionals (cfg.dbUrl != null) [ "--db-url" cfg.dbUrl ]
|
||||
++ lib.optionals (cfg.dbUrlFile != null) [ "--db-url-file" cfg.dbUrlFile ]
|
||||
++ lib.optionals (cfg.adminToken != null) [ "--admin-token" cfg.adminToken ]
|
||||
++ lib.optionals (cfg.stunServers != [ ]) [ "--stun" (lib.concatStringsSep "," cfg.stunServers) ]
|
||||
++ lib.optionals (cfg.turnServers != [ ]) [ "--turn" (lib.concatStringsSep "," cfg.turnServers) ]
|
||||
++ lib.optionals (cfg.streamRelayServers != [ ]) [ "--stream-relay" (lib.concatStringsSep "," cfg.streamRelayServers) ]
|
||||
++ lib.optionals (cfg.udpRelayServers != [ ]) [ "--udp-relay" (lib.concatStringsSep "," cfg.udpRelayServers) ]
|
||||
++ lib.optionals (cfg.udpRelayListen != null) [ "--udp-relay-listen" cfg.udpRelayListen ]
|
||||
++ lib.optionals (cfg.streamRelayListen != null) [ "--stream-relay-listen" cfg.streamRelayListen ]
|
||||
++ lib.optionals (cfg.meshServerId != null) [ "--mesh-server-id" cfg.meshServerId ]
|
||||
++ lib.optionals (cfg.meshListen != null) [ "--mesh-listen" cfg.meshListen ]
|
||||
++ lib.optionals (cfg.meshPeers != [ ]) [ "--mesh-peer" (lib.concatStringsSep "," cfg.meshPeers) ]
|
||||
++ lib.optionals (cfg.meshCaCert != null) [ "--mesh-ca-cert" cfg.meshCaCert ]
|
||||
++ lib.optionals (cfg.meshCert != null) [ "--mesh-cert" cfg.meshCert ]
|
||||
++ lib.optionals (cfg.meshKey != null) [ "--mesh-key" cfg.meshKey ]
|
||||
++ lib.optionals meshEnabled [ "--mesh-max-hops" (toString cfg.meshMaxHops) ]
|
||||
++ cfg.extraArgs;
|
||||
|
||||
startCmd = "${lib.getExe' cfg.package "lightscale-server"} ${lib.escapeShellArgs args}";
|
||||
in
|
||||
{
|
||||
options.services.lightscale-server = {
|
||||
enable = lib.mkEnableOption "lightscale control-plane server";
|
||||
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = defaultPackage;
|
||||
description = "lightscale-server package.";
|
||||
};
|
||||
|
||||
listen = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "0.0.0.0:8080";
|
||||
description = "Listen address for the control plane.";
|
||||
};
|
||||
|
||||
stateFile = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/var/lib/lightscale-server/state.json";
|
||||
description = "State file path when dbUrl is not set.";
|
||||
};
|
||||
|
||||
dbUrl = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Postgres/CockroachDB URL. If set, stateFile is ignored.";
|
||||
};
|
||||
|
||||
dbUrlFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Path to a file containing Postgres/CockroachDB URL. If set, stateFile is ignored.";
|
||||
};
|
||||
|
||||
adminToken = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Admin token passed via CLI. Prefer environmentFiles for secret handling.";
|
||||
};
|
||||
|
||||
environmentFiles = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "systemd EnvironmentFile entries (for LIGHTSCALE_ADMIN_TOKEN, etc.).";
|
||||
};
|
||||
|
||||
environment = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.str;
|
||||
default = { };
|
||||
description = "Additional environment variables for the server service.";
|
||||
};
|
||||
|
||||
stunServers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "STUN servers advertised to clients.";
|
||||
};
|
||||
|
||||
turnServers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "TURN servers advertised to clients.";
|
||||
};
|
||||
|
||||
streamRelayServers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Stream relay servers advertised to clients.";
|
||||
};
|
||||
|
||||
udpRelayServers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "UDP relay servers advertised to clients.";
|
||||
};
|
||||
|
||||
udpRelayListen = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "UDP relay listen address.";
|
||||
};
|
||||
|
||||
streamRelayListen = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Stream relay listen address.";
|
||||
};
|
||||
|
||||
meshServerId = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Server id used for authenticated relay mesh.";
|
||||
};
|
||||
|
||||
meshListen = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Relay mesh listen address (mTLS).";
|
||||
};
|
||||
|
||||
meshPeers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Relay mesh peers in ID=HOST:PORT form.";
|
||||
};
|
||||
|
||||
meshCaCert = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Path to relay mesh CA certificate PEM.";
|
||||
};
|
||||
|
||||
meshCert = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Path to this server's relay mesh certificate PEM.";
|
||||
};
|
||||
|
||||
meshKey = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Path to this server's relay mesh private key PEM.";
|
||||
};
|
||||
|
||||
meshMaxHops = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 4;
|
||||
description = "Maximum relay mesh forwarding hops.";
|
||||
};
|
||||
|
||||
extraArgs = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Additional CLI arguments for lightscale-server.";
|
||||
};
|
||||
|
||||
openFirewall = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Open firewall ports listed in firewallTCPPorts/firewallUDPPorts.";
|
||||
};
|
||||
|
||||
firewallTCPPorts = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.port;
|
||||
default = [ 8080 ];
|
||||
description = "TCP ports to open when openFirewall=true.";
|
||||
};
|
||||
|
||||
firewallUDPPorts = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.port;
|
||||
default = [ ];
|
||||
description = "UDP ports to open when openFirewall=true.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = !(cfg.dbUrl != null && cfg.dbUrlFile != null);
|
||||
message = "services.lightscale-server: set only one of dbUrl and dbUrlFile.";
|
||||
}
|
||||
{
|
||||
assertion = cfg.adminToken != null || cfg.environment ? LIGHTSCALE_ADMIN_TOKEN || cfg.environmentFiles != [ ];
|
||||
message = "services.lightscale-server: provide LIGHTSCALE_ADMIN_TOKEN via adminToken, environment, or environmentFiles.";
|
||||
}
|
||||
{
|
||||
assertion = !meshEnabled || (
|
||||
cfg.meshServerId != null
|
||||
&& cfg.meshListen != null
|
||||
&& cfg.meshCaCert != null
|
||||
&& cfg.meshCert != null
|
||||
&& cfg.meshKey != null
|
||||
);
|
||||
message = "services.lightscale-server: mesh requires meshServerId, meshListen, meshCaCert, meshCert, and meshKey.";
|
||||
}
|
||||
{
|
||||
assertion = !meshEnabled || (cfg.streamRelayListen != null || cfg.udpRelayListen != null);
|
||||
message = "services.lightscale-server: mesh forwarding requires streamRelayListen or udpRelayListen.";
|
||||
}
|
||||
];
|
||||
|
||||
users.groups.lightscale = { };
|
||||
users.users.lightscale = {
|
||||
isSystemUser = true;
|
||||
group = "lightscale";
|
||||
};
|
||||
|
||||
systemd.services.lightscale-server = {
|
||||
description = "lightscale control-plane server";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
environment = cfg.environment;
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
ExecStart = startCmd;
|
||||
Restart = "on-failure";
|
||||
RestartSec = 2;
|
||||
User = "lightscale";
|
||||
Group = "lightscale";
|
||||
WorkingDirectory = "/var/lib/lightscale-server";
|
||||
StateDirectory = "lightscale-server";
|
||||
RuntimeDirectory = "lightscale-server";
|
||||
EnvironmentFile = cfg.environmentFiles;
|
||||
NoNewPrivileges = true;
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall cfg.firewallTCPPorts;
|
||||
networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall cfg.firewallUDPPorts;
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue