{ pkgs, photoncloudPackages, photoncloudModule, nixNosModule, }: let gobgpdConfig = pkgs.writeText "fiberlb-native-bgp-peer.json" (builtins.toJSON { global = { config = { as = 65020; router-id = "192.168.100.1"; }; }; neighbors = [ { config = { neighbor-address = "192.168.100.2"; peer-as = 65010; description = "fiberlb-under-test"; }; } ]; }); iamProtoDir = ../../iam/proto; iamProto = "iam.proto"; fiberlbProtoDir = ../../fiberlb/crates/fiberlb-api/proto; fiberlbProto = "fiberlb.proto"; backendScript = pkgs.writeText "fiberlb-smoke-backend.py" '' from http.server import BaseHTTPRequestHandler, HTTPServer class Handler(BaseHTTPRequestHandler): def do_GET(self): body = b"fiberlb smoke backend\n" self.send_response(200) self.send_header("Content-Type", "text/plain; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, format, *args): return HTTPServer(("127.0.0.1", 18081), Handler).serve_forever() ''; in { name = "fiberlb-native-bgp-vm-smoke"; nodes = { router = { ... }: { networking.hostName = "router"; networking.useDHCP = false; networking.firewall.enable = false; virtualisation.vlans = [ 1 ]; networking.interfaces.eth1.ipv4.addresses = [ { address = "192.168.100.1"; prefixLength = 24; } ]; environment.systemPackages = with pkgs; [ curl gobgp gobgpd jq ]; systemd.services.gobgpd-peer = { description = "GoBGP test peer for FiberLB native BGP smoke"; wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; serviceConfig = { Type = "simple"; ExecStart = "${pkgs.gobgpd}/bin/gobgpd -t json -f ${gobgpdConfig} --api-hosts 127.0.0.1:50051 -p"; Restart = "on-failure"; RestartSec = "2s"; }; }; system.stateVersion = "24.11"; }; lb = { ... }: { imports = [ nixNosModule photoncloudModule ]; networking.hostName = "lb"; networking.useDHCP = false; networking.firewall.enable = false; virtualisation.vlans = [ 1 ]; networking.interfaces.eth1.ipv4.addresses = [ { address = "192.168.100.2"; prefixLength = 24; } ]; environment.systemPackages = with pkgs; [ grpcurl jq python3 ]; services.iam = { enable = true; package = photoncloudPackages.iam-server; port = 50080; httpPort = 8083; storeBackend = "memory"; }; systemd.services.iam.environment = { IAM_ALLOW_RANDOM_SIGNING_KEY = "1"; }; services.fiberlb = { enable = true; package = photoncloudPackages.fiberlb-server; port = 50085; iamAddr = "192.168.100.2:50080"; metadataBackend = "sqlite"; databaseUrl = "sqlite:/var/lib/fiberlb/metadata.db"; singleNode = true; healthCheckIntervalSecs = 1; healthCheckTimeoutSecs = 1; vipCheckIntervalSecs = 1; vipOwnership = { enable = true; interface = "lo"; }; bgp = { enable = true; localAs = 65010; routerId = "192.168.100.2"; nextHop = "192.168.100.2"; holdTimeSecs = 9; keepaliveSecs = 3; peers = [ { address = "192.168.100.1"; port = 179; asn = 65020; description = "router-peer"; } ]; }; }; systemd.services.mock-backend = { description = "FiberLB health-check backend"; wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; serviceConfig = { Type = "simple"; ExecStart = "${pkgs.python3}/bin/python ${backendScript}"; Restart = "always"; RestartSec = "1s"; }; }; system.stateVersion = "24.11"; }; }; testScript = '' import json import shlex import time IAM_PROTO_DIR = "${iamProtoDir}" IAM_PROTO = "${iamProto}" FIBERLB_PROTO_DIR = "${fiberlbProtoDir}" FIBERLB_PROTO = "${fiberlbProto}" def grpcurl_json(machine, endpoint, import_path, proto, service, payload, headers=None): header_args = "" for header in headers or []: header_args += f" -H {shlex.quote(header)}" command = ( f"grpcurl -plaintext{header_args} " f"-import-path {shlex.quote(import_path)} " f"-proto {shlex.quote(proto)} " f"-d {shlex.quote(json.dumps(payload))} " f"{shlex.quote(endpoint)} {shlex.quote(service)}" ) status, output = machine.execute(f"timeout 15 sh -lc {shlex.quote(command + ' 2>&1')}") if status != 0: raise AssertionError( "grpcurl failed" f" service={service}" f" status={status}" f" payload={json.dumps(payload, sort_keys=True)}" f" output={output}" ) return json.loads(output) def issue_project_admin_token(machine, org_id, project_id): principal_id = f"fiberlb-smoke-{int(time.time())}" deadline = time.time() + 120 def retry(action): last_error = None while time.time() < deadline: try: return action() except Exception as exc: last_error = exc time.sleep(2) raise AssertionError(f"IAM bootstrap timed out: {last_error}") retry(lambda: grpcurl_json( machine, "127.0.0.1:50080", IAM_PROTO_DIR, IAM_PROTO, "iam.v1.IamAdmin/CreatePrincipal", { "id": principal_id, "kind": "PRINCIPAL_KIND_SERVICE_ACCOUNT", "name": principal_id, "orgId": org_id, "projectId": project_id, }, )) retry(lambda: grpcurl_json( machine, "127.0.0.1:50080", IAM_PROTO_DIR, IAM_PROTO, "iam.v1.IamAdmin/CreateBinding", { "principal": { "kind": "PRINCIPAL_KIND_SERVICE_ACCOUNT", "id": principal_id, }, "role": "roles/ProjectAdmin", "scope": { "project": { "id": project_id, "orgId": org_id, } }, }, )) token_response = retry(lambda: grpcurl_json( machine, "127.0.0.1:50080", IAM_PROTO_DIR, IAM_PROTO, "iam.v1.IamToken/IssueToken", { "principalId": principal_id, "principalKind": "PRINCIPAL_KIND_SERVICE_ACCOUNT", "scope": { "project": { "id": project_id, "orgId": org_id, } }, "ttlSeconds": 3600, }, )) return token_response["token"] def wait_for_backend_status(status, backend_id, token): lb.wait_until_succeeds( "grpcurl -plaintext " f"-H {shlex.quote('authorization: Bearer ' + token)} " f"-import-path {shlex.quote(FIBERLB_PROTO_DIR)} " f"-proto {shlex.quote(FIBERLB_PROTO)} " f"-d {shlex.quote(json.dumps({'id': backend_id}))} " "127.0.0.1:50085 fiberlb.v1.BackendService/GetBackend " f"| jq -e {shlex.quote(f'.backend.status == \"{status}\"')}" ) def wait_for_route(prefix, present): if present: router.wait_until_succeeds( f"gobgp -u 127.0.0.1 -p 50051 global rib | grep -F {shlex.quote(prefix)}" ) else: deadline = time.time() + 60 while time.time() < deadline: output = router.succeed("gobgp -u 127.0.0.1 -p 50051 global rib || true") if prefix not in output: return time.sleep(1) raise AssertionError(f"route {prefix} still present in GoBGP RIB") def wait_for_local_vip(vip, present): pattern = f"inet {vip}/32" if present: lb.wait_until_succeeds( f"ip -4 addr show dev lo | grep -F {shlex.quote(pattern)}" ) else: deadline = time.time() + 60 while time.time() < deadline: output = lb.succeed("ip -4 addr show dev lo || true") if pattern not in output: return time.sleep(1) raise AssertionError(f"VIP {vip} still present on loopback") def wait_for_http_success(url): router.wait_until_succeeds( f"curl -fsS --max-time 5 {shlex.quote(url)} | grep -F 'fiberlb smoke backend'" ) start_all() serial_stdout_off() router.wait_for_unit("gobgpd-peer.service") router.wait_until_succeeds("ss -ltnH '( sport = :179 )' | grep -q LISTEN") lb.wait_for_unit("iam.service") lb.wait_until_succeeds("ss -ltnH '( sport = :50080 )' | grep -q LISTEN") lb.wait_for_unit("mock-backend.service") lb.wait_for_unit("fiberlb.service") lb.wait_until_succeeds("ss -ltnH '( sport = :50085 )' | grep -q LISTEN") router.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2") token = issue_project_admin_token(lb, "bgp-smoke-org", "bgp-smoke-project") lb_response = grpcurl_json( lb, "127.0.0.1:50085", FIBERLB_PROTO_DIR, FIBERLB_PROTO, "fiberlb.v1.LoadBalancerService/CreateLoadBalancer", { "name": "bgp-smoke-lb", "orgId": "bgp-smoke-org", "projectId": "bgp-smoke-project", "description": "native bgp smoke", }, headers=[f"authorization: Bearer {token}"], ) loadbalancer = lb_response["loadbalancer"] lb_id = loadbalancer["id"] vip = loadbalancer["vipAddress"] vip_prefix = f"{vip}/32" listener_url = f"http://{vip}:18080/" pool_id = grpcurl_json( lb, "127.0.0.1:50085", FIBERLB_PROTO_DIR, FIBERLB_PROTO, "fiberlb.v1.PoolService/CreatePool", { "name": "bgp-smoke-pool", "loadbalancerId": lb_id, "algorithm": "POOL_ALGORITHM_ROUND_ROBIN", "protocol": "POOL_PROTOCOL_TCP", }, headers=[f"authorization: Bearer {token}"], )["pool"]["id"] backend_id = grpcurl_json( lb, "127.0.0.1:50085", FIBERLB_PROTO_DIR, FIBERLB_PROTO, "fiberlb.v1.BackendService/CreateBackend", { "name": "bgp-smoke-backend", "poolId": pool_id, "address": "127.0.0.1", "port": 18081, "weight": 1, }, headers=[f"authorization: Bearer {token}"], )["backend"]["id"] grpcurl_json( lb, "127.0.0.1:50085", FIBERLB_PROTO_DIR, FIBERLB_PROTO, "fiberlb.v1.HealthCheckService/CreateHealthCheck", { "name": "bgp-smoke-health", "poolId": pool_id, "type": "HEALTH_CHECK_TYPE_HTTP", "intervalSeconds": 1, "timeoutSeconds": 1, "healthyThreshold": 1, "unhealthyThreshold": 1, "httpConfig": { "method": "GET", "path": "/", "expectedCodes": [200], }, }, headers=[f"authorization: Bearer {token}"], ) grpcurl_json( lb, "127.0.0.1:50085", FIBERLB_PROTO_DIR, FIBERLB_PROTO, "fiberlb.v1.ListenerService/CreateListener", { "name": "bgp-smoke-listener", "loadbalancerId": lb_id, "protocol": "LISTENER_PROTOCOL_TCP", "port": 18080, "defaultPoolId": pool_id, }, headers=[f"authorization: Bearer {token}"], ) wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token) wait_for_local_vip(vip, True) wait_for_route(vip_prefix, True) router.succeed(f"ip route replace {shlex.quote(vip_prefix)} via 192.168.100.2 dev eth1") wait_for_http_success(listener_url) router.succeed("systemctl restart gobgpd-peer.service") router.wait_for_unit("gobgpd-peer.service") router.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2") wait_for_route(vip_prefix, True) wait_for_http_success(listener_url) lb.succeed("systemctl stop mock-backend.service") wait_for_backend_status("BACKEND_STATUS_OFFLINE", backend_id, token) wait_for_route(vip_prefix, False) wait_for_local_vip(vip, False) router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}") lb.succeed("systemctl start mock-backend.service") lb.wait_for_unit("mock-backend.service") wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token) wait_for_local_vip(vip, True) wait_for_route(vip_prefix, True) wait_for_http_success(listener_url) lb.succeed("systemctl stop fiberlb.service") wait_for_local_vip(vip, False) wait_for_route(vip_prefix, False) router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}") ''; }