photoncloud-monorepo/nix/tests/fiberlb-native-bgp-vm-smoke.nix

{
  pkgs,
  photoncloudPackages,
  photoncloudModule,
  nixNosModule,
}:

let
  gobgpdConfig = pkgs.writeText "fiberlb-native-bgp-peer.json" (builtins.toJSON {
    global = {
      config = {
        as = 65020;
        router-id = "192.168.100.1";
      };
    };

    neighbors = [
      {
        config = {
          neighbor-address = "192.168.100.2";
          peer-as = 65010;
          description = "fiberlb-under-test";
        };
      }
    ];
  });

  iamProtoDir = ../../iam/proto;
  iamProto = "iam.proto";
  fiberlbProtoDir = ../../fiberlb/crates/fiberlb-api/proto;
  fiberlbProto = "fiberlb.proto";
  backendScript = pkgs.writeText "fiberlb-smoke-backend.py" ''
    from http.server import BaseHTTPRequestHandler, HTTPServer


    class Handler(BaseHTTPRequestHandler):
        def do_GET(self):
            body = b"fiberlb smoke backend\n"
            self.send_response(200)
            self.send_header("Content-Type", "text/plain; charset=utf-8")
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)

        def log_message(self, format, *args):
            return


    HTTPServer(("127.0.0.1", 18081), Handler).serve_forever()
  '';
in
{
  name = "fiberlb-native-bgp-vm-smoke";

  nodes = {
    router =
      { ... }:
      {
        networking.hostName = "router";
        networking.useDHCP = false;
        networking.firewall.enable = false;
        virtualisation.vlans = [ 1 ];
        networking.interfaces.eth1.ipv4.addresses = [
          {
            address = "192.168.100.1";
            prefixLength = 24;
          }
        ];

        environment.systemPackages = with pkgs; [
          curl
          gobgp
          gobgpd
          jq
        ];

        systemd.services.gobgpd-peer = {
          description = "GoBGP test peer for FiberLB native BGP smoke";
          wantedBy = [ "multi-user.target" ];
          after = [ "network.target" ];
          serviceConfig = {
            Type = "simple";
            ExecStart = "${pkgs.gobgpd}/bin/gobgpd -t json -f ${gobgpdConfig} --api-hosts 127.0.0.1:50051 -p";
            Restart = "on-failure";
            RestartSec = "2s";
          };
        };

        system.stateVersion = "24.11";
      };

    lb =
      { ... }:
      {
        imports = [
          nixNosModule
          photoncloudModule
        ];

        networking.hostName = "lb";
        networking.useDHCP = false;
        networking.firewall.enable = false;
        virtualisation.vlans = [ 1 ];
        networking.interfaces.eth1.ipv4.addresses = [
          {
            address = "192.168.100.2";
            prefixLength = 24;
          }
        ];

        environment.systemPackages = with pkgs; [
          grpcurl
          jq
          python3
        ];

        services.iam = {
          enable = true;
          package = photoncloudPackages.iam-server;
          port = 50080;
          httpPort = 8083;
          storeBackend = "memory";
        };

        systemd.services.iam.environment = {
          IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
        };

        services.fiberlb = {
          enable = true;
          package = photoncloudPackages.fiberlb-server;
          port = 50085;
          iamAddr = "192.168.100.2:50080";
          metadataBackend = "sqlite";
          databaseUrl = "sqlite:/var/lib/fiberlb/metadata.db";
          singleNode = true;
          healthCheckIntervalSecs = 1;
          healthCheckTimeoutSecs = 1;
          vipCheckIntervalSecs = 1;
          vipOwnership = {
            enable = true;
            interface = "lo";
          };
          bgp = {
            enable = true;
            localAs = 65010;
            routerId = "192.168.100.2";
            nextHop = "192.168.100.2";
            holdTimeSecs = 9;
            keepaliveSecs = 3;
            peers = [
              {
                address = "192.168.100.1";
                port = 179;
                asn = 65020;
                description = "router-peer";
              }
            ];
          };
        };

        systemd.services.mock-backend = {
          description = "FiberLB health-check backend";
          wantedBy = [ "multi-user.target" ];
          after = [ "network.target" ];
          serviceConfig = {
            Type = "simple";
            ExecStart = "${pkgs.python3}/bin/python ${backendScript}";
            Restart = "always";
            RestartSec = "1s";
          };
        };

        system.stateVersion = "24.11";
      };
  };

  testScript = ''
    import json
    import shlex
    import time

    IAM_PROTO_DIR = "${iamProtoDir}"
    IAM_PROTO = "${iamProto}"
    FIBERLB_PROTO_DIR = "${fiberlbProtoDir}"
    FIBERLB_PROTO = "${fiberlbProto}"

    def grpcurl_json(machine, endpoint, import_path, proto, service, payload, headers=None):
        header_args = ""
        for header in headers or []:
            header_args += f" -H {shlex.quote(header)}"
        command = (
            f"grpcurl -plaintext{header_args} "
            f"-import-path {shlex.quote(import_path)} "
            f"-proto {shlex.quote(proto)} "
            f"-d {shlex.quote(json.dumps(payload))} "
            f"{shlex.quote(endpoint)} {shlex.quote(service)}"
        )
        status, output = machine.execute(f"timeout 15 sh -lc {shlex.quote(command + ' 2>&1')}")
        if status != 0:
            raise AssertionError(
                "grpcurl failed"
                f" service={service}"
                f" status={status}"
                f" payload={json.dumps(payload, sort_keys=True)}"
                f" output={output}"
            )
        return json.loads(output)

    def issue_project_admin_token(machine, org_id, project_id):
        principal_id = f"fiberlb-smoke-{int(time.time())}"
        deadline = time.time() + 120

        def retry(action):
            last_error = None
            while time.time() < deadline:
                try:
                    return action()
                except Exception as exc:
                    last_error = exc
                    time.sleep(2)
            raise AssertionError(f"IAM bootstrap timed out: {last_error}")

        retry(lambda: grpcurl_json(
            machine,
            "127.0.0.1:50080",
            IAM_PROTO_DIR,
            IAM_PROTO,
            "iam.v1.IamAdmin/CreatePrincipal",
            {
                "id": principal_id,
                "kind": "PRINCIPAL_KIND_SERVICE_ACCOUNT",
                "name": principal_id,
                "orgId": org_id,
                "projectId": project_id,
            },
        ))
        retry(lambda: grpcurl_json(
            machine,
            "127.0.0.1:50080",
            IAM_PROTO_DIR,
            IAM_PROTO,
            "iam.v1.IamAdmin/CreateBinding",
            {
                "principal": {
                    "kind": "PRINCIPAL_KIND_SERVICE_ACCOUNT",
                    "id": principal_id,
                },
                "role": "roles/ProjectAdmin",
                "scope": {
                    "project": {
                        "id": project_id,
                        "orgId": org_id,
                    }
                },
            },
        ))
        token_response = retry(lambda: grpcurl_json(
            machine,
            "127.0.0.1:50080",
            IAM_PROTO_DIR,
            IAM_PROTO,
            "iam.v1.IamToken/IssueToken",
            {
                "principalId": principal_id,
                "principalKind": "PRINCIPAL_KIND_SERVICE_ACCOUNT",
                "scope": {
                    "project": {
                        "id": project_id,
                        "orgId": org_id,
                    }
                },
                "ttlSeconds": 3600,
            },
        ))
        return token_response["token"]

    def wait_for_backend_status(status, backend_id, token):
        lb.wait_until_succeeds(
            "grpcurl -plaintext "
            f"-H {shlex.quote('authorization: Bearer ' + token)} "
            f"-import-path {shlex.quote(FIBERLB_PROTO_DIR)} "
            f"-proto {shlex.quote(FIBERLB_PROTO)} "
            f"-d {shlex.quote(json.dumps({'id': backend_id}))} "
            "127.0.0.1:50085 fiberlb.v1.BackendService/GetBackend "
            f"| jq -e {shlex.quote(f'.backend.status == \"{status}\"')}"
        )

    def wait_for_route(prefix, present):
        if present:
            router.wait_until_succeeds(
                f"gobgp -u 127.0.0.1 -p 50051 global rib | grep -F {shlex.quote(prefix)}"
            )
        else:
            deadline = time.time() + 60
            while time.time() < deadline:
                output = router.succeed("gobgp -u 127.0.0.1 -p 50051 global rib || true")
                if prefix not in output:
                    return
                time.sleep(1)
            raise AssertionError(f"route {prefix} still present in GoBGP RIB")

    def wait_for_local_vip(vip, present):
        pattern = f"inet {vip}/32"
        if present:
            lb.wait_until_succeeds(
                f"ip -4 addr show dev lo | grep -F {shlex.quote(pattern)}"
            )
        else:
            deadline = time.time() + 60
            while time.time() < deadline:
                output = lb.succeed("ip -4 addr show dev lo || true")
                if pattern not in output:
                    return
                time.sleep(1)
            raise AssertionError(f"VIP {vip} still present on loopback")

    def wait_for_http_success(url):
        router.wait_until_succeeds(
            f"curl -fsS --max-time 5 {shlex.quote(url)} | grep -F 'fiberlb smoke backend'"
        )

    start_all()
    serial_stdout_off()

    router.wait_for_unit("gobgpd-peer.service")
    router.wait_until_succeeds("ss -ltnH '( sport = :179 )' | grep -q LISTEN")
    lb.wait_for_unit("iam.service")
    lb.wait_until_succeeds("ss -ltnH '( sport = :50080 )' | grep -q LISTEN")
    lb.wait_for_unit("mock-backend.service")
    lb.wait_for_unit("fiberlb.service")
    lb.wait_until_succeeds("ss -ltnH '( sport = :50085 )' | grep -q LISTEN")

    router.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2")

    token = issue_project_admin_token(lb, "bgp-smoke-org", "bgp-smoke-project")

    lb_response = grpcurl_json(
        lb,
        "127.0.0.1:50085",
        FIBERLB_PROTO_DIR,
        FIBERLB_PROTO,
        "fiberlb.v1.LoadBalancerService/CreateLoadBalancer",
        {
            "name": "bgp-smoke-lb",
            "orgId": "bgp-smoke-org",
            "projectId": "bgp-smoke-project",
            "description": "native bgp smoke",
        },
        headers=[f"authorization: Bearer {token}"],
    )
    loadbalancer = lb_response["loadbalancer"]
    lb_id = loadbalancer["id"]
    vip = loadbalancer["vipAddress"]
    vip_prefix = f"{vip}/32"
    listener_url = f"http://{vip}:18080/"

    pool_id = grpcurl_json(
        lb,
        "127.0.0.1:50085",
        FIBERLB_PROTO_DIR,
        FIBERLB_PROTO,
        "fiberlb.v1.PoolService/CreatePool",
        {
            "name": "bgp-smoke-pool",
            "loadbalancerId": lb_id,
            "algorithm": "POOL_ALGORITHM_ROUND_ROBIN",
            "protocol": "POOL_PROTOCOL_TCP",
        },
        headers=[f"authorization: Bearer {token}"],
    )["pool"]["id"]

    backend_id = grpcurl_json(
        lb,
        "127.0.0.1:50085",
        FIBERLB_PROTO_DIR,
        FIBERLB_PROTO,
        "fiberlb.v1.BackendService/CreateBackend",
        {
            "name": "bgp-smoke-backend",
            "poolId": pool_id,
            "address": "127.0.0.1",
            "port": 18081,
            "weight": 1,
        },
        headers=[f"authorization: Bearer {token}"],
    )["backend"]["id"]

    grpcurl_json(
        lb,
        "127.0.0.1:50085",
        FIBERLB_PROTO_DIR,
        FIBERLB_PROTO,
        "fiberlb.v1.HealthCheckService/CreateHealthCheck",
        {
            "name": "bgp-smoke-health",
            "poolId": pool_id,
            "type": "HEALTH_CHECK_TYPE_HTTP",
            "intervalSeconds": 1,
            "timeoutSeconds": 1,
            "healthyThreshold": 1,
            "unhealthyThreshold": 1,
            "httpConfig": {
                "method": "GET",
                "path": "/",
                "expectedCodes": [200],
            },
        },
        headers=[f"authorization: Bearer {token}"],
    )

    grpcurl_json(
        lb,
        "127.0.0.1:50085",
        FIBERLB_PROTO_DIR,
        FIBERLB_PROTO,
        "fiberlb.v1.ListenerService/CreateListener",
        {
            "name": "bgp-smoke-listener",
            "loadbalancerId": lb_id,
            "protocol": "LISTENER_PROTOCOL_TCP",
            "port": 18080,
            "defaultPoolId": pool_id,
        },
        headers=[f"authorization: Bearer {token}"],
    )

    wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
    wait_for_local_vip(vip, True)
    wait_for_route(vip_prefix, True)
    router.succeed(f"ip route replace {shlex.quote(vip_prefix)} via 192.168.100.2 dev eth1")
    wait_for_http_success(listener_url)

    router.succeed("systemctl restart gobgpd-peer.service")
    router.wait_for_unit("gobgpd-peer.service")
    router.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2")
    wait_for_route(vip_prefix, True)
    wait_for_http_success(listener_url)

    lb.succeed("systemctl stop mock-backend.service")
    wait_for_backend_status("BACKEND_STATUS_OFFLINE", backend_id, token)
    wait_for_route(vip_prefix, False)
    wait_for_local_vip(vip, False)
    router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")

    lb.succeed("systemctl start mock-backend.service")
    lb.wait_for_unit("mock-backend.service")
    wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
    wait_for_local_vip(vip, True)
    wait_for_route(vip_prefix, True)
    wait_for_http_success(listener_url)

    lb.succeed("systemctl stop fiberlb.service")
    wait_for_local_vip(vip, False)
    wait_for_route(vip_prefix, False)
    router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")
  '';
}