591 lines
19 KiB
Nix
591 lines
19 KiB
Nix
{
|
|
pkgs,
|
|
photoncloudPackages,
|
|
photoncloudModule,
|
|
nixNosModule,
|
|
}:
|
|
|
|
let
|
|
mkGobgpdConfig =
|
|
routerId:
|
|
neighborAddress:
|
|
peerAs:
|
|
localAs:
|
|
pkgs.writeText "fiberlb-native-bgp-peer-${routerId}.json" (builtins.toJSON {
|
|
global = {
|
|
config = {
|
|
as = localAs;
|
|
router-id = routerId;
|
|
};
|
|
};
|
|
|
|
neighbors = [
|
|
{
|
|
config = {
|
|
neighbor-address = neighborAddress;
|
|
peer-as = peerAs;
|
|
description = "fiberlb-under-test";
|
|
};
|
|
}
|
|
];
|
|
});
|
|
|
|
routerAConfig = mkGobgpdConfig "192.168.100.1" "192.168.100.2" 65010 65020;
|
|
routerBConfig = mkGobgpdConfig "192.168.100.3" "192.168.100.2" 65010 65020;
|
|
|
|
iamProtoDir = ../../iam/proto;
|
|
iamProto = "iam.proto";
|
|
fiberlbProtoDir = ../../fiberlb/crates/fiberlb-api/proto;
|
|
fiberlbProto = "fiberlb.proto";
|
|
backendScript = pkgs.writeText "fiberlb-multipath-backend.py" ''
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
body = b"fiberlb multipath backend\n"
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def log_message(self, format, *args):
|
|
return
|
|
|
|
|
|
HTTPServer(("127.0.0.1", 18081), Handler).serve_forever()
|
|
'';
|
|
in
|
|
{
|
|
name = "fiberlb-native-bgp-multipath-vm-smoke";
|
|
|
|
nodes = {
|
|
router_a =
|
|
{ ... }:
|
|
{
|
|
networking.hostName = "router-a";
|
|
networking.useDHCP = false;
|
|
networking.firewall.enable = false;
|
|
virtualisation.vlans = [ 1 ];
|
|
networking.interfaces.eth1.ipv4.addresses = [
|
|
{
|
|
address = "192.168.100.1";
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
curl
|
|
gobgp
|
|
gobgpd
|
|
jq
|
|
];
|
|
|
|
systemd.services.gobgpd-peer = {
|
|
description = "GoBGP router-a test peer for FiberLB native BGP multi-peer smoke";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network.target" ];
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
ExecStart = "${pkgs.gobgpd}/bin/gobgpd -t json -f ${routerAConfig} --api-hosts 127.0.0.1:50051 -p";
|
|
Restart = "on-failure";
|
|
RestartSec = "2s";
|
|
};
|
|
};
|
|
|
|
system.stateVersion = "24.11";
|
|
};
|
|
|
|
router_b =
|
|
{ ... }:
|
|
{
|
|
networking.hostName = "router-b";
|
|
networking.useDHCP = false;
|
|
networking.firewall.enable = false;
|
|
virtualisation.vlans = [ 1 ];
|
|
networking.interfaces.eth1.ipv4.addresses = [
|
|
{
|
|
address = "192.168.100.3";
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
curl
|
|
gobgp
|
|
gobgpd
|
|
jq
|
|
];
|
|
|
|
systemd.services.gobgpd-peer = {
|
|
description = "GoBGP router-b test peer for FiberLB native BGP multi-peer smoke";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network.target" ];
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
ExecStart = "${pkgs.gobgpd}/bin/gobgpd -t json -f ${routerBConfig} --api-hosts 127.0.0.1:50051 -p";
|
|
Restart = "on-failure";
|
|
RestartSec = "2s";
|
|
};
|
|
};
|
|
|
|
system.stateVersion = "24.11";
|
|
};
|
|
|
|
lb =
|
|
{ ... }:
|
|
{
|
|
imports = [
|
|
nixNosModule
|
|
photoncloudModule
|
|
];
|
|
|
|
networking.hostName = "lb";
|
|
networking.useDHCP = false;
|
|
networking.firewall.enable = false;
|
|
virtualisation.vlans = [ 1 ];
|
|
networking.interfaces.eth1.ipv4.addresses = [
|
|
{
|
|
address = "192.168.100.2";
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
curl
|
|
grpcurl
|
|
jq
|
|
python3
|
|
];
|
|
|
|
services.iam = {
|
|
enable = true;
|
|
package = photoncloudPackages.iam-server;
|
|
port = 50080;
|
|
httpPort = 8083;
|
|
storeBackend = "memory";
|
|
};
|
|
|
|
systemd.services.iam.environment = {
|
|
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
|
};
|
|
|
|
services.fiberlb = {
|
|
enable = true;
|
|
package = photoncloudPackages.fiberlb-server;
|
|
port = 50085;
|
|
iamAddr = "192.168.100.2:50080";
|
|
metadataBackend = "sqlite";
|
|
databaseUrl = "sqlite:/var/lib/fiberlb/metadata.db";
|
|
singleNode = true;
|
|
healthCheckIntervalSecs = 1;
|
|
healthCheckTimeoutSecs = 1;
|
|
vipCheckIntervalSecs = 1;
|
|
vipOwnership = {
|
|
enable = true;
|
|
interface = "lo";
|
|
};
|
|
bgp = {
|
|
enable = true;
|
|
localAs = 65010;
|
|
routerId = "192.168.100.2";
|
|
nextHop = "192.168.100.2";
|
|
holdTimeSecs = 9;
|
|
keepaliveSecs = 3;
|
|
peers = [
|
|
{
|
|
address = "192.168.100.1";
|
|
port = 179;
|
|
asn = 65020;
|
|
description = "router-a";
|
|
}
|
|
{
|
|
address = "192.168.100.3";
|
|
port = 179;
|
|
asn = 65020;
|
|
description = "router-b";
|
|
}
|
|
];
|
|
};
|
|
};
|
|
|
|
systemd.services.mock-backend = {
|
|
description = "FiberLB health-check backend";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network.target" ];
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
ExecStart = "${pkgs.python3}/bin/python ${backendScript}";
|
|
Restart = "always";
|
|
RestartSec = "1s";
|
|
};
|
|
};
|
|
|
|
system.stateVersion = "24.11";
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
import json
|
|
import re
|
|
import shlex
|
|
import time
|
|
|
|
IAM_PROTO_DIR = "${iamProtoDir}"
|
|
IAM_PROTO = "${iamProto}"
|
|
FIBERLB_PROTO_DIR = "${fiberlbProtoDir}"
|
|
FIBERLB_PROTO = "${fiberlbProto}"
|
|
METRIC_RE = re.compile(r"^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+([-+0-9.eE]+)$")
|
|
|
|
def grpcurl_json(machine, endpoint, import_path, proto, service, payload, headers=None):
|
|
header_args = ""
|
|
for header in headers or []:
|
|
header_args += f" -H {shlex.quote(header)}"
|
|
command = (
|
|
f"grpcurl -plaintext{header_args} "
|
|
f"-import-path {shlex.quote(import_path)} "
|
|
f"-proto {shlex.quote(proto)} "
|
|
f"-d {shlex.quote(json.dumps(payload))} "
|
|
f"{shlex.quote(endpoint)} {shlex.quote(service)}"
|
|
)
|
|
status, output = machine.execute(f"timeout 15 sh -lc {shlex.quote(command + ' 2>&1')}")
|
|
if status != 0:
|
|
raise AssertionError(
|
|
"grpcurl failed"
|
|
f" service={service}"
|
|
f" status={status}"
|
|
f" payload={json.dumps(payload, sort_keys=True)}"
|
|
f" output={output}"
|
|
)
|
|
return json.loads(output)
|
|
|
|
def issue_project_admin_token(machine, org_id, project_id):
|
|
principal_id = f"fiberlb-multipath-{int(time.time())}"
|
|
deadline = time.time() + 120
|
|
|
|
def retry(action):
|
|
last_error = None
|
|
while time.time() < deadline:
|
|
try:
|
|
return action()
|
|
except Exception as exc:
|
|
last_error = exc
|
|
time.sleep(2)
|
|
raise AssertionError(f"IAM bootstrap timed out: {last_error}")
|
|
|
|
retry(lambda: grpcurl_json(
|
|
machine,
|
|
"127.0.0.1:50080",
|
|
IAM_PROTO_DIR,
|
|
IAM_PROTO,
|
|
"iam.v1.IamAdmin/CreatePrincipal",
|
|
{
|
|
"id": principal_id,
|
|
"kind": "PRINCIPAL_KIND_SERVICE_ACCOUNT",
|
|
"name": principal_id,
|
|
"orgId": org_id,
|
|
"projectId": project_id,
|
|
},
|
|
))
|
|
retry(lambda: grpcurl_json(
|
|
machine,
|
|
"127.0.0.1:50080",
|
|
IAM_PROTO_DIR,
|
|
IAM_PROTO,
|
|
"iam.v1.IamAdmin/CreateBinding",
|
|
{
|
|
"principal": {
|
|
"kind": "PRINCIPAL_KIND_SERVICE_ACCOUNT",
|
|
"id": principal_id,
|
|
},
|
|
"role": "roles/ProjectAdmin",
|
|
"scope": {
|
|
"project": {
|
|
"id": project_id,
|
|
"orgId": org_id,
|
|
}
|
|
},
|
|
},
|
|
))
|
|
token_response = retry(lambda: grpcurl_json(
|
|
machine,
|
|
"127.0.0.1:50080",
|
|
IAM_PROTO_DIR,
|
|
IAM_PROTO,
|
|
"iam.v1.IamToken/IssueToken",
|
|
{
|
|
"principalId": principal_id,
|
|
"principalKind": "PRINCIPAL_KIND_SERVICE_ACCOUNT",
|
|
"scope": {
|
|
"project": {
|
|
"id": project_id,
|
|
"orgId": org_id,
|
|
}
|
|
},
|
|
"ttlSeconds": 3600,
|
|
},
|
|
))
|
|
return token_response["token"]
|
|
|
|
def wait_for_backend_status(status, backend_id, token):
|
|
lb.wait_until_succeeds(
|
|
"grpcurl -plaintext "
|
|
f"-H {shlex.quote('authorization: Bearer ' + token)} "
|
|
f"-import-path {shlex.quote(FIBERLB_PROTO_DIR)} "
|
|
f"-proto {shlex.quote(FIBERLB_PROTO)} "
|
|
f"-d {shlex.quote(json.dumps({'id': backend_id}))} "
|
|
"127.0.0.1:50085 fiberlb.v1.BackendService/GetBackend "
|
|
f"| jq -e {shlex.quote(f'.backend.status == \"{status}\"')}"
|
|
)
|
|
|
|
def wait_for_route(machine, prefix, present):
|
|
command = "gobgp -u 127.0.0.1 -p 50051 global rib || true"
|
|
if present:
|
|
machine.wait_until_succeeds(f"{command} | grep -F {shlex.quote(prefix)}")
|
|
else:
|
|
deadline = time.time() + 60
|
|
while time.time() < deadline:
|
|
output = machine.succeed(command)
|
|
if prefix not in output:
|
|
return
|
|
time.sleep(1)
|
|
raise AssertionError(f"route {prefix} still present in GoBGP RIB")
|
|
|
|
def wait_for_local_vip(vip, present):
|
|
pattern = f"inet {vip}/32"
|
|
if present:
|
|
lb.wait_until_succeeds(
|
|
f"ip -4 addr show dev lo | grep -F {shlex.quote(pattern)}"
|
|
)
|
|
else:
|
|
deadline = time.time() + 60
|
|
while time.time() < deadline:
|
|
output = lb.succeed("ip -4 addr show dev lo || true")
|
|
if pattern not in output:
|
|
return
|
|
time.sleep(1)
|
|
raise AssertionError(f"VIP {vip} still present on loopback")
|
|
|
|
def wait_for_http_success(machine, url):
|
|
machine.wait_until_succeeds(
|
|
f"curl -fsS --max-time 5 {shlex.quote(url)} | grep -F 'fiberlb multipath backend'"
|
|
)
|
|
|
|
def parse_labels(label_blob):
|
|
if not label_blob:
|
|
return {}
|
|
labels = {}
|
|
for part in label_blob.split(","):
|
|
key, value = part.split("=", 1)
|
|
labels[key] = value.strip().strip('"')
|
|
return labels
|
|
|
|
def wait_for_metric(metric_name, expected_value, labels=None):
|
|
expected_labels = labels or {}
|
|
deadline = time.time() + 60
|
|
|
|
while time.time() < deadline:
|
|
exposition = lb.succeed("curl -fsS http://127.0.0.1:9098/metrics")
|
|
for line in exposition.splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
match = METRIC_RE.match(line)
|
|
if not match:
|
|
continue
|
|
name, label_blob, value = match.groups()
|
|
if name != metric_name:
|
|
continue
|
|
if parse_labels(label_blob) != expected_labels:
|
|
continue
|
|
if abs(float(value) - float(expected_value)) < 0.0001:
|
|
return
|
|
time.sleep(1)
|
|
|
|
raise AssertionError(
|
|
f"metric {metric_name} with labels={expected_labels} did not reach {expected_value}"
|
|
)
|
|
|
|
start_all()
|
|
serial_stdout_off()
|
|
|
|
router_a.wait_for_unit("gobgpd-peer.service")
|
|
router_a.wait_until_succeeds("ss -ltnH '( sport = :179 )' | grep -q LISTEN")
|
|
router_b.wait_for_unit("gobgpd-peer.service")
|
|
router_b.wait_until_succeeds("ss -ltnH '( sport = :179 )' | grep -q LISTEN")
|
|
lb.wait_for_unit("iam.service")
|
|
lb.wait_until_succeeds("ss -ltnH '( sport = :50080 )' | grep -q LISTEN")
|
|
lb.wait_for_unit("mock-backend.service")
|
|
lb.wait_for_unit("fiberlb.service")
|
|
lb.wait_until_succeeds("ss -ltnH '( sport = :50085 )' | grep -q LISTEN")
|
|
lb.wait_until_succeeds("ss -ltnH '( sport = :9098 )' | grep -q LISTEN")
|
|
|
|
router_a.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2")
|
|
router_b.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2")
|
|
wait_for_metric("fiberlb_bgp_configured_peers", 2)
|
|
wait_for_metric("fiberlb_bgp_connected_peers", 2)
|
|
wait_for_metric("fiberlb_bgp_peer_session_up", 1, {"peer": "192.168.100.1:179"})
|
|
wait_for_metric("fiberlb_bgp_peer_session_up", 1, {"peer": "192.168.100.3:179"})
|
|
|
|
token = issue_project_admin_token(lb, "bgp-multipath-org", "bgp-multipath-project")
|
|
|
|
lb_response = grpcurl_json(
|
|
lb,
|
|
"127.0.0.1:50085",
|
|
FIBERLB_PROTO_DIR,
|
|
FIBERLB_PROTO,
|
|
"fiberlb.v1.LoadBalancerService/CreateLoadBalancer",
|
|
{
|
|
"name": "bgp-multipath-lb",
|
|
"orgId": "bgp-multipath-org",
|
|
"projectId": "bgp-multipath-project",
|
|
"description": "native bgp multipath smoke",
|
|
},
|
|
headers=[f"authorization: Bearer {token}"],
|
|
)
|
|
loadbalancer = lb_response["loadbalancer"]
|
|
lb_id = loadbalancer["id"]
|
|
vip = loadbalancer["vipAddress"]
|
|
vip_prefix = f"{vip}/32"
|
|
listener_url = f"http://{vip}:18080/"
|
|
|
|
pool_id = grpcurl_json(
|
|
lb,
|
|
"127.0.0.1:50085",
|
|
FIBERLB_PROTO_DIR,
|
|
FIBERLB_PROTO,
|
|
"fiberlb.v1.PoolService/CreatePool",
|
|
{
|
|
"name": "bgp-multipath-pool",
|
|
"loadbalancerId": lb_id,
|
|
"algorithm": "POOL_ALGORITHM_ROUND_ROBIN",
|
|
"protocol": "POOL_PROTOCOL_TCP",
|
|
},
|
|
headers=[f"authorization: Bearer {token}"],
|
|
)["pool"]["id"]
|
|
|
|
backend_id = grpcurl_json(
|
|
lb,
|
|
"127.0.0.1:50085",
|
|
FIBERLB_PROTO_DIR,
|
|
FIBERLB_PROTO,
|
|
"fiberlb.v1.BackendService/CreateBackend",
|
|
{
|
|
"name": "bgp-multipath-backend",
|
|
"poolId": pool_id,
|
|
"address": "127.0.0.1",
|
|
"port": 18081,
|
|
"weight": 1,
|
|
},
|
|
headers=[f"authorization: Bearer {token}"],
|
|
)["backend"]["id"]
|
|
|
|
grpcurl_json(
|
|
lb,
|
|
"127.0.0.1:50085",
|
|
FIBERLB_PROTO_DIR,
|
|
FIBERLB_PROTO,
|
|
"fiberlb.v1.HealthCheckService/CreateHealthCheck",
|
|
{
|
|
"name": "bgp-multipath-health",
|
|
"poolId": pool_id,
|
|
"type": "HEALTH_CHECK_TYPE_HTTP",
|
|
"intervalSeconds": 1,
|
|
"timeoutSeconds": 1,
|
|
"healthyThreshold": 1,
|
|
"unhealthyThreshold": 1,
|
|
"httpConfig": {
|
|
"method": "GET",
|
|
"path": "/",
|
|
"expectedCodes": [200],
|
|
},
|
|
},
|
|
headers=[f"authorization: Bearer {token}"],
|
|
)
|
|
|
|
grpcurl_json(
|
|
lb,
|
|
"127.0.0.1:50085",
|
|
FIBERLB_PROTO_DIR,
|
|
FIBERLB_PROTO,
|
|
"fiberlb.v1.ListenerService/CreateListener",
|
|
{
|
|
"name": "bgp-multipath-listener",
|
|
"loadbalancerId": lb_id,
|
|
"protocol": "LISTENER_PROTOCOL_TCP",
|
|
"port": 18080,
|
|
"defaultPoolId": pool_id,
|
|
},
|
|
headers=[f"authorization: Bearer {token}"],
|
|
)
|
|
|
|
wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
|
|
wait_for_local_vip(vip, True)
|
|
wait_for_metric("fiberlb_bgp_desired_routes", 1)
|
|
wait_for_route(router_a, vip_prefix, True)
|
|
wait_for_route(router_b, vip_prefix, True)
|
|
router_a.succeed(f"ip route replace {shlex.quote(vip_prefix)} via 192.168.100.2 dev eth1")
|
|
router_b.succeed(f"ip route replace {shlex.quote(vip_prefix)} via 192.168.100.2 dev eth1")
|
|
wait_for_http_success(router_a, listener_url)
|
|
wait_for_http_success(router_b, listener_url)
|
|
|
|
router_a.succeed("systemctl stop gobgpd-peer.service")
|
|
wait_for_metric("fiberlb_bgp_connected_peers", 1)
|
|
wait_for_metric("fiberlb_bgp_peer_session_up", 0, {"peer": "192.168.100.1:179"})
|
|
wait_for_metric("fiberlb_bgp_peer_session_up", 1, {"peer": "192.168.100.3:179"})
|
|
wait_for_metric("fiberlb_bgp_desired_routes", 1)
|
|
wait_for_route(router_b, vip_prefix, True)
|
|
wait_for_http_success(router_b, listener_url)
|
|
|
|
router_a.succeed("systemctl start gobgpd-peer.service")
|
|
router_a.wait_for_unit("gobgpd-peer.service")
|
|
router_a.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2")
|
|
wait_for_metric("fiberlb_bgp_connected_peers", 2)
|
|
wait_for_metric("fiberlb_bgp_peer_session_up", 1, {"peer": "192.168.100.1:179"})
|
|
wait_for_route(router_a, vip_prefix, True)
|
|
wait_for_http_success(router_a, listener_url)
|
|
|
|
router_b.succeed("systemctl stop gobgpd-peer.service")
|
|
wait_for_metric("fiberlb_bgp_connected_peers", 1)
|
|
wait_for_metric("fiberlb_bgp_peer_session_up", 0, {"peer": "192.168.100.3:179"})
|
|
wait_for_metric("fiberlb_bgp_peer_session_up", 1, {"peer": "192.168.100.1:179"})
|
|
wait_for_metric("fiberlb_bgp_desired_routes", 1)
|
|
wait_for_route(router_a, vip_prefix, True)
|
|
wait_for_http_success(router_a, listener_url)
|
|
|
|
router_b.succeed("systemctl start gobgpd-peer.service")
|
|
router_b.wait_for_unit("gobgpd-peer.service")
|
|
router_b.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2")
|
|
wait_for_metric("fiberlb_bgp_connected_peers", 2)
|
|
wait_for_metric("fiberlb_bgp_peer_session_up", 1, {"peer": "192.168.100.3:179"})
|
|
wait_for_route(router_b, vip_prefix, True)
|
|
wait_for_http_success(router_b, listener_url)
|
|
|
|
lb.succeed("systemctl stop mock-backend.service")
|
|
wait_for_backend_status("BACKEND_STATUS_OFFLINE", backend_id, token)
|
|
wait_for_metric("fiberlb_bgp_desired_routes", 0)
|
|
wait_for_route(router_a, vip_prefix, False)
|
|
wait_for_route(router_b, vip_prefix, False)
|
|
wait_for_local_vip(vip, False)
|
|
router_a.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")
|
|
router_b.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")
|
|
|
|
lb.succeed("systemctl start mock-backend.service")
|
|
lb.wait_for_unit("mock-backend.service")
|
|
wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
|
|
wait_for_metric("fiberlb_bgp_desired_routes", 1)
|
|
wait_for_local_vip(vip, True)
|
|
wait_for_route(router_a, vip_prefix, True)
|
|
wait_for_route(router_b, vip_prefix, True)
|
|
wait_for_http_success(router_a, listener_url)
|
|
wait_for_http_success(router_b, listener_url)
|
|
|
|
lb.succeed("systemctl stop fiberlb.service")
|
|
wait_for_local_vip(vip, False)
|
|
wait_for_route(router_a, vip_prefix, False)
|
|
wait_for_route(router_b, vip_prefix, False)
|
|
router_a.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")
|
|
router_b.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")
|
|
'';
|
|
}
|