From 290c6ba88a2c65434fed3b09e48e4028be9b75c2 Mon Sep 17 00:00:00 2001 From: centra Date: Fri, 27 Mar 2026 22:51:37 +0900 Subject: [PATCH] Expand gateway matrix coverage and fix test-cluster routing --- docs/component-matrix.md | 4 +- nix/test-cluster/README.md | 2 +- nix/test-cluster/node01.nix | 2 + nix/test-cluster/node06.nix | 14 +- nix/test-cluster/run-cluster.sh | 247 +++++++++++++++++++++++--------- 5 files changed, 196 insertions(+), 73 deletions(-) diff --git a/docs/component-matrix.md b/docs/component-matrix.md index 3927d0b..7c2338a 100644 --- a/docs/component-matrix.md +++ b/docs/component-matrix.md @@ -34,7 +34,9 @@ This split keeps mutable VM volumes on CoronaFS and immutable VM images on Light ## Validated Edge And Tenant Services - `apigateway + iam + prismnet` +- `nightlight + apigateway` - `nightlight` +- `creditservice + iam + apigateway` - `creditservice + iam` - `deployer + iam + chainfire` @@ -42,8 +44,6 @@ This split keeps mutable VM volumes on CoronaFS and immutable VM images on Light - `plasmavmc + prismnet` - `plasmavmc + prismnet + coronafs + lightningstor` -- `nightlight + apigateway` -- `creditservice + iam + apigateway` ## Validation Direction diff --git a/nix/test-cluster/README.md b/nix/test-cluster/README.md index c494a44..c210b3f 100644 --- a/nix/test-cluster/README.md +++ b/nix/test-cluster/README.md @@ -59,7 +59,7 @@ Preferred entrypoint for publishable verification: `nix run ./nix/test-cluster#c `make cluster-smoke` is a convenience wrapper for the same clean host-build VM validation flow. -`nix run ./nix/test-cluster#cluster -- matrix` reuses the current running cluster to exercise composed service scenarios such as `prismnet + flashdns + fiberlb`, VM hosting with `plasmavmc + coronafs + lightningstor`, and the Kubernetes-style hosting bundle. +`nix run ./nix/test-cluster#cluster -- matrix` reuses the current running cluster to exercise composed service scenarios such as `prismnet + flashdns + fiberlb`, VM hosting with `plasmavmc + coronafs + lightningstor`, the Kubernetes-style hosting bundle, and API-gateway-mediated `nightlight` / `creditservice` flows. Preferred entrypoint for publishable matrix verification: `nix run ./nix/test-cluster#cluster -- fresh-matrix` diff --git a/nix/test-cluster/node01.nix b/nix/test-cluster/node01.nix index ff69c89..b00e225 100644 --- a/nix/test-cluster/node01.nix +++ b/nix/test-cluster/node01.nix @@ -77,12 +77,14 @@ enable = true; port = 50081; iamAddr = "10.100.0.11:50080"; + chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs; flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs; }; services.flashdns = { enable = true; iamAddr = "10.100.0.11:50080"; + chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs; flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs; }; diff --git a/nix/test-cluster/node06.nix b/nix/test-cluster/node06.nix index 5e75bbc..f160b64 100644 --- a/nix/test-cluster/node06.nix +++ b/nix/test-cluster/node06.nix @@ -31,7 +31,7 @@ port = 8080; iamAddr = "10.100.0.11:50080"; - # Configure routes to control plane services + # Configure routes to control plane and tenant-facing services routes = [ { name = "iam-auth"; @@ -54,6 +54,18 @@ upstream = "http://${config.plasmacloud.cluster.nodes.node01.ip}:8084"; timeoutMs = 1200000; } + { + name = "nightlight-metrics"; + pathPrefix = "/api/v1/metrics"; + upstream = "http://127.0.0.1:9090/api/v1"; + stripPrefix = true; + } + { + name = "creditservice-rest"; + pathPrefix = "/api/v1/credits"; + upstream = "http://127.0.0.1:3011/api/v1"; + stripPrefix = true; + } ]; }; diff --git a/nix/test-cluster/run-cluster.sh b/nix/test-cluster/run-cluster.sh index 7ae6a4f..ee390ff 100755 --- a/nix/test-cluster/run-cluster.sh +++ b/nix/test-cluster/run-cluster.sh @@ -407,6 +407,49 @@ grpcurl_capture() { return "${status}" } +create_prismnet_vpc_with_retry() { + local token="$1" + local org_id="$2" + local project_id="$3" + local name="$4" + local description="$5" + local cidr_block="$6" + local timeout="${7:-${HTTP_WAIT_TIMEOUT}}" + local deadline=$((SECONDS + timeout)) + local request_json output="" + + wait_for_flaredb_route_metadata node01 + + request_json="$( + jq -cn \ + --arg org "${org_id}" \ + --arg project "${project_id}" \ + --arg name "${name}" \ + --arg description "${description}" \ + --arg cidr "${cidr_block}" \ + '{orgId:$org, projectId:$project, name:$name, description:$description, cidrBlock:$cidr}' + )" + + while true; do + output="$( + grpcurl_capture -plaintext \ + -H "authorization: Bearer ${token}" \ + -import-path "${PRISMNET_PROTO_DIR}" \ + -proto "${PRISMNET_PROTO}" \ + -d "${request_json}" \ + 127.0.0.1:15081 prismnet.VpcService/CreateVpc + )" && { + printf '%s' "${output}" + return 0 + } + + if (( SECONDS >= deadline )); then + die "timed out waiting for PrismNet VPC writes to succeed: ${output}" + fi + sleep 2 + done +} + build_link() { printf '%s/build-%s' "$(vm_dir)" "$1" } @@ -2782,6 +2825,7 @@ for ip in "${nodes[@]}"; do done done EOS + wait_for_flaredb_route_metadata node01 log "Validating FlareDB strong-consistency CAS on the control plane" local flaredb_proto_root="/var/lib/photon-test-protos/flaredb" @@ -2888,13 +2932,13 @@ validate_prismnet_flow() { local vpc_resp subnet_resp port_resp local vpc_id subnet_id port_id - vpc_resp="$(grpcurl -plaintext \ - -H "authorization: Bearer ${token}" \ - -import-path "${PRISMNET_PROTO_DIR}" \ - -proto "${PRISMNET_PROTO}" \ - -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg name "prismnet-smoke-vpc" \ - '{orgId:$org, projectId:$project, name:$name, description:"smoke vpc", cidrBlock:"10.44.0.0/16"}')" \ - 127.0.0.1:15081 prismnet.VpcService/CreateVpc)" + vpc_resp="$(create_prismnet_vpc_with_retry \ + "${token}" \ + "${org_id}" \ + "${project_id}" \ + "prismnet-smoke-vpc" \ + "smoke vpc" \ + "10.44.0.0/16")" vpc_id="$(printf '%s' "${vpc_resp}" | jq -r '.vpc.id')" [[ -n "${vpc_id}" && "${vpc_id}" != "null" ]] || die "PrismNet CreateVpc did not return a VPC ID" @@ -4313,15 +4357,18 @@ validate_gateway() { stop_ssh_tunnel node01 "${iam_tunnel}" } -validate_nightlight_flow() { - log "Validating NightLight remote_write ingestion and query endpoints" +validate_nightlight_flow_with_base() { + local base_url="$1" + local flow_name="$2" + + log "Validating ${flow_name} remote_write ingestion and query endpoints" local metric_name="nightlight_smoke_metric_$(date +%s)" local metric_value metric_value="$(awk 'BEGIN{srand(); printf "%.3f\n", (rand()*100)+1}')" python3 "${REPO_ROOT}/nix/test-cluster/nightlight_remote_write.py" \ - --url http://127.0.0.1:9090/api/v1/write \ + --url "${base_url}/write" \ --metric "${metric_name}" \ --value "${metric_value}" \ --label source=smoke \ @@ -4329,7 +4376,7 @@ validate_nightlight_flow() { local deadline=$((SECONDS + HTTP_WAIT_TIMEOUT)) while true; do - if curl -fsS --get http://127.0.0.1:9090/api/v1/query \ + if curl -fsS --get "${base_url}/query" \ --data-urlencode "query=${metric_name}{source=\"smoke\"}" \ | jq -e --arg name "${metric_name}" --argjson expected "${metric_value}" ' .status == "success" @@ -4339,17 +4386,77 @@ validate_nightlight_flow() { break fi if (( SECONDS >= deadline )); then - die "timed out waiting for NightLight query result for ${metric_name}" + die "timed out waiting for ${flow_name} query result for ${metric_name}" fi sleep 2 done - curl -fsS http://127.0.0.1:9090/api/v1/label/__name__/values \ + curl -fsS "${base_url}/label/__name__/values" \ | jq -e --arg name "${metric_name}" '.status == "success" and (.data | index($name)) != null' >/dev/null - curl -fsS http://127.0.0.1:9090/api/v1/series \ + curl -fsS "${base_url}/series" \ | jq -e --arg name "${metric_name}" '.status == "success" and (.data | any(.__name__ == $name))' >/dev/null } +validate_nightlight_flow() { + validate_nightlight_flow_with_base "http://127.0.0.1:9090/api/v1" "NightLight" +} + +validate_apigateway_nightlight_flow() { + log "Matrix case: NightLight + APIGateway" + wait_for_host_http http://127.0.0.1:8080/health + validate_nightlight_flow_with_base "http://127.0.0.1:8080/api/v1/metrics" "API Gateway -> NightLight" +} + +validate_creditservice_rest_flow() { + local base_url="$1" + local token="$2" + local org_id="$3" + local project_id="$4" + local flow_name="$5" + local reservation_json reservation_id + + log "Validating ${flow_name} REST wallet and reservation flows" + + curl -fsS \ + -X POST "${base_url}/wallets" \ + -H "Authorization: Bearer ${token}" \ + -H 'Content-Type: application/json' \ + -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" '{org_id:$org, project_id:$project, initial_balance:1000}')" \ + | jq -e '.data.project_id != null and .data.balance == 1000 and .data.available == 1000' >/dev/null + + curl -fsS "${base_url}/wallets/${project_id}" \ + -H "Authorization: Bearer ${token}" \ + | jq -e --arg project "${project_id}" '.data.project_id == $project and .data.balance == 1000' >/dev/null + + curl -fsS \ + -X POST "${base_url}/wallets/${project_id}/topup" \ + -H "Authorization: Bearer ${token}" \ + -H 'Content-Type: application/json' \ + -d '{"amount":250,"description":"smoke topup"}' \ + | jq -e '.data.balance == 1250 and .data.total_deposited == 1250' >/dev/null + + reservation_json="$( + curl -fsS \ + -X POST "${base_url}/reservations" \ + -H "Authorization: Bearer ${token}" \ + -H 'Content-Type: application/json' \ + -d "$(jq -cn --arg project "${project_id}" '{project_id:$project, amount:200, description:"smoke reservation", resource_type:"vm", ttl_seconds:120}')" + )" + reservation_id="$(printf '%s' "${reservation_json}" | jq -r '.data.id')" + [[ -n "${reservation_id}" && "${reservation_id}" != "null" ]] || die "${flow_name} reservation did not return an ID" + + curl -fsS \ + -X POST "${base_url}/reservations/${reservation_id}/commit" \ + -H "Authorization: Bearer ${token}" \ + -H 'Content-Type: application/json' \ + -d '{"actual_amount":150,"resource_id":"smoke-vm"}' \ + | jq -e '.data.balance == 1100 and .data.reserved == 0 and .data.available == 1100' >/dev/null + + curl -fsS "${base_url}/wallets/${project_id}/transactions" \ + -H "Authorization: Bearer ${token}" \ + | jq -e '.data.transactions | length >= 3' >/dev/null +} + validate_creditservice_flow() { log "Validating CreditService REST and gRPC quota flows" @@ -4364,47 +4471,15 @@ validate_creditservice_flow() { local org_id="credit-smoke-org-${suffix}" local project_id="credit-smoke-project-${suffix}" local principal_id="credit-smoke-$(date +%s)" - local token reservation_json reservation_id + local token token="$(issue_project_admin_token 15080 "${org_id}" "${project_id}" "${principal_id}")" - curl -fsS \ - -X POST http://127.0.0.1:13011/api/v1/wallets \ - -H "Authorization: Bearer ${token}" \ - -H 'Content-Type: application/json' \ - -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" '{org_id:$org, project_id:$project, initial_balance:1000}')" \ - | jq -e '.data.project_id != null and .data.balance == 1000 and .data.available == 1000' >/dev/null - - curl -fsS http://127.0.0.1:13011/api/v1/wallets/"${project_id}" \ - -H "Authorization: Bearer ${token}" \ - | jq -e --arg project "${project_id}" '.data.project_id == $project and .data.balance == 1000' >/dev/null - - curl -fsS \ - -X POST http://127.0.0.1:13011/api/v1/wallets/"${project_id}"/topup \ - -H "Authorization: Bearer ${token}" \ - -H 'Content-Type: application/json' \ - -d '{"amount":250,"description":"smoke topup"}' \ - | jq -e '.data.balance == 1250 and .data.total_deposited == 1250' >/dev/null - - reservation_json="$( - curl -fsS \ - -X POST http://127.0.0.1:13011/api/v1/reservations \ - -H "Authorization: Bearer ${token}" \ - -H 'Content-Type: application/json' \ - -d "$(jq -cn --arg project "${project_id}" '{project_id:$project, amount:200, description:"smoke reservation", resource_type:"vm", ttl_seconds:120}')" - )" - reservation_id="$(printf '%s' "${reservation_json}" | jq -r '.data.id')" - [[ -n "${reservation_id}" && "${reservation_id}" != "null" ]] || die "CreditService reservation did not return an ID" - - curl -fsS \ - -X POST http://127.0.0.1:13011/api/v1/reservations/"${reservation_id}"/commit \ - -H "Authorization: Bearer ${token}" \ - -H 'Content-Type: application/json' \ - -d '{"actual_amount":150,"resource_id":"smoke-vm"}' \ - | jq -e '.data.balance == 1100 and .data.reserved == 0 and .data.available == 1100' >/dev/null - - curl -fsS http://127.0.0.1:13011/api/v1/wallets/"${project_id}"/transactions \ - -H "Authorization: Bearer ${token}" \ - | jq -e '.data.transactions | length >= 3' >/dev/null + validate_creditservice_rest_flow \ + "http://127.0.0.1:13011/api/v1" \ + "${token}" \ + "${org_id}" \ + "${project_id}" \ + "CreditService" grpcurl -plaintext \ -H "authorization: Bearer ${token}" \ @@ -4450,6 +4525,34 @@ EOS stop_ssh_tunnel node01 "${iam_tunnel}" } +validate_apigateway_creditservice_flow() { + log "Matrix case: CreditService + IAM + APIGateway" + + local iam_tunnel="" + iam_tunnel="$(start_ssh_tunnel node01 15080 50080)" + trap 'stop_ssh_tunnel node01 "${iam_tunnel}"' RETURN + + wait_for_host_http http://127.0.0.1:8080/health + + local suffix + suffix="$(date +%s)" + local org_id="credit-gateway-org-${suffix}" + local project_id="credit-gateway-project-${suffix}" + local principal_id="credit-gateway-$(date +%s)" + local token + token="$(issue_project_admin_token 15080 "${org_id}" "${project_id}" "${principal_id}")" + + validate_creditservice_rest_flow \ + "http://127.0.0.1:8080/api/v1/credits" \ + "${token}" \ + "${org_id}" \ + "${project_id}" \ + "API Gateway -> CreditService" + + trap - RETURN + stop_ssh_tunnel node01 "${iam_tunnel}" +} + validate_deployer_flow() { log "Validating Deployer health, admin registration, and phone-home flows" @@ -5251,61 +5354,61 @@ validate_network_provider_matrix() { local workload_fqdn="" service_fqdn="" cleanup_network_provider_matrix() { - if [[ -n "${service_record_id}" ]]; then + if [[ -n "${service_record_id:-}" ]]; then grpcurl -plaintext -H "authorization: Bearer ${token}" \ -import-path "${FLASHDNS_PROTO_DIR}" -proto "${FLASHDNS_PROTO}" \ -d "$(jq -cn --arg id "${service_record_id}" '{id:$id}')" \ 127.0.0.1:15084 flashdns.v1.RecordService/DeleteRecord >/dev/null 2>&1 || true fi - if [[ -n "${workload_record_id}" ]]; then + if [[ -n "${workload_record_id:-}" ]]; then grpcurl -plaintext -H "authorization: Bearer ${token}" \ -import-path "${FLASHDNS_PROTO_DIR}" -proto "${FLASHDNS_PROTO}" \ -d "$(jq -cn --arg id "${workload_record_id}" '{id:$id}')" \ 127.0.0.1:15084 flashdns.v1.RecordService/DeleteRecord >/dev/null 2>&1 || true fi - if [[ -n "${listener_id}" ]]; then + if [[ -n "${listener_id:-}" ]]; then grpcurl -plaintext -H "authorization: Bearer ${token}" \ -import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \ -d "$(jq -cn --arg id "${listener_id}" '{id:$id}')" \ 127.0.0.1:15085 fiberlb.v1.ListenerService/DeleteListener >/dev/null 2>&1 || true fi - if [[ -n "${backend_id}" ]]; then + if [[ -n "${backend_id:-}" ]]; then grpcurl -plaintext -H "authorization: Bearer ${token}" \ -import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \ -d "$(jq -cn --arg id "${backend_id}" '{id:$id}')" \ 127.0.0.1:15085 fiberlb.v1.BackendService/DeleteBackend >/dev/null 2>&1 || true fi - if [[ -n "${pool_id}" ]]; then + if [[ -n "${pool_id:-}" ]]; then grpcurl -plaintext -H "authorization: Bearer ${token}" \ -import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \ -d "$(jq -cn --arg id "${pool_id}" '{id:$id}')" \ 127.0.0.1:15085 fiberlb.v1.PoolService/DeletePool >/dev/null 2>&1 || true fi - if [[ -n "${lb_id}" ]]; then + if [[ -n "${lb_id:-}" ]]; then grpcurl -plaintext -H "authorization: Bearer ${token}" \ -import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \ -d "$(jq -cn --arg id "${lb_id}" '{id:$id}')" \ 127.0.0.1:15085 fiberlb.v1.LoadBalancerService/DeleteLoadBalancer >/dev/null 2>&1 || true fi - if [[ -n "${port_id}" ]]; then + if [[ -n "${port_id:-}" ]]; then grpcurl -plaintext -H "authorization: Bearer ${token}" \ -import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \ -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg subnet "${subnet_id}" --arg id "${port_id}" '{orgId:$org, projectId:$project, subnetId:$subnet, id:$id}')" \ 127.0.0.1:15081 prismnet.PortService/DeletePort >/dev/null 2>&1 || true fi - if [[ -n "${subnet_id}" ]]; then + if [[ -n "${subnet_id:-}" ]]; then grpcurl -plaintext -H "authorization: Bearer ${token}" \ -import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \ -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg vpc "${vpc_id}" --arg id "${subnet_id}" '{orgId:$org, projectId:$project, vpcId:$vpc, id:$id}')" \ 127.0.0.1:15081 prismnet.SubnetService/DeleteSubnet >/dev/null 2>&1 || true fi - if [[ -n "${vpc_id}" ]]; then + if [[ -n "${vpc_id:-}" ]]; then grpcurl -plaintext -H "authorization: Bearer ${token}" \ -import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \ -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg id "${vpc_id}" '{orgId:$org, projectId:$project, id:$id}')" \ 127.0.0.1:15081 prismnet.VpcService/DeleteVpc >/dev/null 2>&1 || true fi - if [[ -n "${zone_id}" ]]; then + if [[ -n "${zone_id:-}" ]]; then grpcurl -plaintext -H "authorization: Bearer ${token}" \ -import-path "${FLASHDNS_PROTO_DIR}" -proto "${FLASHDNS_PROTO}" \ -d "$(jq -cn --arg id "${zone_id}" '{id:$id, force:true}')" \ @@ -5317,15 +5420,19 @@ validate_network_provider_matrix() { stop_ssh_tunnel node01 "${prism_tunnel}" >/dev/null 2>&1 || true stop_ssh_tunnel node01 "${iam_tunnel}" >/dev/null 2>&1 || true } - trap cleanup_network_provider_matrix RETURN EXIT + trap cleanup_network_provider_matrix RETURN token="$(issue_project_admin_token 15080 "${org_id}" "${project_id}" "${principal_id}")" log "Matrix case: PrismNet only" - vpc_id="$(grpcurl -plaintext -H "authorization: Bearer ${token}" \ - -import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \ - -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" '{orgId:$org, projectId:$project, name:"matrix-vpc", description:"component matrix", cidrBlock:"10.52.0.0/16"}')" \ - 127.0.0.1:15081 prismnet.VpcService/CreateVpc | jq -r '.vpc.id')" + vpc_id="$(create_prismnet_vpc_with_retry \ + "${token}" \ + "${org_id}" \ + "${project_id}" \ + "matrix-vpc" \ + "component matrix" \ + "10.52.0.0/16" \ + 240 | jq -r '.vpc.id')" [[ -n "${vpc_id}" && "${vpc_id}" != "null" ]] || die "component matrix PrismNet VPC creation failed" subnet_id="$(grpcurl -plaintext -H "authorization: Bearer ${token}" \ @@ -5416,7 +5523,7 @@ validate_network_provider_matrix() { ssh_node node01 "curl -fsS --max-time 5 --resolve ${service_fqdn}:${listener_port}:10.100.0.11 http://${service_fqdn}:${listener_port}/health >/dev/null" - trap - RETURN EXIT + trap - RETURN cleanup_network_provider_matrix } @@ -5428,7 +5535,9 @@ validate_component_matrix() { validate_k8shost_flow validate_gateway validate_nightlight_flow + validate_apigateway_nightlight_flow validate_creditservice_flow + validate_apigateway_creditservice_flow validate_deployer_flow validate_native_runtime_flow log "Component matrix validation succeeded"