Expand gateway matrix coverage and fix test-cluster routing

docs/component-matrix.md

@@ -34,7 +34,9 @@ This split keeps mutable VM volumes on CoronaFS and immutable VM images on Light
 ## Validated Edge And Tenant Services
 
 - `apigateway + iam + prismnet`
+- `nightlight + apigateway`
 - `nightlight`
+- `creditservice + iam + apigateway`
 - `creditservice + iam`
 - `deployer + iam + chainfire`
 
@@ -42,8 +44,6 @@ This split keeps mutable VM volumes on CoronaFS and immutable VM images on Light
 
 - `plasmavmc + prismnet`
 - `plasmavmc + prismnet + coronafs + lightningstor`
-- `nightlight + apigateway`
-- `creditservice + iam + apigateway`
 
 ## Validation Direction
 

nix/test-cluster/README.md

@@ -59,7 +59,7 @@ Preferred entrypoint for publishable verification: `nix run ./nix/test-cluster#c
 
 `make cluster-smoke` is a convenience wrapper for the same clean host-build VM validation flow.
 
-`nix run ./nix/test-cluster#cluster -- matrix` reuses the current running cluster to exercise composed service scenarios such as `prismnet + flashdns + fiberlb`, VM hosting with `plasmavmc + coronafs + lightningstor`, and the Kubernetes-style hosting bundle.
+`nix run ./nix/test-cluster#cluster -- matrix` reuses the current running cluster to exercise composed service scenarios such as `prismnet + flashdns + fiberlb`, VM hosting with `plasmavmc + coronafs + lightningstor`, the Kubernetes-style hosting bundle, and API-gateway-mediated `nightlight` / `creditservice` flows.
 
 Preferred entrypoint for publishable matrix verification: `nix run ./nix/test-cluster#cluster -- fresh-matrix`
 

nix/test-cluster/node01.nix

@@ -77,12 +77,14 @@
     enable = true;
     port = 50081;
     iamAddr = "10.100.0.11:50080";
+    chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
     flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
   };
 
   services.flashdns = {
     enable = true;
     iamAddr = "10.100.0.11:50080";
+    chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
     flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
   };
 

nix/test-cluster/node06.nix

@@ -31,7 +31,7 @@
     port = 8080;
     iamAddr = "10.100.0.11:50080";
     
-    # Configure routes to control plane services
+    # Configure routes to control plane and tenant-facing services
     routes = [
       {
         name = "iam-auth";
@@ -54,6 +54,18 @@
         upstream = "http://${config.plasmacloud.cluster.nodes.node01.ip}:8084";
         timeoutMs = 1200000;
       }
+      {
+        name = "nightlight-metrics";
+        pathPrefix = "/api/v1/metrics";
+        upstream = "http://127.0.0.1:9090/api/v1";
+        stripPrefix = true;
+      }
+      {
+        name = "creditservice-rest";
+        pathPrefix = "/api/v1/credits";
+        upstream = "http://127.0.0.1:3011/api/v1";
+        stripPrefix = true;
+      }
     ];
   };
 

nix/test-cluster/run-cluster.sh

@@ -407,6 +407,49 @@ grpcurl_capture() {
   return "${status}"
 }
 
+create_prismnet_vpc_with_retry() {
+  local token="$1"
+  local org_id="$2"
+  local project_id="$3"
+  local name="$4"
+  local description="$5"
+  local cidr_block="$6"
+  local timeout="${7:-${HTTP_WAIT_TIMEOUT}}"
+  local deadline=$((SECONDS + timeout))
+  local request_json output=""
+
+  wait_for_flaredb_route_metadata node01
+
+  request_json="$(
+    jq -cn \
+      --arg org "${org_id}" \
+      --arg project "${project_id}" \
+      --arg name "${name}" \
+      --arg description "${description}" \
+      --arg cidr "${cidr_block}" \
+      '{orgId:$org, projectId:$project, name:$name, description:$description, cidrBlock:$cidr}'
+  )"
+
+  while true; do
+    output="$(
+      grpcurl_capture -plaintext \
+        -H "authorization: Bearer ${token}" \
+        -import-path "${PRISMNET_PROTO_DIR}" \
+        -proto "${PRISMNET_PROTO}" \
+        -d "${request_json}" \
+        127.0.0.1:15081 prismnet.VpcService/CreateVpc
+    )" && {
+      printf '%s' "${output}"
+      return 0
+    }
+
+    if (( SECONDS >= deadline )); then
+      die "timed out waiting for PrismNet VPC writes to succeed: ${output}"
+    fi
+    sleep 2
+  done
+}
+
 build_link() {
   printf '%s/build-%s' "$(vm_dir)" "$1"
 }
@@ -2782,6 +2825,7 @@ for ip in "${nodes[@]}"; do
   done
 done
 EOS
+  wait_for_flaredb_route_metadata node01
 
   log "Validating FlareDB strong-consistency CAS on the control plane"
   local flaredb_proto_root="/var/lib/photon-test-protos/flaredb"
@@ -2888,13 +2932,13 @@ validate_prismnet_flow() {
   local vpc_resp subnet_resp port_resp
   local vpc_id subnet_id port_id
 
-  vpc_resp="$(grpcurl -plaintext \
+  vpc_resp="$(create_prismnet_vpc_with_retry \
-    -H "authorization: Bearer ${token}" \
+    "${token}" \
-    -import-path "${PRISMNET_PROTO_DIR}" \
+    "${org_id}" \
-    -proto "${PRISMNET_PROTO}" \
+    "${project_id}" \
-    -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg name "prismnet-smoke-vpc" \
+    "prismnet-smoke-vpc" \
-      '{orgId:$org, projectId:$project, name:$name, description:"smoke vpc", cidrBlock:"10.44.0.0/16"}')" \
+    "smoke vpc" \
-    127.0.0.1:15081 prismnet.VpcService/CreateVpc)"
+    "10.44.0.0/16")"
   vpc_id="$(printf '%s' "${vpc_resp}" | jq -r '.vpc.id')"
   [[ -n "${vpc_id}" && "${vpc_id}" != "null" ]] || die "PrismNet CreateVpc did not return a VPC ID"
 
@@ -4313,15 +4357,18 @@ validate_gateway() {
   stop_ssh_tunnel node01 "${iam_tunnel}"
 }
 
-validate_nightlight_flow() {
+validate_nightlight_flow_with_base() {
-  log "Validating NightLight remote_write ingestion and query endpoints"
+  local base_url="$1"
+  local flow_name="$2"
+
+  log "Validating ${flow_name} remote_write ingestion and query endpoints"
 
   local metric_name="nightlight_smoke_metric_$(date +%s)"
   local metric_value
   metric_value="$(awk 'BEGIN{srand(); printf "%.3f\n", (rand()*100)+1}')"
 
   python3 "${REPO_ROOT}/nix/test-cluster/nightlight_remote_write.py" \
-    --url http://127.0.0.1:9090/api/v1/write \
+    --url "${base_url}/write" \
     --metric "${metric_name}" \
     --value "${metric_value}" \
     --label source=smoke \
@@ -4329,7 +4376,7 @@ validate_nightlight_flow() {
 
   local deadline=$((SECONDS + HTTP_WAIT_TIMEOUT))
   while true; do
-    if curl -fsS --get http://127.0.0.1:9090/api/v1/query \
+    if curl -fsS --get "${base_url}/query" \
       --data-urlencode "query=${metric_name}{source=\"smoke\"}" \
       | jq -e --arg name "${metric_name}" --argjson expected "${metric_value}" '
         .status == "success"
@@ -4339,17 +4386,77 @@ validate_nightlight_flow() {
       break
     fi
     if (( SECONDS >= deadline )); then
-      die "timed out waiting for NightLight query result for ${metric_name}"
+      die "timed out waiting for ${flow_name} query result for ${metric_name}"
     fi
     sleep 2
   done
 
-  curl -fsS http://127.0.0.1:9090/api/v1/label/__name__/values \
+  curl -fsS "${base_url}/label/__name__/values" \
     | jq -e --arg name "${metric_name}" '.status == "success" and (.data | index($name)) != null' >/dev/null
-  curl -fsS http://127.0.0.1:9090/api/v1/series \
+  curl -fsS "${base_url}/series" \
     | jq -e --arg name "${metric_name}" '.status == "success" and (.data | any(.__name__ == $name))' >/dev/null
 }
 
+validate_nightlight_flow() {
+  validate_nightlight_flow_with_base "http://127.0.0.1:9090/api/v1" "NightLight"
+}
+
+validate_apigateway_nightlight_flow() {
+  log "Matrix case: NightLight + APIGateway"
+  wait_for_host_http http://127.0.0.1:8080/health
+  validate_nightlight_flow_with_base "http://127.0.0.1:8080/api/v1/metrics" "API Gateway -> NightLight"
+}
+
+validate_creditservice_rest_flow() {
+  local base_url="$1"
+  local token="$2"
+  local org_id="$3"
+  local project_id="$4"
+  local flow_name="$5"
+  local reservation_json reservation_id
+
+  log "Validating ${flow_name} REST wallet and reservation flows"
+
+  curl -fsS \
+    -X POST "${base_url}/wallets" \
+    -H "Authorization: Bearer ${token}" \
+    -H 'Content-Type: application/json' \
+    -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" '{org_id:$org, project_id:$project, initial_balance:1000}')" \
+    | jq -e '.data.project_id != null and .data.balance == 1000 and .data.available == 1000' >/dev/null
+
+  curl -fsS "${base_url}/wallets/${project_id}" \
+    -H "Authorization: Bearer ${token}" \
+    | jq -e --arg project "${project_id}" '.data.project_id == $project and .data.balance == 1000' >/dev/null
+
+  curl -fsS \
+    -X POST "${base_url}/wallets/${project_id}/topup" \
+    -H "Authorization: Bearer ${token}" \
+    -H 'Content-Type: application/json' \
+    -d '{"amount":250,"description":"smoke topup"}' \
+    | jq -e '.data.balance == 1250 and .data.total_deposited == 1250' >/dev/null
+
+  reservation_json="$(
+    curl -fsS \
+      -X POST "${base_url}/reservations" \
+      -H "Authorization: Bearer ${token}" \
+      -H 'Content-Type: application/json' \
+      -d "$(jq -cn --arg project "${project_id}" '{project_id:$project, amount:200, description:"smoke reservation", resource_type:"vm", ttl_seconds:120}')"
+  )"
+  reservation_id="$(printf '%s' "${reservation_json}" | jq -r '.data.id')"
+  [[ -n "${reservation_id}" && "${reservation_id}" != "null" ]] || die "${flow_name} reservation did not return an ID"
+
+  curl -fsS \
+    -X POST "${base_url}/reservations/${reservation_id}/commit" \
+    -H "Authorization: Bearer ${token}" \
+    -H 'Content-Type: application/json' \
+    -d '{"actual_amount":150,"resource_id":"smoke-vm"}' \
+    | jq -e '.data.balance == 1100 and .data.reserved == 0 and .data.available == 1100' >/dev/null
+
+  curl -fsS "${base_url}/wallets/${project_id}/transactions" \
+    -H "Authorization: Bearer ${token}" \
+    | jq -e '.data.transactions | length >= 3' >/dev/null
+}
+
 validate_creditservice_flow() {
   log "Validating CreditService REST and gRPC quota flows"
 
@@ -4364,47 +4471,15 @@ validate_creditservice_flow() {
   local org_id="credit-smoke-org-${suffix}"
   local project_id="credit-smoke-project-${suffix}"
   local principal_id="credit-smoke-$(date +%s)"
-  local token reservation_json reservation_id
+  local token
   token="$(issue_project_admin_token 15080 "${org_id}" "${project_id}" "${principal_id}")"
 
-  curl -fsS \
+  validate_creditservice_rest_flow \
-    -X POST http://127.0.0.1:13011/api/v1/wallets \
+    "http://127.0.0.1:13011/api/v1" \
-    -H "Authorization: Bearer ${token}" \
+    "${token}" \
-    -H 'Content-Type: application/json' \
+    "${org_id}" \
-    -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" '{org_id:$org, project_id:$project, initial_balance:1000}')" \
+    "${project_id}" \
-    | jq -e '.data.project_id != null and .data.balance == 1000 and .data.available == 1000' >/dev/null
+    "CreditService"
-
-  curl -fsS http://127.0.0.1:13011/api/v1/wallets/"${project_id}" \
-    -H "Authorization: Bearer ${token}" \
-    | jq -e --arg project "${project_id}" '.data.project_id == $project and .data.balance == 1000' >/dev/null
-
-  curl -fsS \
-    -X POST http://127.0.0.1:13011/api/v1/wallets/"${project_id}"/topup \
-    -H "Authorization: Bearer ${token}" \
-    -H 'Content-Type: application/json' \
-    -d '{"amount":250,"description":"smoke topup"}' \
-    | jq -e '.data.balance == 1250 and .data.total_deposited == 1250' >/dev/null
-
-  reservation_json="$(
-    curl -fsS \
-      -X POST http://127.0.0.1:13011/api/v1/reservations \
-      -H "Authorization: Bearer ${token}" \
-      -H 'Content-Type: application/json' \
-      -d "$(jq -cn --arg project "${project_id}" '{project_id:$project, amount:200, description:"smoke reservation", resource_type:"vm", ttl_seconds:120}')"
-  )"
-  reservation_id="$(printf '%s' "${reservation_json}" | jq -r '.data.id')"
-  [[ -n "${reservation_id}" && "${reservation_id}" != "null" ]] || die "CreditService reservation did not return an ID"
-
-  curl -fsS \
-    -X POST http://127.0.0.1:13011/api/v1/reservations/"${reservation_id}"/commit \
-    -H "Authorization: Bearer ${token}" \
-    -H 'Content-Type: application/json' \
-    -d '{"actual_amount":150,"resource_id":"smoke-vm"}' \
-    | jq -e '.data.balance == 1100 and .data.reserved == 0 and .data.available == 1100' >/dev/null
-
-  curl -fsS http://127.0.0.1:13011/api/v1/wallets/"${project_id}"/transactions \
-    -H "Authorization: Bearer ${token}" \
-    | jq -e '.data.transactions | length >= 3' >/dev/null
 
   grpcurl -plaintext \
     -H "authorization: Bearer ${token}" \
@@ -4450,6 +4525,34 @@ EOS
   stop_ssh_tunnel node01 "${iam_tunnel}"
 }
 
+validate_apigateway_creditservice_flow() {
+  log "Matrix case: CreditService + IAM + APIGateway"
+
+  local iam_tunnel=""
+  iam_tunnel="$(start_ssh_tunnel node01 15080 50080)"
+  trap 'stop_ssh_tunnel node01 "${iam_tunnel}"' RETURN
+
+  wait_for_host_http http://127.0.0.1:8080/health
+
+  local suffix
+  suffix="$(date +%s)"
+  local org_id="credit-gateway-org-${suffix}"
+  local project_id="credit-gateway-project-${suffix}"
+  local principal_id="credit-gateway-$(date +%s)"
+  local token
+  token="$(issue_project_admin_token 15080 "${org_id}" "${project_id}" "${principal_id}")"
+
+  validate_creditservice_rest_flow \
+    "http://127.0.0.1:8080/api/v1/credits" \
+    "${token}" \
+    "${org_id}" \
+    "${project_id}" \
+    "API Gateway -> CreditService"
+
+  trap - RETURN
+  stop_ssh_tunnel node01 "${iam_tunnel}"
+}
+
 validate_deployer_flow() {
   log "Validating Deployer health, admin registration, and phone-home flows"
 
@@ -5251,61 +5354,61 @@ validate_network_provider_matrix() {
   local workload_fqdn="" service_fqdn=""
 
   cleanup_network_provider_matrix() {
-    if [[ -n "${service_record_id}" ]]; then
+    if [[ -n "${service_record_id:-}" ]]; then
       grpcurl -plaintext -H "authorization: Bearer ${token}" \
         -import-path "${FLASHDNS_PROTO_DIR}" -proto "${FLASHDNS_PROTO}" \
         -d "$(jq -cn --arg id "${service_record_id}" '{id:$id}')" \
         127.0.0.1:15084 flashdns.v1.RecordService/DeleteRecord >/dev/null 2>&1 || true
     fi
-    if [[ -n "${workload_record_id}" ]]; then
+    if [[ -n "${workload_record_id:-}" ]]; then
       grpcurl -plaintext -H "authorization: Bearer ${token}" \
         -import-path "${FLASHDNS_PROTO_DIR}" -proto "${FLASHDNS_PROTO}" \
         -d "$(jq -cn --arg id "${workload_record_id}" '{id:$id}')" \
         127.0.0.1:15084 flashdns.v1.RecordService/DeleteRecord >/dev/null 2>&1 || true
     fi
-    if [[ -n "${listener_id}" ]]; then
+    if [[ -n "${listener_id:-}" ]]; then
       grpcurl -plaintext -H "authorization: Bearer ${token}" \
         -import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \
         -d "$(jq -cn --arg id "${listener_id}" '{id:$id}')" \
         127.0.0.1:15085 fiberlb.v1.ListenerService/DeleteListener >/dev/null 2>&1 || true
     fi
-    if [[ -n "${backend_id}" ]]; then
+    if [[ -n "${backend_id:-}" ]]; then
       grpcurl -plaintext -H "authorization: Bearer ${token}" \
         -import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \
         -d "$(jq -cn --arg id "${backend_id}" '{id:$id}')" \
         127.0.0.1:15085 fiberlb.v1.BackendService/DeleteBackend >/dev/null 2>&1 || true
     fi
-    if [[ -n "${pool_id}" ]]; then
+    if [[ -n "${pool_id:-}" ]]; then
       grpcurl -plaintext -H "authorization: Bearer ${token}" \
         -import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \
         -d "$(jq -cn --arg id "${pool_id}" '{id:$id}')" \
         127.0.0.1:15085 fiberlb.v1.PoolService/DeletePool >/dev/null 2>&1 || true
     fi
-    if [[ -n "${lb_id}" ]]; then
+    if [[ -n "${lb_id:-}" ]]; then
       grpcurl -plaintext -H "authorization: Bearer ${token}" \
         -import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \
         -d "$(jq -cn --arg id "${lb_id}" '{id:$id}')" \
         127.0.0.1:15085 fiberlb.v1.LoadBalancerService/DeleteLoadBalancer >/dev/null 2>&1 || true
     fi
-    if [[ -n "${port_id}" ]]; then
+    if [[ -n "${port_id:-}" ]]; then
       grpcurl -plaintext -H "authorization: Bearer ${token}" \
         -import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \
         -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg subnet "${subnet_id}" --arg id "${port_id}" '{orgId:$org, projectId:$project, subnetId:$subnet, id:$id}')" \
         127.0.0.1:15081 prismnet.PortService/DeletePort >/dev/null 2>&1 || true
     fi
-    if [[ -n "${subnet_id}" ]]; then
+    if [[ -n "${subnet_id:-}" ]]; then
       grpcurl -plaintext -H "authorization: Bearer ${token}" \
         -import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \
         -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg vpc "${vpc_id}" --arg id "${subnet_id}" '{orgId:$org, projectId:$project, vpcId:$vpc, id:$id}')" \
         127.0.0.1:15081 prismnet.SubnetService/DeleteSubnet >/dev/null 2>&1 || true
     fi
-    if [[ -n "${vpc_id}" ]]; then
+    if [[ -n "${vpc_id:-}" ]]; then
       grpcurl -plaintext -H "authorization: Bearer ${token}" \
         -import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \
         -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg id "${vpc_id}" '{orgId:$org, projectId:$project, id:$id}')" \
         127.0.0.1:15081 prismnet.VpcService/DeleteVpc >/dev/null 2>&1 || true
     fi
-    if [[ -n "${zone_id}" ]]; then
+    if [[ -n "${zone_id:-}" ]]; then
       grpcurl -plaintext -H "authorization: Bearer ${token}" \
         -import-path "${FLASHDNS_PROTO_DIR}" -proto "${FLASHDNS_PROTO}" \
         -d "$(jq -cn --arg id "${zone_id}" '{id:$id, force:true}')" \
@@ -5317,15 +5420,19 @@ validate_network_provider_matrix() {
     stop_ssh_tunnel node01 "${prism_tunnel}" >/dev/null 2>&1 || true
     stop_ssh_tunnel node01 "${iam_tunnel}" >/dev/null 2>&1 || true
   }
-  trap cleanup_network_provider_matrix RETURN EXIT
+  trap cleanup_network_provider_matrix RETURN
 
   token="$(issue_project_admin_token 15080 "${org_id}" "${project_id}" "${principal_id}")"
 
   log "Matrix case: PrismNet only"
-  vpc_id="$(grpcurl -plaintext -H "authorization: Bearer ${token}" \
+  vpc_id="$(create_prismnet_vpc_with_retry \
-    -import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \
+    "${token}" \
-    -d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" '{orgId:$org, projectId:$project, name:"matrix-vpc", description:"component matrix", cidrBlock:"10.52.0.0/16"}')" \
+    "${org_id}" \
-    127.0.0.1:15081 prismnet.VpcService/CreateVpc | jq -r '.vpc.id')"
+    "${project_id}" \
+    "matrix-vpc" \
+    "component matrix" \
+    "10.52.0.0/16" \
+    240 | jq -r '.vpc.id')"
   [[ -n "${vpc_id}" && "${vpc_id}" != "null" ]] || die "component matrix PrismNet VPC creation failed"
 
   subnet_id="$(grpcurl -plaintext -H "authorization: Bearer ${token}" \
@@ -5416,7 +5523,7 @@ validate_network_provider_matrix() {
 
   ssh_node node01 "curl -fsS --max-time 5 --resolve ${service_fqdn}:${listener_port}:10.100.0.11 http://${service_fqdn}:${listener_port}/health >/dev/null"
 
-  trap - RETURN EXIT
+  trap - RETURN
   cleanup_network_provider_matrix
 }
 
@@ -5428,7 +5535,9 @@ validate_component_matrix() {
   validate_k8shost_flow
   validate_gateway
   validate_nightlight_flow
+  validate_apigateway_nightlight_flow
   validate_creditservice_flow
+  validate_apigateway_creditservice_flow
   validate_deployer_flow
   validate_native_runtime_flow
   log "Component matrix validation succeeded"