centra/photoncloud-monorepo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Expand gateway matrix coverage and fix test-cluster routing

Browse source
This commit is contained in:
centra 2026-03-27 22:51:37 +09:00
parent 6fa172eab1
commit 290c6ba88a
Signed by: centra
GPG key ID: 0C09689D20B25ACA
5 changed files with 196 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docs/component-matrix.md
View file

@ -34,7 +34,9 @@ This split keeps mutable VM volumes on CoronaFS and immutable VM images on Light
## Validated Edge And Tenant Services
- `apigateway + iam + prismnet`
- `nightlight + apigateway`
- `nightlight`
- `creditservice + iam + apigateway`
- `creditservice + iam`
- `deployer + iam + chainfire`
@ -42,8 +44,6 @@ This split keeps mutable VM volumes on CoronaFS and immutable VM images on Light
- `plasmavmc + prismnet`
- `plasmavmc + prismnet + coronafs + lightningstor`
- `nightlight + apigateway`
- `creditservice + iam + apigateway`
## Validation Direction

2
nix/test-cluster/README.md
View file

@ -59,7 +59,7 @@ Preferred entrypoint for publishable verification: `nix run ./nix/test-cluster#c
`make cluster-smoke` is a convenience wrapper for the same clean host-build VM validation flow.
`nix run ./nix/test-cluster#cluster -- matrix` reuses the current running cluster to exercise composed service scenarios such as `prismnet + flashdns + fiberlb`, VM hosting with `plasmavmc + coronafs + lightningstor`, and the Kubernetes-style hosting bundle.
`nix run ./nix/test-cluster#cluster -- matrix` reuses the current running cluster to exercise composed service scenarios such as `prismnet + flashdns + fiberlb`, VM hosting with `plasmavmc + coronafs + lightningstor`, the Kubernetes-style hosting bundle, and API-gateway-mediated `nightlight` / `creditservice` flows.
Preferred entrypoint for publishable matrix verification: `nix run ./nix/test-cluster#cluster -- fresh-matrix`

2
nix/test-cluster/node01.nix
View file

@ -77,12 +77,14 @@
enable = true;
port = 50081;
iamAddr = "10.100.0.11:50080";
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
};
services.flashdns = {
enable = true;
iamAddr = "10.100.0.11:50080";
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
};

14
nix/test-cluster/node06.nix
View file

@ -31,7 +31,7 @@
port = 8080;
iamAddr = "10.100.0.11:50080";
# Configure routes to control plane services
# Configure routes to control plane and tenant-facing services
routes = [
{
name = "iam-auth";
@ -54,6 +54,18 @@
upstream = "http://${config.plasmacloud.cluster.nodes.node01.ip}:8084";
timeoutMs = 1200000;
}
{
name = "nightlight-metrics";
pathPrefix = "/api/v1/metrics";
upstream = "http://127.0.0.1:9090/api/v1";
stripPrefix = true;
}
{
name = "creditservice-rest";
pathPrefix = "/api/v1/credits";
upstream = "http://127.0.0.1:3011/api/v1";
stripPrefix = true;
}
];
};

247
nix/test-cluster/run-cluster.sh
View file

@ -407,6 +407,49 @@ grpcurl_capture() {
return "${status}"
}
create_prismnet_vpc_with_retry() {
local token="$1"
local org_id="$2"
local project_id="$3"
local name="$4"
local description="$5"
local cidr_block="$6"
local timeout="${7:-${HTTP_WAIT_TIMEOUT}}"
local deadline=$((SECONDS + timeout))
local request_json output=""
wait_for_flaredb_route_metadata node01
request_json="$(
jq -cn \
--arg org "${org_id}" \
--arg project "${project_id}" \
--arg name "${name}" \
--arg description "${description}" \
--arg cidr "${cidr_block}" \
'{orgId:$org, projectId:$project, name:$name, description:$description, cidrBlock:$cidr}'
)"
while true; do
output="$(
grpcurl_capture -plaintext \
-H "authorization: Bearer ${token}" \
-import-path "${PRISMNET_PROTO_DIR}" \
-proto "${PRISMNET_PROTO}" \
-d "${request_json}" \
127.0.0.1:15081 prismnet.VpcService/CreateVpc
)" && {
printf '%s' "${output}"
return 0
}
if (( SECONDS >= deadline )); then
die "timed out waiting for PrismNet VPC writes to succeed: ${output}"
fi
sleep 2
done
}
build_link() {
printf '%s/build-%s' "$(vm_dir)" "$1"
}
@ -2782,6 +2825,7 @@ for ip in "${nodes[@]}"; do
done
done
EOS
wait_for_flaredb_route_metadata node01
log "Validating FlareDB strong-consistency CAS on the control plane"
local flaredb_proto_root="/var/lib/photon-test-protos/flaredb"
@ -2888,13 +2932,13 @@ validate_prismnet_flow() {
local vpc_resp subnet_resp port_resp
local vpc_id subnet_id port_id
vpc_resp="$(grpcurl -plaintext \
-H "authorization: Bearer ${token}" \
-import-path "${PRISMNET_PROTO_DIR}" \
-proto "${PRISMNET_PROTO}" \
-d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg name "prismnet-smoke-vpc" \
'{orgId:$org, projectId:$project, name:$name, description:"smoke vpc", cidrBlock:"10.44.0.0/16"}')" \
127.0.0.1:15081 prismnet.VpcService/CreateVpc)"
vpc_resp="$(create_prismnet_vpc_with_retry \
"${token}" \
"${org_id}" \
"${project_id}" \
"prismnet-smoke-vpc" \
"smoke vpc" \
"10.44.0.0/16")"
vpc_id="$(printf '%s' "${vpc_resp}" | jq -r '.vpc.id')"
[[ -n "${vpc_id}" && "${vpc_id}" != "null" ]] || die "PrismNet CreateVpc did not return a VPC ID"
@ -4313,15 +4357,18 @@ validate_gateway() {
stop_ssh_tunnel node01 "${iam_tunnel}"
}
validate_nightlight_flow() {
log "Validating NightLight remote_write ingestion and query endpoints"
validate_nightlight_flow_with_base() {
local base_url="$1"
local flow_name="$2"
log "Validating ${flow_name} remote_write ingestion and query endpoints"
local metric_name="nightlight_smoke_metric_$(date +%s)"
local metric_value
metric_value="$(awk 'BEGIN{srand(); printf "%.3f\n", (rand()*100)+1}')"
python3 "${REPO_ROOT}/nix/test-cluster/nightlight_remote_write.py" \
--url http://127.0.0.1:9090/api/v1/write \
--url "${base_url}/write" \
--metric "${metric_name}" \
--value "${metric_value}" \
--label source=smoke \
@ -4329,7 +4376,7 @@ validate_nightlight_flow() {
local deadline=$((SECONDS + HTTP_WAIT_TIMEOUT))
while true; do
if curl -fsS --get http://127.0.0.1:9090/api/v1/query \
if curl -fsS --get "${base_url}/query" \
--data-urlencode "query=${metric_name}{source=\"smoke\"}" \
| jq -e --arg name "${metric_name}" --argjson expected "${metric_value}" '
.status == "success"
@ -4339,17 +4386,77 @@ validate_nightlight_flow() {
break
fi
if (( SECONDS >= deadline )); then
die "timed out waiting for NightLight query result for ${metric_name}"
die "timed out waiting for ${flow_name} query result for ${metric_name}"
fi
sleep 2
done
curl -fsS http://127.0.0.1:9090/api/v1/label/__name__/values \
curl -fsS "${base_url}/label/__name__/values" \
| jq -e --arg name "${metric_name}" '.status == "success" and (.data | index($name)) != null' >/dev/null
curl -fsS http://127.0.0.1:9090/api/v1/series \
curl -fsS "${base_url}/series" \
| jq -e --arg name "${metric_name}" '.status == "success" and (.data | any(.__name__ == $name))' >/dev/null
}
validate_nightlight_flow() {
validate_nightlight_flow_with_base "http://127.0.0.1:9090/api/v1" "NightLight"
}
validate_apigateway_nightlight_flow() {
log "Matrix case: NightLight + APIGateway"
wait_for_host_http http://127.0.0.1:8080/health
validate_nightlight_flow_with_base "http://127.0.0.1:8080/api/v1/metrics" "API Gateway -> NightLight"
}
validate_creditservice_rest_flow() {
local base_url="$1"
local token="$2"
local org_id="$3"
local project_id="$4"
local flow_name="$5"
local reservation_json reservation_id
log "Validating ${flow_name} REST wallet and reservation flows"
curl -fsS \
-X POST "${base_url}/wallets" \
-H "Authorization: Bearer ${token}" \
-H 'Content-Type: application/json' \
-d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" '{org_id:$org, project_id:$project, initial_balance:1000}')" \
| jq -e '.data.project_id != null and .data.balance == 1000 and .data.available == 1000' >/dev/null
curl -fsS "${base_url}/wallets/${project_id}" \
-H "Authorization: Bearer ${token}" \
| jq -e --arg project "${project_id}" '.data.project_id == $project and .data.balance == 1000' >/dev/null
curl -fsS \
-X POST "${base_url}/wallets/${project_id}/topup" \
-H "Authorization: Bearer ${token}" \
-H 'Content-Type: application/json' \
-d '{"amount":250,"description":"smoke topup"}' \
| jq -e '.data.balance == 1250 and .data.total_deposited == 1250' >/dev/null
reservation_json="$(
curl -fsS \
-X POST "${base_url}/reservations" \
-H "Authorization: Bearer ${token}" \
-H 'Content-Type: application/json' \
-d "$(jq -cn --arg project "${project_id}" '{project_id:$project, amount:200, description:"smoke reservation", resource_type:"vm", ttl_seconds:120}')"
)"
reservation_id="$(printf '%s' "${reservation_json}" | jq -r '.data.id')"
[[ -n "${reservation_id}" && "${reservation_id}" != "null" ]] || die "${flow_name} reservation did not return an ID"
curl -fsS \
-X POST "${base_url}/reservations/${reservation_id}/commit" \
-H "Authorization: Bearer ${token}" \
-H 'Content-Type: application/json' \
-d '{"actual_amount":150,"resource_id":"smoke-vm"}' \
| jq -e '.data.balance == 1100 and .data.reserved == 0 and .data.available == 1100' >/dev/null
curl -fsS "${base_url}/wallets/${project_id}/transactions" \
-H "Authorization: Bearer ${token}" \
| jq -e '.data.transactions | length >= 3' >/dev/null
}
validate_creditservice_flow() {
log "Validating CreditService REST and gRPC quota flows"
@ -4364,47 +4471,15 @@ validate_creditservice_flow() {
local org_id="credit-smoke-org-${suffix}"
local project_id="credit-smoke-project-${suffix}"
local principal_id="credit-smoke-$(date +%s)"
local token reservation_json reservation_id
local token
token="$(issue_project_admin_token 15080 "${org_id}" "${project_id}" "${principal_id}")"
curl -fsS \
-X POST http://127.0.0.1:13011/api/v1/wallets \
-H "Authorization: Bearer ${token}" \
-H 'Content-Type: application/json' \
-d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" '{org_id:$org, project_id:$project, initial_balance:1000}')" \
| jq -e '.data.project_id != null and .data.balance == 1000 and .data.available == 1000' >/dev/null
curl -fsS http://127.0.0.1:13011/api/v1/wallets/"${project_id}" \
-H "Authorization: Bearer ${token}" \
| jq -e --arg project "${project_id}" '.data.project_id == $project and .data.balance == 1000' >/dev/null
curl -fsS \
-X POST http://127.0.0.1:13011/api/v1/wallets/"${project_id}"/topup \
-H "Authorization: Bearer ${token}" \
-H 'Content-Type: application/json' \
-d '{"amount":250,"description":"smoke topup"}' \
| jq -e '.data.balance == 1250 and .data.total_deposited == 1250' >/dev/null
reservation_json="$(
curl -fsS \
-X POST http://127.0.0.1:13011/api/v1/reservations \
-H "Authorization: Bearer ${token}" \
-H 'Content-Type: application/json' \
-d "$(jq -cn --arg project "${project_id}" '{project_id:$project, amount:200, description:"smoke reservation", resource_type:"vm", ttl_seconds:120}')"
)"
reservation_id="$(printf '%s' "${reservation_json}" | jq -r '.data.id')"
[[ -n "${reservation_id}" && "${reservation_id}" != "null" ]] || die "CreditService reservation did not return an ID"
curl -fsS \
-X POST http://127.0.0.1:13011/api/v1/reservations/"${reservation_id}"/commit \
-H "Authorization: Bearer ${token}" \
-H 'Content-Type: application/json' \
-d '{"actual_amount":150,"resource_id":"smoke-vm"}' \
| jq -e '.data.balance == 1100 and .data.reserved == 0 and .data.available == 1100' >/dev/null
curl -fsS http://127.0.0.1:13011/api/v1/wallets/"${project_id}"/transactions \
-H "Authorization: Bearer ${token}" \
| jq -e '.data.transactions | length >= 3' >/dev/null
validate_creditservice_rest_flow \
"http://127.0.0.1:13011/api/v1" \
"${token}" \
"${org_id}" \
"${project_id}" \
"CreditService"
grpcurl -plaintext \
-H "authorization: Bearer ${token}" \
@ -4450,6 +4525,34 @@ EOS
stop_ssh_tunnel node01 "${iam_tunnel}"
}
validate_apigateway_creditservice_flow() {
log "Matrix case: CreditService + IAM + APIGateway"
local iam_tunnel=""
iam_tunnel="$(start_ssh_tunnel node01 15080 50080)"
trap 'stop_ssh_tunnel node01 "${iam_tunnel}"' RETURN
wait_for_host_http http://127.0.0.1:8080/health
local suffix
suffix="$(date +%s)"
local org_id="credit-gateway-org-${suffix}"
local project_id="credit-gateway-project-${suffix}"
local principal_id="credit-gateway-$(date +%s)"
local token
token="$(issue_project_admin_token 15080 "${org_id}" "${project_id}" "${principal_id}")"
validate_creditservice_rest_flow \
"http://127.0.0.1:8080/api/v1/credits" \
"${token}" \
"${org_id}" \
"${project_id}" \
"API Gateway -> CreditService"
trap - RETURN
stop_ssh_tunnel node01 "${iam_tunnel}"
}
validate_deployer_flow() {
log "Validating Deployer health, admin registration, and phone-home flows"
@ -5251,61 +5354,61 @@ validate_network_provider_matrix() {
local workload_fqdn="" service_fqdn=""
cleanup_network_provider_matrix() {
if [[ -n "${service_record_id}" ]]; then
if [[ -n "${service_record_id:-}" ]]; then
grpcurl -plaintext -H "authorization: Bearer ${token}" \
-import-path "${FLASHDNS_PROTO_DIR}" -proto "${FLASHDNS_PROTO}" \
-d "$(jq -cn --arg id "${service_record_id}" '{id:$id}')" \
127.0.0.1:15084 flashdns.v1.RecordService/DeleteRecord >/dev/null 2>&1 || true
fi
if [[ -n "${workload_record_id}" ]]; then
if [[ -n "${workload_record_id:-}" ]]; then
grpcurl -plaintext -H "authorization: Bearer ${token}" \
-import-path "${FLASHDNS_PROTO_DIR}" -proto "${FLASHDNS_PROTO}" \
-d "$(jq -cn --arg id "${workload_record_id}" '{id:$id}')" \
127.0.0.1:15084 flashdns.v1.RecordService/DeleteRecord >/dev/null 2>&1 || true
fi
if [[ -n "${listener_id}" ]]; then
if [[ -n "${listener_id:-}" ]]; then
grpcurl -plaintext -H "authorization: Bearer ${token}" \
-import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \
-d "$(jq -cn --arg id "${listener_id}" '{id:$id}')" \
127.0.0.1:15085 fiberlb.v1.ListenerService/DeleteListener >/dev/null 2>&1 || true
fi
if [[ -n "${backend_id}" ]]; then
if [[ -n "${backend_id:-}" ]]; then
grpcurl -plaintext -H "authorization: Bearer ${token}" \
-import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \
-d "$(jq -cn --arg id "${backend_id}" '{id:$id}')" \
127.0.0.1:15085 fiberlb.v1.BackendService/DeleteBackend >/dev/null 2>&1 || true
fi
if [[ -n "${pool_id}" ]]; then
if [[ -n "${pool_id:-}" ]]; then
grpcurl -plaintext -H "authorization: Bearer ${token}" \
-import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \
-d "$(jq -cn --arg id "${pool_id}" '{id:$id}')" \
127.0.0.1:15085 fiberlb.v1.PoolService/DeletePool >/dev/null 2>&1 || true
fi
if [[ -n "${lb_id}" ]]; then
if [[ -n "${lb_id:-}" ]]; then
grpcurl -plaintext -H "authorization: Bearer ${token}" \
-import-path "${FIBERLB_PROTO_DIR}" -proto "${FIBERLB_PROTO}" \
-d "$(jq -cn --arg id "${lb_id}" '{id:$id}')" \
127.0.0.1:15085 fiberlb.v1.LoadBalancerService/DeleteLoadBalancer >/dev/null 2>&1 || true
fi
if [[ -n "${port_id}" ]]; then
if [[ -n "${port_id:-}" ]]; then
grpcurl -plaintext -H "authorization: Bearer ${token}" \
-import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \
-d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg subnet "${subnet_id}" --arg id "${port_id}" '{orgId:$org, projectId:$project, subnetId:$subnet, id:$id}')" \
127.0.0.1:15081 prismnet.PortService/DeletePort >/dev/null 2>&1 || true
fi
if [[ -n "${subnet_id}" ]]; then
if [[ -n "${subnet_id:-}" ]]; then
grpcurl -plaintext -H "authorization: Bearer ${token}" \
-import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \
-d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg vpc "${vpc_id}" --arg id "${subnet_id}" '{orgId:$org, projectId:$project, vpcId:$vpc, id:$id}')" \
127.0.0.1:15081 prismnet.SubnetService/DeleteSubnet >/dev/null 2>&1 || true
fi
if [[ -n "${vpc_id}" ]]; then
if [[ -n "${vpc_id:-}" ]]; then
grpcurl -plaintext -H "authorization: Bearer ${token}" \
-import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \
-d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" --arg id "${vpc_id}" '{orgId:$org, projectId:$project, id:$id}')" \
127.0.0.1:15081 prismnet.VpcService/DeleteVpc >/dev/null 2>&1 || true
fi
if [[ -n "${zone_id}" ]]; then
if [[ -n "${zone_id:-}" ]]; then
grpcurl -plaintext -H "authorization: Bearer ${token}" \
-import-path "${FLASHDNS_PROTO_DIR}" -proto "${FLASHDNS_PROTO}" \
-d "$(jq -cn --arg id "${zone_id}" '{id:$id, force:true}')" \
@ -5317,15 +5420,19 @@ validate_network_provider_matrix() {
stop_ssh_tunnel node01 "${prism_tunnel}" >/dev/null 2>&1 || true
stop_ssh_tunnel node01 "${iam_tunnel}" >/dev/null 2>&1 || true
}
trap cleanup_network_provider_matrix RETURN EXIT
trap cleanup_network_provider_matrix RETURN
token="$(issue_project_admin_token 15080 "${org_id}" "${project_id}" "${principal_id}")"
log "Matrix case: PrismNet only"
vpc_id="$(grpcurl -plaintext -H "authorization: Bearer ${token}" \
-import-path "${PRISMNET_PROTO_DIR}" -proto "${PRISMNET_PROTO}" \
-d "$(jq -cn --arg org "${org_id}" --arg project "${project_id}" '{orgId:$org, projectId:$project, name:"matrix-vpc", description:"component matrix", cidrBlock:"10.52.0.0/16"}')" \
127.0.0.1:15081 prismnet.VpcService/CreateVpc | jq -r '.vpc.id')"
vpc_id="$(create_prismnet_vpc_with_retry \
"${token}" \
"${org_id}" \
"${project_id}" \
"matrix-vpc" \
"component matrix" \
"10.52.0.0/16" \
240 | jq -r '.vpc.id')"
[[ -n "${vpc_id}" && "${vpc_id}" != "null" ]] || die "component matrix PrismNet VPC creation failed"
subnet_id="$(grpcurl -plaintext -H "authorization: Bearer ${token}" \
@ -5416,7 +5523,7 @@ validate_network_provider_matrix() {
ssh_node node01 "curl -fsS --max-time 5 --resolve ${service_fqdn}:${listener_port}:10.100.0.11 http://${service_fqdn}:${listener_port}/health >/dev/null"
trap - RETURN EXIT
trap - RETURN
cleanup_network_provider_matrix
}
@ -5428,7 +5535,9 @@ validate_component_matrix() {
validate_k8shost_flow
validate_gateway
validate_nightlight_flow
validate_apigateway_nightlight_flow
validate_creditservice_flow
validate_apigateway_creditservice_flow
validate_deployer_flow
validate_native_runtime_flow
log "Component matrix validation succeeded"