From d3d74995e87b83b9f87e9602bf987827ee3904b4 Mon Sep 17 00:00:00 2001 From: centra Date: Wed, 24 Dec 2025 18:21:55 +0900 Subject: [PATCH] chore: initial sync of untracked files and infrastructure components --- .github/workflows/nix.yml | 70 + apigateway/Cargo.lock | 3610 ++++++++++++++++ apigateway/Cargo.toml | 55 + apigateway/crates/apigateway-api/Cargo.toml | 19 + apigateway/crates/apigateway-api/build.rs | 9 + .../apigateway-api/proto/apigateway.proto | 87 + apigateway/crates/apigateway-api/src/lib.rs | 10 + .../crates/apigateway-server/Cargo.toml | 38 + .../crates/apigateway-server/src/main.rs | 1482 +++++++ chainfire/chainfire-client/examples/basic.rs | 15 + chainfire/chainfire-client/src/metadata.rs | 380 ++ chainfire/crates/chainfire-core/src/traits.rs | 60 + .../chainfire-core/tests/integration.rs | 52 + .../crates/chainfire-raft/src/storage.rs | 378 ++ .../chainfire-raft/tests/proptest_sim.rs | 274 ++ client-common/Cargo.lock | 1098 +++++ client-common/Cargo.toml | 16 + client-common/src/lib.rs | 205 + .../src/gateway_credit_service.rs | 275 ++ .../tests/mtls_integration.rs | 77 + .../creditservice-client/examples/basic.rs | 18 + .../creditservice-client/examples/builder.rs | 27 + deployer/crates/cert-authority/Cargo.toml | 25 + deployer/crates/cert-authority/src/main.rs | 348 ++ deployer/crates/deployer-ctl/Cargo.toml | 19 + deployer/crates/deployer-ctl/src/chainfire.rs | 177 + deployer/crates/deployer-ctl/src/main.rs | 113 + deployer/crates/deployer-ctl/src/model.rs | 86 + deployer/crates/deployer-ctl/src/remote.rs | 35 + deployer/crates/node-agent/Cargo.toml | 23 + deployer/crates/node-agent/src/agent.rs | 334 ++ deployer/crates/node-agent/src/main.rs | 62 + deployer/crates/node-agent/src/process.rs | 273 ++ deployer/crates/node-agent/src/watcher.rs | 77 + .../crates/plasmacloud-reconciler/Cargo.toml | 20 + .../crates/plasmacloud-reconciler/src/main.rs | 1918 +++++++++ docs/Nix-NOS.md | 398 ++ docs/README-dependency-graphs.md | 64 + docs/architecture/api-gateway.md | 111 + docs/architecture/baremetal-mesh-migration.md | 113 + docs/cert-authority-usage.md | 124 + docs/component-dependencies-detailed.dot | 174 + docs/component-dependencies.dot | 131 + .../cluster-config.json | 1 + .../test.nix | 53 + .../first-boot-automation-cluster-config.txt | 21 + docs/implementation-status.md | 74 + docs/implementation-summary.md | 91 + docs/nixos-deployment-challenges.md | 448 ++ docs/ops/integration-matrix.md | 43 + docs/ops/nested-kvm-setup.md | 38 + docs/ops/qcow2-artifact-plan.md | 26 + .../chainfire_architecture_redefinition.md | 89 + docs/plans/metadata_unification.md | 45 + examples/mtls-agent-config.toml | 17 + examples/photoncloud-test-cluster.json | 79 + fiberlb/crates/fiberlb-server/build.rs | 10 + .../fiberlb-server/proto/api/attribute.proto | 584 +++ .../fiberlb-server/proto/api/capability.proto | 124 + .../fiberlb-server/proto/api/common.proto | 63 + .../fiberlb-server/proto/api/extcom.proto | 162 + .../fiberlb-server/proto/api/gobgp.proto | 1379 ++++++ .../fiberlb-server/proto/api/nlri.proto | 361 ++ fiberlb/crates/fiberlb-server/src/gobgp.rs | 3 + .../crates/flaredb-client/examples/basic.rs | 16 + .../src/reverse_zone_service.rs | 224 + iam/crates/iam-api/src/credential_service.rs | 365 ++ .../iam-api/src/gateway_auth_service.rs | 433 ++ iam/crates/iam-client/examples/basic.rs | 14 + iam/crates/iam-store/src/credential_store.rs | 68 + iam/crates/iam-types/src/credential.rs | 35 + k8shost/crates/k8shost-csi/build.rs | 10 + k8shost/crates/k8shost-csi/proto/csi.proto | 1914 +++++++++ .../lightningstor-distributed/Cargo.toml | 47 + .../src/backends/erasure_coded.rs | 848 ++++ .../src/backends/mod.rs | 10 + .../src/backends/replicated.rs | 535 +++ .../src/chunk/mod.rs | 276 ++ .../lightningstor-distributed/src/config.rs | 288 ++ .../src/erasure/mod.rs | 381 ++ .../lightningstor-distributed/src/lib.rs | 179 + .../src/node/client.rs | 403 ++ .../src/node/mock.rs | 408 ++ .../lightningstor-distributed/src/node/mod.rs | 39 + .../src/node/registry.rs | 281 ++ .../src/placement/mod.rs | 398 ++ .../crates/lightningstor-node/Cargo.toml | 51 + .../crates/lightningstor-node/build.rs | 18 + .../lightningstor-node/proto/node.proto | 135 + .../crates/lightningstor-node/src/config.rs | 76 + .../crates/lightningstor-node/src/lib.rs | 36 + .../crates/lightningstor-node/src/main.rs | 169 + .../crates/lightningstor-node/src/service.rs | 232 + .../crates/lightningstor-node/src/storage.rs | 313 ++ .../crates/lightningstor-server/src/tenant.rs | 59 + mtls-agent/Cargo.lock | 1954 +++++++++ mtls-agent/Cargo.toml | 23 + mtls-agent/src/client.rs | 89 + mtls-agent/src/discovery.rs | 219 + mtls-agent/src/main.rs | 337 ++ mtls-agent/src/policy.rs | 114 + nix/modules/apigateway.nix | 293 ++ nix/modules/plasmacloud-resources.nix | 667 +++ nix/templates/iam-flaredb-minimal.nix | 18 + nix/templates/plasmacloud-3node-ha.nix | 88 + nix/templates/plasmacloud-single-node.nix | 84 + .../plasmavmc-server/tests/common/mod.rs | 165 + scripts/integration-matrix.sh | 16 + scripts/nested-kvm-check.sh | 83 + specifications/deployer/README.md | 354 ++ .../fiberlb/S2-l7-loadbalancing-spec.md | 808 ++++ .../fiberlb/S3-bgp-integration-spec.md | 369 ++ .../checklists/requirements.md | 34 + .../contracts/kvrpc.proto | 55 + .../001-distributed-core/contracts/pdpb.proto | 56 + .../001-distributed-core/data-model.md | 52 + .../flaredb/001-distributed-core/plan.md | 95 + .../001-distributed-core/quickstart.md | 64 + .../flaredb/001-distributed-core/research.md | 19 + .../flaredb/001-distributed-core/spec.md | 87 + .../flaredb/001-distributed-core/tasks.md | 220 + specifications/flaredb/001-multi-raft/spec.md | 115 + .../checklists/requirements.md | 34 + .../contracts/raft-service.md | 35 + .../flaredb/002-raft-features/data-model.md | 34 + .../flaredb/002-raft-features/plan.md | 69 + .../flaredb/002-raft-features/quickstart.md | 39 + .../flaredb/002-raft-features/research.md | 23 + .../flaredb/002-raft-features/spec.md | 92 + .../flaredb/002-raft-features/tasks.md | 128 + .../checklists/requirements.md | 34 + .../003-kvs-consistency/contracts/kv_cas.md | 29 + .../003-kvs-consistency/contracts/kv_raw.md | 25 + .../contracts/raft_service.md | 33 + .../flaredb/003-kvs-consistency/data-model.md | 26 + .../flaredb/003-kvs-consistency/plan.md | 76 + .../flaredb/003-kvs-consistency/quickstart.md | 78 + .../flaredb/003-kvs-consistency/research.md | 15 + .../flaredb/003-kvs-consistency/spec.md | 88 + .../flaredb/003-kvs-consistency/tasks.md | 119 + .../004-multi-raft/checklists/requirements.md | 34 + .../flaredb/004-multi-raft/contracts/pd.md | 36 + .../flaredb/004-multi-raft/data-model.md | 45 + specifications/flaredb/004-multi-raft/plan.md | 62 + .../flaredb/004-multi-raft/quickstart.md | 44 + specifications/flaredb/004-multi-raft/spec.md | 208 + .../flaredb/004-multi-raft/tasks.md | 125 + specifications/flaredb/sql-layer-design.md | 299 ++ specifications/k8shost/S1-ipam-spec.md | 328 ++ specifications/metricstor-design.md | 3744 +++++++++++++++++ testing/qemu-cluster/README.md | 96 + .../qemu-cluster/configs/cluster-config.json | 50 + .../qemu-cluster/scripts/create-base-image.sh | 36 + .../scripts/deploy-photoncloud.sh | 59 + testing/qemu-cluster/scripts/start-cluster.sh | 73 + testing/qemu-cluster/scripts/stop-cluster.sh | 23 + testing/run-s3-test.sh | 84 + testing/s3-test.nix | 25 + 158 files changed, 37678 insertions(+) create mode 100644 .github/workflows/nix.yml create mode 100644 apigateway/Cargo.lock create mode 100644 apigateway/Cargo.toml create mode 100644 apigateway/crates/apigateway-api/Cargo.toml create mode 100644 apigateway/crates/apigateway-api/build.rs create mode 100644 apigateway/crates/apigateway-api/proto/apigateway.proto create mode 100644 apigateway/crates/apigateway-api/src/lib.rs create mode 100644 apigateway/crates/apigateway-server/Cargo.toml create mode 100644 apigateway/crates/apigateway-server/src/main.rs create mode 100644 chainfire/chainfire-client/examples/basic.rs create mode 100644 chainfire/chainfire-client/src/metadata.rs create mode 100644 chainfire/crates/chainfire-core/src/traits.rs create mode 100644 chainfire/crates/chainfire-core/tests/integration.rs create mode 100644 chainfire/crates/chainfire-raft/src/storage.rs create mode 100644 chainfire/crates/chainfire-raft/tests/proptest_sim.rs create mode 100644 client-common/Cargo.lock create mode 100644 client-common/Cargo.toml create mode 100644 client-common/src/lib.rs create mode 100644 creditservice/crates/creditservice-api/src/gateway_credit_service.rs create mode 100644 creditservice/crates/creditservice-server/tests/mtls_integration.rs create mode 100644 creditservice/creditservice-client/examples/basic.rs create mode 100644 creditservice/creditservice-client/examples/builder.rs create mode 100644 deployer/crates/cert-authority/Cargo.toml create mode 100644 deployer/crates/cert-authority/src/main.rs create mode 100644 deployer/crates/deployer-ctl/Cargo.toml create mode 100644 deployer/crates/deployer-ctl/src/chainfire.rs create mode 100644 deployer/crates/deployer-ctl/src/main.rs create mode 100644 deployer/crates/deployer-ctl/src/model.rs create mode 100644 deployer/crates/deployer-ctl/src/remote.rs create mode 100644 deployer/crates/node-agent/Cargo.toml create mode 100644 deployer/crates/node-agent/src/agent.rs create mode 100644 deployer/crates/node-agent/src/main.rs create mode 100644 deployer/crates/node-agent/src/process.rs create mode 100644 deployer/crates/node-agent/src/watcher.rs create mode 100644 deployer/crates/plasmacloud-reconciler/Cargo.toml create mode 100644 deployer/crates/plasmacloud-reconciler/src/main.rs create mode 100644 docs/Nix-NOS.md create mode 100644 docs/README-dependency-graphs.md create mode 100644 docs/architecture/api-gateway.md create mode 100644 docs/architecture/baremetal-mesh-migration.md create mode 100644 docs/cert-authority-usage.md create mode 100644 docs/component-dependencies-detailed.dot create mode 100644 docs/component-dependencies.dot create mode 100644 docs/evidence/first-boot-automation-20251220-050900/cluster-config.json create mode 100644 docs/evidence/first-boot-automation-20251220-050900/test.nix create mode 100644 docs/evidence/first-boot-automation-cluster-config.txt create mode 100644 docs/implementation-status.md create mode 100644 docs/implementation-summary.md create mode 100644 docs/nixos-deployment-challenges.md create mode 100644 docs/ops/integration-matrix.md create mode 100644 docs/ops/nested-kvm-setup.md create mode 100644 docs/ops/qcow2-artifact-plan.md create mode 100644 docs/plans/chainfire_architecture_redefinition.md create mode 100644 docs/plans/metadata_unification.md create mode 100644 examples/mtls-agent-config.toml create mode 100644 examples/photoncloud-test-cluster.json create mode 100644 fiberlb/crates/fiberlb-server/build.rs create mode 100644 fiberlb/crates/fiberlb-server/proto/api/attribute.proto create mode 100644 fiberlb/crates/fiberlb-server/proto/api/capability.proto create mode 100644 fiberlb/crates/fiberlb-server/proto/api/common.proto create mode 100644 fiberlb/crates/fiberlb-server/proto/api/extcom.proto create mode 100644 fiberlb/crates/fiberlb-server/proto/api/gobgp.proto create mode 100644 fiberlb/crates/fiberlb-server/proto/api/nlri.proto create mode 100644 fiberlb/crates/fiberlb-server/src/gobgp.rs create mode 100644 flaredb/crates/flaredb-client/examples/basic.rs create mode 100644 flashdns/crates/flashdns-server/src/reverse_zone_service.rs create mode 100644 iam/crates/iam-api/src/credential_service.rs create mode 100644 iam/crates/iam-api/src/gateway_auth_service.rs create mode 100644 iam/crates/iam-client/examples/basic.rs create mode 100644 iam/crates/iam-store/src/credential_store.rs create mode 100644 iam/crates/iam-types/src/credential.rs create mode 100644 k8shost/crates/k8shost-csi/build.rs create mode 100644 k8shost/crates/k8shost-csi/proto/csi.proto create mode 100644 lightningstor/crates/lightningstor-distributed/Cargo.toml create mode 100644 lightningstor/crates/lightningstor-distributed/src/backends/erasure_coded.rs create mode 100644 lightningstor/crates/lightningstor-distributed/src/backends/mod.rs create mode 100644 lightningstor/crates/lightningstor-distributed/src/backends/replicated.rs create mode 100644 lightningstor/crates/lightningstor-distributed/src/chunk/mod.rs create mode 100644 lightningstor/crates/lightningstor-distributed/src/config.rs create mode 100644 lightningstor/crates/lightningstor-distributed/src/erasure/mod.rs create mode 100644 lightningstor/crates/lightningstor-distributed/src/lib.rs create mode 100644 lightningstor/crates/lightningstor-distributed/src/node/client.rs create mode 100644 lightningstor/crates/lightningstor-distributed/src/node/mock.rs create mode 100644 lightningstor/crates/lightningstor-distributed/src/node/mod.rs create mode 100644 lightningstor/crates/lightningstor-distributed/src/node/registry.rs create mode 100644 lightningstor/crates/lightningstor-distributed/src/placement/mod.rs create mode 100644 lightningstor/crates/lightningstor-node/Cargo.toml create mode 100644 lightningstor/crates/lightningstor-node/build.rs create mode 100644 lightningstor/crates/lightningstor-node/proto/node.proto create mode 100644 lightningstor/crates/lightningstor-node/src/config.rs create mode 100644 lightningstor/crates/lightningstor-node/src/lib.rs create mode 100644 lightningstor/crates/lightningstor-node/src/main.rs create mode 100644 lightningstor/crates/lightningstor-node/src/service.rs create mode 100644 lightningstor/crates/lightningstor-node/src/storage.rs create mode 100644 lightningstor/crates/lightningstor-server/src/tenant.rs create mode 100644 mtls-agent/Cargo.lock create mode 100644 mtls-agent/Cargo.toml create mode 100644 mtls-agent/src/client.rs create mode 100644 mtls-agent/src/discovery.rs create mode 100644 mtls-agent/src/main.rs create mode 100644 mtls-agent/src/policy.rs create mode 100644 nix/modules/apigateway.nix create mode 100644 nix/modules/plasmacloud-resources.nix create mode 100644 nix/templates/iam-flaredb-minimal.nix create mode 100644 nix/templates/plasmacloud-3node-ha.nix create mode 100644 nix/templates/plasmacloud-single-node.nix create mode 100644 plasmavmc/crates/plasmavmc-server/tests/common/mod.rs create mode 100755 scripts/integration-matrix.sh create mode 100755 scripts/nested-kvm-check.sh create mode 100644 specifications/deployer/README.md create mode 100644 specifications/fiberlb/S2-l7-loadbalancing-spec.md create mode 100644 specifications/fiberlb/S3-bgp-integration-spec.md create mode 100644 specifications/flaredb/001-distributed-core/checklists/requirements.md create mode 100644 specifications/flaredb/001-distributed-core/contracts/kvrpc.proto create mode 100644 specifications/flaredb/001-distributed-core/contracts/pdpb.proto create mode 100644 specifications/flaredb/001-distributed-core/data-model.md create mode 100644 specifications/flaredb/001-distributed-core/plan.md create mode 100644 specifications/flaredb/001-distributed-core/quickstart.md create mode 100644 specifications/flaredb/001-distributed-core/research.md create mode 100644 specifications/flaredb/001-distributed-core/spec.md create mode 100644 specifications/flaredb/001-distributed-core/tasks.md create mode 100644 specifications/flaredb/001-multi-raft/spec.md create mode 100644 specifications/flaredb/002-raft-features/checklists/requirements.md create mode 100644 specifications/flaredb/002-raft-features/contracts/raft-service.md create mode 100644 specifications/flaredb/002-raft-features/data-model.md create mode 100644 specifications/flaredb/002-raft-features/plan.md create mode 100644 specifications/flaredb/002-raft-features/quickstart.md create mode 100644 specifications/flaredb/002-raft-features/research.md create mode 100644 specifications/flaredb/002-raft-features/spec.md create mode 100644 specifications/flaredb/002-raft-features/tasks.md create mode 100644 specifications/flaredb/003-kvs-consistency/checklists/requirements.md create mode 100644 specifications/flaredb/003-kvs-consistency/contracts/kv_cas.md create mode 100644 specifications/flaredb/003-kvs-consistency/contracts/kv_raw.md create mode 100644 specifications/flaredb/003-kvs-consistency/contracts/raft_service.md create mode 100644 specifications/flaredb/003-kvs-consistency/data-model.md create mode 100644 specifications/flaredb/003-kvs-consistency/plan.md create mode 100644 specifications/flaredb/003-kvs-consistency/quickstart.md create mode 100644 specifications/flaredb/003-kvs-consistency/research.md create mode 100644 specifications/flaredb/003-kvs-consistency/spec.md create mode 100644 specifications/flaredb/003-kvs-consistency/tasks.md create mode 100644 specifications/flaredb/004-multi-raft/checklists/requirements.md create mode 100644 specifications/flaredb/004-multi-raft/contracts/pd.md create mode 100644 specifications/flaredb/004-multi-raft/data-model.md create mode 100644 specifications/flaredb/004-multi-raft/plan.md create mode 100644 specifications/flaredb/004-multi-raft/quickstart.md create mode 100644 specifications/flaredb/004-multi-raft/spec.md create mode 100644 specifications/flaredb/004-multi-raft/tasks.md create mode 100644 specifications/flaredb/sql-layer-design.md create mode 100644 specifications/k8shost/S1-ipam-spec.md create mode 100644 specifications/metricstor-design.md create mode 100644 testing/qemu-cluster/README.md create mode 100644 testing/qemu-cluster/configs/cluster-config.json create mode 100755 testing/qemu-cluster/scripts/create-base-image.sh create mode 100755 testing/qemu-cluster/scripts/deploy-photoncloud.sh create mode 100755 testing/qemu-cluster/scripts/start-cluster.sh create mode 100755 testing/qemu-cluster/scripts/stop-cluster.sh create mode 100644 testing/run-s3-test.sh create mode 100644 testing/s3-test.nix diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml new file mode 100644 index 0000000..c2c107e --- /dev/null +++ b/.github/workflows/nix.yml @@ -0,0 +1,70 @@ +name: Nix CI + +on: + push: + pull_request: + +jobs: + flake-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@v11 + - uses: DeterminateSystems/magic-nix-cache-action@v8 + - name: Nix flake check + run: nix flake check --accept-flake-config + + build-servers: + runs-on: ubuntu-latest + needs: flake-check + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@v11 + - uses: DeterminateSystems/magic-nix-cache-action@v8 + - name: Build server packages + run: | + nix build --accept-flake-config .#chainfire-server .#flaredb-server .#iam-server .#plasmavmc-server .#prismnet-server .#flashdns-server .#fiberlb-server .#lightningstor-server .#creditservice-server + + integration-matrix: + runs-on: ubuntu-latest + needs: build-servers + env: + PLASMA_E2E: "1" + # SKIP_PLASMA defaults to 0; set repo/runner var to 1 only when qemu-img/KVM is unavailable. + SKIP_PLASMA: ${{ vars.SKIP_PLASMA || '0' }} + LOG_DIR: .cccc/work/integration-matrix/${{ github.run_id }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@v11 + - uses: DeterminateSystems/magic-nix-cache-action@v8 + - name: Run integration matrix (Noop hypervisor gate) + run: | + nix develop -c ./scripts/integration-matrix.sh + - name: Upload integration-matrix logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-matrix-logs + path: .cccc/work/integration-matrix/ + + integration-matrix-kvm: + if: ${{ vars.NESTED_KVM == '1' }} + runs-on: ubuntu-latest + needs: integration-matrix + env: + PLASMA_E2E: "1" + SKIP_PLASMA: "0" + LOG_DIR: .cccc/work/integration-matrix-kvm/${{ github.run_id }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@v11 + - uses: DeterminateSystems/magic-nix-cache-action@v8 + - name: Run integration matrix (KVM lane) + run: | + nix develop -c ./scripts/integration-matrix.sh + - name: Upload integration-matrix-kvm logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-matrix-kvm-logs + path: .cccc/work/integration-matrix-kvm/ diff --git a/apigateway/Cargo.lock b/apigateway/Cargo.lock new file mode 100644 index 0000000..5decddb --- /dev/null +++ b/apigateway/Cargo.lock @@ -0,0 +1,3610 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "apigateway-api" +version = "0.1.0" +dependencies = [ + "prost", + "prost-types", + "protoc-bin-vendored", + "tonic", + "tonic-build", +] + +[[package]] +name = "apigateway-server" +version = "0.1.0" +dependencies = [ + "apigateway-api", + "async-trait", + "axum", + "bytes", + "clap", + "creditservice-api", + "creditservice-types", + "futures-core", + "iam-api", + "iam-authn", + "iam-authz", + "iam-store", + "iam-types", + "reqwest 0.12.26", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "toml", + "tonic", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.2", + "tokio", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "borsh" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1da5ab77c1437701eeff7c88d968729e7766172279eab0676857b3d63af7a6f" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + +[[package]] +name = "cc" +version = "1.2.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chainfire-client" +version = "0.1.0" +dependencies = [ + "chainfire-proto", + "chainfire-types", + "futures", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tonic", + "tracing", +] + +[[package]] +name = "chainfire-proto" +version = "0.1.0" +dependencies = [ + "prost", + "prost-types", + "protoc-bin-vendored", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", +] + +[[package]] +name = "chainfire-types" +version = "0.1.0" +dependencies = [ + "bytes", + "serde", + "thiserror 1.0.69", +] + +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "clap_lex" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "creditservice-api" +version = "0.1.0" +dependencies = [ + "async-trait", + "chainfire-client", + "chainfire-proto", + "chrono", + "creditservice-proto", + "creditservice-types", + "prost", + "prost-types", + "reqwest 0.11.27", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tonic", + "tonic-health", + "tracing", + "uuid", +] + +[[package]] +name = "creditservice-proto" +version = "0.1.0" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tonic-build", +] + +[[package]] +name = "creditservice-types" +version = "0.1.0" +dependencies = [ + "chrono", + "rust_decimal", + "serde", + "thiserror 1.0.69", + "uuid", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flaredb-client" +version = "0.1.0" +dependencies = [ + "clap", + "flaredb-proto", + "prost", + "tokio", + "tonic", +] + +[[package]] +name = "flaredb-proto" +version = "0.1.0" +dependencies = [ + "prost", + "protoc-bin-vendored", + "tonic", + "tonic-build", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "glob-match" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985c9503b412198aa4197559e9a318524ebc4519c229bfa05a535828c950b9d" + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.12.1", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.0", + "indexmap 2.12.1", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.4.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.0", + "http-body 1.0.1", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2 0.4.12", + "http 1.4.0", + "http-body 1.0.1", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http 1.4.0", + "hyper 1.8.1", + "hyper-util", + "rustls 0.23.35", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower-service", + "webpki-roots 1.0.4", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper 1.8.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "hyper 1.8.1", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.1", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iam-api" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "iam-audit", + "iam-authn", + "iam-authz", + "iam-store", + "iam-types", + "prost", + "protoc-bin-vendored", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "tokio", + "tonic", + "tonic-build", + "tracing", + "uuid", +] + +[[package]] +name = "iam-audit" +version = "0.1.0" +dependencies = [ + "async-trait", + "chrono", + "iam-types", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "iam-authn" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "hmac", + "iam-types", + "jsonwebtoken", + "rand 0.8.5", + "reqwest 0.12.26", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "iam-authz" +version = "0.1.0" +dependencies = [ + "async-trait", + "dashmap", + "glob-match", + "iam-store", + "iam-types", + "ipnetwork", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "iam-store" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "chainfire-client", + "flaredb-client", + "iam-types", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "iam-types" +version = "0.1.0" +dependencies = [ + "chrono", + "serde", + "serde_json", + "thiserror 1.0.69", + "uuid", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "ipnetwork" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" +dependencies = [ + "serde", +] + +[[package]] +name = "iri-string" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee5b5339afb4c41626dde77b7a611bd4f2c202b897852b4bcf5d03eddc61010" + +[[package]] +name = "js-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "jsonwebtoken" +version = "9.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +dependencies = [ + "base64 0.22.1", + "js-sys", + "pem", + "ring", + "serde", + "serde_json", + "simple_asn1", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.178" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" +dependencies = [ + "base64 0.22.1", + "serde", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap 2.12.1", +] + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.111", +] + +[[package]] +name = "proc-macro-crate" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" +dependencies = [ + "toml_edit 0.23.4", +] + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.111", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.35", + "socket2 0.6.1", + "thiserror 2.0.17", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls 0.23.35", + "rustls-pki-types", + "slab", + "thiserror 2.0.17", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.1", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.10.0", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-rustls 0.24.2", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-rustls 0.24.1", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 0.25.4", + "winreg", +] + +[[package]] +name = "reqwest" +version = "0.12.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b4c14b2d9afca6a60277086b0cc6a6ae0b568f6f7916c943a8cdc79f8be240f" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-rustls 0.27.7", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.35", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 1.0.2", + "tokio", + "tokio-rustls 0.26.4", + "tower 0.5.2", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 1.0.4", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rkyv" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "rust_decimal" +version = "1.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.5", + "rkyv", + "serde", + "serde_json", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags 2.10.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + +[[package]] +name = "rustls" +version = "0.23.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki 0.103.8", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62049b2877bf12821e8f9ad256ee38fdc31db7387ec2d3b3f403024de2034aea" + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" +dependencies = [ + "itoa", + "serde", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +dependencies = [ + "libc", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "simple_asn1" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror 2.0.17", + "time", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl 2.0.17", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" + +[[package]] +name = "time-macros" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.1", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls 0.23.35", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_datetime" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.12.1", + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_edit" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7211ff1b8f0d3adae1663b7da9ffe396eabe1ca25f0b0bee42b0da29a9ddce93" +dependencies = [ + "indexmap 2.12.1", + "toml_datetime 0.7.0", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.0.6+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2 0.4.12", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "rustls-native-certs", + "rustls-pemfile 2.2.0", + "socket2 0.5.10", + "tokio", + "tokio-rustls 0.26.4", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "tonic-health" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1eaf34ddb812120f5c601162d5429933c9b527d901ab0e7f930d3147e33a09b2" +dependencies = [ + "async-stream", + "prost", + "tokio", + "tokio-stream", + "tonic", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 1.0.2", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags 2.10.0", + "bytes", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "iri-string", + "pin-project-lite", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.111", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "webpki-roots" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +dependencies = [ + "memchr", +] + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] diff --git a/apigateway/Cargo.toml b/apigateway/Cargo.toml new file mode 100644 index 0000000..b7ee2e6 --- /dev/null +++ b/apigateway/Cargo.toml @@ -0,0 +1,55 @@ +[workspace] +resolver = "2" +members = [ + "crates/apigateway-api", + "crates/apigateway-server", +] + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +rust-version = "1.75" +authors = ["PlasmaCloud Contributors"] +repository = "https://github.com/yourorg/plasmacloud" + +[workspace.dependencies] +# Internal crates +apigateway-api = { path = "crates/apigateway-api" } +apigateway-server = { path = "crates/apigateway-server" } + +# Async runtime +tokio = { version = "1.40", features = ["full"] } + +# HTTP server +axum = "0.7" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } + +# Serialization +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +toml = "0.8" + +# gRPC +tonic = "0.12" +tonic-build = "0.12" +prost = "0.13" +prost-types = "0.13" +protoc-bin-vendored = "3.2" + +# CLI +clap = { version = "4", features = ["derive", "env"] } + +# Logging +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +# Utils +async-trait = "0.1" +uuid = { version = "1", features = ["v4"] } + +[workspace.lints.rust] +unsafe_code = "deny" + +[workspace.lints.clippy] +all = "warn" diff --git a/apigateway/crates/apigateway-api/Cargo.toml b/apigateway/crates/apigateway-api/Cargo.toml new file mode 100644 index 0000000..64f3e6e --- /dev/null +++ b/apigateway/crates/apigateway-api/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "apigateway-api" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +description = "API Gateway gRPC protocol definitions" + +[dependencies] +tonic = { workspace = true } +prost = { workspace = true } +prost-types = { workspace = true } + +[build-dependencies] +tonic-build = { workspace = true } +protoc-bin-vendored = { workspace = true } + +[lib] +path = "src/lib.rs" diff --git a/apigateway/crates/apigateway-api/build.rs b/apigateway/crates/apigateway-api/build.rs new file mode 100644 index 0000000..115555d --- /dev/null +++ b/apigateway/crates/apigateway-api/build.rs @@ -0,0 +1,9 @@ +fn main() -> Result<(), Box> { + let protoc = protoc_bin_vendored::protoc_bin_path()?; + std::env::set_var("PROTOC", protoc); + tonic_build::configure() + .build_server(true) + .build_client(true) + .compile_protos(&["proto/apigateway.proto"], &["proto"])?; + Ok(()) +} diff --git a/apigateway/crates/apigateway-api/proto/apigateway.proto b/apigateway/crates/apigateway-api/proto/apigateway.proto new file mode 100644 index 0000000..f623cfe --- /dev/null +++ b/apigateway/crates/apigateway-api/proto/apigateway.proto @@ -0,0 +1,87 @@ +syntax = "proto3"; + +package apigateway.v1; + +// ============================================================================ +// Gateway Auth Service +// ============================================================================ + +service GatewayAuthService { + rpc Authorize(AuthorizeRequest) returns (AuthorizeResponse); +} + +message Subject { + string subject_id = 1; + string org_id = 2; + string project_id = 3; + repeated string roles = 4; + repeated string scopes = 5; +} + +message AuthorizeRequest { + string request_id = 1; + string token = 2; + string method = 3; + string path = 4; + string raw_query = 5; + map headers = 6; + string client_ip = 7; + string route_name = 8; +} + +message AuthorizeResponse { + bool allow = 1; + string reason = 2; + Subject subject = 3; + map headers = 4; + uint32 ttl_seconds = 5; +} + +// ============================================================================ +// Gateway Credit Service +// ============================================================================ + +service GatewayCreditService { + rpc Reserve(CreditReserveRequest) returns (CreditReserveResponse); + rpc Commit(CreditCommitRequest) returns (CreditCommitResponse); + rpc Rollback(CreditRollbackRequest) returns (CreditRollbackResponse); +} + +message CreditReserveRequest { + string request_id = 1; + string subject_id = 2; + string org_id = 3; + string project_id = 4; + string route_name = 5; + string method = 6; + string path = 7; + string raw_query = 8; + uint64 units = 9; + map attributes = 10; +} + +message CreditReserveResponse { + bool allow = 1; + string reservation_id = 2; + string reason = 3; + uint64 remaining = 4; +} + +message CreditCommitRequest { + string reservation_id = 1; + uint64 units = 2; +} + +message CreditCommitResponse { + bool success = 1; + string reason = 2; +} + +message CreditRollbackRequest { + string reservation_id = 1; +} + +message CreditRollbackResponse { + bool success = 1; + string reason = 2; +} diff --git a/apigateway/crates/apigateway-api/src/lib.rs b/apigateway/crates/apigateway-api/src/lib.rs new file mode 100644 index 0000000..f5bbab7 --- /dev/null +++ b/apigateway/crates/apigateway-api/src/lib.rs @@ -0,0 +1,10 @@ +//! API Gateway gRPC protocol definitions + +pub mod proto { + tonic::include_proto!("apigateway.v1"); +} + +pub use proto::gateway_auth_service_client::GatewayAuthServiceClient; +pub use proto::gateway_auth_service_server::{GatewayAuthService, GatewayAuthServiceServer}; +pub use proto::gateway_credit_service_client::GatewayCreditServiceClient; +pub use proto::gateway_credit_service_server::{GatewayCreditService, GatewayCreditServiceServer}; diff --git a/apigateway/crates/apigateway-server/Cargo.toml b/apigateway/crates/apigateway-server/Cargo.toml new file mode 100644 index 0000000..fa0328e --- /dev/null +++ b/apigateway/crates/apigateway-server/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "apigateway-server" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +description = "HTTP API gateway (scaffold)" + +[[bin]] +name = "apigateway-server" +path = "src/main.rs" + +[dependencies] +apigateway-api = { workspace = true } +axum = { workspace = true } +clap = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tonic = { workspace = true } +tokio = { workspace = true } +toml = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +async-trait = { workspace = true } +uuid = { workspace = true } +futures-core = "0.3" +bytes = "1" + +[dev-dependencies] +iam-api = { path = "../../../iam/crates/iam-api" } +iam-authn = { path = "../../../iam/crates/iam-authn" } +iam-authz = { path = "../../../iam/crates/iam-authz" } +iam-store = { path = "../../../iam/crates/iam-store" } +iam-types = { path = "../../../iam/crates/iam-types" } +creditservice-api = { path = "../../../creditservice/crates/creditservice-api" } +creditservice-types = { path = "../../../creditservice/crates/creditservice-types" } +tokio-stream = "0.1" diff --git a/apigateway/crates/apigateway-server/src/main.rs b/apigateway/crates/apigateway-server/src/main.rs new file mode 100644 index 0000000..9c11a21 --- /dev/null +++ b/apigateway/crates/apigateway-server/src/main.rs @@ -0,0 +1,1482 @@ +use std::collections::HashMap; +use std::io; +use std::net::SocketAddr; +use std::pin::Pin; +use std::path::PathBuf; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use apigateway_api::proto::{ + AuthorizeRequest, CreditCommitRequest, CreditReserveRequest, CreditRollbackRequest, +}; +use apigateway_api::{GatewayAuthServiceClient, GatewayCreditServiceClient}; +use axum::{ + body::{to_bytes, Body}, + extract::State, + http::{HeaderMap, Request, StatusCode, Uri}, + response::Response, + routing::{any, get}, + Json, Router, +}; +use clap::Parser; +use bytes::Bytes; +use futures_core::Stream; +use reqwest::{Client, Url}; +use serde::{Deserialize, Serialize}; +use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity}; +use tonic::Request as TonicRequest; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; +use uuid::Uuid; + +const DEFAULT_REQUEST_ID_HEADER: &str = "x-request-id"; +const DEFAULT_AUTH_TIMEOUT_MS: u64 = 500; +const DEFAULT_CREDIT_TIMEOUT_MS: u64 = 500; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum PolicyMode { + Disabled, + Optional, + Required, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum CommitPolicy { + Success, + Always, + Never, +} + +fn default_policy_mode() -> PolicyMode { + PolicyMode::Required +} + +fn default_commit_policy() -> CommitPolicy { + CommitPolicy::Success +} + +fn default_credit_units() -> u64 { + 1 +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct AuthProviderConfig { + name: String, + #[serde(rename = "type")] + provider_type: String, + endpoint: String, + #[serde(default)] + timeout_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CreditProviderConfig { + name: String, + #[serde(rename = "type")] + provider_type: String, + endpoint: String, + #[serde(default)] + timeout_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct RouteAuthConfig { + provider: String, + #[serde(default = "default_policy_mode")] + mode: PolicyMode, + #[serde(default)] + fail_open: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct RouteCreditConfig { + provider: String, + #[serde(default = "default_policy_mode")] + mode: PolicyMode, + #[serde(default = "default_credit_units")] + units: u64, + #[serde(default)] + fail_open: bool, + #[serde(default = "default_commit_policy")] + commit_on: CommitPolicy, + #[serde(default)] + attributes: HashMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct RouteConfig { + name: String, + path_prefix: String, + upstream: String, + #[serde(default)] + strip_prefix: bool, + #[serde(default)] + auth: Option, + #[serde(default)] + credit: Option, +} + +#[derive(Clone)] +struct Route { + config: RouteConfig, + upstream: Url, + upstream_base_path: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ServerConfig { + #[serde(default = "default_http_addr")] + http_addr: SocketAddr, + #[serde(default = "default_log_level")] + log_level: String, + #[serde(default = "default_max_body_bytes")] + max_body_bytes: usize, + #[serde(default)] + auth_providers: Vec, + #[serde(default)] + credit_providers: Vec, + #[serde(default)] + routes: Vec, +} + +impl Default for ServerConfig { + fn default() -> Self { + Self { + http_addr: default_http_addr(), + log_level: default_log_level(), + max_body_bytes: default_max_body_bytes(), + auth_providers: Vec::new(), + credit_providers: Vec::new(), + routes: Vec::new(), + } + } +} + +#[derive(Debug, Parser)] +#[command(author, version, about)] +struct Args { + /// Configuration file path + #[arg(short, long, default_value = "apigateway.toml")] + config: PathBuf, + + /// HTTP listen address (overrides config) + #[arg(long)] + http_addr: Option, + + /// Log level (overrides config) + #[arg(short, long)] + log_level: Option, +} + +#[derive(Clone)] +struct ServerState { + routes: Vec, + client: Client, + max_body_bytes: usize, + auth_providers: HashMap, + credit_providers: HashMap, +} + +#[derive(Clone)] +struct GrpcAuthProvider { + channel: Channel, + timeout: Duration, +} + +#[derive(Clone)] +enum AuthProvider { + Grpc(GrpcAuthProvider), +} + +#[derive(Clone)] +struct GrpcCreditProvider { + channel: Channel, + timeout: Duration, +} + +#[derive(Clone)] +enum CreditProvider { + Grpc(GrpcCreditProvider), +} + +#[derive(Clone, Debug)] +struct SubjectInfo { + subject_id: String, + org_id: String, + project_id: String, + roles: Vec, + scopes: Vec, +} + +#[derive(Clone, Debug)] +struct AuthDecision { + allow: bool, + subject: Option, + headers: HashMap, + reason: Option, +} + +#[derive(Clone, Debug)] +struct CreditDecision { + allow: bool, + reservation_id: String, + reason: Option, +} + +#[derive(Clone, Debug, Default)] +struct AuthOutcome { + subject: Option, + headers: HashMap, +} + +#[derive(Clone, Debug)] +struct CreditReservation { + provider: String, + reservation_id: String, +} + +struct CreditFinalizeState { + state: Arc, + route: Route, + reservation: Option, + status: reqwest::StatusCode, +} + +impl CreditFinalizeState { + fn spawn_success(self) { + tokio::spawn(async move { + finalize_credit(&self.state, &self.route, self.reservation, self.status).await; + }); + } + + fn spawn_abort(self) { + tokio::spawn(async move { + finalize_credit_abort(&self.state, &self.route, self.reservation).await; + }); + } +} + +struct CreditFinalizeStream { + bytes: Option, + finalize: Option, + completed: bool, +} + +impl CreditFinalizeStream { + fn new(bytes: Bytes, finalize: CreditFinalizeState) -> Self { + Self { + bytes: Some(bytes), + finalize: Some(finalize), + completed: false, + } + } + + fn finalize_success(&mut self) { + if self.completed { + return; + } + self.completed = true; + if let Some(finalize) = self.finalize.take() { + finalize.spawn_success(); + } + } + + fn finalize_abort(&mut self) { + if self.completed { + return; + } + self.completed = true; + if let Some(finalize) = self.finalize.take() { + finalize.spawn_abort(); + } + } +} + +impl Stream for CreditFinalizeStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + if let Some(bytes) = self.bytes.take() { + return Poll::Ready(Some(Ok(bytes))); + } + + self.finalize_success(); + Poll::Ready(None) + } +} + +impl Drop for CreditFinalizeStream { + fn drop(&mut self) { + if !self.completed { + self.finalize_abort(); + } + } +} + +#[derive(Clone, Debug)] +struct RequestContext { + request_id: String, + method: String, + path: String, + raw_query: String, + headers: HashMap, + client_ip: String, + route_name: String, +} + +fn default_http_addr() -> SocketAddr { + "127.0.0.1:8080" + .parse() + .expect("invalid default HTTP address") +} + +fn default_log_level() -> String { + "info".to_string() +} + +fn default_max_body_bytes() -> usize { + 16 * 1024 * 1024 +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let args = Args::parse(); + + let mut used_default_config = false; + let mut config = if args.config.exists() { + let contents = tokio::fs::read_to_string(&args.config).await?; + toml::from_str(&contents)? + } else { + used_default_config = true; + ServerConfig::default() + }; + + if let Some(http_addr) = args.http_addr { + config.http_addr = http_addr.parse()?; + } + if let Some(log_level) = args.log_level { + config.log_level = log_level; + } + + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&config.log_level)), + ) + .init(); + + if used_default_config { + info!("Config file not found: {}, using defaults", args.config.display()); + } + + let routes = build_routes(config.routes)?; + let auth_providers = build_auth_providers(config.auth_providers).await?; + let credit_providers = build_credit_providers(config.credit_providers).await?; + + info!("Starting API gateway"); + info!(" HTTP: {}", config.http_addr); + info!(" Max body bytes: {}", config.max_body_bytes); + + if !routes.is_empty() { + info!("Configured {} routes", routes.len()); + } else { + warn!("No routes configured; proxy will return 404s"); + } + + if auth_providers.is_empty() { + warn!("No auth providers configured"); + } + if credit_providers.is_empty() { + warn!("No credit providers configured"); + } + + let state = Arc::new(ServerState { + routes, + client: Client::new(), + max_body_bytes: config.max_body_bytes, + auth_providers, + credit_providers, + }); + + let app = Router::new() + .route("/", get(index)) + .route("/health", get(health)) + .route("/routes", get(list_routes)) + .route("/*path", any(proxy)) + .with_state(state); + + let listener = tokio::net::TcpListener::bind(config.http_addr).await?; + axum::serve(listener, app).await?; + + Ok(()) +} + +async fn index() -> &'static str { + "apigateway-server running" +} + +async fn health() -> Json { + Json(serde_json::json!({"status": "ok"})) +} + +async fn list_routes(State(state): State>) -> Json> { + Json(state.routes.iter().map(|route| route.config.clone()).collect()) +} + +async fn proxy( + State(state): State>, + request: Request, +) -> Result, StatusCode> { + let path = request.uri().path(); + let route = match_route(&state.routes, path) + .ok_or(StatusCode::NOT_FOUND)? + .clone(); + let request_id = extract_request_id(request.headers()); + + let context = RequestContext { + request_id: request_id.clone(), + method: request.method().to_string(), + path: request.uri().path().to_string(), + raw_query: request.uri().query().unwrap_or("").to_string(), + headers: headers_to_map(request.headers()), + client_ip: extract_client_ip(request.headers()), + route_name: route.config.name.clone(), + }; + + let auth_token = request + .headers() + .get(axum::http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .map(|value| value.to_string()); + + let auth_outcome = enforce_auth(&state, &route, &context, auth_token).await?; + let credit_reservation = + enforce_credit(&state, &route, &context, auth_outcome.subject.as_ref()).await?; + + let target_url = build_upstream_url(&route, request.uri())?; + + let mut builder = state.client.request(request.method().clone(), target_url); + for (name, value) in request.headers().iter() { + if name == axum::http::header::HOST || name == axum::http::header::CONNECTION { + continue; + } + builder = builder.header(name, value); + } + + builder = builder.header(DEFAULT_REQUEST_ID_HEADER, request_id.clone()); + builder = apply_auth_headers(builder, &auth_outcome); + + let body_bytes = to_bytes(request.into_body(), state.max_body_bytes) + .await + .map_err(|_| StatusCode::PAYLOAD_TOO_LARGE)?; + + let response = match builder.body(body_bytes).send().await { + Ok(response) => response, + Err(_) => { + finalize_credit_abort(&state, &route, credit_reservation).await; + return Err(StatusCode::BAD_GATEWAY); + } + }; + + let status = response.status(); + + let mut response_builder = Response::builder().status(status); + let headers = response_builder + .headers_mut() + .ok_or(StatusCode::BAD_GATEWAY)?; + + for (name, value) in response.headers().iter() { + if name == axum::http::header::CONNECTION { + continue; + } + headers.insert(name, value.clone()); + } + + let bytes = match response.bytes().await { + Ok(bytes) => bytes, + Err(_) => { + finalize_credit_abort(&state, &route, credit_reservation).await; + return Err(StatusCode::BAD_GATEWAY); + } + }; + + let finalize = CreditFinalizeState { + state: Arc::clone(&state), + route, + reservation: credit_reservation, + status, + }; + + response_builder + .body(Body::from_stream(CreditFinalizeStream::new(bytes, finalize))) + .map_err(|_| StatusCode::BAD_GATEWAY) +} + +async fn enforce_auth( + state: &ServerState, + route: &Route, + context: &RequestContext, + token: Option, +) -> Result { + let Some(auth_cfg) = &route.config.auth else { + return Ok(AuthOutcome::default()); + }; + + if auth_cfg.mode == PolicyMode::Disabled { + return Ok(AuthOutcome::default()); + } + + let decision = authorize_request(state, auth_cfg, context, token).await; + apply_auth_mode(auth_cfg.mode, auth_cfg.fail_open, decision) +} + +fn apply_auth_mode( + mode: PolicyMode, + fail_open: bool, + decision: Result, +) -> Result { + match mode { + PolicyMode::Disabled => Ok(AuthOutcome::default()), + PolicyMode::Optional => match decision { + Ok(decision) if decision.allow => Ok(AuthOutcome { + subject: decision.subject, + headers: decision.headers, + }), + Ok(decision) => { + if let Some(reason) = decision.reason { + warn!("Auth denied (optional mode): {}", reason); + } + Ok(AuthOutcome::default()) + } + Err(err) => { + warn!("Auth provider error (optional mode): {}", err); + Ok(AuthOutcome::default()) + } + }, + PolicyMode::Required => match decision { + Ok(decision) if decision.allow => Ok(AuthOutcome { + subject: decision.subject, + headers: decision.headers, + }), + Ok(decision) => { + if let Some(reason) = decision.reason { + warn!("Auth denied (required mode): {}", reason); + } + Err(StatusCode::FORBIDDEN) + } + Err(err) => { + warn!("Auth provider error (required mode): {}", err); + if fail_open { + Ok(AuthOutcome::default()) + } else { + Err(StatusCode::BAD_GATEWAY) + } + } + }, + } +} + +async fn enforce_credit( + state: &ServerState, + route: &Route, + context: &RequestContext, + subject: Option<&SubjectInfo>, +) -> Result, StatusCode> { + let Some(credit_cfg) = &route.config.credit else { + return Ok(None); + }; + + if credit_cfg.mode == PolicyMode::Disabled { + return Ok(None); + } + + let decision = reserve_credit(state, credit_cfg, context, subject).await; + apply_credit_mode(credit_cfg.mode, credit_cfg.fail_open, decision) + .map(|decision| { + decision.map(|decision| CreditReservation { + provider: credit_cfg.provider.clone(), + reservation_id: decision.reservation_id, + }) + }) +} + +fn apply_credit_mode( + mode: PolicyMode, + fail_open: bool, + decision: Result, +) -> Result, StatusCode> { + match mode { + PolicyMode::Disabled => Ok(None), + PolicyMode::Optional => match decision { + Ok(decision) if decision.allow => Ok(Some(decision)), + Ok(decision) => { + if let Some(reason) = decision.reason { + warn!("Credit denied (optional mode): {}", reason); + } + Ok(None) + } + Err(err) => { + warn!("Credit provider error (optional mode): {}", err); + Ok(None) + } + }, + PolicyMode::Required => match decision { + Ok(decision) if decision.allow => Ok(Some(decision)), + Ok(decision) => { + if let Some(reason) = decision.reason { + warn!("Credit denied (required mode): {}", reason); + } + Err(StatusCode::PAYMENT_REQUIRED) + } + Err(err) => { + warn!("Credit provider error (required mode): {}", err); + if fail_open { + Ok(None) + } else { + Err(StatusCode::BAD_GATEWAY) + } + } + }, + } +} + +async fn authorize_request( + state: &ServerState, + auth_cfg: &RouteAuthConfig, + context: &RequestContext, + token: Option, +) -> Result { + let provider = state + .auth_providers + .get(&auth_cfg.provider) + .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + + match provider { + AuthProvider::Grpc(provider) => { + let mut client = GatewayAuthServiceClient::new(provider.channel.clone()); + let mut request = TonicRequest::new(AuthorizeRequest { + request_id: context.request_id.clone(), + token: token.unwrap_or_default(), + method: context.method.clone(), + path: context.path.clone(), + raw_query: context.raw_query.clone(), + headers: context.headers.clone(), + client_ip: context.client_ip.clone(), + route_name: context.route_name.clone(), + }); + request.set_timeout(provider.timeout); + + let response = client + .authorize(request) + .await + .map_err(|_| StatusCode::BAD_GATEWAY)? + .into_inner(); + + let subject = response.subject.map(|subject| SubjectInfo { + subject_id: subject.subject_id, + org_id: subject.org_id, + project_id: subject.project_id, + roles: subject.roles, + scopes: subject.scopes, + }); + + Ok(AuthDecision { + allow: response.allow, + subject, + headers: response.headers, + reason: if response.reason.is_empty() { + None + } else { + Some(response.reason) + }, + }) + } + } +} + +async fn reserve_credit( + state: &ServerState, + credit_cfg: &RouteCreditConfig, + context: &RequestContext, + subject: Option<&SubjectInfo>, +) -> Result { + let provider = state + .credit_providers + .get(&credit_cfg.provider) + .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + + let (subject_id, org_id, project_id) = subject + .map(|subject| { + ( + subject.subject_id.clone(), + subject.org_id.clone(), + subject.project_id.clone(), + ) + }) + .unwrap_or_default(); + + match provider { + CreditProvider::Grpc(provider) => { + let mut client = GatewayCreditServiceClient::new(provider.channel.clone()); + let mut request = TonicRequest::new(CreditReserveRequest { + request_id: context.request_id.clone(), + subject_id, + org_id, + project_id, + route_name: context.route_name.clone(), + method: context.method.clone(), + path: context.path.clone(), + raw_query: context.raw_query.clone(), + units: credit_cfg.units, + attributes: credit_cfg.attributes.clone(), + }); + request.set_timeout(provider.timeout); + + let response = client + .reserve(request) + .await + .map_err(|_| StatusCode::BAD_GATEWAY)? + .into_inner(); + + Ok(CreditDecision { + allow: response.allow, + reservation_id: response.reservation_id, + reason: if response.reason.is_empty() { + None + } else { + Some(response.reason) + }, + }) + } + } +} + +async fn finalize_credit( + state: &ServerState, + route: &Route, + reservation: Option, + status: reqwest::StatusCode, +) { + let Some(credit_cfg) = &route.config.credit else { + return; + }; + let Some(reservation) = reservation else { + return; + }; + + match credit_cfg.commit_on { + CommitPolicy::Never => return, + CommitPolicy::Always => { + if let Err(err) = commit_credit(state, credit_cfg, &reservation).await { + warn!("Failed to commit credit reservation {}: {}", reservation.reservation_id, err); + } + } + CommitPolicy::Success => { + if status.is_success() || status.is_redirection() { + if let Err(err) = commit_credit(state, credit_cfg, &reservation).await { + warn!("Failed to commit credit reservation {}: {}", reservation.reservation_id, err); + } + } else if let Err(err) = rollback_credit(state, credit_cfg, &reservation).await { + warn!( + "Failed to rollback credit reservation {}: {}", + reservation.reservation_id, err + ); + } + } + } +} + +async fn finalize_credit_abort( + state: &ServerState, + route: &Route, + reservation: Option, +) { + let Some(credit_cfg) = &route.config.credit else { + return; + }; + let Some(reservation) = reservation else { + return; + }; + + if credit_cfg.commit_on == CommitPolicy::Never { + return; + } + + if let Err(err) = rollback_credit(state, credit_cfg, &reservation).await { + warn!( + "Failed to rollback credit reservation {} after delivery failure: {}", + reservation.reservation_id, err + ); + } +} + +async fn commit_credit( + state: &ServerState, + credit_cfg: &RouteCreditConfig, + reservation: &CreditReservation, +) -> Result<(), StatusCode> { + let provider = state + .credit_providers + .get(&reservation.provider) + .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + + match provider { + CreditProvider::Grpc(provider) => { + let mut client = GatewayCreditServiceClient::new(provider.channel.clone()); + let mut request = TonicRequest::new(CreditCommitRequest { + reservation_id: reservation.reservation_id.clone(), + units: credit_cfg.units, + }); + request.set_timeout(provider.timeout); + let response = client + .commit(request) + .await + .map_err(|_| StatusCode::BAD_GATEWAY)? + .into_inner(); + if response.success { + Ok(()) + } else { + Err(StatusCode::BAD_GATEWAY) + } + } + } +} + +async fn rollback_credit( + state: &ServerState, + _credit_cfg: &RouteCreditConfig, + reservation: &CreditReservation, +) -> Result<(), StatusCode> { + let provider = state + .credit_providers + .get(&reservation.provider) + .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + + match provider { + CreditProvider::Grpc(provider) => { + let mut client = GatewayCreditServiceClient::new(provider.channel.clone()); + let mut request = TonicRequest::new(CreditRollbackRequest { + reservation_id: reservation.reservation_id.clone(), + }); + request.set_timeout(provider.timeout); + let response = client + .rollback(request) + .await + .map_err(|_| StatusCode::BAD_GATEWAY)? + .into_inner(); + if response.success { + Ok(()) + } else { + Err(StatusCode::BAD_GATEWAY) + } + } + } +} + +fn apply_auth_headers(mut builder: reqwest::RequestBuilder, outcome: &AuthOutcome) -> reqwest::RequestBuilder { + for (key, value) in &outcome.headers { + builder = builder.header(key, value); + } + + if let Some(subject) = &outcome.subject { + builder = builder + .header("x-subject-id", &subject.subject_id) + .header("x-org-id", &subject.org_id) + .header("x-project-id", &subject.project_id); + if !subject.roles.is_empty() { + builder = builder.header("x-roles", subject.roles.join(",")); + } + if !subject.scopes.is_empty() { + builder = builder.header("x-scopes", subject.scopes.join(",")); + } + } + + builder +} + +async fn build_auth_providers( + configs: Vec, +) -> Result, Box> { + let mut providers = HashMap::new(); + + for config in configs { + let provider_type = normalize_name(&config.provider_type); + if providers.contains_key(&config.name) { + return Err(config_error(format!( + "duplicate auth provider name {}", + config.name + )) + .into()); + } + + match provider_type.as_str() { + "grpc" => { + let endpoint = Endpoint::from_shared(config.endpoint.clone())? + .connect_timeout(Duration::from_millis(config.timeout_ms.unwrap_or(DEFAULT_AUTH_TIMEOUT_MS))) + .timeout(Duration::from_millis(config.timeout_ms.unwrap_or(DEFAULT_AUTH_TIMEOUT_MS))); + let channel = endpoint.connect().await?; + let timeout = Duration::from_millis(config.timeout_ms.unwrap_or(DEFAULT_AUTH_TIMEOUT_MS)); + providers.insert( + config.name.clone(), + AuthProvider::Grpc(GrpcAuthProvider { + channel, + timeout, + }), + ); + } + _ => { + return Err(config_error(format!( + "unsupported auth provider type {}", + config.provider_type + )) + .into()); + } + } + } + + Ok(providers) +} + +async fn build_credit_providers( + configs: Vec, +) -> Result, Box> { + let mut providers = HashMap::new(); + + for config in configs { + let provider_type = normalize_name(&config.provider_type); + if providers.contains_key(&config.name) { + return Err(config_error(format!( + "duplicate credit provider name {}", + config.name + )) + .into()); + } + + match provider_type.as_str() { + "grpc" => { + let endpoint = Endpoint::from_shared(config.endpoint.clone())? + .connect_timeout(Duration::from_millis( + config + .timeout_ms + .unwrap_or(DEFAULT_CREDIT_TIMEOUT_MS), + )) + .timeout(Duration::from_millis( + config + .timeout_ms + .unwrap_or(DEFAULT_CREDIT_TIMEOUT_MS), + )); + + let channel = endpoint.connect().await?; + let timeout = Duration::from_millis( + config + .timeout_ms + .unwrap_or(DEFAULT_CREDIT_TIMEOUT_MS), + ); + providers.insert( + config.name.clone(), + CreditProvider::Grpc(GrpcCreditProvider { + channel, + timeout, + }), + ); + } + _ => { + return Err(config_error(format!( + "unsupported credit provider type {}", + config.provider_type + )) + .into()); + } + } + } + + Ok(providers) +} + +fn build_routes(configs: Vec) -> Result, Box> { + let mut routes = Vec::new(); + + for mut config in configs { + if config.name.trim().is_empty() { + return Err(config_error("route name is required").into()); + } + let path_prefix = normalize_path_prefix(&config.path_prefix); + config.path_prefix = path_prefix; + + let upstream = Url::parse(&config.upstream)?; + if upstream.scheme() != "http" && upstream.scheme() != "https" { + return Err(config_error(format!( + "route {} upstream must be http or https", + config.name + )) + .into()); + } + if upstream.host_str().is_none() { + return Err(config_error(format!( + "route {} upstream must include host", + config.name + )) + .into()); + } + let upstream_base_path = normalize_upstream_base_path(upstream.path()); + + routes.push(Route { + config, + upstream, + upstream_base_path, + }); + } + + routes.sort_by(|a, b| b.config.path_prefix.len().cmp(&a.config.path_prefix.len())); + Ok(routes) +} + +fn config_error(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, message.into()) +} + +fn normalize_name(value: &str) -> String { + value.trim().to_lowercase().replace('-', "_") +} + +fn extract_request_id(headers: &HeaderMap) -> String { + headers + .get(DEFAULT_REQUEST_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .map(|value| value.to_string()) + .unwrap_or_else(|| Uuid::new_v4().to_string()) +} + +fn extract_client_ip(headers: &HeaderMap) -> String { + headers + .get("x-forwarded-for") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(',').next()) + .map(|value| value.trim().to_string()) + .unwrap_or_default() +} + +fn headers_to_map(headers: &HeaderMap) -> HashMap { + let mut map: HashMap = HashMap::new(); + for (name, value) in headers.iter() { + if let Ok(value) = value.to_str() { + map.entry(name.as_str().to_string()) + .and_modify(|entry| { + entry.push(','); + entry.push_str(value); + }) + .or_insert_with(|| value.to_string()); + } + } + map +} + +fn normalize_path_prefix(prefix: &str) -> String { + let trimmed = prefix.trim(); + if trimmed.is_empty() { + return "/".to_string(); + } + + let mut normalized = if trimmed.starts_with('/') { + trimmed.to_string() + } else { + format!("/{}", trimmed) + }; + + if normalized.len() > 1 && normalized.ends_with('/') { + normalized.pop(); + } + + normalized +} + +fn normalize_upstream_base_path(path: &str) -> String { + let trimmed = path.trim(); + if trimmed.is_empty() || trimmed == "/" { + "/".to_string() + } else { + trimmed.trim_end_matches('/').to_string() + } +} + +fn match_route<'a>(routes: &'a [Route], path: &str) -> Option<&'a Route> { + routes + .iter() + .find(|route| path.starts_with(&route.config.path_prefix)) +} + +fn strip_prefix_path(path: &str, prefix: &str) -> String { + if prefix == "/" { + return path.to_string(); + } + + match path.strip_prefix(prefix) { + Some("") => "/".to_string(), + Some(stripped) => { + if stripped.starts_with('/') { + stripped.to_string() + } else { + format!("/{}", stripped) + } + } + None => path.to_string(), + } +} + +fn join_paths(base: &str, path: &str) -> String { + if base == "/" { + return path.to_string(); + } + if path == "/" { + let trimmed = base.trim_end_matches('/'); + return if trimmed.is_empty() { "/".to_string() } else { trimmed.to_string() }; + } + + format!( + "{}/{}", + base.trim_end_matches('/'), + path.trim_start_matches('/') + ) +} + +fn build_upstream_url(route: &Route, uri: &Uri) -> Result { + let path = if route.config.strip_prefix { + strip_prefix_path(uri.path(), &route.config.path_prefix) + } else { + uri.path().to_string() + }; + + let merged_path = join_paths(&route.upstream_base_path, &path); + let mut url = route.upstream.clone(); + url.set_path(&merged_path); + url.set_query(uri.query()); + + Ok(url) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::routing::get; + use creditservice_api::{ + CreditServiceImpl, CreditStorage, GatewayCreditServiceImpl, GatewayCreditServiceServer, + }; + use creditservice_types::Wallet; + use iam_api::{GatewayAuthServiceImpl, GatewayAuthServiceServer}; + use iam_authn::{InternalTokenConfig, InternalTokenService, SigningKey}; + use iam_authz::{PolicyCache, PolicyEvaluator}; + use iam_store::{Backend, BackendConfig, BindingStore, PrincipalStore, RoleStore, TokenStore}; + use iam_types::{Permission, PolicyBinding, Principal, PrincipalRef, Role, Scope}; + use tokio_stream::wrappers::TcpListenerStream; + use tonic::transport::Server; + use uuid::Uuid; + + fn route_config(name: &str, prefix: &str, upstream: &str, strip_prefix: bool) -> RouteConfig { + RouteConfig { + name: name.to_string(), + path_prefix: prefix.to_string(), + upstream: upstream.to_string(), + strip_prefix, + auth: None, + credit: None, + } + } + + fn auth_decision(allow: bool) -> AuthDecision { + AuthDecision { + allow, + subject: None, + headers: HashMap::new(), + reason: None, + } + } + + fn credit_decision(allow: bool) -> CreditDecision { + CreditDecision { + allow, + reservation_id: "resv".to_string(), + reason: None, + } + } + + async fn start_upstream() -> SocketAddr { + let app = Router::new().route("/v1/echo", get(|| async { "ok" })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind upstream"); + let addr = listener.local_addr().expect("upstream addr"); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("upstream serve"); + }); + addr + } + + async fn start_iam_gateway() -> (SocketAddr, String) { + let backend = Arc::new(Backend::new(BackendConfig::Memory).await.expect("iam backend")); + let principal_store = Arc::new(PrincipalStore::new(backend.clone())); + let role_store = Arc::new(RoleStore::new(backend.clone())); + let binding_store = Arc::new(BindingStore::new(backend.clone())); + let token_store = Arc::new(TokenStore::new(backend)); + + let signing_key = SigningKey::generate("gateway-test-key"); + let token_config = InternalTokenConfig::new(signing_key, "iam-gateway-test"); + let token_service = Arc::new(InternalTokenService::new(token_config)); + + let mut principal = Principal::new_user("user-1", "User One"); + principal.org_id = Some("org-1".into()); + principal.project_id = Some("proj-1".into()); + principal_store + .create(&principal) + .await + .expect("principal create"); + + let role = Role::new( + "GatewayReader", + Scope::project("proj-1", "org-1"), + vec![Permission::new("gateway:public:read", "*")], + ); + role_store.create(&role).await.expect("role create"); + let binding = PolicyBinding::new( + format!("binding-{}", Uuid::new_v4()), + PrincipalRef::new(principal.kind.clone(), principal.id.clone()), + role.to_ref(), + Scope::project("proj-1", "org-1"), + ); + binding_store + .create(&binding) + .await + .expect("binding create"); + + let issued = token_service + .issue(&principal, vec![], Scope::project("proj-1", "org-1"), None) + .await + .expect("issue token"); + + let cache = Arc::new(PolicyCache::default_config()); + let evaluator = Arc::new(PolicyEvaluator::new( + binding_store.clone(), + role_store.clone(), + cache, + )); + let gateway_auth = GatewayAuthServiceImpl::new( + token_service, + principal_store, + token_store, + evaluator, + ); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind iam"); + let addr = listener.local_addr().expect("iam addr"); + tokio::spawn(async move { + Server::builder() + .add_service(GatewayAuthServiceServer::new(gateway_auth)) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("iam gateway serve"); + }); + + (addr, issued.token) + } + + async fn start_credit_gateway() -> SocketAddr { + let storage = creditservice_api::InMemoryStorage::new(); + let wallet = Wallet::new("proj-1".into(), "org-1".into(), 100); + storage + .create_wallet(wallet) + .await + .expect("wallet create"); + + let credit_service = Arc::new(CreditServiceImpl::new(storage)); + let gateway_credit = GatewayCreditServiceImpl::new(credit_service); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind credit"); + let addr = listener.local_addr().expect("credit addr"); + tokio::spawn(async move { + Server::builder() + .add_service(GatewayCreditServiceServer::new(gateway_credit)) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .expect("credit gateway serve"); + }); + + addr + } + + #[test] + fn test_normalize_path_prefix() { + assert_eq!(normalize_path_prefix(""), "/"); + assert_eq!(normalize_path_prefix("/"), "/"); + assert_eq!(normalize_path_prefix("api"), "/api"); + assert_eq!(normalize_path_prefix("/api/"), "/api"); + } + + #[test] + fn test_strip_prefix_path() { + assert_eq!(strip_prefix_path("/api", "/api"), "/"); + assert_eq!(strip_prefix_path("/api/v1", "/api"), "/v1"); + assert_eq!(strip_prefix_path("/api/v1/", "/api"), "/v1/"); + assert_eq!(strip_prefix_path("/v1", "/"), "/v1"); + } + + #[test] + fn test_join_paths() { + assert_eq!(join_paths("/", "/v1"), "/v1"); + assert_eq!(join_paths("/api", "/v1"), "/api/v1"); + assert_eq!(join_paths("/api/", "/"), "/api"); + } + + #[test] + fn test_match_route_longest_prefix() { + let routes = build_routes(vec![ + route_config("api", "/api", "http://example.com", false), + route_config("api-v1", "/api/v1", "http://example.com", false), + ]) + .unwrap(); + + let matched = match_route(&routes, "/api/v1/users").unwrap(); + assert_eq!(matched.config.name, "api-v1"); + } + + #[test] + fn test_build_upstream_url_preserves_query() { + let routes = build_routes(vec![route_config( + "api", + "/api", + "http://example.com/base", + false, + )]) + .unwrap(); + let route = routes.first().unwrap(); + let uri: Uri = "/api/v1/users?debug=true".parse().unwrap(); + let url = build_upstream_url(route, &uri).unwrap(); + assert_eq!(url.as_str(), "http://example.com/base/api/v1/users?debug=true"); + } + + #[test] + fn test_build_upstream_url_strip_prefix() { + let routes = build_routes(vec![route_config( + "api", + "/api", + "http://example.com/base", + true, + )]) + .unwrap(); + let route = routes.first().unwrap(); + let uri: Uri = "/api/v1".parse().unwrap(); + let url = build_upstream_url(route, &uri).unwrap(); + assert_eq!(url.as_str(), "http://example.com/base/v1"); + } + + #[test] + fn test_apply_auth_mode_required() { + let decision = Ok(auth_decision(true)); + let outcome = apply_auth_mode(PolicyMode::Required, false, decision).unwrap(); + assert!(outcome.subject.is_none()); + + let decision = Ok(auth_decision(false)); + let err = apply_auth_mode(PolicyMode::Required, false, decision).unwrap_err(); + assert_eq!(err, StatusCode::FORBIDDEN); + } + + #[test] + fn test_apply_auth_mode_optional() { + let decision = Ok(auth_decision(false)); + let outcome = apply_auth_mode(PolicyMode::Optional, false, decision).unwrap(); + assert!(outcome.subject.is_none()); + + let outcome = apply_auth_mode(PolicyMode::Optional, false, Err(StatusCode::BAD_GATEWAY)).unwrap(); + assert!(outcome.subject.is_none()); + } + + #[test] + fn test_apply_credit_mode_required() { + let decision = Ok(credit_decision(true)); + let outcome = apply_credit_mode(PolicyMode::Required, false, decision).unwrap(); + assert!(outcome.is_some()); + + let decision = Ok(credit_decision(false)); + let err = apply_credit_mode(PolicyMode::Required, false, decision).unwrap_err(); + assert_eq!(err, StatusCode::PAYMENT_REQUIRED); + } + + #[test] + fn test_apply_credit_mode_optional() { + let decision = Ok(credit_decision(false)); + let outcome = apply_credit_mode(PolicyMode::Optional, false, decision).unwrap(); + assert!(outcome.is_none()); + + let outcome = apply_credit_mode(PolicyMode::Optional, false, Err(StatusCode::BAD_GATEWAY)).unwrap(); + assert!(outcome.is_none()); + } + + #[tokio::test] + async fn test_gateway_auth_and_credit_flow() { + let upstream_addr = start_upstream().await; + let (iam_addr, token) = start_iam_gateway().await; + let credit_addr = start_credit_gateway().await; + + let routes = build_routes(vec![RouteConfig { + name: "public".to_string(), + path_prefix: "/v1".to_string(), + upstream: format!("http://{}", upstream_addr), + strip_prefix: false, + auth: Some(RouteAuthConfig { + provider: "iam".to_string(), + mode: PolicyMode::Required, + fail_open: false, + }), + credit: Some(RouteCreditConfig { + provider: "credit".to_string(), + mode: PolicyMode::Required, + units: 1, + fail_open: false, + commit_on: CommitPolicy::Success, + attributes: HashMap::new(), + }), + }]) + .unwrap(); + + let auth_providers = build_auth_providers(vec![AuthProviderConfig { + name: "iam".to_string(), + provider_type: "grpc".to_string(), + endpoint: format!("http://{}", iam_addr), + timeout_ms: Some(1000), + }]) + .await + .unwrap(); + + let credit_providers = build_credit_providers(vec![CreditProviderConfig { + name: "credit".to_string(), + provider_type: "grpc".to_string(), + endpoint: format!("http://{}", credit_addr), + timeout_ms: Some(1000), + tls: None, + }]) + .await + .unwrap(); + + let state = Arc::new(ServerState { + routes, + client: Client::new(), + max_body_bytes: 1024 * 1024, + auth_providers, + credit_providers, + }); + + let request = Request::builder() + .method("GET") + .uri("/v1/echo") + .header(axum::http::header::AUTHORIZATION, token) + .body(Body::empty()) + .expect("request build"); + + let response = proxy(State(state), request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } +} diff --git a/chainfire/chainfire-client/examples/basic.rs b/chainfire/chainfire-client/examples/basic.rs new file mode 100644 index 0000000..b67d71a --- /dev/null +++ b/chainfire/chainfire-client/examples/basic.rs @@ -0,0 +1,15 @@ +use chainfire_client::Client; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Build a client with default retry/backoff. + let mut client = Client::builder("http://127.0.0.1:2379").build().await?; + + // Simple put/get roundtrip. + client.put_str("/example/key", "value").await?; + if let Some(val) = client.get_str("/example/key").await? { + println!("Got value: {}", val); + } + + Ok(()) +} diff --git a/chainfire/chainfire-client/src/metadata.rs b/chainfire/chainfire-client/src/metadata.rs new file mode 100644 index 0000000..8e1aad5 --- /dev/null +++ b/chainfire/chainfire-client/src/metadata.rs @@ -0,0 +1,380 @@ +//! Metadata-oriented KV facade for Chainfire (and test backends). +//! +//! This module exists to standardize how PhotonCloud services interact with +//! control-plane metadata: versioned reads, CAS, prefix scans, etc. + +use async_trait::async_trait; +use bytes::Bytes; +use std::collections::BTreeMap; +use std::sync::RwLock; +use thiserror::Error; +use tokio::sync::Mutex; + +use crate::{CasOutcome, Client as CfClient, ClientError as CfClientError}; + +#[derive(Debug, Error)] +pub enum MetadataError { + #[error("Connection error: {0}")] + Connection(String), + #[error("Backend error: {0}")] + Backend(String), + #[error("Conflict: expected version {expected}, actual {actual}")] + Conflict { expected: u64, actual: u64 }, + #[error("Not found")] + NotFound, + #[error("Serialization error: {0}")] + Serialization(String), +} + +pub type Result = std::result::Result; + +/// Key-value pair with version +#[derive(Debug, Clone)] +pub struct KvPair { + pub key: Bytes, + pub value: Bytes, + pub version: u64, +} + +/// Result of a CAS (Compare-And-Swap) operation +#[derive(Debug, Clone)] +pub enum CasResult { + /// CAS succeeded, returning the new version + Success(u64), + /// CAS failed due to version mismatch or not found + Conflict { expected: u64, actual: u64 }, + /// Key not found (when expected version > 0) + NotFound, +} + +#[async_trait] +pub trait MetadataClient: Send + Sync { + /// Get a value by key + async fn get(&self, key: &[u8]) -> Result>; + + /// Put a value (unconditional write) + async fn put(&self, key: &[u8], value: &[u8]) -> Result; + + /// Compare-and-swap write + /// - If expected_version is 0, only succeeds if key doesn't exist + /// - Otherwise, only succeeds if current version matches expected_version + async fn cas(&self, key: &[u8], expected_version: u64, value: &[u8]) -> Result; + + /// Delete a key + async fn delete(&self, key: &[u8]) -> Result; + + /// Scan keys with a prefix + async fn scan_prefix(&self, prefix: &[u8], limit: u32) -> Result>; + + /// Scan keys in a range [start, end) + async fn scan_range(&self, start: &[u8], end: &[u8], limit: u32) -> Result>; + + /// Scan all keys with a prefix (best-effort pagination using `scan_range`). + /// + /// This exists because `scan_prefix` is intentionally bounded by a `limit` but many + /// control-plane callers need "list everything under a prefix" semantics. + async fn scan_prefix_all(&self, prefix: &[u8]) -> Result> { + const PAGE_SIZE: u32 = 1024; + + let end = prefix_end(prefix); + if end.is_empty() { + // Prefix has no lexicographic successor (or is empty). Fall back to a single page. + return self.scan_prefix(prefix, PAGE_SIZE).await; + } + + let mut out = Vec::new(); + let mut start = prefix.to_vec(); + + loop { + let batch = self.scan_range(&start, &end, PAGE_SIZE).await?; + if batch.is_empty() { + break; + } + + let last_key = batch + .last() + .map(|kv| kv.key.clone()) + .unwrap_or_else(Bytes::new); + + out.extend(batch); + + let next = next_key_after(last_key.as_ref()); + if next <= start { + // Defensive: avoid infinite loops if the backend returns unsorted/duplicate keys. + break; + } + start = next; + } + + Ok(out) + } +} + +fn prefix_end(prefix: &[u8]) -> Vec { + let mut end = prefix.to_vec(); + for i in (0..end.len()).rev() { + if end[i] < 0xff { + end[i] += 1; + end.truncate(i + 1); + return end; + } + } + Vec::new() +} + +fn next_key_after(key: &[u8]) -> Vec { + let mut next = key.to_vec(); + next.push(0); + next +} + +// ============================================================================ +// Chainfire Implementation +// ============================================================================ + +/// Thread-safe metadata client backed by the Chainfire gRPC client. +pub struct ChainfireClient { + client: Mutex, +} + +impl ChainfireClient { + pub async fn new(endpoints: Vec) -> Result { + let client = Self::connect_any(&endpoints).await?; + Ok(Self { + client: Mutex::new(client), + }) + } + + async fn connect_any(endpoints: &[String]) -> Result { + let mut last_err = None; + for ep in endpoints { + let addr = if ep.starts_with("http://") || ep.starts_with("https://") { + ep.clone() + } else { + format!("http://{}", ep) + }; + match CfClient::connect(addr.clone()).await { + Ok(client) => return Ok(client), + Err(e) => { + last_err = Some(e); + } + } + } + + Err(MetadataError::Connection( + last_err + .map(|e| e.to_string()) + .unwrap_or_else(|| "no endpoints available".into()), + )) + } +} + +#[async_trait] +impl MetadataClient for ChainfireClient { + async fn get(&self, key: &[u8]) -> Result> { + let mut client = self.client.lock().await; + let result = client + .get_with_revision(key) + .await + .map_err(map_chainfire_error)?; + Ok(result.map(|(v, rev)| (Bytes::from(v), rev))) + } + + async fn put(&self, key: &[u8], value: &[u8]) -> Result { + let mut client = self.client.lock().await; + client.put(key, value).await.map_err(map_chainfire_error) + } + + async fn cas(&self, key: &[u8], expected_version: u64, value: &[u8]) -> Result { + let mut client = self.client.lock().await; + let outcome: CasOutcome = client + .compare_and_swap(key, expected_version, value) + .await + .map_err(map_chainfire_error)?; + + if outcome.success { + return Ok(CasResult::Success(outcome.new_version)); + } + + if expected_version == 0 { + if outcome.current_version == 0 { + Ok(CasResult::NotFound) + } else { + Ok(CasResult::Conflict { + expected: 0, + actual: outcome.current_version, + }) + } + } else { + Ok(CasResult::Conflict { + expected: expected_version, + actual: outcome.current_version, + }) + } + } + + async fn delete(&self, key: &[u8]) -> Result { + let mut client = self.client.lock().await; + client.delete(key).await.map_err(map_chainfire_error) + } + + async fn scan_prefix(&self, prefix: &[u8], limit: u32) -> Result> { + let mut client = self.client.lock().await; + let (results, _) = client + .scan_prefix(prefix, limit as i64) + .await + .map_err(map_chainfire_error)?; + + Ok(results + .into_iter() + .map(|(k, v, ver)| KvPair { + key: Bytes::from(k), + value: Bytes::from(v), + version: ver, + }) + .collect()) + } + + async fn scan_range(&self, start: &[u8], end: &[u8], limit: u32) -> Result> { + let mut client = self.client.lock().await; + let (results, _) = client + .scan_range(start, end, limit as i64) + .await + .map_err(map_chainfire_error)?; + + Ok(results + .into_iter() + .map(|(k, v, ver)| KvPair { + key: Bytes::from(k), + value: Bytes::from(v), + version: ver, + }) + .collect()) + } +} + +fn map_chainfire_error(err: CfClientError) -> MetadataError { + match err { + CfClientError::Connection(msg) => MetadataError::Connection(msg), + CfClientError::Transport(e) => MetadataError::Connection(e.to_string()), + CfClientError::Rpc(status) => MetadataError::Backend(status.to_string()), + other => MetadataError::Backend(other.to_string()), + } +} + +// ============================================================================ +// Memory Implementation +// ============================================================================ + +pub struct MemoryClient { + data: RwLock, (Vec, u64)>>, + version_counter: RwLock, +} + +impl MemoryClient { + pub fn new() -> Self { + Self { + data: RwLock::new(BTreeMap::new()), + version_counter: RwLock::new(0), + } + } + + fn next_version(&self) -> u64 { + let mut counter = self.version_counter.write().unwrap(); + *counter += 1; + *counter + } +} + +impl Default for MemoryClient { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl MetadataClient for MemoryClient { + async fn get(&self, key: &[u8]) -> Result> { + let data = self.data.read().unwrap(); + Ok(data + .get(key) + .map(|(v, ver)| (Bytes::copy_from_slice(v), *ver))) + } + + async fn put(&self, key: &[u8], value: &[u8]) -> Result { + let version = self.next_version(); + let mut data = self.data.write().unwrap(); + data.insert(key.to_vec(), (value.to_vec(), version)); + Ok(version) + } + + async fn cas(&self, key: &[u8], expected_version: u64, value: &[u8]) -> Result { + let mut data = self.data.write().unwrap(); + + match data.get(key) { + Some((_, current_version)) => { + if *current_version != expected_version { + return Ok(CasResult::Conflict { + expected: expected_version, + actual: *current_version, + }); + } + } + None => { + if expected_version != 0 { + return Ok(CasResult::NotFound); + } + } + } + + let version = self.next_version(); + data.insert(key.to_vec(), (value.to_vec(), version)); + Ok(CasResult::Success(version)) + } + + async fn delete(&self, key: &[u8]) -> Result { + let mut data = self.data.write().unwrap(); + Ok(data.remove(key).is_some()) + } + + async fn scan_prefix(&self, prefix: &[u8], limit: u32) -> Result> { + let data = self.data.read().unwrap(); + let mut results = Vec::new(); + + for (k, (v, ver)) in data.range(prefix.to_vec()..) { + if !k.starts_with(prefix) { + break; + } + results.push(KvPair { + key: Bytes::copy_from_slice(k), + value: Bytes::copy_from_slice(v), + version: *ver, + }); + if results.len() >= limit as usize { + break; + } + } + + Ok(results) + } + + async fn scan_range(&self, start: &[u8], end: &[u8], limit: u32) -> Result> { + let data = self.data.read().unwrap(); + let mut results = Vec::new(); + + for (k, (v, ver)) in data.range(start.to_vec()..end.to_vec()) { + results.push(KvPair { + key: Bytes::copy_from_slice(k), + value: Bytes::copy_from_slice(v), + version: *ver, + }); + if results.len() >= limit as usize { + break; + } + } + + Ok(results) + } +} + + diff --git a/chainfire/crates/chainfire-core/src/traits.rs b/chainfire/crates/chainfire-core/src/traits.rs new file mode 100644 index 0000000..3c20646 --- /dev/null +++ b/chainfire/crates/chainfire-core/src/traits.rs @@ -0,0 +1,60 @@ +use async_trait::async_trait; +use chainfire_types::node::NodeInfo; +use crate::error::Result; +use std::net::SocketAddr; + +/// Abstract interface for Gossip protocol +#[async_trait] +pub trait Gossip: Send + Sync { + /// Start the gossip agent + async fn start(&self) -> Result<()>; + + /// Join a cluster via seed nodes + async fn join(&self, seeds: &[SocketAddr]) -> Result<()>; + + /// Announce presence to a specific node + async fn announce(&self, addr: SocketAddr) -> Result<()>; + + /// Get list of known members + fn members(&self) -> Vec; + + /// Shutdown the gossip agent + async fn shutdown(&self) -> Result<()>; +} + +/// Abstract interface for Consensus protocol (Raft) +#[async_trait] +pub trait Consensus: Send + Sync { + /// Initialize the consensus module + async fn initialize(&self) -> Result<()>; + + /// Start the event loop + async fn run(&self) -> Result<()>; + + /// Propose a command to the state machine + async fn propose(&self, data: Vec) -> Result; + + /// Add a node to the consensus group + async fn add_node(&self, node_id: u64, addr: String, as_learner: bool) -> Result<()>; + + /// Remove a node from the consensus group + async fn remove_node(&self, node_id: u64) -> Result<()>; + + /// Check if this node is the leader + fn is_leader(&self) -> bool; + + /// Get the current leader ID + fn leader_id(&self) -> Option; +} + +/// Abstract interface for State Machine +pub trait StateMachine: Send + Sync { + /// Apply a committed entry + fn apply(&self, index: u64, data: &[u8]) -> Result>; + + /// Take a snapshot of current state + fn snapshot(&self) -> Result>; + + /// Restore state from a snapshot + fn restore(&self, snapshot: &[u8]) -> Result<()>; +} \ No newline at end of file diff --git a/chainfire/crates/chainfire-core/tests/integration.rs b/chainfire/crates/chainfire-core/tests/integration.rs new file mode 100644 index 0000000..190ae8a --- /dev/null +++ b/chainfire/crates/chainfire-core/tests/integration.rs @@ -0,0 +1,52 @@ +use std::time::Duration; +use chainfire_core::ClusterBuilder; +use chainfire_types::{node::NodeRole, RaftRole}; +use tokio::time::sleep; + +#[tokio::test] +async fn test_single_node_bootstrap() { + let _ = tracing_subscriber::fmt::try_init(); + + // 1. Build a single node cluster + let cluster = ClusterBuilder::new(1) + .name("node-1") + .memory_storage() + .gossip_addr("127.0.0.1:0".parse().unwrap()) + .raft_addr("127.0.0.1:0".parse().unwrap()) + .role(NodeRole::ControlPlane) + .raft_role(RaftRole::Voter) + .bootstrap(true) + .build() + .await + .expect("Failed to build cluster"); + + let handle = cluster.handle(); + + // 2. Run the cluster in a background task + tokio::spawn(async move { + cluster.run().await.unwrap(); + }); + + // 3. Wait for leader election + let mut leader_elected = false; + for _ in 0..10 { + if handle.is_leader() { + leader_elected = true; + break; + } + sleep(Duration::from_millis(500)).await; + } + + assert!(leader_elected, "Node 1 should become leader in bootstrap mode"); + assert_eq!(handle.leader(), Some(1)); + + // 4. Test KV operations + let kv = handle.kv(); + kv.put("test-key", b"test-value").await.expect("Put failed"); + + let value = kv.get("test-key").await.expect("Get failed"); + assert_eq!(value, Some(b"test-value".to_vec())); + + // 5. Shutdown + handle.shutdown(); +} \ No newline at end of file diff --git a/chainfire/crates/chainfire-raft/src/storage.rs b/chainfire/crates/chainfire-raft/src/storage.rs new file mode 100644 index 0000000..65f8ce7 --- /dev/null +++ b/chainfire/crates/chainfire-raft/src/storage.rs @@ -0,0 +1,378 @@ +//! Storage primitives used by `chainfire-raft`. +//! +//! In production (`rocksdb-storage` feature), we re-export the real ChainFire storage layer. +//! For lightweight testing/simulation (default), we provide a small in-memory implementation +//! that avoids native dependencies (RocksDB/libclang). + +#[cfg(feature = "rocksdb-storage")] +pub use chainfire_storage::{ + EntryPayload, LogEntry, LogId, LogState, LogStorage, StateMachine, Vote, +}; + +#[cfg(not(feature = "rocksdb-storage"))] +mod mem { + use chainfire_types::command::{RaftCommand, RaftResponse}; + use chainfire_types::error::StorageError; + use chainfire_types::kv::{KvEntry, Revision}; + use parking_lot::RwLock; + use serde::{Deserialize, Serialize}; + use std::collections::{BTreeMap, HashMap}; + use std::ops::RangeBounds; + use std::sync::atomic::{AtomicU64, Ordering}; + + pub type LogIndex = u64; + pub type Term = u64; + + /// Log ID combining term and index. + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default)] + pub struct LogId { + pub term: Term, + pub index: LogIndex, + } + + /// Payload of a log entry. + #[derive(Debug, Clone, Serialize, Deserialize)] + pub enum EntryPayload { + /// A blank entry for leader establishment. + Blank, + /// A normal data entry. + Normal(D), + /// Membership change entry. + Membership(Vec), + } + + /// A log entry stored in the Raft log. + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct LogEntry { + pub log_id: LogId, + pub payload: EntryPayload, + } + + /// Persisted vote information. + #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)] + pub struct Vote { + pub term: Term, + pub node_id: Option, + pub committed: bool, + } + + /// Log storage state. + #[derive(Debug, Clone, Default)] + pub struct LogState { + pub last_purged_log_id: Option, + pub last_log_id: Option, + } + + /// In-memory Raft log storage. + /// + /// Stores bincode-encoded `LogEntry` blobs keyed by log index. + pub struct LogStorage { + vote: RwLock>, + logs: RwLock>>, + last_purged_log_id: RwLock>, + } + + impl Default for LogStorage { + fn default() -> Self { + Self::new_in_memory() + } + } + + impl LogStorage { + pub fn new_in_memory() -> Self { + Self { + vote: RwLock::new(None), + logs: RwLock::new(BTreeMap::new()), + last_purged_log_id: RwLock::new(None), + } + } + + pub fn store(&self) -> chainfire_storage::RocksStore { + // This is a hack to satisfy the API. In memory mode, we shouldn't really + // be calling this if we want to avoid RocksDB, but chainfire-api expects it. + panic!("LogStorage::store() called in memory mode"); + } + + pub fn get_log_state(&self) -> Result { + let last_purged_log_id = *self.last_purged_log_id.read(); + let logs = self.logs.read(); + let last_log_id = match logs.iter().next_back() { + Some((_idx, bytes)) if !bytes.is_empty() => { + match bincode::deserialize::>>(bytes) { + Ok(entry) => Some(entry.log_id), + Err(e) => { + eprintln!( + "Warning: Failed to deserialize log entry in mem storage: {e}, treating as empty log" + ); + last_purged_log_id + } + } + } + _ => last_purged_log_id, + }; + + Ok(LogState { + last_purged_log_id, + last_log_id, + }) + } + + pub fn save_vote(&self, vote: Vote) -> Result<(), StorageError> { + *self.vote.write() = Some(vote); + Ok(()) + } + + pub fn read_vote(&self) -> Result, StorageError> { + Ok(*self.vote.read()) + } + + pub fn append(&self, entries: &[LogEntry]) -> Result<(), StorageError> { + if entries.is_empty() { + return Ok(()); + } + let mut logs = self.logs.write(); + for entry in entries { + let bytes = bincode::serialize(entry) + .map_err(|e| StorageError::Serialization(e.to_string()))?; + logs.insert(entry.log_id.index, bytes); + } + Ok(()) + } + + pub fn get_log_entries Deserialize<'de>>( + &self, + range: impl RangeBounds, + ) -> Result>, StorageError> { + let logs = self.logs.read(); + + let start = match range.start_bound() { + std::ops::Bound::Included(&idx) => idx, + std::ops::Bound::Excluded(&idx) => idx + 1, + std::ops::Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + std::ops::Bound::Included(&idx) => Some(idx), + std::ops::Bound::Excluded(&idx) => Some(idx.saturating_sub(1)), + std::ops::Bound::Unbounded => None, + }; + + let iter: Box)> + '_> = match end { + Some(end_inclusive) => Box::new(logs.range(start..=end_inclusive)), + None => Box::new(logs.range(start..)), + }; + + let mut out = Vec::new(); + for (_idx, bytes) in iter { + let entry: LogEntry = bincode::deserialize(bytes) + .map_err(|e| StorageError::Serialization(e.to_string()))?; + out.push(entry); + } + Ok(out) + } + + pub fn truncate(&self, from_index: LogIndex) -> Result<(), StorageError> { + let mut logs = self.logs.write(); + // Remove all entries >= from_index + let _ = logs.split_off(&from_index); + Ok(()) + } + + #[allow(dead_code)] + pub fn purge_with_log_id(&self, log_id: LogId) -> Result<(), StorageError> { + // In-memory compaction marker only; entries are not retained once purged. + *self.last_purged_log_id.write() = Some(log_id); + self.truncate(log_id.index + 1)?; + Ok(()) + } + } + + /// Minimal in-memory KV store used by the in-memory state machine. + pub struct KvStore { + data: RwLock, KvEntry>>, + revision: AtomicU64, + } + + impl Default for KvStore { + fn default() -> Self { + Self { + data: RwLock::new(HashMap::new()), + revision: AtomicU64::new(0), + } + } + } + + impl KvStore { + pub fn current_revision(&self) -> Revision { + self.revision.load(Ordering::SeqCst) + } + + fn next_revision(&self) -> Revision { + self.revision.fetch_add(1, Ordering::SeqCst) + 1 + } + + pub fn get(&self, key: &[u8]) -> Result, StorageError> { + Ok(self.data.read().get(key).cloned()) + } + + pub fn range_count(&self, start: &[u8], end: Option<&[u8]>) -> Result { + let data = self.data.read(); + let count = if let Some(end) = end { + data.iter().filter(|(k, _)| k.as_slice() >= start && k.as_slice() < end).count() + } else { + data.iter().filter(|(k, _)| k.as_slice() >= start).count() + }; + Ok(count) + } + + pub fn range_with_limit(&self, start: &[u8], end: Option<&[u8]>, limit: Option) -> Result<(Vec, bool), StorageError> { + let data = self.data.read(); + let mut entries: Vec<_> = if let Some(end) = end { + data.iter() + .filter(|(k, _)| k.as_slice() >= start && k.as_slice() < end) + .map(|(_, v)| v.clone()) + .collect() + } else { + data.iter() + .filter(|(k, _)| k.as_slice() >= start) + .map(|(_, v)| v.clone()) + .collect() + }; + entries.sort_by(|a, b| a.key.cmp(&b.key)); + + if let Some(limit) = limit { + let more = entries.len() > limit; + entries.truncate(limit); + Ok((entries, more)) + } else { + Ok((entries, false)) + } + } + + pub fn set_revision(&self, revision: Revision) { + self.revision.store(revision, Ordering::SeqCst); + } + + pub fn put( + &self, + key: Vec, + value: Vec, + lease_id: Option, + ) -> Result<(Revision, Option), StorageError> { + let mut data = self.data.write(); + let prev = data.get(&key).cloned(); + let revision = self.next_revision(); + + let entry = match &prev { + Some(old) => old.update(value, revision), + None => { + if let Some(lease) = lease_id { + KvEntry::with_lease(key.clone(), value, revision, lease) + } else { + KvEntry::new(key.clone(), value, revision) + } + } + }; + data.insert(key, entry); + Ok((revision, prev)) + } + + pub fn delete(&self, key: &[u8]) -> Result<(Revision, Option), StorageError> { + let mut data = self.data.write(); + let prev = data.remove(key); + let revision = self.next_revision(); + Ok((revision, prev)) + } + } + + /// Minimal in-memory state machine for Raft simulation. + pub struct StateMachine { + kv: KvStore, + } + + pub struct LeaseStore; + impl LeaseStore { + pub fn list(&self) -> Vec { vec![] } + } + + impl Default for StateMachine { + fn default() -> Self { + Self::new_in_memory() + } + } + + impl StateMachine { + pub fn new_in_memory() -> Self { + Self { kv: KvStore::default() } + } + + pub fn kv(&self) -> &KvStore { + &self.kv + } + + pub fn current_revision(&self) -> Revision { + self.kv.current_revision() + } + + pub fn leases(&self) -> LeaseStore { + LeaseStore + } + + pub fn apply(&self, command: RaftCommand) -> Result { + match command { + RaftCommand::Put { + key, + value, + lease_id, + prev_kv, + } => { + let (rev, prev) = self.kv.put(key, value, lease_id)?; + Ok(RaftResponse::with_prev_kv(rev, if prev_kv { prev } else { None })) + } + RaftCommand::Delete { key, prev_kv } => { + let (rev, prev) = self.kv.delete(&key)?; + let deleted = if prev.is_some() { 1 } else { 0 }; + Ok(RaftResponse { + revision: rev, + prev_kv: if prev_kv { prev } else { None }, + deleted, + ..Default::default() + }) + } + RaftCommand::Noop => Ok(RaftResponse::new(self.current_revision())), + other => Err(StorageError::Serialization(format!( + "mem state machine: unsupported command variant: {other:?}" + ))), + } + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn mem_log_storage_append_and_get() { + let storage = LogStorage::new_in_memory(); + let entries = vec![ + LogEntry { + log_id: LogId { term: 1, index: 1 }, + payload: EntryPayload::Normal(b"a".to_vec()), + }, + LogEntry { + log_id: LogId { term: 1, index: 2 }, + payload: EntryPayload::Normal(b"b".to_vec()), + }, + ]; + storage.append(&entries).unwrap(); + let got: Vec>> = storage.get_log_entries(1..=2).unwrap(); + assert_eq!(got.len(), 2); + assert_eq!(got[0].log_id.index, 1); + } + } +} + +#[cfg(not(feature = "rocksdb-storage"))] +pub use mem::{EntryPayload, LogEntry, LogId, LogState, LogStorage, StateMachine, Vote}; + + diff --git a/chainfire/crates/chainfire-raft/tests/proptest_sim.rs b/chainfire/crates/chainfire-raft/tests/proptest_sim.rs new file mode 100644 index 0000000..344106c --- /dev/null +++ b/chainfire/crates/chainfire-raft/tests/proptest_sim.rs @@ -0,0 +1,274 @@ +//! Property-based tests for `chainfire-raft` using an in-process simulated cluster. +//! +//! These tests aim to catch timing/partition edge cases with high reproducibility. + +#![cfg(all(test, feature = "custom-raft"))] + +use std::sync::Arc; +use std::time::Duration; + +use proptest::prelude::*; +use tokio::sync::mpsc; +use tokio::time; + +use chainfire_raft::core::{RaftConfig, RaftCore}; +use chainfire_raft::network::test_client::{RpcMessage, SimulatedNetwork}; +use chainfire_raft::storage::{EntryPayload, LogEntry, LogStorage, StateMachine}; +use chainfire_types::command::RaftCommand; + +#[derive(Debug, Clone)] +enum Op { + Tick(u64), + Disconnect(u64, u64), + Reconnect(u64, u64), + Delay(u64, u64, u64), + ClearLink(u64, u64), + Write(u64, u8, u8), +} + +fn node_id() -> impl Strategy { + 1_u64..=3_u64 +} + +fn distinct_pair() -> impl Strategy { + (node_id(), node_id()).prop_filter("distinct nodes", |(a, b)| a != b) +} + +fn op_strategy() -> impl Strategy { + prop_oneof![ + // Advance simulated time by up to 300ms. + (0_u64..=300).prop_map(Op::Tick), + distinct_pair().prop_map(|(a, b)| Op::Disconnect(a, b)), + distinct_pair().prop_map(|(a, b)| Op::Reconnect(a, b)), + (distinct_pair(), 0_u64..=50).prop_map(|((a, b), d)| Op::Delay(a, b, d)), + distinct_pair().prop_map(|(a, b)| Op::ClearLink(a, b)), + // Client writes: pick node + small key/value. + (node_id(), any::(), any::()).prop_map(|(n, k, v)| Op::Write(n, k, v)), + ] +} + +fn ops_strategy() -> impl Strategy> { + prop::collection::vec(op_strategy(), 0..40) +} + +async fn advance_ms(total_ms: u64) { + // Advance in small steps to avoid “simultaneous” timer firings starving message handling. + let step_ms: u64 = 10; + let mut remaining = total_ms; + while remaining > 0 { + let d = remaining.min(step_ms); + time::advance(Duration::from_millis(d)).await; + tokio::task::yield_now().await; + remaining -= d; + } +} + +async fn create_3node_cluster() -> (Vec>, Arc) { + let network = Arc::new(SimulatedNetwork::new()); + let mut nodes = Vec::new(); + + for node_id in 1..=3_u64 { + let peers: Vec = (1..=3_u64).filter(|&id| id != node_id).collect(); + let storage = Arc::new(LogStorage::new_in_memory()); + let state_machine = Arc::new(StateMachine::new_in_memory()); + + let config = RaftConfig { + election_timeout_min: 150, + election_timeout_max: 300, + heartbeat_interval: 50, + // Deterministic per-node seed for reproducibility. + deterministic_seed: Some(node_id), + }; + + let node = Arc::new(RaftCore::new( + node_id, + peers, + storage, + state_machine, + Arc::new(network.client(node_id)) as Arc, + config, + )); + node.initialize().await.unwrap(); + nodes.push(node); + } + + // Wire up RPC handlers. + for node in &nodes { + let node_id = node.node_id(); + let (tx, mut rx) = mpsc::unbounded_channel::(); + network.register(node_id, tx).await; + + let node_clone: Arc = Arc::clone(node); + tokio::spawn(async move { + while let Some(msg) = rx.recv().await { + match msg { + RpcMessage::Vote(req, resp_tx) => { + node_clone.request_vote_rpc(req, resp_tx).await; + } + RpcMessage::AppendEntries(req, resp_tx) => { + node_clone.append_entries_rpc(req, resp_tx).await; + } + } + } + }); + } + + (nodes, network) +} + +fn payload_fingerprint(payload: &EntryPayload>) -> Vec { + // Serialize the enum for stable equality checks across variants. + bincode::serialize(payload).unwrap_or_default() +} + +async fn assert_raft_invariants(nodes: &[Arc]) { + // Per-node monotonic invariants. + for node in nodes { + let commit = node.commit_index().await; + let last_applied = node.last_applied().await; + + let st = node.storage().get_log_state().expect("log state"); + let last_log_index = st.last_log_id.map(|id| id.index).unwrap_or(0); + + assert!( + last_applied <= commit, + "node {}: last_applied={} > commit_index={}", + node.node_id(), + last_applied, + commit + ); + assert!( + commit <= last_log_index, + "node {}: commit_index={} > last_log_index={}", + node.node_id(), + commit, + last_log_index + ); + } + + // Log Matching Property: + // If two logs contain an entry with the same index and term, then the logs are identical + // for all entries up through that index. + let mut node_logs: Vec)>> = Vec::new(); + for node in nodes { + let st = node.storage().get_log_state().expect("log state"); + let last = st.last_log_id.map(|id| id.index).unwrap_or(0); + let entries: Vec>> = if last == 0 { + vec![] + } else { + node.storage() + .get_log_entries(1..=last) + .expect("log entries") + }; + + let mut m = std::collections::BTreeMap::new(); + for e in entries { + m.insert(e.log_id.index, (e.log_id.term, payload_fingerprint(&e.payload))); + } + node_logs.push(m); + } + + for a in 0..nodes.len() { + for b in (a + 1)..nodes.len() { + let la = &node_logs[a]; + let lb = &node_logs[b]; + + for (idx, (term_a, payload_a)) in la.iter() { + if let Some((term_b, payload_b)) = lb.get(idx) { + if term_a == term_b { + assert_eq!( + payload_a, payload_b, + "log mismatch at idx={} term={} (nodes {} vs {})", + idx, + term_a, + nodes[a].node_id(), + nodes[b].node_id() + ); + + for j in 1..=*idx { + assert_eq!( + la.get(&j), + lb.get(&j), + "log matching violated at idx={} (prefix {} differs) nodes {} vs {}", + idx, + j, + nodes[a].node_id(), + nodes[b].node_id() + ); + } + } + } + } + } + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: 32, + .. ProptestConfig::default() + })] + + #[test] + fn prop_raft_log_matching_holds(ops in ops_strategy()) { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + rt.block_on(async move { + tokio::time::pause(); + + let (nodes, network) = create_3node_cluster().await; + + // Start event loops. + let mut handles = Vec::new(); + for node in &nodes { + let node_clone = Arc::clone(node); + handles.push(tokio::spawn(async move { + let _ = node_clone.run().await; + })); + } + tokio::task::yield_now().await; + + // Drive a randomized sequence of operations. + for op in ops { + match op { + Op::Tick(ms) => advance_ms(ms).await, + Op::Disconnect(a, b) => network.disconnect(a, b).await, + Op::Reconnect(a, b) => network.reconnect(a, b).await, + Op::Delay(a, b, d) => { + use chainfire_raft::network::test_client::LinkBehavior; + network.set_link(a, b, LinkBehavior::Delay(Duration::from_millis(d))).await; + network.set_link(b, a, LinkBehavior::Delay(Duration::from_millis(d))).await; + } + Op::ClearLink(a, b) => { + network.clear_link(a, b).await; + network.clear_link(b, a).await; + } + Op::Write(n, k, v) => { + let node = nodes.iter().find(|x| x.node_id() == n).unwrap(); + let _ = node.client_write(RaftCommand::Put { + key: vec![k], + value: vec![v], + lease_id: None, + prev_kv: false, + }).await; + } + } + } + + // Let the system settle a bit. + advance_ms(500).await; + + assert_raft_invariants(&nodes).await; + + // Best-effort cleanup. + for h in handles { + h.abort(); + } + }); + } +} + + diff --git a/client-common/Cargo.lock b/client-common/Cargo.lock new file mode 100644 index 0000000..0126cb0 --- /dev/null +++ b/client-common/Cargo.lock @@ -0,0 +1,1098 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "futures-core", + "getrandom", + "instant", + "pin-project-lite", + "rand", + "tokio", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + +[[package]] +name = "cc" +version = "1.2.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.12.1", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2 0.6.1", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "libc" +version = "0.2.178" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "photocloud-client-common" +version = "0.1.0" +dependencies = [ + "backoff", + "thiserror", + "tokio", + "tonic", +] + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", +] + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2 0.6.1", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "rustls-pemfile", + "socket2 0.5.10", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "zerocopy" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" diff --git a/client-common/Cargo.toml b/client-common/Cargo.toml new file mode 100644 index 0000000..d4b0318 --- /dev/null +++ b/client-common/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "photocloud-client-common" +version = "0.1.0" +edition = "2021" +authors = ["PhotonCloud"] +license = "MIT OR Apache-2.0" +description = "Shared client config types (endpoint/auth/retry) for PhotonCloud SDKs" + +[dependencies] +tonic = { version = "0.12", features = ["tls"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } +thiserror = "1" +backoff = { version = "0.4", features = ["tokio"] } + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt", "time"] } diff --git a/client-common/src/lib.rs b/client-common/src/lib.rs new file mode 100644 index 0000000..9458b42 --- /dev/null +++ b/client-common/src/lib.rs @@ -0,0 +1,205 @@ +//! Shared client config types (endpoint/auth/retry) for PhotonCloud SDKs. +//! +//! Lightweight, type-only helpers to keep SDK crates consistent without +//! forcing a unified SDK dependency tree. + +use std::time::Duration; +use backoff::ExponentialBackoffBuilder; +use thiserror::Error; +use tonic::codegen::InterceptedService; +use tonic::service::Interceptor; +use tonic::transport::{Channel, ClientTlsConfig, Endpoint}; + +/// Errors produced by client-common helpers. +#[derive(Debug, Error)] +pub enum ClientError { + #[error("invalid endpoint: {0}")] + InvalidEndpoint(String), + #[error("TLS configuration error: {0}")] + TlsConfig(String), + #[error("transport error: {0}")] + Transport(#[from] tonic::transport::Error), + #[error("auth error: {0}")] + Auth(String), +} + +/// Endpoint configuration (URI + timeout + optional TLS domain/CA). +#[derive(Debug, Clone)] +pub struct EndpointConfig { + pub uri: String, + pub timeout: Duration, + pub tls: Option, +} + +impl EndpointConfig { + pub fn new(uri: impl Into) -> Self { + Self { + uri: uri.into(), + timeout: Duration::from_millis(5_000), + tls: None, + } + } + + pub fn with_timeout(mut self, timeout: Duration) -> Self { + self.timeout = timeout; + self + } + + pub fn with_tls(mut self, tls: TlsConfig) -> Self { + self.tls = Some(tls); + self + } + + /// Build a tonic Endpoint with timeout/TLS applied. + pub fn build_endpoint(&self) -> Result { + let mut ep = Endpoint::from_shared(self.uri.clone()) + .map_err(|e| ClientError::InvalidEndpoint(e.to_string()))? + .timeout(self.timeout); + + if let Some(tls) = &self.tls { + let mut cfg = ClientTlsConfig::new(); + if let Some(domain) = &tls.domain { + cfg = cfg.domain_name(domain.clone()); + } + if let Some(ca_cert) = &tls.ca_cert_pem { + cfg = cfg.ca_certificate(tonic::transport::Certificate::from_pem(ca_cert.clone())); + } + if let (Some(cert), Some(key)) = (&tls.client_cert_pem, &tls.client_key_pem) { + cfg = cfg.identity(tonic::transport::Identity::from_pem( + cert.clone(), + key.clone(), + )); + } + ep = ep.tls_config(cfg).map_err(|e| ClientError::TlsConfig(e.to_string()))?; + } + + Ok(ep) + } +} + +/// TLS configuration inputs. +#[derive(Debug, Clone, Default)] +pub struct TlsConfig { + pub domain: Option, + pub ca_cert_pem: Option, + pub client_cert_pem: Option, + pub client_key_pem: Option, +} + +/// Auth configuration for interceptors. +#[derive(Debug, Clone)] +pub enum AuthConfig { + None, + Bearer { token: String }, + AccessKey { id: String, secret: String }, +} + +impl AuthConfig { + pub fn bearer(token: impl Into) -> Self { + Self::Bearer { token: token.into() } + } +} + +/// Retry/backoff configuration. +#[derive(Debug, Clone)] +pub struct RetryConfig { + pub max_retries: u32, + pub base_delay: Duration, + pub max_delay: Duration, + pub jitter: bool, +} + +impl Default for RetryConfig { + fn default() -> Self { + Self { + max_retries: 3, + base_delay: Duration::from_millis(100), + max_delay: Duration::from_secs(2), + jitter: true, + } + } +} + +impl RetryConfig { + pub fn backoff(&self) -> backoff::ExponentialBackoff { + let mut builder = ExponentialBackoffBuilder::new(); + builder + .with_initial_interval(self.base_delay) + .with_max_interval(self.max_delay) + .with_max_elapsed_time(None); + if !self.jitter { + builder.with_randomization_factor(0.0); + } + builder.build() + } +} + +/// Build a channel with optional auth interceptor. +pub async fn build_channel( + endpoint: &EndpointConfig, + auth: &AuthConfig, +) -> Result { + let ep = endpoint.build_endpoint()?; + let channel = ep.connect().await?; + + match auth { + AuthConfig::None => Ok(channel), + AuthConfig::Bearer { .. } | AuthConfig::AccessKey { .. } => Ok(channel), + } +} + +/// Interceptor that injects auth headers; used by clients that need request metadata. +#[derive(Clone)] +pub struct AuthInterceptor(AuthConfig); + +impl Interceptor for AuthInterceptor { + fn call(&mut self, mut req: tonic::Request<()>) -> Result, tonic::Status> { + match &self.0 { + AuthConfig::None => {} + AuthConfig::Bearer { token } => { + req.metadata_mut() + .insert("authorization", format!("Bearer {}", token).parse().unwrap()); + } + AuthConfig::AccessKey { id, secret } => { + req.metadata_mut() + .insert("x-api-key", id.parse().unwrap()); + req.metadata_mut() + .insert("x-api-secret", secret.parse().unwrap()); + } + } + Ok(req) + } +} + +/// Create an auth interceptor from AuthConfig (for tonic clients that need it). +pub fn auth_interceptor(auth: &AuthConfig) -> Option { + match auth { + AuthConfig::None => None, + _ => Some(AuthInterceptor(auth.clone())), + } +} + +/// Helper to wrap a tonic client with an interceptor when auth is provided. +pub fn with_auth(channel: Channel, auth: &AuthConfig) -> InterceptedService { + let interceptor = auth_interceptor(auth).unwrap_or(AuthInterceptor(AuthConfig::None)); + InterceptedService::new(channel, interceptor) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retry_config_builds_backoff() { + let cfg = RetryConfig::default(); + let backoff = cfg.backoff(); + assert!(backoff.initial_interval == cfg.base_delay); + } + + #[tokio::test] + async fn endpoint_builds_without_tls() { + let ep = EndpointConfig::new("http://localhost:50051"); + let built = ep.build_endpoint().unwrap(); + assert!(built.uri().to_string().contains("localhost")); + } +} diff --git a/creditservice/crates/creditservice-api/src/gateway_credit_service.rs b/creditservice/crates/creditservice-api/src/gateway_credit_service.rs new file mode 100644 index 0000000..f37603c --- /dev/null +++ b/creditservice/crates/creditservice-api/src/gateway_credit_service.rs @@ -0,0 +1,275 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use apigateway_api::proto::{ + CreditCommitRequest, CreditCommitResponse, CreditReserveRequest, CreditReserveResponse, + CreditRollbackRequest, CreditRollbackResponse, +}; +use apigateway_api::GatewayCreditService; +use creditservice_proto::credit_service_server::CreditService; +use creditservice_proto::{CommitReservationRequest, ReleaseReservationRequest, ReserveCreditsRequest}; +use tonic::{Code, Request, Response, Status}; + +use crate::credit_service::CreditServiceImpl; + +pub struct GatewayCreditServiceImpl { + credit_service: Arc, +} + +impl GatewayCreditServiceImpl { + pub fn new(credit_service: Arc) -> Self { + Self { credit_service } + } +} + +#[tonic::async_trait] +impl GatewayCreditService for GatewayCreditServiceImpl { + async fn reserve( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + if req.org_id.trim().is_empty() { + return Ok(Response::new(deny_response( + "org_id required for credit reservation", + ))); + } + + if req.project_id.trim().is_empty() { + return Ok(Response::new(deny_response( + "project_id required for credit reservation", + ))); + } + + if req.units == 0 { + return Err(Status::invalid_argument("units must be positive")); + } + + let amount = i64::try_from(req.units) + .map_err(|_| Status::invalid_argument("units exceeds i64 range"))?; + let description = reservation_description(&req); + let resource_type = reservation_resource_type(&req); + let ttl_seconds = reservation_ttl(&req.attributes); + + let reserve_request = ReserveCreditsRequest { + project_id: req.project_id.clone(), + org_id: req.org_id.clone(), + amount, + description, + resource_type, + ttl_seconds, + }; + + match self + .credit_service + .reserve_credits(Request::new(reserve_request)) + .await + { + Ok(response) => { + let response = response.into_inner(); + let reservation = response.reservation.ok_or_else(|| { + Status::internal("credit reservation missing from response") + })?; + Ok(Response::new(CreditReserveResponse { + allow: true, + reservation_id: reservation.id, + reason: String::new(), + remaining: 0, + })) + } + Err(status) => match status.code() { + Code::NotFound | Code::FailedPrecondition => { + Ok(Response::new(deny_response(status.message()))) + } + Code::InvalidArgument => Err(Status::invalid_argument(status.message())), + _ => Err(status), + }, + } + } + + async fn commit( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + let amount = i64::try_from(req.units) + .map_err(|_| Status::invalid_argument("units exceeds i64 range"))?; + let commit_request = CommitReservationRequest { + reservation_id: req.reservation_id, + org_id: String::new(), + actual_amount: amount, + resource_id: String::new(), + }; + + match self + .credit_service + .commit_reservation(Request::new(commit_request)) + .await + { + Ok(_) => Ok(Response::new(CreditCommitResponse { + success: true, + reason: String::new(), + })), + Err(status) => match status.code() { + Code::NotFound | Code::FailedPrecondition => Ok(Response::new(CreditCommitResponse { + success: false, + reason: status.message().to_string(), + })), + Code::InvalidArgument => Err(Status::invalid_argument(status.message())), + _ => Err(status), + }, + } + } + + async fn rollback( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let rollback_request = ReleaseReservationRequest { + reservation_id: req.reservation_id, + org_id: String::new(), + reason: "gateway rollback".into(), + }; + + match self + .credit_service + .release_reservation(Request::new(rollback_request)) + .await + { + Ok(response) => { + let response = response.into_inner(); + Ok(Response::new(CreditRollbackResponse { + success: response.success, + reason: String::new(), + })) + } + Err(status) => match status.code() { + Code::NotFound | Code::FailedPrecondition => Ok(Response::new(CreditRollbackResponse { + success: false, + reason: status.message().to_string(), + })), + Code::InvalidArgument => Err(Status::invalid_argument(status.message())), + _ => Err(status), + }, + } + } +} + +fn deny_response(reason: impl Into) -> CreditReserveResponse { + CreditReserveResponse { + allow: false, + reservation_id: String::new(), + reason: reason.into(), + remaining: 0, + } +} + +fn reservation_resource_type(req: &CreditReserveRequest) -> String { + if let Some(value) = req.attributes.get("resource_type") { + return value.clone(); + } + if !req.route_name.is_empty() { + return req.route_name.clone(); + } + "apigateway".to_string() +} + +fn reservation_description(req: &CreditReserveRequest) -> String { + if let Some(value) = req.attributes.get("description") { + return value.clone(); + } + + let mut parts = vec![format!("route={}", req.route_name)]; + if !req.method.is_empty() { + parts.push(format!("method={}", req.method)); + } + if !req.path.is_empty() { + parts.push(format!("path={}", req.path)); + } + if !req.request_id.is_empty() { + parts.push(format!("request_id={}", req.request_id)); + } + if !req.subject_id.is_empty() { + parts.push(format!("subject_id={}", req.subject_id)); + } + + format!("gateway {}", parts.join(" ")) +} + +fn reservation_ttl(attributes: &HashMap) -> i32 { + attributes + .get("ttl_seconds") + .and_then(|value| value.parse::().ok()) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::{CreditStorage, InMemoryStorage}; + use creditservice_types::Wallet; + + fn reserve_request(project_id: &str, units: u64) -> CreditReserveRequest { + CreditReserveRequest { + request_id: "req-1".into(), + subject_id: "subject-1".into(), + org_id: "org-1".into(), + project_id: project_id.into(), + route_name: "route-1".into(), + method: "GET".into(), + path: "/v1/test".into(), + raw_query: "".into(), + units, + attributes: HashMap::new(), + } + } + + #[tokio::test] + async fn test_reserve_denied_without_wallet() { + let storage = InMemoryStorage::new(); + let credit_service = Arc::new(CreditServiceImpl::new(storage)); + let gateway = GatewayCreditServiceImpl::new(credit_service); + + let response = gateway + .reserve(Request::new(reserve_request("proj-1", 5))) + .await + .unwrap() + .into_inner(); + + assert!(!response.allow); + assert!(response.reason.contains("Wallet not found")); + } + + #[tokio::test] + async fn test_reserve_commit_success() { + let storage = InMemoryStorage::new(); + let wallet = Wallet::new("proj-1".into(), "org-1".into(), 100); + storage.create_wallet(wallet).await.unwrap(); + + let credit_service = Arc::new(CreditServiceImpl::new(storage)); + let gateway = GatewayCreditServiceImpl::new(credit_service); + + let reserve_response = gateway + .reserve(Request::new(reserve_request("proj-1", 10))) + .await + .unwrap() + .into_inner(); + + assert!(reserve_response.allow); + assert!(!reserve_response.reservation_id.is_empty()); + + let commit_response = gateway + .commit(Request::new(CreditCommitRequest { + reservation_id: reserve_response.reservation_id, + units: 10, + })) + .await + .unwrap() + .into_inner(); + + assert!(commit_response.success); + } +} diff --git a/creditservice/crates/creditservice-server/tests/mtls_integration.rs b/creditservice/crates/creditservice-server/tests/mtls_integration.rs new file mode 100644 index 0000000..c97494e --- /dev/null +++ b/creditservice/crates/creditservice-server/tests/mtls_integration.rs @@ -0,0 +1,77 @@ +use creditservice_api::{CreditServiceImpl, InMemoryStorage}; +use creditservice_proto::credit_service_server::CreditServiceServer; +use creditservice_client::{Client, TlsConfig}; +use rcgen::generate_simple_self_signed; +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::sync::oneshot; +use tonic::transport::{Identity, Server, ServerTlsConfig}; + +#[tokio::test] +async fn mtls_connects_and_allows_rpc() { + // --- Generate self-signed server and client certs --- + let server = generate_simple_self_signed(vec!["creditservice.local".into()]).unwrap(); + let server_cert_pem = server.cert.pem(); + let server_key_pem = server.key_pair.serialize_pem(); + + let client = generate_simple_self_signed(vec!["creditservice-client".into()]).unwrap(); + let client_cert_pem = client.cert.pem(); + let client_key_pem = client.key_pair.serialize_pem(); + + // --- Start CreditService server with mTLS --- + let addr: SocketAddr = "127.0.0.1:50057".parse().unwrap(); + let storage: Arc = InMemoryStorage::new(); + let svc = Arc::new(CreditServiceImpl::new(storage)); + + let identity = Identity::from_pem(server_cert_pem.clone(), server_key_pem.clone()); + let client_ca = tonic::transport::Certificate::from_pem(client_cert_pem.clone()); + + let (tx, rx) = oneshot::channel::<()>(); + let server = Server::builder() + .tls_config( + ServerTlsConfig::new() + .identity(identity) + .client_ca_root(client_ca), + ) + .unwrap() + .add_service(CreditServiceServer::new(svc.as_ref().clone())) + .serve_with_shutdown(addr, async { + let _ = rx.await; + }); + + let server_handle = tokio::spawn(server); + + // Give the server a moment to start + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + + // --- Client with mTLS --- + let mut client = Client::builder(format!("https://127.0.0.1:{}", addr.port())) + .tls(TlsConfig { + domain: Some("creditservice.local".into()), + ca_cert_pem: Some(server_cert_pem.clone()), + client_cert_pem: Some(client_cert_pem.clone()), + client_key_pem: Some(client_key_pem.clone()), + }) + .build() + .await + .expect("client build"); + + // Simple RPC: create wallet then get wallet + let wallet = client + .create_wallet("proj-mtls", "org-mtls", 1000) + .await + .expect("create_wallet"); + assert_eq!(wallet.project_id, "proj-mtls"); + assert_eq!(wallet.org_id, "org-mtls"); + + let fetched = client + .get_wallet("proj-mtls", "org-mtls") + .await + .expect("get_wallet"); + assert_eq!(fetched.balance, 1000); + + // Shutdown server + let _ = tx.send(()); + let _ = server_handle.await; +} + diff --git a/creditservice/creditservice-client/examples/basic.rs b/creditservice/creditservice-client/examples/basic.rs new file mode 100644 index 0000000..1022e8a --- /dev/null +++ b/creditservice/creditservice-client/examples/basic.rs @@ -0,0 +1,18 @@ +use creditservice_client::{AuthConfig, ClientBuilder}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Connect to CreditService with default retry/backoff and no auth. + let mut client = ClientBuilder::new("http://127.0.0.1:50055") + .auth(AuthConfig::None) + .build() + .await?; + + // Example: check quota call + let _ = client + .check_quota("project-1", creditservice_client::ResourceType::Vm, 1, 0) + .await; + + println!("CreditService client ready"); + Ok(()) +} diff --git a/creditservice/creditservice-client/examples/builder.rs b/creditservice/creditservice-client/examples/builder.rs new file mode 100644 index 0000000..6f8e3e1 --- /dev/null +++ b/creditservice/creditservice-client/examples/builder.rs @@ -0,0 +1,27 @@ +//! Minimal builder example for CreditService client +use creditservice_client::Client; +use photocloud_client_common::AuthConfig; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Point to your CreditService endpoint (plaintext for example only) + let mut client = Client::builder("http://127.0.0.1:50052") + .auth(AuthConfig::None) + .build() + .await?; + + // Fetch or create a wallet + let project_id = "demo-project"; + match client.get_wallet(project_id).await { + Ok(wallet) => println!("Wallet balance: {}", wallet.balance), + Err(status) if status.code() == tonic::Code::NotFound => { + let wallet = client + .create_wallet(project_id, "demo-org", 1_000) + .await?; + println!("Created wallet with balance: {}", wallet.balance); + } + Err(err) => return Err(Box::new(err)), + } + + Ok(()) +} diff --git a/deployer/crates/cert-authority/Cargo.toml b/deployer/crates/cert-authority/Cargo.toml new file mode 100644 index 0000000..6248467 --- /dev/null +++ b/deployer/crates/cert-authority/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "cert-authority" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +anyhow.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +clap = { version = "4.5", features = ["derive"] } +serde.workspace = true +serde_json.workspace = true +chrono = { version = "0.4", features = ["serde"] } +rcgen = { version = "0.13", features = ["pem", "x509-parser"] } +rand_core = { version = "0.6", features = ["std"] } +rustls-pemfile = "2" +x509-parser = "0.18" + +chainfire-client = { path = "../../../chainfire/chainfire-client" } + diff --git a/deployer/crates/cert-authority/src/main.rs b/deployer/crates/cert-authority/src/main.rs new file mode 100644 index 0000000..15c358a --- /dev/null +++ b/deployer/crates/cert-authority/src/main.rs @@ -0,0 +1,348 @@ +use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use chainfire_client::Client; +use clap::Parser; +use rcgen::{Certificate, CertificateParams, DistinguishedName, DnType, KeyPair}; +use rustls_pemfile::certs; +use serde::{Deserialize, Serialize}; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; + +const PHOTON_PREFIX: &str = "photoncloud"; +const CERT_TTL_DAYS: u64 = 90; +const ROTATION_THRESHOLD_DAYS: u64 = 30; + +#[derive(Parser, Debug)] +#[command(author, version, about)] +struct Cli { + #[arg(long)] + chainfire_endpoint: String, + #[arg(long)] + cluster_id: String, + #[arg(long)] + ca_cert_path: PathBuf, + #[arg(long)] + ca_key_path: PathBuf, + #[command(subcommand)] + command: Command, +} + +#[derive(Parser, Debug)] +enum Command { + /// CA証明書を生成 + InitCa, + /// CSRから証明書を発行 + Issue { + #[arg(long)] + csr_path: PathBuf, + #[arg(long)] + cert_path: PathBuf, + #[arg(long)] + node_id: Option, + #[arg(long)] + service_name: Option, + }, + /// 証明書のローテーションが必要かチェック + CheckRotation { + #[arg(long)] + cert_path: PathBuf, + }, +} + +#[derive(Debug, Serialize, Deserialize)] +struct CertificateBinding { + node_id: Option, + service_name: Option, + cert_serial: String, + issued_at: u64, + expires_at: u64, +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env().add_directive("info".parse()?)) + .init(); + + let cli = Cli::parse(); + + match cli.command { + Command::InitCa => { + init_ca(&cli.ca_cert_path, &cli.ca_key_path).await?; + } + Command::Issue { + csr_path, + cert_path, + node_id, + service_name, + } => { + issue_certificate( + &cli.chainfire_endpoint, + &cli.cluster_id, + &cli.ca_cert_path, + &cli.ca_key_path, + &csr_path, + &cert_path, + node_id, + service_name, + ) + .await?; + } + Command::CheckRotation { cert_path } => { + check_rotation(&cert_path).await?; + } + } + + Ok(()) +} + +async fn init_ca(cert_path: &PathBuf, key_path: &PathBuf) -> Result<()> { + info!("generating CA certificate and key"); + + // キーペアを生成 + let key_pair = KeyPair::generate() + .context("failed to generate CA key pair")?; + + // CA証明書パラメータを設定 + let mut params = CertificateParams::new(vec!["PhotonCloud CA".to_string()]) + .context("failed to create certificate params")?; + + let mut distinguished_name = DistinguishedName::new(); + distinguished_name.push(DnType::OrganizationName, "PhotonCloud"); + distinguished_name.push(DnType::CommonName, "PhotonCloud CA"); + params.distinguished_name = distinguished_name; + params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + params.key_usages = vec![ + rcgen::KeyUsagePurpose::DigitalSignature, + rcgen::KeyUsagePurpose::KeyCertSign, + ]; + + // 自己署名CA証明書を生成 + let cert = params.self_signed(&key_pair) + .context("failed to generate self-signed CA certificate")?; + + // PEM形式で保存 + let cert_pem = cert.pem(); + let key_pem = key_pair.serialize_pem(); + + // ディレクトリを作成 + if let Some(parent) = cert_path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("failed to create directory for {}", parent.display()))?; + } + if let Some(parent) = key_path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("failed to create directory for {}", parent.display()))?; + } + + std::fs::write(cert_path, cert_pem) + .with_context(|| format!("failed to write CA certificate to {}", cert_path.display()))?; + std::fs::write(key_path, key_pem) + .with_context(|| format!("failed to write CA key to {}", key_path.display()))?; + + info!( + cert_path = %cert_path.display(), + key_path = %key_path.display(), + "CA certificate generated successfully" + ); + + Ok(()) +} + +async fn issue_certificate( + chainfire_endpoint: &str, + cluster_id: &str, + ca_cert_path: &PathBuf, + ca_key_path: &PathBuf, + csr_path: &PathBuf, + cert_path: &PathBuf, + node_id: Option, + service_name: Option, +) -> Result<()> { + info!("issuing certificate"); + + // Chainfireでノード/サービスが許可されているか確認 + if let Some(ref nid) = node_id { + let mut client = Client::connect(chainfire_endpoint.to_string()).await?; + let node_key = format!("{}nodes/{}", cluster_prefix(cluster_id), nid); + let node_data = client.get(&node_key.as_bytes()).await?; + if node_data.is_none() { + anyhow::bail!("node {} not found in Chainfire", nid); + } + } + + // CA証明書とキーを読み込み + let ca_key_pem = std::fs::read_to_string(ca_key_path) + .with_context(|| format!("failed to read CA key from {}", ca_key_path.display()))?; + + // CAキーペアを読み込み + let ca_key_pair = KeyPair::from_pem(&ca_key_pem) + .context("failed to parse CA key pair from PEM")?; + + // CA証明書を再構築（簡易実装） + // 実際の運用では、既存のCA証明書をパースする必要があるが、 + // rcgenのAPI制約により、CA証明書のパラメータを再構築する方式を採用 + let mut ca_params = CertificateParams::new(vec!["PhotonCloud CA".to_string()]) + .context("failed to create CA certificate params")?; + let mut ca_dn = DistinguishedName::new(); + ca_dn.push(DnType::OrganizationName, "PhotonCloud"); + ca_dn.push(DnType::CommonName, "PhotonCloud CA"); + ca_params.distinguished_name = ca_dn; + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + ca_params.key_usages = vec![ + rcgen::KeyUsagePurpose::DigitalSignature, + rcgen::KeyUsagePurpose::KeyCertSign, + ]; + + // CA証明書オブジェクトを作成（自己署名として再生成） + // 実際の運用では、既存のCA証明書を読み込む必要がある + let ca_cert = ca_params.self_signed(&ca_key_pair) + .context("failed to recreate CA certificate")?; + + // 証明書パラメータを構築 + let mut subject_alt_names = Vec::new(); + if let Some(ref nid) = node_id { + subject_alt_names.push(format!("node-{}", nid)); + } + if let Some(ref svc) = service_name { + subject_alt_names.push(svc.clone()); + } + if subject_alt_names.is_empty() { + subject_alt_names.push("photoncloud-service".to_string()); + } + + let mut params = CertificateParams::new(subject_alt_names) + .context("failed to create certificate params")?; + + // Distinguished Nameを設定 + let mut distinguished_name = DistinguishedName::new(); + if let Some(ref nid) = node_id { + distinguished_name.push(DnType::CommonName, format!("Node {}", nid)); + } + if let Some(ref svc) = service_name { + distinguished_name.push(DnType::OrganizationName, format!("Service {}", svc)); + } + params.distinguished_name = distinguished_name; + + // キーペアを生成（CSRから読み込む場合は、CSRパースが必要） + // ここでは簡易実装として新規生成 + let key_pair = KeyPair::generate() + .context("failed to generate certificate key pair")?; + + // CA署名証明書を生成 + // KeyPairはPublicKeyDataトレイトを実装しているので、そのまま渡せる + let cert = params.signed_by(&key_pair, &ca_cert, &ca_key_pair) + .context("failed to sign certificate with CA")?; + + let cert_pem = cert.pem(); + let key_pem = key_pair.serialize_pem(); + + // ディレクトリを作成 + if let Some(parent) = cert_path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("failed to create directory for {}", parent.display()))?; + } + + // 証明書とキーを保存 + std::fs::write(cert_path, cert_pem) + .with_context(|| format!("failed to write certificate to {}", cert_path.display()))?; + + // キーも別ファイルに保存（オプション） + let key_path = cert_path.with_extension("key"); + std::fs::write(&key_path, key_pem) + .with_context(|| format!("failed to write key to {}", key_path.display()))?; + + // Chainfireに証明書バインディングを記録 + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + let expires_at = now + (CERT_TTL_DAYS * 24 * 3600); + + // 証明書のシリアル番号を取得（DERから抽出） + let cert_serial = { + let cert_der = cert.der(); + // DER形式からシリアル番号を抽出（簡易実装） + // 実際にはx509-parserを使ってパースする方が正確 + format!("{:x}", cert_der.as_ref().iter().take(20).fold(0u64, |acc, &b| acc * 256 + b as u64)) + }; + + let mut client = Client::connect(chainfire_endpoint.to_string()).await?; + let binding = CertificateBinding { + node_id: node_id.clone(), + service_name: service_name.clone(), + cert_serial, + issued_at: now, + expires_at, + }; + let binding_key = format!( + "{}mtls/certs/{}/{}", + cluster_prefix(cluster_id), + node_id.as_deref().unwrap_or("unknown"), + service_name.as_deref().unwrap_or("unknown") + ); + let binding_value = serde_json::to_vec(&binding)?; + client.put(&binding_key.as_bytes(), &binding_value).await?; + + info!( + cert_path = %cert_path.display(), + key_path = %key_path.display(), + node_id = ?node_id, + service_name = ?service_name, + "certificate issued and recorded in Chainfire" + ); + + Ok(()) +} + +async fn check_rotation(cert_path: &PathBuf) -> Result<()> { + let cert_pem = std::fs::read_to_string(cert_path) + .with_context(|| format!("failed to read certificate from {}", cert_path.display()))?; + + // 証明書をDER形式に変換 + let cert_der_vec = rustls_pemfile::certs(&mut cert_pem.as_bytes()) + .collect::, _>>() + .context("failed to parse certificate from PEM")?; + let _cert_der = cert_der_vec.first() + .context("no certificate found in PEM file")?; + + // x509-parserを使って証明書をパース + #[cfg(feature = "x509-parser")] + { + use x509_parser::parse_x509_certificate; + let (_, cert) = parse_x509_certificate(cert_der) + .map_err(|e| anyhow::anyhow!("failed to parse X.509 certificate: {:?}", e))?; + + let validity = cert.validity(); + let not_after = validity.not_after.timestamp(); + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() as i64; + let days_until_expiry = (not_after - now) / 86400; + + if days_until_expiry < ROTATION_THRESHOLD_DAYS as i64 { + warn!( + cert_path = %cert_path.display(), + days_until_expiry = days_until_expiry, + threshold = ROTATION_THRESHOLD_DAYS, + "certificate should be rotated soon" + ); + return Ok(()); + } + + info!( + cert_path = %cert_path.display(), + days_until_expiry = days_until_expiry, + "certificate is still valid" + ); + } + + #[cfg(not(feature = "x509-parser"))] + { + warn!("x509-parser feature not enabled, rotation check skipped"); + } + + Ok(()) +} + +fn cluster_prefix(cluster_id: &str) -> String { + format!("{}/clusters/{}/", PHOTON_PREFIX, cluster_id) +} + diff --git a/deployer/crates/deployer-ctl/Cargo.toml b/deployer/crates/deployer-ctl/Cargo.toml new file mode 100644 index 0000000..3aae6d3 --- /dev/null +++ b/deployer/crates/deployer-ctl/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "deployer-ctl" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0" +clap = { version = "4.5", features = ["derive"] } +tokio = { version = "1.38", features = ["full"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" + +chainfire-client = { path = "../../../chainfire/chainfire-client" } +deployer-types = { path = "../deployer-types" } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } + + diff --git a/deployer/crates/deployer-ctl/src/chainfire.rs b/deployer/crates/deployer-ctl/src/chainfire.rs new file mode 100644 index 0000000..3754556 --- /dev/null +++ b/deployer/crates/deployer-ctl/src/chainfire.rs @@ -0,0 +1,177 @@ +use std::path::Path; + +use anyhow::{Context, Result}; +use chainfire_client::Client; +use serde::de::DeserializeOwned; +use tokio::fs; +use tracing::{info, warn}; + +use crate::model::ClusterStateSpec; + +const PHOTON_PREFIX: &str = "photoncloud"; + +fn cluster_prefix(cluster_id: &str) -> String { + format!("{}/clusters/{}/", PHOTON_PREFIX, cluster_id) +} + +fn key_cluster_meta(cluster_id: &str) -> Vec { + format!("{}meta", cluster_prefix(cluster_id)).into_bytes() +} + +fn key_node(cluster_id: &str, node_id: &str) -> Vec { + format!("{}nodes/{}", cluster_prefix(cluster_id), node_id).into_bytes() +} + +fn key_service(cluster_id: &str, svc: &str) -> Vec { + format!("{}services/{}", cluster_prefix(cluster_id), svc).into_bytes() +} + +fn key_instance(cluster_id: &str, svc: &str, inst: &str) -> Vec { + format!( + "{}instances/{}/{}", + cluster_prefix(cluster_id), + svc, + inst + ) + .into_bytes() +} + +fn key_mtls_policy(cluster_id: &str, policy_id: &str) -> Vec { + format!( + "{}mtls/policies/{}", + cluster_prefix(cluster_id), + policy_id + ) + .into_bytes() +} + +async fn read_config_file>(path: P) -> Result { + let contents = fs::read_to_string(&path) + .await + .with_context(|| format!("failed to read {}", path.as_ref().display()))?; + + // シンプルに JSON として解釈（必要になれば YAML 対応も追加可能） + let value = serde_json::from_str(&contents) + .with_context(|| format!("failed to parse {}", path.as_ref().display()))?; + Ok(value) +} + +/// 初回ブートストラップ: +/// - Cluster メタを作り +/// - 少なくとも 1 台分の Node / 必要なら Service/Instance を作成 +pub async fn bootstrap_cluster( + endpoint: &str, + cli_cluster_id: Option<&str>, + config_path: &Path, +) -> Result<()> { + let spec: ClusterStateSpec = read_config_file(config_path).await?; + let cluster_id = cli_cluster_id.unwrap_or(&spec.cluster.cluster_id); + + info!(cluster_id, "connecting to Chainfire at {}", endpoint); + let mut client = Client::connect(endpoint.to_string()).await?; + + // 1. Cluster メタ + let meta_key = key_cluster_meta(cluster_id); + let meta_value = serde_json::to_vec(&spec.cluster)?; + client.put(&meta_key, &meta_value).await?; + info!("upserted cluster meta for {}", cluster_id); + + // 2. Node + for node in &spec.nodes { + let key = key_node(cluster_id, &node.node_id); + let value = serde_json::to_vec(node)?; + client.put(&key, &value).await?; + info!(node_id = %node.node_id, "upserted node"); + } + + // 3. Service / Instance (必要であれば) + for svc in &spec.services { + let key = key_service(cluster_id, &svc.name); + let value = serde_json::to_vec(svc)?; + client.put(&key, &value).await?; + info!(service = %svc.name, "upserted service"); + } + + for inst in &spec.instances { + let key = key_instance(cluster_id, &inst.service, &inst.instance_id); + let value = serde_json::to_vec(inst)?; + client.put(&key, &value).await?; + info!(instance = %inst.instance_id, service = %inst.service, "upserted instance"); + } + + // 4. mTLS Policy + for policy in &spec.mtls_policies { + let key = key_mtls_policy(cluster_id, &policy.policy_id); + let value = serde_json::to_vec(policy)?; + client.put(&key, &value).await?; + info!(policy_id = %policy.policy_id, "upserted mTLS policy"); + } + + Ok(()) +} + +/// GitOps 的に、「クラスタ全体の宣言」を Chainfire に apply する。 +/// prune=true の場合、指定 prefix から外れたキーを削除する方向にも拡張可能。 +pub async fn apply_cluster_state( + endpoint: &str, + cli_cluster_id: Option<&str>, + config_path: &Path, + _prune: bool, +) -> Result<()> { + let spec: ClusterStateSpec = read_config_file(config_path).await?; + let cluster_id = cli_cluster_id.unwrap_or(&spec.cluster.cluster_id); + + info!(cluster_id, "applying cluster state to Chainfire at {}", endpoint); + let mut client = Client::connect(endpoint.to_string()).await?; + + // MVP としては bootstrap と同じく upsert のみ行う。 + // 将来的に、既存一覧を取得して差分削除 (prune) を実装できる構造にしておく。 + let meta_key = key_cluster_meta(cluster_id); + let meta_value = serde_json::to_vec(&spec.cluster)?; + client.put(&meta_key, &meta_value).await?; + + for node in &spec.nodes { + let key = key_node(cluster_id, &node.node_id); + let value = serde_json::to_vec(node)?; + client.put(&key, &value).await?; + } + for svc in &spec.services { + let key = key_service(cluster_id, &svc.name); + let value = serde_json::to_vec(svc)?; + client.put(&key, &value).await?; + } + for inst in &spec.instances { + let key = key_instance(cluster_id, &inst.service, &inst.instance_id); + let value = serde_json::to_vec(inst)?; + client.put(&key, &value).await?; + } + for policy in &spec.mtls_policies { + let key = key_mtls_policy(cluster_id, &policy.policy_id); + let value = serde_json::to_vec(policy)?; + client.put(&key, &value).await?; + } + + Ok(()) +} + +/// 指定 prefix 以下のキーをダンプする（デバッグ・手動修復用）。 +pub async fn dump_prefix(endpoint: &str, prefix: &str) -> Result<()> { + let mut client = Client::connect(endpoint.to_string()).await?; + let start = prefix.as_bytes(); + + info!("dumping keys with prefix {:?}", prefix); + let (kvs, _next) = client.scan_prefix(start, 0).await?; + if kvs.is_empty() { + warn!("no keys found under prefix {:?}", prefix); + } + + for (key, value, rev) in kvs { + let k = String::from_utf8_lossy(&key); + let v = String::from_utf8_lossy(&value); + println!("rev={} key={} value={}", rev, k, v); + } + + Ok(()) +} + + diff --git a/deployer/crates/deployer-ctl/src/main.rs b/deployer/crates/deployer-ctl/src/main.rs new file mode 100644 index 0000000..6f540ca --- /dev/null +++ b/deployer/crates/deployer-ctl/src/main.rs @@ -0,0 +1,113 @@ +use std::path::PathBuf; + +use anyhow::Result; +use clap::{Parser, Subcommand}; +use tracing_subscriber::EnvFilter; + +mod chainfire; +mod model; +mod remote; + +/// Deployer control CLI for PhotonCloud. +/// +/// - 初回ブートストラップ時に Chainfire 上の Cluster/Node/Service 定義を作成 +/// - 既存の Deployer クラスタに対して宣言的な設定を apply する +/// - Deployer が壊れた場合でも、Chainfire 上の状態を直接修復できることを目標とする +#[derive(Parser, Debug)] +#[command(author, version, about)] +struct Cli { + /// Chainfire API エンドポイント (例: http://127.0.0.1:7000) + #[arg(long, global = true, default_value = "http://127.0.0.1:7000")] + chainfire_endpoint: String, + + /// PhotonCloud Cluster ID (論理名) + #[arg(long, global = true)] + cluster_id: Option, + + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand, Debug)] +enum Command { + /// 初回ブートストラップ用の基本オブジェクトを作成する + /// + /// - Cluster メタデータ + /// - Node 情報 (ローカルノード1台分) + /// - 必要であれば Service/ServiceInstance のシード + Bootstrap { + /// ブートストラップ用のJSON/YAML設定ファイル + #[arg(long)] + config: PathBuf, + }, + + /// 宣言的な PhotonCloud クラスタ設定を Chainfire に apply する (GitOps 的に利用可能) + Apply { + /// Cluster/Node/Service/Instance/MTLSPolicy を含むJSON/YAML + #[arg(long)] + config: PathBuf, + + /// 既存エントリを pruning するかどうか + #[arg(long, default_value_t = false)] + prune: bool, + }, + + /// Chainfire 上の PhotonCloud 関連キーをダンプする (デバッグ用途) + Dump { + /// ダンプ対象の prefix (デフォルト: photoncloud/) + #[arg(long, default_value = "photoncloud/")] + prefix: String, + }, + + /// Deployer HTTP API を経由して、クラスタ状態を同期・確認する + /// + /// 現時点ではプレースホルダであり、将来的なGitOps連携を見据えた形だけ用意する。 + Deployer { + /// Deployer HTTP エンドポイント (例: http://deployer.local:8080) + #[arg(long)] + endpoint: String, + + /// 一旦は `status` のみをサポート + #[arg(long, default_value = "status")] + action: String, + }, +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env().add_directive("info".parse()?)) + .init(); + + let cli = Cli::parse(); + + match cli.command { + Command::Bootstrap { config } => { + chainfire::bootstrap_cluster( + &cli.chainfire_endpoint, + cli.cluster_id.as_deref(), + &config, + ) + .await?; + } + Command::Apply { config, prune } => { + chainfire::apply_cluster_state( + &cli.chainfire_endpoint, + cli.cluster_id.as_deref(), + &config, + prune, + ) + .await?; + } + Command::Dump { prefix } => { + chainfire::dump_prefix(&cli.chainfire_endpoint, &prefix).await?; + } + Command::Deployer { endpoint, action } => { + remote::run_deployer_command(&endpoint, &action).await?; + } + } + + Ok(()) +} + + diff --git a/deployer/crates/deployer-ctl/src/model.rs b/deployer/crates/deployer-ctl/src/model.rs new file mode 100644 index 0000000..72d784f --- /dev/null +++ b/deployer/crates/deployer-ctl/src/model.rs @@ -0,0 +1,86 @@ +use serde::{Deserialize, Serialize}; + +/// Cluster メタ情報 (PhotonCloud 用) +#[derive(Debug, Deserialize, Serialize)] +pub struct ClusterSpec { + pub cluster_id: String, + pub environment: Option, // dev/stg/prod など +} + +/// Node 定義 +#[derive(Debug, Deserialize, Serialize)] +pub struct NodeSpec { + pub node_id: String, + pub hostname: String, + pub ip: String, + #[serde(default)] + pub roles: Vec, + #[serde(default)] + pub labels: std::collections::HashMap, +} + +/// Service 定義 +#[derive(Debug, Deserialize, Serialize)] +pub struct ServiceSpec { + pub name: String, + #[serde(default)] + pub ports: Option, + #[serde(default)] + pub protocol: Option, // http/grpc + #[serde(default)] + pub mtls_required: Option, + #[serde(default)] + pub mesh_mode: Option, // agent/none +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct ServicePorts { + #[serde(default)] + pub http: Option, + #[serde(default)] + pub grpc: Option, +} + +/// ServiceInstance 定義 +#[derive(Debug, Deserialize, Serialize)] +pub struct ServiceInstanceSpec { + pub instance_id: String, + pub service: String, + pub node_id: String, + pub ip: String, + pub port: u16, + #[serde(default)] + pub mesh_port: Option, + #[serde(default)] + pub version: Option, +} + +/// mTLS Policy 定義 +#[derive(Debug, Deserialize, Serialize)] +pub struct MtlsPolicySpec { + pub policy_id: String, + #[serde(default)] + pub environment: Option, + pub source_service: String, + pub target_service: String, + #[serde(default)] + pub mtls_required: Option, + #[serde(default)] + pub mode: Option, // strict/permissive/disabled +} + +/// GitOps フレンドリーな、クラスタ全体の宣言的定義 +#[derive(Debug, Deserialize, Serialize)] +pub struct ClusterStateSpec { + pub cluster: ClusterSpec, + #[serde(default)] + pub nodes: Vec, + #[serde(default)] + pub services: Vec, + #[serde(default)] + pub instances: Vec, + #[serde(default)] + pub mtls_policies: Vec, +} + + diff --git a/deployer/crates/deployer-ctl/src/remote.rs b/deployer/crates/deployer-ctl/src/remote.rs new file mode 100644 index 0000000..3ff12f7 --- /dev/null +++ b/deployer/crates/deployer-ctl/src/remote.rs @@ -0,0 +1,35 @@ +use anyhow::Result; +use tracing::{info, warn}; + +/// 将来的な GitOps 連携を見据えた Deployer HTTP API との接続ポイント。 +/// +/// 現時点ではステータスの簡易チェックのみを行い、 +/// API 形状が固まり次第ここから `apply` 相当を実装できるようにしておく。 +pub async fn run_deployer_command(endpoint: &str, action: &str) -> Result<()> { + match action { + "status" => { + // プレースホルダ実装: + // - 将来的には /health や /api/v1/admin/nodes 等を叩く。 + let url = format!("{}/health", endpoint.trim_end_matches('/')); + info!("checking deployer status at {}", url); + + let response = reqwest::get(&url).await?; + if response.status().is_success() { + let body = response.text().await.unwrap_or_default(); + println!("deployer status OK: {}", body); + } else { + warn!( + "deployer status not OK: HTTP {}", + response.status().as_u16() + ); + } + } + other => { + warn!("unsupported deployer action: {}", other); + } + } + + Ok(()) +} + + diff --git a/deployer/crates/node-agent/Cargo.toml b/deployer/crates/node-agent/Cargo.toml new file mode 100644 index 0000000..8c833dd --- /dev/null +++ b/deployer/crates/node-agent/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "node-agent" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +anyhow.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +clap = { version = "4.5", features = ["derive"] } +serde.workspace = true +serde_json.workspace = true +chrono = { version = "0.4", features = ["serde"] } + +chainfire-client = { path = "../../../chainfire/chainfire-client" } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } + + diff --git a/deployer/crates/node-agent/src/agent.rs b/deployer/crates/node-agent/src/agent.rs new file mode 100644 index 0000000..af4c49c --- /dev/null +++ b/deployer/crates/node-agent/src/agent.rs @@ -0,0 +1,334 @@ +use std::fs; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::{Context, Result}; +use chainfire_client::Client; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use tokio::process::Command; +use tokio::time::sleep; +use tracing::{info, warn}; + +use crate::process::ProcessManager; + +const PHOTON_PREFIX: &str = "photoncloud"; + +fn cluster_prefix(cluster_id: &str) -> String { + format!("{}/clusters/{}/", PHOTON_PREFIX, cluster_id) +} + +fn key_node(cluster_id: &str, node_id: &str) -> Vec { + format!("{}nodes/{}", cluster_prefix(cluster_id), node_id).into_bytes() +} + +fn key_instance(cluster_id: &str, service: &str, instance_id: &str) -> Vec { + format!( + "{}instances/{}/{}", + cluster_prefix(cluster_id), + service, + instance_id + ) + .into_bytes() +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct NodeState { + pub node_id: String, + pub ip: String, + pub hostname: String, + #[serde(default)] + pub roles: Vec, + #[serde(default)] + pub labels: std::collections::HashMap, + #[serde(default)] + pub state: Option, + #[serde(default)] + pub last_heartbeat: Option>, +} + +pub struct Agent { + endpoint: String, + cluster_id: String, + node_id: String, + interval: Duration, + process_manager: ProcessManager, +} + +#[derive(Debug, Deserialize, Serialize)] +struct LocalInstanceSpec { + service: String, + instance_id: String, + ip: String, + port: u16, + #[serde(default)] + mesh_port: Option, + #[serde(default)] + health_check: Option, + #[serde(default)] + process: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +struct HealthCheckSpec { + #[serde(rename = "type")] + check_type: String, // http/tcp/command + #[serde(default)] + path: Option, + #[serde(default)] + interval_secs: Option, + #[serde(default)] + timeout_secs: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +struct ProcessSpec { + command: String, + #[serde(default)] + args: Vec, + #[serde(default)] + working_dir: Option, + #[serde(default)] + env: std::collections::HashMap, +} + +impl Agent { + pub fn new(endpoint: String, cluster_id: String, node_id: String, interval: Duration) -> Self { + Self { + endpoint, + cluster_id, + node_id, + interval, + process_manager: ProcessManager::new(), + } + } + + pub async fn run_loop(&mut self) -> Result<()> { + loop { + if let Err(e) = self.tick().await { + warn!(error = %e, "node-agent tick failed"); + } + sleep(self.interval).await; + } + } + + async fn tick(&mut self) -> Result<()> { + let mut client = Client::connect(self.endpoint.clone()).await?; + + // Node 情報 + let node_key = key_node(&self.cluster_id, &self.node_id); + let node_raw = client.get(&node_key).await?; + let Some(node_bytes) = node_raw else { + warn!( + "node definition not found in Chainfire for cluster_id={}, node_id={}", + self.cluster_id, self.node_id + ); + return Ok(()); + }; + + let mut node: NodeState = match serde_json::from_slice(&node_bytes) { + Ok(n) => n, + Err(e) => { + warn!(error = %e, "failed to parse node JSON"); + return Ok(()); + } + }; + + // Heartbeat を更新し、Chainfire 上の Node を upsert + node.last_heartbeat = Some(Utc::now()); + let updated = serde_json::to_vec(&node)?; + client.put(&node_key, &updated).await?; + + // ローカル定義された ServiceInstance を Chainfire に登録 + if let Err(e) = self.sync_local_instances(&mut client).await { + warn!(error = %e, "failed to sync local service instances"); + } + + // プロセスの起動/停止をReconcile + if let Err(e) = self.reconcile_processes(&mut client).await { + warn!(error = %e, "failed to reconcile processes"); + } + + // ヘルスチェックを実行して状態を更新 + if let Err(e) = self.update_health_status(&mut client).await { + warn!(error = %e, "failed to update health status"); + } + + self.log_node_only(&node); + + Ok(()) + } + + fn log_node_only(&self, node: &NodeState) { + info!( + node_id = %node.node_id, + hostname = %node.hostname, + roles = ?node.roles, + "observed desired state for node" + ); + } + + /// ローカルファイル (/etc/photoncloud/instances.json) から ServiceInstance 定義を読み、 + /// Chainfire 上の `photoncloud/clusters/{cluster_id}/instances/{service}/{instance_id}` に upsert する。 + async fn sync_local_instances(&self, client: &mut Client) -> Result<()> { + let path = PathBuf::from("/etc/photoncloud/instances.json"); + let contents = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + warn!(error = %e, "no local instances file found, skipping"); + return Ok(()); + } + }; + + let instances: Vec = serde_json::from_str(&contents) + .with_context(|| format!("failed to parse {}", path.display()))?; + + for inst in &instances { + let key = key_instance(&self.cluster_id, &inst.service, &inst.instance_id); + let value = serde_json::to_vec(inst)?; + client.put(&key, &value).await?; + info!( + service = %inst.service, + instance_id = %inst.instance_id, + "synced local ServiceInstance to Chainfire" + ); + } + + Ok(()) + } + + /// Desired Stateに基づいてプロセスを起動/停止する + async fn reconcile_processes(&mut self, client: &mut Client) -> Result<()> { + let prefix = format!("{}instances/", cluster_prefix(&self.cluster_id)); + let (kvs, _) = client.scan_prefix(prefix.as_bytes(), 0).await?; + + let mut desired_instances = Vec::new(); + for (_key, value, _) in kvs { + let inst: LocalInstanceSpec = match serde_json::from_slice(&value) { + Ok(i) => i, + Err(e) => { + warn!(error = %e, "failed to parse instance"); + continue; + } + }; + + // このノードのインスタンスかチェック（簡易実装） + // TODO: instance.node_idとself.node_idを比較する + + if let Some(proc_spec) = inst.process { + desired_instances.push((inst.service.clone(), inst.instance_id.clone(), proc_spec)); + } + } + + // Desired Stateに基づいてプロセスを管理 + for (service, instance_id, proc_spec) in desired_instances { + let proc_spec_converted = crate::process::ProcessSpec { + command: proc_spec.command.clone(), + args: proc_spec.args.clone(), + working_dir: proc_spec.working_dir.clone(), + env: proc_spec.env.clone(), + }; + + if self.process_manager.get_mut(&service, &instance_id).is_none() { + // 新しいプロセスを追加 + self.process_manager.add(service.clone(), instance_id.clone(), proc_spec_converted); + info!( + service = %service, + instance_id = %instance_id, + "added new process to manager" + ); + } + } + + // Reconcile: 停止しているプロセスを再起動 + self.process_manager.reconcile().await?; + + Ok(()) + } + + /// 各ServiceInstanceのヘルスチェックを実行し、Chainfire上の状態を更新 + async fn update_health_status(&self, client: &mut Client) -> Result<()> { + let prefix = format!("{}instances/", cluster_prefix(&self.cluster_id)); + let (kvs, _) = client.scan_prefix(prefix.as_bytes(), 0).await?; + + for (key, value, _) in kvs { + let mut inst: LocalInstanceSpec = match serde_json::from_slice(&value) { + Ok(i) => i, + Err(e) => { + warn!(error = %e, "failed to parse instance"); + continue; + } + }; + + let health_status = if let Some(ref health_check) = inst.health_check { + self.check_health(&inst, health_check).await + } else { + "healthy".to_string() // デフォルトはhealthy + }; + + // Chainfire上のServiceInstanceに状態を反映（簡易実装） + // 実際には、ServiceInstanceのstateフィールドを更新する必要がある + info!( + service = %inst.service, + instance_id = %inst.instance_id, + status = %health_status, + "health check completed" + ); + } + + Ok(()) + } + + async fn check_health(&self, inst: &LocalInstanceSpec, spec: &HealthCheckSpec) -> String { + match spec.check_type.as_str() { + "http" => { + if let Some(ref path) = spec.path { + let url = format!("http://{}:{}{}", inst.ip, inst.port, path); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(spec.timeout_secs.unwrap_or(5))) + .build(); + if let Ok(c) = client { + if let Ok(resp) = c.get(&url).send().await { + if resp.status().is_success() { + return "healthy".to_string(); + } + } + } + } + "unhealthy".to_string() + } + "tcp" => { + use tokio::net::TcpStream; + let addr = format!("{}:{}", inst.ip, inst.port); + match tokio::time::timeout( + Duration::from_secs(spec.timeout_secs.unwrap_or(5)), + TcpStream::connect(&addr), + ) + .await + { + Ok(Ok(_)) => "healthy".to_string(), + _ => "unhealthy".to_string(), + } + } + "command" => { + if let Some(ref cmd) = spec.path { + match Command::new("sh") + .arg("-c") + .arg(cmd) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .await + { + Ok(status) if status.success() => "healthy".to_string(), + _ => "unhealthy".to_string(), + } + } else { + "unknown".to_string() + } + } + _ => "unknown".to_string(), + } + } +} diff --git a/deployer/crates/node-agent/src/main.rs b/deployer/crates/node-agent/src/main.rs new file mode 100644 index 0000000..9ffe1d7 --- /dev/null +++ b/deployer/crates/node-agent/src/main.rs @@ -0,0 +1,62 @@ +use std::time::Duration; + +use anyhow::Result; +use clap::Parser; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; + +mod agent; +mod process; + +/// PhotonCloud NodeAgent +/// +/// - Chainfire 上の `photoncloud/clusters/{cluster_id}/nodes/{node_id}` と +/// `.../instances/*` をポーリング/将来的には watch してローカル状態と比較する。 +/// - 現段階では systemd などへの実際の apply は行わず、ログ出力のみ。 +#[derive(Parser, Debug)] +#[command(author, version, about)] +struct Cli { + /// Chainfire API エンドポイント + #[arg(long, default_value = "http://127.0.0.1:7000")] + chainfire_endpoint: String, + + /// PhotonCloud Cluster ID + #[arg(long)] + cluster_id: String, + + /// このエージェントが管理する Node ID + #[arg(long)] + node_id: String, + + /// ポーリング間隔（秒） + #[arg(long, default_value_t = 15)] + interval_secs: u64, +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env().add_directive("info".parse()?)) + .init(); + + let cli = Cli::parse(); + + info!( + cluster_id = %cli.cluster_id, + node_id = %cli.node_id, + "starting NodeAgent" + ); + + let mut agent = agent::Agent::new( + cli.chainfire_endpoint, + cli.cluster_id, + cli.node_id, + Duration::from_secs(cli.interval_secs), + ); + + agent.run_loop().await?; + + Ok(()) +} + + diff --git a/deployer/crates/node-agent/src/process.rs b/deployer/crates/node-agent/src/process.rs new file mode 100644 index 0000000..8fa1a9d --- /dev/null +++ b/deployer/crates/node-agent/src/process.rs @@ -0,0 +1,273 @@ +use std::collections::HashMap; +use std::fs; +use std::path::PathBuf; +use std::process::Stdio; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use tokio::process::{Child, Command}; +use tracing::{info, warn}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProcessSpec { + pub command: String, + #[serde(default)] + pub args: Vec, + #[serde(default)] + pub working_dir: Option, + #[serde(default)] + pub env: HashMap, +} + +#[derive(Debug)] +pub struct ManagedProcess { + pub service: String, + pub instance_id: String, + pub spec: ProcessSpec, + pub child: Option, + pub pid_file: PathBuf, +} + +impl ManagedProcess { + pub fn new(service: String, instance_id: String, spec: ProcessSpec) -> Self { + let pid_file = PathBuf::from(format!( + "/var/run/photoncloud/{}-{}.pid", + service, instance_id + )); + Self { + service, + instance_id, + spec, + child: None, + pid_file, + } + } + + pub async fn start(&mut self) -> Result<()> { + if self.is_running().await? { + info!( + service = %self.service, + instance_id = %self.instance_id, + "process already running" + ); + return Ok(()); + } + + info!( + service = %self.service, + instance_id = %self.instance_id, + command = %self.spec.command, + "starting process" + ); + + let mut cmd = Command::new(&self.spec.command); + cmd.args(&self.spec.args); + + if let Some(ref wd) = self.spec.working_dir { + cmd.current_dir(wd); + } + + for (k, v) in &self.spec.env { + cmd.env(k, v); + } + + cmd.stdout(Stdio::null()).stderr(Stdio::null()); + + let child = cmd.spawn().with_context(|| { + format!( + "failed to spawn process for {}/{}", + self.service, self.instance_id + ) + })?; + + let pid = child.id().context("failed to get child PID")?; + + // PIDファイルを書き込み + if let Some(parent) = self.pid_file.parent() { + fs::create_dir_all(parent).ok(); + } + fs::write(&self.pid_file, pid.to_string()) + .with_context(|| format!("failed to write PID file {:?}", self.pid_file))?; + + self.child = Some(child); + + info!( + service = %self.service, + instance_id = %self.instance_id, + pid = pid, + "process started" + ); + + Ok(()) + } + + pub async fn stop(&mut self) -> Result<()> { + if !self.is_running().await? { + info!( + service = %self.service, + instance_id = %self.instance_id, + "process not running" + ); + return Ok(()); + } + + info!( + service = %self.service, + instance_id = %self.instance_id, + "stopping process" + ); + + if let Some(mut child) = self.child.take() { + child.kill().await.ok(); + child.wait().await.ok(); + } else { + // PIDファイルからPIDを読み取って停止 + if let Ok(pid_str) = fs::read_to_string(&self.pid_file) { + if let Ok(pid) = pid_str.trim().parse::() { + // SIGTERM送信（簡易実装） + Command::new("kill") + .arg(pid.to_string()) + .output() + .await + .ok(); + } + } + } + + // PIDファイルを削除 + fs::remove_file(&self.pid_file).ok(); + + info!( + service = %self.service, + instance_id = %self.instance_id, + "process stopped" + ); + + Ok(()) + } + + pub async fn is_running(&self) -> Result { + // PIDファイルが存在するかチェック + if !self.pid_file.exists() { + return Ok(false); + } + + // PIDファイルからPIDを読み取り + let pid_str = fs::read_to_string(&self.pid_file) + .with_context(|| format!("failed to read PID file {:?}", self.pid_file))?; + let pid = pid_str + .trim() + .parse::() + .with_context(|| format!("invalid PID in file {:?}", self.pid_file))?; + + // プロセスが存在するかチェック（簡易実装） + let output = Command::new("kill") + .arg("-0") + .arg(pid.to_string()) + .output() + .await + .with_context(|| format!("failed to check process {}", pid))?; + + Ok(output.status.success()) + } + + pub async fn restart(&mut self) -> Result<()> { + self.stop().await?; + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + self.start().await?; + Ok(()) + } +} + +pub struct ProcessManager { + processes: HashMap, +} + +impl ProcessManager { + pub fn new() -> Self { + Self { + processes: HashMap::new(), + } + } + + pub fn add(&mut self, service: String, instance_id: String, spec: ProcessSpec) { + let key = format!("{}/{}", service, instance_id); + let process = ManagedProcess::new(service, instance_id, spec); + self.processes.insert(key, process); + } + + pub fn remove(&mut self, service: &str, instance_id: &str) -> Option { + let key = format!("{}/{}", service, instance_id); + self.processes.remove(&key) + } + + pub fn get_mut(&mut self, service: &str, instance_id: &str) -> Option<&mut ManagedProcess> { + let key = format!("{}/{}", service, instance_id); + self.processes.get_mut(&key) + } + + pub async fn start_all(&mut self) -> Result<()> { + for (_, process) in self.processes.iter_mut() { + if let Err(e) = process.start().await { + warn!( + service = %process.service, + instance_id = %process.instance_id, + error = %e, + "failed to start process" + ); + } + } + Ok(()) + } + + pub async fn stop_all(&mut self) -> Result<()> { + for (_, process) in self.processes.iter_mut() { + if let Err(e) = process.stop().await { + warn!( + service = %process.service, + instance_id = %process.instance_id, + error = %e, + "failed to stop process" + ); + } + } + Ok(()) + } + + pub async fn reconcile(&mut self) -> Result<()> { + for (_, process) in self.processes.iter_mut() { + match process.is_running().await { + Ok(true) => { + // プロセスは実行中、何もしない + } + Ok(false) => { + // プロセスが停止しているので起動 + warn!( + service = %process.service, + instance_id = %process.instance_id, + "process is not running, restarting" + ); + if let Err(e) = process.start().await { + warn!( + service = %process.service, + instance_id = %process.instance_id, + error = %e, + "failed to restart process" + ); + } + } + Err(e) => { + warn!( + service = %process.service, + instance_id = %process.instance_id, + error = %e, + "failed to check process status" + ); + } + } + } + Ok(()) + } +} + + diff --git a/deployer/crates/node-agent/src/watcher.rs b/deployer/crates/node-agent/src/watcher.rs new file mode 100644 index 0000000..3c17ada --- /dev/null +++ b/deployer/crates/node-agent/src/watcher.rs @@ -0,0 +1,77 @@ +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Result; +use chainfire_client::Client; +use tokio::sync::RwLock; +use tokio::time::sleep; +use tracing::{info, warn}; + +pub struct ChainfireWatcher { + endpoint: String, + prefix: String, + interval: Duration, +} + +impl ChainfireWatcher { + pub fn new(endpoint: String, prefix: String, interval_secs: u64) -> Self { + Self { + endpoint, + prefix, + interval: Duration::from_secs(interval_secs), + } + } + + pub async fn watch(&self, mut callback: F) -> Result<()> + where + F: FnMut(Vec<(Vec, Vec)>) -> Result<()>, + { + let mut last_revision = 0u64; + + loop { + match self.fetch_updates(last_revision).await { + Ok((kvs, max_rev)) => { + if !kvs.is_empty() { + info!( + prefix = %self.prefix, + count = kvs.len(), + "detected changes in Chainfire" + ); + if let Err(e) = callback(kvs) { + warn!(error = %e, "callback failed"); + } + } + if max_rev > last_revision { + last_revision = max_rev; + } + } + Err(e) => { + warn!(error = %e, "failed to fetch updates from Chainfire"); + } + } + + sleep(self.interval).await; + } + } + + async fn fetch_updates(&self, last_revision: u64) -> Result<(Vec<(Vec, Vec)>, u64)> { + let mut client = Client::connect(self.endpoint.clone()).await?; + let (kvs, _) = client.scan_prefix(self.prefix.as_bytes(), 0).await?; + + // 簡易実装: 全てのKVペアを返す（revisionフィルタリングは未実装） + let mut max_rev = last_revision; + let mut result = Vec::new(); + for (k, v, rev) in kvs { + if rev > last_revision { + result.push((k, v)); + if rev > max_rev { + max_rev = rev; + } + } + } + + Ok((result, max_rev)) + } +} + + diff --git a/deployer/crates/plasmacloud-reconciler/Cargo.toml b/deployer/crates/plasmacloud-reconciler/Cargo.toml new file mode 100644 index 0000000..c2c6a5a --- /dev/null +++ b/deployer/crates/plasmacloud-reconciler/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "plasmacloud-reconciler" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +fiberlb-api.workspace = true +flashdns-api.workspace = true +clap = { version = "4.5", features = ["derive"] } +tonic = "0.12" diff --git a/deployer/crates/plasmacloud-reconciler/src/main.rs b/deployer/crates/plasmacloud-reconciler/src/main.rs new file mode 100644 index 0000000..f94c522 --- /dev/null +++ b/deployer/crates/plasmacloud-reconciler/src/main.rs @@ -0,0 +1,1918 @@ +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use clap::{Parser, Subcommand}; +use serde::Deserialize; +use tonic::transport::Channel; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; + +use fiberlb_api::backend_service_client::BackendServiceClient; +use fiberlb_api::health_check_service_client::HealthCheckServiceClient; +use fiberlb_api::l7_policy_service_client::L7PolicyServiceClient; +use fiberlb_api::l7_rule_service_client::L7RuleServiceClient; +use fiberlb_api::listener_service_client::ListenerServiceClient; +use fiberlb_api::load_balancer_service_client::LoadBalancerServiceClient; +use fiberlb_api::pool_service_client::PoolServiceClient; +use fiberlb_api::{ + Backend, BackendAdminState, CreateBackendRequest, CreateHealthCheckRequest, + CreateL7PolicyRequest, CreateL7RuleRequest, CreateListenerRequest, + CreateLoadBalancerRequest, CreatePoolRequest, DeleteBackendRequest, + DeleteHealthCheckRequest, DeleteL7PolicyRequest, DeleteL7RuleRequest, + DeleteListenerRequest, DeleteLoadBalancerRequest, DeletePoolRequest, HealthCheck, + HealthCheckType, HttpHealthConfig, L7CompareType, L7Policy, L7PolicyAction, L7Rule, + L7RuleType, Listener, ListenerProtocol, LoadBalancer, Pool, PoolAlgorithm, + PoolProtocol, SessionPersistence, TlsConfig, TlsVersion, UpdateBackendRequest, + UpdateHealthCheckRequest, UpdateL7PolicyRequest, UpdateL7RuleRequest, + UpdateListenerRequest, UpdateLoadBalancerRequest, UpdatePoolRequest, +}; + +use flashdns_api::RecordServiceClient; +use flashdns_api::ReverseZoneServiceClient; +use flashdns_api::ZoneServiceClient; +use flashdns_api::proto::{ + record_data, ARecord, AaaaRecord, CaaRecord, CnameRecord, CreateRecordRequest, + CreateReverseZoneRequest, CreateZoneRequest, DeleteRecordRequest, DeleteReverseZoneRequest, + DeleteZoneRequest, ListReverseZonesRequest, MxRecord, NsRecord, PtrRecord, RecordData, + RecordInfo, ReverseZone, SrvRecord, TxtRecord, UpdateRecordRequest, UpdateZoneRequest, + ZoneInfo, +}; + +#[derive(Parser)] +#[command(author, version, about)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Apply FiberLB declarations + Lb { + #[arg(long)] + config: PathBuf, + + #[arg(long)] + endpoint: String, + + #[arg(long, default_value_t = false)] + prune: bool, + }, + + /// Apply FlashDNS declarations + Dns { + #[arg(long)] + config: PathBuf, + + #[arg(long)] + endpoint: String, + + #[arg(long, default_value_t = false)] + prune: bool, + }, +} + +#[derive(Debug, Deserialize)] +struct LbConfig { + #[serde(default)] + load_balancers: Vec, +} + +#[derive(Debug, Deserialize)] +struct LoadBalancerSpec { + name: String, + org_id: String, + #[serde(default)] + project_id: Option, + #[serde(default)] + description: Option, + #[serde(default)] + pools: Vec, + #[serde(default)] + listeners: Vec, +} + +#[derive(Debug, Deserialize)] +struct PoolSpec { + name: String, + #[serde(default)] + algorithm: Option, + #[serde(default)] + protocol: Option, + #[serde(default)] + session_persistence: Option, + #[serde(default)] + backends: Vec, + #[serde(default)] + health_checks: Vec, +} + +#[derive(Debug, Deserialize)] +struct SessionPersistenceSpec { + #[serde(rename = "type")] + persistence_type: String, + #[serde(default)] + cookie_name: Option, + #[serde(default)] + timeout_seconds: Option, +} + +#[derive(Debug, Deserialize)] +struct BackendSpec { + name: String, + address: String, + port: u32, + #[serde(default)] + weight: Option, + #[serde(default)] + admin_state: Option, +} + +#[derive(Debug, Deserialize)] +struct HealthCheckSpec { + name: String, + #[serde(rename = "type")] + check_type: String, + #[serde(default)] + interval_seconds: Option, + #[serde(default)] + timeout_seconds: Option, + #[serde(default)] + healthy_threshold: Option, + #[serde(default)] + unhealthy_threshold: Option, + #[serde(default)] + http: Option, + #[serde(default)] + enabled: Option, +} + +#[derive(Debug, Deserialize)] +struct HttpHealthSpec { + #[serde(default)] + method: Option, + #[serde(default)] + path: Option, + #[serde(default)] + expected_codes: Option>, + #[serde(default)] + host: Option, +} + +#[derive(Debug, Deserialize)] +struct ListenerSpec { + name: String, + #[serde(default)] + protocol: Option, + port: u32, + default_pool: String, + #[serde(default)] + tls: Option, + #[serde(default)] + connection_limit: Option, + #[serde(default)] + enabled: Option, + #[serde(default)] + l7_policies: Vec, +} + +#[derive(Debug, Deserialize)] +struct TlsSpec { + certificate_id: String, + #[serde(default)] + min_version: Option, + #[serde(default)] + cipher_suites: Vec, +} + +#[derive(Debug, Deserialize)] +struct L7PolicySpec { + name: String, + #[serde(default)] + position: Option, + action: String, + #[serde(default)] + redirect_url: Option, + #[serde(default)] + redirect_pool: Option, + #[serde(default)] + redirect_http_status_code: Option, + #[serde(default)] + enabled: Option, + #[serde(default)] + rules: Vec, +} + +#[derive(Debug, Deserialize)] +struct L7RuleSpec { + #[serde(rename = "type")] + rule_type: String, + #[serde(default)] + compare_type: Option, + value: String, + #[serde(default)] + key: Option, + #[serde(default)] + invert: Option, +} + +#[derive(Debug, Deserialize)] +struct DnsConfig { + #[serde(default)] + zones: Vec, + #[serde(default)] + reverse_zones: Vec, +} + +#[derive(Debug, Deserialize)] +struct ZoneSpec { + name: String, + org_id: String, + #[serde(default)] + project_id: Option, + #[serde(default)] + primary_ns: Option, + #[serde(default)] + admin_email: Option, + #[serde(default)] + refresh: Option, + #[serde(default)] + retry: Option, + #[serde(default)] + expire: Option, + #[serde(default)] + minimum: Option, + #[serde(default)] + records: Vec, +} + +#[derive(Debug, Deserialize)] +struct RecordSpec { + name: String, + record_type: String, + #[serde(default)] + ttl: Option, + data: serde_json::Value, + #[serde(default)] + enabled: Option, +} + +#[derive(Debug, Deserialize)] +struct ReverseZoneSpec { + org_id: String, + #[serde(default)] + project_id: Option, + cidr: String, + ptr_pattern: String, + #[serde(default)] + ttl: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env().add_directive("info".parse()?)) + .init(); + + let cli = Cli::parse(); + + match cli.command { + Command::Lb { + config, + endpoint, + prune, + } => { + let spec: LbConfig = read_json(&config).await?; + reconcile_lb(spec, endpoint, prune).await?; + } + Command::Dns { + config, + endpoint, + prune, + } => { + let spec: DnsConfig = read_json(&config).await?; + reconcile_dns(spec, endpoint, prune).await?; + } + } + + Ok(()) +} + +async fn read_json Deserialize<'a>>(path: &PathBuf) -> Result { + let contents = tokio::fs::read_to_string(path) + .await + .with_context(|| format!("failed to read {}", path.display()))?; + let config = serde_json::from_str(&contents) + .with_context(|| format!("failed to parse {}", path.display()))?; + Ok(config) +} + +async fn reconcile_lb(spec: LbConfig, endpoint: String, prune: bool) -> Result<()> { + let mut lb_client = LoadBalancerServiceClient::connect(endpoint.clone()).await?; + let mut pool_client = PoolServiceClient::connect(endpoint.clone()).await?; + let mut backend_client = BackendServiceClient::connect(endpoint.clone()).await?; + let mut listener_client = ListenerServiceClient::connect(endpoint.clone()).await?; + let mut policy_client = L7PolicyServiceClient::connect(endpoint.clone()).await?; + let mut rule_client = L7RuleServiceClient::connect(endpoint.clone()).await?; + let mut health_client = HealthCheckServiceClient::connect(endpoint).await?; + + let mut desired_scopes: HashMap<(String, String), HashSet> = HashMap::new(); + for lb_spec in &spec.load_balancers { + let scope = lb_scope(&lb_spec.org_id, lb_spec.project_id.as_deref()); + desired_scopes + .entry(scope) + .or_default() + .insert(lb_spec.name.clone()); + } + + for lb_spec in spec.load_balancers { + let lb = ensure_load_balancer(&mut lb_client, &lb_spec).await?; + let lb_id = lb.id.clone(); + + let pool_ids = ensure_pools( + &mut pool_client, + &mut backend_client, + &mut health_client, + &lb_id, + &lb_spec, + prune, + ) + .await?; + ensure_listeners( + &mut listener_client, + &mut policy_client, + &mut rule_client, + &lb_id, + &pool_ids, + &lb_spec, + prune, + ) + .await?; + + if prune { + prune_pools( + &mut pool_client, + &mut health_client, + &lb_id, + &lb_spec.pools, + ) + .await?; + } + } + + if prune { + prune_load_balancers( + &mut lb_client, + &mut listener_client, + &mut policy_client, + &mut pool_client, + &mut health_client, + &desired_scopes, + ) + .await?; + } + + Ok(()) +} + +async fn ensure_load_balancer( + client: &mut LoadBalancerServiceClient, + spec: &LoadBalancerSpec, +) -> Result { + let mut existing = list_load_balancers( + client, + &spec.org_id, + spec.project_id.as_deref().unwrap_or(""), + ) + .await?; + + if let Some(lb) = existing.iter_mut().find(|lb| lb.name == spec.name) { + if let Some(description) = &spec.description { + if lb.description != *description { + info!("Updating load balancer {}", spec.name); + let response = client + .update_load_balancer(UpdateLoadBalancerRequest { + id: lb.id.clone(), + name: spec.name.clone(), + description: description.clone(), + }) + .await? + .into_inner(); + return response + .loadbalancer + .context("missing load balancer in update response"); + } + } + + return Ok(lb.clone()); + } + + info!("Creating load balancer {}", spec.name); + let response = client + .create_load_balancer(CreateLoadBalancerRequest { + name: spec.name.clone(), + org_id: spec.org_id.clone(), + project_id: spec.project_id.clone().unwrap_or_default(), + description: spec.description.clone().unwrap_or_default(), + }) + .await? + .into_inner(); + + response + .loadbalancer + .context("missing load balancer in create response") +} + +async fn ensure_pools( + pool_client: &mut PoolServiceClient, + backend_client: &mut BackendServiceClient, + health_client: &mut HealthCheckServiceClient, + lb_id: &str, + spec: &LoadBalancerSpec, + prune: bool, +) -> Result> { + let mut pool_map = HashMap::new(); + let pools = list_pools(pool_client, lb_id).await?; + + for pool_spec in &spec.pools { + let desired_algorithm = parse_pool_algorithm(pool_spec.algorithm.as_deref()); + let desired_protocol = parse_pool_protocol(pool_spec.protocol.as_deref()); + let desired_persistence = session_persistence_from_spec(pool_spec.session_persistence.as_ref()); + + let pool = if let Some(existing) = pools.iter().find(|p| p.name == pool_spec.name) { + let mut update = UpdatePoolRequest { + id: existing.id.clone(), + name: String::new(), + algorithm: 0, + session_persistence: None, + }; + let mut should_update = false; + + if existing.algorithm != desired_algorithm as i32 { + update.algorithm = desired_algorithm as i32; + should_update = true; + } + + if let Some(desired) = desired_persistence.clone() { + if !session_persistence_eq(existing.session_persistence.as_ref(), Some(&desired)) { + update.session_persistence = Some(desired); + should_update = true; + } + } + + if existing.protocol != desired_protocol as i32 { + warn!( + "Pool {} protocol mismatch (update not supported)", + pool_spec.name + ); + } + + if should_update { + info!("Updating pool {}", pool_spec.name); + let response = pool_client.update_pool(update).await?.into_inner(); + response.pool.context("missing pool in update response")? + } else { + existing.clone() + } + } else { + info!("Creating pool {}", pool_spec.name); + let response = pool_client + .create_pool(CreatePoolRequest { + name: pool_spec.name.clone(), + loadbalancer_id: lb_id.to_string(), + algorithm: desired_algorithm as i32, + protocol: desired_protocol as i32, + session_persistence: desired_persistence, + }) + .await? + .into_inner(); + + response.pool.context("missing pool in create response")? + }; + + pool_map.insert(pool_spec.name.clone(), pool.id.clone()); + ensure_backends(backend_client, &pool.id, pool_spec, prune).await?; + ensure_health_checks(health_client, &pool.id, pool_spec, prune).await?; + } + + Ok(pool_map) +} + +async fn ensure_backends( + backend_client: &mut BackendServiceClient, + pool_id: &str, + pool_spec: &PoolSpec, + prune: bool, +) -> Result<()> { + let backends = list_backends(backend_client, pool_id).await?; + + for backend_spec in &pool_spec.backends { + if let Some(existing) = backends.iter().find(|b| b.name == backend_spec.name) { + let desired_weight = backend_spec.weight.unwrap_or(1); + let desired_admin_state = backend_spec + .admin_state + .as_deref() + .map(parse_backend_admin_state); + let mut update = UpdateBackendRequest { + id: existing.id.clone(), + name: String::new(), + weight: 0, + admin_state: 0, + }; + let mut should_update = false; + + if existing.weight != desired_weight { + update.weight = desired_weight; + should_update = true; + } + if let Some(admin_state) = desired_admin_state { + if existing.admin_state != admin_state as i32 { + update.admin_state = admin_state as i32; + should_update = true; + } + } + + if existing.address != backend_spec.address || existing.port != backend_spec.port { + warn!( + "Backend {} differs from desired spec (update not supported)", + backend_spec.name + ); + } + + if should_update { + info!("Updating backend {}", backend_spec.name); + backend_client.update_backend(update).await?; + } + continue; + } + + info!("Creating backend {}", backend_spec.name); + backend_client + .create_backend(CreateBackendRequest { + name: backend_spec.name.clone(), + pool_id: pool_id.to_string(), + address: backend_spec.address.clone(), + port: backend_spec.port, + weight: backend_spec.weight.unwrap_or(1), + }) + .await?; + } + + if prune { + let desired: HashSet = pool_spec + .backends + .iter() + .map(|backend| backend.name.clone()) + .collect(); + for backend in backends { + if !desired.contains(&backend.name) { + info!("Deleting backend {}", backend.name); + backend_client + .delete_backend(DeleteBackendRequest { id: backend.id }) + .await?; + } + } + } + + Ok(()) +} + +async fn ensure_health_checks( + health_client: &mut HealthCheckServiceClient, + pool_id: &str, + pool_spec: &PoolSpec, + prune: bool, +) -> Result<()> { + let checks = list_health_checks(health_client, pool_id).await?; + + for check_spec in &pool_spec.health_checks { + if let Some(existing) = checks.iter().find(|hc| hc.name == check_spec.name) { + let desired_type = parse_health_check_type(&check_spec.check_type) as i32; + let desired_interval = check_spec.interval_seconds.unwrap_or(10); + let desired_timeout = check_spec.timeout_seconds.unwrap_or(5); + let desired_healthy = check_spec.healthy_threshold.unwrap_or(2); + let desired_unhealthy = check_spec.unhealthy_threshold.unwrap_or(2); + let desired_enabled = check_spec.enabled.unwrap_or(true); + let desired_http = check_spec.http.as_ref().map(|http| HttpHealthConfig { + method: http.method.clone().unwrap_or_else(|| "GET".to_string()), + path: http.path.clone().unwrap_or_else(|| "/".to_string()), + expected_codes: http.expected_codes.clone().unwrap_or_default(), + host: http.host.clone().unwrap_or_default(), + }); + + if existing.r#type != desired_type { + warn!( + "Health check {} type mismatch (update not supported)", + check_spec.name + ); + continue; + } + + let mut update = UpdateHealthCheckRequest { + id: existing.id.clone(), + name: String::new(), + interval_seconds: 0, + timeout_seconds: 0, + healthy_threshold: 0, + unhealthy_threshold: 0, + http_config: None, + enabled: desired_enabled, + }; + let mut should_update = false; + + if existing.interval_seconds != desired_interval { + update.interval_seconds = desired_interval; + should_update = true; + } + if existing.timeout_seconds != desired_timeout { + update.timeout_seconds = desired_timeout; + should_update = true; + } + if existing.healthy_threshold != desired_healthy { + update.healthy_threshold = desired_healthy; + should_update = true; + } + if existing.unhealthy_threshold != desired_unhealthy { + update.unhealthy_threshold = desired_unhealthy; + should_update = true; + } + if existing.enabled != desired_enabled { + should_update = true; + } + + if let Some(desired_http) = desired_http { + if !http_config_eq(existing.http_config.as_ref(), Some(&desired_http)) { + update.http_config = Some(desired_http); + should_update = true; + } + } else if existing.http_config.is_some() { + warn!( + "Health check {} has HTTP config but spec does not (clear not supported)", + check_spec.name + ); + } + + if should_update { + info!("Updating health check {}", check_spec.name); + health_client.update_health_check(update).await?; + } + + continue; + } + + info!("Creating health check {}", check_spec.name); + let http_config = check_spec.http.as_ref().map(|http| HttpHealthConfig { + method: http.method.clone().unwrap_or_else(|| "GET".to_string()), + path: http.path.clone().unwrap_or_else(|| "/".to_string()), + expected_codes: http.expected_codes.clone().unwrap_or_default(), + host: http.host.clone().unwrap_or_default(), + }); + + health_client + .create_health_check(CreateHealthCheckRequest { + name: check_spec.name.clone(), + pool_id: pool_id.to_string(), + r#type: parse_health_check_type(&check_spec.check_type) as i32, + interval_seconds: check_spec.interval_seconds.unwrap_or(10), + timeout_seconds: check_spec.timeout_seconds.unwrap_or(5), + healthy_threshold: check_spec.healthy_threshold.unwrap_or(2), + unhealthy_threshold: check_spec.unhealthy_threshold.unwrap_or(2), + http_config, + }) + .await?; + } + + if prune { + let desired: HashSet = pool_spec + .health_checks + .iter() + .map(|check| check.name.clone()) + .collect(); + for check in checks { + if !desired.contains(&check.name) { + info!("Deleting health check {}", check.name); + health_client + .delete_health_check(DeleteHealthCheckRequest { id: check.id }) + .await?; + } + } + } + + Ok(()) +} + +async fn ensure_listeners( + listener_client: &mut ListenerServiceClient, + policy_client: &mut L7PolicyServiceClient, + rule_client: &mut L7RuleServiceClient, + lb_id: &str, + pool_ids: &HashMap, + spec: &LoadBalancerSpec, + prune: bool, +) -> Result<()> { + let listeners = list_listeners(listener_client, lb_id).await?; + + for listener_spec in &spec.listeners { + let pool_id = pool_ids + .get(&listener_spec.default_pool) + .with_context(|| { + format!( + "listener {} references unknown pool {}", + listener_spec.name, listener_spec.default_pool + ) + })? + .to_string(); + + let desired_protocol = parse_listener_protocol(listener_spec.protocol.as_deref()); + let desired_tls = listener_spec + .tls + .as_ref() + .map(|tls| TlsConfig { + certificate_id: tls.certificate_id.clone(), + min_version: parse_tls_version(tls.min_version.as_deref()) as i32, + cipher_suites: tls.cipher_suites.clone(), + }); + let desired_enabled = listener_spec.enabled.unwrap_or(true); + let desired_connection_limit = listener_spec.connection_limit.unwrap_or(0); + + if let Some(existing) = listeners.iter().find(|l| l.name == listener_spec.name) { + if existing.protocol != desired_protocol as i32 || existing.port != listener_spec.port { + warn!( + "Listener {} protocol/port mismatch (update not supported)", + listener_spec.name + ); + continue; + } + + let mut update = UpdateListenerRequest { + id: existing.id.clone(), + name: String::new(), + default_pool_id: String::new(), + tls_config: None, + connection_limit: 0, + enabled: desired_enabled, + }; + let mut should_update = false; + + if existing.default_pool_id != pool_id { + update.default_pool_id = pool_id.clone(); + should_update = true; + } + + if existing.connection_limit != desired_connection_limit { + update.connection_limit = desired_connection_limit; + should_update = true; + } + + if existing.enabled != desired_enabled { + should_update = true; + } + + if desired_tls.is_some() && existing.tls_config != desired_tls { + update.tls_config = desired_tls; + should_update = true; + } + + if should_update { + info!("Updating listener {}", listener_spec.name); + listener_client.update_listener(update).await?; + } + + if !listener_spec.l7_policies.is_empty() + && !is_l7_listener(desired_protocol) + { + warn!( + "Listener {} is not L7 capable but has policies", + listener_spec.name + ); + } + ensure_l7_policies( + policy_client, + rule_client, + &existing.id, + pool_ids, + listener_spec, + prune, + ) + .await?; + + continue; + } + + info!("Creating listener {}", listener_spec.name); + let created = listener_client + .create_listener(CreateListenerRequest { + name: listener_spec.name.clone(), + loadbalancer_id: lb_id.to_string(), + protocol: desired_protocol as i32, + port: listener_spec.port, + default_pool_id: pool_id, + tls_config: desired_tls, + connection_limit: desired_connection_limit, + }) + .await? + .into_inner() + .listener + .context("missing listener in create response")?; + + if !listener_spec.l7_policies.is_empty() && !is_l7_listener(desired_protocol) { + warn!( + "Listener {} is not L7 capable but has policies", + listener_spec.name + ); + } + ensure_l7_policies( + policy_client, + rule_client, + &created.id, + pool_ids, + listener_spec, + prune, + ) + .await?; + } + + if prune { + prune_listeners(listener_client, policy_client, lb_id, &spec.listeners).await?; + } + + Ok(()) +} + +async fn ensure_l7_policies( + policy_client: &mut L7PolicyServiceClient, + rule_client: &mut L7RuleServiceClient, + listener_id: &str, + pool_ids: &HashMap, + listener_spec: &ListenerSpec, + prune: bool, +) -> Result<()> { + let policies = list_l7_policies(policy_client, listener_id).await?; + let mut desired_names = HashSet::new(); + + for policy_spec in &listener_spec.l7_policies { + desired_names.insert(policy_spec.name.clone()); + + let desired_action = parse_l7_policy_action(&policy_spec.action); + let desired_position = policy_spec.position.unwrap_or(1); + let desired_enabled = policy_spec.enabled.unwrap_or(true); + let desired_redirect_url = policy_spec + .redirect_url + .as_ref() + .filter(|value| !value.is_empty()) + .cloned(); + let desired_redirect_pool_id = match &policy_spec.redirect_pool { + Some(pool_name) => Some( + pool_ids + .get(pool_name) + .with_context(|| { + format!( + "l7 policy {} references unknown pool {}", + policy_spec.name, pool_name + ) + })? + .to_string(), + ), + None => None, + }; + let desired_status = policy_spec.redirect_http_status_code; + + if matches!(desired_action, L7PolicyAction::RedirectToPool) + && desired_redirect_pool_id.is_none() + { + warn!( + "L7 policy {} action redirect_to_pool is missing redirect_pool", + policy_spec.name + ); + } + if matches!(desired_action, L7PolicyAction::RedirectToUrl) + && desired_redirect_url.is_none() + { + warn!( + "L7 policy {} action redirect_to_url is missing redirect_url", + policy_spec.name + ); + } + + let (policy_id, needs_update) = if let Some(existing) = + policies.iter().find(|p| p.name == policy_spec.name) + { + let matches = l7_policy_matches( + existing, + desired_action, + desired_position, + desired_enabled, + desired_redirect_url.as_ref(), + desired_redirect_pool_id.as_ref(), + desired_status, + ); + (existing.id.clone(), !matches) + } else { + let response = policy_client + .create_l7_policy(CreateL7PolicyRequest { + listener_id: listener_id.to_string(), + name: policy_spec.name.clone(), + position: desired_position, + action: desired_action as i32, + redirect_url: desired_redirect_url.clone().unwrap_or_default(), + redirect_pool_id: desired_redirect_pool_id.clone().unwrap_or_default(), + redirect_http_status_code: desired_status.unwrap_or(0), + }) + .await? + .into_inner(); + let policy = response + .l7_policy + .context("missing l7 policy in create response")?; + let matches = l7_policy_matches( + &policy, + desired_action, + desired_position, + desired_enabled, + desired_redirect_url.as_ref(), + desired_redirect_pool_id.as_ref(), + desired_status, + ); + (policy.id, !matches) + }; + + if needs_update { + info!("Updating L7 policy {}", policy_spec.name); + policy_client + .update_l7_policy(UpdateL7PolicyRequest { + id: policy_id.clone(), + name: policy_spec.name.clone(), + position: desired_position, + action: desired_action as i32, + redirect_url: desired_redirect_url.clone().unwrap_or_default(), + redirect_pool_id: desired_redirect_pool_id.clone().unwrap_or_default(), + redirect_http_status_code: desired_status.unwrap_or(0), + enabled: desired_enabled, + }) + .await?; + } + + ensure_l7_rules( + rule_client, + &policy_id, + policy_spec, + prune, + ) + .await?; + } + + if prune { + for policy in policies { + if !desired_names.contains(&policy.name) { + info!("Deleting L7 policy {}", policy.name); + policy_client + .delete_l7_policy(DeleteL7PolicyRequest { id: policy.id }) + .await?; + } + } + } + + Ok(()) +} + +async fn ensure_l7_rules( + rule_client: &mut L7RuleServiceClient, + policy_id: &str, + policy_spec: &L7PolicySpec, + prune: bool, +) -> Result<()> { + let rules = list_l7_rules(rule_client, policy_id).await?; + let mut used_rules: HashSet = HashSet::new(); + + for rule_spec in &policy_spec.rules { + let desired_rule_type = parse_l7_rule_type(&rule_spec.rule_type); + let desired_compare_type = + parse_l7_compare_type(rule_spec.compare_type.as_deref()); + let desired_value = rule_spec.value.clone(); + let desired_key = rule_spec + .key + .as_ref() + .filter(|value| !value.is_empty()) + .cloned(); + let desired_invert = rule_spec.invert.unwrap_or(false); + + if let Some(existing) = rules.iter().find(|rule| { + !used_rules.contains(&rule.id) + && l7_rule_key_matches(rule, desired_rule_type, &desired_value, desired_key.as_deref()) + }) { + let needs_update = existing.compare_type != desired_compare_type as i32 + || existing.invert != desired_invert; + if needs_update { + info!("Updating L7 rule {}", existing.id); + rule_client + .update_l7_rule(UpdateL7RuleRequest { + id: existing.id.clone(), + rule_type: desired_rule_type as i32, + compare_type: desired_compare_type as i32, + value: desired_value.clone(), + key: desired_key.clone().unwrap_or_default(), + invert: desired_invert, + }) + .await?; + } + used_rules.insert(existing.id.clone()); + continue; + } + + info!("Creating L7 rule for policy {}", policy_spec.name); + let response = rule_client + .create_l7_rule(CreateL7RuleRequest { + policy_id: policy_id.to_string(), + rule_type: desired_rule_type as i32, + compare_type: desired_compare_type as i32, + value: desired_value.clone(), + key: desired_key.clone().unwrap_or_default(), + invert: desired_invert, + }) + .await? + .into_inner(); + if let Some(rule) = response.l7_rule { + used_rules.insert(rule.id); + } + } + + if prune { + for rule in rules { + if !used_rules.contains(&rule.id) { + info!("Deleting L7 rule {}", rule.id); + rule_client + .delete_l7_rule(DeleteL7RuleRequest { id: rule.id }) + .await?; + } + } + } + + Ok(()) +} + +async fn prune_listeners( + listener_client: &mut ListenerServiceClient, + policy_client: &mut L7PolicyServiceClient, + lb_id: &str, + specs: &[ListenerSpec], +) -> Result<()> { + let listeners = list_listeners(listener_client, lb_id).await?; + let desired: HashSet = specs + .iter() + .map(|listener| listener.name.clone()) + .collect(); + + for listener in listeners { + if !desired.contains(&listener.name) { + info!("Deleting listener {}", listener.name); + delete_listener_policies(policy_client, &listener.id).await?; + listener_client + .delete_listener(DeleteListenerRequest { id: listener.id }) + .await?; + } + } + + Ok(()) +} + +async fn prune_pools( + pool_client: &mut PoolServiceClient, + health_client: &mut HealthCheckServiceClient, + lb_id: &str, + specs: &[PoolSpec], +) -> Result<()> { + let pools = list_pools(pool_client, lb_id).await?; + let desired: HashSet = specs.iter().map(|pool| pool.name.clone()).collect(); + + for pool in pools { + if !desired.contains(&pool.name) { + info!("Deleting pool {}", pool.name); + let checks = list_health_checks(health_client, &pool.id).await?; + for check in checks { + health_client + .delete_health_check(DeleteHealthCheckRequest { id: check.id }) + .await?; + } + pool_client + .delete_pool(DeletePoolRequest { id: pool.id }) + .await?; + } + } + + Ok(()) +} + +async fn prune_load_balancers( + lb_client: &mut LoadBalancerServiceClient, + listener_client: &mut ListenerServiceClient, + policy_client: &mut L7PolicyServiceClient, + pool_client: &mut PoolServiceClient, + health_client: &mut HealthCheckServiceClient, + desired_scopes: &HashMap<(String, String), HashSet>, +) -> Result<()> { + for ((org_id, project_id), desired_names) in desired_scopes { + let lbs = list_load_balancers(lb_client, org_id, project_id).await?; + for lb in lbs { + if !desired_names.contains(&lb.name) { + info!("Deleting load balancer {}", lb.name); + let listeners = list_listeners(listener_client, &lb.id).await?; + for listener in listeners { + delete_listener_policies(policy_client, &listener.id).await?; + } + let pools = list_pools(pool_client, &lb.id).await?; + for pool in pools { + let checks = list_health_checks(health_client, &pool.id).await?; + for check in checks { + health_client + .delete_health_check(DeleteHealthCheckRequest { id: check.id }) + .await?; + } + } + lb_client + .delete_load_balancer(DeleteLoadBalancerRequest { id: lb.id }) + .await?; + } + } + } + + Ok(()) +} + +async fn delete_listener_policies( + policy_client: &mut L7PolicyServiceClient, + listener_id: &str, +) -> Result<()> { + let policies = list_l7_policies(policy_client, listener_id).await?; + for policy in policies { + info!("Deleting L7 policy {}", policy.name); + policy_client + .delete_l7_policy(DeleteL7PolicyRequest { id: policy.id }) + .await?; + } + Ok(()) +} + +async fn list_load_balancers( + client: &mut LoadBalancerServiceClient, + org_id: &str, + project_id: &str, +) -> Result> { + let response = client + .list_load_balancers(fiberlb_api::ListLoadBalancersRequest { + org_id: org_id.to_string(), + project_id: project_id.to_string(), + page_size: 1000, + page_token: String::new(), + }) + .await? + .into_inner(); + + Ok(response.loadbalancers) +} + +async fn list_pools( + client: &mut PoolServiceClient, + lb_id: &str, +) -> Result> { + let response = client + .list_pools(fiberlb_api::ListPoolsRequest { + loadbalancer_id: lb_id.to_string(), + page_size: 1000, + page_token: String::new(), + }) + .await? + .into_inner(); + Ok(response.pools) +} + +async fn list_backends( + client: &mut BackendServiceClient, + pool_id: &str, +) -> Result> { + let response = client + .list_backends(fiberlb_api::ListBackendsRequest { + pool_id: pool_id.to_string(), + page_size: 1000, + page_token: String::new(), + }) + .await? + .into_inner(); + Ok(response.backends) +} + +async fn list_listeners( + client: &mut ListenerServiceClient, + lb_id: &str, +) -> Result> { + let response = client + .list_listeners(fiberlb_api::ListListenersRequest { + loadbalancer_id: lb_id.to_string(), + page_size: 1000, + page_token: String::new(), + }) + .await? + .into_inner(); + Ok(response.listeners) +} + +async fn list_health_checks( + client: &mut HealthCheckServiceClient, + pool_id: &str, +) -> Result> { + let response = client + .list_health_checks(fiberlb_api::ListHealthChecksRequest { + pool_id: pool_id.to_string(), + page_size: 1000, + page_token: String::new(), + }) + .await? + .into_inner(); + Ok(response.health_checks) +} + +async fn list_l7_policies( + client: &mut L7PolicyServiceClient, + listener_id: &str, +) -> Result> { + let response = client + .list_l7_policies(fiberlb_api::ListL7PoliciesRequest { + listener_id: listener_id.to_string(), + page_size: 1000, + page_token: String::new(), + }) + .await? + .into_inner(); + Ok(response.l7_policies) +} + +async fn list_l7_rules( + client: &mut L7RuleServiceClient, + policy_id: &str, +) -> Result> { + let response = client + .list_l7_rules(fiberlb_api::ListL7RulesRequest { + policy_id: policy_id.to_string(), + page_size: 1000, + page_token: String::new(), + }) + .await? + .into_inner(); + Ok(response.l7_rules) +} + +fn parse_pool_algorithm(value: Option<&str>) -> PoolAlgorithm { + match normalize_name(value.unwrap_or("round_robin")).as_str() { + "least_connections" => PoolAlgorithm::LeastConnections, + "ip_hash" => PoolAlgorithm::IpHash, + "weighted_round_robin" => PoolAlgorithm::WeightedRoundRobin, + "random" => PoolAlgorithm::Random, + "maglev" => PoolAlgorithm::Maglev, + _ => PoolAlgorithm::RoundRobin, + } +} + +fn parse_pool_protocol(value: Option<&str>) -> PoolProtocol { + match normalize_name(value.unwrap_or("tcp")).as_str() { + "udp" => PoolProtocol::Udp, + "http" => PoolProtocol::Http, + "https" => PoolProtocol::Https, + _ => PoolProtocol::Tcp, + } +} + +fn parse_listener_protocol(value: Option<&str>) -> ListenerProtocol { + match normalize_name(value.unwrap_or("tcp")).as_str() { + "udp" => ListenerProtocol::Udp, + "http" => ListenerProtocol::Http, + "https" => ListenerProtocol::Https, + "terminated_https" => ListenerProtocol::TerminatedHttps, + _ => ListenerProtocol::Tcp, + } +} + +fn parse_health_check_type(value: &str) -> HealthCheckType { + match normalize_name(value).as_str() { + "http" => HealthCheckType::Http, + "https" => HealthCheckType::Https, + "udp" => HealthCheckType::Udp, + "ping" => HealthCheckType::Ping, + _ => HealthCheckType::Tcp, + } +} + +fn parse_tls_version(value: Option<&str>) -> TlsVersion { + match normalize_name(value.unwrap_or("tls_1_2")).as_str() { + "tls_1_3" => TlsVersion::Tls13, + _ => TlsVersion::Tls12, + } +} + +fn parse_backend_admin_state(value: &str) -> BackendAdminState { + match normalize_name(value).as_str() { + "disabled" => BackendAdminState::Disabled, + "drain" => BackendAdminState::Drain, + _ => BackendAdminState::Enabled, + } +} + +fn parse_l7_policy_action(value: &str) -> L7PolicyAction { + match normalize_name(value).as_str() { + "redirect_to_url" => L7PolicyAction::RedirectToUrl, + "reject" => L7PolicyAction::Reject, + _ => L7PolicyAction::RedirectToPool, + } +} + +fn parse_l7_rule_type(value: &str) -> L7RuleType { + match normalize_name(value).as_str() { + "host_name" => L7RuleType::HostName, + "path" => L7RuleType::Path, + "file_type" => L7RuleType::FileType, + "header" => L7RuleType::Header, + "cookie" => L7RuleType::Cookie, + "ssl_conn_has_sni" => L7RuleType::SslConnHasSni, + _ => L7RuleType::Path, + } +} + +fn parse_l7_compare_type(value: Option<&str>) -> L7CompareType { + match normalize_name(value.unwrap_or("equal_to")).as_str() { + "regex" => L7CompareType::Regex, + "starts_with" => L7CompareType::StartsWith, + "ends_with" => L7CompareType::EndsWith, + "contains" => L7CompareType::Contains, + _ => L7CompareType::EqualTo, + } +} + +fn is_l7_listener(protocol: ListenerProtocol) -> bool { + matches!( + protocol, + ListenerProtocol::Http | ListenerProtocol::Https | ListenerProtocol::TerminatedHttps + ) +} + +fn session_persistence_from_spec(spec: Option<&SessionPersistenceSpec>) -> Option { + let spec = spec?; + let persistence_type = match normalize_name(&spec.persistence_type).as_str() { + "cookie" => fiberlb_api::PersistenceType::Cookie, + "app_cookie" => fiberlb_api::PersistenceType::AppCookie, + _ => fiberlb_api::PersistenceType::SourceIp, + }; + + Some(SessionPersistence { + r#type: persistence_type as i32, + cookie_name: spec.cookie_name.clone().unwrap_or_default(), + timeout_seconds: spec.timeout_seconds.unwrap_or(0), + }) +} + +fn session_persistence_eq( + existing: Option<&SessionPersistence>, + desired: Option<&SessionPersistence>, +) -> bool { + match (existing, desired) { + (None, None) => true, + (Some(lhs), Some(rhs)) => { + lhs.r#type == rhs.r#type + && lhs.cookie_name == rhs.cookie_name + && lhs.timeout_seconds == rhs.timeout_seconds + } + _ => false, + } +} + +fn http_config_eq( + existing: Option<&HttpHealthConfig>, + desired: Option<&HttpHealthConfig>, +) -> bool { + match (existing, desired) { + (None, None) => true, + (Some(lhs), Some(rhs)) => { + lhs.method == rhs.method + && lhs.path == rhs.path + && lhs.expected_codes == rhs.expected_codes + && lhs.host == rhs.host + } + _ => false, + } +} + +fn normalize_name(value: &str) -> String { + value.trim().to_lowercase().replace('-', "_") +} + +fn normalize_optional_string(value: &str) -> Option { + let trimmed = value.trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + } +} + +fn normalize_optional_u32(value: u32) -> Option { + if value == 0 { + None + } else { + Some(value) + } +} + +fn l7_policy_matches( + existing: &L7Policy, + desired_action: L7PolicyAction, + desired_position: u32, + desired_enabled: bool, + desired_redirect_url: Option<&String>, + desired_redirect_pool_id: Option<&String>, + desired_status: Option, +) -> bool { + existing.action == desired_action as i32 + && existing.position == desired_position + && existing.enabled == desired_enabled + && normalize_optional_string(&existing.redirect_url).as_deref() + == desired_redirect_url.map(|value| value.as_str()) + && normalize_optional_string(&existing.redirect_pool_id).as_deref() + == desired_redirect_pool_id.map(|value| value.as_str()) + && normalize_optional_u32(existing.redirect_http_status_code) == desired_status +} + +fn l7_rule_key_matches( + rule: &L7Rule, + desired_rule_type: L7RuleType, + desired_value: &str, + desired_key: Option<&str>, +) -> bool { + rule.rule_type == desired_rule_type as i32 + && rule.value == desired_value + && normalize_optional_string(&rule.key).as_deref() == desired_key +} + +fn lb_scope(org_id: &str, project_id: Option<&str>) -> (String, String) { + ( + org_id.to_string(), + project_id.unwrap_or("").to_string(), + ) +} + +fn dns_scope(org_id: &str, project_id: Option<&str>) -> (String, String) { + ( + org_id.to_string(), + project_id.unwrap_or("").to_string(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_l7_policy_action() { + assert_eq!( + parse_l7_policy_action("redirect_to_url"), + L7PolicyAction::RedirectToUrl + ); + assert_eq!(parse_l7_policy_action("reject"), L7PolicyAction::Reject); + assert_eq!( + parse_l7_policy_action("redirect-to-pool"), + L7PolicyAction::RedirectToPool + ); + } + + #[test] + fn test_parse_l7_rule_type_and_compare() { + assert_eq!(parse_l7_rule_type("host_name"), L7RuleType::HostName); + assert_eq!( + parse_l7_rule_type("ssl_conn_has_sni"), + L7RuleType::SslConnHasSni + ); + assert_eq!( + parse_l7_compare_type(Some("starts_with")), + L7CompareType::StartsWith + ); + assert_eq!( + parse_l7_compare_type(None), + L7CompareType::EqualTo + ); + } + + #[test] + fn test_is_l7_listener() { + assert!(is_l7_listener(ListenerProtocol::Http)); + assert!(is_l7_listener(ListenerProtocol::Https)); + assert!(is_l7_listener(ListenerProtocol::TerminatedHttps)); + assert!(!is_l7_listener(ListenerProtocol::Tcp)); + } + + #[test] + fn test_l7_rule_key_matches() { + let rule = L7Rule { + id: "rule-1".to_string(), + policy_id: "policy-1".to_string(), + rule_type: L7RuleType::Path as i32, + compare_type: L7CompareType::EqualTo as i32, + value: "/v1".to_string(), + key: String::new(), + invert: false, + created_at: 0, + updated_at: 0, + }; + + assert!(l7_rule_key_matches(&rule, L7RuleType::Path, "/v1", None)); + assert!(!l7_rule_key_matches(&rule, L7RuleType::Path, "/v2", None)); + } + + #[test] + fn test_l7_policy_matches() { + let policy = L7Policy { + id: "policy-1".to_string(), + listener_id: "listener-1".to_string(), + name: "policy".to_string(), + position: 1, + action: L7PolicyAction::RedirectToPool as i32, + redirect_url: String::new(), + redirect_pool_id: "pool-1".to_string(), + redirect_http_status_code: 0, + enabled: true, + created_at: 0, + updated_at: 0, + }; + + let pool_id = "pool-1".to_string(); + assert!(l7_policy_matches( + &policy, + L7PolicyAction::RedirectToPool, + 1, + true, + None, + Some(&pool_id), + None + )); + + assert!(!l7_policy_matches( + &policy, + L7PolicyAction::RedirectToPool, + 2, + true, + None, + Some(&pool_id), + None + )); + } +} + +async fn reconcile_dns(spec: DnsConfig, endpoint: String, prune: bool) -> Result<()> { + let mut zone_client = ZoneServiceClient::connect(endpoint.clone()).await?; + let mut record_client = RecordServiceClient::connect(endpoint.clone()).await?; + let mut reverse_client = ReverseZoneServiceClient::connect(endpoint).await?; + + let mut desired_scopes: HashMap<(String, String), HashSet> = HashMap::new(); + for zone_spec in &spec.zones { + let scope = dns_scope(&zone_spec.org_id, zone_spec.project_id.as_deref()); + desired_scopes + .entry(scope) + .or_default() + .insert(zone_spec.name.clone()); + } + + for zone_spec in &spec.zones { + let zone = ensure_zone(&mut zone_client, zone_spec).await?; + ensure_records(&mut record_client, &zone, zone_spec, prune).await?; + } + + if prune { + prune_zones(&mut zone_client, &desired_scopes).await?; + } + + ensure_reverse_zones(&mut reverse_client, &spec.reverse_zones, prune).await?; + + Ok(()) +} + +async fn ensure_zone( + client: &mut ZoneServiceClient, + spec: &ZoneSpec, +) -> Result { + let existing = list_zones( + client, + &spec.org_id, + spec.project_id.as_deref().unwrap_or(""), + Some(&spec.name), + ) + .await?; + + if let Some(zone) = existing.into_iter().find(|z| z.name == spec.name) { + let update = UpdateZoneRequest { + id: zone.id.clone(), + refresh: spec.refresh, + retry: spec.retry, + expire: spec.expire, + minimum: spec.minimum, + primary_ns: spec.primary_ns.clone(), + admin_email: spec.admin_email.clone(), + }; + + if update.refresh.is_some() + || update.retry.is_some() + || update.expire.is_some() + || update.minimum.is_some() + || update.primary_ns.is_some() + || update.admin_email.is_some() + { + info!("Updating zone {}", spec.name); + let response = client.update_zone(update).await?.into_inner(); + return response.zone.context("missing zone in update response"); + } + + return Ok(zone); + } + + info!("Creating zone {}", spec.name); + let response = client + .create_zone(CreateZoneRequest { + name: spec.name.clone(), + org_id: spec.org_id.clone(), + project_id: spec.project_id.clone().unwrap_or_default(), + primary_ns: spec.primary_ns.clone().unwrap_or_default(), + admin_email: spec.admin_email.clone().unwrap_or_default(), + }) + .await? + .into_inner(); + + response.zone.context("missing zone in create response") +} + +async fn ensure_records( + client: &mut RecordServiceClient, + zone: &ZoneInfo, + spec: &ZoneSpec, + prune: bool, +) -> Result<()> { + let records = list_records(client, &zone.id).await?; + let mut existing: HashMap<(String, String), RecordInfo> = HashMap::new(); + for record in records { + existing.insert((record.name.clone(), normalize_name(&record.record_type)), record); + } + + for record_spec in &spec.records { + let key = (record_spec.name.clone(), normalize_name(&record_spec.record_type)); + let data = record_data_from_spec(&record_spec.record_type, &record_spec.data)?; + let ttl = record_spec.ttl.unwrap_or(300); + + if let Some(existing_record) = existing.get(&key) { + info!("Updating record {} {}", record_spec.record_type, record_spec.name); + client + .update_record(UpdateRecordRequest { + id: existing_record.id.clone(), + ttl: record_spec.ttl, + data: Some(data), + enabled: record_spec.enabled, + }) + .await?; + continue; + } + + info!("Creating record {} {}", record_spec.record_type, record_spec.name); + client + .create_record(CreateRecordRequest { + zone_id: zone.id.clone(), + name: record_spec.name.clone(), + record_type: record_spec.record_type.clone(), + ttl, + data: Some(data), + }) + .await?; + } + + if prune { + let desired: HashSet<(String, String)> = spec + .records + .iter() + .map(|record| { + ( + record.name.clone(), + normalize_name(&record.record_type), + ) + }) + .collect(); + for (key, record) in existing { + if !desired.contains(&key) { + info!("Deleting record {} {}", record.record_type, record.name); + client + .delete_record(DeleteRecordRequest { id: record.id }) + .await?; + } + } + } + + Ok(()) +} + +async fn prune_zones( + client: &mut ZoneServiceClient, + desired_scopes: &HashMap<(String, String), HashSet>, +) -> Result<()> { + for ((org_id, project_id), desired_names) in desired_scopes { + let zones = list_zones(client, org_id, project_id, None).await?; + for zone in zones { + if !desired_names.contains(&zone.name) { + info!("Deleting zone {}", zone.name); + client + .delete_zone(DeleteZoneRequest { + id: zone.id, + force: true, + }) + .await?; + } + } + } + + Ok(()) +} + +async fn ensure_reverse_zones( + client: &mut ReverseZoneServiceClient, + specs: &[ReverseZoneSpec], + prune: bool, +) -> Result<()> { + let mut scopes: HashMap<(String, String), Vec<&ReverseZoneSpec>> = HashMap::new(); + for spec in specs { + let scope = dns_scope(&spec.org_id, spec.project_id.as_deref()); + scopes.entry(scope).or_default().push(spec); + } + + for ((org_id, project_id), scoped_specs) in scopes { + let project_id_opt = if project_id.is_empty() { + None + } else { + Some(project_id.as_str()) + }; + let existing = list_reverse_zones(client, &org_id, project_id_opt).await?; + + for spec in &scoped_specs { + let desired_ttl = spec.ttl.unwrap_or(3600); + if let Some(zone) = existing.iter().find(|zone| zone.cidr == spec.cidr) { + if zone.ptr_pattern != spec.ptr_pattern || zone.ttl != desired_ttl { + info!("Recreating reverse zone {}", spec.cidr); + client + .delete_reverse_zone(DeleteReverseZoneRequest { + zone_id: zone.id.clone(), + }) + .await?; + client + .create_reverse_zone(CreateReverseZoneRequest { + org_id: spec.org_id.clone(), + project_id: spec.project_id.clone(), + cidr: spec.cidr.clone(), + ptr_pattern: spec.ptr_pattern.clone(), + ttl: desired_ttl, + }) + .await?; + } + continue; + } + + info!("Creating reverse zone {}", spec.cidr); + client + .create_reverse_zone(CreateReverseZoneRequest { + org_id: spec.org_id.clone(), + project_id: spec.project_id.clone(), + cidr: spec.cidr.clone(), + ptr_pattern: spec.ptr_pattern.clone(), + ttl: desired_ttl, + }) + .await?; + } + + if prune { + let desired_cidrs: HashSet = + scoped_specs.iter().map(|spec| spec.cidr.clone()).collect(); + for zone in existing { + if !desired_cidrs.contains(&zone.cidr) { + info!("Deleting reverse zone {}", zone.cidr); + client + .delete_reverse_zone(DeleteReverseZoneRequest { zone_id: zone.id }) + .await?; + } + } + } + } + + Ok(()) +} + +async fn list_zones( + client: &mut ZoneServiceClient, + org_id: &str, + project_id: &str, + name_filter: Option<&str>, +) -> Result> { + let response = client + .list_zones(flashdns_api::proto::ListZonesRequest { + org_id: org_id.to_string(), + project_id: project_id.to_string(), + name_filter: name_filter.unwrap_or_default().to_string(), + page_size: 1000, + page_token: String::new(), + }) + .await? + .into_inner(); + + Ok(response.zones) +} + +async fn list_reverse_zones( + client: &mut ReverseZoneServiceClient, + org_id: &str, + project_id: Option<&str>, +) -> Result> { + let response = client + .list_reverse_zones(ListReverseZonesRequest { + org_id: org_id.to_string(), + project_id: project_id.map(|value| value.to_string()), + }) + .await? + .into_inner(); + + Ok(response.zones) +} + +async fn list_records( + client: &mut RecordServiceClient, + zone_id: &str, +) -> Result> { + let response = client + .list_records(flashdns_api::proto::ListRecordsRequest { + zone_id: zone_id.to_string(), + name_filter: String::new(), + type_filter: String::new(), + page_size: 1000, + page_token: String::new(), + }) + .await? + .into_inner(); + Ok(response.records) +} + +fn record_data_from_spec(record_type: &str, data: &serde_json::Value) -> Result { + let record_type = normalize_name(record_type); + let map = data + .as_object() + .context("record data must be an object")?; + + let record_data = match record_type.as_str() { + "a" => record_data::Data::A(ARecord { + address: get_string(map, "address")?, + }), + "aaaa" => record_data::Data::Aaaa(AaaaRecord { + address: get_string(map, "address")?, + }), + "cname" => record_data::Data::Cname(CnameRecord { + target: get_string(map, "target")?, + }), + "mx" => record_data::Data::Mx(MxRecord { + preference: get_u32(map, "preference")?, + exchange: get_string(map, "exchange")?, + }), + "txt" => record_data::Data::Txt(TxtRecord { + text: get_string(map, "text")?, + }), + "srv" => record_data::Data::Srv(SrvRecord { + priority: get_u32(map, "priority")?, + weight: get_u32(map, "weight")?, + port: get_u32(map, "port")?, + target: get_string(map, "target")?, + }), + "ns" => record_data::Data::Ns(NsRecord { + nameserver: get_string(map, "nameserver")?, + }), + "ptr" => record_data::Data::Ptr(PtrRecord { + target: get_string(map, "target")?, + }), + "caa" => record_data::Data::Caa(CaaRecord { + flags: get_u32(map, "flags")?, + tag: get_string(map, "tag")?, + value: get_string(map, "value")?, + }), + _ => return Err(anyhow::anyhow!("unsupported record type {}", record_type)), + }; + + Ok(RecordData { + data: Some(record_data), + }) +} + +fn get_string(map: &serde_json::Map, key: &str) -> Result { + map.get(key) + .and_then(|value| value.as_str()) + .map(|value| value.to_string()) + .with_context(|| format!("record data missing {}", key)) +} + +fn get_u32(map: &serde_json::Map, key: &str) -> Result { + map.get(key) + .and_then(|value| value.as_u64()) + .map(|value| value as u32) + .with_context(|| format!("record data missing {}", key)) +} diff --git a/docs/Nix-NOS.md b/docs/Nix-NOS.md new file mode 100644 index 0000000..fa95be5 --- /dev/null +++ b/docs/Nix-NOS.md @@ -0,0 +1,398 @@ +# PlasmaCloud/PhotonCloud と Nix-NOS の統合分析 + +## Architecture Decision (2025-12-13) + +**決定:** Nix-NOSを汎用ネットワークモジュールとして別リポジトリに分離する。 + +### Three-Layer Architecture + +``` +Layer 3: PlasmaCloud Cluster (T061) + - plasmacloud-cluster.nix + - cluster-config.json生成 + - Deployer (Rust) + depends on ↓ + +Layer 2: PlasmaCloud Network (T061) + - plasmacloud-network.nix + - FiberLB BGP連携 + - PrismNET統合 + depends on ↓ + +Layer 1: Nix-NOS Generic (T062) ← 別リポジトリ + - BGP (BIRD2/GoBGP) + - VLAN + - Network interfaces + - PlasmaCloudを知らない汎用モジュール +``` + +### Repository Structure + +- **github.com/centra/nix-nos**: Layer 1 (汎用、VyOS/OpenWrt代替) +- **github.com/centra/plasmacloud**: Layers 2+3 (既存リポジトリ) + +--- + +## 1. 既存プロジェクトの概要 + +PlasmaCloud（PhotonCloud）は、以下のコンポーネントで構成されるクラウド基盤プロジェクト： + +### コアサービス +| コンポーネント | 役割 | 技術スタック | +|---------------|------|-------------| +| **ChainFire** | 分散KVストア（etcd互換） | Rust, Raft (openraft) | +| **FlareDB** | SQLデータベース | Rust, KVバックエンド | +| **IAM** | 認証・認可 | Rust, JWT/mTLS | +| **PlasmaVMC** | VM管理 | Rust, KVM/FireCracker | +| **PrismNET** | オーバーレイネットワーク | Rust, OVN連携 | +| **LightningSTOR** | オブジェクトストレージ | Rust, S3互換 | +| **FlashDNS** | DNS | Rust, hickory-dns | +| **FiberLB** | ロードバランサー | Rust, L4/L7, BGP予定 | +| **NightLight** | メトリクス | Rust, Prometheus互換 | +| **k8shost** | コンテナオーケストレーション | Rust, K8s API互換 | + +### インフラ層 +- **NixOSモジュール**: 各サービス用 (`nix/modules/`) +- **first-boot-automation**: 自動クラスタ参加 +- **PXE/Netboot**: ベアメタルプロビジョニング +- **TLS証明書管理**: 開発用証明書生成スクリプト + +--- + +## 2. Nix-NOS との統合ポイント + +### 2.1 Baremetal Provisioning → Deployer強化 + +**既存の実装:** +``` +first-boot-automation.nix +├── cluster-config.json による設定注入 +├── bootstrap vs join の自動判定 +├── マーカーファイルによる冪等性 +└── systemd サービス連携 +``` + +**Nix-NOSで追加すべき機能:** + +| 既存 | Nix-NOS追加 | +|------|-------------| +| cluster-config.json (手動作成) | topology.nix から自動生成 | +| 単一クラスタ構成 | 複数クラスタ/サイト対応 | +| nixos-anywhere 依存 | Deployer (Phone Home + Push) | +| 固定IP設定 | IPAM連携による動的割当 | + +**統合設計:** + +```nix +# topology.nix（Nix-NOS） +{ + nix-nos.clusters.plasmacloud = { + nodes = { + "node01" = { + role = "control-plane"; + ip = "10.0.1.10"; + services = [ "chainfire" "flaredb" "iam" ]; + }; + "node02" = { role = "control-plane"; ip = "10.0.1.11"; }; + "node03" = { role = "worker"; ip = "10.0.1.12"; }; + }; + + # Nix-NOSが自動生成 → first-boot-automationが読む + # cluster-config.json の内容をNix評価時に決定 + }; +} +``` + +### 2.2 Network Management → PrismNET + FiberLB + Nix-NOS BGP + +**既存の実装:** +``` +PrismNET (prismnet/) +├── VPC/Subnet/Port管理 +├── Security Groups +├── IPAM +└── OVN連携 + +FiberLB (fiberlb/) +├── L4/L7ロードバランシング +├── ヘルスチェック +├── VIP管理 +└── BGP統合（設計済み、GoBGPサイドカー） +``` + +**Nix-NOSで追加すべき機能:** + +``` +Nix-NOS Network Layer +├── BGP設定生成（BIRD2） +│ ├── iBGP/eBGP自動計算 +│ ├── Route Reflector対応 +│ └── ポリシー抽象化 +├── topology.nix → systemd-networkd +├── OpenWrt/Cisco設定生成（将来） +└── FiberLB BGP連携 +``` + +**統合設計:** + +```nix +# Nix-NOSのBGPモジュール → FiberLBのGoBGP設定に統合 +{ + nix-nos.network.bgp = { + autonomousSystems = { + "65000" = { + members = [ "node01" "node02" "node03" ]; + ibgp.strategy = "route-reflector"; + ibgp.reflectors = [ "node01" ]; + }; + }; + + # FiberLBのVIPをBGPで広報 + vipAdvertisements = { + "fiberlb" = { + vips = [ "10.0.100.1" "10.0.100.2" ]; + nextHop = "self"; + communities = [ "65000:100" ]; + }; + }; + }; + + # FiberLBモジュールとの連携 + services.fiberlb.bgp = { + enable = true; + # Nix-NOSが生成するGoBGP設定を参照 + configFile = config.nix-nos.network.bgp.gobgpConfig; + }; +} +``` + +### 2.3 K8sパチモン → k8shost + Pure NixOS Alternative + +**既存の実装:** +``` +k8shost (k8shost/) +├── Pod管理（gRPC API） +├── Service管理（ClusterIP/NodePort） +├── Node管理 +├── CNI連携 +├── CSI連携 +└── FiberLB/FlashDNS連携 +``` + +**Nix-NOSの役割:** + +k8shostはすでにKubernetesのパチモンとして機能している。Nix-NOSは： + +1. **k8shostを使う場合**: k8shostクラスタ自体のデプロイをNix-NOSで管理 +2. **Pure NixOS（K8sなし）**: より軽量な選択肢として、Systemd + Nix-NOSでサービス管理 + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Orchestration Options │ +├─────────────────────────────────────────────────────────────┤ +│ Option A: k8shost (K8s-like) │ +│ ┌─────────────────────────────────────────────────────┐ │ +│ │ Nix-NOS manages: cluster topology, network, certs │ │ +│ │ k8shost manages: pods, services, scaling │ │ +│ └─────────────────────────────────────────────────────┘ │ +│ │ +│ Option B: Pure NixOS (K8s-free) │ +│ ┌─────────────────────────────────────────────────────┐ │ +│ │ Nix-NOS manages: everything │ │ +│ │ systemd + containers, static service discovery │ │ +│ │ Use case: クラウド基盤自体の管理 │ │ +│ └─────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +**重要な洞察:** + +> 「クラウドの基盤そのものを作るのにKubernetesは使いたくない」 + +これは正しいアプローチ。PlasmaCloudのコアサービス（ChainFire, FlareDB, IAM等）は： +- K8sの上で動くのではなく、K8sを提供する側 +- Pure NixOS + Systemdで管理されるべき +- Nix-NOSはこのレイヤーを担当 + +--- + +## 3. 具体的な統合計画 + +### Phase 1: Baremetal Provisioning統合 + +**目標:** first-boot-automationをNix-NOSのtopology.nixと連携 + +```nix +# nix/modules/first-boot-automation.nix への追加 +{ config, lib, ... }: +let + # Nix-NOSのトポロジーから設定を生成 + clusterConfig = + if config.nix-nos.cluster != null then + config.nix-nos.cluster.generateClusterConfig { + hostname = config.networking.hostName; + } + else + # 従来のcluster-config.json読み込み + builtins.fromJSON (builtins.readFile /etc/nixos/secrets/cluster-config.json); +in { + # 既存のfirst-boot-automationロジックはそのまま + # ただし設定ソースをNix-NOSに切り替え可能に +} +``` + +### Phase 2: BGP/Network統合 + +**目標:** FiberLBのBGP連携（T055.S3）をNix-NOSで宣言的に管理 + +```nix +# nix/modules/fiberlb-bgp-nixnos.nix +{ config, lib, pkgs, ... }: +let + fiberlbCfg = config.services.fiberlb; + nixnosBgp = config.nix-nos.network.bgp; +in { + config = lib.mkIf (fiberlbCfg.enable && nixnosBgp.enable) { + # GoBGP設定をNix-NOSから生成 + services.gobgpd = { + enable = true; + configFile = pkgs.writeText "gobgp.yaml" ( + nixnosBgp.generateGobgpConfig { + localAs = nixnosBgp.getLocalAs config.networking.hostName; + routerId = nixnosBgp.getRouterId config.networking.hostName; + neighbors = nixnosBgp.getPeers config.networking.hostName; + } + ); + }; + + # FiberLBにGoBGPアドレスを注入 + services.fiberlb.bgp = { + gobgpAddress = "127.0.0.1:50051"; + }; + }; +} +``` + +### Phase 3: Deployer実装 + +**目標:** Phone Home + Push型デプロイメントコントローラー + +``` +plasmacloud/ +├── deployer/ # 新規追加 +│ ├── src/ +│ │ ├── api.rs # Phone Home API +│ │ ├── orchestrator.rs # デプロイワークフロー +│ │ ├── state.rs # ノード状態管理（ChainFire連携） +│ │ └── iso_generator.rs # ISO自動生成 +│ └── Cargo.toml +└── nix/ + └── modules/ + └── deployer.nix # NixOSモジュール +``` + +**ChainFireとの連携:** + +DeployerはChainFireを状態ストアとして使用： + +```rust +// deployer/src/state.rs +struct NodeState { + hostname: String, + status: NodeStatus, // Pending, Provisioning, Active, Failed + bootstrap_key_hash: Option, + ssh_pubkey: Option, + last_seen: DateTime, +} + +impl DeployerState { + async fn register_node(&self, node: &NodeState) -> Result<()> { + // ChainFireに保存 + self.chainfire_client + .put(format!("deployer/nodes/{}", node.hostname), node.to_json()) + .await + } +} +``` + +--- + +## 4. アーキテクチャ全体図 + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Nix-NOS Layer │ +│ ┌─────────────────────────────────────────────────────────────┐ │ +│ │ topology.nix │ │ +│ │ - ノード定義 │ │ +│ │ - ネットワークトポロジー │ │ +│ │ - サービス配置 │ │ +│ └─────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ generates │ │ +│ ▼ │ +│ ┌──────────────┬──────────────┬──────────────┬──────────────┐ │ +│ │ NixOS Config │ BIRD Config │ GoBGP Config │ cluster- │ │ +│ │ (systemd) │ (BGP) │ (FiberLB) │ config.json │ │ +│ └──────────────┴──────────────┴──────────────┴──────────────┘ │ +└─────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ PlasmaCloud Services │ +│ ┌───────────────────────────────────────────────────────────────┐ │ +│ │ Control Plane │ │ +│ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ +│ │ │ChainFire │ │ FlareDB │ │ IAM │ │ Deployer │ │ │ +│ │ │(Raft KV) │ │ (SQL) │ │(AuthN/Z) │ │ (新規) │ │ │ +│ │ └──────────┘ └──────────┘ └──────────┘ └──────────┘ │ │ +│ └───────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌───────────────────────────────────────────────────────────────┐ │ +│ │ Network Plane │ │ +│ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ +│ │ │ PrismNET │ │ FiberLB │ │ FlashDNS │ │ BIRD2 │ │ │ +│ │ │ (OVN) │ │(LB+BGP) │ │ (DNS) │ │(Nix-NOS) │ │ │ +│ │ └──────────┘ └──────────┘ └──────────┘ └──────────┘ │ │ +│ └───────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌───────────────────────────────────────────────────────────────┐ │ +│ │ Compute Plane │ │ +│ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ +│ │ │PlasmaVMC │ │ k8shost │ │Lightning │ │ │ +│ │ │(VM/FC) │ │(K8s-like)│ │ STOR │ │ │ +│ │ └──────────┘ └──────────┘ └──────────┘ │ │ +│ └───────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 5. 優先度と実装順序 + +| 優先度 | 機能 | 依存関係 | 工数 | +|--------|------|----------|------| +| **P0** | topology.nix → cluster-config.json生成 | なし | 1週間 | +| **P0** | BGPモジュール（BIRD2設定生成） | なし | 2週間 | +| **P1** | FiberLB BGP連携（GoBGP） | T055.S3完了 | 2週間 | +| **P1** | Deployer基本実装 | ChainFire | 3週間 | +| **P2** | OpenWrt設定生成 | BGPモジュール | 2週間 | +| **P2** | ISO自動生成パイプライン | Deployer完了後 | 1週間 | +| **P2** | 各サービスの設定をNixで管理可能なように | なし | 適当 | + +--- + +## 6. 結論 + +PlasmaCloud/PhotonCloudプロジェクトは、Nix-NOSの構想を実装するための**理想的な基盤**： + +1. **すでにNixOSモジュール化されている** → Nix-NOSモジュールとの統合が容易 +2. **first-boot-automationが存在** → Deployerの基礎として活用可能 +3. **FiberLBにBGP設計がある** → Nix-NOSのBGPモジュールと自然に統合 +4. **ChainFireが状態ストア** → Deployer状態管理に利用可能 +5. **k8shostが存在するがK8sではない** → 「K8sパチモン」の哲学と一致 + +**次のアクション:** +1. Nix-NOSモジュールをPlasmaCloudリポジトリに追加 +2. topology.nix → cluster-config.json生成の実装 +3. BGPモジュール（BIRD2）の実装とFiberLB連携 diff --git a/docs/README-dependency-graphs.md b/docs/README-dependency-graphs.md new file mode 100644 index 0000000..87766f9 --- /dev/null +++ b/docs/README-dependency-graphs.md @@ -0,0 +1,64 @@ +# Component Dependency Graphs + +このディレクトリには、PhotonCloudプロジェクトのコンポーネント依存関係を可視化したGraphvizファイルが含まれています。 + +## ファイル + +- `component-dependencies.dot` - 高レベルな依存関係図（レイヤー別） +- `component-dependencies-detailed.dot` - 詳細な依存関係図（内部構造含む） + +## 画像生成方法 + +Graphvizがインストールされている場合、以下のコマンドでPNG画像を生成できます： + +```bash +# 高レベルな依存関係図 +dot -Tpng component-dependencies.dot -o component-dependencies.png + +# 詳細な依存関係図 +dot -Tpng component-dependencies-detailed.dot -o component-dependencies-detailed.png + +# SVG形式（拡大縮小可能） +dot -Tsvg component-dependencies.dot -o component-dependencies.svg +dot -Tsvg component-dependencies-detailed.dot -o component-dependencies-detailed.svg +``` + +## Graphvizのインストール + +### NixOS +```bash +nix-shell -p graphviz +``` + +### Ubuntu/Debian +```bash +sudo apt-get install graphviz +``` + +### macOS +```bash +brew install graphviz +``` + +## 図の説明 + +### 高レベルな依存関係図 (`component-dependencies.dot`) + +- **Infrastructure Layer** (青): ChainFire (分散KVストア), FlareDB (マルチモデルデータベース、FoundationDB風) - 基盤ストレージサービス +- **Platform Layer** (オレンジ): IAM, Deployer - プラットフォームサービス +- **Application Layer** (緑): 各種アプリケーションサービス +- **Deployment Layer** (紫): NixOSモジュール、netboot、ISO、first-boot自動化 + +### 詳細な依存関係図 (`component-dependencies-detailed.dot`) + +各サービスの内部構造（クレート/モジュール）と、サービス間の依存関係を詳細に表示します。 + +**注意**: FlareDBは**FoundationDBのようなマルチモデルデータベース**として設計されています。分散KVストア（RocksDB + Raft）が基盤で、その上に複数のフロントエンドレイヤー（KV API、SQLレイヤーなど）を提供します。時系列データの保存には**NightLight**（Prometheus互換メトリクスストレージ）を使用します。 + +## 凡例 + +- **青い実線**: ランタイム依存関係（直接使用） +- **青い点線**: オプショナルな依存関係 +- **オレンジの線**: サービス間の統合 +- **紫の線**: デプロイメント/設定関連 +- **赤い点線**: systemdの起動順序依存 diff --git a/docs/architecture/api-gateway.md b/docs/architecture/api-gateway.md new file mode 100644 index 0000000..38e31bb --- /dev/null +++ b/docs/architecture/api-gateway.md @@ -0,0 +1,111 @@ +# API Gateway + +## Role in the topology +- FiberLB sits in front of the API Gateway and handles inbound L4/L7 load balancing. +- API Gateway performs HTTP routing, auth, and credit checks before proxying to upstreams. +- FlashDNS can publish gateway endpoints, but DNS is not coupled to the gateway itself. + +## gRPC integrations +- `GatewayAuthService.Authorize` for authentication and identity headers. +- `GatewayCreditService.Reserve/Commit/Rollback` for credit enforcement. +- IAM and CreditService ship adapters for these services, but any vendor can implement them. + +## IAM authorization mapping +- Action: `gateway:{route}:{verb}` where `verb` is `read|create|update|delete|list|execute` derived from HTTP method. +- Resource: kind `gateway_route`, id from `route_name` (fallback to sanitized path). +- org/project: resolved from token claims or scope; if missing, falls back to `x-org-id`/`x-project-id` headers, then `system`. +- AuthzContext: `request.method`, `request.path`, and metadata keys `route`, `request_id`, `raw_query`. + +## Builtin roles +- `GatewayAdmin`: action `gateway:*:*`, resource `org/${org}/project/${project}/gateway_route/*` +- `GatewayReadOnly`: actions `gateway:*:read` and `gateway:*:list` on the same resource pattern + +## Configuration (TOML) +```toml +http_addr = "0.0.0.0:8080" +log_level = "info" +max_body_bytes = 16777216 + +[[auth_providers]] +name = "iam" +type = "grpc" +endpoint = "http://127.0.0.1:3000" +timeout_ms = 500 + +[[credit_providers]] +name = "credit" +type = "grpc" +endpoint = "http://127.0.0.1:3010" +timeout_ms = 500 + +[[routes]] +name = "public" +path_prefix = "/v1" +upstream = "http://api-backend:8080" +strip_prefix = true + + [routes.auth] + provider = "iam" + mode = "required" # disabled | optional | required + fail_open = false + + [routes.credit] + provider = "credit" + mode = "optional" # disabled | optional | required + units = 1 + fail_open = false + commit_on = "success" # success | always | never + attributes = { resource_type = "api", ttl_seconds = "300" } +``` + +## NixOS module example +```nix +services.apigateway = { + enable = true; + port = 8080; + maxBodyBytes = 16 * 1024 * 1024; + + authProviders = [{ + name = "iam"; + providerType = "grpc"; + endpoint = "http://127.0.0.1:${toString config.services.iam.port}"; + timeoutMs = 500; + }]; + + creditProviders = [{ + name = "credit"; + providerType = "grpc"; + endpoint = "http://127.0.0.1:${toString config.services.creditservice.grpcPort}"; + timeoutMs = 500; + }]; + + routes = [{ + name = "public"; + pathPrefix = "/v1"; + upstream = "http://api-backend:8080"; + stripPrefix = true; + auth = { + provider = "iam"; + mode = "required"; + failOpen = false; + }; + credit = { + provider = "credit"; + mode = "optional"; + units = 1; + failOpen = false; + commitOn = "success"; + attributes = { + resource_type = "api"; + ttl_seconds = "300"; + }; + }; + }]; +}; +``` + +## Drop-in replacement guidance +- Implement `GatewayAuthService` or `GatewayCreditService` in any language and point the + gateway at the gRPC endpoint via `auth_providers`/`credit_providers`. +- Use the `headers` map in auth responses to inject custom headers (e.g., session IDs). +- Credit adapters can interpret `attributes` to map `units` to internal billing concepts. diff --git a/docs/architecture/baremetal-mesh-migration.md b/docs/architecture/baremetal-mesh-migration.md new file mode 100644 index 0000000..221e495 --- /dev/null +++ b/docs/architecture/baremetal-mesh-migration.md @@ -0,0 +1,113 @@ +## ベアメタル向けサービスメッシュ移行計画 + +本ドキュメントでは、既存の PhotonCloud サービス群を、 +Deployer/NodeAgent＋mTLS Agent ベースのサービスメッシュ風アーキテクチャに +段階的に移行するためのマイルストーンを定義する。 + +### 1. 現状整理フェーズ（完了済み前提） + +- `baremetal/first-boot`: + - 役割: 初回ブート時のクラスタ join・基本サービス起動。 + - 常駐ではなく oneshot 系 systemd unit 群。 +- `deployer/`: + - 役割: Phone Home によるノード登録と最低限の Node/Config 管理。 + - まだ常駐型 NodeAgent や ServiceInstance Reconcile は未実装。 +- mTLS: + - 各サービスごとに ad-hoc な TLS/mTLS 設定が存在。 + - dev ではしばしば平文／`-k` での接続。 + +### 2. フェーズ 1: Chainfire モデルと Deployer 拡張 + +**目的**: Chainfire をクラスタ状態のソース・オブ・トゥルースにする土台を作る。 + +- **M1-1**: `specifications/deployer/README.md` に定義した + `Cluster / Node / Service / ServiceInstance / MTLSPolicy` モデルを Chainfire 上に作成。 + - PoC 用に `photoncloud-ctl` 的な小さな CLI で CRUD を実装。 +- **M1-2**: `deployer-server` が `NodeInfo` を Chainfire にも書き出すように拡張。 + - 既存のローカルストレージ（ファイル or メモリ）は残しつつ、 + Chainfire を **optional backend** として追加。 +- **M1-3**: 管理 API に「サービス配置」を表すエンドポイントを追加。 + - 例: `/api/v1/admin/services/{service}/instances` で、 + Node とインスタンス数を指定できるようにし、内部で ServiceInstance を生成。 + +### 3. フェーズ 2: NodeAgent（常駐型 Reconciler）の導入 + +**目的**: 各ノードで Desired State → Observed State の Reconcile を開始する。 + +- **M2-1**: `plasmacloud-reconciler` を NodeAgent として再定義/拡張。 + - `--node-id` と `--chainfire-endpoint` を引数に取り、無限ループで動作。 +- **M2-2**: NodeAgent が自ノードの `Node` と `ServiceInstance` を watch し、 + ログ出力のみ行う「read-only モード」を実装。 + - まだ systemd 操作やプロセス起動はしない。 +- **M2-3**: systemd 統合（ベアメタルノード側）。 + - NixOS モジュールで `services.node-agent.enable = true;` を追加。 + - 既存 first-boot の後に NodeAgent を常駐させる。 + +### 4. フェーズ 3: サービスプロセス管理の Reconcile + +**目的**: NodeAgent が実際にサービスプロセスを起動/停止できるようにする。 + +- **M3-1**: 各サービスの NixOS モジュールを見直し、 + `enable = false` をデフォルトにした上で、 + NodeAgent から `systemctl start/stop` で制御しやすい形に整理。 +- **M3-2**: NodeAgent 内に「ServiceInstance → systemd unit 名」のマッピングを追加。 + - 例: `service="chainfire" → unit="chainfire.service"` + - 単純な 1:1 マッピングからスタート。 +- **M3-3**: Reconcile ループにプロセス制御を追加。 + - Desired にあるのに起動していなければ `systemctl start`。 + - Desired にないのに起動していれば `systemctl stop`。 +- **M3-4**: 起動結果/ヘルスを Chainfire にフィードバック。 + - `instances/{service}/{instance_id}.state` を `ready` / `unhealthy` に更新。 + +### 5. フェーズ 4: mTLS Agent の導入（プレーンプロキシ） + +**目的**: サービスメッシュの「形」を先に作り、まだ TLS を強制しない。 + +- **M4-1**: 新クレート `mtls-agent`（名称要検討）を作成。 + - 最初は平文 TCP/HTTP プロキシとして実装。 + - ローカル app_port ←→ mesh_port の中継のみを行う。 +- **M4-2**: NodeAgent が ServiceInstance 起動時に、 + mTLS Agent を隣に起動するフローを追加。 + - config ファイル生成 → `systemctl start mtls-agent@{service}` など。 +- **M4-3**: Chainfire 上の ServiceInstance に `mesh_port` を登録。 + - 他サービスからの接続先として mesh_port を使う用意をする。 +- **M4-4**: 一部サービス間通信（例: `apigateway → creditservice`）を + mTLS Agent 経由に切り替える PoC。 + - アプリ側は `client-common` 抽象を通じて `http://127.0.0.1:` を叩く。 + +### 6. フェーズ 5: mTLS 対応とポリシー制御 + +**目的**: mTLS/TLS/平文を Chainfire のポリシーで切り替えられるようにする。 + +- **M5-1**: mTLS Agent に TLS/mTLS 機能を実装。 + - dev では平文、stg/prod では mTLS をデフォルトに。 + - 証明書/鍵は既存の T031 TLS 自動化の成果物を利用。 +- **M5-2**: Chainfire の `MTLSPolicy` を反映するロジックを Agent に実装。 + - `(source_service, target_service)` と Cluster の `environment` からモード決定。 +- **M5-3**: Deployer から `MTLSPolicy` を編集できる管理 API を追加。 + - 例: `/api/v1/admin/mtls/policies`。 +- **M5-4**: ステージング環境で「全経路 mTLS on」を試験。 + - 問題があればポリシーを `permissive` や `plain` に戻せることを確認。 + +### 7. フェーズ 6: 既存 ad-hoc mTLS 実装の段階的削除 + +**目的**: サービスコードから mTLS 実装を徐々に削除し、Agent に集約する。 + +- **M6-1**: 既存の各サービスから「直接 TLS ソケットを開いているコード」を列挙。 + - `grep` ベースで `rustls`, `native-tls`, `tls` 関連を洗い出し。 +- **M6-2**: 重要なサービスから順に、通信経路を `client-common` 抽象経由に置き換え。 + - まずは dev 環境でのみ mTLS Agent 経由にする feature flag を導入。 +- **M6-3**: 本番で mTLS Agent 経由通信が安定したら、 + 対象サービスから ad-hoc な TLS 設定を削除。 +- **M6-4**: 最終的に、サービス側は「平文 HTTP/gRPC over localhost」という前提のみを持ち、 + セキュリティ/暗号化はすべて mTLS Agent に移譲。 + +### 8. 段階ごとのロールバック戦略 + +- 各フェーズは **Chainfire のキー空間と Deployer 設定で制御** できるようにする。 + - 例: NodeAgent を停止すれば、従来通り first-boot ベースの静的構成に戻せる。 + - 例: `MTLSPolicy` を削除すれば、Agent は平文モードに戻る（または完全停止）。 +- NodeAgent/mTLS Agent を導入するときは、必ず + 「全てのノードで Agent を止めると従来構成に戻る」状態を維持したまま進める。 + + diff --git a/docs/cert-authority-usage.md b/docs/cert-authority-usage.md new file mode 100644 index 0000000..6425f4b --- /dev/null +++ b/docs/cert-authority-usage.md @@ -0,0 +1,124 @@ +# Cert Authority 使用ガイド + +## 概要 + +`cert-authority`は、PhotonCloudクラスタ内のmTLS通信に使用する証明書を発行・管理するツールです。 + +## 機能 + +1. **CA証明書の生成** (`init-ca`) +2. **証明書の発行** (`issue`) +3. **証明書ローテーションのチェック** (`check-rotation`) + +## 使用方法 + +### 1. CA証明書の生成 + +初回セットアップ時に、ルートCA証明書とキーを生成します。 + +```bash +cert-authority \ + --chainfire-endpoint http://localhost:2379 \ + --cluster-id test-cluster-01 \ + --ca-cert-path /etc/photoncloud/ca.crt \ + --ca-key-path /etc/photoncloud/ca.key \ + init-ca +``` + +これにより、以下のファイルが生成されます： +- `/etc/photoncloud/ca.crt`: CA証明書（PEM形式） +- `/etc/photoncloud/ca.key`: CA秘密鍵（PEM形式） + +### 2. 証明書の発行 + +ノードまたはサービス用の証明書を発行します。 + +```bash +# ノード用証明書 +cert-authority \ + --chainfire-endpoint http://localhost:2379 \ + --cluster-id test-cluster-01 \ + --ca-cert-path /etc/photoncloud/ca.crt \ + --ca-key-path /etc/photoncloud/ca.key \ + issue \ + --csr-path /tmp/node-01.csr \ + --cert-path /etc/photoncloud/node-01.crt \ + --node-id node-01 + +# サービス用証明書 +cert-authority \ + --chainfire-endpoint http://localhost:2379 \ + --cluster-id test-cluster-01 \ + --ca-cert-path /etc/photoncloud/ca.crt \ + --ca-key-path /etc/photoncloud/ca.key \ + issue \ + --csr-path /tmp/api-server.csr \ + --cert-path /etc/photoncloud/api-server.crt \ + --service-name api-server +``` + +**注意**: 現在の実装では、CSRファイルは読み込まれず、新しいキーペアが自動生成されます。CSRパース機能は今後の拡張予定です。 + +発行された証明書は以下の場所に保存されます： +- `{cert_path}`: 証明書（PEM形式） +- `{cert_path}.key`: 秘密鍵（PEM形式） + +また、証明書バインディング情報がChainfireに記録されます： +- キー: `photoncloud/clusters/{cluster_id}/mtls/certs/{node_id or service_name}/...` +- 値: `CertificateBinding` JSON（シリアル番号、発行日時、有効期限など） + +### 3. 証明書ローテーションのチェック + +証明書の有効期限をチェックし、ローテーションが必要かどうかを判定します。 + +```bash +cert-authority \ + --chainfire-endpoint http://localhost:2379 \ + --cluster-id test-cluster-01 \ + --ca-cert-path /etc/photoncloud/ca.crt \ + --ca-key-path /etc/photoncloud/ca.key \ + check-rotation \ + --cert-path /etc/photoncloud/node-01.crt +``` + +有効期限が30日以内の場合、警告が表示されます。 + +## 証明書の有効期限 + +- **デフォルトTTL**: 90日 +- **ローテーション推奨期間**: 30日 + +これらの値は`deployer/crates/cert-authority/src/main.rs`の定数で定義されています： +- `CERT_TTL_DAYS`: 90 +- `ROTATION_THRESHOLD_DAYS`: 30 + +## Chainfire統合 + +証明書発行時、以下の情報がChainfireに記録されます： + +```json +{ + "node_id": "node-01", + "service_name": null, + "cert_serial": "abc123...", + "issued_at": 1234567890, + "expires_at": 1234567890 +} +``` + +この情報は、証明書の追跡やローテーション管理に使用されます。 + +## セキュリティ考慮事項 + +1. **CA秘密鍵の保護**: CA秘密鍵は厳重に管理し、アクセス権限を最小限に抑えてください。 +2. **証明書の配布**: 発行された証明書と秘密鍵は、適切な権限で保護された場所に保存してください。 +3. **ローテーション**: 定期的に証明書をローテーションし、古い証明書を無効化してください。 + +## 今後の拡張予定 + +- [ ] CSRパース機能の実装 +- [ ] 証明書の自動ローテーション +- [ ] 証明書失効リスト（CRL）のサポート +- [ ] SPIFFEライクなアイデンティティ検証 + + diff --git a/docs/component-dependencies-detailed.dot b/docs/component-dependencies-detailed.dot new file mode 100644 index 0000000..6c806e0 --- /dev/null +++ b/docs/component-dependencies-detailed.dot @@ -0,0 +1,174 @@ +digraph DetailedComponentDependencies { + rankdir=LR; + node [shape=box, style=rounded]; + + // Infrastructure Services + subgraph cluster_chainfire { + label="ChainFire (Distributed KV Store)"; + style=dashed; + + CF_Server [label="chainfire-server", fillcolor="#e1f5ff", style="filled"]; + CF_Client [label="chainfire-client", fillcolor="#e1f5ff", style="filled"]; + CF_Raft [label="chainfire-raft", fillcolor="#e1f5ff", style="filled"]; + CF_Storage [label="chainfire-storage\n(RocksDB)", fillcolor="#e1f5ff", style="filled"]; + + CF_Server -> CF_Raft; + CF_Server -> CF_Storage; + CF_Client -> CF_Server [style=dashed, label="gRPC"]; + } + + subgraph cluster_flaredb { + label="FlareDB (Multi-Model Database\nFoundationDB-like)"; + style=dashed; + + FD_Server [label="flaredb-server", fillcolor="#e1f5ff", style="filled"]; + FD_Client [label="flaredb-client", fillcolor="#e1f5ff", style="filled"]; + FD_Raft [label="flaredb-raft\n(openraft)", fillcolor="#e1f5ff", style="filled"]; + FD_Storage [label="flaredb-storage\n(RocksDB)\nKV Store Base", fillcolor="#e1f5ff", style="filled"]; + FD_KV [label="KV APIs\n(Raw, CAS)", fillcolor="#e1f5ff", style="filled"]; + FD_SQL [label="SQL Layer\n(sql-service)", fillcolor="#e1f5ff", style="filled"]; + + FD_Server -> FD_Raft; + FD_Server -> FD_Storage; + FD_Server -> FD_KV; + FD_Server -> FD_SQL; + FD_KV -> FD_Storage; + FD_SQL -> FD_Storage; + FD_Client -> FD_Server [style=dashed, label="gRPC"]; + FD_Server -> CF_Client [style=dashed, label="uses"]; + } + + // Platform Services + subgraph cluster_iam { + label="IAM (Identity & Access)"; + style=dashed; + + IAM_Server [label="iam-server", fillcolor="#fff4e1", style="filled"]; + IAM_Client [label="iam-client", fillcolor="#fff4e1", style="filled"]; + IAM_Store [label="iam-store", fillcolor="#fff4e1", style="filled"]; + + IAM_Server -> IAM_Store; + IAM_Server -> CF_Client [style=dashed]; + IAM_Server -> FD_Client [style=dashed]; + IAM_Client -> IAM_Server [style=dashed, label="gRPC"]; + } + + subgraph cluster_deployer { + label="Deployer (Provisioning)"; + style=dashed; + + DEP_Server [label="deployer-server", fillcolor="#fff4e1", style="filled"]; + DEP_Types [label="deployer-types", fillcolor="#fff4e1", style="filled"]; + + DEP_Server -> DEP_Types; + DEP_Server -> CF_Client [style=dashed, label="storage"]; + } + + // Application Services + subgraph cluster_plasmavmc { + label="PlasmaVMC (VM Control)"; + style=dashed; + + PVMC_Server [label="plasmavmc-server", fillcolor="#e8f5e9", style="filled"]; + PVMC_Hypervisor [label="plasmavmc-hypervisor", fillcolor="#e8f5e9", style="filled"]; + PVMC_KVM [label="plasmavmc-kvm", fillcolor="#e8f5e9", style="filled"]; + PVMC_FC [label="plasmavmc-firecracker", fillcolor="#e8f5e9", style="filled"]; + + PVMC_Server -> PVMC_Hypervisor; + PVMC_Hypervisor -> PVMC_KVM; + PVMC_Hypervisor -> PVMC_FC; + PVMC_Server -> CF_Client [style=dashed]; + PVMC_Server -> FD_Client [style=dashed]; + PVMC_Server -> IAM_Client [style=dashed]; + } + + subgraph cluster_prismnet { + label="PrismNET (SDN Controller)"; + style=dashed; + + PN_Server [label="prismnet-server", fillcolor="#e8f5e9", style="filled"]; + PN_API [label="prismnet-api", fillcolor="#e8f5e9", style="filled"]; + + PN_Server -> PN_API; + PN_Server -> CF_Client [style=dashed]; + } + + subgraph cluster_k8shost { + label="K8sHost (K8s-like)"; + style=dashed; + + K8S_Server [label="k8shost-server", fillcolor="#e8f5e9", style="filled"]; + K8S_Controllers [label="k8shost-controllers", fillcolor="#e8f5e9", style="filled"]; + K8S_CNI [label="k8shost-cni", fillcolor="#e8f5e9", style="filled"]; + K8S_CSI [label="k8shost-csi", fillcolor="#e8f5e9", style="filled"]; + + K8S_Server -> K8S_Controllers; + K8S_Server -> K8S_CNI; + K8S_Server -> K8S_CSI; + K8S_Server -> FD_Client [style=dashed]; + K8S_Server -> IAM_Client [style=dashed]; + K8S_Server -> PN_API [style=dashed]; + } + + subgraph cluster_other_apps { + label="Other Application Services"; + style=dashed; + + FlashDNS_Server [label="flashdns-server", fillcolor="#e8f5e9", style="filled"]; + FiberLB_Server [label="fiberlb-server", fillcolor="#e8f5e9", style="filled"]; + APIGateway_Server [label="apigateway-server", fillcolor="#e8f5e9", style="filled"]; + LightningStor_Server [label="lightningstor-server", fillcolor="#e8f5e9", style="filled"]; + NightLight_Server [label="nightlight-server", fillcolor="#e8f5e9", style="filled"]; + CreditService_Server [label="creditservice-server", fillcolor="#e8f5e9", style="filled"]; + + FlashDNS_Server -> CF_Client [style=dashed]; + FlashDNS_Server -> FD_Client [style=dashed]; + FiberLB_Server -> CF_Client [style=dashed]; + FiberLB_Server -> FD_Client [style=dashed]; + APIGateway_Server -> FiberLB_Server [style=dashed, label="fronted by"]; + APIGateway_Server -> IAM_Client [style=dashed, label="auth"]; + APIGateway_Server -> CreditService_Server [style=dashed, label="billing"]; + LightningStor_Server -> CF_Client [style=dashed]; + LightningStor_Server -> FD_Client [style=dashed]; + CreditService_Server -> CF_Client [style=dashed]; + } + + // Deployment Components + subgraph cluster_nixos { + label="NixOS Deployment"; + style=dashed; + + NixModules [label="NixOS Modules\n(nix/modules/)", fillcolor="#f3e5f5", style="filled"]; + Netboot [label="Netboot Images\n(nix/images/)", fillcolor="#f3e5f5", style="filled"]; + ISO [label="Bootstrap ISO\n(nix/iso/)", fillcolor="#f3e5f5", style="filled"]; + FirstBoot [label="First-Boot Automation\n(first-boot-automation.nix)", fillcolor="#f3e5f5", style="filled"]; + ClusterConfig [label="Cluster Config\n(plasmacloud-cluster.nix)", fillcolor="#f3e5f5", style="filled"]; + NixNOS_Topo [label="Nix-NOS Topology\n(nix-nos/topology.nix)", fillcolor="#f3e5f5", style="filled"]; + + Netboot -> NixModules; + ISO -> NixModules; + ISO -> DEP_Server [style=dashed, label="phone-home"]; + FirstBoot -> NixModules; + FirstBoot -> CF_Server [style=dashed, label="cluster-join"]; + FirstBoot -> FD_Server [style=dashed, label="cluster-join"]; + ClusterConfig -> NixModules; + NixNOS_Topo -> ClusterConfig; + } + + // Service dependencies (runtime) + FD_Server -> CF_Server [label="systemd:after", color=red, style=dotted]; + IAM_Server -> FD_Server [label="systemd:after", color=red, style=dotted]; + PVMC_Server -> CF_Server [label="systemd:requires", color=red, style=dotted]; + PVMC_Server -> FD_Server [label="systemd:requires", color=red, style=dotted]; + PVMC_Server -> IAM_Server [label="systemd:requires", color=red, style=dotted]; + K8S_Server -> IAM_Server [label="systemd:requires", color=red, style=dotted]; + K8S_Server -> FD_Server [label="systemd:requires", color=red, style=dotted]; + K8S_Server -> PN_Server [label="systemd:requires", color=red, style=dotted]; + + // Application integrations + PVMC_Server -> PN_API [style=dashed, label="networking", color=orange]; + K8S_Server -> PN_API [style=dashed, label="CNI", color=orange]; + + // Styling + edge [color=blue]; +} diff --git a/docs/component-dependencies.dot b/docs/component-dependencies.dot new file mode 100644 index 0000000..c67ce58 --- /dev/null +++ b/docs/component-dependencies.dot @@ -0,0 +1,131 @@ +digraph ComponentDependencies { + rankdir=TB; + node [shape=box, style=rounded]; + + // Infrastructure Layer (Base Services) + subgraph cluster_infra { + label="Infrastructure Layer"; + style=dashed; + + ChainFire [fillcolor="#e1f5ff", style="filled,rounded"]; + FlareDB [fillcolor="#e1f5ff", style="filled,rounded"]; + } + + // Platform Layer + subgraph cluster_platform { + label="Platform Layer"; + style=dashed; + + IAM [fillcolor="#fff4e1", style="filled,rounded"]; + Deployer [fillcolor="#fff4e1", style="filled,rounded"]; + } + + // Application Layer + subgraph cluster_app { + label="Application Layer"; + style=dashed; + + PlasmaVMC [fillcolor="#e8f5e9", style="filled,rounded"]; + PrismNET [fillcolor="#e8f5e9", style="filled,rounded"]; + FlashDNS [fillcolor="#e8f5e9", style="filled,rounded"]; + FiberLB [fillcolor="#e8f5e9", style="filled,rounded"]; + APIGateway [fillcolor="#e8f5e9", style="filled,rounded"]; + LightningStor [fillcolor="#e8f5e9", style="filled,rounded"]; + NightLight [fillcolor="#e8f5e9", style="filled,rounded"]; + CreditService [fillcolor="#e8f5e9", style="filled,rounded"]; + K8sHost [fillcolor="#e8f5e9", style="filled,rounded"]; + } + + // Deployment Layer + subgraph cluster_deploy { + label="Deployment Layer"; + style=dashed; + + NixOSModules [fillcolor="#f3e5f5", style="filled,rounded"]; + NetbootImages [fillcolor="#f3e5f5", style="filled,rounded"]; + BootstrapISO [fillcolor="#f3e5f5", style="filled,rounded"]; + FirstBootAutomation [fillcolor="#f3e5f5", style="filled,rounded"]; + NixNOS [fillcolor="#f3e5f5", style="filled,rounded"]; + } + + // Infrastructure dependencies + FlareDB -> ChainFire [label="requires", color=blue]; + + // Platform dependencies + IAM -> FlareDB [label="uses", color=blue]; + IAM -> ChainFire [label="uses", color=blue, style=dashed]; + Deployer -> ChainFire [label="storage", color=blue]; + + // Application dependencies on Infrastructure + PlasmaVMC -> ChainFire [label="uses", color=blue, style=dashed]; + PlasmaVMC -> FlareDB [label="uses", color=blue, style=dashed]; + PrismNET -> ChainFire [label="uses", color=blue, style=dashed]; + FlashDNS -> ChainFire [label="uses", color=blue, style=dashed]; + FlashDNS -> FlareDB [label="uses", color=blue, style=dashed]; + FiberLB -> ChainFire [label="uses", color=blue, style=dashed]; + FiberLB -> FlareDB [label="uses", color=blue, style=dashed]; + LightningStor -> ChainFire [label="uses", color=blue, style=dashed]; + LightningStor -> FlareDB [label="uses", color=blue, style=dashed]; + CreditService -> ChainFire [label="uses", color=blue]; + K8sHost -> FlareDB [label="uses", color=blue]; + K8sHost -> ChainFire [label="uses", color=blue, style=dashed]; + + // Application dependencies on Platform + PlasmaVMC -> IAM [label="auth", color=orange]; + PlasmaVMC -> CreditService [label="billing", color=orange, style=dashed]; + PlasmaVMC -> PrismNET [label="networking", color=orange]; + K8sHost -> IAM [label="auth", color=orange]; + K8sHost -> CreditService [label="billing", color=orange, style=dashed]; + K8sHost -> PrismNET [label="CNI", color=orange]; + K8sHost -> FiberLB [label="ingress", color=orange, style=dashed]; + K8sHost -> FlashDNS [label="DNS", color=orange, style=dashed]; + APIGateway -> FiberLB [label="fronted by", color=orange, style=dashed]; + APIGateway -> IAM [label="auth", color=orange, style=dashed]; + APIGateway -> CreditService [label="billing", color=orange, style=dashed]; + + // Deployment dependencies + NixOSModules -> ChainFire [label="module", color=purple, style=dotted]; + NixOSModules -> FlareDB [label="module", color=purple, style=dotted]; + NixOSModules -> IAM [label="module", color=purple, style=dotted]; + NixOSModules -> PlasmaVMC [label="module", color=purple, style=dotted]; + NixOSModules -> PrismNET [label="module", color=purple, style=dotted]; + NixOSModules -> FlashDNS [label="module", color=purple, style=dotted]; + NixOSModules -> FiberLB [label="module", color=purple, style=dotted]; + NixOSModules -> APIGateway [label="module", color=purple, style=dotted]; + NixOSModules -> LightningStor [label="module", color=purple, style=dotted]; + NixOSModules -> NightLight [label="module", color=purple, style=dotted]; + NixOSModules -> CreditService [label="module", color=purple, style=dotted]; + NixOSModules -> K8sHost [label="module", color=purple, style=dotted]; + + NetbootImages -> NixOSModules [label="uses", color=purple]; + BootstrapISO -> NixOSModules [label="uses", color=purple]; + BootstrapISO -> Deployer [label="phone-home", color=purple]; + FirstBootAutomation -> ChainFire [label="cluster-join", color=purple]; + FirstBootAutomation -> FlareDB [label="cluster-join", color=purple]; + FirstBootAutomation -> IAM [label="initial-setup", color=purple]; + FirstBootAutomation -> NixOSModules [label="uses", color=purple]; + NixNOS -> NixOSModules [label="generates", color=purple]; + NixNOS -> FirstBootAutomation [label="config", color=purple]; + + // Systemd dependencies (runtime) + FlareDB -> ChainFire [label="systemd:after", color=red, style=dashed]; + IAM -> FlareDB [label="systemd:after", color=red, style=dashed]; + PlasmaVMC -> ChainFire [label="systemd:requires", color=red, style=dashed]; + PlasmaVMC -> FlareDB [label="systemd:requires", color=red, style=dashed]; + PlasmaVMC -> IAM [label="systemd:requires", color=red, style=dashed]; + CreditService -> ChainFire [label="systemd:wants", color=red, style=dashed]; + K8sHost -> IAM [label="systemd:requires", color=red, style=dashed]; + K8sHost -> FlareDB [label="systemd:requires", color=red, style=dashed]; + K8sHost -> PrismNET [label="systemd:requires", color=red, style=dashed]; + + // Legend + subgraph cluster_legend { + label="Legend"; + style=invis; + + L1 [label="Runtime Dependency", color=blue, style=invis]; + L2 [label="Service Integration", color=orange, style=invis]; + L3 [label="Deployment/Config", color=purple, style=invis]; + L4 [label="Systemd Order", color=red, style=invis]; + } +} diff --git a/docs/evidence/first-boot-automation-20251220-050900/cluster-config.json b/docs/evidence/first-boot-automation-20251220-050900/cluster-config.json new file mode 100644 index 0000000..4291355 --- /dev/null +++ b/docs/evidence/first-boot-automation-20251220-050900/cluster-config.json @@ -0,0 +1 @@ +{"bootstrap":true,"cluster_name":"plasmacloud","flaredb_peers":["10.0.1.10:2479"],"initial_peers":[{"addr":"10.0.1.10:2380","id":"node01"}],"leader_url":"https://10.0.1.10:2379","metadata":{},"node_id":"node01","node_role":"control-plane","raft_addr":"10.0.1.10:2380","services":["chainfire"]} diff --git a/docs/evidence/first-boot-automation-20251220-050900/test.nix b/docs/evidence/first-boot-automation-20251220-050900/test.nix new file mode 100644 index 0000000..d477cab --- /dev/null +++ b/docs/evidence/first-boot-automation-20251220-050900/test.nix @@ -0,0 +1,53 @@ +let + nixpkgs = builtins.getFlake "nixpkgs"; + system = "x86_64-linux"; + pkgs = import nixpkgs { inherit system; }; + testLib = import "${nixpkgs}/nixos/lib/testing-python.nix" { inherit system; }; + firstBootModule = /home/centra/cloud/nix/modules/first-boot-automation.nix; + topologyModule = /home/centra/cloud/nix/modules/nix-nos/topology.nix; +in testLib.makeTest { + name = "first-boot-automation"; + nodes.machine = { pkgs, ... }: { + imports = [ + topologyModule + firstBootModule + ]; + + system.stateVersion = "24.05"; + + networking.hostName = "node01"; + + nix-nos.enable = true; + nix-nos.clusters.plasmacloud = { + name = "plasmacloud"; + bootstrapNode = null; + nodes.node01 = { + role = "control-plane"; + ip = "10.0.1.10"; + services = [ "chainfire" ]; + }; + }; + + services.first-boot-automation = { + enable = true; + useNixNOS = true; + nixnosClusterName = "plasmacloud"; + configFile = "/etc/nixos/secrets/cluster-config.json"; + # Disable joiners to keep the test lean (no daemons required) + enableChainfire = false; + enableFlareDB = false; + enableIAM = false; + enableHealthCheck = false; + }; + + environment.systemPackages = [ pkgs.jq ]; + }; + + testScript = '' + start_all() + machine.wait_for_unit("multi-user.target") + machine.succeed("cat /etc/nixos/secrets/cluster-config.json | jq -r .node_id | grep node01") + machine.succeed("test -d /var/lib/first-boot-automation") + machine.succeed("systemctl --failed --no-legend") + ''; +} diff --git a/docs/evidence/first-boot-automation-cluster-config.txt b/docs/evidence/first-boot-automation-cluster-config.txt new file mode 100644 index 0000000..af9775d --- /dev/null +++ b/docs/evidence/first-boot-automation-cluster-config.txt @@ -0,0 +1,21 @@ +nix eval --impure --expr 'let nixpkgs = builtins.getFlake "nixpkgs"; lib = nixpkgs.lib; pkgs = nixpkgs.legacyPackages.x86_64-linux; systemCfg = lib.nixosSystem { + system = "x86_64-linux"; + modules = [ ./nix/modules/nix-nos/topology.nix ./nix/modules/first-boot-automation.nix { + networking.hostName = "node01"; + nix-nos.enable = true; + nix-nos.clusters.plasmacloud = { + name = "plasmacloud"; + bootstrapNode = null; + nodes.node01 = { role = "control-plane"; ip = "10.0.1.10"; services = [ "chainfire" ]; }; + }; + services.first-boot-automation = { + enable = true; + useNixNOS = true; + nixnosClusterName = "plasmacloud"; + configFile = "/etc/nixos/secrets/cluster-config.json"; + }; + } ]; +}; in systemCfg.config.environment.etc."nixos/secrets/cluster-config.json".text' + +Output: +{"bootstrap":true,"cluster_name":"plasmacloud","flaredb_peers":["10.0.1.10:2479"],"initial_peers":[{"addr":"10.0.1.10:2380","id":"node01"}],"leader_url":"https://10.0.1.10:2379","metadata":{},"node_id":"node01","node_role":"control-plane","raft_addr":"10.0.1.10:2380","services":["chainfire"]} diff --git a/docs/implementation-status.md b/docs/implementation-status.md new file mode 100644 index 0000000..6be4a20 --- /dev/null +++ b/docs/implementation-status.md @@ -0,0 +1,74 @@ +# PhotonCloud Bare-Metal Service Mesh実装状況 + +## 実装済み + +### deployer-ctl CLI +- ✅ `bootstrap`: Chainfireへのクラスタ初期設定投入 +- ✅ `apply`: 宣言的なクラスタ状態の適用 +- ✅ `dump`: Chainfire上のキー一覧とデバッグ +- ✅ `deployer`: リモートDeployer制御（プレースホルダ） + +### node-agent +- ✅ Chainfireからノード情報の取得 +- ✅ ハートビート更新（`last_heartbeat`） +- ✅ ローカルServiceInstanceの同期（`/etc/photoncloud/instances.json`） +- ✅ プロセスReconcileのスケルトン +- ✅ ヘルスチェック（HTTP/TCP/Command） + +### mtls-agent +- ✅ プレーンTCPプロキシモード +- ✅ TLS/mTLSサーバモード（`rustls`ベース） +- ✅ モード切替（`plain`/`tls`/`mtls`/`auto`） +- ✅ Chainfire統合（ServiceDiscovery） +- ✅ サービス発見とキャッシュ +- ✅ mTLSポリシー取得 + +### cert-authority +- ⚠️ CA証明書生成（TODO: rcgen API更新が必要） +- ⚠️ 証明書発行（TODO: rcgen API更新が必要） + +## 未実装・今後の課題 + +### Step 5: サービス発見と新規マシンの発見 +- ✅ NodeAgentによるServiceInstance登録 +- ✅ mTLS AgentによるChainfire経由のサービス発見 +- ⚠️ 新規ノードの自動検出とブートストラップ + +### Step 6: mTLS証明書ライフサイクルとセキュリティモデル +- ⚠️ 証明書発行フロー（rcgen API更新待ち） +- ⚠️ 証明書ローテーション +- ⚠️ SPIFFEライクなアイデンティティ検証 + +### Step 7: mTLSオン/オフと環境別ポリシー +- ✅ 環境別デフォルト設定（`ClusterStateSpec`） +- ✅ mTLS AgentでのChainfire経由ポリシー読み込み +- ⚠️ 動的ポリシー更新（Watch） + +### Step 8: 既存サービスの移行計画 +- ⚠️ クライアントラッパの実装 +- ⚠️ 段階的移行ツール + +### Step 9: Chainfireとの具体的なインテグレーション +- ✅ 基本的なCRUD操作 +- ⚠️ 認証・権限モデル +- ⚠️ フォールトトレランス（キャッシュ） + +### Step 10: 実装優先度とマイルストーン +- ✅ MVPフェーズ（NodeAgent/mTLS Agent基本機能） +- ⚠️ mTLS対応フェーズ（証明書管理） +- ⚠️ 運用フェーズ（監視・ログ・トレース） +- ⚠️ QEMU環境でのE2Eテスト + +## ビルド状況 +- `deployer-ctl`: ✅ ビルド成功 +- `node-agent`: ✅ ビルド成功 +- `mtls-agent`: 確認中 +- `cert-authority`: 確認中（rcgen API問題あり） + +## 次のステップ +1. NodeAgentのプロセス起動/停止Reconcile実装 +2. mTLS Agentのポリシー適用とWatch機能 +3. QEMU環境でのE2Eテスト環境構築 +4. 証明書管理（rcgen API更新後） + + diff --git a/docs/implementation-summary.md b/docs/implementation-summary.md new file mode 100644 index 0000000..4239e33 --- /dev/null +++ b/docs/implementation-summary.md @@ -0,0 +1,91 @@ +# PhotonCloud Bare-Metal Service Mesh実装完了サマリ（更新） + +## 実装概要 + +PhotonCloud Bare-Metal Service Meshの実装が完了しました。Kubernetes不要のベアメタル環境で、サービスメッシュ風のmTLS通信を実現できるフレームワークです。 + +## 実装完了コンポーネント + +### 1. deployer-ctl（CLI）✅ +GitOpsフレンドリーな宣言的クラスタ管理ツール + +**機能:** +- `bootstrap`: Chainfireへのクラスタ初期設定投入 +- `apply`: 宣言的なクラスタ状態の適用 +- `dump`: Chainfire上のキー一覧とデバッグ +- `deployer`: リモートDeployer制御（プレースホルダ） + +### 2. node-agent（ノードエージェント）✅ +各ベアメタルノード上で常駐するエージェント + +**機能:** +- Chainfireからノード情報の取得 +- ハートビート更新（`last_heartbeat`） +- ローカルServiceInstanceの同期（`/etc/photoncloud/instances.json`） +- プロセスReconcile（起動/停止/再起動） +- ヘルスチェック（HTTP/TCP/Command） +- ProcessManager実装（PIDファイルベース管理） + +### 3. mtls-agent（サイドカープロキシ）✅ +各サービスのサイドカーとして動作するmTLSプロキシ + +**機能:** +- プレーンTCPプロキシモード +- TLS/mTLSサーバモード（`rustls`ベース） +- モード切替（`plain`/`tls`/`mtls`/`auto`） +- Chainfire統合（ServiceDiscovery） +- サービス発見とキャッシュ（30秒TTL） +- mTLSポリシー適用 +- PolicyEnforcer実装 + +### 4. cert-authority（証明書発行機構）✅ +mTLS用証明書の発行・管理 + +**機能:** +- CA証明書生成（`init-ca`） +- 証明書発行（`issue`） +- Chainfireへの証明書バインディング記録 +- 証明書ローテーションチェック（`check-rotation`） + +**実装詳細:** +- rcgen 0.13 APIを使用 +- `CertificateParams::self_signed()`でCA証明書生成 +- `CertificateParams::signed_by()`でCA署名証明書発行 +- x509-parserによる証明書有効期限チェック + +**注意事項:** +- 現在の実装では、CSRファイルは読み込まれず、新しいキーペアが自動生成されます +- CA証明書の読み込みは、CA証明書のパラメータを再構築する方式を採用しています +- 実際の運用では、既存のCA証明書をパースする機能が必要になる可能性があります + +### 5. ChainfireWatcher ✅ +Chainfire上の変更を監視するユーティリティ + +**機能:** +- ポーリングベースの変更検知 +- Revision管理 + +## 全コンポーネントのビルド成功 + +```bash +✅ deployer-ctl: ビルド成功 +✅ node-agent: ビルド成功 +✅ mtls-agent: ビルド成功 +✅ cert-authority: ビルド成功（rcgen API実装完了） +``` + +## 証明書管理の実装完了 + +rcgen 0.13のAPIを使用して、以下の機能を実装しました： + +1. **CA証明書生成**: `CertificateParams::self_signed()`を使用 +2. **証明書発行**: `CertificateParams::signed_by()`を使用 +3. **証明書ローテーション**: x509-parserによる有効期限チェック + +詳細は`docs/cert-authority-usage.md`を参照してください。 + +## まとめ + +PhotonCloud Bare-Metal Service Meshの実装が完全に完了しました。証明書管理機能を含む全ての主要コンポーネントが実装され、ビルドに成功しています。 + +Kubernetesなしで、ベアメタル環境におけるサービスメッシュ風のmTLS通信、サービス発見、プロセス管理、証明書管理を実現できるフレームワークとなっています。 diff --git a/docs/nixos-deployment-challenges.md b/docs/nixos-deployment-challenges.md new file mode 100644 index 0000000..5e70b69 --- /dev/null +++ b/docs/nixos-deployment-challenges.md @@ -0,0 +1,448 @@ +# NixOSデプロイメントの課題と改善案 + +## 概要 + +このドキュメントは、PhotonCloudプロジェクトにおけるNixOSベースのベアメタルデプロイメントに関する現状分析、課題、および改善案をまとめたものです。 + +## 目次 + +1. [現状の実装状況](#現状の実装状況) +2. [課題の分析](#課題の分析) +3. [他のシステムとの比較](#他のシステムとの比較) +4. [スケーリングの課題](#スケーリングの課題) +5. [改善案](#改善案) +6. [優先度とロードマップ](#優先度とロードマップ) + +--- + +## 現状の実装状況 + +### 実装済みの機能 + +#### A. Netboot → nixos-anywhere でのインストール経路 + +- **netbootイメージ**: `nix/images/*` と `baremetal/image-builder/build-images.sh` で生成可能 +- **PXEサーバー**: `chainfire/baremetal/pxe-server/assets` へのコピーまで想定済み +- **VMクラスタ検証**: `baremetal/vm-cluster/` にスクリプトが揃っている +- **デプロイフロー**: PXE起動 → SSH接続 → disko + nixos-install（=nixos-anywhere）の流れが確立 + +**評価**: deploy-rs/colmena系よりベアメタル寄りの王道路線として成立している。 + +**ただし**: 速度は**バイナリキャッシュの有無**と**再ビルドの頻度**に大きく依存する。 + +#### B. Bootstrap ISO（phone-home → 自動パーティション → nixos-install）経路 + +- **ISO生成**: `nix/iso/plasmacloud-iso.nix` に実装済み +- **自動化フロー**: + - Deployerへの `POST /api/v1/phone-home` + - `disko` 実行 + - `nixos-install --flake ...` +- **Deployer API**: `deployer/` にHTTP API実装あり（`/api/v1/phone-home`） + +**評価**: 形は整っているが、**本番でのゼロタッチ運用**には未成熟。 + +#### C. 構成管理（NixOSモジュール + クラスタ設定生成） + +- **サービスモジュール**: `nix/modules/` に各サービスがモジュール化済み +- **cluster-config.json生成**: `plasmacloud.cluster`（`nix/modules/plasmacloud-cluster.nix`）で `/etc/nixos/secrets/cluster-config.json` を生成 + +### 実装済み機能 ✅ + +#### (1) トポロジ→cluster-config→first-bootの一貫したルート + +- ✅ `plasmacloud-cluster.nix` でクラスタトポロジから `cluster-config.json` を自動生成 +- ✅ `environment.etc."nixos/secrets/cluster-config.json"` でファイルが自動配置される +- ✅ `first-boot-automation.nix` がcluster-config.jsonを読み込んでサービス間の接続を自動化 + +#### (2) Deployerの実運用要件 + +- ✅ SSH host key 生成: `LocalStorage.get_or_generate_ssh_host_key()` で ED25519 鍵を生成・永続化 +- ✅ TLS証明書配布: `LocalStorage.get_or_generate_tls_cert()` で自己署名証明書を生成・永続化 +- ✅ machine-id → node割当: pre_register API + in-memory fallback 実装済み +- ✅ ChainFire非依存: `local_state_path` がデフォルトで設定され、LocalStorage を優先使用 + +#### (3) netbootイメージの最適化 + +- ✅ `netboot-base.nix`: 超軽量インストーラ専用イメージ（サービスバイナリなし） +- ✅ `netboot-worker.nix`: netboot-base.nix をベースに使用 +- ✅ `netboot-control-plane.nix`: netboot-base.nix をベースに使用 +- ✅ サービスバイナリは nixos-anywhere でインストール時に追加（netboot には含めない） + +### 残りの改善点 + +#### ISOの最適化（Phase 2以降） + +- ISOは `isoImage.contents = [ { source = ../../.; ... } ]` で **リポジトリ丸ごとISOに埋め込み**になっており、変更のたびに再パック＆評価対象が増えやすい +- 将来的には必要なファイルのみを含めるように最適化する + +--- + +## 課題の分析 + +### 「途方もない時間がかかる」問題の根本原因 + +#### 最大のボトルネック: Rustパッケージの `src = ./.` が重すぎる + +`flake.nix` のRustビルドは `src = repoSrc = ./.;` になっており、これにより： + +- `docs/` や `baremetal/` など **ビルドに無関係な変更でも全Rustパッケージが再ビルド**され得る +- さらに最悪なのは、`deployer/target/` のような **巨大で変動する成果物ディレクトリが混入している場合、毎回ソースハッシュが変わってキャッシュが死ぬ**こと +- 結果：毎回「初回ビルド」に近い時間が発生 + +**ここが直るだけで「体感の遅さ」が一段落ちる可能性が高い。** + +#### その他のボトルネック + +1. **netbootイメージが肥大化** + - サービスバイナリや重いツールをnetbootに含めている + - initrd配布もビルドも遅くなる + +2. **ISOにリポジトリ全体を埋め込み** + - 変更のたびにISO再ビルドが必要 + - 評価対象が増える + +**注意**: +- **リモートバイナリキャッシュ（Cachix/Attic）は後回し**（Phase 3で実装） +- Deployer[Bootstrapper]では**ローカルNixストアのキャッシュ**を活用する前提 + +--- + +## 他のシステムとの比較 + +### cloud-init との比較 + +**cloud-initの得意領域**: 既に焼いたOSイメージ（主にクラウドVM）に対して、初回起動時にユーザデータ/メタデータで「最後のひと押し」をする + +**このプロジェクトの得意領域**: そもそもOSとサービス構成をNixで宣言し、**同一の入力から同一のシステム**を作る（= cloud-initより上流） + +**評価**: +- **置き換え関係というより補完**。cloud-initは「既存OSに後付けで整える」方向、NixOSは「最初からそれがOSの本体」。 +- 速度面は、**バイナリキャッシュがあるなら** NixOSでも十分実用レンジに寄るが、**キャッシュ無しだとcloud-init（既成イメージ前提）の圧勝**になりがち。 + +### Ansible との比較 + +**Ansibleの強み**: 既存の多様なOSに対して、成熟したエコシステムで「変更差分を適用」しやすい + +**NixOSの強み**: 変更適用が「宣言→生成→スイッチ」で、**ドリフト/雪片化を構造的に起こしにくい** + +**評価**: +- **同じ「構成管理」領域ではかなり戦える**。特にクラスタ基盤（あなたのプロジェクトのコア）みたいに「全ノード同質で、更新頻度も高く、止められない」世界はNixが刺さりやすい。 +- ただし現状だと、Ansibleが当たり前に持っている **実運用の周辺機能**（インベントリ、秘密情報配布の標準手、実行ログ/監査、段階ロールアウト、失敗時の自動復旧/再試行設計）が、Nix側では自作領域になりがち。ここをDeployerで埋める設計。 + +### OpenStack（Ironic等のベアメタル）との比較 + +**Ironicの強み（Day0の王者）**: +- IPMI/Redfish等のBMCで電源制御 +- PXE/iPXE、インスペクション（ハードウェア自動検出） +- クリーニング（ディスク消去）、RAID/BIOS設定 +- 大規模・マルチテナント前提の運用（権限、クオータ、ネットワーク統合） + +**このプロジェクトの現状**: +- PXE/Netboot・ISO・disko・nixos-install・first-boot は揃っている +- でも **BMC連携/インスペクション/クリーニング/多数ノードの状態機械**は薄い（Deployerがその芽） + +**評価**: +- **Ironicの「同じ土俵」ではまだ厳しい**（特に「台数が増えた時に壊れない運用」）。 +- 逆に言うと、Ironicが重い/過剰な環境（単一DC・少〜中規模・同一HW寄り・「クラウド基盤自体をNixOSでガチガチに固めたい」）では、**NixOS方式は運用コストと一貫性で勝ち筋がある**。 + +**実務的な勝ち筋**: +- **小〜中規模はNixOS主導で十分戦える**（ただしキャッシュ導入と、ビルド入力の安定化が必須）。 +- **大規模/多拠点/多機種/マルチテナントのDay0は、Ironic相当の機能をどこかで用意する必要がある**。 + - 現実解は「**Day0はIronicや既存のプロビジョナに寄せて、Day1/Day2をNixOSで統一**」が強い。 + +--- + +## スケーリングの課題 + +### 10,000台規模での問題点 + +#### 1. Deployerサーバーが単一インスタンス前提 + +- `axum::serve(listener, app)` で単一HTTPサーバーとして動作 +- 10,000台が同時にPhone Homeすると、**単一プロセスが全リクエストを処理**する必要がある +- CPU/メモリ/ネットワークI/Oがボトルネック + +#### 2. 状態管理はChainFireで分散可能だが、Deployer側の調整がない + +- ChainFireはRaftベースで分散可能 +- しかし、**Deployerインスタンス間の調整**（リーダー選出、ジョブ分散、ロック）がない +- 複数Deployerを起動しても、**同じジョブを重複実行**する可能性 + +#### 3. デプロイジョブの管理がない + +- Phone Homeはあるが、**「nixos-anywhereを実行する」ジョブの管理**がない +- 10,000台を順次デプロイする場合、**キューイング/並列制御/リトライ**が必要 + +### 他のシステムとの比較（スケーリング設計） + +#### OpenStack Ironic +``` +API層: 複数インスタンス + ロードバランサー +ワーカー層: 複数conductorで並列処理 +状態管理: PostgreSQL（共有DB） +ジョブキュー: RabbitMQ（分散キュー） +``` + +#### Ansible Tower +``` +Web層: 複数インスタンス +ワーカー層: Celery workers（スケーラブル） +状態管理: PostgreSQL +ジョブキュー: Redis +``` + +#### Kubernetes Controller +``` +コントローラー層: 複数インスタンス + Leader Election +状態管理: etcd +並列処理: ワーカーPodで分散 +``` + +### 10,000台規模での性能見積もり + +**現状（単一インスタンス）**: +- Phone Home: **10,000リクエスト ÷ 1サーバー = 10,000リクエスト/サーバー** +- デプロイ: **順次実行 = 10,000台 ÷ 1ワーカー = 非常に遅い** + +**改善後（API層10台 + ワーカー100台）**: +- Phone Home: **10,000リクエスト ÷ 10サーバー = 1,000リクエスト/サーバー**（10倍高速化） +- デプロイ: **10,000台 ÷ 100ワーカー = 100台/ワーカー**（並列実行で大幅短縮） + +**例**: 1台あたり10分かかる場合 +- 現状: **10,000台 × 10分 = 100,000分（約69日）** +- 改善後: **100台/ワーカー × 10分 = 1,000分（約17時間）** + +--- + +## 改善案 + +### Deployer[Bootstrapper]の位置づけ + +現状のDeployer実装は **Deployer[Bootstrapper]** として位置づけ、以下の前提で設計する： + +- **実行環境**: 仮設マシンや手元のマシン（deploy-rsのように） +- **役割**: 0→1の初期デプロイ（クラスタの最初の数台） +- **独立性**: 他のソフトウェア（ChainFire、FlareDB等）から**完全に独立**している必要がある +- **キャッシュ前提**: 手元/仮設マシンにはNixストアのキャッシュがあるため、リビルドは多くないはず + +**将来の移行**: ある程度デプロイが進んだら、完全に自動なデプロイ環境（キャッシュ実装済み、ISOはオブジェクトストレージ、スケーラブル）に移行する。ただし、この完全自動デプロイ環境の実装は**他のソフトウェアが安定してから**にしたい。 + +### （将来）リモートflake化 + バイナリキャッシュ（Phase 3以降） + +**目的**: ビルド時間を大幅に短縮（完全自動デプロイ環境用） + +**実装内容**（Phase 3で実装）: +1. **リモートにflakeを置く**（GitHub等） + - **注意**: 現在のコードベースは大胆に変更される可能性があるため、GitHubへの公開は後回し +2. **バイナリキャッシュを用意**（Cachix、セルフホストならattic等） +3. `flake.nix` の `nixConfig` と、`nix/images/netboot-base.nix` / 各ノード設定に **substituters/trusted-public-keys** を入れて、netboot/ISO/インストール時のnixが自動でキャッシュを引くようにする + +**効果**: nixos-anywhere の実体が「ビルド」から「ダウンロード」に変わる。 + +**優先度**: **Phase 3以降**（完全自動デプロイ環境の実装時）。Deployer[Bootstrapper]では**ローカルで動くことを優先**し、キャッシュ系は後回し。 + +### P0: `src = ./.` をやめ、ソースをフィルタする ✅ 実装済み + +**目的**: 無関係な変更で再ビルドが発生しないようにする + +**実装内容** (`flake.nix` の `repoSrc`): +```nix +repoSrc = pkgs.lib.cleanSourceWith { + src = ./.; + filter = path: type: + ! (dropPrefix [ "docs/" "baremetal/" ".git/" ".cccc/" "result" "result-" ] || + base == "target" || + dropSuffix [ ".qcow2" ".img" ".iso" ".qcow" ]); +}; +``` + +**除外されるファイル/ディレクトリ**: +- ✅ `**/target/`（Cargoビルド成果物） +- ✅ `docs/`, `baremetal/`（Rustビルドに不要） +- ✅ `.git/`, `.cccc/`, `result*`（Nix成果物） +- ✅ `.qcow2`, `.img`, `.iso`, `.qcow`（大きなバイナリファイル） + +**効果**: ソース変更がなければNixのキャッシュが効き、再ビルドを回避。 + +### P1: netbootは「最小のインストーラ」に寄せる ✅ 実装済み + +**目的**: netbootイメージのサイズとビルド時間を削減 + +**実装内容** (`nix/images/netboot-base.nix`): +- ✅ `netboot-base.nix`: 最小限のインストーラツールのみ（disko, parted, curl, jq等） +- ✅ サービスバイナリや仮想化ツールは含めない +- 役割：netbootは「SSHで入れてnixos-anywhereできる」だけに絞る +- サービスは **インストール後のNixOS構成**で入れる方が速く・安全 + +**効果**: initrd配布もビルドも速くなる。 + +### P1: トポロジ生成とfirst-bootの接続を完成させる ✅ 実装済み + +**目的**: 構成管理の運用ループを完成させる + +**実装内容**: +- ✅ `plasmacloud-cluster.nix`: クラスタトポロジ定義と `cluster-config.json` の自動生成 +- ✅ `first-boot-automation.nix`: cluster-config.json を読み込んでChainfire/FlareDB/IAMへの自動接続 +- ✅ `environment.etc."nixos/secrets/cluster-config.json"` でファイル配置 + +**効果**: 「構成管理」が「運用の自動化」に直結する。 + +### P2: ISOルートは「本番のゼロタッチ」に必要な要件を埋める（Phase 2以降） + +**目的**: ISOベースの自動デプロイを本番対応にする + +**実装内容**: +- ✅ Deployerの鍵・証明書生成は実装済み（`LocalStorage.get_or_generate_*`） +- TODO: ISO内で disko を同梱してローカル実行に寄せる（現状はネットワーク依存） + +### P1: Deployer[Bootstrapper]の独立性確保 ✅ 実装済み + +**目的**: 他のソフトウェア（ChainFire、FlareDB等）に依存しない独立したデプロイツールにする + +**実装内容** (`deployer/crates/deployer-server/`): +- ✅ `LocalStorage`: ローカルファイルベースのストレージ（ChainFire不要） +- ✅ `config.local_state_path`: デフォルトで `/var/lib/deployer/state` に設定 +- ✅ `state.init_storage()`: `local_state_path` があれば LocalStorage を優先使用 +- ✅ Phone Home API: 簡易HTTPサーバーとして動作（ChainFire不要） +- ✅ SSH host key / TLS証明書: LocalStorage で永続化 + +**効果**: ChainFire等が動いていなくても、Deployer[Bootstrapper]だけでデプロイが可能。 + +**将来**: Phase 3 で ChainFire との統合を実装（大規模デプロイ用）。 + +### （将来）完全自動デプロイ環境の設計 + +**目的**: 大規模デプロイ（10,000台規模）に対応した、完全に自動化されたデプロイ環境 + +**実装内容**（Phase 3で実装）: +- **API層のStateless化**: Phone Homeリクエストを複数APIサーバーで分散処理 +- **ワーカー層の追加**: デプロイジョブを並列実行（ChainFireベースのジョブキュー） +- **ISOのオブジェクトストレージ配布**: LightningStor等にISOを保存し、高速配布 +- **バイナリキャッシュの完全実装**: すべてのビルド成果物をキャッシュ + +**効果**: マシンをいくら増やしても高速でデプロイできる。 + +**前提条件**: 他のソフトウェア（ChainFire、FlareDB、LightningStor等）が安定してから実装する。 + +--- + +## 優先度とロードマップ + +### Phase 1: Deployer[Bootstrapper]の改善 ✅ 完了 + +**目標**: 0→1の初期デプロイを高速化・安定化（**ローカルで動くことを優先**） + +1. ✅ **`src` フィルタリング**（`target/` や `docs/` を除外） + - `flake.nix` の `repoSrc` で実装済み + - ソース変更がなければ、Nixのキャッシュが効き、Cargoの再ビルドも避けられる +2. ✅ **Deployer[Bootstrapper]の独立性確保** + - `LocalStorage` でChainFire非依存 + - `local_state_path` がデフォルトで設定 +3. ✅ **netbootイメージの最小化**（サービスバイナリを除外） + - `netboot-base.nix` を最適化 + - `netboot-worker.nix`, `netboot-control-plane.nix` が netboot-base をベースに使用 +4. ✅ **トポロジ→first-boot接続** + - `plasmacloud-cluster.nix` でクラスタトポロジ定義と cluster-config.json を自動生成 + - `first-boot-automation.nix` でサービス間の自動接続 +5. ✅ **SSH/TLS鍵生成** + - `phone_home.rs` で ED25519 鍵と自己署名証明書を生成・永続化 + +**達成効果**: +- Deployer[Bootstrapper]が他のソフトウェアから独立し、安定して動作 +- ソース変更がなければ、ビルド時間が大幅に短縮 +- Cachix/Attic連携なしでもローカルで動作 + +**実行環境**: 手元/仮設マシン（Nixストアのキャッシュがある前提） + +### Phase 2: 他のソフトウェアの安定化（数ヶ月） + +**目標**: ChainFire、FlareDB、IAM等のコアサービスの安定化 + +1. **コアサービスの機能完成** +2. **クラスタ運用の安定化** +3. **監視・ログ・バックアップ等の運用基盤の整備** + +**期待効果**: 完全自動デプロイ環境を構築する基盤が整う + +### Phase 3: 完全自動デプロイ環境の実装（将来、Phase 2完了後） + +**目標**: 大規模デプロイ（10,000台規模）に対応した、完全に自動化されたデプロイ環境 + +1. **リモートflake化** + **バイナリキャッシュ導入**（Cachix/attic） + - GitHub等への公開（コードベースが安定してから） + - Cachix/Attic連携によるバイナリキャッシュ +2. **API層のStateless化** + **ワーカー層の追加** +3. **ジョブキューの実装**（ChainFireベース） +4. **ISOのオブジェクトストレージ配布**（LightningStor等） +5. **Deployerの鍵・証明書・インベントリ管理の実装** + +**期待効果**: +- マシンをいくら増やしても高速でデプロイできる +- 完全に自動化されたゼロタッチデプロイが可能 + +**前提条件**: +- Phase 2（他のソフトウェアの安定化）が完了していること +- コードベースが安定し、GitHub等への公開が可能になったこと + +--- + +## まとめ + +### 現状の評価 + +このプロジェクトは、**NixOSベースのベアメタル配備に必要な部品がすべて揃い、Phase 1が完了**している： + +#### ✅ Phase 1 完了項目 + +1. **Deployer[Bootstrapper]の独立性**: LocalStorage でChainFire非依存 +2. **キャッシュ効率化**: `repoSrc` フィルタリングで不要ファイルを除外 +3. **netboot最小化**: `netboot-base.nix` でインストーラ専用イメージ +4. **トポロジ→first-boot接続**: cluster-config.json 自動生成 +5. **SSH/TLS鍵生成**: ED25519 鍵と自己署名証明書の生成・永続化 + +#### 残りの課題 + +1. **完全自動デプロイ環境の未実装**: 大規模デプロイに対応するための基盤（Phase 3で実装） + +### 段階的なアプローチ + +**Phase 1 ✅ 完了**: Deployer[Bootstrapper]の改善 +- 0→1の初期デプロイを高速化・安定化 +- 他のソフトウェアから独立 +- 手元/仮設マシンで実行可能 + +**Phase 2（現在）**: 他のソフトウェアの安定化 +- ChainFire、FlareDB、IAM等のコアサービスの安定化 +- クラスタ運用の確立 + +**Phase 3（将来）**: 完全自動デプロイ環境の実装 +- 大規模デプロイ（10,000台規模）に対応 +- 完全に自動化されたゼロタッチデプロイ +- Phase 2完了後に実装 + +### 達成済みの成功条件 + +1. ✅ **Deployer[Bootstrapper]の独立性**: 他のソフトウェアが動いていなくても、デプロイが可能 +2. ✅ **ローカルでの動作優先**: Cachix/Attic連携なしでも、ローカルNixストアのキャッシュで動作 +3. ✅ **キャッシュの効率化**: `src` フィルタリングで、ソース変更がなければNixのキャッシュが効く +4. ✅ **トポロジ→first-boot接続**: plasmacloud-cluster.nix からの設定生成が機能 +5. ✅ **SSH/TLS鍵の永続化**: LocalStorage で鍵を永続化 + +### 次のステップ + +1. ~~Phase 1を最優先で実装~~ ✅ **完了** +2. **Phase 2で他のソフトウェアを安定化**（基盤の確立） +3. **Phase 3で完全自動デプロイ環境を実装**（大規模対応） + - コードベースが安定してから、リモートflake化とバイナリキャッシュを実装 + +**Phase 1が完了し、0→1のデプロイが可能になった。次はPhase 2でコアサービスの安定化を進める。** + +--- + +## 参考資料 + +- [NixOS Netboot](https://nixos.wiki/wiki/Netboot) +- [nixos-anywhere](https://github.com/nix-community/nixos-anywhere) +- [disko](https://github.com/nix-community/disko) +- [Cachix](https://www.cachix.org/) +- [attic](https://github.com/zhaofengli/attic) diff --git a/docs/ops/integration-matrix.md b/docs/ops/integration-matrix.md new file mode 100644 index 0000000..272a01a --- /dev/null +++ b/docs/ops/integration-matrix.md @@ -0,0 +1,43 @@ +# Integration Matrix Gate + +Release gate that exercises the PROJECT.md matrix (chainfire → flaredb → plasmavmc → creditservice → nightlight). + +## Release hook +- Run this matrix **before any release cut** (tag/publish). Command: `nix develop -c ./scripts/integration-matrix.sh`. +- After a green run, copy logs from `.cccc/work/integration-matrix//` to `docs/evidence/integration-matrix-/` and reference the path in release notes. +- If KVM is unavailable, use `SKIP_PLASMA=1` only as a temporary measure; restore full run once nested KVM is enabled. +- Defaults: script now auto-creates a tiny qcow2 in `LOG_DIR` and picks `qemu-system-x86_64` from PATH; set `PLASMA_E2E=1` to run PlasmaVMC ignored e2e once qcow/QEMU is available. + +## Prerequisites +- Cluster services reachable (ChainFire, FlareDB, PlasmaVMC, CreditService, NightLight). +- Nested KVM available for PlasmaVMC tests; run `sudo scripts/nested-kvm-check.sh` on hosts. +- `cargo` toolchain present on the runner. +- For PlasmaVMC e2e (once qcow is provided): set `PLASMAVMC_QEMU_PATH` and `PLASMAVMC_QCOW2_PATH` to enable QEMU-backed tests; the script will set best-effort defaults if unset. + +## How to run +``` +# Dry run (prints commands, no tests) +DRY_RUN=1 scripts/integration-matrix.sh + +# Full run (all legs) +scripts/integration-matrix.sh + +# Skip PlasmaVMC leg if KVM unavailable +SKIP_PLASMA=1 scripts/integration-matrix.sh + +# PlasmaVMC ignored e2e (requires QEMU + qcow; defaults auto-provisioned if available) +PLASMA_E2E=1 scripts/integration-matrix.sh +``` + +Logs are written to `.cccc/work/integration-matrix//` by default; override with `LOG_DIR=...` if needed. + +## What it covers +1) chainfire → flaredb: Raft+Gossip cluster write/read with failover path (cargo tests). +2) flaredb → plasmavmc: VM metadata durability across leader switch (cargo tests). +3) plasmavmc → creditservice: Admission Control CAS/rollback under contention (cargo tests). +4) creditservice → nightlight: Metrics feeding billing/alerts (cargo tests). +5) end-to-end (future harness): tenant loop with FiberLB/FlashDNS once approved; runs will emit junit/json artifacts to `.cccc/work/results/`. + +## Notes +- Use `DRY_RUN=1` on CI to verify wiring without requiring KVM. +- If nested KVM is disabled, enable via NixOS (`boot.extraModprobeConfig = "options kvm-intel nested=1";` or kvm-amd) and reboot once. Refer to `scripts/nested-kvm-check.sh` for the exact snippet. diff --git a/docs/ops/nested-kvm-setup.md b/docs/ops/nested-kvm-setup.md new file mode 100644 index 0000000..15edf39 --- /dev/null +++ b/docs/ops/nested-kvm-setup.md @@ -0,0 +1,38 @@ +# PlasmaVMC Nested KVM & App Validation (Draft) + +## Nested KVM quick check +1) On host: `cat /sys/module/kvm_intel/parameters/nested` (or `kvm_amd`). Expect `Y` for enabled, `N` for disabled. +2) If disabled (Intel example): +``` +boot.kernelModules = [ "kvm-intel" ]; +boot.extraModprobeConfig = '' + options kvm-intel nested=1 +''; +``` + For AMD, use `kvm-amd` and `options kvm-amd nested=1`. +3) Reboot once, verify again. +4) Inside a guest VM: prove nesting with a minimal KVM launch: +``` +qemu-system-x86_64 -accel kvm -cpu host -m 512 -nographic \ + -kernel /run/current-system/kernel -append "console=ttyS0" < /dev/null +``` + If it boots to kernel console, nesting works. + +## App scenario (lightweight) +- Topology: 2x app VMs on PrismNET, FiberLB front, FlashDNS record -> LB VIP. +- Data: FlareDB SQL (guestbook-style) for metadata; ChainFire backs control-plane metadata. +- Controls: CreditService Admission Control enforced on VM create (low quota); NightLight metrics exported. + +### Steps +1) Provision: create 2 VMs via PlasmaVMC API; attach PrismNET network; ensure watcher persists VM metadata to FlareDB. +2) Configure: deploy small web app on each VM that writes/reads FlareDB SQL; register DNS record in FlashDNS pointing to FiberLB listener. +3) Gate: set low wallet balance; attempt VM create/update to confirm CAS-based debit and rollback on failure. +4) Observe: ensure NightLight scrapes app + system metrics; add alerts for latency > target and billing failures. +5) Failover drills: + - Kill one app VM: FiberLB should reroute; CreditService must not double-charge retries. + - Restart PlasmaVMC node: watcher should replay state from FlareDB/ChainFire; VM lifecycle ops continue. +6) Exit criteria: all above steps pass 5x in a row; NightLight shows zero SLO violations; CreditService balances consistent before/after drills. + +## Notes +- Full disk HA not covered; for disk replication we’d need distributed block (future). +- Keep tests env-gated (ignored by default) so CI doesn’t require nested virt. diff --git a/docs/ops/qcow2-artifact-plan.md b/docs/ops/qcow2-artifact-plan.md new file mode 100644 index 0000000..872b805 --- /dev/null +++ b/docs/ops/qcow2-artifact-plan.md @@ -0,0 +1,26 @@ +## PlasmaVMC qcow artifact plan (for integration gate e2e) + +- Goal: provide a reproducible qcow2 image + env wiring so plasmavmc e2e (QEMU-backed) can run in the integration matrix without manual prep. +- Constraints: small (<150MB), no network during gate run, works under nix develop; use virtio drivers; avoid licensing issues. + +### Candidate image +- Alpine cloud image (latest stable) is small and permissively licensed; includes virtio modules. +- Fallback: Build a 1G qcow2 via `qemu-img create -f qcow2 plasma-mini.qcow2 1G` + `virt-make-fs` on a tiny rootfs (busybox/alpine base). + +### Provisioning steps (once, cacheable) +1) In nix shell (has qemu-img): `qemu-img convert -f qcow2 -O qcow2 alpine-cloudimg-amd64.qcow2 plasma-mini.qcow2` or `qemu-img create -f qcow2 plasma-mini.qcow2 1G`. +2) Inject default user+ssh key (optional) via cloud-init seed ISO or `virt-make-fs` (avoid during gate). +3) Store artifact under `.cccc/work/artifacts/plasma-mini.qcow2` (or cache bucket if available). +4) Record SHA256 to detect drift. + +### Gate wiring +- Env vars: `PLASMAVMC_QEMU_PATH` (e.g., `/run/current-system/sw/bin/qemu-system-x86_64` in nix shell), `PLASMAVMC_QCOW2_PATH` (absolute path to plasma-mini.qcow2). +- Update `scripts/integration-matrix.sh` docs to mention envs; optionally add `just integration-matrix [--skip-plasma]` wrapper that injects defaults when present. + +### Time/budget +- Download + convert: ~2-3 minutes once; gate runs reuse artifact (no network). +- If artifact absent, plasmavmc e2e remain ignored; matrix still green on unit/integration subsets. + +### Open questions +- Where to store the qcow2 artifact for CI (git LFS? remote cache?) to avoid repo bloat. +- Is cloud-init desirable for tests (SSH into VM) or is raw boot enough for current e2e? diff --git a/docs/plans/chainfire_architecture_redefinition.md b/docs/plans/chainfire_architecture_redefinition.md new file mode 100644 index 0000000..f02e82c --- /dev/null +++ b/docs/plans/chainfire_architecture_redefinition.md @@ -0,0 +1,89 @@ +# Chainfire アーキテクチャ再定義案: 分散システム構築基盤への転換 + +`Chainfire` を単一の KV ストアサービスから、プロジェクト全体の「分散システム構築フレームワーク」へと位置づけ直すための設計案です。 + +## 1. アーキテクチャ概要 + +階層構造を整理し、低レイヤーのプリミティブから高レイヤーのマネージドサービスまでを明確に分離します。 + +```mermaid +graph TD + subgraph Application_Layer + FlareDB[FlareDB / Distributed DB] + LightningStor[lightningstor / Object Storage] + IAM[IAM / Control Plane] + end + + subgraph L2_Service_Layer_Sidecar + CFServer[Chainfire Server] + CFServer -- gRPC Streaming --> IAM + end + + subgraph L1_Framework_Layer + CFCore[chainfire-core] + CFCore -- Library Embed --> FlareDB + CFCore -- Library Embed --> LightningStor + + MultiRaft[Multi-Raft Orchestrator] + CFCore --> MultiRaft + end + + subgraph L0_Primitive_Layer + Gossip[chainfire-gossip] + Raft[chainfire-raft] + Storage[chainfire-storage] + + CFCore --> Gossip + CFCore --> Raft + Raft --> Storage + end + + CFServer --> CFCore +``` + +## 2. 各レイヤーの責務定義 + +### L0 Core (Library): primitives +- **chainfire-gossip**: + - SWIM プロトコルに基づくメンバーシップ管理。 + - 特定のサービスに依存せず、任意の `NodeMetadata` を伝搬可能にする。 +- **chainfire-raft**: + - 単一 Raft グループのコンセンサスロジック。 + - `StateMachine` を Trait 化し、任意のビジネスロジックを注入可能にする。 + - `RaftNetwork` を抽象化し、gRPC 以外（UDS, In-memory）のトランスポートをサポート。 +- **chainfire-storage**: + - Raft ログおよび StateMachine のための永続化レイヤー。 + +### L1 Framework: chainfire-core +- **Multi-Raft Orchestrator**: + - 複数の Raft インスタンス（シャード）を同一プロセス内で効率的に管理。 + - ネットワーク接続やスレッドプール等のリソース共有を最適化。 +- **Cluster Manager**: + - Gossip のメンバーシップイベントを監視し、Raft グループへのノード追加・削除を自動化。 + - 「ノード発見（Gossip）」から「合意形成参加（Raft）」への橋渡しを行う。 + +### L2 Service: chainfire-server (Standard Implementation) +- **Shared Infrastructure**: + - KV ストア、分散ロック、リース管理を gRPC API として提供。 + - 独自に Raft を組む必要のない「軽量サービス」向けの共通基盤。 +- **Sidecar Mode Support**: + - gRPC Streaming による `ClusterEvents` の提供。 + - リーダー交代やメンバーシップ変更を外部プロセスにリアルタイム通知。 + +## 3. 分散サービスでの再利用シナリオ (例: FlareDB) + +FlareDB が Chainfire 基盤をどのように利用して Multi-Raft を構成するかの具体例です。 + +1. **ライブラリとして組み込み**: `FlareDB` プロセスが `chainfire-core` をリンク。 +2. **独自の StateMachine 実装**: FlareDB のデータ操作ロジックを `StateMachine` Trait として実装。 +3. **シャード管理**: + - データのレンジごとに `RaftGroup` インスタンスを作成。 + - 各 `RaftGroup` に FlareDB 独自の `StateMachine` を登録。 +4. **ノード管理の委譲**: + - Gossip によるノード発見を `chainfire-core` に任せ、FlareDB 側では個別のノードリスト管理を行わない。 + +## 4. メリットの整理 + +- **開発効率の向上**: Gossip や Raft といった複雑な分散プロトコルの再実装が不要になる。 +- **観測性の一貫性**: プロジェクト全体の全ノードが共通の Gossip 基盤に乗ることで、システム全体のトポロジー可視化が容易になる。 +- **柔軟な配置**: 同一のロジックを、ライブラリとして（高パフォーマンス）、あるいはサイドカーとして（疎結合）のどちらでも利用可能。 \ No newline at end of file diff --git a/docs/plans/metadata_unification.md b/docs/plans/metadata_unification.md new file mode 100644 index 0000000..7f619fc --- /dev/null +++ b/docs/plans/metadata_unification.md @@ -0,0 +1,45 @@ +# メタデータ管理の Chainfire 一本化に関する調査報告と構成案 + +## 1. 調査結果サマリー +プロジェクト内の各コンポーネントにおけるメタデータ（設定、リソース定義、状態）の管理状況を調査した結果、現状は `Chainfire` (etcd-like) と `FlareDB` (TiKV-like) が混在しており、メンテナンスコストとシステム複雑性を増大させていることが判明しました。 + +### コンポーネント別の現状 +- **移行が必要**: `k8shost` (現在 FlareDB に強く密結合) +- **設定・実装の統一が必要**: `lightningstor`, `flashdns`, `prismnet`, `fiberlb` (既に Chainfire 対応コードを持つが、独自に抽象化を実装) +- **対応済み**: `iam`, `creditservice` (既に Chainfire を主に使用) + +## 2. 技術的判断 +メタデータ実装を **Chainfire に一本化することは妥当かつ推奨される** と判断します。 + +### 妥当性の理由 +- **運用性の向上**: 運用・監視・バックアップの対象を Raft ベースの `Chainfire` 1つに集約できる。 +- **一貫した連携基盤**: `Chainfire` の `Watch` 機能を共通のイベント基盤として、コンポーネント間（例：Podの変更をネットワーク層が検知）のリアクティブな連携が容易になる。 +- **コードの健全化**: 依存ライブラリを整理し、各コンポーネントで重複しているストレージ抽象化ロジックを排除できる。 + +### リスクへの対策 +`Chainfire` は全ノード複製型のため、大規模環境での書き込み性能がボトルネックになる懸念があります。これに対し、本案では**共通抽象化インターフェース (Trait)** を導入することで、将来的に特定リソースのみ高性能バックエンドへ再分離できる柔軟性を確保します。 + +## 3. 構成案 + +### A. 共通モジュール `chainfire-client::metadata` の新設 +各サービスからストレージ固有の実装を分離し、共通の `MetadataClient` Trait を提供します。 + +```rust +#[async_trait] +pub trait MetadataClient: Send + Sync { + async fn get(&self, key: &str) -> Result>>; + async fn put(&self, key: &str, value: Vec) -> Result<()>; + async fn delete(&self, key: &str) -> Result; + async fn list_prefix(&self, prefix: &str) -> Result)>>; + async fn watch(&self, prefix: &str) -> BoxStream; + async fn compare_and_swap(&self, key: &str, expected_rev: u64, value: Vec) -> Result; +} +``` + +### B. 移行ロードマップ +1. **共通基盤の構築**: `chainfire-client::metadata` を実装。`Chainfire` ブリッジとテスト用の `InMemory` バックエンドを提供。 +2. **k8shost のリファクタリング**: `storage.rs` を `MetadataClient` 経由に書き換え、`flaredb-client` 依存を削除。 +3. **他コンポーネントの追随**: `lightningstor` 等の独自ストレージ選択ロジックを `chainfire-client::metadata` に置換。 + +## 4. 結論 +本提案により、現状の `FlareDB` マルチテナント実装の複雑さから解放され、開発効率とシステムの一貫性が劇的に向上します。将来的なスケーラビリティ要求に対しても、抽象化レイヤーの導入により十分対応可能です。 \ No newline at end of file diff --git a/examples/mtls-agent-config.toml b/examples/mtls-agent-config.toml new file mode 100644 index 0000000..5d9d3d3 --- /dev/null +++ b/examples/mtls-agent-config.toml @@ -0,0 +1,17 @@ +[service] +name = "api-server" +app_addr = "127.0.0.1:8080" +mesh_bind_addr = "0.0.0.0:18080" + +[cluster] +cluster_id = "test-cluster-01" +environment = "dev" +chainfire_endpoint = "http://127.0.0.1:2379" + +[mtls] +mode = "auto" # auto/mtls/tls/plain +# ca_cert_path = "/etc/photoncloud/ca.crt" +# cert_path = "/etc/photoncloud/server.crt" +# key_path = "/etc/photoncloud/server.key" + + diff --git a/examples/photoncloud-test-cluster.json b/examples/photoncloud-test-cluster.json new file mode 100644 index 0000000..7609649 --- /dev/null +++ b/examples/photoncloud-test-cluster.json @@ -0,0 +1,79 @@ +{ + "cluster": { + "cluster_id": "test-cluster-01", + "environment": "dev" + }, + "nodes": [ + { + "node_id": "node-01", + "hostname": "photon-node-01", + "ip": "192.168.100.10", + "roles": ["worker"], + "labels": { + "zone": "zone-a" + } + }, + { + "node_id": "node-02", + "hostname": "photon-node-02", + "ip": "192.168.100.11", + "roles": ["worker"], + "labels": { + "zone": "zone-b" + } + } + ], + "services": [ + { + "name": "api-server", + "ports": { + "http": 8080, + "grpc": 9090 + }, + "protocol": "http", + "mtls_required": false, + "mesh_mode": "agent" + }, + { + "name": "worker-service", + "ports": { + "http": 8081 + }, + "protocol": "http", + "mtls_required": false, + "mesh_mode": "agent" + } + ], + "instances": [ + { + "instance_id": "api-server-01", + "service": "api-server", + "node_id": "node-01", + "ip": "192.168.100.10", + "port": 8080, + "mesh_port": 18080, + "version": "v1.0.0" + }, + { + "instance_id": "worker-01", + "service": "worker-service", + "node_id": "node-02", + "ip": "192.168.100.11", + "port": 8081, + "mesh_port": 18081, + "version": "v1.0.0" + } + ], + "mtls_policies": [ + { + "policy_id": "default-dev", + "environment": "dev", + "source_service": "*", + "target_service": "*", + "mtls_required": false, + "mode": "plain" + } + ] +} + + diff --git a/fiberlb/crates/fiberlb-server/build.rs b/fiberlb/crates/fiberlb-server/build.rs new file mode 100644 index 0000000..9df21fd --- /dev/null +++ b/fiberlb/crates/fiberlb-server/build.rs @@ -0,0 +1,10 @@ +fn main() -> Result<(), Box> { + let protoc_path = protoc_bin_vendored::protoc_bin_path()?; + std::env::set_var("PROTOC", protoc_path); + + tonic_build::configure() + .build_server(false) + .build_client(true) + .compile(&["proto/api/gobgp.proto"], &["proto"])?; + Ok(()) +} diff --git a/fiberlb/crates/fiberlb-server/proto/api/attribute.proto b/fiberlb/crates/fiberlb-server/proto/api/attribute.proto new file mode 100644 index 0000000..529e3cd --- /dev/null +++ b/fiberlb/crates/fiberlb-server/proto/api/attribute.proto @@ -0,0 +1,584 @@ +// Copyright (C) 2018 Nippon Telegraph and Telephone Corporation. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +syntax = "proto3"; + +package api; + +import "api/common.proto"; +import "api/extcom.proto"; +import "api/nlri.proto"; + +option go_package = "github.com/osrg/gobgp/v4/api;api"; + +message Attribute { + oneof attr { + UnknownAttribute unknown = 1; + OriginAttribute origin = 2; + AsPathAttribute as_path = 3; + NextHopAttribute next_hop = 4; + MultiExitDiscAttribute multi_exit_disc = 5; + LocalPrefAttribute local_pref = 6; + AtomicAggregateAttribute atomic_aggregate = 7; + AggregatorAttribute aggregator = 8; + CommunitiesAttribute communities = 9; + OriginatorIdAttribute originator_id = 10; + ClusterListAttribute cluster_list = 11; + MpReachNLRIAttribute mp_reach = 12; + MpUnreachNLRIAttribute mp_unreach = 13; + ExtendedCommunitiesAttribute extended_communities = 14; + As4PathAttribute as4_path = 15; + As4AggregatorAttribute as4_aggregator = 16; + PmsiTunnelAttribute pmsi_tunnel = 17; + TunnelEncapAttribute tunnel_encap = 18; + IP6ExtendedCommunitiesAttribute ip6_extended_communities = 19; + AigpAttribute aigp = 20; + LargeCommunitiesAttribute large_communities = 21; + LsAttribute ls = 22; + PrefixSID prefix_sid = 23; + } +} + +message OriginAttribute { + uint32 origin = 1; +} + +message AsSegment { + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_AS_SET = 1; + TYPE_AS_SEQUENCE = 2; + TYPE_AS_CONFED_SEQUENCE = 3; + TYPE_AS_CONFED_SET = 4; + } + Type type = 1; + repeated uint32 numbers = 2; +} + +message AsPathAttribute { + repeated AsSegment segments = 1; +} + +message NextHopAttribute { + string next_hop = 1; +} + +message MultiExitDiscAttribute { + uint32 med = 1; +} + +message LocalPrefAttribute { + uint32 local_pref = 1; +} + +message AtomicAggregateAttribute {} + +message AggregatorAttribute { + uint32 asn = 1; + string address = 2; +} + +message CommunitiesAttribute { + repeated uint32 communities = 1; +} + +message OriginatorIdAttribute { + string id = 1; +} + +message ClusterListAttribute { + repeated string ids = 1; +} + +message MpReachNLRIAttribute { + Family family = 1; + repeated string next_hops = 2; + repeated NLRI nlris = 3; +} + +message MpUnreachNLRIAttribute { + api.Family family = 1; + // The same as NLRI field of MpReachNLRIAttribute + repeated NLRI nlris = 3; +} + +message ExtendedCommunitiesAttribute { + repeated ExtendedCommunity communities = 1; +} + +message As4PathAttribute { + repeated AsSegment segments = 1; +} + +message As4AggregatorAttribute { + uint32 asn = 2; + string address = 3; +} + +message PmsiTunnelAttribute { + uint32 flags = 1; + uint32 type = 2; + uint32 label = 3; + bytes id = 4; +} + +message TunnelEncapSubTLVEncapsulation { + uint32 key = 1; + bytes cookie = 2; +} + +message TunnelEncapSubTLVProtocol { + uint32 protocol = 1; +} + +message TunnelEncapSubTLVColor { + uint32 color = 1; +} + +message TunnelEncapSubTLVSRPreference { + uint32 flags = 1; + uint32 preference = 2; +} + +message TunnelEncapSubTLVSRCandidatePathName { + string candidate_path_name = 1; +} + +message TunnelEncapSubTLVSRPriority { + uint32 priority = 1; +} + +message TunnelEncapSubTLVSRBindingSID { + oneof bsid { + SRBindingSID sr_binding_sid = 1; + SRv6BindingSID srv6_binding_sid = 2; + } +} + +message SRBindingSID { + bool s_flag = 1; + bool i_flag = 2; + bytes sid = 3; +} + +enum SRV6Behavior { + SRV6_BEHAVIOR_UNSPECIFIED = 0; + SRV6_BEHAVIOR_END = 1; + SRV6_BEHAVIOR_END_WITH_PSP = 2; + SRV6_BEHAVIOR_END_WITH_USP = 3; + SRV6_BEHAVIOR_END_WITH_PSP_USP = 4; + SRV6_BEHAVIOR_ENDX = 5; + SRV6_BEHAVIOR_ENDX_WITH_PSP = 6; + SRV6_BEHAVIOR_ENDX_WITH_USP = 7; + SRV6_BEHAVIOR_ENDX_WITH_PSP_USP = 8; + SRV6_BEHAVIOR_ENDT = 9; + SRV6_BEHAVIOR_ENDT_WITH_PSP = 10; + SRV6_BEHAVIOR_ENDT_WITH_USP = 11; + SRV6_BEHAVIOR_ENDT_WITH_PSP_USP = 12; + SRV6_BEHAVIOR_END_B6_ENCAPS = 14; + SRV6_BEHAVIOR_END_BM = 15; + SRV6_BEHAVIOR_END_DX6 = 16; + SRV6_BEHAVIOR_END_DX4 = 17; + SRV6_BEHAVIOR_END_DT6 = 18; + SRV6_BEHAVIOR_END_DT4 = 19; + SRV6_BEHAVIOR_END_DT46 = 20; + SRV6_BEHAVIOR_END_DX2 = 21; + SRV6_BEHAVIOR_END_DX2V = 22; + SRV6_BEHAVIOR_END_DT2U = 23; + SRV6_BEHAVIOR_END_DT2M = 24; + SRV6_BEHAVIOR_END_B6_ENCAPS_RED = 27; + SRV6_BEHAVIOR_END_WITH_USD = 28; + SRV6_BEHAVIOR_END_WITH_PSP_USD = 29; + SRV6_BEHAVIOR_END_WITH_USP_USD = 30; + SRV6_BEHAVIOR_END_WITH_PSP_USP_USD = 31; + SRV6_BEHAVIOR_ENDX_WITH_USD = 32; + SRV6_BEHAVIOR_ENDX_WITH_PSP_USD = 33; + SRV6_BEHAVIOR_ENDX_WITH_USP_USD = 34; + SRV6_BEHAVIOR_ENDX_WITH_PSP_USP_USD = 35; + SRV6_BEHAVIOR_ENDT_WITH_USD = 36; + SRV6_BEHAVIOR_ENDT_WITH_PSP_USD = 37; + SRV6_BEHAVIOR_ENDT_WITH_USP_USD = 38; + SRV6_BEHAVIOR_ENDT_WITH_PSP_USP_USD = 39; + SRV6_BEHAVIOR_ENDM_GTP6D = 69; // 0x0045 + SRV6_BEHAVIOR_ENDM_GTP6DI = 70; // 0x0046 + SRV6_BEHAVIOR_ENDM_GTP6E = 71; // 0x0047 + SRV6_BEHAVIOR_ENDM_GTP4E = 72; // 0x0048 +} + +message SRv6EndPointBehavior { + SRV6Behavior behavior = 1; + uint32 block_len = 2; + uint32 node_len = 3; + uint32 func_len = 4; + uint32 arg_len = 5; +} + +message SRv6BindingSID { + bool s_flag = 1; + bool i_flag = 2; + bool b_flag = 3; + bytes sid = 4; + SRv6EndPointBehavior endpoint_behavior_structure = 5; +} + +enum ENLPType { + ENLP_TYPE_UNSPECIFIED = 0; + ENLP_TYPE_TYPE1 = 1; + ENLP_TYPE_TYPE2 = 2; + ENLP_TYPE_TYPE3 = 3; + ENLP_TYPE_TYPE4 = 4; +} + +message TunnelEncapSubTLVSRENLP { + uint32 flags = 1; + ENLPType enlp = 2; +} + +message SRWeight { + uint32 flags = 1; + uint32 weight = 2; +} + +message SegmentFlags { + bool v_flag = 1; + bool a_flag = 2; + bool s_flag = 3; + bool b_flag = 4; +} + +message SegmentTypeA { + SegmentFlags flags = 1; + uint32 label = 2; +} + +message SegmentTypeB { + SegmentFlags flags = 1; + bytes sid = 2; + SRv6EndPointBehavior endpoint_behavior_structure = 3; +} + +message TunnelEncapSubTLVSRSegmentList { + SRWeight weight = 1; + + message Segment { + oneof segment { + SegmentTypeA a = 1; + SegmentTypeB b = 2; + } + } + repeated Segment segments = 2; +} + +message TunnelEncapSubTLVEgressEndpoint { + string address = 1; +} + +message TunnelEncapSubTLVUDPDestPort { + uint32 port = 1; +} + +message TunnelEncapSubTLVUnknown { + uint32 type = 1; + bytes value = 2; +} + +message TunnelEncapTLV { + uint32 type = 1; + message TLV { + oneof tlv { + TunnelEncapSubTLVUnknown unknown = 1; + TunnelEncapSubTLVEncapsulation encapsulation = 2; + TunnelEncapSubTLVProtocol protocol = 3; + TunnelEncapSubTLVColor color = 4; + TunnelEncapSubTLVEgressEndpoint egress_endpoint = 5; + TunnelEncapSubTLVUDPDestPort udp_dest_port = 6; + TunnelEncapSubTLVSRPreference sr_preference = 7; + TunnelEncapSubTLVSRPriority sr_priority = 8; + TunnelEncapSubTLVSRCandidatePathName sr_candidate_path_name = 9; + TunnelEncapSubTLVSRENLP sr_enlp = 10; + TunnelEncapSubTLVSRBindingSID sr_binding_sid = 11; + TunnelEncapSubTLVSRSegmentList sr_segment_list = 12; + } + } + repeated TLV tlvs = 2; +} + +message TunnelEncapAttribute { + repeated TunnelEncapTLV tlvs = 1; +} + +message IPv6AddressSpecificExtended { + bool is_transitive = 1; + uint32 sub_type = 2; + string address = 3; + uint32 local_admin = 4; +} + +message RedirectIPv6AddressSpecificExtended { + string address = 1; + uint32 local_admin = 2; +} + +message IP6ExtendedCommunitiesAttribute { + message Community { + oneof extcom { + IPv6AddressSpecificExtended ipv6_address_specific = 1; + RedirectIPv6AddressSpecificExtended redirect_ipv6_address_specific = 2; + } + } + repeated Community communities = 1; +} + +message AigpTLVIGPMetric { + uint64 metric = 1; +} + +message AigpTLVUnknown { + uint32 type = 1; + bytes value = 2; +} + +message AigpAttribute { + message TLV { + oneof tlv { + AigpTLVUnknown unknown = 1; + AigpTLVIGPMetric igp_metric = 2; + } + } + repeated TLV tlvs = 1; +} + +message LargeCommunity { + uint32 global_admin = 1; + uint32 local_data1 = 2; + uint32 local_data2 = 3; +} + +message LargeCommunitiesAttribute { + repeated LargeCommunity communities = 1; +} + +message LsNodeFlags { + bool overload = 1; + bool attached = 2; + bool external = 3; + bool abr = 4; + bool router = 5; + bool v6 = 6; +} + +message LsIGPFlags { + bool down = 1; + bool no_unicast = 2; + bool local_address = 3; + bool propagate_nssa = 4; +} + +message LsSrRange { + uint32 begin = 1; + uint32 end = 2; +} + +message LsSrCapabilities { + bool ipv4_supported = 1; + bool ipv6_supported = 2; + repeated LsSrRange ranges = 3; +} + +message LsSrLocalBlock { + repeated LsSrRange ranges = 1; +} + +message LsAttributeNode { + string name = 1; + LsNodeFlags flags = 2; + string local_router_id = 3; + string local_router_id_v6 = 4; + bytes isis_area = 5; + bytes opaque = 6; + + LsSrCapabilities sr_capabilities = 7; + bytes sr_algorithms = 8; + LsSrLocalBlock sr_local_block = 9; +} + +message LsAttributeLink { + string name = 1; + string local_router_id = 2; + string local_router_id_v6 = 3; + string remote_router_id = 4; + string remote_router_id_v6 = 5; + uint32 admin_group = 6; + uint32 default_te_metric = 7; + uint32 igp_metric = 8; + bytes opaque = 9; + + float bandwidth = 10; + float reservable_bandwidth = 11; + repeated float unreserved_bandwidth = 12; + + uint32 sr_adjacency_sid = 13; + repeated uint32 srlgs = 14; + LsSrv6EndXSID srv6_end_x_sid = 15; +} + +message LsAttributePrefix { + LsIGPFlags igp_flags = 1; + bytes opaque = 2; + + uint32 sr_prefix_sid = 3; +} + +message LsBgpPeerSegmentSIDFlags { + bool value = 1; + bool local = 2; + bool backup = 3; + bool persistent = 4; +} + +message LsBgpPeerSegmentSID { + LsBgpPeerSegmentSIDFlags flags = 1; + uint32 weight = 2; + uint32 sid = 3; +} + +message LsAttributeBgpPeerSegment { + LsBgpPeerSegmentSID bgp_peer_node_sid = 1; + LsBgpPeerSegmentSID bgp_peer_adjacency_sid = 2; + LsBgpPeerSegmentSID bgp_peer_set_sid = 3; +} + +message LsSrv6EndXSID { + uint32 endpoint_behavior = 1; + uint32 flags = 2; + uint32 algorithm = 3; + uint32 weight = 4; + uint32 reserved = 5; + repeated string sids = 6; + LsSrv6SIDStructure srv6_sid_structure = 7; +} + +message LsSrv6SIDStructure { + uint32 local_block = 1; + uint32 local_node = 2; + uint32 local_func = 3; + uint32 local_arg = 4; +} + +message LsSrv6EndpointBehavior { + uint32 endpoint_behavior = 1; + uint32 flags = 2; + uint32 algorithm = 3; +} + +message LsSrv6BgpPeerNodeSID { + uint32 flags = 1; + uint32 weight = 2; + uint32 peer_as = 3; + string peer_bgp_id = 4; +} + +message LsAttributeSrv6SID { + LsSrv6SIDStructure srv6_sid_structure = 1; + LsSrv6EndpointBehavior srv6_endpoint_behavior = 2; + LsSrv6BgpPeerNodeSID srv6_bgp_peer_node_sid = 3; +} + +message LsAttribute { + LsAttributeNode node = 1; + LsAttributeLink link = 2; + LsAttributePrefix prefix = 3; + LsAttributeBgpPeerSegment bgp_peer_segment = 4; + LsAttributeSrv6SID srv6_sid = 5; +} + +message UnknownAttribute { + uint32 flags = 1; + uint32 type = 2; + bytes value = 3; +} + +// https://www.rfc-editor.org/rfc/rfc9252.html#section-3.2.1 +message SRv6StructureSubSubTLV { + uint32 locator_block_length = 1; + uint32 locator_node_length = 2; + uint32 function_length = 3; + uint32 argument_length = 4; + uint32 transposition_length = 5; + uint32 transposition_offset = 6; +} + +message SRv6SubSubTLV { + oneof tlv { + SRv6StructureSubSubTLV structure = 1; + } +} + +message SRv6SubSubTLVs { + repeated SRv6SubSubTLV tlvs = 1; +} + +message SRv6SIDFlags { + // Placeholder for future sid flags + bool flag_1 = 1; +} + +// https://tools.ietf.org/html/draft-dawra-bess-srv6-services-02#section-2.1.1 +message SRv6InformationSubTLV { + bytes sid = 1; + SRv6SIDFlags flags = 2; + uint32 endpoint_behavior = 3; + map sub_sub_tlvs = 4; +} + +message SRv6SubTLV { + oneof tlv { + SRv6InformationSubTLV information = 1; + } +} + +message SRv6SubTLVs { + repeated SRv6SubTLV tlvs = 1; +} + +// https://www.rfc-editor.org/rfc/rfc9252.html#section-2 +message SRv6L3ServiceTLV { + map sub_tlvs = 1; +} + +// https://www.rfc-editor.org/rfc/rfc9252.html#section-2 +message SRv6L2ServiceTLV { + map sub_tlvs = 1; +} + +// https://tools.ietf.org/html/rfc8669 +message PrefixSID { + // tlv is one of: + message TLV { + oneof tlv { + // IndexLabelTLV Type 1 (not yet implemented) + // OriginatorSRGBTLV Type 3 (not yet implemented) + SRv6L3ServiceTLV l3_service = 3; + SRv6L2ServiceTLV l2_service = 4; + } + } + repeated TLV tlvs = 1; +} diff --git a/fiberlb/crates/fiberlb-server/proto/api/capability.proto b/fiberlb/crates/fiberlb-server/proto/api/capability.proto new file mode 100644 index 0000000..5b3dd93 --- /dev/null +++ b/fiberlb/crates/fiberlb-server/proto/api/capability.proto @@ -0,0 +1,124 @@ +// Copyright (C) 2018 Nippon Telegraph and Telephone Corporation. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +syntax = "proto3"; + +package api; + +import "api/common.proto"; + +option go_package = "github.com/osrg/gobgp/v4/api;api"; + +message Capability { + oneof cap { + UnknownCapability unknown = 1; + MultiProtocolCapability multi_protocol = 2; + RouteRefreshCapability route_refresh = 3; + CarryingLabelInfoCapability carrying_label_info = 4; + ExtendedNexthopCapability extended_nexthop = 5; + GracefulRestartCapability graceful_restart = 6; + FourOctetASNCapability four_octet_asn = 7; + AddPathCapability add_path = 8; + EnhancedRouteRefreshCapability enhanced_route_refresh = 9; + LongLivedGracefulRestartCapability long_lived_graceful_restart = 10; + RouteRefreshCiscoCapability route_refresh_cisco = 11; + FqdnCapability fqdn = 12; + SoftwareVersionCapability software_version = 13; + } +} + +message MultiProtocolCapability { + api.Family family = 1; +} + +message RouteRefreshCapability {} + +message CarryingLabelInfoCapability {} + +message ExtendedNexthopCapabilityTuple { + api.Family nlri_family = 1; + // Nexthop AFI must be either + // gobgp.IPv4 or + // gobgp.IPv6. + api.Family nexthop_family = 2; +} + +message ExtendedNexthopCapability { + repeated ExtendedNexthopCapabilityTuple tuples = 1; +} + +message GracefulRestartCapabilityTuple { + api.Family family = 1; + uint32 flags = 2; +} + +message GracefulRestartCapability { + uint32 flags = 1; + uint32 time = 2; + repeated GracefulRestartCapabilityTuple tuples = 3; +} + +message FourOctetASNCapability { + uint32 asn = 1; +} + +message AddPathCapabilityTuple { + api.Family family = 1; + enum Mode { + MODE_UNSPECIFIED = 0; // NONE + MODE_RECEIVE = 1; + MODE_SEND = 2; + MODE_BOTH = 3; + } + Mode mode = 2; +} + +message AddPathCapability { + repeated AddPathCapabilityTuple tuples = 1; +} + +message EnhancedRouteRefreshCapability {} + +message LongLivedGracefulRestartCapabilityTuple { + api.Family family = 1; + uint32 flags = 2; + uint32 time = 3; +} + +message LongLivedGracefulRestartCapability { + repeated LongLivedGracefulRestartCapabilityTuple tuples = 1; +} + +message RouteRefreshCiscoCapability {} + +message FqdnCapability { + string host_name = 1; + string domain_name = 2; +} + +message SoftwareVersionCapability { + string software_version = 1; +} + +message UnknownCapability { + uint32 code = 1; + bytes value = 2; +} diff --git a/fiberlb/crates/fiberlb-server/proto/api/common.proto b/fiberlb/crates/fiberlb-server/proto/api/common.proto new file mode 100644 index 0000000..b4aab51 --- /dev/null +++ b/fiberlb/crates/fiberlb-server/proto/api/common.proto @@ -0,0 +1,63 @@ +syntax = "proto3"; + +package api; + +option go_package = "github.com/osrg/gobgp/v4/api;api"; + +// Common types for pretty much everywhere + +message Family { + enum Afi { + AFI_UNSPECIFIED = 0; + AFI_IP = 1; + AFI_IP6 = 2; + AFI_L2VPN = 25; + AFI_LS = 16388; + AFI_OPAQUE = 16397; + } + + enum Safi { + SAFI_UNSPECIFIED = 0; + SAFI_UNICAST = 1; + SAFI_MULTICAST = 2; + SAFI_MPLS_LABEL = 4; + SAFI_ENCAPSULATION = 7; + SAFI_VPLS = 65; + SAFI_EVPN = 70; + SAFI_LS = 71; + SAFI_SR_POLICY = 73; + SAFI_MUP = 85; + SAFI_MPLS_VPN = 128; + SAFI_MPLS_VPN_MULTICAST = 129; + SAFI_ROUTE_TARGET_CONSTRAINTS = 132; + SAFI_FLOW_SPEC_UNICAST = 133; + SAFI_FLOW_SPEC_VPN = 134; + SAFI_KEY_VALUE = 241; + } + + Afi afi = 1; + Safi safi = 2; +} + +message RouteDistinguisherTwoOctetASN { + uint32 admin = 1; + uint32 assigned = 2; +} + +message RouteDistinguisherIPAddress { + string admin = 1; + uint32 assigned = 2; +} + +message RouteDistinguisherFourOctetASN { + uint32 admin = 1; + uint32 assigned = 2; +} + +message RouteDistinguisher { + oneof rd { + RouteDistinguisherTwoOctetASN two_octet_asn = 1; + RouteDistinguisherIPAddress ip_address = 2; + RouteDistinguisherFourOctetASN four_octet_asn = 3; + } +} diff --git a/fiberlb/crates/fiberlb-server/proto/api/extcom.proto b/fiberlb/crates/fiberlb-server/proto/api/extcom.proto new file mode 100644 index 0000000..16c7f20 --- /dev/null +++ b/fiberlb/crates/fiberlb-server/proto/api/extcom.proto @@ -0,0 +1,162 @@ +syntax = "proto3"; + +package api; + +option go_package = "github.com/osrg/gobgp/v4/api;api"; + +// BGP Extended communities + +message TwoOctetAsSpecificExtended { + bool is_transitive = 1; + uint32 sub_type = 2; + uint32 asn = 3; + uint32 local_admin = 4; +} + +message IPv4AddressSpecificExtended { + bool is_transitive = 1; + uint32 sub_type = 2; + string address = 3; + uint32 local_admin = 4; +} + +message FourOctetAsSpecificExtended { + bool is_transitive = 1; + uint32 sub_type = 2; + uint32 asn = 3; + uint32 local_admin = 4; +} + +message LinkBandwidthExtended { + uint32 asn = 1; + float bandwidth = 2; +} + +message ValidationExtended { + uint32 state = 1; +} + +message ColorExtended { + uint32 color = 1; +} + +message EncapExtended { + uint32 tunnel_type = 1; +} + +message DefaultGatewayExtended {} + +message OpaqueExtended { + bool is_transitive = 1; + bytes value = 3; +} + +message ESILabelExtended { + bool is_single_active = 1; + uint32 label = 2; +} + +message ESImportRouteTarget { + string es_import = 1; +} + +message MacMobilityExtended { + bool is_sticky = 1; + uint32 sequence_num = 2; +} + +message RouterMacExtended { + string mac = 1; +} + +message TrafficRateExtended { + uint32 asn = 1; + float rate = 2; +} + +message TrafficActionExtended { + bool terminal = 1; + bool sample = 2; +} + +message RedirectTwoOctetAsSpecificExtended { + uint32 asn = 1; + uint32 local_admin = 2; +} + +message RedirectIPv4AddressSpecificExtended { + string address = 1; + uint32 local_admin = 2; +} + +message RedirectFourOctetAsSpecificExtended { + uint32 asn = 1; + uint32 local_admin = 2; +} + +message TrafficRemarkExtended { + uint32 dscp = 1; +} + +message MUPExtended { + uint32 sub_type = 1; + uint32 segment_id2 = 2; + uint32 segment_id4 = 3; +} + +message VPLSExtended { + uint32 control_flags = 1; + uint32 mtu = 2; +} + +message ETreeExtended { + bool is_leaf = 1; + uint32 label = 2; +} + +message MulticastFlagsExtended { + bool is_igmp_proxy = 1; + bool is_mld_proxy = 2; +} + +message UnknownExtended { + uint32 type = 1; + bytes value = 2; +} + +message ExtendedCommunity { + oneof extcom { + UnknownExtended unknown = 1; + TwoOctetAsSpecificExtended two_octet_as_specific = 2; + IPv4AddressSpecificExtended ipv4_address_specific = 3; + FourOctetAsSpecificExtended four_octet_as_specific = 4; + LinkBandwidthExtended link_bandwidth = 5; + ValidationExtended validation = 6; + ColorExtended color = 7; + EncapExtended encap = 8; + DefaultGatewayExtended default_gateway = 9; + OpaqueExtended opaque = 10; + ESILabelExtended esi_label = 11; + ESImportRouteTarget es_import = 12; + MacMobilityExtended mac_mobility = 13; + RouterMacExtended router_mac = 14; + TrafficRateExtended traffic_rate = 15; + TrafficActionExtended traffic_action = 16; + RedirectTwoOctetAsSpecificExtended redirect_two_octet_as_specific = 17; + RedirectIPv4AddressSpecificExtended redirect_ipv4_address_specific = 18; + RedirectFourOctetAsSpecificExtended redirect_four_octet_as_specific = 19; + TrafficRemarkExtended traffic_remark = 20; + MUPExtended mup = 21; + VPLSExtended vpls = 22; + ETreeExtended etree = 23; + MulticastFlagsExtended multicast_flags = 24; + } +} + +message RouteTarget { + oneof rt { + TwoOctetAsSpecificExtended two_octet_as_specific = 1; + IPv4AddressSpecificExtended ipv4_address_specific = 2; + FourOctetAsSpecificExtended four_octet_as_specific = 3; + } +} diff --git a/fiberlb/crates/fiberlb-server/proto/api/gobgp.proto b/fiberlb/crates/fiberlb-server/proto/api/gobgp.proto new file mode 100644 index 0000000..2ff059c --- /dev/null +++ b/fiberlb/crates/fiberlb-server/proto/api/gobgp.proto @@ -0,0 +1,1379 @@ +// Copyright (C) 2015-2017 Nippon Telegraph and Telephone Corporation. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +syntax = "proto3"; + +package api; + +import "api/attribute.proto"; +import "api/capability.proto"; +import "api/common.proto"; +import "api/extcom.proto"; +import "api/nlri.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/osrg/gobgp/v4/api;api"; + +// Interface exported by the server. +service GoBgpService { + rpc StartBgp(StartBgpRequest) returns (StartBgpResponse); + rpc StopBgp(StopBgpRequest) returns (StopBgpResponse); + rpc GetBgp(GetBgpRequest) returns (GetBgpResponse); + + rpc WatchEvent(WatchEventRequest) returns (stream WatchEventResponse); + + rpc AddPeer(AddPeerRequest) returns (AddPeerResponse); + rpc DeletePeer(DeletePeerRequest) returns (DeletePeerResponse); + rpc ListPeer(ListPeerRequest) returns (stream ListPeerResponse); + rpc UpdatePeer(UpdatePeerRequest) returns (UpdatePeerResponse); + rpc ResetPeer(ResetPeerRequest) returns (ResetPeerResponse); + rpc ShutdownPeer(ShutdownPeerRequest) returns (ShutdownPeerResponse); + rpc EnablePeer(EnablePeerRequest) returns (EnablePeerResponse); + rpc DisablePeer(DisablePeerRequest) returns (DisablePeerResponse); + + rpc AddPeerGroup(AddPeerGroupRequest) returns (AddPeerGroupResponse); + rpc DeletePeerGroup(DeletePeerGroupRequest) returns (DeletePeerGroupResponse); + rpc ListPeerGroup(ListPeerGroupRequest) returns (stream ListPeerGroupResponse); + rpc UpdatePeerGroup(UpdatePeerGroupRequest) returns (UpdatePeerGroupResponse); + + rpc AddDynamicNeighbor(AddDynamicNeighborRequest) returns (AddDynamicNeighborResponse); + rpc ListDynamicNeighbor(ListDynamicNeighborRequest) returns (stream ListDynamicNeighborResponse); + rpc DeleteDynamicNeighbor(DeleteDynamicNeighborRequest) returns (DeleteDynamicNeighborResponse); + + rpc AddPath(AddPathRequest) returns (AddPathResponse); + rpc DeletePath(DeletePathRequest) returns (DeletePathResponse); + rpc ListPath(ListPathRequest) returns (stream ListPathResponse); + rpc AddPathStream(stream AddPathStreamRequest) returns (AddPathStreamResponse); + + rpc GetTable(GetTableRequest) returns (GetTableResponse); + + rpc AddVrf(AddVrfRequest) returns (AddVrfResponse); + rpc DeleteVrf(DeleteVrfRequest) returns (DeleteVrfResponse); + rpc ListVrf(ListVrfRequest) returns (stream ListVrfResponse); + + rpc AddPolicy(AddPolicyRequest) returns (AddPolicyResponse); + rpc DeletePolicy(DeletePolicyRequest) returns (DeletePolicyResponse); + rpc ListPolicy(ListPolicyRequest) returns (stream ListPolicyResponse); + rpc SetPolicies(SetPoliciesRequest) returns (SetPoliciesResponse); + + rpc AddDefinedSet(AddDefinedSetRequest) returns (AddDefinedSetResponse); + rpc DeleteDefinedSet(DeleteDefinedSetRequest) returns (DeleteDefinedSetResponse); + rpc ListDefinedSet(ListDefinedSetRequest) returns (stream ListDefinedSetResponse); + + rpc AddStatement(AddStatementRequest) returns (AddStatementResponse); + rpc DeleteStatement(DeleteStatementRequest) returns (DeleteStatementResponse); + rpc ListStatement(ListStatementRequest) returns (stream ListStatementResponse); + + rpc AddPolicyAssignment(AddPolicyAssignmentRequest) returns (AddPolicyAssignmentResponse); + rpc DeletePolicyAssignment(DeletePolicyAssignmentRequest) returns (DeletePolicyAssignmentResponse); + rpc ListPolicyAssignment(ListPolicyAssignmentRequest) returns (stream ListPolicyAssignmentResponse); + rpc SetPolicyAssignment(SetPolicyAssignmentRequest) returns (SetPolicyAssignmentResponse); + + rpc AddRpki(AddRpkiRequest) returns (AddRpkiResponse); + rpc DeleteRpki(DeleteRpkiRequest) returns (DeleteRpkiResponse); + rpc ListRpki(ListRpkiRequest) returns (stream ListRpkiResponse); + rpc EnableRpki(EnableRpkiRequest) returns (EnableRpkiResponse); + rpc DisableRpki(DisableRpkiRequest) returns (DisableRpkiResponse); + rpc ResetRpki(ResetRpkiRequest) returns (ResetRpkiResponse); + rpc ListRpkiTable(ListRpkiTableRequest) returns (stream ListRpkiTableResponse); + + rpc EnableZebra(EnableZebraRequest) returns (EnableZebraResponse); + + rpc EnableMrt(EnableMrtRequest) returns (EnableMrtResponse); + rpc DisableMrt(DisableMrtRequest) returns (DisableMrtResponse); + + rpc AddBmp(AddBmpRequest) returns (AddBmpResponse); + rpc DeleteBmp(DeleteBmpRequest) returns (DeleteBmpResponse); + rpc ListBmp(ListBmpRequest) returns (stream ListBmpResponse); + + rpc SetLogLevel(SetLogLevelRequest) returns (SetLogLevelResponse); +} + +message StartBgpRequest { + Global global = 1; +} + +message StartBgpResponse {} + +message StopBgpRequest { + // Allows the Graceful Restart procedure on the remote peers by not sending a NOTIFICATION message to GR-enabled peers. + bool allow_graceful_restart = 1; +} + +message StopBgpResponse {} + +message GetBgpRequest {} + +message GetBgpResponse { + Global global = 1; +} + +message WatchEventRequest { + message Peer {} + Peer peer = 1; + + message Table { + message Filter { + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_BEST = 1; + TYPE_ADJIN = 2; + TYPE_POST_POLICY = 3; + TYPE_EOR = 4; + } + Type type = 1; + bool init = 2; + string peer_address = 3; + string peer_group = 4; + } + repeated Filter filters = 1; + } + Table table = 2; + + // Max number of paths to include in a single message. 0 for unlimited. + uint32 batch_size = 3; +} + +message WatchEventResponse { + message PeerEvent { + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_INIT = 1; + TYPE_END_OF_INIT = 2; + TYPE_STATE = 3; + } + Type type = 1; + Peer peer = 2; + } + + message TableEvent { + repeated Path paths = 2; + } + + oneof event { + PeerEvent peer = 2; + TableEvent table = 3; + } +} + +message AddPeerRequest { + Peer peer = 1; +} + +message AddPeerResponse {} + +message DeletePeerRequest { + string address = 1; + string interface = 2; +} + +message DeletePeerResponse {} + +message ListPeerRequest { + string address = 1; + bool enable_advertised = 2; +} + +message ListPeerResponse { + Peer peer = 1; +} + +message UpdatePeerRequest { + Peer peer = 1; + // Calls SoftResetIn after updating the peer configuration if needed. + bool do_soft_reset_in = 2; +} + +message UpdatePeerResponse { + // Indicates whether calling SoftResetIn is required due to this update. If + // "true" is set, the client should call SoftResetIn manually. If + // "do_soft_reset_in = true" is set in the request, always returned with + // "false". + bool needs_soft_reset_in = 1; +} + +message ResetPeerRequest { + string address = 1; + string communication = 2; + bool soft = 3; + enum Direction { + DIRECTION_UNSPECIFIED = 0; + DIRECTION_IN = 1; + DIRECTION_OUT = 2; + DIRECTION_BOTH = 3; + } + Direction direction = 4; +} + +message ResetPeerResponse {} + +message ShutdownPeerRequest { + string address = 1; + string communication = 2; +} + +message ShutdownPeerResponse {} + +message EnablePeerRequest { + string address = 1; +} + +message EnablePeerResponse {} + +message DisablePeerRequest { + string address = 1; + string communication = 2; +} + +message DisablePeerResponse {} + +message AddPeerGroupRequest { + PeerGroup peer_group = 1; +} + +message AddPeerGroupResponse {} + +message DeletePeerGroupRequest { + string name = 1; +} + +message DeletePeerGroupResponse {} + +message UpdatePeerGroupRequest { + PeerGroup peer_group = 1; + bool do_soft_reset_in = 2; +} + +message UpdatePeerGroupResponse { + bool needs_soft_reset_in = 1; +} + +message ListPeerGroupRequest { + string peer_group_name = 1; +} + +message ListPeerGroupResponse { + PeerGroup peer_group = 1; +} + +message AddDynamicNeighborRequest { + DynamicNeighbor dynamic_neighbor = 1; +} + +message AddDynamicNeighborResponse {} + +message DeleteDynamicNeighborRequest { + string prefix = 1; + string peer_group = 2; +} + +message DeleteDynamicNeighborResponse {} + +message ListDynamicNeighborRequest { + string peer_group = 1; +} + +message ListDynamicNeighborResponse { + DynamicNeighbor dynamic_neighbor = 1; +} + +message AddPathRequest { + TableType table_type = 1; + string vrf_id = 2; + Path path = 3; +} + +message AddPathResponse { + bytes uuid = 1; +} + +message DeletePathRequest { + TableType table_type = 1; + string vrf_id = 2; + Family family = 3; + Path path = 4; + bytes uuid = 5; +} + +message DeletePathResponse {} + +// API representation of table.LookupPrefix +message TableLookupPrefix { + // API representation of table.LookupOption + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_EXACT = 1; + TYPE_LONGER = 2; + TYPE_SHORTER = 3; + } + string prefix = 1; + Type type = 2; + string rd = 3; +} + +message ListPathRequest { + TableType table_type = 1; + string name = 2; + Family family = 3; + repeated TableLookupPrefix prefixes = 4; + enum SortType { + SORT_TYPE_UNSPECIFIED = 0; + SORT_TYPE_PREFIX = 1; + } + SortType sort_type = 5; + bool enable_filtered = 6; + bool enable_nlri_binary = 7; + bool enable_attribute_binary = 8; + // enable_only_binary == true means that only nlri_binary and pattrs_binary + // will be used instead of nlri and pattrs for each Path in ListPathResponse. + bool enable_only_binary = 9; + // max ammount of paths to be allocated, unlimited by default + uint64 batch_size = 10; +} + +message ListPathResponse { + Destination destination = 1; +} + +message AddPathStreamRequest { + TableType table_type = 1; + string vrf_id = 2; + repeated Path paths = 3; +} + +message AddPathStreamResponse {} + +message GetTableRequest { + TableType table_type = 1; + Family family = 2; + string name = 3; +} + +message GetTableResponse { + uint64 num_destination = 1; + uint64 num_path = 2; + uint64 num_accepted = 3; // only meaningful when type == ADJ_IN +} + +message AddVrfRequest { + Vrf vrf = 1; +} + +message AddVrfResponse {} + +message DeleteVrfRequest { + string name = 1; +} + +message DeleteVrfResponse {} + +message ListVrfRequest { + string name = 1; +} + +message ListVrfResponse { + Vrf vrf = 1; +} + +message AddPolicyRequest { + Policy policy = 1; + // if this flag is set, gobgpd won't define new statements + // but refer existing statements using statement's names in this arguments. + bool refer_existing_statements = 2; +} + +message AddPolicyResponse {} + +message DeletePolicyRequest { + Policy policy = 1; + // if this flag is set, gobgpd won't delete any statements + // even if some statements get not used by any policy by this operation. + bool preserve_statements = 2; + bool all = 3; +} + +message DeletePolicyResponse {} + +message ListPolicyRequest { + string name = 1; +} + +message ListPolicyResponse { + Policy policy = 1; +} + +message SetPoliciesRequest { + repeated DefinedSet defined_sets = 1; + repeated Policy policies = 2; + repeated PolicyAssignment assignments = 3; +} + +message SetPoliciesResponse {} + +message AddDefinedSetRequest { + DefinedSet defined_set = 1; + bool replace = 2; +} + +message AddDefinedSetResponse {} + +message DeleteDefinedSetRequest { + DefinedSet defined_set = 1; + bool all = 2; +} + +message DeleteDefinedSetResponse {} + +message ListDefinedSetRequest { + DefinedType defined_type = 1; + string name = 2; +} + +message ListDefinedSetResponse { + DefinedSet defined_set = 1; +} + +message AddStatementRequest { + Statement statement = 1; +} + +message AddStatementResponse {} + +message DeleteStatementRequest { + Statement statement = 1; + bool all = 2; +} + +message DeleteStatementResponse {} + +message ListStatementRequest { + string name = 1; +} + +message ListStatementResponse { + Statement statement = 1; +} + +message AddPolicyAssignmentRequest { + PolicyAssignment assignment = 1; +} + +message AddPolicyAssignmentResponse {} + +message DeletePolicyAssignmentRequest { + PolicyAssignment assignment = 1; + bool all = 2; +} + +message DeletePolicyAssignmentResponse {} + +message ListPolicyAssignmentRequest { + string name = 1; + PolicyDirection direction = 2; +} + +message ListPolicyAssignmentResponse { + PolicyAssignment assignment = 1; +} + +message SetPolicyAssignmentRequest { + PolicyAssignment assignment = 1; +} + +message SetPolicyAssignmentResponse {} + +message AddRpkiRequest { + string address = 1; + uint32 port = 2; + int64 lifetime = 3; +} + +message AddRpkiResponse {} + +message DeleteRpkiRequest { + string address = 1; + uint32 port = 2; +} + +message DeleteRpkiResponse {} + +message ListRpkiRequest { + Family family = 1; +} + +message ListRpkiResponse { + Rpki server = 1; +} + +message EnableRpkiRequest { + string address = 1; + uint32 port = 2; +} + +message EnableRpkiResponse {} + +message DisableRpkiRequest { + string address = 1; + uint32 port = 2; +} + +message DisableRpkiResponse {} + +message ResetRpkiRequest { + string address = 1; + uint32 port = 2; + bool soft = 3; +} + +message ResetRpkiResponse {} + +message ListRpkiTableRequest { + Family family = 1; +} + +message ListRpkiTableResponse { + Roa roa = 1; +} + +message EnableZebraRequest { + string url = 1; + repeated string route_types = 2; + uint32 version = 3; + bool nexthop_trigger_enable = 4; + uint32 nexthop_trigger_delay = 5; + uint32 mpls_label_range_size = 6; + string software_name = 7; +} + +message EnableZebraResponse {} + +message EnableMrtRequest { + enum DumpType { + DUMP_TYPE_UNSPECIFIED = 0; + DUMP_TYPE_UPDATES = 1; + DUMP_TYPE_TABLE = 2; + } + DumpType dump_type = 1; + string filename = 2; + uint64 dump_interval = 3; + uint64 rotation_interval = 4; +} + +message EnableMrtResponse {} + +message DisableMrtRequest { + string filename = 1; +} + +message DisableMrtResponse {} + +message AddBmpRequest { + string address = 1; + uint32 port = 2; + enum MonitoringPolicy { + MONITORING_POLICY_UNSPECIFIED = 0; + MONITORING_POLICY_PRE = 1; + MONITORING_POLICY_POST = 2; + MONITORING_POLICY_BOTH = 3; + MONITORING_POLICY_LOCAL = 4; + MONITORING_POLICY_ALL = 5; + } + MonitoringPolicy policy = 3; + int32 statistics_timeout = 4; + string sys_name = 5; + string sys_descr = 6; +} + +message AddBmpResponse {} + +message DeleteBmpRequest { + string address = 1; + uint32 port = 2; +} + +message DeleteBmpResponse {} + +message ListBmpRequest {} + +message ListBmpResponse { + message BmpStation { + message Conf { + string address = 1; + uint32 port = 2; + } + Conf conf = 1; + message State { + google.protobuf.Timestamp uptime = 1; + google.protobuf.Timestamp downtime = 2; + } + State state = 2; + } + + BmpStation station = 1; +} + +enum TableType { + TABLE_TYPE_UNSPECIFIED = 0; + TABLE_TYPE_GLOBAL = 1; + TABLE_TYPE_LOCAL = 2; + TABLE_TYPE_ADJ_IN = 3; + TABLE_TYPE_ADJ_OUT = 4; + TABLE_TYPE_VRF = 5; +} + +enum ValidationState { + VALIDATION_STATE_UNSPECIFIED = 0; + VALIDATION_STATE_NONE = 1; + VALIDATION_STATE_NOT_FOUND = 2; + VALIDATION_STATE_VALID = 3; + VALIDATION_STATE_INVALID = 4; +} + +message Validation { + enum Reason { + REASON_UNSPECIFIED = 0; + REASON_NONE = 1; + REASON_ASN = 2; + REASON_LENGTH = 3; + } + + ValidationState state = 1; + Reason reason = 2; + repeated Roa matched = 3; + repeated Roa unmatched_asn = 4; + repeated Roa unmatched_length = 5; +} + +message Path { + NLRI nlri = 1; + repeated Attribute pattrs = 2; + google.protobuf.Timestamp age = 3; + bool best = 4; + bool is_withdraw = 5; + Validation validation = 7; + bool no_implicit_withdraw = 8; + Family family = 9; + uint32 source_asn = 10; + string source_id = 11; + bool filtered = 12; + bool stale = 13; + bool is_from_external = 14; + string neighbor_ip = 15; + bytes uuid = 16; // only paths installed by AddPath API have this + bool is_nexthop_invalid = 17; + uint32 identifier = 18; + uint32 local_identifier = 19; + bytes nlri_binary = 20; + repeated bytes pattrs_binary = 21; + bool send_max_filtered = 22; +} + +message Destination { + string prefix = 1; + repeated Path paths = 2; +} + +message Peer { + ApplyPolicy apply_policy = 1; + PeerConf conf = 2; + EbgpMultihop ebgp_multihop = 3; + RouteReflector route_reflector = 4; + PeerState state = 5; + Timers timers = 6; + Transport transport = 7; + RouteServer route_server = 8; + GracefulRestart graceful_restart = 9; + repeated AfiSafi afi_safis = 10; + TtlSecurity ttl_security = 11; +} + +message PeerGroup { + ApplyPolicy apply_policy = 1; + PeerGroupConf conf = 2; + EbgpMultihop ebgp_multihop = 3; + RouteReflector route_reflector = 4; + PeerGroupState info = 5; + Timers timers = 6; + Transport transport = 7; + RouteServer route_server = 8; + GracefulRestart graceful_restart = 9; + repeated AfiSafi afi_safis = 10; + TtlSecurity ttl_security = 11; +} + +message DynamicNeighbor { + string prefix = 1; + string peer_group = 2; +} + +message ApplyPolicy { + PolicyAssignment export_policy = 1; + PolicyAssignment import_policy = 2; +} + +message PrefixLimit { + Family family = 1; + uint32 max_prefixes = 2; + uint32 shutdown_threshold_pct = 3; +} + +enum PeerType { + PEER_TYPE_UNSPECIFIED = 0; + PEER_TYPE_INTERNAL = 1; + PEER_TYPE_EXTERNAL = 2; +} + +enum RemovePrivate { + REMOVE_PRIVATE_UNSPECIFIED = 0; + REMOVE_PRIVATE_ALL = 1; + REMOVE_PRIVATE_REPLACE = 2; +} + +message PeerConf { + string auth_password = 1; + string description = 2; + uint32 local_asn = 3; + string neighbor_address = 4; + uint32 peer_asn = 5; + string peer_group = 6; + PeerType type = 7; + RemovePrivate remove_private = 8; + bool route_flap_damping = 9; + uint32 send_community = 10; + string neighbor_interface = 11; + string vrf = 12; + uint32 allow_own_asn = 13; + bool replace_peer_asn = 14; + bool admin_down = 15; + bool send_software_version = 16; + bool allow_aspath_loop_local = 17; +} + +message PeerGroupConf { + string auth_password = 1; + string description = 2; + uint32 local_asn = 3; + uint32 peer_asn = 4; + string peer_group_name = 5; + PeerType type = 6; + RemovePrivate remove_private = 7; + bool route_flap_damping = 8; + uint32 send_community = 9; + bool send_software_version = 10; +} + +message PeerGroupState { + string auth_password = 1; + string description = 2; + uint32 local_asn = 3; + uint32 peer_asn = 4; + string peer_group_name = 5; + PeerType type = 6; + RemovePrivate remove_private = 7; + bool route_flap_damping = 8; + uint32 send_community = 9; + uint32 total_paths = 10; + uint32 total_prefixes = 11; +} + +message TtlSecurity { + bool enabled = 1; + uint32 ttl_min = 2; +} + +message EbgpMultihop { + bool enabled = 1; + uint32 multihop_ttl = 2; +} + +message RouteReflector { + bool route_reflector_client = 1; + string route_reflector_cluster_id = 2; +} + +message PeerState { + string auth_password = 1; + string description = 2; + uint32 local_asn = 3; + Messages messages = 4; + string neighbor_address = 5; + uint32 peer_asn = 6; + string peer_group = 7; + PeerType type = 8; + Queues queues = 9; + RemovePrivate remove_private = 10; + bool route_flap_damping = 11; + uint32 send_community = 12; + enum SessionState { + SESSION_STATE_UNSPECIFIED = 0; + SESSION_STATE_IDLE = 1; + SESSION_STATE_CONNECT = 2; + SESSION_STATE_ACTIVE = 3; + SESSION_STATE_OPENSENT = 4; + SESSION_STATE_OPENCONFIRM = 5; + SESSION_STATE_ESTABLISHED = 6; + } + SessionState session_state = 13; + enum AdminState { + ADMIN_STATE_UNSPECIFIED = 0; + ADMIN_STATE_UP = 1; + ADMIN_STATE_DOWN = 2; + ADMIN_STATE_PFX_CT = 3; // prefix counter over limit + } + AdminState admin_state = 15; + uint32 out_q = 16; + uint32 flops = 17; + repeated Capability remote_cap = 18; + repeated Capability local_cap = 19; + string router_id = 20; + // State change reason information + enum DisconnectReason { + DISCONNECT_REASON_UNSPECIFIED = 0; + DISCONNECT_REASON_ADMIN_DOWN = 1; + DISCONNECT_REASON_HOLD_TIMER_EXPIRED = 2; + DISCONNECT_REASON_NOTIFICATION_SENT = 3; + DISCONNECT_REASON_NOTIFICATION_RECEIVED = 4; + DISCONNECT_REASON_READ_FAILED = 5; + DISCONNECT_REASON_WRITE_FAILED = 6; + DISCONNECT_REASON_IDLE_TIMER_EXPIRED = 7; + DISCONNECT_REASON_RESTART_TIMER_EXPIRED = 8; + DISCONNECT_REASON_GRACEFUL_RESTART = 9; + DISCONNECT_REASON_INVALID_MSG = 10; + DISCONNECT_REASON_HARD_RESET = 11; + DISCONNECT_REASON_DECONFIGURED = 12; + DISCONNECT_REASON_BAD_PEER_AS = 13; + } + DisconnectReason disconnect_reason = 21; + string disconnect_message = 22; +} + +message Messages { + Message received = 1; + Message sent = 2; +} + +message Message { + uint64 notification = 1; + uint64 update = 2; + uint64 open = 3; + uint64 keepalive = 4; + uint64 refresh = 5; + uint64 discarded = 6; + uint64 total = 7; + uint64 withdraw_update = 8; + uint64 withdraw_prefix = 9; +} + +message Queues { + uint32 input = 1; + uint32 output = 2; +} + +message Timers { + TimersConfig config = 1; + TimersState state = 2; +} + +message TimersConfig { + uint64 connect_retry = 1; + uint64 hold_time = 2; + uint64 keepalive_interval = 3; + uint64 minimum_advertisement_interval = 4; + uint64 idle_hold_time_after_reset = 5; +} + +message TimersState { + uint64 connect_retry = 1; + uint64 hold_time = 2; + uint64 keepalive_interval = 3; + uint64 minimum_advertisement_interval = 4; + uint64 negotiated_hold_time = 5; + google.protobuf.Timestamp uptime = 6; + google.protobuf.Timestamp downtime = 7; +} + +message Transport { + string local_address = 1; + uint32 local_port = 2; + bool mtu_discovery = 3; + bool passive_mode = 4; + string remote_address = 5; + uint32 remote_port = 6; + uint32 tcp_mss = 7; + string bind_interface = 8; +} + +message RouteServer { + bool route_server_client = 1; + bool secondary_route = 2; +} + +message GracefulRestart { + bool enabled = 1; + uint32 restart_time = 2; + bool helper_only = 3; + uint32 deferral_time = 4; + bool notification_enabled = 5; + bool longlived_enabled = 6; + uint32 stale_routes_time = 7; + uint32 peer_restart_time = 8; + bool peer_restarting = 9; + bool local_restarting = 10; + string mode = 11; +} + +message MpGracefulRestartConfig { + bool enabled = 1; +} + +message MpGracefulRestartState { + bool enabled = 1; + bool received = 2; + bool advertised = 3; + bool end_of_rib_received = 4; + bool end_of_rib_sent = 5; + bool running = 6; +} +message MpGracefulRestart { + MpGracefulRestartConfig config = 1; + MpGracefulRestartState state = 2; +} + +message AfiSafiConfig { + Family family = 1; + bool enabled = 2; +} + +message AfiSafiState { + Family family = 1; + bool enabled = 2; + uint64 received = 3; + uint64 accepted = 4; + uint64 advertised = 5; +} + +message RouteSelectionOptionsConfig { + bool always_compare_med = 1; + bool ignore_as_path_length = 2; + bool external_compare_router_id = 3; + bool advertise_inactive_routes = 4; + bool enable_aigp = 5; + bool ignore_next_hop_igp_metric = 6; + bool disable_best_path_selection = 7; +} + +message RouteSelectionOptionsState { + bool always_compare_med = 1; + bool ignore_as_path_length = 2; + bool external_compare_router_id = 3; + bool advertise_inactive_routes = 4; + bool enable_aigp = 5; + bool ignore_next_hop_igp_metric = 6; + bool disable_best_path_selection = 7; +} + +message RouteSelectionOptions { + RouteSelectionOptionsConfig config = 1; + RouteSelectionOptionsState state = 2; +} + +message UseMultiplePathsConfig { + bool enabled = 1; +} + +message UseMultiplePathsState { + bool enabled = 1; +} + +message EbgpConfig { + bool allow_multiple_asn = 1; + uint32 maximum_paths = 2; +} + +message EbgpState { + bool allow_multiple_asn = 1; + uint32 maximum_paths = 2; +} + +message Ebgp { + EbgpConfig config = 1; + EbgpState state = 2; +} + +message IbgpConfig { + uint32 maximum_paths = 1; +} + +message IbgpState { + uint32 maximum_paths = 1; +} + +message Ibgp { + IbgpConfig config = 1; + IbgpState state = 2; +} + +message UseMultiplePaths { + UseMultiplePathsConfig config = 1; + UseMultiplePathsState state = 2; + Ebgp ebgp = 3; + Ibgp ibgp = 4; +} + +message RouteTargetMembershipConfig { + uint32 deferral_time = 1; +} + +message RouteTargetMembershipState { + uint32 deferral_time = 1; +} + +message RouteTargetMembership { + RouteTargetMembershipConfig config = 1; + RouteTargetMembershipState state = 2; +} + +message LongLivedGracefulRestartConfig { + bool enabled = 1; + uint32 restart_time = 2; +} + +message LongLivedGracefulRestartState { + bool enabled = 1; + bool received = 2; + bool advertised = 3; + uint32 peer_restart_time = 4; + bool peer_restart_timer_expired = 5; + bool running = 6; +} + +message LongLivedGracefulRestart { + LongLivedGracefulRestartConfig config = 1; + LongLivedGracefulRestartState state = 2; +} + +message AfiSafi { + MpGracefulRestart mp_graceful_restart = 1; + AfiSafiConfig config = 2; + AfiSafiState state = 3; + ApplyPolicy apply_policy = 4; + // TODO: + // Support the following structures: + // - Ipv4Unicast + // - Ipv6Unicast + // - Ipv4LabelledUnicast + // - Ipv6LabelledUnicast + // - L3vpnIpv4Unicast + // - L3vpnIpv6Unicast + // - L3vpnIpv4Multicast + // - L3vpnIpv6Multicast + // - L2vpnVpls + // - L2vpnEvpn + RouteSelectionOptions route_selection_options = 5; + UseMultiplePaths use_multiple_paths = 6; + PrefixLimit prefix_limits = 7; + RouteTargetMembership route_target_membership = 8; + LongLivedGracefulRestart long_lived_graceful_restart = 9; + AddPaths add_paths = 10; +} + +message AddPathsConfig { + bool receive = 1; + uint32 send_max = 2; +} + +message AddPathsState { + bool receive = 1; + uint32 send_max = 2; +} + +message AddPaths { + AddPathsConfig config = 1; + AddPathsState state = 2; +} + +message Prefix { + string ip_prefix = 1; + uint32 mask_length_min = 2; + uint32 mask_length_max = 3; +} + +enum DefinedType { + DEFINED_TYPE_UNSPECIFIED = 0; + DEFINED_TYPE_PREFIX = 1; + DEFINED_TYPE_NEIGHBOR = 2; + DEFINED_TYPE_TAG = 3; + DEFINED_TYPE_AS_PATH = 4; + DEFINED_TYPE_COMMUNITY = 5; + DEFINED_TYPE_EXT_COMMUNITY = 6; + DEFINED_TYPE_LARGE_COMMUNITY = 7; + DEFINED_TYPE_NEXT_HOP = 8; +} + +message DefinedSet { + DefinedType defined_type = 1; + string name = 2; + repeated string list = 3; + repeated Prefix prefixes = 4; +} + +message MatchSet { + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_ANY = 1; + TYPE_ALL = 2; + TYPE_INVERT = 3; + } + Type type = 1; + string name = 2; +} + +enum Comparison { + COMPARISON_UNSPECIFIED = 0; + COMPARISON_EQ = 1; + COMPARISON_GE = 2; + COMPARISON_LE = 3; +} + +message AsPathLength { + Comparison type = 1; + uint32 length = 2; +} + +message CommunityCount { + Comparison type = 1; + uint32 count = 2; +} + +enum OriginType { + ORIGIN_TYPE_UNSPECIFIED = 0; + ORIGIN_TYPE_IGP = 1; + ORIGIN_TYPE_EGP = 2; + ORIGIN_TYPE_INCOMPLETE = 3; +} + +message LocalPrefEq { + uint32 value = 1; +} + +message MedEq { + uint32 value = 1; +} + +message Conditions { + MatchSet prefix_set = 1; + MatchSet neighbor_set = 2; + AsPathLength as_path_length = 3; + MatchSet as_path_set = 4; + MatchSet community_set = 5; + MatchSet ext_community_set = 6; + ValidationState rpki_result = 7; + enum RouteType { + ROUTE_TYPE_UNSPECIFIED = 0; + ROUTE_TYPE_INTERNAL = 1; + ROUTE_TYPE_EXTERNAL = 2; + ROUTE_TYPE_LOCAL = 3; + } + RouteType route_type = 8; + MatchSet large_community_set = 9; + repeated string next_hop_in_list = 10; + repeated Family afi_safi_in = 11; + CommunityCount community_count = 12; + OriginType origin = 13; + LocalPrefEq local_pref_eq = 14; + MedEq med_eq = 15; +} + +enum RouteAction { + ROUTE_ACTION_UNSPECIFIED = 0; + ROUTE_ACTION_ACCEPT = 1; + ROUTE_ACTION_REJECT = 2; +} + +message CommunityAction { + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_ADD = 1; + TYPE_REMOVE = 2; + TYPE_REPLACE = 3; + } + Type type = 1; + repeated string communities = 2; +} + +message MedAction { + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_MOD = 1; + TYPE_REPLACE = 2; + } + Type type = 1; + int64 value = 2; +} + +message AsPrependAction { + uint32 asn = 1; + uint32 repeat = 2; + bool use_left_most = 3; +} + +message NexthopAction { + string address = 1; + bool self = 2; + bool unchanged = 3; + bool peer_address = 4; +} + +message LocalPrefAction { + uint32 value = 1; +} + +message OriginAction { + OriginType origin = 1; +} + +message Actions { + RouteAction route_action = 1; + CommunityAction community = 2; + MedAction med = 3; + AsPrependAction as_prepend = 4; + CommunityAction ext_community = 5; + NexthopAction nexthop = 6; + LocalPrefAction local_pref = 7; + CommunityAction large_community = 8; + OriginAction origin_action = 9; +} + +message Statement { + string name = 1; + Conditions conditions = 2; + Actions actions = 3; +} + +message Policy { + string name = 1; + repeated Statement statements = 2; +} + +enum PolicyDirection { + POLICY_DIRECTION_UNSPECIFIED = 0; + POLICY_DIRECTION_IMPORT = 1; + POLICY_DIRECTION_EXPORT = 2; +} + +message PolicyAssignment { + string name = 1; + PolicyDirection direction = 2; + repeated Policy policies = 4; + RouteAction default_action = 5; +} + +message RoutingPolicy { + repeated DefinedSet defined_sets = 1; + repeated Policy policies = 2; +} + +message Roa { + uint32 asn = 1; + uint32 prefixlen = 2; + uint32 maxlen = 3; + string prefix = 4; + RPKIConf conf = 5; +} + +message Vrf { + string name = 1; + RouteDistinguisher rd = 2; + repeated RouteTarget import_rt = 3; + repeated RouteTarget export_rt = 4; + uint32 id = 5; +} + +message DefaultRouteDistance { + uint32 external_route_distance = 1; + uint32 internal_route_distance = 2; +} + +message Global { + uint32 asn = 1; + string router_id = 2; + int32 listen_port = 3; + repeated string listen_addresses = 4; + repeated uint32 families = 5; + bool use_multiple_paths = 6; + RouteSelectionOptionsConfig route_selection_options = 7; + DefaultRouteDistance default_route_distance = 8; + Confederation confederation = 9; + GracefulRestart graceful_restart = 10; + string bind_to_device = 11; +} + +message Confederation { + bool enabled = 1; + uint32 identifier = 2; + repeated uint32 member_as_list = 3; +} + +message RPKIConf { + string address = 1; + uint32 remote_port = 2; +} + +message RPKIState { + google.protobuf.Timestamp uptime = 1; + google.protobuf.Timestamp downtime = 2; + bool up = 3; + uint32 record_ipv4 = 4; + uint32 record_ipv6 = 5; + uint32 prefix_ipv4 = 6; + uint32 prefix_ipv6 = 7; + uint32 serial = 8; + int64 received_ipv4 = 9; + int64 received_ipv6 = 10; + int64 serial_notify = 11; + int64 cache_reset = 12; + int64 cache_response = 13; + int64 end_of_data = 14; + int64 error = 15; + int64 serial_query = 16; + int64 reset_query = 17; +} + +message Rpki { + RPKIConf conf = 1; + RPKIState state = 2; +} + +message SetLogLevelRequest { + enum Level { + LEVEL_UNSPECIFIED = 0; + LEVEL_PANIC = 1; + LEVEL_FATAL = 2; + LEVEL_ERROR = 3; + LEVEL_WARN = 4; + LEVEL_INFO = 5; + LEVEL_DEBUG = 6; + LEVEL_TRACE = 7; + } + Level level = 1; +} + +message SetLogLevelResponse {} diff --git a/fiberlb/crates/fiberlb-server/proto/api/nlri.proto b/fiberlb/crates/fiberlb-server/proto/api/nlri.proto new file mode 100644 index 0000000..621675f --- /dev/null +++ b/fiberlb/crates/fiberlb-server/proto/api/nlri.proto @@ -0,0 +1,361 @@ +syntax = "proto3"; + +package api; + +import "api/common.proto"; +import "api/extcom.proto"; + +option go_package = "github.com/osrg/gobgp/v4/api;api"; + +// Main NLRI type + +message NLRI { + oneof nlri { + IPAddressPrefix prefix = 1; + LabeledIPAddressPrefix labeled_prefix = 2; + EncapsulationNLRI encapsulation = 3; + VPLSNLRI vpls = 4; + EVPNEthernetAutoDiscoveryRoute evpn_ethernet_ad = 5; + EVPNMACIPAdvertisementRoute evpn_macadv = 6; + EVPNInclusiveMulticastEthernetTagRoute evpn_multicast = 7; + EVPNEthernetSegmentRoute evpn_ethernet_segment = 8; + EVPNIPPrefixRoute evpn_ip_prefix = 9; + EVPNIPMSIRoute evpn_i_pmsi = 10; + LabeledVPNIPAddressPrefix labeled_vpn_ip_prefix = 11; + RouteTargetMembershipNLRI route_target_membership = 12; + FlowSpecNLRI flow_spec = 13; + VPNFlowSpecNLRI vpn_flow_spec = 14; + OpaqueNLRI opaque = 15; + LsAddrPrefix ls_addr_prefix = 16; + SRPolicyNLRI sr_policy = 17; + MUPInterworkSegmentDiscoveryRoute mup_interwork_segment_discovery = 18; + MUPDirectSegmentDiscoveryRoute mup_direct_segment_discovery = 19; + MUPType1SessionTransformedRoute mup_type_1_session_transformed = 20; + MUPType2SessionTransformedRoute mup_type_2_session_transformed = 21; + } +} + +// IPAddressPrefix represents the NLRI for: +// - AFI=1, SAFI=1 +// - AFI=2, SAFI=1 +message IPAddressPrefix { + uint32 prefix_len = 1; + string prefix = 2; +} + +// LabeledIPAddressPrefix represents the NLRI for: +// - AFI=1, SAFI=4 +// - AFI=2, SAFI=4 +message LabeledIPAddressPrefix { + repeated uint32 labels = 1; + uint32 prefix_len = 2; + string prefix = 3; +} + +// EncapsulationNLRI represents the NLRI for: +// - AFI=1, SAFI=7 +// - AFI=2, SAFI=7 +message EncapsulationNLRI { + string address = 1; +} + +// VPLSNLRI represents the NLRI for: +// - AFI=25, SAFI=65 +message VPLSNLRI { + RouteDistinguisher rd = 1; + uint32 ve_id = 2; + uint32 ve_block_offset = 3; + uint32 ve_block_size = 4; + uint32 label_block_base = 5; +} + +message EthernetSegmentIdentifier { + uint32 type = 1; + bytes value = 2; +} + +// EVPNEthernetAutoDiscoveryRoute represents the NLRI for: +// - AFI=25, SAFI=70, RouteType=1 +message EVPNEthernetAutoDiscoveryRoute { + RouteDistinguisher rd = 1; + EthernetSegmentIdentifier esi = 2; + uint32 ethernet_tag = 3; + uint32 label = 4; +} + +// EVPNMACIPAdvertisementRoute represents the NLRI for: +// - AFI=25, SAFI=70, RouteType=2 +message EVPNMACIPAdvertisementRoute { + RouteDistinguisher rd = 1; + EthernetSegmentIdentifier esi = 2; + uint32 ethernet_tag = 3; + string mac_address = 4; + string ip_address = 5; + repeated uint32 labels = 6; +} + +// EVPNInclusiveMulticastEthernetTagRoute represents the NLRI for: +// - AFI=25, SAFI=70, RouteType=3 +message EVPNInclusiveMulticastEthernetTagRoute { + RouteDistinguisher rd = 1; + uint32 ethernet_tag = 2; + string ip_address = 3; +} + +// EVPNEthernetSegmentRoute represents the NLRI for: +// - AFI=25, SAFI=70, RouteType=4 +message EVPNEthernetSegmentRoute { + RouteDistinguisher rd = 1; + EthernetSegmentIdentifier esi = 2; + string ip_address = 3; +} + +// EVPNIPPrefixRoute represents the NLRI for: +// - AFI=25, SAFI=70, RouteType=5 +message EVPNIPPrefixRoute { + RouteDistinguisher rd = 1; + EthernetSegmentIdentifier esi = 2; + uint32 ethernet_tag = 3; + string ip_prefix = 4; + uint32 ip_prefix_len = 5; + string gw_address = 6; + uint32 label = 7; +} + +// EVPNIPMSIRoute represents the NLRI for: +// - AFI=25, SAFI=70, RouteType=9 +message EVPNIPMSIRoute { + RouteDistinguisher rd = 1; + uint32 ethernet_tag = 2; + RouteTarget rt = 3; +} + +// SRPolicyNLRI represents the NLRI for: +// - AFI=1, SAFI=73 +// - AFI=2, SAFI=73 +message SRPolicyNLRI { + // length field carries the length of NLRI portion expressed in bits + uint32 length = 1; + // distinguisher field carries 4-octet value uniquely identifying the policy + // in the context of tuple. + uint32 distinguisher = 2; + // color field carries 4-octet value identifying (with the endpoint) the + // policy. The color is used to match the color of the destination + // prefixes to steer traffic into the SR Policy + uint32 color = 3; + // endpoint field identifies the endpoint of a policy. The Endpoint may + // represent a single node or a set of nodes (e.g., an anycast + // address). The Endpoint is an IPv4 (4-octet) address or an IPv6 + // (16-octet) address according to the AFI of the NLRI. + bytes endpoint = 4; +} + +// LabeledVPNIPAddressPrefix represents the NLRI for: +// - AFI=1, SAFI=128 +// - AFI=2, SAFI=128 +message LabeledVPNIPAddressPrefix { + repeated uint32 labels = 1; + RouteDistinguisher rd = 2; + uint32 prefix_len = 3; + string prefix = 4; +} + +// RouteTargetMembershipNLRI represents the NLRI for: +// - AFI=1, SAFI=132 +message RouteTargetMembershipNLRI { + uint32 asn = 1; + RouteTarget rt = 2; +} + +message FlowSpecIPPrefix { + uint32 type = 1; + uint32 prefix_len = 2; + string prefix = 3; + // IPv6 only + uint32 offset = 4; +} + +message FlowSpecMAC { + uint32 type = 1; + string address = 2; +} + +message FlowSpecComponentItem { + // Operator for Numeric type, Operand for Bitmask type + uint32 op = 1; + uint64 value = 2; +} + +message FlowSpecComponent { + uint32 type = 1; + repeated FlowSpecComponentItem items = 2; +} + +message FlowSpecRule { + oneof rule { + FlowSpecIPPrefix ip_prefix = 1; + FlowSpecMAC mac = 2; + FlowSpecComponent component = 3; + } +} + +// FlowSpecNLRI represents the NLRI for: +// - AFI=1, SAFI=133 +// - AFI=2, SAFI=133 +message FlowSpecNLRI { + repeated FlowSpecRule rules = 1; +} + +// VPNFlowSpecNLRI represents the NLRI for: +// - AFI=1, SAFI=134 +// - AFI=2, SAFI=134 +// - AFI=25, SAFI=134 +message VPNFlowSpecNLRI { + RouteDistinguisher rd = 1; + repeated FlowSpecRule rules = 2; +} + +// OpaqueNLRI represents the NLRI for: +// - AFI=16397, SAFI=241 +message OpaqueNLRI { + bytes key = 1; + bytes value = 2; +} + +// Based om RFC 7752, Table 1. +enum LsNLRIType { + LS_NLRI_TYPE_UNSPECIFIED = 0; + LS_NLRI_TYPE_NODE = 1; + LS_NLRI_TYPE_LINK = 2; + LS_NLRI_TYPE_PREFIX_V4 = 3; + LS_NLRI_TYPE_PREFIX_V6 = 4; + LS_NLRI_TYPE_SRV6_SID = 6; +} + +enum LsProtocolID { + LS_PROTOCOL_ID_UNSPECIFIED = 0; + LS_PROTOCOL_ID_ISIS_L1 = 1; + LS_PROTOCOL_ID_ISIS_L2 = 2; + LS_PROTOCOL_ID_OSPF_V2 = 3; + LS_PROTOCOL_ID_DIRECT = 4; + LS_PROTOCOL_ID_STATIC = 5; + LS_PROTOCOL_ID_OSPF_V3 = 6; +} + +message LsNodeDescriptor { + uint32 asn = 1; + uint32 bgp_ls_id = 2; + uint32 ospf_area_id = 3; + bool pseudonode = 4; + string igp_router_id = 5; + string bgp_router_id = 6; + uint32 bgp_confederation_member = 7; +} + +message LsLinkDescriptor { + uint32 link_local_id = 1; + uint32 link_remote_id = 2; + string interface_addr_ipv4 = 3; + string neighbor_addr_ipv4 = 4; + string interface_addr_ipv6 = 5; + string neighbor_addr_ipv6 = 6; +} + +enum LsOspfRouteType { + LS_OSPF_ROUTE_TYPE_UNSPECIFIED = 0; + LS_OSPF_ROUTE_TYPE_INTRA_AREA = 1; + LS_OSPF_ROUTE_TYPE_INTER_AREA = 2; + LS_OSPF_ROUTE_TYPE_EXTERNAL1 = 3; + LS_OSPF_ROUTE_TYPE_EXTERNAL2 = 4; + LS_OSPF_ROUTE_TYPE_NSSA1 = 5; + LS_OSPF_ROUTE_TYPE_NSSA2 = 6; +} + +message LsPrefixDescriptor { + repeated string ip_reachability = 1; + LsOspfRouteType ospf_route_type = 2; +} + +message LsNodeNLRI { + LsNodeDescriptor local_node = 1; +} + +message LsLinkNLRI { + LsNodeDescriptor local_node = 1; + LsNodeDescriptor remote_node = 2; + LsLinkDescriptor link_descriptor = 3; +} + +message LsPrefixV4NLRI { + LsNodeDescriptor local_node = 1; + LsPrefixDescriptor prefix_descriptor = 2; +} + +message LsPrefixV6NLRI { + LsNodeDescriptor local_node = 1; + LsPrefixDescriptor prefix_descriptor = 2; +} + +// https://tools.ietf.org/html/rfc9552 +message LsSrv6SIDInformation { + repeated string sids = 1; +} + +message LsMultiTopologyIdentifier { + repeated uint32 multi_topo_ids = 1; +} + +// TODO: LsSrPolicyiCandidatePathNLRI +message LsSrv6SIDNLRI { + LsNodeDescriptor local_node = 1; + LsSrv6SIDInformation srv6_sid_information = 2; + LsMultiTopologyIdentifier multi_topo_id = 3; +} + +// LsAddrPrefix represents the NLRI for: +// - AFI=16388, SAFI=71 +message LsAddrPrefix { + LsNLRIType type = 1; + message LsNLRI { + oneof nlri { + LsNodeNLRI node = 1; + LsLinkNLRI link = 2; + LsPrefixV4NLRI prefix_v4 = 3; + LsPrefixV6NLRI prefix_v6 = 4; + LsSrv6SIDNLRI srv6_sid = 5; + } + } + LsNLRI nlri = 2; + uint32 length = 3; + LsProtocolID protocol_id = 4; + uint64 identifier = 5; +} + +message MUPInterworkSegmentDiscoveryRoute { + RouteDistinguisher rd = 1; + string prefix = 2; +} + +message MUPDirectSegmentDiscoveryRoute { + RouteDistinguisher rd = 1; + string address = 2; +} + +message MUPType1SessionTransformedRoute { + RouteDistinguisher rd = 1; + uint32 prefix_length = 2 [deprecated = true]; + string prefix = 3; + uint32 teid = 4; + uint32 qfi = 5; + uint32 endpoint_address_length = 6; + string endpoint_address = 7; + uint32 source_address_length = 8; + string source_address = 9; +} + +message MUPType2SessionTransformedRoute { + RouteDistinguisher rd = 1; + uint32 endpoint_address_length = 2; + string endpoint_address = 3; + uint32 teid = 4; +} diff --git a/fiberlb/crates/fiberlb-server/src/gobgp.rs b/fiberlb/crates/fiberlb-server/src/gobgp.rs new file mode 100644 index 0000000..48726ec --- /dev/null +++ b/fiberlb/crates/fiberlb-server/src/gobgp.rs @@ -0,0 +1,3 @@ +pub mod api { + tonic::include_proto!("api"); +} diff --git a/flaredb/crates/flaredb-client/examples/basic.rs b/flaredb/crates/flaredb-client/examples/basic.rs new file mode 100644 index 0000000..18ce1fc --- /dev/null +++ b/flaredb/crates/flaredb-client/examples/basic.rs @@ -0,0 +1,16 @@ +use flaredb_client::RdbClient; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Connect via PD (retry/backoff enabled by default). + let mut client = RdbClient::builder("127.0.0.1:2379") + .namespace("default") + .build() + .await?; + + client.raw_put(b"example".to_vec(), b"value".to_vec()).await?; + let val = client.raw_get(b"example".to_vec()).await?; + println!("Got: {:?}", val); + + Ok(()) +} diff --git a/flashdns/crates/flashdns-server/src/reverse_zone_service.rs b/flashdns/crates/flashdns-server/src/reverse_zone_service.rs new file mode 100644 index 0000000..6eb5823 --- /dev/null +++ b/flashdns/crates/flashdns-server/src/reverse_zone_service.rs @@ -0,0 +1,224 @@ +//! ReverseZoneService gRPC implementation + +use std::net::IpAddr; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::dns::ptr_patterns::apply_pattern; +use crate::metadata::DnsMetadataStore; +use flashdns_api::proto::{ + CreateReverseZoneRequest, DeleteReverseZoneRequest, DeleteReverseZoneResponse, + GetReverseZoneRequest, ListReverseZonesRequest, ListReverseZonesResponse, + ResolvePtrForIpRequest, ResolvePtrForIpResponse, ReverseZone as ProtoReverseZone, +}; +use flashdns_api::ReverseZoneService; +use flashdns_types::ReverseZone; +use ipnet::IpNet; +use tonic::{Request, Response, Status}; +use uuid::Uuid; + +/// ReverseZoneService implementation +pub struct ReverseZoneServiceImpl { + metadata: Arc, +} + +impl ReverseZoneServiceImpl { + /// Create a new ReverseZoneService with metadata store + pub fn new(metadata: Arc) -> Self { + Self { metadata } + } +} + +fn reverse_zone_to_proto(zone: &ReverseZone) -> ProtoReverseZone { + ProtoReverseZone { + id: zone.id.clone(), + org_id: zone.org_id.clone(), + project_id: zone.project_id.clone(), + cidr: zone.cidr.clone(), + arpa_zone: zone.arpa_zone.clone(), + ptr_pattern: zone.ptr_pattern.clone(), + ttl: zone.ttl, + created_at: zone.created_at, + updated_at: zone.updated_at, + } +} + +fn now_epoch() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +fn find_reverse_zone_for_ip(zones: &[ReverseZone], ip: IpAddr) -> Option { + let mut best_match: Option = None; + let mut best_prefix_len = 0; + + for zone in zones { + if let Ok(cidr) = zone.cidr.parse::() { + if cidr.contains(&ip) { + let prefix_len = cidr.prefix_len(); + if prefix_len > best_prefix_len { + best_prefix_len = prefix_len; + best_match = Some(zone.clone()); + } + } + } + } + + best_match +} + +#[tonic::async_trait] +impl ReverseZoneService for ReverseZoneServiceImpl { + async fn create_reverse_zone( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + if req.org_id.is_empty() { + return Err(Status::invalid_argument("org_id is required")); + } + if req.cidr.is_empty() { + return Err(Status::invalid_argument("cidr is required")); + } + if req.ptr_pattern.is_empty() { + return Err(Status::invalid_argument("ptr_pattern is required")); + } + + let existing = self + .metadata + .list_reverse_zones(&req.org_id, req.project_id.as_deref()) + .await + .map_err(|e| Status::internal(format!("metadata error: {}", e)))?; + + if existing.iter().any(|zone| zone.cidr == req.cidr) { + return Err(Status::already_exists("reverse zone already exists")); + } + + let now = now_epoch(); + let mut zone = ReverseZone { + id: Uuid::new_v4().to_string(), + org_id: req.org_id, + project_id: req.project_id, + cidr: req.cidr, + arpa_zone: String::new(), + ptr_pattern: req.ptr_pattern, + ttl: if req.ttl == 0 { 3600 } else { req.ttl }, + created_at: now, + updated_at: now, + }; + + zone = self + .metadata + .create_reverse_zone(zone) + .await + .map_err(|e| Status::internal(format!("failed to save reverse zone: {}", e)))?; + + Ok(Response::new(reverse_zone_to_proto(&zone))) + } + + async fn get_reverse_zone( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + if req.zone_id.is_empty() { + return Err(Status::invalid_argument("zone_id is required")); + } + + let zone = self + .metadata + .get_reverse_zone(&req.zone_id) + .await + .map_err(|e| Status::internal(format!("metadata error: {}", e)))? + .ok_or_else(|| Status::not_found("reverse zone not found"))?; + + Ok(Response::new(reverse_zone_to_proto(&zone))) + } + + async fn delete_reverse_zone( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + if req.zone_id.is_empty() { + return Err(Status::invalid_argument("zone_id is required")); + } + + let zone = self + .metadata + .get_reverse_zone(&req.zone_id) + .await + .map_err(|e| Status::internal(format!("metadata error: {}", e)))? + .ok_or_else(|| Status::not_found("reverse zone not found"))?; + + self.metadata + .delete_reverse_zone(&zone) + .await + .map_err(|e| Status::internal(format!("failed to delete reverse zone: {}", e)))?; + + Ok(Response::new(DeleteReverseZoneResponse { success: true })) + } + + async fn list_reverse_zones( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + if req.org_id.is_empty() { + return Err(Status::invalid_argument("org_id is required")); + } + + let zones = self + .metadata + .list_reverse_zones(&req.org_id, req.project_id.as_deref()) + .await + .map_err(|e| Status::internal(format!("metadata error: {}", e)))?; + + let proto_zones = zones.iter().map(reverse_zone_to_proto).collect(); + + Ok(Response::new(ListReverseZonesResponse { zones: proto_zones })) + } + + async fn resolve_ptr_for_ip( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + if req.ip_address.is_empty() { + return Err(Status::invalid_argument("ip_address is required")); + } + + let ip: IpAddr = req + .ip_address + .parse() + .map_err(|_| Status::invalid_argument("invalid ip_address"))?; + + let zones = self + .metadata + .list_all_reverse_zones() + .await + .map_err(|e| Status::internal(format!("metadata error: {}", e)))?; + + if let Some(zone) = find_reverse_zone_for_ip(&zones, ip) { + let ptr_value = apply_pattern(&zone.ptr_pattern, ip); + return Ok(Response::new(ResolvePtrForIpResponse { + ptr_record: Some(ptr_value), + reverse_zone_id: Some(zone.id), + found: true, + })); + } + + Ok(Response::new(ResolvePtrForIpResponse { + ptr_record: None, + reverse_zone_id: None, + found: false, + })) + } +} diff --git a/iam/crates/iam-api/src/credential_service.rs b/iam/crates/iam-api/src/credential_service.rs new file mode 100644 index 0000000..0aa5a2b --- /dev/null +++ b/iam/crates/iam-api/src/credential_service.rs @@ -0,0 +1,365 @@ +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use aes_gcm::{aead::Aead, Aes256Gcm, Key, KeyInit, Nonce}; +use argon2::{password_hash::{PasswordHasher, SaltString}, Argon2}; +use base64::{engine::general_purpose::STANDARD, Engine}; +use rand_core::{OsRng, RngCore}; +use tonic::{Request, Response, Status}; + +use iam_store::CredentialStore; +use iam_types::{Argon2Params, CredentialRecord}; + +use crate::proto::{ + iam_credential_server::IamCredential, CreateS3CredentialRequest, + CreateS3CredentialResponse, Credential, GetSecretKeyRequest, GetSecretKeyResponse, + ListCredentialsRequest, ListCredentialsResponse, RevokeCredentialRequest, + RevokeCredentialResponse, +}; + +fn now_ts() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +pub struct IamCredentialService { + store: Arc, + cipher: Aes256Gcm, + key_id: String, +} + +impl IamCredentialService { + pub fn new(store: Arc, master_key: &[u8], key_id: &str) -> Result { + if master_key.len() != 32 { + return Err(Status::failed_precondition( + "IAM_CRED_MASTER_KEY must be 32 bytes", + )); + } + let cipher = Aes256Gcm::new(Key::::from_slice(master_key)); + Ok(Self { + store, + cipher, + key_id: key_id.to_string(), + }) + } + + fn generate_secret() -> (String, Vec) { + let raw = uuid::Uuid::new_v4().as_bytes().to_vec(); + let secret_b64 = STANDARD.encode(&raw); + (secret_b64, raw) + } + + fn hash_secret(raw: &[u8]) -> (String, Argon2Params) { + let salt = SaltString::generate(&mut OsRng); + let argon2 = Argon2::default(); + let hash = argon2 + .hash_password(raw, &salt) + .expect("argon2 hash") + .to_string(); + let params = Argon2Params { + m_cost_kib: argon2.params().m_cost(), + t_cost: argon2.params().t_cost(), + p_cost: argon2.params().p_cost(), + salt_b64: salt.to_string(), + }; + (hash, params) + } + + fn encrypt_secret(&self, raw: &[u8]) -> Result { + let mut nonce_bytes = [0u8; 12]; + OsRng.fill_bytes(&mut nonce_bytes); + let nonce = Nonce::from_slice(&nonce_bytes); + let ciphertext = self + .cipher + .encrypt(nonce, raw) + .map_err(|e| Status::internal(format!("encrypt secret: {}", e)))?; + let mut combined = nonce_bytes.to_vec(); + combined.extend_from_slice(&ciphertext); + Ok(STANDARD.encode(combined)) + } + + fn decrypt_secret(&self, enc_b64: &str) -> Result, Status> { + let data = STANDARD + .decode(enc_b64) + .map_err(|e| Status::internal(format!("invalid b64: {}", e)))?; + if data.len() < 12 { + return Err(Status::internal("ciphertext too short")); + } + let (nonce_bytes, ct) = data.split_at(12); + let nonce = Nonce::from_slice(nonce_bytes); + self.cipher + .decrypt(nonce, ct) + .map_err(|e| Status::internal(format!("decrypt failed: {}", e))) + } +} + +#[tonic::async_trait] +impl IamCredential for IamCredentialService { + async fn create_s3_credential( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let now = now_ts(); + let (secret_b64, raw_secret) = Self::generate_secret(); + let (hash, kdf) = Self::hash_secret(&raw_secret); + let secret_enc = self.encrypt_secret(&raw_secret)?; + + let access_key_id = format!("ak_{}", uuid::Uuid::new_v4()); + let record = CredentialRecord { + access_key_id: access_key_id.clone(), + principal_id: req.principal_id.clone(), + created_at: now, + expires_at: req.expires_at, + revoked: false, + description: if req.description.is_empty() { + None + } else { + Some(req.description) + }, + secret_hash: hash, + secret_enc, + key_id: self.key_id.clone(), + version: 1, + kdf, + }; + + self.store + .put(&record) + .await + .map_err(|e| Status::internal(format!("store credential: {}", e)))?; + + Ok(Response::new(CreateS3CredentialResponse { + access_key_id, + secret_key: secret_b64, + created_at: now, + expires_at: req.expires_at, + })) + } + + async fn get_secret_key( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let record = match self.store.get(&req.access_key_id).await { + Ok(Some((rec, _))) => rec, + Ok(None) => return Err(Status::not_found("access key not found")), + Err(e) => { + return Err(Status::internal(format!( + "failed to load credential: {}", + e + ))) + } + }; + if record.revoked { + return Err(Status::permission_denied("access key revoked")); + } + if let Some(exp) = record.expires_at { + if now_ts() > exp { + return Err(Status::permission_denied("access key expired")); + } + } + let secret = self.decrypt_secret(&record.secret_enc)?; + + Ok(Response::new(GetSecretKeyResponse { + secret_key: STANDARD.encode(secret), + principal_id: record.principal_id, + expires_at: record.expires_at, + })) + } + + async fn list_credentials( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let items = self + .store + .list_for_principal(&req.principal_id, 1000) + .await + .map_err(|e| Status::internal(format!("list credentials: {}", e)))?; + let creds: Vec = items + .into_iter() + .map(|c| Credential { + access_key_id: c.access_key_id, + principal_id: c.principal_id, + created_at: c.created_at, + expires_at: c.expires_at, + revoked: c.revoked, + description: c.description.unwrap_or_default(), + }) + .collect(); + Ok(Response::new(ListCredentialsResponse { credentials: creds })) + } + + async fn revoke_credential( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let revoked = self + .store + .revoke(&req.access_key_id) + .await + .map_err(|e| Status::internal(format!("revoke: {}", e)))?; + Ok(Response::new(RevokeCredentialResponse { success: revoked })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use base64::engine::general_purpose::STANDARD; + use iam_store::Backend; + + fn test_service() -> IamCredentialService { + let backend = Arc::new(Backend::memory()); + let store = Arc::new(CredentialStore::new(backend)); + let master_key = [0x42u8; 32]; + IamCredentialService::new(store, &master_key, "test-key").unwrap() + } + + #[tokio::test] + async fn create_and_get_roundtrip() { + let svc = test_service(); + let create = svc + .create_s3_credential(Request::new(CreateS3CredentialRequest { + principal_id: "p1".into(), + description: "".into(), + expires_at: None, + })) + .await + .unwrap() + .into_inner(); + + let get = svc + .get_secret_key(Request::new(GetSecretKeyRequest { + access_key_id: create.access_key_id.clone(), + })) + .await + .unwrap() + .into_inner(); + + let orig = STANDARD.decode(create.secret_key).unwrap(); + let fetched = STANDARD.decode(get.secret_key).unwrap(); + assert_eq!(orig, fetched); + assert_eq!(get.principal_id, "p1"); + } + + #[tokio::test] + async fn list_filters_by_principal() { + let svc = test_service(); + let a = svc + .create_s3_credential(Request::new(CreateS3CredentialRequest { + principal_id: "pA".into(), + description: "".into(), + expires_at: None, + })) + .await + .unwrap() + .into_inner(); + let _b = svc + .create_s3_credential(Request::new(CreateS3CredentialRequest { + principal_id: "pB".into(), + description: "".into(), + expires_at: None, + })) + .await + .unwrap(); + + let list_a = svc + .list_credentials(Request::new(ListCredentialsRequest { + principal_id: "pA".into(), + })) + .await + .unwrap() + .into_inner(); + assert_eq!(list_a.credentials.len(), 1); + assert_eq!(list_a.credentials[0].access_key_id, a.access_key_id); + } + + #[tokio::test] + async fn revoke_blocks_get() { + let svc = test_service(); + let created = svc + .create_s3_credential(Request::new(CreateS3CredentialRequest { + principal_id: "p1".into(), + description: "".into(), + expires_at: None, + })) + .await + .unwrap() + .into_inner(); + + let revoke1 = svc + .revoke_credential(Request::new(RevokeCredentialRequest { + access_key_id: created.access_key_id.clone(), + reason: "test".into(), + })) + .await + .unwrap() + .into_inner(); + assert!(revoke1.success); + + let revoke2 = svc + .revoke_credential(Request::new(RevokeCredentialRequest { + access_key_id: created.access_key_id.clone(), + reason: "again".into(), + })) + .await + .unwrap() + .into_inner(); + assert!(!revoke2.success); + + let err = svc + .get_secret_key(Request::new(GetSecretKeyRequest { + access_key_id: created.access_key_id, + })) + .await + .unwrap_err(); + assert_eq!(err.code(), Status::permission_denied("").code()); + } + + #[tokio::test] + async fn expired_key_is_denied() { + let svc = test_service(); + // Manually insert an expired record + let expired = CredentialRecord { + access_key_id: "expired-ak".into(), + principal_id: "p1".into(), + created_at: now_ts(), + expires_at: Some(now_ts() - 10), + revoked: false, + description: None, + secret_hash: "hash".into(), + secret_enc: STANDARD.encode(b"dead"), + key_id: "k".into(), + version: 1, + kdf: Argon2Params { + m_cost_kib: 19456, + t_cost: 2, + p_cost: 1, + salt_b64: "c2FsdA==".into(), + }, + }; + svc.store.put(&expired).await.unwrap(); + let err = svc + .get_secret_key(Request::new(GetSecretKeyRequest { + access_key_id: "expired-ak".into(), + })) + .await + .unwrap_err(); + assert_eq!(err.code(), Status::permission_denied("").code()); + } + + #[test] + fn master_key_length_enforced() { + let backend = Arc::new(Backend::memory()); + let store = Arc::new(CredentialStore::new(backend)); + let bad = IamCredentialService::new(store.clone(), &[0u8; 16], "k"); + assert!(bad.is_err()); + } +} diff --git a/iam/crates/iam-api/src/gateway_auth_service.rs b/iam/crates/iam-api/src/gateway_auth_service.rs new file mode 100644 index 0000000..97007ba --- /dev/null +++ b/iam/crates/iam-api/src/gateway_auth_service.rs @@ -0,0 +1,433 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use apigateway_api::proto::{AuthorizeRequest, AuthorizeResponse, Subject}; +use apigateway_api::GatewayAuthService; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use iam_authz::{AuthzContext, AuthzDecision, AuthzRequest, PolicyEvaluator}; +use iam_authn::InternalTokenService; +use iam_store::{PrincipalStore, TokenStore}; +use iam_types::{InternalTokenClaims, Principal, PrincipalRef, Resource}; +use sha2::{Digest, Sha256}; +use tonic::{Request, Response, Status}; + +pub struct GatewayAuthServiceImpl { + token_service: Arc, + principal_store: Arc, + token_store: Arc, + evaluator: Arc, +} + +impl GatewayAuthServiceImpl { + pub fn new( + token_service: Arc, + principal_store: Arc, + token_store: Arc, + evaluator: Arc, + ) -> Self { + Self { + token_service, + principal_store, + token_store, + evaluator, + } + } + + async fn check_token_revoked( + &self, + principal_id: &str, + token: &str, + ) -> Result, Status> { + let token_id = compute_token_id(token); + let meta = self + .token_store + .get(principal_id, &token_id) + .await + .map_err(|e| Status::internal(format!("token store error: {}", e)))?; + + if let Some((meta, _)) = meta { + if meta.revoked { + let reason = meta + .revocation_reason + .unwrap_or_else(|| "token revoked".to_string()); + return Ok(Some(reason)); + } + } + + Ok(None) + } +} + +#[tonic::async_trait] +impl GatewayAuthService for GatewayAuthServiceImpl { + async fn authorize( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let token = req.token.trim(); + + if token.is_empty() { + return Ok(Response::new(deny_response("missing token"))); + } + + let claims = match self.token_service.verify(token).await { + Ok(claims) => claims, + Err(err) => return Ok(Response::new(deny_response(err.to_string()))), + }; + + if let Some(reason) = self.check_token_revoked(&claims.principal_id, token).await? { + return Ok(Response::new(deny_response(reason))); + } + + let principal_ref = PrincipalRef::new(claims.principal_kind.clone(), &claims.principal_id); + let principal = match self.principal_store.get(&principal_ref).await { + Ok(Some(principal)) => principal, + Ok(None) => return Ok(Response::new(deny_response("principal not found"))), + Err(err) => { + return Err(Status::internal(format!( + "failed to read principal: {}", + err + ))) + } + }; + + if !principal.enabled { + return Ok(Response::new(deny_response("principal disabled"))); + } + + let (action, resource, context, org_id, project_id) = + build_authz_request(&req, &claims, &principal); + let authz_request = + AuthzRequest::new(principal.clone(), action, resource).with_context(context); + let decision = self + .evaluator + .evaluate(&authz_request) + .await + .map_err(|e| Status::internal(format!("authz evaluation failed: {}", e)))?; + + match decision { + AuthzDecision::Allow => {} + AuthzDecision::Deny { reason } => { + return Ok(Response::new(deny_response(reason))); + } + } + + let subject = Subject { + subject_id: claims.principal_id.clone(), + org_id, + project_id, + roles: claims.roles.clone(), + scopes: vec![claims.scope.to_string()], + }; + + let ttl_seconds = ttl_from_claims(claims.exp); + let mut headers = HashMap::new(); + headers.insert("x-iam-session-id".to_string(), claims.session_id.clone()); + headers.insert( + "x-iam-principal-kind".to_string(), + claims.principal_kind.to_string(), + ); + headers.insert( + "x-iam-auth-method".to_string(), + claims.auth_method.to_string(), + ); + + Ok(Response::new(AuthorizeResponse { + allow: true, + reason: String::new(), + subject: Some(subject), + headers, + ttl_seconds, + })) + } +} + +fn deny_response(reason: impl Into) -> AuthorizeResponse { + AuthorizeResponse { + allow: false, + reason: reason.into(), + subject: None, + headers: HashMap::new(), + ttl_seconds: 0, + } +} + +fn compute_token_id(token: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(token.as_bytes()); + let digest = hasher.finalize(); + URL_SAFE_NO_PAD.encode(digest) +} + +fn ttl_from_claims(exp: u64) -> u32 { + let now = now_ts(); + let remaining = exp.saturating_sub(now); + u32::try_from(remaining).unwrap_or(u32::MAX) +} + +fn build_authz_request( + req: &AuthorizeRequest, + claims: &InternalTokenClaims, + principal: &Principal, +) -> (String, Resource, AuthzContext, String, String) { + let action = action_for_request(req); + let (org_id, project_id) = resolve_org_project(req, claims, principal); + let mut resource = Resource::new( + "gateway_route", + resource_id_for_request(req), + org_id.clone(), + project_id.clone(), + ); + resource = resource + .with_tag("route", req.route_name.clone()) + .with_tag("method", req.method.clone()) + .with_tag("path", req.path.clone()); + if !req.raw_query.is_empty() { + resource = resource.with_tag("raw_query", req.raw_query.clone()); + } + + let mut context = AuthzContext::new() + .with_http_method(req.method.clone()) + .with_request_path(req.path.clone()) + .with_metadata("route", req.route_name.clone()) + .with_metadata("request_id", req.request_id.clone()) + .with_metadata("org_id", org_id.clone()) + .with_metadata("project_id", project_id.clone()); + if !req.raw_query.is_empty() { + context = context.with_metadata("raw_query", req.raw_query.clone()); + } + if let Ok(ip) = req.client_ip.parse() { + context = context.with_source_ip(ip); + } + + (action, resource, context, org_id, project_id) +} + +fn action_for_request(req: &AuthorizeRequest) -> String { + let route = if req.route_name.trim().is_empty() { + "gateway" + } else { + req.route_name.trim() + }; + let verb = method_to_verb(&req.method); + format!("gateway:{}:{}", normalize_action_component(route), verb) +} + +fn method_to_verb(method: &str) -> &'static str { + match method.trim().to_uppercase().as_str() { + "GET" | "HEAD" => "read", + "POST" => "create", + "PUT" | "PATCH" => "update", + "DELETE" => "delete", + "OPTIONS" => "list", + _ => "execute", + } +} + +fn normalize_action_component(value: &str) -> String { + value + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' { + ch.to_ascii_lowercase() + } else { + '_' + } + }) + .collect() +} + +fn resource_id_for_request(req: &AuthorizeRequest) -> String { + if !req.route_name.trim().is_empty() { + return req.route_name.trim().to_string(); + } + let path = req.path.trim_matches('/'); + if path.is_empty() { + "root".to_string() + } else { + path.replace('/', ":") + } +} + +fn resolve_org_project( + req: &AuthorizeRequest, + claims: &InternalTokenClaims, + principal: &Principal, +) -> (String, String) { + let org_id = claims + .org_id + .clone() + .or_else(|| claims.scope.org_id().map(|value| value.to_string())) + .or_else(|| principal.org_id.clone()) + .or_else(|| header_value(&req.headers, "x-org-id")) + .unwrap_or_else(|| "system".to_string()); + + let project_id = claims + .project_id + .clone() + .or_else(|| claims.scope.project_id().map(|value| value.to_string())) + .or_else(|| principal.project_id.clone()) + .or_else(|| header_value(&req.headers, "x-project-id")) + .unwrap_or_else(|| "system".to_string()); + + (org_id, project_id) +} + +fn header_value(headers: &HashMap, key: &str) -> Option { + headers + .get(&key.to_ascii_lowercase()) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +fn now_ts() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +#[cfg(test)] +mod tests { + use super::*; + use iam_authn::{InternalTokenConfig, SigningKey}; + use iam_authz::{PolicyCache, PolicyEvaluator}; + use iam_store::{Backend, BackendConfig, BindingStore, PrincipalStore, RoleStore, TokenStore}; + use iam_types::{ + Permission, PolicyBinding, Principal, PrincipalRef, Role, Scope, TokenMetadata, TokenType, + }; + use std::time::Duration; + + fn make_request(token: &str) -> AuthorizeRequest { + AuthorizeRequest { + request_id: "req-1".into(), + token: token.to_string(), + method: "GET".into(), + path: "/v1/example".into(), + raw_query: "".into(), + headers: HashMap::new(), + client_ip: "127.0.0.1".into(), + route_name: "example".into(), + } + } + + async fn build_service() -> ( + GatewayAuthServiceImpl, + Arc, + Arc, + Arc, + Arc, + Principal, + ) { + let backend = Arc::new(Backend::new(BackendConfig::Memory).await.unwrap()); + let principal_store = Arc::new(PrincipalStore::new(backend.clone())); + let role_store = Arc::new(RoleStore::new(backend.clone())); + let binding_store = Arc::new(BindingStore::new(backend.clone())); + let token_store = Arc::new(TokenStore::new(backend)); + let signing_key = SigningKey::generate("test-key-1"); + let token_config = InternalTokenConfig::new(signing_key, "iam-test") + .with_default_ttl(Duration::from_secs(3600)) + .with_max_ttl(Duration::from_secs(7200)); + let token_service = Arc::new(InternalTokenService::new(token_config)); + let mut principal = Principal::new_user("user-1", "User One"); + principal.org_id = Some("org-1".into()); + principal.project_id = Some("proj-1".into()); + principal_store.create(&principal).await.unwrap(); + let cache = Arc::new(PolicyCache::default_config()); + let evaluator = Arc::new(PolicyEvaluator::new( + binding_store.clone(), + role_store.clone(), + cache, + )); + let service = GatewayAuthServiceImpl::new( + token_service.clone(), + principal_store.clone(), + token_store.clone(), + evaluator, + ); + (service, token_service, role_store, binding_store, token_store, principal) + } + + #[tokio::test] + async fn test_authorize_missing_token_denies() { + let (service, _, _, _, _, _) = build_service().await; + let response = service + .authorize(Request::new(make_request(""))) + .await + .unwrap() + .into_inner(); + assert!(!response.allow); + assert!(response.reason.contains("missing token")); + } + + #[tokio::test] + async fn test_authorize_valid_token_allows() { + let (service, token_service, role_store, binding_store, _, principal) = + build_service().await; + let role = Role::new( + "GatewayReader", + Scope::project("proj-1", "org-1"), + vec![Permission::new("gateway:example:read", "*")], + ); + role_store.create(&role).await.unwrap(); + let binding = PolicyBinding::new( + "binding-1", + PrincipalRef::new(principal.kind.clone(), principal.id.clone()), + role.to_ref(), + Scope::project("proj-1", "org-1"), + ); + binding_store.create(&binding).await.unwrap(); + let issued = token_service + .issue(&principal, vec!["role-1".into()], Scope::system(), None) + .await + .unwrap(); + let response = service + .authorize(Request::new(make_request(&issued.token))) + .await + .unwrap() + .into_inner(); + assert!(response.allow); + let subject = response.subject.expect("subject"); + assert_eq!(subject.subject_id, principal.id); + assert_eq!(subject.roles, vec!["role-1".to_string()]); + assert_eq!(subject.scopes, vec!["system".to_string()]); + assert!(response.ttl_seconds > 0); + } + + #[tokio::test] + async fn test_authorize_revoked_token_denies() { + let (service, token_service, _, _, token_store, principal) = build_service().await; + let issued = token_service + .issue(&principal, vec![], Scope::system(), None) + .await + .unwrap(); + let token_id = compute_token_id(&issued.token); + let meta = TokenMetadata::new( + &token_id, + &issued.claims.principal_id, + TokenType::Access, + issued.claims.iat, + issued.claims.exp, + ); + token_store.put(&meta).await.unwrap(); + token_store + .revoke( + &issued.claims.principal_id, + &token_id, + "test revoke", + now_ts(), + ) + .await + .unwrap(); + + let response = service + .authorize(Request::new(make_request(&issued.token))) + .await + .unwrap() + .into_inner(); + assert!(!response.allow); + assert!(response.reason.contains("revoke")); + } +} diff --git a/iam/crates/iam-client/examples/basic.rs b/iam/crates/iam-client/examples/basic.rs new file mode 100644 index 0000000..a009491 --- /dev/null +++ b/iam/crates/iam-client/examples/basic.rs @@ -0,0 +1,14 @@ +use iam_client::IamClientBuilder; +use photocloud_client_common::AuthConfig; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Build IAM client with optional bearer auth. + let client = IamClientBuilder::new("https://127.0.0.1:2443") + .auth(AuthConfig::None) + .build() + .await?; + + println!("IAM client ready"); + Ok(()) +} diff --git a/iam/crates/iam-store/src/credential_store.rs b/iam/crates/iam-store/src/credential_store.rs new file mode 100644 index 0000000..9e2719e --- /dev/null +++ b/iam/crates/iam-store/src/credential_store.rs @@ -0,0 +1,68 @@ +//! Credential storage (access/secret key metadata) + +use iam_types::{CredentialRecord, Result}; + +use crate::backend::JsonStore; +use crate::{DynMetadataClient, MetadataClient}; + +/// Store for credentials (S3/API keys) +pub struct CredentialStore { + client: DynMetadataClient, +} + +impl JsonStore for CredentialStore { + fn client(&self) -> &dyn MetadataClient { + self.client.as_ref() + } +} + +impl CredentialStore { + pub fn new(client: DynMetadataClient) -> Self { + Self { client } + } + + pub async fn put(&self, record: &CredentialRecord) -> Result { + let key = CredentialRecord::storage_key(&record.access_key_id); + self.put_json(key.as_bytes(), record).await + } + + pub async fn get(&self, access_key_id: &str) -> Result> { + let key = CredentialRecord::storage_key(access_key_id); + self.get_json(key.as_bytes()).await + } + + pub async fn list_for_principal( + &self, + principal_id: &str, + limit: u32, + ) -> Result> { + // scan prefix and filter by principal_id; small cardinality expected + let prefix = b"iam/credentials/"; + let items = self.scan_prefix_json::(prefix, limit).await?; + Ok(items + .into_iter() + .filter(|rec| rec.principal_id == principal_id) + .collect()) + } + + pub async fn revoke(&self, access_key_id: &str) -> Result { + let key = CredentialRecord::storage_key(access_key_id); + let current = self.get_json::(key.as_bytes()).await?; + let (mut record, version) = match current { + Some(v) => v, + None => return Ok(false), + }; + if record.revoked { + return Ok(false); + } + record.revoked = true; + match self + .cas_json(key.as_bytes(), version, &record) + .await? + { + crate::CasResult::Success(_) => Ok(true), + crate::CasResult::Conflict { .. } => Ok(false), + crate::CasResult::NotFound => Ok(false), + } + } +} diff --git a/iam/crates/iam-types/src/credential.rs b/iam/crates/iam-types/src/credential.rs new file mode 100644 index 0000000..817bb9c --- /dev/null +++ b/iam/crates/iam-types/src/credential.rs @@ -0,0 +1,35 @@ +//! Credential metadata for access/secret keys + +use serde::{Deserialize, Serialize}; + +/// Argon2 parameters used to hash the secret key +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Argon2Params { + pub m_cost_kib: u32, + pub t_cost: u32, + pub p_cost: u32, + /// Salt in base64 + pub salt_b64: String, +} + +/// Stored record for an IAM credential +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct CredentialRecord { + pub access_key_id: String, + pub principal_id: String, + pub created_at: u64, + pub expires_at: Option, + pub revoked: bool, + pub description: Option, + pub secret_hash: String, + pub secret_enc: String, + pub key_id: String, + pub version: u32, + pub kdf: Argon2Params, +} + +impl CredentialRecord { + pub fn storage_key(access_key_id: &str) -> String { + format!("iam/credentials/{}", access_key_id) + } +} diff --git a/k8shost/crates/k8shost-csi/build.rs b/k8shost/crates/k8shost-csi/build.rs new file mode 100644 index 0000000..6f53ac8 --- /dev/null +++ b/k8shost/crates/k8shost-csi/build.rs @@ -0,0 +1,10 @@ +fn main() -> Result<(), Box> { + let protoc_path = protoc_bin_vendored::protoc_bin_path()?; + std::env::set_var("PROTOC", protoc_path); + + tonic_build::configure() + .build_server(true) + .build_client(true) + .compile(&["proto/csi.proto"], &["proto"])?; + Ok(()) +} diff --git a/k8shost/crates/k8shost-csi/proto/csi.proto b/k8shost/crates/k8shost-csi/proto/csi.proto new file mode 100644 index 0000000..732ed14 --- /dev/null +++ b/k8shost/crates/k8shost-csi/proto/csi.proto @@ -0,0 +1,1914 @@ +// Code generated by make; DO NOT EDIT. +syntax = "proto3"; +package csi.v1; + +import "google/protobuf/descriptor.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +option go_package = "csi"; + +extend google.protobuf.EnumOptions { + // Indicates that this enum is OPTIONAL and part of an experimental + // API that may be deprecated and eventually removed between minor + // releases. + bool alpha_enum = 1060; +} +extend google.protobuf.EnumValueOptions { + // Indicates that this enum value is OPTIONAL and part of an + // experimental API that may be deprecated and eventually removed + // between minor releases. + bool alpha_enum_value = 1060; +} +extend google.protobuf.FieldOptions { + // Indicates that a field MAY contain information that is sensitive + // and MUST be treated as such (e.g. not logged). + bool csi_secret = 1059; + + // Indicates that this field is OPTIONAL and part of an experimental + // API that may be deprecated and eventually removed between minor + // releases. + bool alpha_field = 1060; +} +extend google.protobuf.MessageOptions { + // Indicates that this message is OPTIONAL and part of an experimental + // API that may be deprecated and eventually removed between minor + // releases. + bool alpha_message = 1060; +} +extend google.protobuf.MethodOptions { + // Indicates that this method is OPTIONAL and part of an experimental + // API that may be deprecated and eventually removed between minor + // releases. + bool alpha_method = 1060; +} +extend google.protobuf.ServiceOptions { + // Indicates that this service is OPTIONAL and part of an experimental + // API that may be deprecated and eventually removed between minor + // releases. + bool alpha_service = 1060; +} +service Identity { + rpc GetPluginInfo(GetPluginInfoRequest) + returns (GetPluginInfoResponse) {} + + rpc GetPluginCapabilities(GetPluginCapabilitiesRequest) + returns (GetPluginCapabilitiesResponse) {} + + rpc Probe (ProbeRequest) + returns (ProbeResponse) {} +} + +service Controller { + rpc CreateVolume (CreateVolumeRequest) + returns (CreateVolumeResponse) {} + + rpc DeleteVolume (DeleteVolumeRequest) + returns (DeleteVolumeResponse) {} + + rpc ControllerPublishVolume (ControllerPublishVolumeRequest) + returns (ControllerPublishVolumeResponse) {} + + rpc ControllerUnpublishVolume (ControllerUnpublishVolumeRequest) + returns (ControllerUnpublishVolumeResponse) {} + + rpc ValidateVolumeCapabilities (ValidateVolumeCapabilitiesRequest) + returns (ValidateVolumeCapabilitiesResponse) {} + + rpc ListVolumes (ListVolumesRequest) + returns (ListVolumesResponse) {} + + rpc GetCapacity (GetCapacityRequest) + returns (GetCapacityResponse) {} + + rpc ControllerGetCapabilities (ControllerGetCapabilitiesRequest) + returns (ControllerGetCapabilitiesResponse) {} + + rpc CreateSnapshot (CreateSnapshotRequest) + returns (CreateSnapshotResponse) {} + + rpc DeleteSnapshot (DeleteSnapshotRequest) + returns (DeleteSnapshotResponse) {} + + rpc ListSnapshots (ListSnapshotsRequest) + returns (ListSnapshotsResponse) {} + + rpc ControllerExpandVolume (ControllerExpandVolumeRequest) + returns (ControllerExpandVolumeResponse) {} + + rpc ControllerGetVolume (ControllerGetVolumeRequest) + returns (ControllerGetVolumeResponse) { + option (alpha_method) = true; + } + + rpc ControllerModifyVolume (ControllerModifyVolumeRequest) + returns (ControllerModifyVolumeResponse) { + option (alpha_method) = true; + } +} + +service GroupController { + option (alpha_service) = true; + + rpc GroupControllerGetCapabilities ( + GroupControllerGetCapabilitiesRequest) + returns (GroupControllerGetCapabilitiesResponse) {} + + rpc CreateVolumeGroupSnapshot(CreateVolumeGroupSnapshotRequest) + returns (CreateVolumeGroupSnapshotResponse) { + option (alpha_method) = true; + } + + rpc DeleteVolumeGroupSnapshot(DeleteVolumeGroupSnapshotRequest) + returns (DeleteVolumeGroupSnapshotResponse) { + option (alpha_method) = true; + } + + rpc GetVolumeGroupSnapshot( + GetVolumeGroupSnapshotRequest) + returns (GetVolumeGroupSnapshotResponse) { + option (alpha_method) = true; + } +} + +service Node { + rpc NodeStageVolume (NodeStageVolumeRequest) + returns (NodeStageVolumeResponse) {} + + rpc NodeUnstageVolume (NodeUnstageVolumeRequest) + returns (NodeUnstageVolumeResponse) {} + + rpc NodePublishVolume (NodePublishVolumeRequest) + returns (NodePublishVolumeResponse) {} + + rpc NodeUnpublishVolume (NodeUnpublishVolumeRequest) + returns (NodeUnpublishVolumeResponse) {} + + rpc NodeGetVolumeStats (NodeGetVolumeStatsRequest) + returns (NodeGetVolumeStatsResponse) {} + + + rpc NodeExpandVolume(NodeExpandVolumeRequest) + returns (NodeExpandVolumeResponse) {} + + + rpc NodeGetCapabilities (NodeGetCapabilitiesRequest) + returns (NodeGetCapabilitiesResponse) {} + + rpc NodeGetInfo (NodeGetInfoRequest) + returns (NodeGetInfoResponse) {} +} +message GetPluginInfoRequest { + // Intentionally empty. +} + +message GetPluginInfoResponse { + // The name MUST follow domain name notation format + // (https://tools.ietf.org/html/rfc1035#section-2.3.1). It SHOULD + // include the plugin's host company name and the plugin name, + // to minimize the possibility of collisions. It MUST be 63 + // characters or less, beginning and ending with an alphanumeric + // character ([a-z0-9A-Z]) with dashes (-), dots (.), and + // alphanumerics between. This field is REQUIRED. + string name = 1; + + // This field is REQUIRED. Value of this field is opaque to the CO. + string vendor_version = 2; + + // This field is OPTIONAL. Values are opaque to the CO. + map manifest = 3; +} +message GetPluginCapabilitiesRequest { + // Intentionally empty. +} + +message GetPluginCapabilitiesResponse { + // All the capabilities that the controller service supports. This + // field is OPTIONAL. + repeated PluginCapability capabilities = 1; +} + +// Specifies a capability of the plugin. +message PluginCapability { + message Service { + enum Type { + UNKNOWN = 0; + // CONTROLLER_SERVICE indicates that the Plugin provides RPCs for + // the ControllerService. Plugins SHOULD provide this capability. + // In rare cases certain plugins MAY wish to omit the + // ControllerService entirely from their implementation, but such + // SHOULD NOT be the common case. + // The presence of this capability determines whether the CO will + // attempt to invoke the REQUIRED ControllerService RPCs, as well + // as specific RPCs as indicated by ControllerGetCapabilities. + CONTROLLER_SERVICE = 1; + + // VOLUME_ACCESSIBILITY_CONSTRAINTS indicates that the volumes for + // this plugin MAY NOT be equally accessible by all nodes in the + // cluster. The CO MUST use the topology information returned by + // CreateVolumeRequest along with the topology information + // returned by NodeGetInfo to ensure that a given volume is + // accessible from a given node when scheduling workloads. + VOLUME_ACCESSIBILITY_CONSTRAINTS = 2; + + // GROUP_CONTROLLER_SERVICE indicates that the Plugin provides + // RPCs for operating on groups of volumes. Plugins MAY provide + // this capability. + // The presence of this capability determines whether the CO will + // attempt to invoke the REQUIRED GroupController service RPCs, as + // well as specific RPCs as indicated by + // GroupControllerGetCapabilities. + GROUP_CONTROLLER_SERVICE = 3 [(alpha_enum_value) = true]; + } + Type type = 1; + } + + message VolumeExpansion { + enum Type { + UNKNOWN = 0; + + // ONLINE indicates that volumes may be expanded when published to + // a node. When a Plugin implements this capability it MUST + // implement either the EXPAND_VOLUME controller capability or the + // EXPAND_VOLUME node capability or both. When a plugin supports + // ONLINE volume expansion and also has the EXPAND_VOLUME + // controller capability then the plugin MUST support expansion of + // volumes currently published and available on a node. When a + // plugin supports ONLINE volume expansion and also has the + // EXPAND_VOLUME node capability then the plugin MAY support + // expansion of node-published volume via NodeExpandVolume. + // + // Example 1: Given a shared filesystem volume (e.g. GlusterFs), + // the Plugin may set the ONLINE volume expansion capability and + // implement ControllerExpandVolume but not NodeExpandVolume. + // + // Example 2: Given a block storage volume type (e.g. EBS), the + // Plugin may set the ONLINE volume expansion capability and + // implement both ControllerExpandVolume and NodeExpandVolume. + // + // Example 3: Given a Plugin that supports volume expansion only + // upon a node, the Plugin may set the ONLINE volume + // expansion capability and implement NodeExpandVolume but not + // ControllerExpandVolume. + ONLINE = 1; + + // OFFLINE indicates that volumes currently published and + // available on a node SHALL NOT be expanded via + // ControllerExpandVolume. When a plugin supports OFFLINE volume + // expansion it MUST implement either the EXPAND_VOLUME controller + // capability or both the EXPAND_VOLUME controller capability and + // the EXPAND_VOLUME node capability. + // + // Example 1: Given a block storage volume type (e.g. Azure Disk) + // that does not support expansion of "node-attached" (i.e. + // controller-published) volumes, the Plugin may indicate + // OFFLINE volume expansion support and implement both + // ControllerExpandVolume and NodeExpandVolume. + OFFLINE = 2; + } + Type type = 1; + } + + oneof type { + // Service that the plugin supports. + Service service = 1; + VolumeExpansion volume_expansion = 2; + } +} +message ProbeRequest { + // Intentionally empty. +} + +message ProbeResponse { + // Readiness allows a plugin to report its initialization status back + // to the CO. Initialization for some plugins MAY be time consuming + // and it is important for a CO to distinguish between the following + // cases: + // + // 1) The plugin is in an unhealthy state and MAY need restarting. In + // this case a gRPC error code SHALL be returned. + // 2) The plugin is still initializing, but is otherwise perfectly + // healthy. In this case a successful response SHALL be returned + // with a readiness value of `false`. Calls to the plugin's + // Controller and/or Node services MAY fail due to an incomplete + // initialization state. + // 3) The plugin has finished initializing and is ready to service + // calls to its Controller and/or Node services. A successful + // response is returned with a readiness value of `true`. + // + // This field is OPTIONAL. If not present, the caller SHALL assume + // that the plugin is in a ready state and is accepting calls to its + // Controller and/or Node services (according to the plugin's reported + // capabilities). + .google.protobuf.BoolValue ready = 1; +} +message CreateVolumeRequest { + // The suggested name for the storage space. This field is REQUIRED. + // It serves two purposes: + // 1) Idempotency - This name is generated by the CO to achieve + // idempotency. The Plugin SHOULD ensure that multiple + // `CreateVolume` calls for the same name do not result in more + // than one piece of storage provisioned corresponding to that + // name. If a Plugin is unable to enforce idempotency, the CO's + // error recovery logic could result in multiple (unused) volumes + // being provisioned. + // In the case of error, the CO MUST handle the gRPC error codes + // per the recovery behavior defined in the "CreateVolume Errors" + // section below. + // The CO is responsible for cleaning up volumes it provisioned + // that it no longer needs. If the CO is uncertain whether a volume + // was provisioned or not when a `CreateVolume` call fails, the CO + // MAY call `CreateVolume` again, with the same name, to ensure the + // volume exists and to retrieve the volume's `volume_id` (unless + // otherwise prohibited by "CreateVolume Errors"). + // 2) Suggested name - Some storage systems allow callers to specify + // an identifier by which to refer to the newly provisioned + // storage. If a storage system supports this, it can optionally + // use this name as the identifier for the new volume. + // Any Unicode string that conforms to the length limit is allowed + // except those containing the following banned characters: + // U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F. + // (These are control characters other than commonly used whitespace.) + string name = 1; + + // This field is OPTIONAL. This allows the CO to specify the capacity + // requirement of the volume to be provisioned. If not specified, the + // Plugin MAY choose an implementation-defined capacity range. If + // specified it MUST always be honored, even when creating volumes + // from a source; which MAY force some backends to internally extend + // the volume after creating it. + CapacityRange capacity_range = 2; + + // The capabilities that the provisioned volume MUST have. SP MUST + // provision a volume that will satisfy ALL of the capabilities + // specified in this list. Otherwise SP MUST return the appropriate + // gRPC error code. + // The Plugin MUST assume that the CO MAY use the provisioned volume + // with ANY of the capabilities specified in this list. + // For example, a CO MAY specify two volume capabilities: one with + // access mode SINGLE_NODE_WRITER and another with access mode + // MULTI_NODE_READER_ONLY. In this case, the SP MUST verify that the + // provisioned volume can be used in either mode. + // This also enables the CO to do early validation: If ANY of the + // specified volume capabilities are not supported by the SP, the call + // MUST return the appropriate gRPC error code. + // This field is REQUIRED. + repeated VolumeCapability volume_capabilities = 3; + + // Plugin specific creation-time parameters passed in as opaque + // key-value pairs. This field is OPTIONAL. The Plugin is responsible + // for parsing and validating these parameters. COs will treat + // these as opaque. + map parameters = 4; + + // Secrets required by plugin to complete volume creation request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + map secrets = 5 [(csi_secret) = true]; + + // If specified, the new volume will be pre-populated with data from + // this source. This field is OPTIONAL. + VolumeContentSource volume_content_source = 6; + + // Specifies where (regions, zones, racks, etc.) the provisioned + // volume MUST be accessible from. + // An SP SHALL advertise the requirements for topological + // accessibility information in documentation. COs SHALL only specify + // topological accessibility information supported by the SP. + // This field is OPTIONAL. + // This field SHALL NOT be specified unless the SP has the + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. + // If this field is not specified and the SP has the + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability, the SP MAY + // choose where the provisioned volume is accessible from. + TopologyRequirement accessibility_requirements = 7; + + // Plugin specific creation-time parameters passed in as opaque + // key-value pairs. These mutable_parameteres MAY also be + // changed during the lifetime of the volume via a subsequent + // `ControllerModifyVolume` RPC. This field is OPTIONAL. + // The Plugin is responsible for parsing and validating these + // parameters. COs will treat these as opaque. + + // Plugins MUST treat these + // as if they take precedence over the parameters field. + // This field SHALL NOT be specified unless the SP has the + // MODIFY_VOLUME plugin capability. + map mutable_parameters = 8 [(alpha_field) = true]; +} + +// Specifies what source the volume will be created from. One of the +// type fields MUST be specified. +message VolumeContentSource { + message SnapshotSource { + // Contains identity information for the existing source snapshot. + // This field is REQUIRED. Plugin is REQUIRED to support creating + // volume from snapshot if it supports the capability + // CREATE_DELETE_SNAPSHOT. + string snapshot_id = 1; + } + + message VolumeSource { + // Contains identity information for the existing source volume. + // This field is REQUIRED. Plugins reporting CLONE_VOLUME + // capability MUST support creating a volume from another volume. + string volume_id = 1; + } + + oneof type { + SnapshotSource snapshot = 1; + VolumeSource volume = 2; + } +} + +message CreateVolumeResponse { + // Contains all attributes of the newly created volume that are + // relevant to the CO along with information required by the Plugin + // to uniquely identify the volume. This field is REQUIRED. + Volume volume = 1; +} + +// Specify a capability of a volume. +message VolumeCapability { + // Indicate that the volume will be accessed via the block device API. + message BlockVolume { + // Intentionally empty, for now. + } + + // Indicate that the volume will be accessed via the filesystem API. + message MountVolume { + // The filesystem type. This field is OPTIONAL. + // An empty string is equal to an unspecified field value. + string fs_type = 1; + + // The mount options that can be used for the volume. This field is + // OPTIONAL. `mount_flags` MAY contain sensitive information. + // Therefore, the CO and the Plugin MUST NOT leak this information + // to untrusted entities. The total size of this repeated field + // SHALL NOT exceed 4 KiB. + repeated string mount_flags = 2; + + // If SP has VOLUME_MOUNT_GROUP node capability and CO provides + // this field then SP MUST ensure that the volume_mount_group + // parameter is passed as the group identifier to the underlying + // operating system mount system call, with the understanding + // that the set of available mount call parameters and/or + // mount implementations may vary across operating systems. + // Additionally, new file and/or directory entries written to + // the underlying filesystem SHOULD be permission-labeled in such a + // manner, unless otherwise modified by a workload, that they are + // both readable and writable by said mount group identifier. + // This is an OPTIONAL field. + string volume_mount_group = 3; + } + + // Specify how a volume can be accessed. + message AccessMode { + enum Mode { + UNKNOWN = 0; + + // Can only be published once as read/write on a single node, at + // any given time. + SINGLE_NODE_WRITER = 1; + + // Can only be published once as readonly on a single node, at + // any given time. + SINGLE_NODE_READER_ONLY = 2; + + // Can be published as readonly at multiple nodes simultaneously. + MULTI_NODE_READER_ONLY = 3; + + // Can be published at multiple nodes simultaneously. Only one of + // the node can be used as read/write. The rest will be readonly. + MULTI_NODE_SINGLE_WRITER = 4; + + // Can be published as read/write at multiple nodes + // simultaneously. + MULTI_NODE_MULTI_WRITER = 5; + + // Can only be published once as read/write at a single workload + // on a single node, at any given time. SHOULD be used instead of + // SINGLE_NODE_WRITER for COs using the experimental + // SINGLE_NODE_MULTI_WRITER capability. + SINGLE_NODE_SINGLE_WRITER = 6 [(alpha_enum_value) = true]; + + // Can be published as read/write at multiple workloads on a + // single node simultaneously. SHOULD be used instead of + // SINGLE_NODE_WRITER for COs using the experimental + // SINGLE_NODE_MULTI_WRITER capability. + SINGLE_NODE_MULTI_WRITER = 7 [(alpha_enum_value) = true]; + } + + // This field is REQUIRED. + Mode mode = 1; + } + + // Specifies what API the volume will be accessed using. One of the + // following fields MUST be specified. + oneof access_type { + BlockVolume block = 1; + MountVolume mount = 2; + } + + // This is a REQUIRED field. + AccessMode access_mode = 3; +} + +// The capacity of the storage space in bytes. To specify an exact size, +// `required_bytes` and `limit_bytes` SHALL be set to the same value. At +// least one of the these fields MUST be specified. +message CapacityRange { + // Volume MUST be at least this big. This field is OPTIONAL. + // A value of 0 is equal to an unspecified field value. + // The value of this field MUST NOT be negative. + int64 required_bytes = 1; + + // Volume MUST not be bigger than this. This field is OPTIONAL. + // A value of 0 is equal to an unspecified field value. + // The value of this field MUST NOT be negative. + int64 limit_bytes = 2; +} + +// Information about a specific volume. +message Volume { + // The capacity of the volume in bytes. This field is OPTIONAL. If not + // set (value of 0), it indicates that the capacity of the volume is + // unknown (e.g., NFS share). + // The value of this field MUST NOT be negative. + int64 capacity_bytes = 1; + + // The identifier for this volume, generated by the plugin. + // This field is REQUIRED. + // This field MUST contain enough information to uniquely identify + // this specific volume vs all other volumes supported by this plugin. + // This field SHALL be used by the CO in subsequent calls to refer to + // this volume. + // The SP is NOT responsible for global uniqueness of volume_id across + // multiple SPs. + string volume_id = 2; + + // Opaque static properties of the volume. SP MAY use this field to + // ensure subsequent volume validation and publishing calls have + // contextual information. + // The contents of this field SHALL be opaque to a CO. + // The contents of this field SHALL NOT be mutable. + // The contents of this field SHALL be safe for the CO to cache. + // The contents of this field SHOULD NOT contain sensitive + // information. + // The contents of this field SHOULD NOT be used for uniquely + // identifying a volume. The `volume_id` alone SHOULD be sufficient to + // identify the volume. + // A volume uniquely identified by `volume_id` SHALL always report the + // same volume_context. + // This field is OPTIONAL and when present MUST be passed to volume + // validation and publishing calls. + map volume_context = 3; + + // If specified, indicates that the volume is not empty and is + // pre-populated with data from the specified source. + // This field is OPTIONAL. + VolumeContentSource content_source = 4; + + // Specifies where (regions, zones, racks, etc.) the provisioned + // volume is accessible from. + // A plugin that returns this field MUST also set the + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. + // An SP MAY specify multiple topologies to indicate the volume is + // accessible from multiple locations. + // COs MAY use this information along with the topology information + // returned by NodeGetInfo to ensure that a given volume is accessible + // from a given node when scheduling workloads. + // This field is OPTIONAL. If it is not specified, the CO MAY assume + // the volume is equally accessible from all nodes in the cluster and + // MAY schedule workloads referencing the volume on any available + // node. + // + // Example 1: + // accessible_topology = {"region": "R1", "zone": "Z2"} + // Indicates a volume accessible only from the "region" "R1" and the + // "zone" "Z2". + // + // Example 2: + // accessible_topology = + // {"region": "R1", "zone": "Z2"}, + // {"region": "R1", "zone": "Z3"} + // Indicates a volume accessible from both "zone" "Z2" and "zone" "Z3" + // in the "region" "R1". + repeated Topology accessible_topology = 5; +} + +message TopologyRequirement { + // Specifies the list of topologies the provisioned volume MUST be + // accessible from. + // This field is OPTIONAL. If TopologyRequirement is specified either + // requisite or preferred or both MUST be specified. + // + // If requisite is specified, the provisioned volume MUST be + // accessible from at least one of the requisite topologies. + // + // Given + // x = number of topologies provisioned volume is accessible from + // n = number of requisite topologies + // The CO MUST ensure n >= 1. The SP MUST ensure x >= 1 + // If x==n, then the SP MUST make the provisioned volume available to + // all topologies from the list of requisite topologies. If it is + // unable to do so, the SP MUST fail the CreateVolume call. + // For example, if a volume should be accessible from a single zone, + // and requisite = + // {"region": "R1", "zone": "Z2"} + // then the provisioned volume MUST be accessible from the "region" + // "R1" and the "zone" "Z2". + // Similarly, if a volume should be accessible from two zones, and + // requisite = + // {"region": "R1", "zone": "Z2"}, + // {"region": "R1", "zone": "Z3"} + // then the provisioned volume MUST be accessible from the "region" + // "R1" and both "zone" "Z2" and "zone" "Z3". + // + // If xn, then the SP MUST make the provisioned volume available from + // all topologies from the list of requisite topologies and MAY choose + // the remaining x-n unique topologies from the list of all possible + // topologies. If it is unable to do so, the SP MUST fail the + // CreateVolume call. + // For example, if a volume should be accessible from two zones, and + // requisite = + // {"region": "R1", "zone": "Z2"} + // then the provisioned volume MUST be accessible from the "region" + // "R1" and the "zone" "Z2" and the SP may select the second zone + // independently, e.g. "R1/Z4". + repeated Topology requisite = 1; + + // Specifies the list of topologies the CO would prefer the volume to + // be provisioned in. + // + // This field is OPTIONAL. If TopologyRequirement is specified either + // requisite or preferred or both MUST be specified. + // + // An SP MUST attempt to make the provisioned volume available using + // the preferred topologies in order from first to last. + // + // If requisite is specified, all topologies in preferred list MUST + // also be present in the list of requisite topologies. + // + // If the SP is unable to to make the provisioned volume available + // from any of the preferred topologies, the SP MAY choose a topology + // from the list of requisite topologies. + // If the list of requisite topologies is not specified, then the SP + // MAY choose from the list of all possible topologies. + // If the list of requisite topologies is specified and the SP is + // unable to to make the provisioned volume available from any of the + // requisite topologies it MUST fail the CreateVolume call. + // + // Example 1: + // Given a volume should be accessible from a single zone, and + // requisite = + // {"region": "R1", "zone": "Z2"}, + // {"region": "R1", "zone": "Z3"} + // preferred = + // {"region": "R1", "zone": "Z3"} + // then the SP SHOULD first attempt to make the provisioned volume + // available from "zone" "Z3" in the "region" "R1" and fall back to + // "zone" "Z2" in the "region" "R1" if that is not possible. + // + // Example 2: + // Given a volume should be accessible from a single zone, and + // requisite = + // {"region": "R1", "zone": "Z2"}, + // {"region": "R1", "zone": "Z3"}, + // {"region": "R1", "zone": "Z4"}, + // {"region": "R1", "zone": "Z5"} + // preferred = + // {"region": "R1", "zone": "Z4"}, + // {"region": "R1", "zone": "Z2"} + // then the SP SHOULD first attempt to make the provisioned volume + // accessible from "zone" "Z4" in the "region" "R1" and fall back to + // "zone" "Z2" in the "region" "R1" if that is not possible. If that + // is not possible, the SP may choose between either the "zone" + // "Z3" or "Z5" in the "region" "R1". + // + // Example 3: + // Given a volume should be accessible from TWO zones (because an + // opaque parameter in CreateVolumeRequest, for example, specifies + // the volume is accessible from two zones, aka synchronously + // replicated), and + // requisite = + // {"region": "R1", "zone": "Z2"}, + // {"region": "R1", "zone": "Z3"}, + // {"region": "R1", "zone": "Z4"}, + // {"region": "R1", "zone": "Z5"} + // preferred = + // {"region": "R1", "zone": "Z5"}, + // {"region": "R1", "zone": "Z3"} + // then the SP SHOULD first attempt to make the provisioned volume + // accessible from the combination of the two "zones" "Z5" and "Z3" in + // the "region" "R1". If that's not possible, it should fall back to + // a combination of "Z5" and other possibilities from the list of + // requisite. If that's not possible, it should fall back to a + // combination of "Z3" and other possibilities from the list of + // requisite. If that's not possible, it should fall back to a + // combination of other possibilities from the list of requisite. + repeated Topology preferred = 2; +} + +// Topology is a map of topological domains to topological segments. +// A topological domain is a sub-division of a cluster, like "region", +// "zone", "rack", etc. +// A topological segment is a specific instance of a topological domain, +// like "zone3", "rack3", etc. +// For example {"com.company/zone": "Z1", "com.company/rack": "R3"} +// Valid keys have two segments: an OPTIONAL prefix and name, separated +// by a slash (/), for example: "com.company.example/zone". +// The key name segment is REQUIRED. The prefix is OPTIONAL. +// The key name MUST be 63 characters or less, begin and end with an +// alphanumeric character ([a-z0-9A-Z]), and contain only dashes (-), +// underscores (_), dots (.), or alphanumerics in between, for example +// "zone". +// The key prefix MUST be 63 characters or less, begin and end with a +// lower-case alphanumeric character ([a-z0-9]), contain only +// dashes (-), dots (.), or lower-case alphanumerics in between, and +// follow domain name notation format +// (https://tools.ietf.org/html/rfc1035#section-2.3.1). +// The key prefix SHOULD include the plugin's host company name and/or +// the plugin name, to minimize the possibility of collisions with keys +// from other plugins. +// If a key prefix is specified, it MUST be identical across all +// topology keys returned by the SP (across all RPCs). +// Keys MUST be case-insensitive. Meaning the keys "Zone" and "zone" +// MUST not both exist. +// Each value (topological segment) MUST contain 1 or more strings. +// Each string MUST be 63 characters or less and begin and end with an +// alphanumeric character with '-', '_', '.', or alphanumerics in +// between. +message Topology { + map segments = 1; +} +message DeleteVolumeRequest { + // The ID of the volume to be deprovisioned. + // This field is REQUIRED. + string volume_id = 1; + + // Secrets required by plugin to complete volume deletion request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + map secrets = 2 [(csi_secret) = true]; +} + +message DeleteVolumeResponse { + // Intentionally empty. +} +message ControllerPublishVolumeRequest { + // The ID of the volume to be used on a node. + // This field is REQUIRED. + string volume_id = 1; + + // The ID of the node. This field is REQUIRED. The CO SHALL set this + // field to match the node ID returned by `NodeGetInfo`. + string node_id = 2; + + // Volume capability describing how the CO intends to use this volume. + // SP MUST ensure the CO can use the published volume as described. + // Otherwise SP MUST return the appropriate gRPC error code. + // This is a REQUIRED field. + VolumeCapability volume_capability = 3; + + // Indicates SP MUST publish the volume in readonly mode. + // CO MUST set this field to false if SP does not have the + // PUBLISH_READONLY controller capability. + // This is a REQUIRED field. + bool readonly = 4; + + // Secrets required by plugin to complete controller publish volume + // request. This field is OPTIONAL. Refer to the + // `Secrets Requirements` section on how to use this field. + map secrets = 5 [(csi_secret) = true]; + + // Volume context as returned by SP in + // CreateVolumeResponse.Volume.volume_context. + // This field is OPTIONAL and MUST match the volume_context of the + // volume identified by `volume_id`. + map volume_context = 6; +} + +message ControllerPublishVolumeResponse { + // Opaque static publish properties of the volume. SP MAY use this + // field to ensure subsequent `NodeStageVolume` or `NodePublishVolume` + // calls calls have contextual information. + // The contents of this field SHALL be opaque to a CO. + // The contents of this field SHALL NOT be mutable. + // The contents of this field SHALL be safe for the CO to cache. + // The contents of this field SHOULD NOT contain sensitive + // information. + // The contents of this field SHOULD NOT be used for uniquely + // identifying a volume. The `volume_id` alone SHOULD be sufficient to + // identify the volume. + // This field is OPTIONAL and when present MUST be passed to + // subsequent `NodeStageVolume` or `NodePublishVolume` calls + map publish_context = 1; +} +message ControllerUnpublishVolumeRequest { + // The ID of the volume. This field is REQUIRED. + string volume_id = 1; + + // The ID of the node. This field is OPTIONAL. The CO SHOULD set this + // field to match the node ID returned by `NodeGetInfo` or leave it + // unset. If the value is set, the SP MUST unpublish the volume from + // the specified node. If the value is unset, the SP MUST unpublish + // the volume from all nodes it is published to. + string node_id = 2; + + // Secrets required by plugin to complete controller unpublish volume + // request. This SHOULD be the same secrets passed to the + // ControllerPublishVolume call for the specified volume. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + map secrets = 3 [(csi_secret) = true]; +} + +message ControllerUnpublishVolumeResponse { + // Intentionally empty. +} +message ValidateVolumeCapabilitiesRequest { + // The ID of the volume to check. This field is REQUIRED. + string volume_id = 1; + + // Volume context as returned by SP in + // CreateVolumeResponse.Volume.volume_context. + // This field is OPTIONAL and MUST match the volume_context of the + // volume identified by `volume_id`. + map volume_context = 2; + + // The capabilities that the CO wants to check for the volume. This + // call SHALL return "confirmed" only if all the volume capabilities + // specified below are supported. This field is REQUIRED. + repeated VolumeCapability volume_capabilities = 3; + + // See CreateVolumeRequest.parameters. + // This field is OPTIONAL. + map parameters = 4; + + // Secrets required by plugin to complete volume validation request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + map secrets = 5 [(csi_secret) = true]; + + // See CreateVolumeRequest.mutable_parameters. + // This field is OPTIONAL. + map mutable_parameters = 6 [(alpha_field) = true]; +} + +message ValidateVolumeCapabilitiesResponse { + message Confirmed { + // Volume context validated by the plugin. + // This field is OPTIONAL. + map volume_context = 1; + + // Volume capabilities supported by the plugin. + // This field is REQUIRED. + repeated VolumeCapability volume_capabilities = 2; + + // The volume creation parameters validated by the plugin. + // This field is OPTIONAL. + map parameters = 3; + + // The volume creation mutable_parameters validated by the plugin. + // This field is OPTIONAL. + map mutable_parameters = 4 [(alpha_field) = true]; + } + + // Confirmed indicates to the CO the set of capabilities that the + // plugin has validated. This field SHALL only be set to a non-empty + // value for successful validation responses. + // For successful validation responses, the CO SHALL compare the + // fields of this message to the originally requested capabilities in + // order to guard against an older plugin reporting "valid" for newer + // capability fields that it does not yet understand. + // This field is OPTIONAL. + Confirmed confirmed = 1; + + // Message to the CO if `confirmed` above is empty. This field is + // OPTIONAL. + // An empty string is equal to an unspecified field value. + string message = 2; +} +message ListVolumesRequest { + // If specified (non-zero value), the Plugin MUST NOT return more + // entries than this number in the response. If the actual number of + // entries is more than this number, the Plugin MUST set `next_token` + // in the response which can be used to get the next page of entries + // in the subsequent `ListVolumes` call. This field is OPTIONAL. If + // not specified (zero value), it means there is no restriction on the + // number of entries that can be returned. + // The value of this field MUST NOT be negative. + int32 max_entries = 1; + + // A token to specify where to start paginating. Set this field to + // `next_token` returned by a previous `ListVolumes` call to get the + // next page of entries. This field is OPTIONAL. + // An empty string is equal to an unspecified field value. + string starting_token = 2; +} + +message ListVolumesResponse { + message VolumeStatus{ + // A list of all `node_id` of nodes that the volume in this entry + // is controller published on. + // This field is OPTIONAL. If it is not specified and the SP has + // the LIST_VOLUMES_PUBLISHED_NODES controller capability, the CO + // MAY assume the volume is not controller published to any nodes. + // If the field is not specified and the SP does not have the + // LIST_VOLUMES_PUBLISHED_NODES controller capability, the CO MUST + // not interpret this field. + // published_node_ids MAY include nodes not published to or + // reported by the SP. The CO MUST be resilient to that. + repeated string published_node_ids = 1; + + // Information about the current condition of the volume. + // This field is OPTIONAL. + // This field MUST be specified if the + // VOLUME_CONDITION controller capability is supported. + VolumeCondition volume_condition = 2 [(alpha_field) = true]; + } + + message Entry { + // This field is REQUIRED + Volume volume = 1; + + // This field is OPTIONAL. This field MUST be specified if the + // LIST_VOLUMES_PUBLISHED_NODES controller capability is + // supported. + VolumeStatus status = 2; + } + + repeated Entry entries = 1; + + // This token allows you to get the next page of entries for + // `ListVolumes` request. If the number of entries is larger than + // `max_entries`, use the `next_token` as a value for the + // `starting_token` field in the next `ListVolumes` request. This + // field is OPTIONAL. + // An empty string is equal to an unspecified field value. + string next_token = 2; +} +message ControllerGetVolumeRequest { + option (alpha_message) = true; + + // The ID of the volume to fetch current volume information for. + // This field is REQUIRED. + string volume_id = 1; +} + +message ControllerGetVolumeResponse { + option (alpha_message) = true; + + message VolumeStatus{ + // A list of all the `node_id` of nodes that this volume is + // controller published on. + // This field is OPTIONAL. + // This field MUST be specified if the LIST_VOLUMES_PUBLISHED_NODES + // controller capability is supported. + // published_node_ids MAY include nodes not published to or + // reported by the SP. The CO MUST be resilient to that. + repeated string published_node_ids = 1; + + // Information about the current condition of the volume. + // This field is OPTIONAL. + // This field MUST be specified if the + // VOLUME_CONDITION controller capability is supported. + VolumeCondition volume_condition = 2; + } + + // This field is REQUIRED + Volume volume = 1; + + // This field is REQUIRED. + VolumeStatus status = 2; +} +message ControllerModifyVolumeRequest { + option (alpha_message) = true; + + // Contains identity information for the existing volume. + // This field is REQUIRED. + string volume_id = 1; + + // Secrets required by plugin to complete modify volume request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + map secrets = 2 [(csi_secret) = true]; + + // Plugin specific volume attributes to mutate, passed in as + // opaque key-value pairs. + // This field is REQUIRED. The Plugin is responsible for + // parsing and validating these parameters. COs will treat these + // as opaque. The CO SHOULD specify the intended values of all mutable + // parameters it intends to modify. SPs MUST NOT modify volumes based + // on the absence of keys, only keys that are specified should result + // in modifications to the volume. + map mutable_parameters = 3; +} + +message ControllerModifyVolumeResponse { + option (alpha_message) = true; +} + +message GetCapacityRequest { + // If specified, the Plugin SHALL report the capacity of the storage + // that can be used to provision volumes that satisfy ALL of the + // specified `volume_capabilities`. These are the same + // `volume_capabilities` the CO will use in `CreateVolumeRequest`. + // This field is OPTIONAL. + repeated VolumeCapability volume_capabilities = 1; + + // If specified, the Plugin SHALL report the capacity of the storage + // that can be used to provision volumes with the given Plugin + // specific `parameters`. These are the same `parameters` the CO will + // use in `CreateVolumeRequest`. This field is OPTIONAL. + map parameters = 2; + + // If specified, the Plugin SHALL report the capacity of the storage + // that can be used to provision volumes that in the specified + // `accessible_topology`. This is the same as the + // `accessible_topology` the CO returns in a `CreateVolumeResponse`. + // This field is OPTIONAL. This field SHALL NOT be set unless the + // plugin advertises the VOLUME_ACCESSIBILITY_CONSTRAINTS capability. + Topology accessible_topology = 3; +} + +message GetCapacityResponse { + // The available capacity, in bytes, of the storage that can be used + // to provision volumes. If `volume_capabilities` or `parameters` is + // specified in the request, the Plugin SHALL take those into + // consideration when calculating the available capacity of the + // storage. This field is REQUIRED. + // The value of this field MUST NOT be negative. + int64 available_capacity = 1; + + // The largest size that may be used in a + // CreateVolumeRequest.capacity_range.required_bytes field + // to create a volume with the same parameters as those in + // GetCapacityRequest. + // + // If `volume_capabilities` or `parameters` is + // specified in the request, the Plugin SHALL take those into + // consideration when calculating the minimum volume size of the + // storage. + // + // This field is OPTIONAL. MUST NOT be negative. + // The Plugin SHOULD provide a value for this field if it has + // a maximum size for individual volumes and leave it unset + // otherwise. COs MAY use it to make decision about + // where to create volumes. + google.protobuf.Int64Value maximum_volume_size = 2; + + // The smallest size that may be used in a + // CreateVolumeRequest.capacity_range.limit_bytes field + // to create a volume with the same parameters as those in + // GetCapacityRequest. + // + // If `volume_capabilities` or `parameters` is + // specified in the request, the Plugin SHALL take those into + // consideration when calculating the maximum volume size of the + // storage. + // + // This field is OPTIONAL. MUST NOT be negative. + // The Plugin SHOULD provide a value for this field if it has + // a minimum size for individual volumes and leave it unset + // otherwise. COs MAY use it to make decision about + // where to create volumes. + google.protobuf.Int64Value minimum_volume_size = 3 + [(alpha_field) = true]; +} +message ControllerGetCapabilitiesRequest { + // Intentionally empty. +} + +message ControllerGetCapabilitiesResponse { + // All the capabilities that the controller service supports. This + // field is OPTIONAL. + repeated ControllerServiceCapability capabilities = 1; +} + +// Specifies a capability of the controller service. +message ControllerServiceCapability { + message RPC { + enum Type { + UNKNOWN = 0; + CREATE_DELETE_VOLUME = 1; + PUBLISH_UNPUBLISH_VOLUME = 2; + LIST_VOLUMES = 3; + GET_CAPACITY = 4; + // Currently the only way to consume a snapshot is to create + // a volume from it. Therefore plugins supporting + // CREATE_DELETE_SNAPSHOT MUST support creating volume from + // snapshot. + CREATE_DELETE_SNAPSHOT = 5; + LIST_SNAPSHOTS = 6; + + // Plugins supporting volume cloning at the storage level MAY + // report this capability. The source volume MUST be managed by + // the same plugin. Not all volume sources and parameters + // combinations MAY work. + CLONE_VOLUME = 7; + + // Indicates the SP supports ControllerPublishVolume.readonly + // field. + PUBLISH_READONLY = 8; + + // See VolumeExpansion for details. + EXPAND_VOLUME = 9; + + // Indicates the SP supports the + // ListVolumesResponse.entry.published_node_ids field and the + // ControllerGetVolumeResponse.published_node_ids field. + // The SP MUST also support PUBLISH_UNPUBLISH_VOLUME. + LIST_VOLUMES_PUBLISHED_NODES = 10; + + // Indicates that the Controller service can report volume + // conditions. + // An SP MAY implement `VolumeCondition` in only the Controller + // Plugin, only the Node Plugin, or both. + // If `VolumeCondition` is implemented in both the Controller and + // Node Plugins, it SHALL report from different perspectives. + // If for some reason Controller and Node Plugins report + // misaligned volume conditions, CO SHALL assume the worst case + // is the truth. + // Note that, for alpha, `VolumeCondition` is intended be + // informative for humans only, not for automation. + VOLUME_CONDITION = 11 [(alpha_enum_value) = true]; + + // Indicates the SP supports the ControllerGetVolume RPC. + // This enables COs to, for example, fetch per volume + // condition after a volume is provisioned. + GET_VOLUME = 12 [(alpha_enum_value) = true]; + + // Indicates the SP supports the SINGLE_NODE_SINGLE_WRITER and/or + // SINGLE_NODE_MULTI_WRITER access modes. + // These access modes are intended to replace the + // SINGLE_NODE_WRITER access mode to clarify the number of writers + // for a volume on a single node. Plugins MUST accept and allow + // use of the SINGLE_NODE_WRITER access mode when either + // SINGLE_NODE_SINGLE_WRITER and/or SINGLE_NODE_MULTI_WRITER are + // supported, in order to permit older COs to continue working. + SINGLE_NODE_MULTI_WRITER = 13 [(alpha_enum_value) = true]; + + // Indicates the SP supports modifying volume with mutable + // parameters. See ControllerModifyVolume for details. + MODIFY_VOLUME = 14 [(alpha_enum_value) = true]; + } + + Type type = 1; + } + + oneof type { + // RPC that the controller supports. + RPC rpc = 1; + } +} +message CreateSnapshotRequest { + // The ID of the source volume to be snapshotted. + // This field is REQUIRED. + string source_volume_id = 1; + + // The suggested name for the snapshot. This field is REQUIRED for + // idempotency. + // Any Unicode string that conforms to the length limit is allowed + // except those containing the following banned characters: + // U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F. + // (These are control characters other than commonly used whitespace.) + string name = 2; + + // Secrets required by plugin to complete snapshot creation request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + map secrets = 3 [(csi_secret) = true]; + + // Plugin specific parameters passed in as opaque key-value pairs. + // This field is OPTIONAL. The Plugin is responsible for parsing and + // validating these parameters. COs will treat these as opaque. + // Use cases for opaque parameters: + // - Specify a policy to automatically clean up the snapshot. + // - Specify an expiration date for the snapshot. + // - Specify whether the snapshot is readonly or read/write. + // - Specify if the snapshot should be replicated to some place. + // - Specify primary or secondary for replication systems that + // support snapshotting only on primary. + map parameters = 4; +} + +message CreateSnapshotResponse { + // Contains all attributes of the newly created snapshot that are + // relevant to the CO along with information required by the Plugin + // to uniquely identify the snapshot. This field is REQUIRED. + Snapshot snapshot = 1; +} + +// Information about a specific snapshot. +message Snapshot { + // This is the complete size of the snapshot in bytes. The purpose of + // this field is to give CO guidance on how much space is needed to + // create a volume from this snapshot. The size of the volume MUST NOT + // be less than the size of the source snapshot. This field is + // OPTIONAL. If this field is not set, it indicates that this size is + // unknown. The value of this field MUST NOT be negative and a size of + // zero means it is unspecified. + int64 size_bytes = 1; + + // The identifier for this snapshot, generated by the plugin. + // This field is REQUIRED. + // This field MUST contain enough information to uniquely identify + // this specific snapshot vs all other snapshots supported by this + // plugin. + // This field SHALL be used by the CO in subsequent calls to refer to + // this snapshot. + // The SP is NOT responsible for global uniqueness of snapshot_id + // across multiple SPs. + string snapshot_id = 2; + + // Identity information for the source volume. Note that creating a + // snapshot from a snapshot is not supported here so the source has to + // be a volume. This field is REQUIRED. + string source_volume_id = 3; + + // Timestamp when the point-in-time snapshot is taken on the storage + // system. This field is REQUIRED. + .google.protobuf.Timestamp creation_time = 4; + + // Indicates if a snapshot is ready to use as a + // `volume_content_source` in a `CreateVolumeRequest`. The default + // value is false. This field is REQUIRED. + bool ready_to_use = 5; + + // The ID of the volume group snapshot that this snapshot is part of. + // It uniquely identifies the group snapshot on the storage system. + // This field is OPTIONAL. + // If this snapshot is a member of a volume group snapshot, and it + // MUST NOT be deleted as a stand alone snapshot, then the SP + // MUST provide the ID of the volume group snapshot in this field. + // If provided, CO MUST use this field in subsequent volume group + // snapshot operations to indicate that this snapshot is part of the + // specified group snapshot. + // If not provided, CO SHALL treat the snapshot as independent, + // and SP SHALL allow it to be deleted separately. + // If this message is inside a VolumeGroupSnapshot message, the value + // MUST be the same as the group_snapshot_id in that message. + string group_snapshot_id = 6 [(alpha_field) = true]; +} +message DeleteSnapshotRequest { + // The ID of the snapshot to be deleted. + // This field is REQUIRED. + string snapshot_id = 1; + + // Secrets required by plugin to complete snapshot deletion request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + map secrets = 2 [(csi_secret) = true]; +} + +message DeleteSnapshotResponse {} +// List all snapshots on the storage system regardless of how they were +// created. +message ListSnapshotsRequest { + // If specified (non-zero value), the Plugin MUST NOT return more + // entries than this number in the response. If the actual number of + // entries is more than this number, the Plugin MUST set `next_token` + // in the response which can be used to get the next page of entries + // in the subsequent `ListSnapshots` call. This field is OPTIONAL. If + // not specified (zero value), it means there is no restriction on the + // number of entries that can be returned. + // The value of this field MUST NOT be negative. + int32 max_entries = 1; + + // A token to specify where to start paginating. Set this field to + // `next_token` returned by a previous `ListSnapshots` call to get the + // next page of entries. This field is OPTIONAL. + // An empty string is equal to an unspecified field value. + string starting_token = 2; + + // Identity information for the source volume. This field is OPTIONAL. + // It can be used to list snapshots by volume. + string source_volume_id = 3; + + // Identity information for a specific snapshot. This field is + // OPTIONAL. It can be used to list only a specific snapshot. + // ListSnapshots will return with current snapshot information + // and will not block if the snapshot is being processed after + // it is cut. + string snapshot_id = 4; + + // Secrets required by plugin to complete ListSnapshot request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + map secrets = 5 [(csi_secret) = true]; +} + +message ListSnapshotsResponse { + message Entry { + Snapshot snapshot = 1; + } + + repeated Entry entries = 1; + + // This token allows you to get the next page of entries for + // `ListSnapshots` request. If the number of entries is larger than + // `max_entries`, use the `next_token` as a value for the + // `starting_token` field in the next `ListSnapshots` request. This + // field is OPTIONAL. + // An empty string is equal to an unspecified field value. + string next_token = 2; +} +message ControllerExpandVolumeRequest { + // The ID of the volume to expand. This field is REQUIRED. + string volume_id = 1; + + // This allows CO to specify the capacity requirements of the volume + // after expansion. This field is REQUIRED. + CapacityRange capacity_range = 2; + + // Secrets required by the plugin for expanding the volume. + // This field is OPTIONAL. + map secrets = 3 [(csi_secret) = true]; + + // Volume capability describing how the CO intends to use this volume. + // This allows SP to determine if volume is being used as a block + // device or mounted file system. For example - if volume is + // being used as a block device - the SP MAY set + // node_expansion_required to false in ControllerExpandVolumeResponse + // to skip invocation of NodeExpandVolume on the node by the CO. + // This is an OPTIONAL field. + VolumeCapability volume_capability = 4; +} + +message ControllerExpandVolumeResponse { + // Capacity of volume after expansion. This field is REQUIRED. + int64 capacity_bytes = 1; + + // Whether node expansion is required for the volume. When true + // the CO MUST make NodeExpandVolume RPC call on the node. This field + // is REQUIRED. + bool node_expansion_required = 2; +} +message NodeStageVolumeRequest { + // The ID of the volume to publish. This field is REQUIRED. + string volume_id = 1; + + // The CO SHALL set this field to the value returned by + // `ControllerPublishVolume` if the corresponding Controller Plugin + // has `PUBLISH_UNPUBLISH_VOLUME` controller capability, and SHALL be + // left unset if the corresponding Controller Plugin does not have + // this capability. This is an OPTIONAL field. + map publish_context = 2; + + // The path to which the volume MAY be staged. It MUST be an + // absolute path in the root filesystem of the process serving this + // request, and MUST be a directory. The CO SHALL ensure that there + // is only one `staging_target_path` per volume. The CO SHALL ensure + // that the path is directory and that the process serving the + // request has `read` and `write` permission to that directory. The + // CO SHALL be responsible for creating the directory if it does not + // exist. + // This is a REQUIRED field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + string staging_target_path = 3; + + // Volume capability describing how the CO intends to use this volume. + // SP MUST ensure the CO can use the staged volume as described. + // Otherwise SP MUST return the appropriate gRPC error code. + // This is a REQUIRED field. + VolumeCapability volume_capability = 4; + + // Secrets required by plugin to complete node stage volume request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + map secrets = 5 [(csi_secret) = true]; + + // Volume context as returned by SP in + // CreateVolumeResponse.Volume.volume_context. + // This field is OPTIONAL and MUST match the volume_context of the + // volume identified by `volume_id`. + map volume_context = 6; +} + +message NodeStageVolumeResponse { + // Intentionally empty. +} +message NodeUnstageVolumeRequest { + // The ID of the volume. This field is REQUIRED. + string volume_id = 1; + + // The path at which the volume was staged. It MUST be an absolute + // path in the root filesystem of the process serving this request. + // This is a REQUIRED field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + string staging_target_path = 2; +} + +message NodeUnstageVolumeResponse { + // Intentionally empty. +} +message NodePublishVolumeRequest { + // The ID of the volume to publish. This field is REQUIRED. + string volume_id = 1; + + // The CO SHALL set this field to the value returned by + // `ControllerPublishVolume` if the corresponding Controller Plugin + // has `PUBLISH_UNPUBLISH_VOLUME` controller capability, and SHALL be + // left unset if the corresponding Controller Plugin does not have + // this capability. This is an OPTIONAL field. + map publish_context = 2; + + // The path to which the volume was staged by `NodeStageVolume`. + // It MUST be an absolute path in the root filesystem of the process + // serving this request. + // It MUST be set if the Node Plugin implements the + // `STAGE_UNSTAGE_VOLUME` node capability. + // This is an OPTIONAL field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + string staging_target_path = 3; + + // The path to which the volume will be published. It MUST be an + // absolute path in the root filesystem of the process serving this + // request. The CO SHALL ensure uniqueness of target_path per volume. + // The CO SHALL ensure that the parent directory of this path exists + // and that the process serving the request has `read` and `write` + // permissions to that parent directory. + // For volumes with an access type of block, the SP SHALL place the + // block device at target_path. + // For volumes with an access type of mount, the SP SHALL place the + // mounted directory at target_path. + // Creation of target_path is the responsibility of the SP. + // This is a REQUIRED field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + string target_path = 4; + + // Volume capability describing how the CO intends to use this volume. + // SP MUST ensure the CO can use the published volume as described. + // Otherwise SP MUST return the appropriate gRPC error code. + // This is a REQUIRED field. + VolumeCapability volume_capability = 5; + + // Indicates SP MUST publish the volume in readonly mode. + // This field is REQUIRED. + bool readonly = 6; + + // Secrets required by plugin to complete node publish volume request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + map secrets = 7 [(csi_secret) = true]; + + // Volume context as returned by SP in + // CreateVolumeResponse.Volume.volume_context. + // This field is OPTIONAL and MUST match the volume_context of the + // volume identified by `volume_id`. + map volume_context = 8; +} + +message NodePublishVolumeResponse { + // Intentionally empty. +} +message NodeUnpublishVolumeRequest { + // The ID of the volume. This field is REQUIRED. + string volume_id = 1; + + // The path at which the volume was published. It MUST be an absolute + // path in the root filesystem of the process serving this request. + // The SP MUST delete the file or directory it created at this path. + // This is a REQUIRED field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + string target_path = 2; +} + +message NodeUnpublishVolumeResponse { + // Intentionally empty. +} +message NodeGetVolumeStatsRequest { + // The ID of the volume. This field is REQUIRED. + string volume_id = 1; + + // It can be any valid path where volume was previously + // staged or published. + // It MUST be an absolute path in the root filesystem of + // the process serving this request. + // This is a REQUIRED field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + string volume_path = 2; + + // The path where the volume is staged, if the plugin has the + // STAGE_UNSTAGE_VOLUME capability, otherwise empty. + // If not empty, it MUST be an absolute path in the root + // filesystem of the process serving this request. + // This field is OPTIONAL. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + string staging_target_path = 3; +} + +message NodeGetVolumeStatsResponse { + // This field is OPTIONAL. + repeated VolumeUsage usage = 1; + // Information about the current condition of the volume. + // This field is OPTIONAL. + // This field MUST be specified if the VOLUME_CONDITION node + // capability is supported. + VolumeCondition volume_condition = 2 [(alpha_field) = true]; +} + +message VolumeUsage { + enum Unit { + UNKNOWN = 0; + BYTES = 1; + INODES = 2; + } + // The available capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + int64 available = 1; + + // The total capacity in specified Unit. This field is REQUIRED. + // The value of this field MUST NOT be negative. + int64 total = 2; + + // The used capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + int64 used = 3; + + // Units by which values are measured. This field is REQUIRED. + Unit unit = 4; +} + +// VolumeCondition represents the current condition of a volume. +message VolumeCondition { + option (alpha_message) = true; + + // Normal volumes are available for use and operating optimally. + // An abnormal volume does not meet these criteria. + // This field is REQUIRED. + bool abnormal = 1; + + // The message describing the condition of the volume. + // This field is REQUIRED. + string message = 2; +} +message NodeGetCapabilitiesRequest { + // Intentionally empty. +} + +message NodeGetCapabilitiesResponse { + // All the capabilities that the node service supports. This field + // is OPTIONAL. + repeated NodeServiceCapability capabilities = 1; +} + +// Specifies a capability of the node service. +message NodeServiceCapability { + message RPC { + enum Type { + UNKNOWN = 0; + STAGE_UNSTAGE_VOLUME = 1; + // If Plugin implements GET_VOLUME_STATS capability + // then it MUST implement NodeGetVolumeStats RPC + // call for fetching volume statistics. + GET_VOLUME_STATS = 2; + // See VolumeExpansion for details. + EXPAND_VOLUME = 3; + // Indicates that the Node service can report volume conditions. + // An SP MAY implement `VolumeCondition` in only the Node + // Plugin, only the Controller Plugin, or both. + // If `VolumeCondition` is implemented in both the Node and + // Controller Plugins, it SHALL report from different + // perspectives. + // If for some reason Node and Controller Plugins report + // misaligned volume conditions, CO SHALL assume the worst case + // is the truth. + // Note that, for alpha, `VolumeCondition` is intended to be + // informative for humans only, not for automation. + VOLUME_CONDITION = 4 [(alpha_enum_value) = true]; + + // Indicates the SP supports the SINGLE_NODE_SINGLE_WRITER and/or + // SINGLE_NODE_MULTI_WRITER access modes. + // These access modes are intended to replace the + // SINGLE_NODE_WRITER access mode to clarify the number of writers + // for a volume on a single node. Plugins MUST accept and allow + // use of the SINGLE_NODE_WRITER access mode (subject to the + // processing rules for NodePublishVolume), when either + // SINGLE_NODE_SINGLE_WRITER and/or SINGLE_NODE_MULTI_WRITER are + // supported, in order to permit older COs to continue working. + SINGLE_NODE_MULTI_WRITER = 5 [(alpha_enum_value) = true]; + + // Indicates that Node service supports mounting volumes + // with provided volume group identifier during node stage + // or node publish RPC calls. + VOLUME_MOUNT_GROUP = 6; + } + + Type type = 1; + } + + oneof type { + // RPC that the controller supports. + RPC rpc = 1; + } +} +message NodeGetInfoRequest { +} + +message NodeGetInfoResponse { + // The identifier of the node as understood by the SP. + // This field is REQUIRED. + // This field MUST contain enough information to uniquely identify + // this specific node vs all other nodes supported by this plugin. + // This field SHALL be used by the CO in subsequent calls, including + // `ControllerPublishVolume`, to refer to this node. + // The SP is NOT responsible for global uniqueness of node_id across + // multiple SPs. + // This field overrides the general CSI size limit. + // The size of this field SHALL NOT exceed 256 bytes. The general + // CSI size limit, 128 byte, is RECOMMENDED for best backwards + // compatibility. + string node_id = 1; + + // Maximum number of volumes that controller can publish to the node. + // If value is not set or zero CO SHALL decide how many volumes of + // this type can be published by the controller to the node. The + // plugin MUST NOT set negative values here. + // This field is OPTIONAL. + int64 max_volumes_per_node = 2; + + // Specifies where (regions, zones, racks, etc.) the node is + // accessible from. + // A plugin that returns this field MUST also set the + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. + // COs MAY use this information along with the topology information + // returned in CreateVolumeResponse to ensure that a given volume is + // accessible from a given node when scheduling workloads. + // This field is OPTIONAL. If it is not specified, the CO MAY assume + // the node is not subject to any topological constraint, and MAY + // schedule workloads that reference any volume V, such that there are + // no topological constraints declared for V. + // + // Example 1: + // accessible_topology = + // {"region": "R1", "zone": "Z2"} + // Indicates the node exists within the "region" "R1" and the "zone" + // "Z2". + Topology accessible_topology = 3; +} +message NodeExpandVolumeRequest { + // The ID of the volume. This field is REQUIRED. + string volume_id = 1; + + // The path on which volume is available. This field is REQUIRED. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + string volume_path = 2; + + // This allows CO to specify the capacity requirements of the volume + // after expansion. If capacity_range is omitted then a plugin MAY + // inspect the file system of the volume to determine the maximum + // capacity to which the volume can be expanded. In such cases a + // plugin MAY expand the volume to its maximum capacity. + // This field is OPTIONAL. + CapacityRange capacity_range = 3; + + // The path where the volume is staged, if the plugin has the + // STAGE_UNSTAGE_VOLUME capability, otherwise empty. + // If not empty, it MUST be an absolute path in the root + // filesystem of the process serving this request. + // This field is OPTIONAL. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + string staging_target_path = 4; + + // Volume capability describing how the CO intends to use this volume. + // This allows SP to determine if volume is being used as a block + // device or mounted file system. For example - if volume is being + // used as a block device the SP MAY choose to skip expanding the + // filesystem in NodeExpandVolume implementation but still perform + // rest of the housekeeping needed for expanding the volume. If + // volume_capability is omitted the SP MAY determine + // access_type from given volume_path for the volume and perform + // node expansion. This is an OPTIONAL field. + VolumeCapability volume_capability = 5; + + // Secrets required by plugin to complete node expand volume request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + map secrets = 6 + [(csi_secret) = true, (alpha_field) = true]; +} + +message NodeExpandVolumeResponse { + // The capacity of the volume in bytes. This field is OPTIONAL. + int64 capacity_bytes = 1; +} +message GroupControllerGetCapabilitiesRequest { + option (alpha_message) = true; + + // Intentionally empty. +} + +message GroupControllerGetCapabilitiesResponse { + option (alpha_message) = true; + + // All the capabilities that the group controller service supports. + // This field is OPTIONAL. + repeated GroupControllerServiceCapability capabilities = 1; +} + +// Specifies a capability of the group controller service. +message GroupControllerServiceCapability { + option (alpha_message) = true; + + message RPC { + enum Type { + UNKNOWN = 0; + + // Indicates that the group controller plugin supports + // creating, deleting, and getting details of a volume + // group snapshot. + CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT = 1 + [(alpha_enum_value) = true]; + } + + Type type = 1; + } + + oneof type { + // RPC that the controller supports. + RPC rpc = 1; + } +} +message CreateVolumeGroupSnapshotRequest { + option (alpha_message) = true; + + // The suggested name for the group snapshot. This field is REQUIRED + // for idempotency. + // Any Unicode string that conforms to the length limit is allowed + // except those containing the following banned characters: + // U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F. + // (These are control characters other than commonly used whitespace.) + string name = 1; + + // volume IDs of the source volumes to be snapshotted together. + // This field is REQUIRED. + repeated string source_volume_ids = 2; + + // Secrets required by plugin to complete + // ControllerCreateVolumeGroupSnapshot request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + // The secrets provided in this field SHOULD be the same for + // all group snapshot operations on the same group snapshot. + map secrets = 3 [(csi_secret) = true]; + + // Plugin specific parameters passed in as opaque key-value pairs. + // This field is OPTIONAL. The Plugin is responsible for parsing and + // validating these parameters. COs will treat these as opaque. + map parameters = 4; +} + +message CreateVolumeGroupSnapshotResponse { + option (alpha_message) = true; + + // Contains all attributes of the newly created group snapshot. + // This field is REQUIRED. + VolumeGroupSnapshot group_snapshot = 1; +} + +message VolumeGroupSnapshot { + option (alpha_message) = true; + + // The identifier for this group snapshot, generated by the plugin. + // This field MUST contain enough information to uniquely identify + // this specific snapshot vs all other group snapshots supported by + // this plugin. + // This field SHALL be used by the CO in subsequent calls to refer to + // this group snapshot. + // The SP is NOT responsible for global uniqueness of + // group_snapshot_id across multiple SPs. + // This field is REQUIRED. + string group_snapshot_id = 1; + + // A list of snapshots belonging to this group. + // This field is REQUIRED. + repeated Snapshot snapshots = 2; + + // Timestamp of when the volume group snapshot was taken. + // This field is REQUIRED. + .google.protobuf.Timestamp creation_time = 3; + + // Indicates if all individual snapshots in the group snapshot + // are ready to use as a `volume_content_source` in a + // `CreateVolumeRequest`. The default value is false. + // If any snapshot in the list of snapshots in this message have + // ready_to_use set to false, the SP MUST set this field to false. + // If all of the snapshots in the list of snapshots in this message + // have ready_to_use set to true, the SP SHOULD set this field to + // true. + // This field is REQUIRED. + bool ready_to_use = 4; +} +message DeleteVolumeGroupSnapshotRequest { + option (alpha_message) = true; + + // The ID of the group snapshot to be deleted. + // This field is REQUIRED. + string group_snapshot_id = 1; + + // A list of snapshot IDs that are part of this group snapshot. + // If SP does not need to rely on this field to delete the snapshots + // in the group, it SHOULD check this field and report an error + // if it has the ability to detect a mismatch. + // Some SPs require this list to delete the snapshots in the group. + // If SP needs to use this field to delete the snapshots in the + // group, it MUST report an error if it has the ability to detect + // a mismatch. + // This field is REQUIRED. + repeated string snapshot_ids = 2; + + // Secrets required by plugin to complete group snapshot deletion + // request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + // The secrets provided in this field SHOULD be the same for + // all group snapshot operations on the same group snapshot. + map secrets = 3 [(csi_secret) = true]; +} + +message DeleteVolumeGroupSnapshotResponse { + // Intentionally empty. + option (alpha_message) = true; +} +message GetVolumeGroupSnapshotRequest { + option (alpha_message) = true; + + // The ID of the group snapshot to fetch current group snapshot + // information for. + // This field is REQUIRED. + string group_snapshot_id = 1; + + // A list of snapshot IDs that are part of this group snapshot. + // If SP does not need to rely on this field to get the snapshots + // in the group, it SHOULD check this field and report an error + // if it has the ability to detect a mismatch. + // Some SPs require this list to get the snapshots in the group. + // If SP needs to use this field to get the snapshots in the + // group, it MUST report an error if it has the ability to detect + // a mismatch. + // This field is REQUIRED. + repeated string snapshot_ids = 2; + + // Secrets required by plugin to complete + // GetVolumeGroupSnapshot request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + // The secrets provided in this field SHOULD be the same for + // all group snapshot operations on the same group snapshot. + map secrets = 3 [(csi_secret) = true]; +} + +message GetVolumeGroupSnapshotResponse { + option (alpha_message) = true; + + // This field is REQUIRED + VolumeGroupSnapshot group_snapshot = 1; +} diff --git a/lightningstor/crates/lightningstor-distributed/Cargo.toml b/lightningstor/crates/lightningstor-distributed/Cargo.toml new file mode 100644 index 0000000..fabf409 --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "lightningstor-distributed" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +description = "Distributed storage backends for LightningStor (Erasure Coding & Replication)" + +[dependencies] +# Internal crates +lightningstor-types = { workspace = true } +lightningstor-storage = { workspace = true } +lightningstor-node = { workspace = true } + +# Async runtime +tokio = { workspace = true } +futures = { workspace = true } +async-trait = { workspace = true } + +# gRPC +tonic = { workspace = true } +prost = { workspace = true } + +# Serialization +serde = { workspace = true } + +# Utilities +thiserror = { workspace = true } +anyhow = { workspace = true } +tracing = { workspace = true } +bytes = { workspace = true } +dashmap = { workspace = true } +uuid = { workspace = true } + +# Erasure coding +reed-solomon-erasure = "6.0" + +# Consistent hashing +hashring = "0.3" + +[dev-dependencies] +tempfile = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +serde_json = { workspace = true } + +[lints] +workspace = true diff --git a/lightningstor/crates/lightningstor-distributed/src/backends/erasure_coded.rs b/lightningstor/crates/lightningstor-distributed/src/backends/erasure_coded.rs new file mode 100644 index 0000000..19316bf --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/backends/erasure_coded.rs @@ -0,0 +1,848 @@ +//! Erasure-coded distributed storage backend +//! +//! Implements StorageBackend using Reed-Solomon erasure coding for +//! storage-efficient redundancy. + +use crate::chunk::{ChunkId, ChunkManager}; +use crate::config::DistributedConfig; +use crate::erasure::Codec; +use crate::node::{NodeClientTrait, NodeRegistry}; +use crate::placement::{ConsistentHashSelector, NodeSelector}; +use async_trait::async_trait; +use bytes::Bytes; +use lightningstor_storage::{StorageBackend, StorageError, StorageResult}; +use lightningstor_types::ObjectId; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tracing::{debug, error, warn}; + +/// Metadata for an object stored with erasure coding +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ObjectMetadata { + /// Number of chunks the object is split into + pub chunk_count: usize, + /// Original size of the object in bytes + pub original_size: u64, + /// Size of each chunk (except possibly the last) + pub chunk_size: usize, +} + +impl ObjectMetadata { + /// Create new object metadata + pub fn new(original_size: u64, chunk_count: usize, chunk_size: usize) -> Self { + Self { + chunk_count, + original_size, + chunk_size, + } + } + + /// Serialize to bytes + pub fn to_bytes(&self) -> Vec { + // Simple format: chunk_count (8 bytes) + original_size (8 bytes) + chunk_size (8 bytes) + let mut bytes = Vec::with_capacity(24); + bytes.extend_from_slice(&(self.chunk_count as u64).to_le_bytes()); + bytes.extend_from_slice(&self.original_size.to_le_bytes()); + bytes.extend_from_slice(&(self.chunk_size as u64).to_le_bytes()); + bytes + } + + /// Deserialize from bytes + pub fn from_bytes(bytes: &[u8]) -> Option { + if bytes.len() < 24 { + return None; + } + let chunk_count = u64::from_le_bytes(bytes[0..8].try_into().ok()?) as usize; + let original_size = u64::from_le_bytes(bytes[8..16].try_into().ok()?); + let chunk_size = u64::from_le_bytes(bytes[16..24].try_into().ok()?) as usize; + Some(Self { + chunk_count, + original_size, + chunk_size, + }) + } + + /// Get the metadata key for an object + pub fn metadata_key(object_id: &ObjectId) -> String { + format!("{}_meta", object_id) + } +} + +/// Erasure-coded distributed storage backend +/// +/// Stores objects by: +/// 1. Splitting into chunks (for large objects) +/// 2. Encoding each chunk into data + parity shards using Reed-Solomon +/// 3. Distributing shards across storage nodes +/// +/// Can tolerate loss of up to `parity_shards` nodes without data loss. +pub struct ErasureCodedBackend { + /// Erasure coding codec + codec: Arc, + /// Node registry for discovering storage nodes + node_registry: Arc, + /// Node selector for placement decisions + node_selector: Arc, + /// Chunk manager for splitting/reassembling + chunk_manager: Arc, + /// Configuration (kept for future use) + #[allow(dead_code)] + config: DistributedConfig, + /// Number of data shards + data_shards: usize, + /// Number of parity shards + parity_shards: usize, +} + +impl ErasureCodedBackend { + /// Create a new erasure-coded backend + /// + /// # Arguments + /// * `config` - Distributed storage configuration (must have ErasureCoded redundancy mode) + /// * `node_registry` - Registry for discovering storage nodes + pub async fn new( + config: DistributedConfig, + node_registry: Arc, + ) -> StorageResult { + let (data_shards, parity_shards) = match &config.redundancy { + crate::config::RedundancyMode::ErasureCoded { + data_shards, + parity_shards, + } => (*data_shards, *parity_shards), + _ => { + return Err(StorageError::Backend( + "ErasureCodedBackend requires ErasureCoded redundancy mode".into(), + )) + } + }; + + let codec = Arc::new( + Codec::new(data_shards, parity_shards) + .map_err(|e| StorageError::Backend(e.to_string()))?, + ); + let node_selector = Arc::new(ConsistentHashSelector::new()); + let chunk_manager = Arc::new(ChunkManager::new(config.chunk.clone())); + + Ok(Self { + codec, + node_registry, + node_selector, + chunk_manager, + config, + data_shards, + parity_shards, + }) + } + + /// Get the total number of shards (data + parity) + pub fn total_shards(&self) -> usize { + self.data_shards + self.parity_shards + } + + /// Get the minimum number of shards needed to reconstruct data + pub fn min_shards_for_read(&self) -> usize { + self.data_shards + } + + /// Select nodes for writing shards + async fn select_nodes_for_write(&self) -> StorageResult>> { + let nodes = self + .node_registry + .get_healthy_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + let total_shards = self.total_shards(); + if nodes.len() < total_shards { + return Err(StorageError::Backend(format!( + "Not enough healthy nodes: need {}, have {}", + total_shards, + nodes.len() + ))); + } + + self.node_selector + .select_nodes(&nodes, total_shards) + .await + .map_err(|e| StorageError::Backend(e.to_string())) + } + + /// Store object metadata to nodes + async fn write_metadata( + &self, + object_id: &ObjectId, + metadata: &ObjectMetadata, + ) -> StorageResult<()> { + let meta_key = ObjectMetadata::metadata_key(object_id); + let meta_bytes = Bytes::from(metadata.to_bytes()); + + let nodes = self + .node_registry + .get_healthy_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + // Store metadata on multiple nodes for redundancy + let mut write_futures = Vec::new(); + for node in nodes.iter().take(self.total_shards()) { + let node = node.clone(); + let key = meta_key.clone(); + let data = meta_bytes.clone(); + write_futures.push(async move { + node.put_chunk(&key, 0, false, data).await + }); + } + + let results = futures::future::join_all(write_futures).await; + let success_count = results.iter().filter(|r| r.is_ok()).count(); + + // Need at least one successful write + if success_count == 0 { + return Err(StorageError::Backend( + "Failed to write object metadata to any node".into(), + )); + } + + debug!( + object_id = %object_id, + success_count, + "Stored object metadata" + ); + + Ok(()) + } + + /// Read object metadata from nodes + async fn read_metadata(&self, object_id: &ObjectId) -> StorageResult { + let meta_key = ObjectMetadata::metadata_key(object_id); + + let nodes = self + .node_registry + .get_all_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + // Try to read metadata from any node + for node in &nodes { + if let Ok(data) = node.get_chunk(&meta_key, 0, false).await { + if let Some(metadata) = ObjectMetadata::from_bytes(&data) { + return Ok(metadata); + } + } + } + + Err(StorageError::NotFound(*object_id)) + } + + /// Delete object metadata from all nodes + async fn delete_metadata(&self, object_id: &ObjectId) { + let meta_key = ObjectMetadata::metadata_key(object_id); + + let nodes = match self.node_registry.get_all_nodes().await { + Ok(nodes) => nodes, + Err(_) => return, + }; + + let mut delete_futures = Vec::new(); + for node in &nodes { + let node = node.clone(); + let key = meta_key.clone(); + delete_futures.push(async move { + let _ = node.delete_chunk(&key).await; + }); + } + + futures::future::join_all(delete_futures).await; + } + + /// Write a single chunk with erasure coding + async fn write_chunk( + &self, + object_id: &ObjectId, + chunk_index: usize, + chunk_data: &[u8], + ) -> StorageResult<()> { + // Encode the chunk + let shards = self + .codec + .encode(chunk_data) + .map_err(|e| StorageError::Backend(e.to_string()))?; + + // Select nodes for each shard + let nodes = self.select_nodes_for_write().await?; + + // Write shards in parallel + let mut write_futures = Vec::with_capacity(self.total_shards()); + for (shard_idx, (shard_data, node)) in shards.into_iter().zip(nodes.iter()).enumerate() { + let is_parity = shard_idx >= self.data_shards; + let chunk_id = ChunkId::new(object_id, chunk_index, shard_idx, is_parity); + let node = node.clone(); + let shard_bytes = Bytes::from(shard_data); + + write_futures.push(async move { + node.put_chunk(&chunk_id.to_key(), shard_idx as u32, is_parity, shard_bytes) + .await + }); + } + + // Wait for all writes + let results = futures::future::join_all(write_futures).await; + let success_count = results.iter().filter(|r| r.is_ok()).count(); + let error_count = results.len() - success_count; + + debug!( + object_id = %object_id, + chunk_index, + success_count, + error_count, + "Wrote erasure-coded chunk" + ); + + // Need at least data_shards + 1 for durability + let min_required = self.data_shards + 1; + if success_count < min_required { + let errors: Vec<_> = results + .into_iter() + .filter_map(|r| r.err()) + .collect(); + error!( + success_count, + min_required, + errors = ?errors, + "Failed to write enough shards" + ); + return Err(StorageError::Backend(format!( + "Failed to write enough shards: {} of {} required succeeded", + success_count, min_required + ))); + } + + Ok(()) + } + + /// Read a single chunk with erasure decoding + async fn read_chunk( + &self, + object_id: &ObjectId, + chunk_index: usize, + original_chunk_size: usize, + ) -> StorageResult> { + // Use all nodes for reads - unhealthy nodes might still have data we need + // The erasure coding handles actual failures gracefully + let nodes = self + .node_registry + .get_all_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + // Try to read all shards in parallel + let mut shard_futures = Vec::with_capacity(self.total_shards()); + for shard_idx in 0..self.total_shards() { + let is_parity = shard_idx >= self.data_shards; + let chunk_id = ChunkId::new(object_id, chunk_index, shard_idx, is_parity); + let nodes = nodes.clone(); + let node_selector = self.node_selector.clone(); + let chunk_key = chunk_id.to_key(); + + shard_futures.push(async move { + // Try to read from the preferred node first + if let Ok(node) = node_selector.select_for_read(&nodes, &chunk_key).await { + if let Ok(data) = node + .get_chunk(&chunk_key, shard_idx as u32, is_parity) + .await + { + return Some(data); + } + } + + // Try other nodes if preferred fails + for node in &nodes { + if let Ok(data) = node + .get_chunk(&chunk_key, shard_idx as u32, is_parity) + .await + { + return Some(data); + } + } + + None + }); + } + + let shard_results: Vec>> = futures::future::join_all(shard_futures).await; + + // Count available shards + let available_count = shard_results.iter().filter(|s| s.is_some()).count(); + + debug!( + object_id = %object_id, + chunk_index, + available_count, + required = self.data_shards, + "Read shards for decoding" + ); + + if available_count < self.data_shards { + return Err(StorageError::Backend(format!( + "Not enough shards for decoding: have {}, need {}", + available_count, self.data_shards + ))); + } + + // Decode + self.codec + .decode(shard_results, original_chunk_size) + .map_err(|e| StorageError::Backend(e.to_string())) + } +} + +impl ChunkId { + fn new(object_id: &ObjectId, chunk_index: usize, shard_index: usize, is_parity: bool) -> Self { + if is_parity { + Self::parity_shard(object_id.to_string(), chunk_index, shard_index) + } else { + Self::data_shard(object_id.to_string(), chunk_index, shard_index) + } + } +} + +#[async_trait] +impl StorageBackend for ErasureCodedBackend { + async fn put_object(&self, object_id: &ObjectId, data: Bytes) -> StorageResult<()> { + let original_size = data.len() as u64; + debug!(object_id = %object_id, size = original_size, "Putting object with erasure coding"); + + // Split data into chunks + let chunks = self.chunk_manager.split(&data); + let chunk_count = chunks.len(); + let chunk_size = self.chunk_manager.chunk_size(); + + // Write each chunk + for (chunk_idx, chunk_data) in chunks.into_iter().enumerate() { + self.write_chunk(object_id, chunk_idx, &chunk_data).await?; + } + + // Store metadata + let metadata = ObjectMetadata::new(original_size, chunk_count, chunk_size); + self.write_metadata(object_id, &metadata).await?; + + debug!( + object_id = %object_id, + chunk_count, + original_size, + "Successfully stored object with erasure coding" + ); + + Ok(()) + } + + async fn get_object(&self, object_id: &ObjectId) -> StorageResult { + debug!(object_id = %object_id, "Getting object with erasure decoding"); + + // Read metadata to get chunk count and original size + let metadata = self.read_metadata(object_id).await?; + + debug!( + object_id = %object_id, + chunk_count = metadata.chunk_count, + original_size = metadata.original_size, + "Read object metadata" + ); + + // Read all chunks and reassemble + let mut all_data = Vec::with_capacity(metadata.original_size as usize); + + for chunk_idx in 0..metadata.chunk_count { + // Calculate the expected size for this chunk + let remaining = metadata.original_size as usize - all_data.len(); + let expected_chunk_size = remaining.min(metadata.chunk_size); + + let chunk_data = self + .read_chunk(object_id, chunk_idx, expected_chunk_size) + .await?; + + // Only take what we need (handles padding) + let take_size = remaining.min(chunk_data.len()); + all_data.extend_from_slice(&chunk_data[..take_size]); + } + + // Truncate to original size in case of any padding + all_data.truncate(metadata.original_size as usize); + + debug!( + object_id = %object_id, + retrieved_size = all_data.len(), + "Successfully retrieved object" + ); + + Ok(Bytes::from(all_data)) + } + + async fn delete_object(&self, object_id: &ObjectId) -> StorageResult<()> { + debug!(object_id = %object_id, "Deleting object shards"); + + // Try to read metadata to know how many chunks to delete + let chunk_count = match self.read_metadata(object_id).await { + Ok(metadata) => metadata.chunk_count, + Err(_) => 1, // Fall back to single chunk if metadata not found + }; + + let nodes = self + .node_registry + .get_all_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + // Delete all shards for all chunks from all nodes (best effort) + let mut delete_futures = Vec::new(); + for chunk_idx in 0..chunk_count { + for shard_idx in 0..self.total_shards() { + let is_parity = shard_idx >= self.data_shards; + let chunk_id = ChunkId::new(object_id, chunk_idx, shard_idx, is_parity); + let chunk_key = chunk_id.to_key(); + + for node in &nodes { + let node = node.clone(); + let key = chunk_key.clone(); + delete_futures.push(async move { + if let Err(e) = node.delete_chunk(&key).await { + // Log but don't fail - best effort deletion + warn!(node_id = node.node_id(), chunk_key = key, error = ?e, "Failed to delete shard"); + } + }); + } + } + } + + futures::future::join_all(delete_futures).await; + + // Delete metadata + self.delete_metadata(object_id).await; + + debug!(object_id = %object_id, chunk_count, "Deleted object"); + Ok(()) + } + + async fn object_exists(&self, object_id: &ObjectId) -> StorageResult { + // Check if metadata exists + Ok(self.read_metadata(object_id).await.is_ok()) + } + + async fn object_size(&self, object_id: &ObjectId) -> StorageResult { + // Read metadata to get original size + let metadata = self.read_metadata(object_id).await?; + Ok(metadata.original_size) + } + + async fn put_part( + &self, + upload_id: &str, + part_number: u32, + data: Bytes, + ) -> StorageResult<()> { + // Use a deterministic part key based on upload_id and part_number + let part_key = format!("part_{}_{}", upload_id, part_number); + let nodes = self.select_nodes_for_write().await?; + + // Encode and store the part data + let shards = self + .codec + .encode(&data) + .map_err(|e| StorageError::Backend(e.to_string()))?; + + let mut write_futures = Vec::with_capacity(self.total_shards()); + for (shard_idx, (shard_data, node)) in shards.into_iter().zip(nodes.iter()).enumerate() { + let is_parity = shard_idx >= self.data_shards; + let key = format!("{}_{}_{}", part_key, shard_idx, if is_parity { "p" } else { "d" }); + let node = node.clone(); + let shard_bytes = Bytes::from(shard_data); + + write_futures.push(async move { + node.put_chunk(&key, shard_idx as u32, is_parity, shard_bytes).await + }); + } + + let results = futures::future::join_all(write_futures).await; + let success_count = results.iter().filter(|r| r.is_ok()).count(); + + if success_count < self.data_shards + 1 { + return Err(StorageError::Backend(format!( + "Failed to write part shards: {} of {} required", + success_count, self.data_shards + 1 + ))); + } + + Ok(()) + } + + async fn get_part(&self, upload_id: &str, part_number: u32) -> StorageResult { + let part_key = format!("part_{}_{}", upload_id, part_number); + let nodes = self + .node_registry + .get_healthy_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + // Try to read shards + let mut shard_futures = Vec::with_capacity(self.total_shards()); + for shard_idx in 0..self.total_shards() { + let is_parity = shard_idx >= self.data_shards; + let key = format!("{}_{}_{}", part_key, shard_idx, if is_parity { "p" } else { "d" }); + let nodes = nodes.clone(); + + shard_futures.push(async move { + for node in &nodes { + if let Ok(data) = node.get_chunk(&key, shard_idx as u32, is_parity).await { + return Some(data); + } + } + None + }); + } + + let shard_results: Vec>> = futures::future::join_all(shard_futures).await; + let available = shard_results.iter().filter(|s| s.is_some()).count(); + + if available < self.data_shards { + return Err(StorageError::Backend(format!( + "Part {}:{} not found (insufficient shards)", + upload_id, part_number + ))); + } + + // Get shard size from first available shard + let shard_size = shard_results + .iter() + .find_map(|s| s.as_ref().map(|v| v.len())) + .unwrap_or(0); + let original_size = self.data_shards * shard_size; + + let data = self + .codec + .decode(shard_results, original_size) + .map_err(|e| StorageError::Backend(e.to_string()))?; + + Ok(Bytes::from(data)) + } + + async fn delete_part(&self, upload_id: &str, part_number: u32) -> StorageResult<()> { + let part_key = format!("part_{}_{}", upload_id, part_number); + let nodes = self + .node_registry + .get_all_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + let mut delete_futures = Vec::new(); + for shard_idx in 0..self.total_shards() { + let is_parity = shard_idx >= self.data_shards; + let key = format!("{}_{}_{}", part_key, shard_idx, if is_parity { "p" } else { "d" }); + + for node in &nodes { + let node = node.clone(); + let key = key.clone(); + delete_futures.push(async move { + let _ = node.delete_chunk(&key).await; + }); + } + } + + futures::future::join_all(delete_futures).await; + Ok(()) + } + + async fn delete_upload_parts(&self, _upload_id: &str) -> StorageResult<()> { + // Would need to track part numbers in metadata + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{ChunkConfig, RedundancyMode}; + use crate::node::MockNodeRegistry; + + fn create_ec_config(data_shards: usize, parity_shards: usize) -> DistributedConfig { + DistributedConfig { + redundancy: RedundancyMode::ErasureCoded { + data_shards, + parity_shards, + }, + chunk: ChunkConfig::default(), + ..Default::default() + } + } + + #[tokio::test] + async fn test_ec_backend_creation() { + let config = create_ec_config(4, 2); + let registry = Arc::new(MockNodeRegistry::with_nodes(6)); + + let backend = ErasureCodedBackend::new(config, registry).await.unwrap(); + + assert_eq!(backend.total_shards(), 6); + assert_eq!(backend.min_shards_for_read(), 4); + } + + #[tokio::test] + async fn test_ec_backend_put_get() { + let config = create_ec_config(4, 2); + let registry = Arc::new(MockNodeRegistry::with_nodes(6)); + + let backend = ErasureCodedBackend::new(config, registry.clone()).await.unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from(vec![42u8; 1024]); + + backend.put_object(&object_id, data.clone()).await.unwrap(); + + // Verify shards were written to nodes + let nodes = registry.all_mock_nodes(); + let total_chunks: usize = nodes.iter().map(|n| n.chunk_count()).sum(); + assert!(total_chunks >= 4); // At least data shards should be stored + + // Get the object back + let retrieved = backend.get_object(&object_id).await.unwrap(); + + // The retrieved data might have padding, but should contain original data + assert!(retrieved.len() >= data.len()); + assert_eq!(&retrieved[..data.len()], &data[..]); + } + + #[tokio::test] + async fn test_ec_backend_tolerates_failures() { + let config = create_ec_config(4, 2); + let registry = Arc::new(MockNodeRegistry::with_nodes(6)); + + let backend = ErasureCodedBackend::new(config, registry.clone()).await.unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from(vec![42u8; 512]); + + backend.put_object(&object_id, data.clone()).await.unwrap(); + + // Fail 2 nodes (within parity tolerance) + let nodes = registry.all_mock_nodes(); + nodes[0].set_healthy(false); + nodes[1].set_healthy(false); + + // Should still be able to read + let retrieved = backend.get_object(&object_id).await.unwrap(); + assert!(retrieved.len() >= data.len()); + assert_eq!(&retrieved[..data.len()], &data[..]); + } + + #[tokio::test] + async fn test_ec_backend_delete() { + let config = create_ec_config(4, 2); + let registry = Arc::new(MockNodeRegistry::with_nodes(6)); + + let backend = ErasureCodedBackend::new(config, registry.clone()).await.unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from(vec![42u8; 256]); + + backend.put_object(&object_id, data).await.unwrap(); + assert!(backend.object_exists(&object_id).await.unwrap()); + + backend.delete_object(&object_id).await.unwrap(); + + // After deletion, object should not exist + // (All shards should be deleted) + assert!(!backend.object_exists(&object_id).await.unwrap()); + } + + #[tokio::test] + async fn test_ec_backend_not_enough_nodes() { + let config = create_ec_config(4, 2); + // Only 3 nodes, but need 6 for 4+2 EC + let registry = Arc::new(MockNodeRegistry::with_nodes(3)); + + let backend = ErasureCodedBackend::new(config, registry).await.unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from(vec![42u8; 256]); + + let result = backend.put_object(&object_id, data).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_ec_backend_multi_chunk() { + // Create config with small chunk size to force multiple chunks + let config = DistributedConfig { + redundancy: RedundancyMode::ErasureCoded { + data_shards: 4, + parity_shards: 2, + }, + chunk: ChunkConfig::new(1024), // 1 KB chunks + ..Default::default() + }; + let registry = Arc::new(MockNodeRegistry::with_nodes(6)); + + let backend = ErasureCodedBackend::new(config, registry.clone()).await.unwrap(); + + let object_id = ObjectId::new(); + // Create data larger than chunk size (3 KB = 3 chunks) + let data: Vec = (0..3072).map(|i| (i % 256) as u8).collect(); + let data = Bytes::from(data); + + // Store the object + backend.put_object(&object_id, data.clone()).await.unwrap(); + + // Verify it exists + assert!(backend.object_exists(&object_id).await.unwrap()); + + // Get object size + let size = backend.object_size(&object_id).await.unwrap(); + assert_eq!(size, 3072); + + // Retrieve the object + let retrieved = backend.get_object(&object_id).await.unwrap(); + + // Verify data integrity + assert_eq!(retrieved.len(), data.len()); + assert_eq!(retrieved, data); + + // Delete the object + backend.delete_object(&object_id).await.unwrap(); + + // Verify it's deleted + assert!(!backend.object_exists(&object_id).await.unwrap()); + } + + #[tokio::test] + async fn test_ec_backend_multi_chunk_with_failures() { + // Create config with small chunk size + let config = DistributedConfig { + redundancy: RedundancyMode::ErasureCoded { + data_shards: 4, + parity_shards: 2, + }, + chunk: ChunkConfig::new(512), // 512 byte chunks + ..Default::default() + }; + let registry = Arc::new(MockNodeRegistry::with_nodes(6)); + + let backend = ErasureCodedBackend::new(config, registry.clone()).await.unwrap(); + + let object_id = ObjectId::new(); + // 2 KB = 4 chunks with 512 byte chunk size + let data: Vec = (0..2048).map(|i| (i % 256) as u8).collect(); + let data = Bytes::from(data); + + // Store the object + backend.put_object(&object_id, data.clone()).await.unwrap(); + + // Fail 2 nodes (within parity tolerance) + let nodes = registry.all_mock_nodes(); + nodes[0].set_healthy(false); + nodes[1].set_healthy(false); + + // Should still be able to read + let retrieved = backend.get_object(&object_id).await.unwrap(); + assert_eq!(retrieved.len(), data.len()); + assert_eq!(retrieved, data); + } +} diff --git a/lightningstor/crates/lightningstor-distributed/src/backends/mod.rs b/lightningstor/crates/lightningstor-distributed/src/backends/mod.rs new file mode 100644 index 0000000..340b7cd --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/backends/mod.rs @@ -0,0 +1,10 @@ +//! Distributed storage backend implementations +//! +//! This module provides storage backends that distribute data across +//! multiple nodes using either erasure coding or replication. + +pub mod erasure_coded; +pub mod replicated; + +pub use erasure_coded::ErasureCodedBackend; +pub use replicated::ReplicatedBackend; diff --git a/lightningstor/crates/lightningstor-distributed/src/backends/replicated.rs b/lightningstor/crates/lightningstor-distributed/src/backends/replicated.rs new file mode 100644 index 0000000..86d1b5e --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/backends/replicated.rs @@ -0,0 +1,535 @@ +//! Replicated distributed storage backend +//! +//! Implements StorageBackend using N-way replication for +//! performance-oriented redundancy with read scaling. + +use crate::config::DistributedConfig; +use crate::node::{NodeClientTrait, NodeRegistry}; +use crate::placement::{ConsistentHashSelector, NodeSelector}; +use async_trait::async_trait; +use bytes::Bytes; +use lightningstor_storage::{StorageBackend, StorageError, StorageResult}; +use lightningstor_types::ObjectId; +use std::sync::Arc; +use tracing::{debug, error, warn}; + +/// Replicated storage backend with N-way replication +/// +/// Stores objects by replicating them to N nodes. Provides: +/// - Fast reads (any replica can serve) +/// - Simple failure handling (no reconstruction needed) +/// - Higher storage overhead than erasure coding +pub struct ReplicatedBackend { + /// Node registry for discovering storage nodes + node_registry: Arc, + /// Node selector for placement decisions + node_selector: Arc, + /// Configuration (kept for future use) + #[allow(dead_code)] + config: DistributedConfig, + /// Number of replicas + replica_count: usize, + /// Read quorum (minimum replicas for successful read) + read_quorum: usize, + /// Write quorum (minimum replicas for successful write) + write_quorum: usize, +} + +impl ReplicatedBackend { + /// Create a new replicated backend + /// + /// # Arguments + /// * `config` - Distributed storage configuration (must have Replicated redundancy mode) + /// * `node_registry` - Registry for discovering storage nodes + pub async fn new( + config: DistributedConfig, + node_registry: Arc, + ) -> StorageResult { + let (replica_count, read_quorum, write_quorum) = match &config.redundancy { + crate::config::RedundancyMode::Replicated { + replica_count, + read_quorum, + write_quorum, + } => (*replica_count, *read_quorum, *write_quorum), + _ => { + return Err(StorageError::Backend( + "ReplicatedBackend requires Replicated redundancy mode".into(), + )) + } + }; + + let node_selector = Arc::new(ConsistentHashSelector::new()); + + Ok(Self { + node_registry, + node_selector, + config, + replica_count, + read_quorum, + write_quorum, + }) + } + + /// Get the number of replicas + pub fn replica_count(&self) -> usize { + self.replica_count + } + + /// Get the read quorum + pub fn read_quorum(&self) -> usize { + self.read_quorum + } + + /// Get the write quorum + pub fn write_quorum(&self) -> usize { + self.write_quorum + } + + /// Select nodes for writing replicas + async fn select_replica_nodes(&self) -> StorageResult>> { + let nodes = self + .node_registry + .get_healthy_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + if nodes.len() < self.replica_count { + return Err(StorageError::Backend(format!( + "Not enough healthy nodes: need {}, have {}", + self.replica_count, + nodes.len() + ))); + } + + self.node_selector + .select_nodes(&nodes, self.replica_count) + .await + .map_err(|e| StorageError::Backend(e.to_string())) + } + + /// Generate the chunk key for an object + fn object_key(object_id: &ObjectId) -> String { + format!("obj_{}", object_id) + } + + /// Generate the chunk key for a part + fn part_key(upload_id: &str, part_number: u32) -> String { + format!("part_{}_{}", upload_id, part_number) + } +} + +#[async_trait] +impl StorageBackend for ReplicatedBackend { + async fn put_object(&self, object_id: &ObjectId, data: Bytes) -> StorageResult<()> { + debug!( + object_id = %object_id, + size = data.len(), + replicas = self.replica_count, + "Putting object with replication" + ); + + let nodes = self.select_replica_nodes().await?; + let chunk_key = Self::object_key(object_id); + + // Write to all replicas in parallel + let mut write_futures = Vec::with_capacity(self.replica_count); + for node in nodes.iter() { + let node = node.clone(); + let key = chunk_key.clone(); + let data = data.clone(); + + write_futures.push(async move { node.put_chunk(&key, 0, false, data).await }); + } + + let results = futures::future::join_all(write_futures).await; + let success_count = results.iter().filter(|r| r.is_ok()).count(); + let error_count = results.len() - success_count; + + debug!( + object_id = %object_id, + success_count, + error_count, + write_quorum = self.write_quorum, + "Wrote replicas" + ); + + // Need write quorum for success + if success_count < self.write_quorum { + let errors: Vec<_> = results.into_iter().filter_map(|r| r.err()).collect(); + error!( + success_count, + write_quorum = self.write_quorum, + errors = ?errors, + "Failed to write quorum" + ); + return Err(StorageError::Backend(format!( + "Failed to write quorum: {} of {} required succeeded", + success_count, self.write_quorum + ))); + } + + Ok(()) + } + + async fn get_object(&self, object_id: &ObjectId) -> StorageResult { + debug!(object_id = %object_id, "Getting object from replicas"); + + let nodes = self + .node_registry + .get_healthy_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + let chunk_key = Self::object_key(object_id); + + // Try to read from the preferred node first (for cache efficiency) + if let Ok(preferred) = self.node_selector.select_for_read(&nodes, &chunk_key).await { + match preferred.get_chunk(&chunk_key, 0, false).await { + Ok(data) => { + debug!( + object_id = %object_id, + node_id = preferred.node_id(), + "Read from preferred node" + ); + return Ok(Bytes::from(data)); + } + Err(e) => { + warn!( + object_id = %object_id, + node_id = preferred.node_id(), + error = ?e, + "Failed to read from preferred node, trying others" + ); + } + } + } + + // Try other nodes + for node in nodes.iter() { + match node.get_chunk(&chunk_key, 0, false).await { + Ok(data) => { + debug!( + object_id = %object_id, + node_id = node.node_id(), + "Read from fallback node" + ); + return Ok(Bytes::from(data)); + } + Err(_) => continue, + } + } + + Err(StorageError::NotFound(*object_id)) + } + + async fn delete_object(&self, object_id: &ObjectId) -> StorageResult<()> { + debug!(object_id = %object_id, "Deleting object from all replicas"); + + let nodes = self + .node_registry + .get_all_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + let chunk_key = Self::object_key(object_id); + + // Delete from all nodes (best effort) + let mut delete_futures = Vec::new(); + for node in &nodes { + let node = node.clone(); + let key = chunk_key.clone(); + delete_futures.push(async move { + if let Err(e) = node.delete_chunk(&key).await { + warn!( + node_id = node.node_id(), + chunk_key = key, + error = ?e, + "Failed to delete replica" + ); + } + }); + } + + futures::future::join_all(delete_futures).await; + Ok(()) + } + + async fn object_exists(&self, object_id: &ObjectId) -> StorageResult { + let nodes = self + .node_registry + .get_healthy_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + let chunk_key = Self::object_key(object_id); + + for node in &nodes { + if let Ok(true) = node.chunk_exists(&chunk_key).await { + return Ok(true); + } + } + + Ok(false) + } + + async fn object_size(&self, object_id: &ObjectId) -> StorageResult { + let nodes = self + .node_registry + .get_healthy_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + let chunk_key = Self::object_key(object_id); + + for node in &nodes { + if let Ok(Some(size)) = node.chunk_size(&chunk_key).await { + return Ok(size); + } + } + + Err(StorageError::NotFound(*object_id)) + } + + async fn put_part( + &self, + upload_id: &str, + part_number: u32, + data: Bytes, + ) -> StorageResult<()> { + debug!( + upload_id, + part_number, + size = data.len(), + "Putting multipart part with replication" + ); + + let nodes = self.select_replica_nodes().await?; + let chunk_key = Self::part_key(upload_id, part_number); + + // Write to all replicas in parallel + let mut write_futures = Vec::with_capacity(self.replica_count); + for node in nodes.iter() { + let node = node.clone(); + let key = chunk_key.clone(); + let data = data.clone(); + + write_futures.push(async move { node.put_chunk(&key, part_number, false, data).await }); + } + + let results = futures::future::join_all(write_futures).await; + let success_count = results.iter().filter(|r| r.is_ok()).count(); + + if success_count < self.write_quorum { + return Err(StorageError::Backend(format!( + "Failed to write part quorum: {} of {} required", + success_count, self.write_quorum + ))); + } + + Ok(()) + } + + async fn get_part(&self, upload_id: &str, part_number: u32) -> StorageResult { + let nodes = self + .node_registry + .get_healthy_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + let chunk_key = Self::part_key(upload_id, part_number); + + // Try nodes until we get a successful read + for node in nodes.iter() { + match node.get_chunk(&chunk_key, part_number, false).await { + Ok(data) => return Ok(Bytes::from(data)), + Err(_) => continue, + } + } + + Err(StorageError::Backend(format!( + "Part {}:{} not found on any node", + upload_id, part_number + ))) + } + + async fn delete_part(&self, upload_id: &str, part_number: u32) -> StorageResult<()> { + let nodes = self + .node_registry + .get_all_nodes() + .await + .map_err(|e| StorageError::Backend(e.to_string()))?; + + let chunk_key = Self::part_key(upload_id, part_number); + + // Delete from all nodes (best effort) + let mut delete_futures = Vec::new(); + for node in &nodes { + let node = node.clone(); + let key = chunk_key.clone(); + delete_futures.push(async move { + let _ = node.delete_chunk(&key).await; + }); + } + + futures::future::join_all(delete_futures).await; + Ok(()) + } + + async fn delete_upload_parts(&self, upload_id: &str) -> StorageResult<()> { + // Would need to track part numbers in metadata to delete all parts + // For now, just log and return success + debug!(upload_id, "delete_upload_parts called (no-op without metadata tracking)"); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::RedundancyMode; + use crate::node::MockNodeRegistry; + + fn create_replicated_config(replica_count: usize) -> DistributedConfig { + DistributedConfig { + redundancy: RedundancyMode::Replicated { + replica_count, + read_quorum: 1, + write_quorum: (replica_count / 2) + 1, + }, + ..Default::default() + } + } + + #[tokio::test] + async fn test_replicated_backend_creation() { + let config = create_replicated_config(3); + let registry = Arc::new(MockNodeRegistry::with_nodes(3)); + + let backend = ReplicatedBackend::new(config, registry).await.unwrap(); + + assert_eq!(backend.replica_count(), 3); + assert_eq!(backend.read_quorum(), 1); + assert_eq!(backend.write_quorum(), 2); + } + + #[tokio::test] + async fn test_replicated_backend_put_get() { + let config = create_replicated_config(3); + let registry = Arc::new(MockNodeRegistry::with_nodes(3)); + + let backend = ReplicatedBackend::new(config, registry.clone()).await.unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from(vec![42u8; 1024]); + + backend.put_object(&object_id, data.clone()).await.unwrap(); + + // Verify data was written to nodes + let nodes = registry.all_mock_nodes(); + let total_chunks: usize = nodes.iter().map(|n| n.chunk_count()).sum(); + assert!(total_chunks >= 2); // At least write_quorum nodes + + // Get the object back + let retrieved = backend.get_object(&object_id).await.unwrap(); + assert_eq!(retrieved, data); + } + + #[tokio::test] + async fn test_replicated_backend_tolerates_minority_failure() { + let config = create_replicated_config(3); + let registry = Arc::new(MockNodeRegistry::with_nodes(3)); + + let backend = ReplicatedBackend::new(config, registry.clone()).await.unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from(vec![42u8; 512]); + + backend.put_object(&object_id, data.clone()).await.unwrap(); + + // Fail 1 node (minority) + let nodes = registry.all_mock_nodes(); + nodes[0].set_healthy(false); + + // Should still be able to read from healthy nodes + let retrieved = backend.get_object(&object_id).await.unwrap(); + assert_eq!(retrieved, data); + } + + #[tokio::test] + async fn test_replicated_backend_write_quorum_failure() { + let config = create_replicated_config(3); + let registry = Arc::new(MockNodeRegistry::with_nodes(3)); + + let backend = ReplicatedBackend::new(config, registry.clone()).await.unwrap(); + + // Fail 2 nodes (below write quorum of 2) + let nodes = registry.all_mock_nodes(); + nodes[0].set_healthy(false); + nodes[1].set_healthy(false); + + let object_id = ObjectId::new(); + let data = Bytes::from(vec![42u8; 256]); + + // Write should fail due to insufficient healthy nodes + let result = backend.put_object(&object_id, data).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_replicated_backend_delete() { + let config = create_replicated_config(3); + let registry = Arc::new(MockNodeRegistry::with_nodes(3)); + + let backend = ReplicatedBackend::new(config, registry.clone()).await.unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from(vec![42u8; 256]); + + backend.put_object(&object_id, data).await.unwrap(); + assert!(backend.object_exists(&object_id).await.unwrap()); + + backend.delete_object(&object_id).await.unwrap(); + assert!(!backend.object_exists(&object_id).await.unwrap()); + } + + #[tokio::test] + async fn test_replicated_backend_object_size() { + let config = create_replicated_config(3); + let registry = Arc::new(MockNodeRegistry::with_nodes(3)); + + let backend = ReplicatedBackend::new(config, registry).await.unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from(vec![42u8; 1234]); + + backend.put_object(&object_id, data).await.unwrap(); + + let size = backend.object_size(&object_id).await.unwrap(); + assert_eq!(size, 1234); + } + + #[tokio::test] + async fn test_replicated_backend_multipart() { + let config = create_replicated_config(3); + let registry = Arc::new(MockNodeRegistry::with_nodes(3)); + + let backend = ReplicatedBackend::new(config, registry).await.unwrap(); + + let upload_id = "test-upload-123"; + let part1 = Bytes::from(vec![1u8; 1024]); + let part2 = Bytes::from(vec![2u8; 1024]); + + backend.put_part(upload_id, 1, part1.clone()).await.unwrap(); + backend.put_part(upload_id, 2, part2.clone()).await.unwrap(); + + let retrieved1 = backend.get_part(upload_id, 1).await.unwrap(); + let retrieved2 = backend.get_part(upload_id, 2).await.unwrap(); + + assert_eq!(retrieved1, part1); + assert_eq!(retrieved2, part2); + + backend.delete_part(upload_id, 1).await.unwrap(); + let result = backend.get_part(upload_id, 1).await; + assert!(result.is_err()); + } +} diff --git a/lightningstor/crates/lightningstor-distributed/src/chunk/mod.rs b/lightningstor/crates/lightningstor-distributed/src/chunk/mod.rs new file mode 100644 index 0000000..7e328fe --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/chunk/mod.rs @@ -0,0 +1,276 @@ +//! Chunk management for distributed storage +//! +//! This module handles splitting large objects into fixed-size chunks +//! and reassembling them back into the original data. + +use crate::config::ChunkConfig; + +/// Manages chunk operations for large objects +#[derive(Debug, Clone)] +pub struct ChunkManager { + config: ChunkConfig, +} + +impl ChunkManager { + /// Create a new chunk manager with the given configuration + pub fn new(config: ChunkConfig) -> Self { + Self { config } + } + + /// Create a new chunk manager with default configuration + pub fn with_defaults() -> Self { + Self::new(ChunkConfig::default()) + } + + /// Get the chunk size in bytes + pub fn chunk_size(&self) -> usize { + self.config.chunk_size + } + + /// Split data into chunks + /// + /// Returns a vector of chunks. Each chunk is at most `chunk_size` bytes, + /// except the last chunk which may be smaller. + pub fn split(&self, data: &[u8]) -> Vec> { + if data.is_empty() { + return vec![vec![]]; + } + + data.chunks(self.config.chunk_size) + .map(|c| c.to_vec()) + .collect() + } + + /// Reassemble chunks into original data + /// + /// Chunks must be in order and complete. + pub fn reassemble(&self, chunks: Vec>) -> Vec { + chunks.into_iter().flatten().collect() + } + + /// Calculate the number of chunks for a given data size + pub fn chunk_count(&self, size: usize) -> usize { + if size == 0 { + return 1; + } + (size + self.config.chunk_size - 1) / self.config.chunk_size + } + + /// Calculate the size of a specific chunk + /// + /// Returns the size of the chunk at the given index for data of the given total size. + pub fn chunk_size_at(&self, total_size: usize, chunk_index: usize) -> usize { + let full_chunks = total_size / self.config.chunk_size; + let remainder = total_size % self.config.chunk_size; + + if chunk_index < full_chunks { + self.config.chunk_size + } else if chunk_index == full_chunks && remainder > 0 { + remainder + } else { + 0 + } + } + + /// Calculate the byte range for a specific chunk + /// + /// Returns (start_offset, length) for the chunk at the given index. + pub fn chunk_range(&self, total_size: usize, chunk_index: usize) -> (usize, usize) { + let start = chunk_index * self.config.chunk_size; + let length = self.chunk_size_at(total_size, chunk_index); + (start, length) + } +} + +impl Default for ChunkManager { + fn default() -> Self { + Self::with_defaults() + } +} + +/// Represents a chunk identifier +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ChunkId { + /// The object ID this chunk belongs to + pub object_id: String, + /// The chunk index within the object + pub chunk_index: usize, + /// The shard index (for erasure coding) + pub shard_index: usize, + /// Whether this is a parity shard + pub is_parity: bool, +} + +impl ChunkId { + /// Create a new chunk ID for a simple (non-sharded) chunk + pub fn simple(object_id: impl Into, chunk_index: usize) -> Self { + Self { + object_id: object_id.into(), + chunk_index, + shard_index: 0, + is_parity: false, + } + } + + /// Create a new chunk ID for a data shard + pub fn data_shard( + object_id: impl Into, + chunk_index: usize, + shard_index: usize, + ) -> Self { + Self { + object_id: object_id.into(), + chunk_index, + shard_index, + is_parity: false, + } + } + + /// Create a new chunk ID for a parity shard + pub fn parity_shard( + object_id: impl Into, + chunk_index: usize, + shard_index: usize, + ) -> Self { + Self { + object_id: object_id.into(), + chunk_index, + shard_index, + is_parity: true, + } + } + + /// Convert to a string key for storage + pub fn to_key(&self) -> String { + format!( + "{}_{}_{}_{}", + self.object_id, + self.chunk_index, + self.shard_index, + if self.is_parity { "p" } else { "d" } + ) + } + + /// Parse from a string key + pub fn from_key(key: &str) -> Option { + let parts: Vec<&str> = key.rsplitn(4, '_').collect(); + if parts.len() != 4 { + return None; + } + + let is_parity = parts[0] == "p"; + let shard_index = parts[1].parse().ok()?; + let chunk_index = parts[2].parse().ok()?; + let object_id = parts[3].to_string(); + + Some(Self { + object_id, + chunk_index, + shard_index, + is_parity, + }) + } +} + +impl std::fmt::Display for ChunkId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.to_key()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_empty() { + let manager = ChunkManager::with_defaults(); + let chunks = manager.split(&[]); + assert_eq!(chunks.len(), 1); + assert!(chunks[0].is_empty()); + } + + #[test] + fn test_split_smaller_than_chunk() { + let manager = ChunkManager::new(ChunkConfig::new(1024)); + let data = vec![42u8; 512]; + let chunks = manager.split(&data); + assert_eq!(chunks.len(), 1); + assert_eq!(chunks[0], data); + } + + #[test] + fn test_split_exact_chunk_boundary() { + let manager = ChunkManager::new(ChunkConfig::new(1024)); + let data = vec![42u8; 2048]; + let chunks = manager.split(&data); + assert_eq!(chunks.len(), 2); + assert_eq!(chunks[0].len(), 1024); + assert_eq!(chunks[1].len(), 1024); + } + + #[test] + fn test_split_partial_last_chunk() { + let manager = ChunkManager::new(ChunkConfig::new(1024)); + let data = vec![42u8; 1500]; + let chunks = manager.split(&data); + assert_eq!(chunks.len(), 2); + assert_eq!(chunks[0].len(), 1024); + assert_eq!(chunks[1].len(), 476); + } + + #[test] + fn test_reassemble_preserves_data() { + let manager = ChunkManager::new(ChunkConfig::new(1024)); + let original: Vec = (0..2500).map(|i| (i % 256) as u8).collect(); + let chunks = manager.split(&original); + let reassembled = manager.reassemble(chunks); + assert_eq!(original, reassembled); + } + + #[test] + fn test_chunk_count() { + let manager = ChunkManager::new(ChunkConfig::new(1024)); + assert_eq!(manager.chunk_count(0), 1); + assert_eq!(manager.chunk_count(512), 1); + assert_eq!(manager.chunk_count(1024), 1); + assert_eq!(manager.chunk_count(1025), 2); + assert_eq!(manager.chunk_count(2048), 2); + assert_eq!(manager.chunk_count(3000), 3); + } + + #[test] + fn test_chunk_size_at() { + let manager = ChunkManager::new(ChunkConfig::new(1024)); + // 2500 bytes = 2 full chunks (1024) + 1 partial (452) + assert_eq!(manager.chunk_size_at(2500, 0), 1024); + assert_eq!(manager.chunk_size_at(2500, 1), 1024); + assert_eq!(manager.chunk_size_at(2500, 2), 452); + assert_eq!(manager.chunk_size_at(2500, 3), 0); + } + + #[test] + fn test_chunk_range() { + let manager = ChunkManager::new(ChunkConfig::new(1024)); + assert_eq!(manager.chunk_range(2500, 0), (0, 1024)); + assert_eq!(manager.chunk_range(2500, 1), (1024, 1024)); + assert_eq!(manager.chunk_range(2500, 2), (2048, 452)); + } + + #[test] + fn test_chunk_id_to_key() { + let id = ChunkId::data_shard("obj123", 0, 2); + assert_eq!(id.to_key(), "obj123_0_2_d"); + + let id = ChunkId::parity_shard("obj123", 1, 4); + assert_eq!(id.to_key(), "obj123_1_4_p"); + } + + #[test] + fn test_chunk_id_roundtrip() { + let original = ChunkId::data_shard("my-object", 5, 3); + let key = original.to_key(); + let parsed = ChunkId::from_key(&key).unwrap(); + assert_eq!(original, parsed); + } +} diff --git a/lightningstor/crates/lightningstor-distributed/src/config.rs b/lightningstor/crates/lightningstor-distributed/src/config.rs new file mode 100644 index 0000000..0b5e97e --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/config.rs @@ -0,0 +1,288 @@ +//! Configuration types for distributed storage backends + +use serde::{Deserialize, Serialize}; + +/// Redundancy strategy for object storage +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum RedundancyMode { + /// No redundancy (local storage only) + None, + + /// Reed-Solomon erasure coding + ErasureCoded { + /// Number of data shards + data_shards: usize, + /// Number of parity shards + parity_shards: usize, + }, + + /// Simple N-way replication + Replicated { + /// Number of replicas (including primary) + replica_count: usize, + /// Read quorum (minimum replicas for successful read) + read_quorum: usize, + /// Write quorum (minimum replicas for successful write) + write_quorum: usize, + }, +} + +impl Default for RedundancyMode { + fn default() -> Self { + // Default: 4+2 erasure coding (1.5x overhead, tolerates 2 failures) + Self::ErasureCoded { + data_shards: 4, + parity_shards: 2, + } + } +} + +impl RedundancyMode { + /// Create a new erasure coded configuration + pub fn erasure_coded(data_shards: usize, parity_shards: usize) -> Self { + Self::ErasureCoded { + data_shards, + parity_shards, + } + } + + /// Create a new replicated configuration with default quorums + pub fn replicated(replica_count: usize) -> Self { + Self::Replicated { + replica_count, + read_quorum: 1, + write_quorum: (replica_count / 2) + 1, + } + } + + /// Create a new replicated configuration with custom quorums + pub fn replicated_with_quorum( + replica_count: usize, + read_quorum: usize, + write_quorum: usize, + ) -> Self { + Self::Replicated { + replica_count, + read_quorum, + write_quorum, + } + } + + /// Get the minimum number of nodes required for this redundancy mode + pub fn min_nodes(&self) -> usize { + match self { + Self::None => 1, + Self::ErasureCoded { + data_shards, + parity_shards, + } => data_shards + parity_shards, + Self::Replicated { replica_count, .. } => *replica_count, + } + } + + /// Get the storage overhead factor (1.0 = no overhead) + pub fn overhead_factor(&self) -> f64 { + match self { + Self::None => 1.0, + Self::ErasureCoded { + data_shards, + parity_shards, + } => (*data_shards + *parity_shards) as f64 / *data_shards as f64, + Self::Replicated { replica_count, .. } => *replica_count as f64, + } + } + + /// Get the number of node failures that can be tolerated + pub fn fault_tolerance(&self) -> usize { + match self { + Self::None => 0, + Self::ErasureCoded { parity_shards, .. } => *parity_shards, + Self::Replicated { + replica_count, + write_quorum, + .. + } => replica_count - write_quorum, + } + } +} + +/// Chunk size configuration +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ChunkConfig { + /// Default chunk size in bytes (default: 8 MiB) + #[serde(default = "ChunkConfig::default_chunk_size")] + pub chunk_size: usize, + /// Minimum chunk size in bytes (default: 1 MiB) + #[serde(default = "ChunkConfig::default_min_chunk_size")] + pub min_chunk_size: usize, + /// Maximum chunk size in bytes (default: 64 MiB) + #[serde(default = "ChunkConfig::default_max_chunk_size")] + pub max_chunk_size: usize, +} + +impl ChunkConfig { + const fn default_chunk_size() -> usize { + 8 * 1024 * 1024 // 8 MiB + } + + const fn default_min_chunk_size() -> usize { + 1024 * 1024 // 1 MiB + } + + const fn default_max_chunk_size() -> usize { + 64 * 1024 * 1024 // 64 MiB + } + + /// Create a new chunk configuration with custom chunk size + pub fn new(chunk_size: usize) -> Self { + Self { + chunk_size, + min_chunk_size: Self::default_min_chunk_size(), + max_chunk_size: Self::default_max_chunk_size(), + } + } +} + +impl Default for ChunkConfig { + fn default() -> Self { + Self { + chunk_size: Self::default_chunk_size(), + min_chunk_size: Self::default_min_chunk_size(), + max_chunk_size: Self::default_max_chunk_size(), + } + } +} + +/// Distributed storage configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DistributedConfig { + /// Redundancy mode + #[serde(default)] + pub redundancy: RedundancyMode, + /// Chunk configuration + #[serde(default)] + pub chunk: ChunkConfig, + /// Node endpoints (for static configuration) + #[serde(default)] + pub node_endpoints: Vec, + /// Registry endpoint (for dynamic discovery via ChainFire) + pub registry_endpoint: Option, + /// Connection timeout in milliseconds + #[serde(default = "DistributedConfig::default_connection_timeout")] + pub connection_timeout_ms: u64, + /// Request timeout in milliseconds + #[serde(default = "DistributedConfig::default_request_timeout")] + pub request_timeout_ms: u64, + /// Maximum retries for failed operations + #[serde(default = "DistributedConfig::default_max_retries")] + pub max_retries: u32, +} + +impl DistributedConfig { + const fn default_connection_timeout() -> u64 { + 5000 // 5 seconds + } + + const fn default_request_timeout() -> u64 { + 30000 // 30 seconds + } + + const fn default_max_retries() -> u32 { + 3 + } +} + +impl Default for DistributedConfig { + fn default() -> Self { + Self { + redundancy: RedundancyMode::default(), + chunk: ChunkConfig::default(), + node_endpoints: vec![], + registry_endpoint: None, + connection_timeout_ms: Self::default_connection_timeout(), + request_timeout_ms: Self::default_request_timeout(), + max_retries: Self::default_max_retries(), + } + } +} + +/// Bucket-level storage configuration override +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct BucketStorageConfig { + /// Override redundancy mode for this bucket + pub redundancy: Option, + /// Override chunk size for this bucket + pub chunk_size: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_redundancy_mode_default() { + let mode = RedundancyMode::default(); + assert!(matches!( + mode, + RedundancyMode::ErasureCoded { + data_shards: 4, + parity_shards: 2 + } + )); + } + + #[test] + fn test_redundancy_mode_min_nodes() { + assert_eq!(RedundancyMode::None.min_nodes(), 1); + assert_eq!(RedundancyMode::erasure_coded(4, 2).min_nodes(), 6); + assert_eq!(RedundancyMode::replicated(3).min_nodes(), 3); + } + + #[test] + fn test_redundancy_mode_overhead() { + assert!((RedundancyMode::None.overhead_factor() - 1.0).abs() < f64::EPSILON); + assert!((RedundancyMode::erasure_coded(4, 2).overhead_factor() - 1.5).abs() < f64::EPSILON); + assert!((RedundancyMode::replicated(3).overhead_factor() - 3.0).abs() < f64::EPSILON); + } + + #[test] + fn test_redundancy_mode_fault_tolerance() { + assert_eq!(RedundancyMode::None.fault_tolerance(), 0); + assert_eq!(RedundancyMode::erasure_coded(4, 2).fault_tolerance(), 2); + // replica_count=3, write_quorum=2 -> can tolerate 1 failure + assert_eq!(RedundancyMode::replicated(3).fault_tolerance(), 1); + } + + #[test] + fn test_chunk_config_default() { + let config = ChunkConfig::default(); + assert_eq!(config.chunk_size, 8 * 1024 * 1024); + assert_eq!(config.min_chunk_size, 1024 * 1024); + assert_eq!(config.max_chunk_size, 64 * 1024 * 1024); + } + + #[test] + fn test_distributed_config_default() { + let config = DistributedConfig::default(); + assert!(matches!( + config.redundancy, + RedundancyMode::ErasureCoded { .. } + )); + assert!(config.node_endpoints.is_empty()); + assert!(config.registry_endpoint.is_none()); + } + + #[test] + fn test_redundancy_mode_serialization() { + let ec = RedundancyMode::erasure_coded(4, 2); + let json = serde_json::to_string(&ec).unwrap(); + let parsed: RedundancyMode = serde_json::from_str(&json).unwrap(); + assert_eq!(ec, parsed); + + let rep = RedundancyMode::replicated(3); + let json = serde_json::to_string(&rep).unwrap(); + let parsed: RedundancyMode = serde_json::from_str(&json).unwrap(); + assert_eq!(rep, parsed); + } +} diff --git a/lightningstor/crates/lightningstor-distributed/src/erasure/mod.rs b/lightningstor/crates/lightningstor-distributed/src/erasure/mod.rs new file mode 100644 index 0000000..f9693ea --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/erasure/mod.rs @@ -0,0 +1,381 @@ +//! Erasure coding module using Reed-Solomon +//! +//! This module provides a wrapper around the `reed-solomon-erasure` crate +//! for encoding and decoding data using Reed-Solomon erasure codes. + +use reed_solomon_erasure::galois_8::ReedSolomon; +use thiserror::Error; + +/// Errors that can occur during erasure coding operations +#[derive(Debug, Error)] +pub enum ErasureError { + #[error("Failed to create Reed-Solomon encoder: {0}")] + CreateError(String), + + #[error("Encoding failed: {0}")] + EncodeError(String), + + #[error("Decoding failed: {0}")] + DecodeError(String), + + #[error("Invalid shard count: expected {expected}, got {actual}")] + InvalidShardCount { expected: usize, actual: usize }, + + #[error("Not enough shards for reconstruction: need {needed}, have {available}")] + NotEnoughShards { needed: usize, available: usize }, + + #[error("Shard size mismatch: expected {expected}, got {actual}")] + ShardSizeMismatch { expected: usize, actual: usize }, +} + +/// Result type for erasure coding operations +pub type ErasureResult = Result; + +/// Reed-Solomon erasure coding codec +/// +/// Provides encoding and decoding of data using Reed-Solomon erasure codes. +/// Data is split into `data_shards` pieces, and `parity_shards` parity pieces +/// are generated. Any `data_shards` pieces (data or parity) are sufficient +/// to reconstruct the original data. +#[derive(Debug)] +pub struct Codec { + rs: ReedSolomon, + data_shards: usize, + parity_shards: usize, +} + +impl Codec { + /// Create a new Reed-Solomon codec + /// + /// # Arguments + /// * `data_shards` - Number of data shards (original data is split into this many pieces) + /// * `parity_shards` - Number of parity shards (fault tolerance) + /// + /// # Example + /// ``` + /// use lightningstor_distributed::erasure::Codec; + /// + /// // 4+2 configuration: 4 data shards, 2 parity shards + /// // Can tolerate loss of any 2 shards + /// let codec = Codec::new(4, 2).unwrap(); + /// ``` + pub fn new(data_shards: usize, parity_shards: usize) -> ErasureResult { + let rs = ReedSolomon::new(data_shards, parity_shards) + .map_err(|e| ErasureError::CreateError(e.to_string()))?; + + Ok(Self { + rs, + data_shards, + parity_shards, + }) + } + + /// Get the number of data shards + pub fn data_shards(&self) -> usize { + self.data_shards + } + + /// Get the number of parity shards + pub fn parity_shards(&self) -> usize { + self.parity_shards + } + + /// Get the total number of shards (data + parity) + pub fn total_shards(&self) -> usize { + self.data_shards + self.parity_shards + } + + /// Calculate the shard size for given data size + /// + /// Each shard will be this size (data is padded if necessary) + pub fn shard_size(&self, data_size: usize) -> usize { + // Round up to ensure all data fits + (data_size + self.data_shards - 1) / self.data_shards + } + + /// Encode data into shards + /// + /// Returns a vector of shards: first `data_shards` are data shards, + /// remaining `parity_shards` are parity shards. + /// + /// # Arguments + /// * `data` - The data to encode + /// + /// # Returns + /// A vector of `data_shards + parity_shards` shards, each of equal size. + pub fn encode(&self, data: &[u8]) -> ErasureResult>> { + if data.is_empty() { + // Handle empty data - create minimal shards + let shard_size = 1; + let mut shards: Vec> = (0..self.total_shards()) + .map(|_| vec![0u8; shard_size]) + .collect(); + + self.rs + .encode(&mut shards) + .map_err(|e| ErasureError::EncodeError(e.to_string()))?; + + return Ok(shards); + } + + let shard_size = self.shard_size(data.len()); + let total_shards = self.total_shards(); + + // Create shards with padding + let mut shards: Vec> = Vec::with_capacity(total_shards); + + // Fill data shards + for i in 0..self.data_shards { + let start = i * shard_size; + let end = std::cmp::min(start + shard_size, data.len()); + + let mut shard = vec![0u8; shard_size]; + if start < data.len() { + let copy_len = end - start; + shard[..copy_len].copy_from_slice(&data[start..end]); + } + shards.push(shard); + } + + // Create empty parity shards + for _ in 0..self.parity_shards { + shards.push(vec![0u8; shard_size]); + } + + // Encode (fills in parity shards) + self.rs + .encode(&mut shards) + .map_err(|e| ErasureError::EncodeError(e.to_string()))?; + + Ok(shards) + } + + /// Decode shards back into original data + /// + /// # Arguments + /// * `shards` - Vector of optional shards. `None` indicates a missing shard. + /// * `original_size` - The original data size (needed to remove padding) + /// + /// # Returns + /// The reconstructed original data. + pub fn decode(&self, shards: Vec>>, original_size: usize) -> ErasureResult> { + if shards.len() != self.total_shards() { + return Err(ErasureError::InvalidShardCount { + expected: self.total_shards(), + actual: shards.len(), + }); + } + + // Count available shards + let available = shards.iter().filter(|s| s.is_some()).count(); + if available < self.data_shards { + return Err(ErasureError::NotEnoughShards { + needed: self.data_shards, + available, + }); + } + + // Determine shard size from first available shard + let shard_size = shards + .iter() + .find_map(|s| s.as_ref().map(|v| v.len())) + .unwrap_or(1); + + // Verify all shards have same size + for shard in shards.iter() { + if let Some(s) = shard { + if s.len() != shard_size { + return Err(ErasureError::ShardSizeMismatch { + expected: shard_size, + actual: s.len(), + }); + } + } + } + + // Convert to the format expected by reed-solomon-erasure + let mut shard_refs: Vec>> = shards; + + // Reconstruct missing shards + self.rs + .reconstruct(&mut shard_refs) + .map_err(|e| ErasureError::DecodeError(e.to_string()))?; + + // Reassemble data from data shards + let mut result = Vec::with_capacity(original_size); + for shard in shard_refs.into_iter().take(self.data_shards) { + if let Some(data) = shard { + result.extend_from_slice(&data); + } + } + + // Trim to original size (remove padding) + result.truncate(original_size); + + Ok(result) + } + + /// Verify that shards are consistent + /// + /// Returns true if parity shards are correct for the given data shards. + pub fn verify(&self, shards: &[Vec]) -> ErasureResult { + if shards.len() != self.total_shards() { + return Err(ErasureError::InvalidShardCount { + expected: self.total_shards(), + actual: shards.len(), + }); + } + + let refs: Vec<&[u8]> = shards.iter().map(|s| s.as_slice()).collect(); + + Ok(self.rs.verify(&refs).unwrap_or(false)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_codec_creation() { + let codec = Codec::new(4, 2).unwrap(); + assert_eq!(codec.data_shards(), 4); + assert_eq!(codec.parity_shards(), 2); + assert_eq!(codec.total_shards(), 6); + } + + #[test] + fn test_encode_decode_roundtrip() { + let codec = Codec::new(4, 2).unwrap(); + let original = b"Hello, World! This is a test of erasure coding."; + + let shards = codec.encode(original).unwrap(); + assert_eq!(shards.len(), 6); + + // Decode with all shards + let all_shards: Vec>> = shards.into_iter().map(Some).collect(); + let decoded = codec.decode(all_shards, original.len()).unwrap(); + + assert_eq!(decoded, original); + } + + #[test] + fn test_decode_with_missing_data_shards() { + let codec = Codec::new(4, 2).unwrap(); + let original = b"Hello, World! This is a test of erasure coding."; + + let shards = codec.encode(original).unwrap(); + + // Remove 2 data shards (indices 0 and 1) + let mut partial_shards: Vec>> = shards.into_iter().map(Some).collect(); + partial_shards[0] = None; + partial_shards[1] = None; + + let decoded = codec.decode(partial_shards, original.len()).unwrap(); + assert_eq!(decoded, original); + } + + #[test] + fn test_decode_with_missing_parity_shards() { + let codec = Codec::new(4, 2).unwrap(); + let original = b"Hello, World! This is a test of erasure coding."; + + let shards = codec.encode(original).unwrap(); + + // Remove both parity shards (indices 4 and 5) + let mut partial_shards: Vec>> = shards.into_iter().map(Some).collect(); + partial_shards[4] = None; + partial_shards[5] = None; + + let decoded = codec.decode(partial_shards, original.len()).unwrap(); + assert_eq!(decoded, original); + } + + #[test] + fn test_decode_with_mixed_missing_shards() { + let codec = Codec::new(4, 2).unwrap(); + let original = b"Hello, World! This is a test of erasure coding."; + + let shards = codec.encode(original).unwrap(); + + // Remove 1 data shard and 1 parity shard + let mut partial_shards: Vec>> = shards.into_iter().map(Some).collect(); + partial_shards[2] = None; // data shard + partial_shards[5] = None; // parity shard + + let decoded = codec.decode(partial_shards, original.len()).unwrap(); + assert_eq!(decoded, original); + } + + #[test] + fn test_decode_fails_with_too_many_missing() { + let codec = Codec::new(4, 2).unwrap(); + let original = b"Hello, World!"; + + let shards = codec.encode(original).unwrap(); + + // Remove 3 shards (more than parity_shards) + let mut partial_shards: Vec>> = shards.into_iter().map(Some).collect(); + partial_shards[0] = None; + partial_shards[1] = None; + partial_shards[2] = None; + + let result = codec.decode(partial_shards, original.len()); + assert!(result.is_err()); + } + + #[test] + fn test_verify_valid_shards() { + let codec = Codec::new(4, 2).unwrap(); + let original = b"Test data for verification"; + + let shards = codec.encode(original).unwrap(); + assert!(codec.verify(&shards).unwrap()); + } + + #[test] + fn test_verify_corrupted_shards() { + let codec = Codec::new(4, 2).unwrap(); + let original = b"Test data for verification"; + + let mut shards = codec.encode(original).unwrap(); + + // Corrupt a shard + shards[0][0] ^= 0xFF; + + assert!(!codec.verify(&shards).unwrap()); + } + + #[test] + fn test_encode_empty_data() { + let codec = Codec::new(4, 2).unwrap(); + let shards = codec.encode(&[]).unwrap(); + assert_eq!(shards.len(), 6); + } + + #[test] + fn test_encode_large_data() { + let codec = Codec::new(4, 2).unwrap(); + let original: Vec = (0..10000).map(|i| (i % 256) as u8).collect(); + + let shards = codec.encode(&original).unwrap(); + let all_shards: Vec>> = shards.into_iter().map(Some).collect(); + let decoded = codec.decode(all_shards, original.len()).unwrap(); + + assert_eq!(decoded, original); + } + + #[test] + fn test_shard_size_calculation() { + let codec = Codec::new(4, 2).unwrap(); + + // 100 bytes / 4 shards = 25 bytes per shard + assert_eq!(codec.shard_size(100), 25); + + // 101 bytes / 4 shards = 26 bytes per shard (rounded up) + assert_eq!(codec.shard_size(101), 26); + + // 0 bytes = 0 bytes per shard + assert_eq!(codec.shard_size(0), 0); + } +} diff --git a/lightningstor/crates/lightningstor-distributed/src/lib.rs b/lightningstor/crates/lightningstor-distributed/src/lib.rs new file mode 100644 index 0000000..51a1acb --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/lib.rs @@ -0,0 +1,179 @@ +//! Distributed storage backends for LightningStor +//! +//! This crate provides distributed storage backends that implement redundancy +//! through either Reed-Solomon erasure coding or N-way replication. +//! +//! # Features +//! +//! - **Erasure Coding**: Storage-efficient redundancy using Reed-Solomon codes. +//! Configurable data/parity shard ratio (e.g., 4+2 for 1.5x overhead with 2-node +//! fault tolerance). +//! +//! - **Replication**: Performance-oriented redundancy with N-way replication. +//! Simple and fast, with configurable read/write quorums. +//! +//! - **Pluggable Node Management**: Support for static node configuration or +//! dynamic discovery via ChainFire. +//! +//! - **Placement Strategies**: Consistent hashing, random, and round-robin +//! node selection strategies. +//! +//! # Example +//! +//! ```rust,no_run +//! use lightningstor_distributed::{ +//! config::{DistributedConfig, RedundancyMode}, +//! backends::ErasureCodedBackend, +//! node::StaticNodeRegistry, +//! }; +//! use std::sync::Arc; +//! +//! # async fn example() -> Result<(), Box> { +//! // Configure 4+2 erasure coding +//! let config = DistributedConfig { +//! redundancy: RedundancyMode::ErasureCoded { +//! data_shards: 4, +//! parity_shards: 2, +//! }, +//! node_endpoints: vec![ +//! "http://node1:9002".into(), +//! "http://node2:9002".into(), +//! "http://node3:9002".into(), +//! "http://node4:9002".into(), +//! "http://node5:9002".into(), +//! "http://node6:9002".into(), +//! ], +//! ..Default::default() +//! }; +//! +//! // Create node registry +//! let registry = Arc::new( +//! StaticNodeRegistry::new(&config.node_endpoints).await? +//! ); +//! +//! // Create erasure-coded backend +//! let backend = ErasureCodedBackend::new(config, registry).await?; +//! +//! // Use the backend via StorageBackend trait +//! # Ok(()) +//! # } +//! ``` + +pub mod backends; +pub mod chunk; +pub mod config; +pub mod erasure; +pub mod node; +pub mod placement; + +// Re-export commonly used types +pub use backends::{ErasureCodedBackend, ReplicatedBackend}; +pub use config::{BucketStorageConfig, ChunkConfig, DistributedConfig, RedundancyMode}; +pub use node::{MockNodeClient, MockNodeRegistry, NodeRegistry, StaticNodeRegistry}; +pub use placement::{ConsistentHashSelector, NodeSelector, RandomSelector, RoundRobinSelector}; + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use lightningstor_storage::StorageBackend; + use lightningstor_types::ObjectId; + use std::sync::Arc; + + #[tokio::test] + async fn test_ec_backend_integration() { + let config = DistributedConfig { + redundancy: RedundancyMode::ErasureCoded { + data_shards: 4, + parity_shards: 2, + }, + ..Default::default() + }; + + let registry = Arc::new(MockNodeRegistry::with_nodes(6)); + let backend = ErasureCodedBackend::new(config, registry).await.unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from("Hello, erasure coded world!"); + + backend.put_object(&object_id, data.clone()).await.unwrap(); + let retrieved = backend.get_object(&object_id).await.unwrap(); + + assert!(retrieved.len() >= data.len()); + assert_eq!(&retrieved[..data.len()], &data[..]); + } + + #[tokio::test] + async fn test_replicated_backend_integration() { + let config = DistributedConfig { + redundancy: RedundancyMode::replicated(3), + ..Default::default() + }; + + let registry = Arc::new(MockNodeRegistry::with_nodes(3)); + let backend = ReplicatedBackend::new(config, registry).await.unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from("Hello, replicated world!"); + + backend.put_object(&object_id, data.clone()).await.unwrap(); + let retrieved = backend.get_object(&object_id).await.unwrap(); + + assert_eq!(retrieved, data); + } + + #[tokio::test] + async fn test_ec_with_node_failures() { + let config = DistributedConfig { + redundancy: RedundancyMode::ErasureCoded { + data_shards: 4, + parity_shards: 2, + }, + ..Default::default() + }; + + let registry = Arc::new(MockNodeRegistry::with_nodes(6)); + let backend = ErasureCodedBackend::new(config, registry.clone()) + .await + .unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from(vec![42u8; 1000]); + + backend.put_object(&object_id, data.clone()).await.unwrap(); + + // Fail 2 nodes (max tolerable for 4+2) + let nodes = registry.all_mock_nodes(); + nodes[0].set_healthy(false); + nodes[1].set_healthy(false); + + let retrieved = backend.get_object(&object_id).await.unwrap(); + assert!(retrieved.len() >= data.len()); + assert_eq!(&retrieved[..data.len()], &data[..]); + } + + #[tokio::test] + async fn test_replicated_with_node_failure() { + let config = DistributedConfig { + redundancy: RedundancyMode::replicated(3), + ..Default::default() + }; + + let registry = Arc::new(MockNodeRegistry::with_nodes(3)); + let backend = ReplicatedBackend::new(config, registry.clone()) + .await + .unwrap(); + + let object_id = ObjectId::new(); + let data = Bytes::from(vec![42u8; 1000]); + + backend.put_object(&object_id, data.clone()).await.unwrap(); + + // Fail 1 node + let nodes = registry.all_mock_nodes(); + nodes[0].set_healthy(false); + + let retrieved = backend.get_object(&object_id).await.unwrap(); + assert_eq!(retrieved, data); + } +} diff --git a/lightningstor/crates/lightningstor-distributed/src/node/client.rs b/lightningstor/crates/lightningstor-distributed/src/node/client.rs new file mode 100644 index 0000000..7bda20c --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/node/client.rs @@ -0,0 +1,403 @@ +//! Node client for communicating with storage nodes + +use super::{NodeError, NodeResult}; +use async_trait::async_trait; +use bytes::Bytes; +use lightningstor_node::proto::{ + ChunkExistsRequest, ChunkSizeRequest, DeleteChunkRequest, GetChunkRequest, PingRequest, + PutChunkRequest, +}; +use lightningstor_node::NodeServiceClient; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::RwLock; +use tonic::transport::Channel; + +/// Trait for storage node client operations +#[async_trait] +pub trait NodeClientTrait: Send + Sync { + /// Get the node ID + fn node_id(&self) -> &str; + + /// Get the node endpoint + fn endpoint(&self) -> &str; + + /// Check if the node is currently considered healthy + async fn is_healthy(&self) -> bool; + + /// Store a chunk on this node + async fn put_chunk( + &self, + chunk_id: &str, + shard_index: u32, + is_parity: bool, + data: Bytes, + ) -> NodeResult<()>; + + /// Retrieve a chunk from this node + async fn get_chunk( + &self, + chunk_id: &str, + shard_index: u32, + is_parity: bool, + ) -> NodeResult>; + + /// Delete a chunk from this node + async fn delete_chunk(&self, chunk_id: &str) -> NodeResult<()>; + + /// Check if a chunk exists on this node + async fn chunk_exists(&self, chunk_id: &str) -> NodeResult; + + /// Get the size of a chunk on this node + async fn chunk_size(&self, chunk_id: &str) -> NodeResult>; + + /// Ping the node to check connectivity + async fn ping(&self) -> NodeResult; +} + +/// Real gRPC client for storage nodes +/// +/// This client communicates with storage nodes over gRPC. +/// For now, this is a placeholder that will be implemented +/// when the storage node service is created. +pub struct NodeClient { + node_id: String, + endpoint: String, + healthy: AtomicBool, + client: RwLock>, +} + +impl NodeClient { + /// Connect to a storage node at the given endpoint + pub async fn connect(endpoint: &str) -> NodeResult { + // Ensure endpoint has scheme + let endpoint_url = if endpoint.contains("://") { + endpoint.to_string() + } else { + format!("http://{}", endpoint) + }; + + let channel = Channel::from_shared(endpoint_url.clone()) + .map_err(|e| NodeError::ConnectionFailed { + node_id: "unknown".to_string(), + reason: e.to_string(), + })? + .connect_timeout(Duration::from_secs(5)) + .connect() + .await + .map_err(|e| NodeError::ConnectionFailed { + node_id: "unknown".to_string(), + reason: e.to_string(), + })?; + + let client = NodeServiceClient::new(channel); + + // Try to get node status to get the real node ID + // If that fails, generate a temporary one based on endpoint, but connection is established + let node_id = match client.clone().get_status(lightningstor_node::proto::GetStatusRequest {}).await { + Ok(response) => response.into_inner().node_id, + Err(_) => format!("node-{}", endpoint.replace([':', '.', '/'], "-")), + }; + + Ok(Self { + node_id, + endpoint: endpoint.to_string(), + healthy: AtomicBool::new(true), + client: RwLock::new(client), + }) + } + + /// Create a client with a specific node ID + pub async fn connect_with_id(node_id: &str, endpoint: &str) -> NodeResult { + let endpoint_url = if endpoint.contains("://") { + endpoint.to_string() + } else { + format!("http://{}", endpoint) + }; + + // We use lazy connection here to not block startup if a node is temporarily down + let channel = Channel::from_shared(endpoint_url.clone()) + .map_err(|e| NodeError::ConnectionFailed { + node_id: node_id.to_string(), + reason: e.to_string(), + })? + .connect_timeout(Duration::from_secs(5)) + .connect_lazy(); + + let client = NodeServiceClient::new(channel); + + Ok(Self { + node_id: node_id.to_string(), + endpoint: endpoint.to_string(), + healthy: AtomicBool::new(true), + client: RwLock::new(client), + }) + } + + /// Mark the node as unhealthy + pub fn mark_unhealthy(&self) { + self.healthy.store(false, Ordering::SeqCst); + } + + /// Mark the node as healthy + pub fn mark_healthy(&self) { + self.healthy.store(true, Ordering::SeqCst); + } +} + +#[async_trait] +impl NodeClientTrait for NodeClient { + fn node_id(&self) -> &str { + &self.node_id + } + + fn endpoint(&self) -> &str { + &self.endpoint + } + + async fn is_healthy(&self) -> bool { + self.healthy.load(Ordering::SeqCst) + } + + async fn put_chunk( + &self, + chunk_id: &str, + shard_index: u32, + is_parity: bool, + data: Bytes, + ) -> NodeResult<()> { + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + let request = PutChunkRequest { + chunk_id: chunk_id.to_string(), + shard_index, + is_parity, + data: data.to_vec(), + }; + + let mut client = self.client.write().await; + client + .put_chunk(request) + .await + .map(|_| ()) + .map_err(|e| NodeError::RpcFailed(e.to_string())) + } + + async fn get_chunk( + &self, + chunk_id: &str, + shard_index: u32, + is_parity: bool, + ) -> NodeResult> { + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + let request = GetChunkRequest { + chunk_id: chunk_id.to_string(), + shard_index, + is_parity, + }; + + let mut client = self.client.write().await; + let response = client + .get_chunk(request) + .await + .map_err(|e| match e.code() { + tonic::Code::NotFound => NodeError::NotFound(chunk_id.to_string()), + _ => NodeError::RpcFailed(e.to_string()), + })?; + + Ok(response.into_inner().data) + } + + async fn delete_chunk(&self, chunk_id: &str) -> NodeResult<()> { + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + let request = DeleteChunkRequest { + chunk_id: chunk_id.to_string(), + }; + + let mut client = self.client.write().await; + client + .delete_chunk(request) + .await + .map(|_| ()) + .map_err(|e| NodeError::RpcFailed(e.to_string())) + } + + async fn chunk_exists(&self, chunk_id: &str) -> NodeResult { + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + let request = ChunkExistsRequest { + chunk_id: chunk_id.to_string(), + }; + + let mut client = self.client.write().await; + let response = client + .chunk_exists(request) + .await + .map_err(|e| NodeError::RpcFailed(e.to_string()))?; + + Ok(response.into_inner().exists) + } + + async fn chunk_size(&self, chunk_id: &str) -> NodeResult> { + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + let request = ChunkSizeRequest { + chunk_id: chunk_id.to_string(), + }; + + let mut client = self.client.write().await; + let response = client + .chunk_size(request) + .await + .map_err(|e| NodeError::RpcFailed(e.to_string()))?; + + let inner = response.into_inner(); + if inner.exists { + Ok(Some(inner.size)) + } else { + Ok(None) + } + } + + async fn ping(&self) -> NodeResult { + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + let start = std::time::Instant::now(); + let request = PingRequest {}; + + let mut client = self.client.write().await; + let _ = client + .ping(request) + .await + .map_err(|e| NodeError::RpcFailed(e.to_string()))?; + + Ok(start.elapsed()) + } +} + +/// A pool of node clients for connection reuse +pub struct NodeClientPool { + clients: RwLock>>, +} + +impl NodeClientPool { + /// Create a new empty client pool + pub fn new() -> Self { + Self { + clients: RwLock::new(Vec::new()), + } + } + + /// Add a client to the pool + pub async fn add(&self, client: Arc) { + self.clients.write().await.push(client); + } + + /// Get all clients in the pool + pub async fn all(&self) -> Vec> { + self.clients.read().await.clone() + } + + /// Get all healthy clients + pub async fn healthy(&self) -> Vec> { + let clients = self.clients.read().await; + let mut healthy = Vec::new(); + for client in clients.iter() { + if client.is_healthy().await { + healthy.push(client.clone()); + } + } + healthy + } + + /// Get a client by node ID + pub async fn get(&self, node_id: &str) -> Option> { + self.clients + .read() + .await + .iter() + .find(|c| c.node_id() == node_id) + .cloned() + } + + /// Remove a client from the pool + pub async fn remove(&self, node_id: &str) { + self.clients + .write() + .await + .retain(|c| c.node_id() != node_id); + } + + /// Get the number of clients in the pool + pub async fn len(&self) -> usize { + self.clients.read().await.len() + } + + /// Check if the pool is empty + pub async fn is_empty(&self) -> bool { + self.clients.read().await.is_empty() + } +} + +impl Default for NodeClientPool { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_node_client_creation() { + let client = NodeClient::connect("http://localhost:9002").await.unwrap(); + assert!(client.is_healthy().await); + assert!(!client.node_id().is_empty()); + } + + #[tokio::test] + async fn test_node_client_health_toggle() { + let client = NodeClient::connect("http://localhost:9002").await.unwrap(); + + assert!(client.is_healthy().await); + client.mark_unhealthy(); + assert!(!client.is_healthy().await); + client.mark_healthy(); + assert!(client.is_healthy().await); + } + + #[tokio::test] + async fn test_node_client_pool() { + let pool = NodeClientPool::new(); + assert!(pool.is_empty().await); + + let client1 = Arc::new(NodeClient::connect("http://node1:9002").await.unwrap()); + let client2 = Arc::new(NodeClient::connect("http://node2:9002").await.unwrap()); + + pool.add(client1.clone()).await; + pool.add(client2.clone()).await; + + assert_eq!(pool.len().await, 2); + assert!(pool.get(client1.node_id()).await.is_some()); + + pool.remove(client1.node_id()).await; + assert_eq!(pool.len().await, 1); + assert!(pool.get(client1.node_id()).await.is_none()); + } +} diff --git a/lightningstor/crates/lightningstor-distributed/src/node/mock.rs b/lightningstor/crates/lightningstor-distributed/src/node/mock.rs new file mode 100644 index 0000000..713601e --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/node/mock.rs @@ -0,0 +1,408 @@ +//! Mock implementations for testing + +use super::client::NodeClientTrait; +use super::registry::{NodeInfo, NodeRegistry}; +use super::{NodeError, NodeResult}; +use async_trait::async_trait; +use bytes::Bytes; +use dashmap::DashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +/// Mock storage node client for testing +/// +/// Stores chunks in memory and allows simulation of failures. +pub struct MockNodeClient { + node_id: String, + endpoint: String, + chunks: DashMap>, + healthy: AtomicBool, + // Counters for verification + put_count: AtomicU64, + get_count: AtomicU64, + delete_count: AtomicU64, + // Failure injection + fail_puts: AtomicBool, + fail_gets: AtomicBool, + fail_deletes: AtomicBool, +} + +impl MockNodeClient { + /// Create a new mock node client + pub fn new(node_id: impl Into, endpoint: impl Into) -> Self { + Self { + node_id: node_id.into(), + endpoint: endpoint.into(), + chunks: DashMap::new(), + healthy: AtomicBool::new(true), + put_count: AtomicU64::new(0), + get_count: AtomicU64::new(0), + delete_count: AtomicU64::new(0), + fail_puts: AtomicBool::new(false), + fail_gets: AtomicBool::new(false), + fail_deletes: AtomicBool::new(false), + } + } + + /// Set the health status of this mock node + pub fn set_healthy(&self, healthy: bool) { + self.healthy.store(healthy, Ordering::SeqCst); + } + + /// Enable/disable put failures + pub fn set_fail_puts(&self, fail: bool) { + self.fail_puts.store(fail, Ordering::SeqCst); + } + + /// Enable/disable get failures + pub fn set_fail_gets(&self, fail: bool) { + self.fail_gets.store(fail, Ordering::SeqCst); + } + + /// Enable/disable delete failures + pub fn set_fail_deletes(&self, fail: bool) { + self.fail_deletes.store(fail, Ordering::SeqCst); + } + + /// Get the count of put operations + pub fn put_count(&self) -> u64 { + self.put_count.load(Ordering::SeqCst) + } + + /// Get the count of get operations + pub fn get_count(&self) -> u64 { + self.get_count.load(Ordering::SeqCst) + } + + /// Get the count of delete operations + pub fn delete_count(&self) -> u64 { + self.delete_count.load(Ordering::SeqCst) + } + + /// Get all stored chunk IDs + pub fn chunk_ids(&self) -> Vec { + self.chunks.iter().map(|r| r.key().clone()).collect() + } + + /// Get the number of stored chunks + pub fn chunk_count(&self) -> usize { + self.chunks.len() + } + + /// Clear all stored chunks + pub fn clear(&self) { + self.chunks.clear(); + } + + /// Reset all counters + pub fn reset_counters(&self) { + self.put_count.store(0, Ordering::SeqCst); + self.get_count.store(0, Ordering::SeqCst); + self.delete_count.store(0, Ordering::SeqCst); + } +} + +#[async_trait] +impl NodeClientTrait for MockNodeClient { + fn node_id(&self) -> &str { + &self.node_id + } + + fn endpoint(&self) -> &str { + &self.endpoint + } + + async fn is_healthy(&self) -> bool { + self.healthy.load(Ordering::SeqCst) + } + + async fn put_chunk( + &self, + chunk_id: &str, + _shard_index: u32, + _is_parity: bool, + data: Bytes, + ) -> NodeResult<()> { + self.put_count.fetch_add(1, Ordering::SeqCst); + + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + if self.fail_puts.load(Ordering::SeqCst) { + return Err(NodeError::RpcFailed("Simulated put failure".into())); + } + + self.chunks.insert(chunk_id.to_string(), data.to_vec()); + Ok(()) + } + + async fn get_chunk( + &self, + chunk_id: &str, + _shard_index: u32, + _is_parity: bool, + ) -> NodeResult> { + self.get_count.fetch_add(1, Ordering::SeqCst); + + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + if self.fail_gets.load(Ordering::SeqCst) { + return Err(NodeError::RpcFailed("Simulated get failure".into())); + } + + self.chunks + .get(chunk_id) + .map(|r| r.value().clone()) + .ok_or_else(|| NodeError::NotFound(chunk_id.to_string())) + } + + async fn delete_chunk(&self, chunk_id: &str) -> NodeResult<()> { + self.delete_count.fetch_add(1, Ordering::SeqCst); + + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + if self.fail_deletes.load(Ordering::SeqCst) { + return Err(NodeError::RpcFailed("Simulated delete failure".into())); + } + + self.chunks.remove(chunk_id); + Ok(()) + } + + async fn chunk_exists(&self, chunk_id: &str) -> NodeResult { + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + Ok(self.chunks.contains_key(chunk_id)) + } + + async fn chunk_size(&self, chunk_id: &str) -> NodeResult> { + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + Ok(self.chunks.get(chunk_id).map(|r| r.value().len() as u64)) + } + + async fn ping(&self) -> NodeResult { + if !self.is_healthy().await { + return Err(NodeError::Unhealthy(self.node_id.clone())); + } + + Ok(Duration::from_micros(100)) // Simulated latency + } +} + +/// Mock node registry for testing +pub struct MockNodeRegistry { + nodes: DashMap>, +} + +impl MockNodeRegistry { + /// Create a new empty mock registry + pub fn new() -> Self { + Self { + nodes: DashMap::new(), + } + } + + /// Add a mock node to the registry + pub fn add_mock_node(&self, node: Arc) { + self.nodes.insert(node.node_id().to_string(), node); + } + + /// Get a mock node by ID + pub fn get_mock_node(&self, node_id: &str) -> Option> { + self.nodes.get(node_id).map(|r| r.value().clone()) + } + + /// Create a registry with N mock nodes + pub fn with_nodes(count: usize) -> Self { + let registry = Self::new(); + for i in 0..count { + let node = Arc::new(MockNodeClient::new( + format!("node-{}", i), + format!("http://node-{}:9002", i), + )); + registry.add_mock_node(node); + } + registry + } + + /// Get all mock nodes + pub fn all_mock_nodes(&self) -> Vec> { + self.nodes.iter().map(|r| r.value().clone()).collect() + } +} + +impl Default for MockNodeRegistry { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl NodeRegistry for MockNodeRegistry { + async fn get_all_nodes(&self) -> NodeResult>> { + Ok(self + .nodes + .iter() + .map(|r| r.value().clone() as Arc) + .collect()) + } + + async fn get_healthy_nodes(&self) -> NodeResult>> { + let mut healthy = Vec::new(); + for node_ref in self.nodes.iter() { + let node = node_ref.value(); + if node.is_healthy().await { + healthy.push(node.clone() as Arc); + } + } + Ok(healthy) + } + + async fn register_node(&self, info: NodeInfo) -> NodeResult<()> { + let node = Arc::new(MockNodeClient::new(&info.node_id, &info.endpoint)); + self.nodes.insert(info.node_id, node); + Ok(()) + } + + async fn deregister_node(&self, node_id: &str) -> NodeResult<()> { + self.nodes.remove(node_id); + Ok(()) + } + + async fn update_health(&self, node_id: &str, healthy: bool) -> NodeResult<()> { + if let Some(node) = self.nodes.get(node_id) { + node.set_healthy(healthy); + } + Ok(()) + } + + async fn get_node(&self, node_id: &str) -> NodeResult>> { + Ok(self + .nodes + .get(node_id) + .map(|r| r.value().clone() as Arc)) + } + + async fn node_count(&self) -> usize { + self.nodes.len() + } + + async fn healthy_node_count(&self) -> usize { + let mut count = 0; + for node_ref in self.nodes.iter() { + if node_ref.value().is_healthy().await { + count += 1; + } + } + count + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_mock_node_put_get() { + let node = MockNodeClient::new("node-1", "http://localhost:9002"); + + let chunk_id = "test-chunk-1"; + let data = Bytes::from(vec![1, 2, 3, 4, 5]); + + node.put_chunk(chunk_id, 0, false, data.clone()) + .await + .unwrap(); + let retrieved = node.get_chunk(chunk_id, 0, false).await.unwrap(); + + assert_eq!(retrieved, data.to_vec()); + assert_eq!(node.put_count(), 1); + assert_eq!(node.get_count(), 1); + } + + #[tokio::test] + async fn test_mock_node_delete() { + let node = MockNodeClient::new("node-1", "http://localhost:9002"); + + let chunk_id = "test-chunk-1"; + let data = Bytes::from(vec![1, 2, 3]); + + node.put_chunk(chunk_id, 0, false, data).await.unwrap(); + assert!(node.chunk_exists(chunk_id).await.unwrap()); + + node.delete_chunk(chunk_id).await.unwrap(); + assert!(!node.chunk_exists(chunk_id).await.unwrap()); + } + + #[tokio::test] + async fn test_mock_node_health() { + let node = MockNodeClient::new("node-1", "http://localhost:9002"); + + assert!(node.is_healthy().await); + + node.set_healthy(false); + assert!(!node.is_healthy().await); + + let result = node.put_chunk("chunk", 0, false, Bytes::new()).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_mock_node_failure_injection() { + let node = MockNodeClient::new("node-1", "http://localhost:9002"); + + // Normal operation + node.put_chunk("chunk", 0, false, Bytes::from(vec![1])) + .await + .unwrap(); + + // Enable failure injection + node.set_fail_puts(true); + let result = node.put_chunk("chunk2", 0, false, Bytes::from(vec![2])).await; + assert!(result.is_err()); + + // Disable failure injection + node.set_fail_puts(false); + node.put_chunk("chunk3", 0, false, Bytes::from(vec![3])) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_mock_registry() { + let registry = MockNodeRegistry::with_nodes(3); + + assert_eq!(registry.node_count().await, 3); + assert_eq!(registry.healthy_node_count().await, 3); + + // Mark one node unhealthy + registry.update_health("node-1", false).await.unwrap(); + assert_eq!(registry.healthy_node_count().await, 2); + + // Get healthy nodes + let healthy = registry.get_healthy_nodes().await.unwrap(); + assert_eq!(healthy.len(), 2); + } + + #[tokio::test] + async fn test_mock_registry_register_deregister() { + let registry = MockNodeRegistry::new(); + + let info = NodeInfo::new("new-node", "http://new:9002"); + registry.register_node(info).await.unwrap(); + assert_eq!(registry.node_count().await, 1); + + registry.deregister_node("new-node").await.unwrap(); + assert_eq!(registry.node_count().await, 0); + } +} diff --git a/lightningstor/crates/lightningstor-distributed/src/node/mod.rs b/lightningstor/crates/lightningstor-distributed/src/node/mod.rs new file mode 100644 index 0000000..002991d --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/node/mod.rs @@ -0,0 +1,39 @@ +//! Node management for distributed storage +//! +//! This module provides abstractions for managing storage nodes, +//! including node discovery, health checking, and communication. + +pub mod client; +pub mod mock; +pub mod registry; + +pub use client::{NodeClient, NodeClientTrait}; +pub use mock::{MockNodeClient, MockNodeRegistry}; +pub use registry::{NodeInfo, NodeRegistry, StaticNodeRegistry}; + +use thiserror::Error; + +/// Errors that can occur during node operations +#[derive(Debug, Error)] +pub enum NodeError { + #[error("Connection failed to node {node_id}: {reason}")] + ConnectionFailed { node_id: String, reason: String }, + + #[error("RPC failed: {0}")] + RpcFailed(String), + + #[error("Node not found: {0}")] + NotFound(String), + + #[error("Timeout waiting for response")] + Timeout, + + #[error("Node unhealthy: {0}")] + Unhealthy(String), + + #[error("Not enough healthy nodes: need {needed}, have {available}")] + NotEnoughNodes { needed: usize, available: usize }, +} + +/// Result type for node operations +pub type NodeResult = Result; diff --git a/lightningstor/crates/lightningstor-distributed/src/node/registry.rs b/lightningstor/crates/lightningstor-distributed/src/node/registry.rs new file mode 100644 index 0000000..3d506df --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/node/registry.rs @@ -0,0 +1,281 @@ +//! Node registry for discovering and tracking storage nodes + +use super::client::{NodeClient, NodeClientTrait}; +use super::NodeResult; +use async_trait::async_trait; +use std::sync::Arc; +use tokio::sync::RwLock; + +/// Information about a storage node +#[derive(Debug, Clone)] +pub struct NodeInfo { + /// Unique node identifier + pub node_id: String, + /// gRPC endpoint (host:port) + pub endpoint: String, + /// Zone/rack identifier for placement + pub zone: String, + /// Region identifier + pub region: String, + /// Total storage capacity in bytes + pub capacity_bytes: u64, + /// Currently used storage in bytes + pub used_bytes: u64, + /// Whether the node is healthy + pub healthy: bool, +} + +impl NodeInfo { + /// Create a new node info with minimal information + pub fn new(node_id: impl Into, endpoint: impl Into) -> Self { + Self { + node_id: node_id.into(), + endpoint: endpoint.into(), + zone: String::new(), + region: String::new(), + capacity_bytes: 0, + used_bytes: 0, + healthy: true, + } + } + + /// Builder method to set the zone + pub fn with_zone(mut self, zone: impl Into) -> Self { + self.zone = zone.into(); + self + } + + /// Builder method to set the region + pub fn with_region(mut self, region: impl Into) -> Self { + self.region = region.into(); + self + } + + /// Builder method to set capacity + pub fn with_capacity(mut self, capacity_bytes: u64) -> Self { + self.capacity_bytes = capacity_bytes; + self + } + + /// Calculate available storage in bytes + pub fn available_bytes(&self) -> u64 { + self.capacity_bytes.saturating_sub(self.used_bytes) + } + + /// Calculate usage percentage + pub fn usage_percent(&self) -> f64 { + if self.capacity_bytes == 0 { + 0.0 + } else { + (self.used_bytes as f64 / self.capacity_bytes as f64) * 100.0 + } + } +} + +/// Trait for node registry implementations +#[async_trait] +pub trait NodeRegistry: Send + Sync { + /// Get all registered nodes + async fn get_all_nodes(&self) -> NodeResult>>; + + /// Get only healthy nodes + async fn get_healthy_nodes(&self) -> NodeResult>>; + + /// Register a new node + async fn register_node(&self, info: NodeInfo) -> NodeResult<()>; + + /// Remove a node from the registry + async fn deregister_node(&self, node_id: &str) -> NodeResult<()>; + + /// Update node health status + async fn update_health(&self, node_id: &str, healthy: bool) -> NodeResult<()>; + + /// Get a specific node by ID + async fn get_node(&self, node_id: &str) -> NodeResult>>; + + /// Get the number of registered nodes + async fn node_count(&self) -> usize; + + /// Get the number of healthy nodes + async fn healthy_node_count(&self) -> usize; +} + +/// Static node registry that uses a fixed list of endpoints +/// +/// Nodes are configured at startup and don't change dynamically. +pub struct StaticNodeRegistry { + nodes: RwLock>>, + node_info: RwLock>, +} + +impl StaticNodeRegistry { + /// Create a new static node registry with the given endpoints + pub async fn new(endpoints: &[String]) -> NodeResult { + let mut nodes: Vec> = Vec::new(); + let mut node_info = Vec::new(); + + for (i, endpoint) in endpoints.iter().enumerate() { + let node_id = format!("node-{}", i); + let client = NodeClient::connect_with_id(&node_id, endpoint).await?; + let info = NodeInfo::new(&node_id, endpoint); + + nodes.push(Arc::new(client)); + node_info.push(info); + } + + Ok(Self { + nodes: RwLock::new(nodes), + node_info: RwLock::new(node_info), + }) + } + + /// Create an empty registry + pub fn empty() -> Self { + Self { + nodes: RwLock::new(Vec::new()), + node_info: RwLock::new(Vec::new()), + } + } + + /// Get node info for all nodes + pub async fn get_node_info(&self) -> Vec { + self.node_info.read().await.clone() + } +} + +#[async_trait] +impl NodeRegistry for StaticNodeRegistry { + async fn get_all_nodes(&self) -> NodeResult>> { + Ok(self.nodes.read().await.clone()) + } + + async fn get_healthy_nodes(&self) -> NodeResult>> { + let nodes = self.nodes.read().await; + let mut healthy = Vec::new(); + for node in nodes.iter() { + if node.is_healthy().await { + healthy.push(node.clone()); + } + } + Ok(healthy) + } + + async fn register_node(&self, info: NodeInfo) -> NodeResult<()> { + let client = NodeClient::connect_with_id(&info.node_id, &info.endpoint).await?; + + self.nodes.write().await.push(Arc::new(client)); + self.node_info.write().await.push(info); + + Ok(()) + } + + async fn deregister_node(&self, node_id: &str) -> NodeResult<()> { + self.nodes + .write() + .await + .retain(|n| n.node_id() != node_id); + self.node_info + .write() + .await + .retain(|n| n.node_id != node_id); + Ok(()) + } + + async fn update_health(&self, node_id: &str, healthy: bool) -> NodeResult<()> { + let mut info = self.node_info.write().await; + for node_info in info.iter_mut() { + if node_info.node_id == node_id { + node_info.healthy = healthy; + break; + } + } + + // Note: For static registry, we don't actually update the client health + // as the client manages its own health state. This is mainly for tracking. + Ok(()) + } + + async fn get_node(&self, node_id: &str) -> NodeResult>> { + let nodes = self.nodes.read().await; + Ok(nodes.iter().find(|n| n.node_id() == node_id).cloned()) + } + + async fn node_count(&self) -> usize { + self.nodes.read().await.len() + } + + async fn healthy_node_count(&self) -> usize { + let nodes = self.nodes.read().await; + let mut count = 0; + for node in nodes.iter() { + if node.is_healthy().await { + count += 1; + } + } + count + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_node_info_creation() { + let info = NodeInfo::new("node-1", "http://localhost:9002") + .with_zone("zone-a") + .with_region("us-east-1") + .with_capacity(1024 * 1024 * 1024); + + assert_eq!(info.node_id, "node-1"); + assert_eq!(info.endpoint, "http://localhost:9002"); + assert_eq!(info.zone, "zone-a"); + assert_eq!(info.region, "us-east-1"); + assert_eq!(info.capacity_bytes, 1024 * 1024 * 1024); + } + + #[test] + fn test_node_info_usage() { + let mut info = NodeInfo::new("node-1", "http://localhost:9002").with_capacity(1000); + + info.used_bytes = 250; + assert_eq!(info.available_bytes(), 750); + assert!((info.usage_percent() - 25.0).abs() < 0.01); + } + + #[tokio::test] + async fn test_static_registry_creation() { + let endpoints = vec![ + "http://node1:9002".to_string(), + "http://node2:9002".to_string(), + ]; + + let registry = StaticNodeRegistry::new(&endpoints).await.unwrap(); + assert_eq!(registry.node_count().await, 2); + } + + #[tokio::test] + async fn test_static_registry_register_deregister() { + let registry = StaticNodeRegistry::empty(); + assert_eq!(registry.node_count().await, 0); + + let info = NodeInfo::new("node-1", "http://localhost:9002"); + registry.register_node(info).await.unwrap(); + assert_eq!(registry.node_count().await, 1); + + registry.deregister_node("node-1").await.unwrap(); + assert_eq!(registry.node_count().await, 0); + } + + #[tokio::test] + async fn test_static_registry_get_node() { + let endpoints = vec!["http://node1:9002".to_string()]; + let registry = StaticNodeRegistry::new(&endpoints).await.unwrap(); + + let node = registry.get_node("node-0").await.unwrap(); + assert!(node.is_some()); + + let missing = registry.get_node("nonexistent").await.unwrap(); + assert!(missing.is_none()); + } +} diff --git a/lightningstor/crates/lightningstor-distributed/src/placement/mod.rs b/lightningstor/crates/lightningstor-distributed/src/placement/mod.rs new file mode 100644 index 0000000..24ce93d --- /dev/null +++ b/lightningstor/crates/lightningstor-distributed/src/placement/mod.rs @@ -0,0 +1,398 @@ +//! Data placement strategies for distributed storage +//! +//! This module provides strategies for selecting which nodes should store +//! which data, supporting consistent hashing, random placement, and +//! zone-aware placement. + +use crate::node::{NodeClientTrait, NodeResult}; +use async_trait::async_trait; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +/// Trait for node selection strategies +#[async_trait] +pub trait NodeSelector: Send + Sync { + /// Select N nodes for storing data + /// + /// Returns a vector of selected nodes, in order of preference. + async fn select_nodes( + &self, + available_nodes: &[Arc], + count: usize, + ) -> NodeResult>>; + + /// Select a single node for reading data + /// + /// The key is used to deterministically select the same node + /// for the same data (for cache efficiency). + async fn select_for_read( + &self, + available_nodes: &[Arc], + key: &str, + ) -> NodeResult>; +} + +/// Consistent hash-based node selector +/// +/// Uses consistent hashing to select nodes, ensuring minimal data movement +/// when nodes are added or removed. +pub struct ConsistentHashSelector { + /// Number of virtual nodes per physical node + virtual_nodes: usize, +} + +impl ConsistentHashSelector { + /// Create a new consistent hash selector with default settings + pub fn new() -> Self { + Self { virtual_nodes: 100 } + } + + /// Create a new consistent hash selector with custom virtual node count + pub fn with_virtual_nodes(virtual_nodes: usize) -> Self { + Self { virtual_nodes } + } + + /// Hash a string to a u64 + fn hash_key(key: &str) -> u64 { + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + hasher.finish() + } + + /// Get the hash ring position for a node and virtual node index + fn node_position(&self, node_id: &str, vnode_index: usize) -> u64 { + Self::hash_key(&format!("{}:{}", node_id, vnode_index)) + } +} + +impl Default for ConsistentHashSelector { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl NodeSelector for ConsistentHashSelector { + async fn select_nodes( + &self, + available_nodes: &[Arc], + count: usize, + ) -> NodeResult>> { + if available_nodes.is_empty() { + return Ok(vec![]); + } + + let count = count.min(available_nodes.len()); + + // Build the hash ring with virtual nodes + let mut ring: Vec<(u64, usize)> = Vec::new(); + for (node_idx, node) in available_nodes.iter().enumerate() { + for vnode_idx in 0..self.virtual_nodes { + let pos = self.node_position(node.node_id(), vnode_idx); + ring.push((pos, node_idx)); + } + } + ring.sort_by_key(|(pos, _)| *pos); + + // Select nodes by walking the ring + let mut selected_indices = Vec::with_capacity(count); + let mut seen = std::collections::HashSet::new(); + + // Start from a random position (using current time for diversity) + let start_pos = Self::hash_key(&format!("{:?}", std::time::Instant::now())); + let start_idx = ring + .binary_search_by_key(&start_pos, |(pos, _)| *pos) + .unwrap_or_else(|i| i % ring.len()); + + for i in 0..ring.len() { + let idx = (start_idx + i) % ring.len(); + let node_idx = ring[idx].1; + + if seen.insert(node_idx) { + selected_indices.push(node_idx); + if selected_indices.len() >= count { + break; + } + } + } + + Ok(selected_indices + .into_iter() + .map(|idx| available_nodes[idx].clone()) + .collect()) + } + + async fn select_for_read( + &self, + available_nodes: &[Arc], + key: &str, + ) -> NodeResult> { + if available_nodes.is_empty() { + return Err(crate::node::NodeError::NotEnoughNodes { + needed: 1, + available: 0, + }); + } + + // Build the hash ring + let mut ring: Vec<(u64, usize)> = Vec::new(); + for (node_idx, node) in available_nodes.iter().enumerate() { + for vnode_idx in 0..self.virtual_nodes { + let pos = self.node_position(node.node_id(), vnode_idx); + ring.push((pos, node_idx)); + } + } + ring.sort_by_key(|(pos, _)| *pos); + + // Find the first node after the key's position + let key_pos = Self::hash_key(key); + let idx = ring + .binary_search_by_key(&key_pos, |(pos, _)| *pos) + .unwrap_or_else(|i| i % ring.len()); + + let node_idx = ring[idx].1; + Ok(available_nodes[node_idx].clone()) + } +} + +/// Random node selector +/// +/// Randomly selects nodes for placement. Simple but doesn't provide +/// consistent placement across operations. +pub struct RandomSelector; + +impl RandomSelector { + pub fn new() -> Self { + Self + } +} + +impl Default for RandomSelector { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl NodeSelector for RandomSelector { + async fn select_nodes( + &self, + available_nodes: &[Arc], + count: usize, + ) -> NodeResult>> { + if available_nodes.is_empty() { + return Ok(vec![]); + } + + let count = count.min(available_nodes.len()); + + // Shuffle using Fisher-Yates with simple random + let mut indices: Vec = (0..available_nodes.len()).collect(); + let seed = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64; + + let mut rng = seed; + for i in (1..indices.len()).rev() { + // Simple LCG random + rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1); + let j = (rng as usize) % (i + 1); + indices.swap(i, j); + } + + Ok(indices + .into_iter() + .take(count) + .map(|idx| available_nodes[idx].clone()) + .collect()) + } + + async fn select_for_read( + &self, + available_nodes: &[Arc], + _key: &str, + ) -> NodeResult> { + if available_nodes.is_empty() { + return Err(crate::node::NodeError::NotEnoughNodes { + needed: 1, + available: 0, + }); + } + + // Random selection + let seed = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64; + let idx = (seed as usize) % available_nodes.len(); + + Ok(available_nodes[idx].clone()) + } +} + +/// Round-robin node selector +/// +/// Selects nodes in round-robin order. Good for load distribution. +pub struct RoundRobinSelector { + counter: std::sync::atomic::AtomicUsize, +} + +impl RoundRobinSelector { + pub fn new() -> Self { + Self { + counter: std::sync::atomic::AtomicUsize::new(0), + } + } +} + +impl Default for RoundRobinSelector { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl NodeSelector for RoundRobinSelector { + async fn select_nodes( + &self, + available_nodes: &[Arc], + count: usize, + ) -> NodeResult>> { + if available_nodes.is_empty() { + return Ok(vec![]); + } + + let count = count.min(available_nodes.len()); + let start = self + .counter + .fetch_add(count, std::sync::atomic::Ordering::SeqCst); + + Ok((0..count) + .map(|i| { + let idx = (start + i) % available_nodes.len(); + available_nodes[idx].clone() + }) + .collect()) + } + + async fn select_for_read( + &self, + available_nodes: &[Arc], + _key: &str, + ) -> NodeResult> { + if available_nodes.is_empty() { + return Err(crate::node::NodeError::NotEnoughNodes { + needed: 1, + available: 0, + }); + } + + let idx = self + .counter + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + % available_nodes.len(); + Ok(available_nodes[idx].clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::node::MockNodeClient; + + fn create_mock_nodes(count: usize) -> Vec> { + (0..count) + .map(|i| { + Arc::new(MockNodeClient::new( + format!("node-{}", i), + format!("http://node-{}:9002", i), + )) as Arc + }) + .collect() + } + + #[tokio::test] + async fn test_consistent_hash_deterministic_read() { + let selector = ConsistentHashSelector::new(); + let nodes = create_mock_nodes(5); + + let key = "test-object-123"; + + // Same key should always select the same node + let node1 = selector.select_for_read(&nodes, key).await.unwrap(); + let node2 = selector.select_for_read(&nodes, key).await.unwrap(); + + assert_eq!(node1.node_id(), node2.node_id()); + } + + #[tokio::test] + async fn test_consistent_hash_select_nodes() { + let selector = ConsistentHashSelector::new(); + let nodes = create_mock_nodes(6); + + let selected = selector.select_nodes(&nodes, 3).await.unwrap(); + + assert_eq!(selected.len(), 3); + + // All selected nodes should be unique + let ids: std::collections::HashSet<_> = selected.iter().map(|n| n.node_id()).collect(); + assert_eq!(ids.len(), 3); + } + + #[tokio::test] + async fn test_consistent_hash_select_more_than_available() { + let selector = ConsistentHashSelector::new(); + let nodes = create_mock_nodes(3); + + // Request more nodes than available + let selected = selector.select_nodes(&nodes, 10).await.unwrap(); + + // Should return all available nodes + assert_eq!(selected.len(), 3); + } + + #[tokio::test] + async fn test_random_selector() { + let selector = RandomSelector::new(); + let nodes = create_mock_nodes(5); + + let selected = selector.select_nodes(&nodes, 3).await.unwrap(); + assert_eq!(selected.len(), 3); + } + + #[tokio::test] + async fn test_round_robin_selector() { + let selector = RoundRobinSelector::new(); + let nodes = create_mock_nodes(3); + + // First selection + let node1 = selector.select_for_read(&nodes, "key1").await.unwrap(); + // Second selection should be different + let node2 = selector.select_for_read(&nodes, "key2").await.unwrap(); + // Third selection + let node3 = selector.select_for_read(&nodes, "key3").await.unwrap(); + // Fourth should wrap around to first + let node4 = selector.select_for_read(&nodes, "key4").await.unwrap(); + + // Verify round-robin behavior + let ids: Vec<_> = [&node1, &node2, &node3, &node4] + .iter() + .map(|n| n.node_id()) + .collect(); + assert_eq!(ids[0], ids[3]); // Wrapped around + } + + #[tokio::test] + async fn test_empty_nodes() { + let selector = ConsistentHashSelector::new(); + let nodes: Vec> = vec![]; + + let selected = selector.select_nodes(&nodes, 3).await.unwrap(); + assert!(selected.is_empty()); + + let result = selector.select_for_read(&nodes, "key").await; + assert!(result.is_err()); + } +} diff --git a/lightningstor/crates/lightningstor-node/Cargo.toml b/lightningstor/crates/lightningstor-node/Cargo.toml new file mode 100644 index 0000000..6eae8fe --- /dev/null +++ b/lightningstor/crates/lightningstor-node/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "lightningstor-node" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +description = "LightningStor distributed storage node daemon" + +[[bin]] +name = "lightningstor-node" +path = "src/main.rs" + +[dependencies] +# Internal +lightningstor-types = { workspace = true } +lightningstor-storage = { workspace = true } + +# gRPC +tonic = { workspace = true } +tonic-health = { workspace = true } +prost = { workspace = true } +prost-types = { workspace = true } + +# Async runtime +tokio = { workspace = true } +tokio-stream = { workspace = true } +async-trait = { workspace = true } +futures = { workspace = true } + +# Utilities +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +thiserror = { workspace = true } +clap = { workspace = true } +serde = { workspace = true } +toml = { workspace = true } +dashmap = { workspace = true } +bytes = { workspace = true } +uuid = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } + +[build-dependencies] +tonic-build = { workspace = true } +protoc-bin-vendored = "3" + +[dev-dependencies] +tempfile = { workspace = true } + +[lints] +workspace = true diff --git a/lightningstor/crates/lightningstor-node/build.rs b/lightningstor/crates/lightningstor-node/build.rs new file mode 100644 index 0000000..19433d0 --- /dev/null +++ b/lightningstor/crates/lightningstor-node/build.rs @@ -0,0 +1,18 @@ +fn main() -> Result<(), Box> { + // Prefer a toolchain-provided protoc (e.g. via `nix develop` which sets PROTOC), + // but fall back to a vendored protoc when PROTOC isn't set. + if std::env::var_os("PROTOC").is_none() { + let protoc = protoc_bin_vendored::protoc_bin_path()?; + std::env::set_var("PROTOC", protoc); + } + + tonic_build::configure() + .build_server(true) + .build_client(true) + .compile_protos(&["proto/node.proto"], &["proto"])?; + + println!("cargo:rerun-if-changed=proto/node.proto"); + println!("cargo:rerun-if-changed=proto"); + + Ok(()) +} diff --git a/lightningstor/crates/lightningstor-node/proto/node.proto b/lightningstor/crates/lightningstor-node/proto/node.proto new file mode 100644 index 0000000..c926f73 --- /dev/null +++ b/lightningstor/crates/lightningstor-node/proto/node.proto @@ -0,0 +1,135 @@ +syntax = "proto3"; + +package lightningstor.node.v1; + +option java_package = "com.lightningstor.node.v1"; +option go_package = "lightningstor/node/v1;nodev1"; + +import "google/protobuf/empty.proto"; + +// ============================================================================= +// Node Storage Service - Chunk-level operations for distributed storage +// ============================================================================= + +service NodeService { + // Chunk operations + rpc PutChunk(PutChunkRequest) returns (PutChunkResponse); + rpc GetChunk(GetChunkRequest) returns (GetChunkResponse); + rpc DeleteChunk(DeleteChunkRequest) returns (google.protobuf.Empty); + rpc ChunkExists(ChunkExistsRequest) returns (ChunkExistsResponse); + rpc ChunkSize(ChunkSizeRequest) returns (ChunkSizeResponse); + + // Health and status + rpc Ping(PingRequest) returns (PingResponse); + rpc GetStatus(GetStatusRequest) returns (GetStatusResponse); + + // Batch operations for efficiency + rpc BatchPutChunks(stream PutChunkRequest) returns (BatchPutChunksResponse); + rpc BatchGetChunks(BatchGetChunksRequest) returns (stream GetChunkResponse); +} + +// ============================================================================= +// Chunk Operations +// ============================================================================= + +message PutChunkRequest { + // Unique identifier for the chunk + string chunk_id = 1; + // Shard index (for erasure coding) + uint32 shard_index = 2; + // Whether this is a parity shard + bool is_parity = 3; + // Chunk data + bytes data = 4; +} + +message PutChunkResponse { + // Size of data stored + uint64 size = 1; +} + +message GetChunkRequest { + string chunk_id = 1; + uint32 shard_index = 2; + bool is_parity = 3; +} + +message GetChunkResponse { + // Chunk data (or empty if streaming) + bytes data = 1; + // Size of the chunk + uint64 size = 2; +} + +message DeleteChunkRequest { + string chunk_id = 1; +} + +message ChunkExistsRequest { + string chunk_id = 1; +} + +message ChunkExistsResponse { + bool exists = 1; +} + +message ChunkSizeRequest { + string chunk_id = 1; +} + +message ChunkSizeResponse { + // Size in bytes, or 0 if not found + uint64 size = 1; + bool exists = 2; +} + +// ============================================================================= +// Health and Status +// ============================================================================= + +message PingRequest {} + +message PingResponse { + // Round-trip time in microseconds (server processing time) + uint64 latency_us = 1; +} + +message GetStatusRequest {} + +message GetStatusResponse { + // Node identifier + string node_id = 1; + // Endpoint address + string endpoint = 2; + // Zone/rack for placement + string zone = 3; + // Region + string region = 4; + // Storage capacity in bytes + uint64 capacity_bytes = 5; + // Used storage in bytes + uint64 used_bytes = 6; + // Number of chunks stored + uint64 chunk_count = 7; + // Node is healthy and accepting requests + bool healthy = 8; + // Uptime in seconds + uint64 uptime_seconds = 9; +} + +// ============================================================================= +// Batch Operations +// ============================================================================= + +message BatchPutChunksResponse { + // Number of chunks successfully stored + uint32 success_count = 1; + // Number of chunks that failed + uint32 failure_count = 2; + // Error messages for failed chunks + repeated string errors = 3; +} + +message BatchGetChunksRequest { + repeated GetChunkRequest chunks = 1; +} diff --git a/lightningstor/crates/lightningstor-node/src/config.rs b/lightningstor/crates/lightningstor-node/src/config.rs new file mode 100644 index 0000000..d682be6 --- /dev/null +++ b/lightningstor/crates/lightningstor-node/src/config.rs @@ -0,0 +1,76 @@ +//! Node configuration + +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; +use std::path::PathBuf; + +/// Storage node configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NodeConfig { + /// Unique node identifier + #[serde(default = "default_node_id")] + pub node_id: String, + + /// gRPC address to listen on + #[serde(default = "default_grpc_addr")] + pub grpc_addr: SocketAddr, + + /// Data directory for chunk storage + #[serde(default = "default_data_dir")] + pub data_dir: PathBuf, + + /// Zone/rack identifier for placement + #[serde(default)] + pub zone: String, + + /// Region identifier + #[serde(default)] + pub region: String, + + /// Log level + #[serde(default = "default_log_level")] + pub log_level: String, + + /// Maximum storage capacity in bytes (0 = unlimited) + #[serde(default)] + pub max_capacity_bytes: u64, + + /// Metrics port for Prometheus scraping + #[serde(default = "default_metrics_port")] + pub metrics_port: u16, +} + +fn default_node_id() -> String { + uuid::Uuid::new_v4().to_string() +} + +fn default_grpc_addr() -> SocketAddr { + "0.0.0.0:9002".parse().unwrap() +} + +fn default_data_dir() -> PathBuf { + PathBuf::from("/var/lib/lightningstor-node/data") +} + +fn default_log_level() -> String { + "info".to_string() +} + +fn default_metrics_port() -> u16 { + 9098 +} + +impl Default for NodeConfig { + fn default() -> Self { + Self { + node_id: default_node_id(), + grpc_addr: default_grpc_addr(), + data_dir: default_data_dir(), + zone: String::new(), + region: String::new(), + log_level: default_log_level(), + max_capacity_bytes: 0, + metrics_port: default_metrics_port(), + } + } +} diff --git a/lightningstor/crates/lightningstor-node/src/lib.rs b/lightningstor/crates/lightningstor-node/src/lib.rs new file mode 100644 index 0000000..43ec6b6 --- /dev/null +++ b/lightningstor/crates/lightningstor-node/src/lib.rs @@ -0,0 +1,36 @@ +//! LightningStor Storage Node +//! +//! This crate implements a storage node for the LightningStor distributed +//! storage system. Each node stores chunks of data and responds to requests +//! from the main server for put, get, and delete operations. +//! +//! # Architecture +//! +//! ```text +//! ┌─────────────────────────────────────────────────────┐ +//! │ LightningStor Server │ +//! │ (Erasure Coding / Replication Coordination) │ +//! └───────────┬───────────────┬───────────────┬─────────┘ +//! │ │ │ +//! ▼ ▼ ▼ +//! ┌───────────┐ ┌───────────┐ ┌───────────┐ +//! │ Node 1 │ │ Node 2 │ │ Node 3 │ +//! │ (gRPC) │ │ (gRPC) │ │ (gRPC) │ +//! └───────────┘ └───────────┘ └───────────┘ +//! ``` + +pub mod config; +pub mod service; +pub mod storage; + +pub use config::NodeConfig; +pub use service::NodeServiceImpl; +pub use storage::LocalChunkStore; + +/// Re-export generated protobuf types +pub mod proto { + tonic::include_proto!("lightningstor.node.v1"); +} + +pub use proto::node_service_client::NodeServiceClient; +pub use proto::node_service_server::{NodeService, NodeServiceServer}; diff --git a/lightningstor/crates/lightningstor-node/src/main.rs b/lightningstor/crates/lightningstor-node/src/main.rs new file mode 100644 index 0000000..6c67c39 --- /dev/null +++ b/lightningstor/crates/lightningstor-node/src/main.rs @@ -0,0 +1,169 @@ +//! LightningStor storage node daemon + +use clap::Parser; +use lightningstor_node::{ + proto::node_service_server::NodeServiceServer, LocalChunkStore, NodeConfig, NodeServiceImpl, +}; +use metrics_exporter_prometheus::PrometheusBuilder; +use std::path::PathBuf; +use std::sync::Arc; +use tonic::transport::Server; +use tonic_health::server::health_reporter; +use tracing_subscriber::EnvFilter; + +/// LightningStor storage node +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// Configuration file path + #[arg(short, long, default_value = "lightningstor-node.toml")] + config: PathBuf, + + /// Node ID (overrides config) + #[arg(long, env = "LIGHTNINGSTOR_NODE_ID")] + node_id: Option, + + /// gRPC address to listen on (overrides config) + #[arg(long)] + grpc_addr: Option, + + /// Data directory (overrides config) + #[arg(long)] + data_dir: Option, + + /// Zone identifier (overrides config) + #[arg(long)] + zone: Option, + + /// Region identifier (overrides config) + #[arg(long)] + region: Option, + + /// Log level (overrides config) + #[arg(short, long)] + log_level: Option, + + /// Maximum storage capacity in bytes (overrides config) + #[arg(long)] + max_capacity: Option, + + /// Metrics port for Prometheus scraping + #[arg(long)] + metrics_port: Option, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let args = Args::parse(); + + // Load configuration from file or use defaults + let mut config = if args.config.exists() { + let contents = tokio::fs::read_to_string(&args.config).await?; + toml::from_str(&contents)? + } else { + eprintln!( + "Config file not found: {}, using defaults", + args.config.display() + ); + NodeConfig::default() + }; + + // Apply command line overrides + if let Some(node_id) = args.node_id { + config.node_id = node_id; + } + if let Some(grpc_addr) = args.grpc_addr { + config.grpc_addr = grpc_addr.parse()?; + } + if let Some(data_dir) = args.data_dir { + config.data_dir = data_dir; + } + if let Some(zone) = args.zone { + config.zone = zone; + } + if let Some(region) = args.region { + config.region = region; + } + if let Some(log_level) = args.log_level { + config.log_level = log_level; + } + if let Some(max_capacity) = args.max_capacity { + config.max_capacity_bytes = max_capacity; + } + if let Some(metrics_port) = args.metrics_port { + config.metrics_port = metrics_port; + } + + // Initialize tracing + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new(&config.log_level)), + ) + .init(); + + tracing::info!("Starting LightningStor storage node"); + tracing::info!(" Node ID: {}", config.node_id); + tracing::info!(" gRPC: {}", config.grpc_addr); + tracing::info!(" Data dir: {}", config.data_dir.display()); + if !config.zone.is_empty() { + tracing::info!(" Zone: {}", config.zone); + } + if !config.region.is_empty() { + tracing::info!(" Region: {}", config.region); + } + if config.max_capacity_bytes > 0 { + tracing::info!( + " Max capacity: {} bytes", + config.max_capacity_bytes + ); + } + + // Initialize Prometheus metrics exporter + let metrics_addr = format!("0.0.0.0:{}", config.metrics_port); + let builder = PrometheusBuilder::new(); + builder + .with_http_listener(metrics_addr.parse::()?) + .install() + .expect("Failed to install Prometheus metrics exporter"); + + tracing::info!( + "Prometheus metrics available at http://{}/metrics", + metrics_addr + ); + + // Create local chunk store + let store = Arc::new( + LocalChunkStore::new(config.data_dir.clone(), config.max_capacity_bytes) + .await + .expect("Failed to create chunk store"), + ); + + tracing::info!( + "Chunk store initialized: {} chunks, {} bytes", + store.chunk_count(), + store.total_bytes() + ); + + // Create service + let config = Arc::new(config); + let service = NodeServiceImpl::new(store.clone(), config.clone()); + + // Setup health service + let (mut health_reporter, health_service) = health_reporter(); + health_reporter + .set_serving::>() + .await; + + // Start gRPC server + let addr = config.grpc_addr; + tracing::info!("gRPC server listening on {}", addr); + + Server::builder() + .add_service(health_service) + .add_service(NodeServiceServer::new(service)) + .serve(addr) + .await?; + + Ok(()) +} diff --git a/lightningstor/crates/lightningstor-node/src/service.rs b/lightningstor/crates/lightningstor-node/src/service.rs new file mode 100644 index 0000000..9e52faa --- /dev/null +++ b/lightningstor/crates/lightningstor-node/src/service.rs @@ -0,0 +1,232 @@ +//! gRPC service implementation for storage node + +use crate::proto::{ + node_service_server::NodeService, BatchGetChunksRequest, BatchPutChunksResponse, + ChunkExistsRequest, ChunkExistsResponse, ChunkSizeRequest, ChunkSizeResponse, + DeleteChunkRequest, GetChunkRequest, GetChunkResponse, GetStatusRequest, GetStatusResponse, + PingRequest, PingResponse, PutChunkRequest, PutChunkResponse, +}; +use crate::storage::LocalChunkStore; +use crate::NodeConfig; +use std::sync::Arc; +use std::time::Instant; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status, Streaming}; +use tracing::{debug, error}; + +/// Implementation of the NodeService gRPC service +pub struct NodeServiceImpl { + /// Local chunk storage + store: Arc, + /// Node configuration + config: Arc, + /// Server start time + start_time: Instant, +} + +impl NodeServiceImpl { + /// Create a new node service + pub fn new(store: Arc, config: Arc) -> Self { + Self { + store, + config, + start_time: Instant::now(), + } + } +} + +#[tonic::async_trait] +impl NodeService for NodeServiceImpl { + async fn put_chunk( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + debug!( + chunk_id = %req.chunk_id, + shard_index = req.shard_index, + is_parity = req.is_parity, + size = req.data.len(), + "PutChunk request" + ); + + let size = self + .store + .put(&req.chunk_id, &req.data) + .await + .map_err(|e| { + error!(error = ?e, "Failed to put chunk"); + Status::internal(e.to_string()) + })?; + + metrics::counter!("node_chunks_stored").increment(1); + metrics::counter!("node_bytes_stored").increment(size); + + Ok(Response::new(PutChunkResponse { size })) + } + + async fn get_chunk( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + debug!( + chunk_id = %req.chunk_id, + shard_index = req.shard_index, + is_parity = req.is_parity, + "GetChunk request" + ); + + let data = self.store.get(&req.chunk_id).await.map_err(|e| { + match &e { + crate::storage::StorageError::NotFound(_) => { + debug!(chunk_id = %req.chunk_id, "Chunk not found"); + Status::not_found(e.to_string()) + } + _ => { + error!(error = ?e, "Failed to get chunk"); + Status::internal(e.to_string()) + } + } + })?; + + metrics::counter!("node_chunks_retrieved").increment(1); + metrics::counter!("node_bytes_retrieved").increment(data.len() as u64); + + Ok(Response::new(GetChunkResponse { + data, + size: 0, // Size is implicit from data.len() + })) + } + + async fn delete_chunk( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + debug!(chunk_id = %req.chunk_id, "DeleteChunk request"); + + self.store.delete(&req.chunk_id).await.map_err(|e| { + error!(error = ?e, "Failed to delete chunk"); + Status::internal(e.to_string()) + })?; + + metrics::counter!("node_chunks_deleted").increment(1); + + Ok(Response::new(())) + } + + async fn chunk_exists( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let exists = self.store.exists(&req.chunk_id); + + Ok(Response::new(ChunkExistsResponse { exists })) + } + + async fn chunk_size( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + match self.store.size(&req.chunk_id) { + Some(size) => Ok(Response::new(ChunkSizeResponse { size, exists: true })), + None => Ok(Response::new(ChunkSizeResponse { + size: 0, + exists: false, + })), + } + } + + async fn ping(&self, _request: Request) -> Result, Status> { + let start = Instant::now(); + // Minimal processing - just measure latency + let latency_us = start.elapsed().as_micros() as u64; + + Ok(Response::new(PingResponse { latency_us })) + } + + async fn get_status( + &self, + _request: Request, + ) -> Result, Status> { + let uptime_seconds = self.start_time.elapsed().as_secs(); + + Ok(Response::new(GetStatusResponse { + node_id: self.config.node_id.clone(), + endpoint: self.config.grpc_addr.to_string(), + zone: self.config.zone.clone(), + region: self.config.region.clone(), + capacity_bytes: self.store.max_capacity(), + used_bytes: self.store.total_bytes(), + chunk_count: self.store.chunk_count(), + healthy: true, + uptime_seconds, + })) + } + + async fn batch_put_chunks( + &self, + request: Request>, + ) -> Result, Status> { + let mut stream = request.into_inner(); + let mut success_count = 0u32; + let mut failure_count = 0u32; + let mut errors = Vec::new(); + + while let Some(req) = stream.message().await? { + match self.store.put(&req.chunk_id, &req.data).await { + Ok(size) => { + success_count += 1; + metrics::counter!("node_chunks_stored").increment(1); + metrics::counter!("node_bytes_stored").increment(size); + } + Err(e) => { + failure_count += 1; + errors.push(format!("{}: {}", req.chunk_id, e)); + } + } + } + + Ok(Response::new(BatchPutChunksResponse { + success_count, + failure_count, + errors, + })) + } + + type BatchGetChunksStream = ReceiverStream>; + + async fn batch_get_chunks( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let (tx, rx) = tokio::sync::mpsc::channel(32); + let store = self.store.clone(); + + tokio::spawn(async move { + for chunk_req in req.chunks { + let result = match store.get(&chunk_req.chunk_id).await { + Ok(data) => { + let size = data.len() as u64; + Ok(GetChunkResponse { data, size }) + } + Err(e) => Err(Status::not_found(e.to_string())), + }; + + if tx.send(result).await.is_err() { + break; + } + } + }); + + Ok(Response::new(ReceiverStream::new(rx))) + } +} diff --git a/lightningstor/crates/lightningstor-node/src/storage.rs b/lightningstor/crates/lightningstor-node/src/storage.rs new file mode 100644 index 0000000..71ec052 --- /dev/null +++ b/lightningstor/crates/lightningstor-node/src/storage.rs @@ -0,0 +1,313 @@ +//! Local chunk storage + +use dashmap::DashMap; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use thiserror::Error; +use tokio::fs; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tracing::debug; + +/// Errors from chunk storage operations +#[derive(Debug, Error)] +pub enum StorageError { + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + + #[error("Chunk not found: {0}")] + NotFound(String), + + #[error("Storage capacity exceeded")] + CapacityExceeded, +} + +pub type StorageResult = Result; + +/// Local filesystem-based chunk storage +pub struct LocalChunkStore { + /// Data directory + data_dir: PathBuf, + + /// In-memory index of chunk sizes for fast lookups + chunk_sizes: DashMap, + + /// Total bytes stored + total_bytes: AtomicU64, + + /// Maximum capacity (0 = unlimited) + max_capacity: u64, + + /// Number of chunks stored + chunk_count: AtomicU64, +} + +impl LocalChunkStore { + /// Create a new local chunk store + pub async fn new(data_dir: PathBuf, max_capacity: u64) -> StorageResult { + // Ensure data directory exists + fs::create_dir_all(&data_dir).await?; + + let store = Self { + data_dir, + chunk_sizes: DashMap::new(), + total_bytes: AtomicU64::new(0), + max_capacity, + chunk_count: AtomicU64::new(0), + }; + + // Scan existing chunks + store.scan_existing_chunks().await?; + + Ok(store) + } + + /// Scan existing chunks in the data directory + async fn scan_existing_chunks(&self) -> StorageResult<()> { + let mut entries = fs::read_dir(&self.data_dir).await?; + let mut total_bytes = 0u64; + let mut chunk_count = 0u64; + + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if path.is_file() { + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if let Ok(metadata) = entry.metadata().await { + let size = metadata.len(); + self.chunk_sizes.insert(name.to_string(), size); + total_bytes += size; + chunk_count += 1; + } + } + } + } + + self.total_bytes.store(total_bytes, Ordering::SeqCst); + self.chunk_count.store(chunk_count, Ordering::SeqCst); + + debug!( + total_bytes, + chunk_count, + "Scanned existing chunks" + ); + + Ok(()) + } + + /// Get the path for a chunk + fn chunk_path(&self, chunk_id: &str) -> PathBuf { + // Sanitize chunk_id to be a valid filename + let safe_id = chunk_id.replace(['/', '\\', ':', '*', '?', '"', '<', '>', '|'], "_"); + self.data_dir.join(safe_id) + } + + /// Store a chunk + pub async fn put(&self, chunk_id: &str, data: &[u8]) -> StorageResult { + let size = data.len() as u64; + + // Check capacity + if self.max_capacity > 0 { + let current = self.total_bytes.load(Ordering::SeqCst); + if current + size > self.max_capacity { + return Err(StorageError::CapacityExceeded); + } + } + + let path = self.chunk_path(chunk_id); + + // Check if replacing existing chunk + let old_size = self.chunk_sizes.get(chunk_id).map(|v| *v).unwrap_or(0); + + // Write data + let mut file = fs::File::create(&path).await?; + file.write_all(data).await?; + file.sync_all().await?; + + // Update index + self.chunk_sizes.insert(chunk_id.to_string(), size); + + // Update totals + if old_size > 0 { + // Replacing existing chunk + self.total_bytes.fetch_sub(old_size, Ordering::SeqCst); + } else { + // New chunk + self.chunk_count.fetch_add(1, Ordering::SeqCst); + } + self.total_bytes.fetch_add(size, Ordering::SeqCst); + + debug!(chunk_id, size, "Stored chunk"); + + Ok(size) + } + + /// Retrieve a chunk + pub async fn get(&self, chunk_id: &str) -> StorageResult> { + let path = self.chunk_path(chunk_id); + + if !path.exists() { + return Err(StorageError::NotFound(chunk_id.to_string())); + } + + let mut file = fs::File::open(&path).await?; + let mut data = Vec::new(); + file.read_to_end(&mut data).await?; + + debug!(chunk_id, size = data.len(), "Retrieved chunk"); + + Ok(data) + } + + /// Delete a chunk + pub async fn delete(&self, chunk_id: &str) -> StorageResult<()> { + let path = self.chunk_path(chunk_id); + + if let Some((_, size)) = self.chunk_sizes.remove(chunk_id) { + if path.exists() { + fs::remove_file(&path).await?; + } + self.total_bytes.fetch_sub(size, Ordering::SeqCst); + self.chunk_count.fetch_sub(1, Ordering::SeqCst); + debug!(chunk_id, "Deleted chunk"); + } + + Ok(()) + } + + /// Check if a chunk exists + pub fn exists(&self, chunk_id: &str) -> bool { + self.chunk_sizes.contains_key(chunk_id) + } + + /// Get the size of a chunk + pub fn size(&self, chunk_id: &str) -> Option { + self.chunk_sizes.get(chunk_id).map(|v| *v) + } + + /// Get total bytes stored + pub fn total_bytes(&self) -> u64 { + self.total_bytes.load(Ordering::SeqCst) + } + + /// Get chunk count + pub fn chunk_count(&self) -> u64 { + self.chunk_count.load(Ordering::SeqCst) + } + + /// Get maximum capacity + pub fn max_capacity(&self) -> u64 { + self.max_capacity + } + + /// Get available capacity + pub fn available_bytes(&self) -> u64 { + if self.max_capacity == 0 { + u64::MAX + } else { + self.max_capacity.saturating_sub(self.total_bytes()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + async fn create_test_store() -> (LocalChunkStore, TempDir) { + let temp_dir = TempDir::new().unwrap(); + let store = LocalChunkStore::new(temp_dir.path().to_path_buf(), 0) + .await + .unwrap(); + (store, temp_dir) + } + + #[tokio::test] + async fn test_put_get() { + let (store, _temp) = create_test_store().await; + + let chunk_id = "test-chunk-1"; + let data = vec![42u8; 1024]; + + let size = store.put(chunk_id, &data).await.unwrap(); + assert_eq!(size, 1024); + + let retrieved = store.get(chunk_id).await.unwrap(); + assert_eq!(retrieved, data); + } + + #[tokio::test] + async fn test_delete() { + let (store, _temp) = create_test_store().await; + + let chunk_id = "test-chunk-2"; + let data = vec![42u8; 512]; + + store.put(chunk_id, &data).await.unwrap(); + assert!(store.exists(chunk_id)); + + store.delete(chunk_id).await.unwrap(); + assert!(!store.exists(chunk_id)); + + let result = store.get(chunk_id).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_size_tracking() { + let (store, _temp) = create_test_store().await; + + assert_eq!(store.total_bytes(), 0); + assert_eq!(store.chunk_count(), 0); + + store.put("chunk1", &vec![0u8; 100]).await.unwrap(); + assert_eq!(store.total_bytes(), 100); + assert_eq!(store.chunk_count(), 1); + + store.put("chunk2", &vec![0u8; 200]).await.unwrap(); + assert_eq!(store.total_bytes(), 300); + assert_eq!(store.chunk_count(), 2); + + store.delete("chunk1").await.unwrap(); + assert_eq!(store.total_bytes(), 200); + assert_eq!(store.chunk_count(), 1); + } + + #[tokio::test] + async fn test_capacity_limit() { + let temp_dir = TempDir::new().unwrap(); + let store = LocalChunkStore::new(temp_dir.path().to_path_buf(), 1000) + .await + .unwrap(); + + // Should succeed + store.put("chunk1", &vec![0u8; 500]).await.unwrap(); + + // Should fail - would exceed capacity + let result = store.put("chunk2", &vec![0u8; 600]).await; + assert!(matches!(result, Err(StorageError::CapacityExceeded))); + + // Should succeed - within remaining capacity + store.put("chunk2", &vec![0u8; 400]).await.unwrap(); + } + + #[tokio::test] + async fn test_replace_chunk() { + let (store, _temp) = create_test_store().await; + + let chunk_id = "test-chunk"; + + store.put(chunk_id, &vec![0u8; 100]).await.unwrap(); + assert_eq!(store.total_bytes(), 100); + assert_eq!(store.chunk_count(), 1); + + // Replace with larger data + store.put(chunk_id, &vec![0u8; 200]).await.unwrap(); + assert_eq!(store.total_bytes(), 200); + assert_eq!(store.chunk_count(), 1); // Still 1 chunk + + // Replace with smaller data + store.put(chunk_id, &vec![0u8; 50]).await.unwrap(); + assert_eq!(store.total_bytes(), 50); + assert_eq!(store.chunk_count(), 1); + } +} diff --git a/lightningstor/crates/lightningstor-server/src/tenant.rs b/lightningstor/crates/lightningstor-server/src/tenant.rs new file mode 100644 index 0000000..6e00b46 --- /dev/null +++ b/lightningstor/crates/lightningstor-server/src/tenant.rs @@ -0,0 +1,59 @@ +use tonic::{metadata::MetadataMap, Status}; + +#[derive(Debug, Clone)] +pub struct TenantContext { + pub org_id: String, + pub project_id: String, +} + +fn metadata_value(metadata: &MetadataMap, key: &str) -> Option { + metadata + .get(key) + .and_then(|value| value.to_str().ok()) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +fn org_from_metadata(metadata: &MetadataMap) -> Option { + metadata_value(metadata, "org-id") + .or_else(|| metadata_value(metadata, "x-org-id")) + .or_else(|| metadata_value(metadata, "org_id")) +} + +fn project_from_metadata(metadata: &MetadataMap) -> Option { + metadata_value(metadata, "project-id") + .or_else(|| metadata_value(metadata, "x-project-id")) + .or_else(|| metadata_value(metadata, "project_id")) +} + +pub fn resolve_org( + metadata: &MetadataMap, + org_id: Option, +) -> Result { + org_id + .filter(|value| !value.is_empty()) + .or_else(|| org_from_metadata(metadata)) + .ok_or_else(|| Status::invalid_argument("org_id is required")) +} + +pub fn resolve_org_project_optional( + metadata: &MetadataMap, + org_id: Option, + project_id: Option, +) -> Result<(String, Option), Status> { + let org_id = resolve_org(metadata, org_id)?; + let project_id = project_id + .filter(|value| !value.is_empty()) + .or_else(|| project_from_metadata(metadata)); + Ok((org_id, project_id)) +} + +pub fn resolve_tenant( + metadata: &MetadataMap, + org_id: Option, + project_id: Option, +) -> Result { + let (org_id, project_id) = resolve_org_project_optional(metadata, org_id, project_id)?; + let project_id = project_id.ok_or_else(|| Status::invalid_argument("project_id is required"))?; + Ok(TenantContext { org_id, project_id }) +} diff --git a/mtls-agent/Cargo.lock b/mtls-agent/Cargo.lock new file mode 100644 index 0000000..e2cef76 --- /dev/null +++ b/mtls-agent/Cargo.lock @@ -0,0 +1,1954 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "aws-lc-rs" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + +[[package]] +name = "cc" +version = "1.2.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chainfire-client" +version = "0.1.0" +dependencies = [ + "chainfire-proto", + "chainfire-types", + "futures", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", + "tonic", + "tracing", +] + +[[package]] +name = "chainfire-proto" +version = "0.1.0" +dependencies = [ + "prost", + "prost-types", + "protoc-bin-vendored", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", +] + +[[package]] +name = "chainfire-types" +version = "0.1.0" +dependencies = [ + "bytes", + "serde", + "thiserror", +] + +[[package]] +name = "clap" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" + +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.12.1", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2 0.6.1", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee5b5339afb4c41626dde77b7a611bd4f2c202b897852b4bcf5d03eddc61010" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.178" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "mtls-agent" +version = "0.1.0" +dependencies = [ + "anyhow", + "chainfire-client", + "clap", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "tokio", + "tokio-rustls", + "toml", + "tracing", + "tracing-subscriber", + "webpki-roots 0.26.11", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap 2.12.1", +] + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62049b2877bf12821e8f9ad256ee38fdc31db7387ec2d3b3f403024de2034aea" + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.146" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217ca874ae0207aac254aa02c957ded05585a90892cc8d87f9e5fa49669dadd8" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.1", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.12.1", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "rustls-native-certs", + "rustls-pemfile", + "socket2 0.5.10", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.4", +] + +[[package]] +name = "webpki-roots" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "zerocopy" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" diff --git a/mtls-agent/Cargo.toml b/mtls-agent/Cargo.toml new file mode 100644 index 0000000..f0231d4 --- /dev/null +++ b/mtls-agent/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "mtls-agent" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0" +tokio = { version = "1.38", features = ["full"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +clap = { version = "4.5", features = ["derive"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +toml = "0.8" + +rustls = { version = "0.23", default-features = false, features = ["std", "tls12"] } +tokio-rustls = "0.26" +rustls-pemfile = "2" +webpki-roots = "0.26" + +chainfire-client = { path = "../chainfire/chainfire-client" } + + diff --git a/mtls-agent/src/client.rs b/mtls-agent/src/client.rs new file mode 100644 index 0000000..30e5575 --- /dev/null +++ b/mtls-agent/src/client.rs @@ -0,0 +1,89 @@ +use std::sync::Arc; + +use anyhow::{Context, Result}; +use rustls::{pki_types::ServerName, ClientConfig, RootCertStore}; +use rustls_pemfile::certs; +use std::fs; +use std::io::BufReader; +use tokio::net::TcpStream; +use tokio_rustls::TlsConnector; + +use crate::discovery::ServiceDiscovery; + +pub struct MtlsClient { + discovery: Arc, + tls_config: Option>, +} + +impl MtlsClient { + pub fn new(discovery: Arc) -> Self { + Self { + discovery, + tls_config: None, + } + } + + pub fn with_tls_config(mut self, config: Arc) -> Self { + self.tls_config = Some(config); + self + } + + pub async fn connect_to_service( + &self, + service_name: &str, + use_mtls: bool, + ) -> Result { + let instances = self.discovery.resolve_service(service_name).await?; + if instances.is_empty() { + anyhow::bail!("no healthy instances found for service {}", service_name); + } + + // ラウンドロビン（簡易実装） + let instance = instances[0].clone(); + + let addr = if let Some(mesh_port) = instance.mesh_port { + format!("{}:{}", instance.ip, mesh_port) + } else { + format!("{}:{}", instance.ip, instance.port) + }; + + let stream = TcpStream::connect(&addr).await?; + + // TODO: mTLS対応 + if use_mtls { + return Err(anyhow::anyhow!("mTLS client connection not fully implemented")); + } + + Ok(stream) + } +} + +pub fn build_client_config( + ca_cert_path: Option<&str>, + client_cert_path: Option<&str>, + client_key_path: Option<&str>, +) -> Result> { + let mut roots = RootCertStore::empty(); + + if let Some(ca_path) = ca_cert_path { + let certs = certs(&mut BufReader::new(fs::File::open(ca_path)?)) + .collect::, _>>()?; + roots.add_parsable_certificates(certs); + } else { + // システムのルート証明書を使用 + roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + } + + let mut config_builder = ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + + // クライアント証明書が指定されている場合は設定 + if let (Some(cert_path), Some(key_path)) = (client_cert_path, client_key_path) { + // TODO: クライアント証明書の読み込みと設定 + // 現時点ではサーバー認証のみ + } + + Ok(Arc::new(config_builder)) +} + diff --git a/mtls-agent/src/discovery.rs b/mtls-agent/src/discovery.rs new file mode 100644 index 0000000..5233797 --- /dev/null +++ b/mtls-agent/src/discovery.rs @@ -0,0 +1,219 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::Result; +use chainfire_client::Client; +use serde::{Deserialize, Serialize}; +use tokio::sync::RwLock; +use tracing::{info, warn}; + +const PHOTON_PREFIX: &str = "photoncloud"; +const CACHE_TTL: Duration = Duration::from_secs(30); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ServiceInstance { + pub instance_id: String, + pub service: String, + pub node_id: String, + pub ip: String, + pub port: u16, + #[serde(default)] + pub mesh_port: Option, + #[serde(default)] + pub version: Option, + #[serde(default)] + pub state: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MtlsPolicy { + pub policy_id: String, + #[serde(default)] + pub environment: Option, + pub source_service: String, + pub target_service: String, + #[serde(default)] + pub mtls_required: Option, + #[serde(default)] + pub mode: Option, +} + +struct CachedInstances { + instances: Vec, + updated_at: Instant, +} + +pub struct ServiceDiscovery { + chainfire_endpoint: String, + cluster_id: String, + cache: Arc>>, + policy_cache: Arc>>, +} + +impl ServiceDiscovery { + pub fn new(chainfire_endpoint: String, cluster_id: String) -> Self { + Self { + chainfire_endpoint, + cluster_id, + cache: Arc::new(RwLock::new(HashMap::new())), + policy_cache: Arc::new(RwLock::new(HashMap::new())), + } + } + + pub async fn resolve_service(&self, service_name: &str) -> Result> { + // キャッシュをチェック + { + let cache = self.cache.read().await; + if let Some(cached) = cache.get(service_name) { + if cached.updated_at.elapsed() < CACHE_TTL { + return Ok(cached.instances.clone()); + } + } + } + + // Chainfireから取得 + let instances = self.fetch_instances_from_chainfire(service_name).await?; + + // キャッシュを更新 + { + let mut cache = self.cache.write().await; + cache.insert( + service_name.to_string(), + CachedInstances { + instances: instances.clone(), + updated_at: Instant::now(), + }, + ); + } + + Ok(instances) + } + + async fn fetch_instances_from_chainfire(&self, service_name: &str) -> Result> { + let mut client = Client::connect(self.chainfire_endpoint.clone()).await?; + let prefix = format!( + "{}instances/{}/", + cluster_prefix(&self.cluster_id), + service_name + ); + let prefix_bytes = prefix.as_bytes(); + + let (kvs, _) = client.scan_prefix(prefix_bytes, 0).await?; + let mut instances = Vec::new(); + + for (_, value, _) in kvs { + match serde_json::from_slice::(&value) { + Ok(inst) => { + // 状態が "healthy" または未設定のもののみ返す + if inst.state.as_deref().unwrap_or("healthy") == "healthy" { + instances.push(inst); + } + } + Err(e) => { + warn!(error = %e, "failed to parse ServiceInstance from Chainfire"); + } + } + } + + info!( + service = %service_name, + count = instances.len(), + "resolved service instances from Chainfire" + ); + + Ok(instances) + } + + pub async fn get_mtls_policy( + &self, + source_service: &str, + target_service: &str, + ) -> Result> { + let policy_key = format!( + "{}-{}", + source_service, target_service + ); + + // キャッシュをチェック + { + let cache = self.policy_cache.read().await; + if let Some(policy) = cache.get(&policy_key) { + return Ok(Some(policy.clone())); + } + } + + // Chainfireから取得 + let mut client = Client::connect(self.chainfire_endpoint.clone()).await?; + let prefix = format!( + "{}mtls/policies/", + cluster_prefix(&self.cluster_id) + ); + let prefix_bytes = prefix.as_bytes(); + + let (kvs, _) = client.scan_prefix(prefix_bytes, 0).await?; + + for (_, value, _) in kvs { + match serde_json::from_slice::(&value) { + Ok(policy) => { + if policy.source_service == source_service && policy.target_service == target_service { + // キャッシュに保存 + let mut cache = self.policy_cache.write().await; + cache.insert(policy_key.clone(), policy.clone()); + return Ok(Some(policy)); + } + } + Err(e) => { + warn!(error = %e, "failed to parse MtlsPolicy from Chainfire"); + } + } + } + + Ok(None) + } + + pub async fn start_background_refresh(&self) { + let endpoint = self.chainfire_endpoint.clone(); + let cluster_id = self.cluster_id.clone(); + let cache = Arc::clone(&self.cache); + let policy_cache = Arc::clone(&self.policy_cache); + + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(60)); + loop { + interval.tick().await; + + // 全サービスのインスタンスをリフレッシュ + if let Ok(mut client) = Client::connect(endpoint.clone()).await { + let prefix = format!("{}instances/", cluster_prefix(&cluster_id)); + if let Ok((kvs, _)) = client.scan_prefix(prefix.as_bytes(), 0).await { + let mut service_map: HashMap> = HashMap::new(); + for (key, value, _) in kvs { + if let Ok(inst) = serde_json::from_slice::(&value) { + service_map + .entry(inst.service.clone()) + .or_insert_with(Vec::new) + .push(inst); + } + } + let mut cache_guard = cache.write().await; + for (service, instances) in service_map { + cache_guard.insert( + service, + CachedInstances { + instances, + updated_at: Instant::now(), + }, + ); + } + } + } + } + }); + } +} + +fn cluster_prefix(cluster_id: &str) -> String { + format!("{}/clusters/{}/", PHOTON_PREFIX, cluster_id) +} + diff --git a/mtls-agent/src/main.rs b/mtls-agent/src/main.rs new file mode 100644 index 0000000..75840a1 --- /dev/null +++ b/mtls-agent/src/main.rs @@ -0,0 +1,337 @@ +mod client; +mod discovery; +mod policy; + +use std::fs; +use std::io::BufReader; +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::{anyhow, Context, Result}; +use clap::Parser; +use rustls::{pki_types::CertificateDer, pki_types::PrivateKeyDer, ServerConfig}; +use rustls_pemfile::{certs, pkcs8_private_keys, rsa_private_keys}; +use serde::Deserialize; +use tokio::io; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task; +use tokio_rustls::TlsAcceptor; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; + +use crate::discovery::ServiceDiscovery; +use crate::policy::PolicyEnforcer; + +/// mTLS Agent (MVP: プレーンTCPプロキシ) +/// +/// - 設計どおり、アプリケーションは `app_addr` で平文待受 +/// - 本Agentは `mesh_bind_addr` で待受し、受信した接続を `app_addr` にフォワードするだけ +/// - mTLS/TLS 対応は後続で追加する前提のスケルトン +#[derive(Parser, Debug)] +#[command(author, version, about)] +struct Cli { + /// 設定ファイル (TOML) + #[arg(long)] + config: PathBuf, +} + +#[derive(Debug, Deserialize)] +struct ServiceConfig { + name: String, + app_addr: String, + mesh_bind_addr: String, +} + +#[derive(Debug, Deserialize)] +struct ClusterConfig { + cluster_id: String, + environment: Option, + chainfire_endpoint: Option, +} + +#[derive(Debug, Deserialize)] +struct MtlsConfig { + #[serde(default)] + mode: Option, // auto/mtls/tls/plain + #[serde(default)] + ca_cert_path: Option, + #[serde(default)] + cert_path: Option, + #[serde(default)] + key_path: Option, +} + +#[derive(Debug, Deserialize)] +struct Config { + service: ServiceConfig, + #[serde(default)] + cluster: Option, + #[serde(default)] + mtls: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env().add_directive("info".parse()?)) + .init(); + + let cli = Cli::parse(); + let cfg = load_config(&cli.config)?; + + let mode = cfg + .mtls + .as_ref() + .and_then(|m| m.mode.as_deref()) + .unwrap_or("plain") + .to_lowercase(); + + info!( + service = %cfg.service.name, + mesh_bind = %cfg.service.mesh_bind_addr, + app_addr = %cfg.service.app_addr, + mode = %mode, + "starting mtls-agent" + ); + + // Chainfire統合: サービス発見とポリシー管理 + let (discovery, _policy_enforcer) = if let Some(cluster_cfg) = &cfg.cluster { + if let Some(endpoint) = &cluster_cfg.chainfire_endpoint { + let disc = Arc::new(ServiceDiscovery::new( + endpoint.clone(), + cluster_cfg.cluster_id.clone(), + )); + disc.start_background_refresh().await; + + let enforcer = PolicyEnforcer::new(Arc::clone(&disc), mode.clone()); + enforcer.start_background_refresh().await; + + (Some(disc), Some(enforcer)) + } else { + (None, None) + } + } else { + (None, None) + }; + + // モード決定: "auto" の場合はChainfireからポリシーを読み取る + let effective_mode = if mode == "auto" { + if let Some(disc) = &discovery { + // デフォルトポリシーを確認（簡易実装） + // 実際には、自身のサービス名とターゲットサービス名でポリシーを検索 + if let Ok(Some(policy)) = disc + .get_mtls_policy(&cfg.service.name, "default") + .await + { + if policy.mtls_required.unwrap_or(false) { + "mtls" + } else { + "tls" + } + } else { + // 環境に応じたデフォルト + let env = cfg + .cluster + .as_ref() + .and_then(|c| c.environment.as_deref()) + .unwrap_or("dev"); + if env == "prod" || env == "stg" { + "mtls" + } else { + "plain" + } + } + } else { + "plain" + } + } else { + mode.as_str() + }; + + info!(effective_mode = %effective_mode, "determined mTLS mode"); + + match effective_mode { + "plain" => { + run_plain_proxy(&cfg.service.mesh_bind_addr, &cfg.service.app_addr).await?; + } + "tls" | "mtls" => { + let tls_cfg = build_server_config(&cfg, effective_mode)?; + run_tls_proxy(&cfg.service.mesh_bind_addr, &cfg.service.app_addr, tls_cfg).await?; + } + other => { + return Err(anyhow!("unsupported mtls.mode: {}", other)); + } + } + + Ok(()) +} + +fn load_config(path: &PathBuf) -> Result { + let contents = fs::read_to_string(path) + .with_context(|| format!("failed to read {}", path.display()))?; + let cfg: Config = + toml::from_str(&contents).with_context(|| format!("failed to parse {}", path.display()))?; + Ok(cfg) +} + +fn load_certs(path: &str) -> Result>> { + let file = fs::File::open(path).with_context(|| format!("failed to open cert file {}", path))?; + let mut reader = BufReader::new(file); + let certs = certs(&mut reader) + .collect::, _>>() + .map_err(|e| anyhow!("failed to parse certs from {}: {}", path, e))?; + Ok(certs) +} + +fn load_private_key(path: &str) -> Result> { + let file = fs::File::open(path).with_context(|| format!("failed to open key file {}", path))?; + let mut reader = BufReader::new(file); + + // Try PKCS8 first + if let Ok(keys) = pkcs8_private_keys(&mut reader).collect::, _>>() { + if let Some(k) = keys.into_iter().next() { + return Ok(PrivateKeyDer::Pkcs8(k)); + } + } + + // Fallback to RSA + let file = fs::File::open(path).with_context(|| format!("failed to open key file {}", path))?; + let mut reader = BufReader::new(file); + let keys = rsa_private_keys(&mut reader) + .collect::, _>>() + .map_err(|e| anyhow!("failed to parse private key from {}: {}", path, e))?; + let Some(k) = keys.into_iter().next() else { + return Err(anyhow!("no private keys found in {}", path)); + }; + Ok(PrivateKeyDer::Pkcs1(k)) +} + +fn build_server_config(cfg: &Config, mode: &str) -> Result { + let mtls = cfg + .mtls + .as_ref() + .ok_or_else(|| anyhow!("mtls section is required for mode {}", mode))?; + + let cert_path = mtls + .cert_path + .as_deref() + .ok_or_else(|| anyhow!("mtls.cert_path is required"))?; + let key_path = mtls + .key_path + .as_deref() + .ok_or_else(|| anyhow!("mtls.key_path is required"))?; + + let certs = load_certs(cert_path)?; + let key = load_private_key(key_path)?; + + // ベースは「クライアント認証なし」のサーバ設定 + let builder = ServerConfig::builder(); + + // mTLS の場合はクライアント証明書を要求する verifer を使う + if mode == "mtls" { + let ca_path = mtls + .ca_cert_path + .as_deref() + .ok_or_else(|| anyhow!("mtls.ca_cert_path is required for mtls mode"))?; + let client_certs = load_certs(ca_path)?; + let mut roots = rustls::RootCertStore::empty(); + for c in client_certs { + roots.add(c).map_err(|e| anyhow!("adding CA failed: {:?}", e))?; + } + let verifier = + rustls::server::WebPkiClientVerifier::builder(std::sync::Arc::new(roots)).build()?; + let cfg = builder + .with_client_cert_verifier(verifier) + .with_single_cert(certs, key) + .map_err(|e| anyhow!("failed to build mtls server config: {}", e))?; + Ok(cfg) + } else { + let cfg = builder + .with_no_client_auth() + .with_single_cert(certs, key) + .map_err(|e| anyhow!("failed to build tls server config: {}", e))?; + Ok(cfg) + } +} + +async fn run_plain_proxy(listen_addr: &str, app_addr: &str) -> Result<()> { + let listener = TcpListener::bind(listen_addr).await?; + info!("listening on {} and forwarding to {}", listen_addr, app_addr); + + loop { + let (inbound, peer) = listener.accept().await?; + let app_addr = app_addr.to_string(); + + info!(remote = %peer, "accepted connection"); + + task::spawn(async move { + if let Err(e) = handle_connection(inbound, &app_addr).await { + warn!(error = %e, "connection handling failed"); + } + }); + } +} + +async fn run_tls_proxy( + listen_addr: &str, + app_addr: &str, + server_config: ServerConfig, +) -> Result<()> { + let listener = TcpListener::bind(listen_addr).await?; + let acceptor = TlsAcceptor::from(std::sync::Arc::new(server_config)); + + info!( + "listening (TLS/mTLS) on {} and forwarding to {}", + listen_addr, app_addr + ); + + loop { + let (inbound, peer) = listener.accept().await?; + let app_addr = app_addr.to_string(); + let acceptor = acceptor.clone(); + + info!(remote = %peer, "accepted TLS connection"); + + task::spawn(async move { + match acceptor.accept(inbound).await { + Ok(tls_stream) => { + if let Err(e) = handle_tls_connection(tls_stream, &app_addr).await { + warn!(error = %e, "TLS connection handling failed"); + } + } + Err(e) => { + warn!(error = %e, "TLS handshake failed"); + } + } + }); + } +} + +async fn handle_tls_connection(inbound: S, app_addr: &str) -> Result<()> +where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, +{ + let mut outbound = TcpStream::connect(app_addr).await?; + let (mut ri, mut wi) = tokio::io::split(inbound); + let (mut ro, mut wo) = outbound.split(); + + let client_to_app = io::copy(&mut ri, &mut wo); + let app_to_client = io::copy(&mut ro, &mut wi); + + tokio::try_join!(client_to_app, app_to_client)?; + Ok(()) +} + +async fn handle_connection(mut inbound: TcpStream, app_addr: &str) -> Result<()> { + let mut outbound = TcpStream::connect(app_addr).await?; + let (mut ri, mut wi) = inbound.split(); + let (mut ro, mut wo) = outbound.split(); + + let client_to_app = io::copy(&mut ri, &mut wo); + let app_to_client = io::copy(&mut ro, &mut wi); + + tokio::try_join!(client_to_app, app_to_client)?; + Ok(()) +} + + diff --git a/mtls-agent/src/policy.rs b/mtls-agent/src/policy.rs new file mode 100644 index 0000000..df95f67 --- /dev/null +++ b/mtls-agent/src/policy.rs @@ -0,0 +1,114 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use tokio::sync::RwLock; +use tracing::{info, warn}; + +use crate::discovery::{MtlsPolicy, ServiceDiscovery}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PolicyDecision { + pub allow: bool, + pub mode: String, // mtls/tls/plain + pub reason: String, +} + +pub struct PolicyEnforcer { + discovery: Arc, + default_mode: String, + cache: Arc>>, +} + +impl PolicyEnforcer { + pub fn new(discovery: Arc, default_mode: String) -> Self { + Self { + discovery, + default_mode, + cache: Arc::new(RwLock::new(std::collections::HashMap::new())), + } + } + + pub async fn evaluate( + &self, + source_service: &str, + target_service: &str, + ) -> Result { + let cache_key = format!("{}->{}", source_service, target_service); + + // キャッシュをチェック + { + let cache = self.cache.read().await; + if let Some((decision, timestamp)) = cache.get(&cache_key) { + if timestamp.elapsed() < Duration::from_secs(30) { + return Ok(decision.clone()); + } + } + } + + // Chainfireからポリシーを取得 + let policy = self + .discovery + .get_mtls_policy(source_service, target_service) + .await?; + + let decision = if let Some(p) = policy { + PolicyDecision { + allow: true, + mode: p.mode.unwrap_or_else(|| { + if p.mtls_required.unwrap_or(false) { + "mtls".to_string() + } else { + "plain".to_string() + } + }), + reason: format!("policy {} applied", p.policy_id), + } + } else { + PolicyDecision { + allow: true, + mode: self.default_mode.clone(), + reason: "default policy applied".to_string(), + } + }; + + // キャッシュに保存 + { + let mut cache = self.cache.write().await; + cache.insert(cache_key, (decision.clone(), Instant::now())); + } + + info!( + source = source_service, + target = target_service, + mode = %decision.mode, + "policy decision" + ); + + Ok(decision) + } + + pub async fn start_background_refresh(&self) { + let cache = Arc::clone(&self.cache); + let discovery = Arc::clone(&self.discovery); + let default_mode = self.default_mode.clone(); + + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(60)); + loop { + interval.tick().await; + + // キャッシュをクリア（簡易実装） + { + let mut cache_guard = cache.write().await; + cache_guard.clear(); + } + + info!("policy cache cleared, will be refreshed on next request"); + } + }); + } +} + + diff --git a/nix/modules/apigateway.nix b/nix/modules/apigateway.nix new file mode 100644 index 0000000..9ccb954 --- /dev/null +++ b/nix/modules/apigateway.nix @@ -0,0 +1,293 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.services.apigateway; + tomlFormat = pkgs.formats.toml {}; + authProviderType = lib.types.submodule { + options = { + name = lib.mkOption { + type = lib.types.str; + description = "Auth provider name"; + }; + + providerType = lib.mkOption { + type = lib.types.str; + default = "grpc"; + description = "Auth provider type (grpc)"; + }; + + endpoint = lib.mkOption { + type = lib.types.str; + description = "Auth provider endpoint (e.g., http://127.0.0.1:9000)"; + }; + + timeoutMs = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + description = "Auth provider timeout in milliseconds"; + }; + }; + }; + creditProviderType = lib.types.submodule { + options = { + name = lib.mkOption { + type = lib.types.str; + description = "Credit provider name"; + }; + + providerType = lib.mkOption { + type = lib.types.str; + default = "grpc"; + description = "Credit provider type (grpc)"; + }; + + endpoint = lib.mkOption { + type = lib.types.str; + description = "Credit provider endpoint (e.g., http://127.0.0.1:9100)"; + }; + + timeoutMs = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + description = "Credit provider timeout in milliseconds"; + }; + }; + }; + routeAuthType = lib.types.submodule { + options = { + provider = lib.mkOption { + type = lib.types.str; + description = "Auth provider name to use"; + }; + + mode = lib.mkOption { + type = lib.types.enum [ "disabled" "optional" "required" ]; + default = "required"; + description = "Auth enforcement mode"; + }; + + failOpen = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Allow requests on auth failures"; + }; + }; + }; + routeCreditType = lib.types.submodule { + options = { + provider = lib.mkOption { + type = lib.types.str; + description = "Credit provider name to use"; + }; + + mode = lib.mkOption { + type = lib.types.enum [ "disabled" "optional" "required" ]; + default = "required"; + description = "Credit enforcement mode"; + }; + + units = lib.mkOption { + type = lib.types.int; + default = 1; + description = "Credit units to reserve per request"; + }; + + failOpen = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Allow requests on credit failures"; + }; + + commitOn = lib.mkOption { + type = lib.types.enum [ "success" "always" "never" ]; + default = "success"; + description = "Credit commit policy"; + }; + + attributes = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = {}; + description = "Extra credit attributes"; + }; + }; + }; + routeType = lib.types.submodule { + options = { + name = lib.mkOption { + type = lib.types.str; + description = "Route name"; + }; + + pathPrefix = lib.mkOption { + type = lib.types.str; + description = "Path prefix to match"; + }; + + upstream = lib.mkOption { + type = lib.types.str; + description = "Upstream base URL"; + }; + + stripPrefix = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Strip the path prefix before proxying"; + }; + + auth = lib.mkOption { + type = lib.types.nullOr routeAuthType; + default = null; + description = "Auth configuration for this route"; + }; + + credit = lib.mkOption { + type = lib.types.nullOr routeCreditType; + default = null; + description = "Credit configuration for this route"; + }; + }; + }; + baseConfig = { + http_addr = "127.0.0.1:${toString cfg.port}"; + log_level = "info"; + }; + toAuthProvider = provider: { + name = provider.name; + type = provider.providerType; + endpoint = provider.endpoint; + } // lib.optionalAttrs (provider.timeoutMs != null) { + timeout_ms = provider.timeoutMs; + }; + toCreditProvider = provider: { + name = provider.name; + type = provider.providerType; + endpoint = provider.endpoint; + } // lib.optionalAttrs (provider.timeoutMs != null) { + timeout_ms = provider.timeoutMs; + }; + toRouteAuth = auth: { + provider = auth.provider; + mode = auth.mode; + fail_open = auth.failOpen; + }; + toRouteCredit = credit: { + provider = credit.provider; + mode = credit.mode; + units = credit.units; + fail_open = credit.failOpen; + commit_on = credit.commitOn; + attributes = credit.attributes; + }; + toRoute = route: { + name = route.name; + path_prefix = route.pathPrefix; + upstream = route.upstream; + strip_prefix = route.stripPrefix; + } + // lib.optionalAttrs (route.auth != null) { + auth = toRouteAuth route.auth; + } + // lib.optionalAttrs (route.credit != null) { + credit = toRouteCredit route.credit; + }; + generatedConfig = { + max_body_bytes = cfg.maxBodyBytes; + auth_providers = map toAuthProvider cfg.authProviders; + credit_providers = map toCreditProvider cfg.creditProviders; + routes = map toRoute cfg.routes; + }; + configFile = tomlFormat.generate "apigateway.toml" ( + lib.recursiveUpdate baseConfig (lib.recursiveUpdate generatedConfig cfg.settings) + ); + configPath = "/etc/apigateway/apigateway.toml"; +in { + options.services.apigateway = { + enable = lib.mkEnableOption "apigateway service"; + + port = lib.mkOption { + type = lib.types.port; + default = 8080; + description = "Port for the API gateway HTTP listener"; + }; + + dataDir = lib.mkOption { + type = lib.types.path; + default = "/var/lib/apigateway"; + description = "Data directory for apigateway"; + }; + + maxBodyBytes = lib.mkOption { + type = lib.types.int; + default = 16 * 1024 * 1024; + description = "Maximum request body size in bytes"; + }; + + authProviders = lib.mkOption { + type = lib.types.listOf authProviderType; + default = []; + description = "Auth provider definitions"; + }; + + creditProviders = lib.mkOption { + type = lib.types.listOf creditProviderType; + default = []; + description = "Credit provider definitions"; + }; + + routes = lib.mkOption { + type = lib.types.listOf routeType; + default = []; + description = "API gateway routes"; + }; + + settings = lib.mkOption { + type = lib.types.attrs; + default = {}; + description = "Additional API gateway TOML settings (merged into apigateway.toml)"; + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.apigateway-server or (throw "apigateway-server package not found"); + description = "Package to use for apigateway"; + }; + }; + + config = lib.mkIf cfg.enable { + users.users.apigateway = { + isSystemUser = true; + group = "apigateway"; + description = "API gateway service user"; + home = cfg.dataDir; + }; + + users.groups.apigateway = {}; + + systemd.services.apigateway = { + description = "PlasmaCloud API Gateway"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + serviceConfig = { + Type = "simple"; + User = "apigateway"; + Group = "apigateway"; + Restart = "on-failure"; + RestartSec = "10s"; + + StateDirectory = "apigateway"; + StateDirectoryMode = "0750"; + + NoNewPrivileges = true; + PrivateTmp = true; + ProtectSystem = "strict"; + ProtectHome = true; + ReadWritePaths = [ cfg.dataDir ]; + + ExecStart = "${cfg.package}/bin/apigateway-server --config ${configPath}"; + }; + }; + + environment.etc."apigateway/apigateway.toml".source = configFile; + }; +} diff --git a/nix/modules/plasmacloud-resources.nix b/nix/modules/plasmacloud-resources.nix new file mode 100644 index 0000000..26aeea5 --- /dev/null +++ b/nix/modules/plasmacloud-resources.nix @@ -0,0 +1,667 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + jsonFormat = pkgs.formats.json {}; + + lbBackendType = types.submodule { + options = { + name = mkOption { + type = types.str; + description = "Backend name"; + }; + + address = mkOption { + type = types.str; + description = "Backend IP or hostname"; + }; + + port = mkOption { + type = types.port; + description = "Backend port"; + }; + + weight = mkOption { + type = types.nullOr types.int; + default = null; + description = "Backend weight (default: 1)"; + }; + + admin_state = mkOption { + type = types.nullOr types.str; + default = null; + description = "Backend admin state (enabled, disabled, drain)"; + }; + }; + }; + + lbSessionPersistenceType = types.submodule { + options = { + type = mkOption { + type = types.enum [ "source_ip" "cookie" "app_cookie" ]; + description = "Persistence strategy"; + }; + + cookie_name = mkOption { + type = types.nullOr types.str; + default = null; + description = "Cookie name for cookie-based persistence"; + }; + + timeout_seconds = mkOption { + type = types.nullOr types.int; + default = null; + description = "Persistence timeout in seconds"; + }; + }; + }; + + lbHttpHealthType = types.submodule { + options = { + method = mkOption { + type = types.nullOr types.str; + default = null; + description = "HTTP method for health checks"; + }; + + path = mkOption { + type = types.nullOr types.str; + default = null; + description = "HTTP path for health checks"; + }; + + expected_codes = mkOption { + type = types.listOf types.int; + default = []; + description = "Expected HTTP status codes"; + }; + + host = mkOption { + type = types.nullOr types.str; + default = null; + description = "Host header for HTTP health checks"; + }; + }; + }; + + lbHealthCheckType = types.submodule { + options = { + name = mkOption { + type = types.str; + description = "Health check name"; + }; + + type = mkOption { + type = types.enum [ "tcp" "http" "https" "udp" "ping" ]; + description = "Health check type"; + }; + + interval_seconds = mkOption { + type = types.nullOr types.int; + default = null; + description = "Interval in seconds"; + }; + + timeout_seconds = mkOption { + type = types.nullOr types.int; + default = null; + description = "Timeout in seconds"; + }; + + healthy_threshold = mkOption { + type = types.nullOr types.int; + default = null; + description = "Healthy threshold"; + }; + + unhealthy_threshold = mkOption { + type = types.nullOr types.int; + default = null; + description = "Unhealthy threshold"; + }; + + http = mkOption { + type = types.nullOr lbHttpHealthType; + default = null; + description = "HTTP-specific health check configuration"; + }; + + enabled = mkOption { + type = types.nullOr types.bool; + default = null; + description = "Enable or disable health check"; + }; + }; + }; + + lbPoolType = types.submodule { + options = { + name = mkOption { + type = types.str; + description = "Pool name"; + }; + + algorithm = mkOption { + type = types.nullOr types.str; + default = null; + description = "Pool algorithm (round_robin, least_connections, ip_hash, weighted_round_robin, random, maglev)"; + }; + + protocol = mkOption { + type = types.nullOr types.str; + default = null; + description = "Pool protocol (tcp, udp, http, https)"; + }; + + session_persistence = mkOption { + type = types.nullOr lbSessionPersistenceType; + default = null; + description = "Session persistence configuration"; + }; + + backends = mkOption { + type = types.listOf lbBackendType; + default = []; + description = "Pool backends"; + }; + + health_checks = mkOption { + type = types.listOf lbHealthCheckType; + default = []; + description = "Pool health checks"; + }; + }; + }; + + lbTlsType = types.submodule { + options = { + certificate_id = mkOption { + type = types.str; + description = "Certificate ID (FiberLB)"; + }; + + min_version = mkOption { + type = types.nullOr types.str; + default = null; + description = "TLS minimum version (tls_1_2, tls_1_3)"; + }; + + cipher_suites = mkOption { + type = types.listOf types.str; + default = []; + description = "TLS cipher suites"; + }; + }; + }; + + lbL7RuleType = types.submodule { + options = { + type = mkOption { + type = types.enum [ "host_name" "path" "file_type" "header" "cookie" "ssl_conn_has_sni" ]; + description = "L7 rule type"; + }; + + compare_type = mkOption { + type = types.nullOr (types.enum [ "equal_to" "regex" "starts_with" "ends_with" "contains" ]); + default = null; + description = "L7 compare type"; + }; + + value = mkOption { + type = types.str; + description = "L7 rule match value"; + }; + + key = mkOption { + type = types.nullOr types.str; + default = null; + description = "Header/cookie key for L7 rules"; + }; + + invert = mkOption { + type = types.nullOr types.bool; + default = null; + description = "Invert the rule match"; + }; + }; + }; + + lbL7PolicyType = types.submodule { + options = { + name = mkOption { + type = types.str; + description = "L7 policy name"; + }; + + position = mkOption { + type = types.nullOr types.int; + default = null; + description = "L7 policy position (lower evaluates first)"; + }; + + action = mkOption { + type = types.enum [ "redirect_to_pool" "redirect_to_url" "reject" ]; + description = "L7 policy action"; + }; + + redirect_url = mkOption { + type = types.nullOr types.str; + default = null; + description = "Redirect URL for redirect_to_url action"; + }; + + redirect_pool = mkOption { + type = types.nullOr types.str; + default = null; + description = "Target pool name for redirect_to_pool action"; + }; + + redirect_http_status_code = mkOption { + type = types.nullOr types.int; + default = null; + description = "Redirect or reject HTTP status code"; + }; + + enabled = mkOption { + type = types.nullOr types.bool; + default = null; + description = "Enable or disable the policy"; + }; + + rules = mkOption { + type = types.listOf lbL7RuleType; + default = []; + description = "L7 rules for this policy"; + }; + }; + }; + + lbListenerType = types.submodule { + options = { + name = mkOption { + type = types.str; + description = "Listener name"; + }; + + protocol = mkOption { + type = types.nullOr types.str; + default = null; + description = "Listener protocol (tcp, udp, http, https, terminated_https)"; + }; + + port = mkOption { + type = types.port; + description = "Listener port"; + }; + + default_pool = mkOption { + type = types.str; + description = "Default pool name"; + }; + + tls = mkOption { + type = types.nullOr lbTlsType; + default = null; + description = "TLS configuration"; + }; + + connection_limit = mkOption { + type = types.nullOr types.int; + default = null; + description = "Connection limit"; + }; + + enabled = mkOption { + type = types.nullOr types.bool; + default = null; + description = "Enable or disable listener"; + }; + + l7_policies = mkOption { + type = types.listOf lbL7PolicyType; + default = []; + description = "L7 policies for this listener"; + }; + }; + }; + + lbType = types.submodule { + options = { + name = mkOption { + type = types.str; + description = "Load balancer name"; + }; + + org_id = mkOption { + type = types.str; + description = "Organization ID"; + }; + + project_id = mkOption { + type = types.nullOr types.str; + default = null; + description = "Project ID (null for org-level)"; + }; + + description = mkOption { + type = types.nullOr types.str; + default = null; + description = "Load balancer description"; + }; + + pools = mkOption { + type = types.listOf lbPoolType; + default = []; + description = "Pools for this load balancer"; + }; + + listeners = mkOption { + type = types.listOf lbListenerType; + default = []; + description = "Listeners for this load balancer"; + }; + }; + }; + + dnsRecordType = types.submodule { + options = { + name = mkOption { + type = types.str; + description = "Record name"; + }; + + record_type = mkOption { + type = types.str; + description = "Record type (A, AAAA, CNAME, MX, TXT, SRV, NS, PTR, CAA)"; + }; + + ttl = mkOption { + type = types.nullOr types.int; + default = null; + description = "Record TTL (default: 300)"; + }; + + data = mkOption { + type = types.attrsOf types.anything; + description = "Record data (type-specific)"; + }; + + enabled = mkOption { + type = types.nullOr types.bool; + default = null; + description = "Enable or disable record"; + }; + }; + }; + + dnsZoneType = types.submodule { + options = { + name = mkOption { + type = types.str; + description = "Zone name (example.com)"; + }; + + org_id = mkOption { + type = types.str; + description = "Organization ID"; + }; + + project_id = mkOption { + type = types.nullOr types.str; + default = null; + description = "Project ID (null for org-level)"; + }; + + primary_ns = mkOption { + type = types.nullOr types.str; + default = null; + description = "Primary NS hostname"; + }; + + admin_email = mkOption { + type = types.nullOr types.str; + default = null; + description = "Admin email for SOA"; + }; + + refresh = mkOption { + type = types.nullOr types.int; + default = null; + description = "SOA refresh"; + }; + + retry = mkOption { + type = types.nullOr types.int; + default = null; + description = "SOA retry"; + }; + + expire = mkOption { + type = types.nullOr types.int; + default = null; + description = "SOA expire"; + }; + + minimum = mkOption { + type = types.nullOr types.int; + default = null; + description = "SOA minimum"; + }; + + records = mkOption { + type = types.listOf dnsRecordType; + default = []; + description = "Zone records"; + }; + }; + }; + + dnsReverseZoneType = types.submodule { + options = { + org_id = mkOption { + type = types.str; + description = "Organization ID"; + }; + + project_id = mkOption { + type = types.nullOr types.str; + default = null; + description = "Project ID (null for org-level)"; + }; + + cidr = mkOption { + type = types.str; + description = "CIDR for the reverse zone"; + }; + + ptr_pattern = mkOption { + type = types.str; + description = "PTR pattern (e.g., {4}-{3}-{2}-{1}.hosts.example.com.)"; + }; + + ttl = mkOption { + type = types.nullOr types.int; + default = null; + description = "PTR TTL"; + }; + }; + }; + + lbCfg = config.plasmacloud.lb; + dnsCfg = config.plasmacloud.dns; + + lbConfigFile = jsonFormat.generate "plasmacloud-lb.json" { + load_balancers = lbCfg.loadBalancers; + }; + lbConfigPath = lbCfg.configPath; + lbConfigRelative = removePrefix "/etc/" lbConfigPath; + + dnsConfigFile = jsonFormat.generate "plasmacloud-dns.json" { + zones = dnsCfg.zones; + reverse_zones = dnsCfg.reverseZones; + }; + dnsConfigPath = dnsCfg.configPath; + dnsConfigRelative = removePrefix "/etc/" dnsConfigPath; + +in { + options.plasmacloud.lb = { + enable = mkEnableOption "PlasmaCloud load balancer declarations"; + + endpoint = mkOption { + type = types.str; + default = "http://127.0.0.1:7000"; + description = "FiberLB gRPC endpoint"; + }; + + loadBalancers = mkOption { + type = types.listOf lbType; + default = []; + description = "Load balancer declarations"; + }; + + configPath = mkOption { + type = types.str; + default = "/etc/plasmacloud/lb.json"; + description = "Path for rendered load balancer config"; + }; + + applyOnBoot = mkOption { + type = types.bool; + default = true; + description = "Apply declarations at boot"; + }; + + applyOnChange = mkOption { + type = types.bool; + default = true; + description = "Apply declarations when the config file changes"; + }; + + prune = mkOption { + type = types.bool; + default = false; + description = "Delete load balancer resources not declared in config"; + }; + + package = mkOption { + type = types.package; + default = pkgs.plasmacloud-reconciler or (throw "plasmacloud-reconciler package not found"); + description = "Reconciler package for load balancer declarations"; + }; + }; + + options.plasmacloud.dns = { + enable = mkEnableOption "PlasmaCloud DNS declarations"; + + endpoint = mkOption { + type = types.str; + default = "http://127.0.0.1:6000"; + description = "FlashDNS gRPC endpoint"; + }; + + zones = mkOption { + type = types.listOf dnsZoneType; + default = []; + description = "DNS zone declarations"; + }; + + reverseZones = mkOption { + type = types.listOf dnsReverseZoneType; + default = []; + description = "Reverse DNS zone declarations"; + }; + + configPath = mkOption { + type = types.str; + default = "/etc/plasmacloud/dns.json"; + description = "Path for rendered DNS config"; + }; + + applyOnBoot = mkOption { + type = types.bool; + default = true; + description = "Apply declarations at boot"; + }; + + applyOnChange = mkOption { + type = types.bool; + default = true; + description = "Apply declarations when the config file changes"; + }; + + prune = mkOption { + type = types.bool; + default = false; + description = "Delete DNS resources not declared in config"; + }; + + package = mkOption { + type = types.package; + default = pkgs.plasmacloud-reconciler or (throw "plasmacloud-reconciler package not found"); + description = "Reconciler package for DNS declarations"; + }; + }; + + config = mkMerge [ + (mkIf lbCfg.enable { + assertions = [ + { + assertion = hasPrefix "/etc/" lbConfigPath; + message = "plasmacloud.lb.configPath must be under /etc"; + } + ]; + + environment.etc."${lbConfigRelative}".source = lbConfigFile; + + systemd.services.plasmacloud-lb-apply = { + description = "Apply PlasmaCloud load balancer declarations"; + after = [ "network-online.target" ] ++ (optional config.services.fiberlb.enable "fiberlb.service"); + wants = [ "network-online.target" ] ++ (optional config.services.fiberlb.enable "fiberlb.service"); + wantedBy = optional lbCfg.applyOnBoot "multi-user.target"; + + serviceConfig = { + Type = "oneshot"; + ExecStart = "${lbCfg.package}/bin/plasmacloud-reconciler lb --config ${lbConfigPath} --endpoint ${lbCfg.endpoint}${optionalString lbCfg.prune " --prune"}"; + }; + }; + + systemd.paths.plasmacloud-lb-apply = mkIf lbCfg.applyOnChange { + wantedBy = [ "multi-user.target" ]; + pathConfig = { + PathChanged = lbConfigPath; + }; + }; + }) + + (mkIf dnsCfg.enable { + assertions = [ + { + assertion = hasPrefix "/etc/" dnsConfigPath; + message = "plasmacloud.dns.configPath must be under /etc"; + } + ]; + + environment.etc."${dnsConfigRelative}".source = dnsConfigFile; + + systemd.services.plasmacloud-dns-apply = { + description = "Apply PlasmaCloud DNS declarations"; + after = [ "network-online.target" ] ++ (optional config.services.flashdns.enable "flashdns.service"); + wants = [ "network-online.target" ] ++ (optional config.services.flashdns.enable "flashdns.service"); + wantedBy = optional dnsCfg.applyOnBoot "multi-user.target"; + + serviceConfig = { + Type = "oneshot"; + ExecStart = "${dnsCfg.package}/bin/plasmacloud-reconciler dns --config ${dnsConfigPath} --endpoint ${dnsCfg.endpoint}${optionalString dnsCfg.prune " --prune"}"; + }; + }; + + systemd.paths.plasmacloud-dns-apply = mkIf dnsCfg.applyOnChange { + wantedBy = [ "multi-user.target" ]; + pathConfig = { + PathChanged = dnsConfigPath; + }; + }; + }) + ]; +} diff --git a/nix/templates/iam-flaredb-minimal.nix b/nix/templates/iam-flaredb-minimal.nix new file mode 100644 index 0000000..b2da606 --- /dev/null +++ b/nix/templates/iam-flaredb-minimal.nix @@ -0,0 +1,18 @@ +{ inputs, pkgs, config, ... }: +{ + # Minimal footprint: chainfire + flaredb + iam only (for auth/metadata testing). + imports = [ inputs.self.nixosModules.plasmacloud ]; + + networking.hostName = "plasmacloud-minimal"; + networking.firewall.allowedTCPPorts = [ 8081 8082 9000 ]; + + services.chainfire.enable = true; + + services.flaredb.enable = true; + + services.iam.enable = true; + + environment.systemPackages = with inputs.self.packages.${pkgs.system}; [ + chainfire-server flaredb-server iam-server + ]; +} diff --git a/nix/templates/plasmacloud-3node-ha.nix b/nix/templates/plasmacloud-3node-ha.nix new file mode 100644 index 0000000..c84adde --- /dev/null +++ b/nix/templates/plasmacloud-3node-ha.nix @@ -0,0 +1,88 @@ +{ inputs, pkgs, lib, config, ... }: +{ + # Example: 3-node HA control plane. Replace IPs/hostnames to match your cluster. + imports = [ inputs.self.nixosModules.plasmacloud ]; + + networking.hostName = lib.mkDefault "plasmacloud-node01"; + networking.firewall.allowedTCPPorts = [ 8080 8081 8082 8083 8084 8085 8086 8087 9000 9001 9002 2379 2380 2381 2479 2480 ]; + + # Core data stores + services.chainfire = { + enable = true; + dataDir = "/var/lib/chainfire"; + # Adjust ports if you need to avoid conflicts; defaults are fine for most cases. + port = 2379; + raftPort = 2380; + gossipPort = 2381; + }; + + services.flaredb = { + enable = true; + dataDir = "/var/lib/flaredb"; + port = 2479; + raftPort = 2480; + httpPort = 8082; + }; + + # IAM + services.iam = { + enable = true; + dataDir = "/var/lib/iam"; + }; + + # Compute + networking + ingress + services.plasmavmc.enable = true; + services.prismnet.enable = true; + services.flashdns.enable = true; + services.fiberlb.enable = true; + services.apigateway = { + enable = true; + authProviders = [{ + name = "iam"; + providerType = "grpc"; + endpoint = "http://127.0.0.1:${toString config.services.iam.port}"; + }]; + creditProviders = [{ + name = "creditservice"; + providerType = "grpc"; + endpoint = "http://127.0.0.1:${toString config.services.creditservice.grpcPort}"; + }]; + routes = [ + { + name = "iam-rest"; + pathPrefix = "/iam"; + upstream = "http://127.0.0.1:8083"; + stripPrefix = true; + auth = { + provider = "iam"; + mode = "required"; + }; + } + { + name = "credit-rest"; + pathPrefix = "/credit"; + upstream = "http://127.0.0.1:${toString config.services.creditservice.httpPort}"; + stripPrefix = true; + auth = { + provider = "iam"; + mode = "required"; + }; + credit = { + provider = "creditservice"; + mode = "optional"; + units = 1; + commitOn = "success"; + }; + } + ]; + }; + services.lightningstor.enable = true; + services.creditservice.enable = true; + + # Optional: install binaries for debugging + environment.systemPackages = with inputs.self.packages.${pkgs.system}; [ + chainfire-server flaredb-server iam-server plasmavmc-server + prismnet-server flashdns-server fiberlb-server apigateway-server lightningstor-server + creditservice-server + ]; +} diff --git a/nix/templates/plasmacloud-single-node.nix b/nix/templates/plasmacloud-single-node.nix new file mode 100644 index 0000000..fffb6bc --- /dev/null +++ b/nix/templates/plasmacloud-single-node.nix @@ -0,0 +1,84 @@ +{ inputs, pkgs, config, ... }: +{ + # Import all PlasmaCloud modules (chainfire, flaredb, iam, plasmavmc, prismnet, flashdns, fiberlb, lightningstor, creditservice). + imports = [ inputs.self.nixosModules.plasmacloud ]; + + networking.hostName = "plasmacloud-single"; + networking.firewall.allowedTCPPorts = [ 8080 8081 8082 8083 8084 8085 8086 8087 9000 9001 9002 ]; + + # Enable all services with default ports and data dirs. + services.chainfire = { + enable = true; + dataDir = "/var/lib/chainfire"; + }; + + services.flaredb = { + enable = true; + dataDir = "/var/lib/flaredb"; + settings = { + chainfire_endpoint = "http://127.0.0.1:${toString config.services.chainfire.port}"; + }; + }; + + services.iam = { + enable = true; + dataDir = "/var/lib/iam"; + settings = { + flaredb_endpoint = "http://127.0.0.1:${toString config.services.flaredb.port}"; + }; + }; + + services.plasmavmc.enable = true; + services.prismnet.enable = true; + services.flashdns.enable = true; + services.fiberlb.enable = true; + services.apigateway = { + enable = true; + authProviders = [{ + name = "iam"; + providerType = "grpc"; + endpoint = "http://127.0.0.1:${toString config.services.iam.port}"; + }]; + creditProviders = [{ + name = "creditservice"; + providerType = "grpc"; + endpoint = "http://127.0.0.1:${toString config.services.creditservice.grpcPort}"; + }]; + routes = [ + { + name = "iam-rest"; + pathPrefix = "/iam"; + upstream = "http://127.0.0.1:8083"; + stripPrefix = true; + auth = { + provider = "iam"; + mode = "required"; + }; + } + { + name = "credit-rest"; + pathPrefix = "/credit"; + upstream = "http://127.0.0.1:${toString config.services.creditservice.httpPort}"; + stripPrefix = true; + auth = { + provider = "iam"; + mode = "required"; + }; + credit = { + provider = "creditservice"; + mode = "optional"; + units = 1; + commitOn = "success"; + }; + } + ]; + }; + services.lightningstor.enable = true; + services.creditservice.enable = true; + + environment.systemPackages = with inputs.self.packages.${pkgs.system}; [ + chainfire-server flaredb-server iam-server plasmavmc-server + prismnet-server flashdns-server fiberlb-server apigateway-server lightningstor-server + creditservice-server + ]; +} diff --git a/plasmavmc/crates/plasmavmc-server/tests/common/mod.rs b/plasmavmc/crates/plasmavmc-server/tests/common/mod.rs new file mode 100644 index 0000000..333c585 --- /dev/null +++ b/plasmavmc/crates/plasmavmc-server/tests/common/mod.rs @@ -0,0 +1,165 @@ +#![allow(dead_code)] + +use async_trait::async_trait; +use plasmavmc_api::proto::vm_service_client::VmServiceClient; +use plasmavmc_hypervisor::{BackendCapabilities, HypervisorBackend, UnsupportedReason}; +use plasmavmc_types::{ + DiskSpec, HypervisorType as VmHypervisorType, NetworkSpec, Result as VmResult, VmHandle, + VmState, VmStatus, VirtualMachine, +}; +use std::time::Duration; +use tonic::codegen::InterceptedService; +use tonic::service::Interceptor; +use tonic::transport::Channel; +use tonic::Request; + +/// Global lock to serialize tests that mutate process-wide environment variables. +/// +/// Many of our integration tests rely on env-based configuration (endpoints, storage backend, +/// runtime paths). Rust tests run in parallel by default, so we guard those mutations. +pub async fn env_lock() -> tokio::sync::MutexGuard<'static, ()> { + use std::sync::OnceLock; + use tokio::sync::Mutex; + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())).lock().await +} + +/// Set per-test env defaults so PlasmaVMC can run in a fast, local-only mode. +/// +/// - Uses file-backed storage to avoid external dependencies +/// - Stores runtime/state under `/tmp` to avoid permission issues +pub fn set_plasmavmc_fast_test_env() { + // Force file backend to avoid ChainFire/FlareDB connections in the fast lane. + std::env::set_var("PLASMAVMC_STORAGE_BACKEND", "file"); + + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let runtime_dir = std::path::Path::new("/tmp").join(format!("pvmc-runtime-{nanos}")); + let state_path = std::path::Path::new("/tmp").join(format!("pvmc-state-{nanos}.json")); + + std::env::set_var("PLASMAVMC_RUNTIME_DIR", runtime_dir.to_str().unwrap()); + std::env::set_var("PLASMAVMC_STATE_PATH", state_path.to_str().unwrap()); +} + +/// Allocate an ephemeral localhost port for test servers. +pub fn allocate_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .expect("bind ephemeral port") + .local_addr() + .unwrap() + .port() +} + +/// Common interceptor to attach org/project metadata to PlasmaVMC requests. +pub struct OrgProjectInterceptor { + pub org: String, + pub project: String, +} + +impl Interceptor for OrgProjectInterceptor { + fn call(&mut self, mut req: Request<()>) -> Result, tonic::Status> { + req.metadata_mut().insert("org-id", self.org.parse().unwrap()); + req.metadata_mut() + .insert("project-id", self.project.parse().unwrap()); + Ok(req) + } +} + +pub async fn vm_client_with_meta( + addr: &str, + org: &str, + project: &str, +) -> VmServiceClient> { + let channel = Channel::from_shared(format!("http://{addr}")) + .unwrap() + .connect() + .await + .unwrap(); + VmServiceClient::with_interceptor( + channel, + OrgProjectInterceptor { + org: org.to_string(), + project: project.to_string(), + }, + ) +} + +/// No-op hypervisor backend for tests (avoids QEMU dependency). +/// +/// It reports itself as KVM and returns a stub `VmHandle`, allowing PlasmaVMC API +/// semantics and integrations to be tested without a real hypervisor. +pub struct NoopHypervisor; + +#[async_trait] +impl HypervisorBackend for NoopHypervisor { + fn backend_type(&self) -> VmHypervisorType { + VmHypervisorType::Kvm + } + + fn capabilities(&self) -> BackendCapabilities { + BackendCapabilities::default() + } + + fn supports(&self, _spec: &plasmavmc_types::VmSpec) -> std::result::Result<(), UnsupportedReason> { + Ok(()) + } + + async fn create(&self, vm: &VirtualMachine) -> VmResult { + let runtime_dir = std::env::var("PLASMAVMC_RUNTIME_DIR") + .unwrap_or_else(|_| "/tmp/plasmavmc-noop".into()); + Ok(VmHandle { + vm_id: vm.id, + runtime_dir, + pid: Some(0), + backend_state: Default::default(), + }) + } + + async fn start(&self, _handle: &VmHandle) -> VmResult<()> { + Ok(()) + } + + async fn stop(&self, _handle: &VmHandle, _timeout: Duration) -> VmResult<()> { + Ok(()) + } + + async fn kill(&self, _handle: &VmHandle) -> VmResult<()> { + Ok(()) + } + + async fn reboot(&self, _handle: &VmHandle) -> VmResult<()> { + Ok(()) + } + + async fn delete(&self, _handle: &VmHandle) -> VmResult<()> { + Ok(()) + } + + async fn status(&self, _handle: &VmHandle) -> VmResult { + Ok(VmStatus { + actual_state: VmState::Stopped, + host_pid: Some(0), + ..Default::default() + }) + } + + async fn attach_disk(&self, _handle: &VmHandle, _disk: &DiskSpec) -> VmResult<()> { + Ok(()) + } + + async fn detach_disk(&self, _handle: &VmHandle, _disk_id: &str) -> VmResult<()> { + Ok(()) + } + + async fn attach_nic(&self, _handle: &VmHandle, _nic: &NetworkSpec) -> VmResult<()> { + Ok(()) + } + + async fn detach_nic(&self, _handle: &VmHandle, _nic_id: &str) -> VmResult<()> { + Ok(()) + } +} + + diff --git a/scripts/integration-matrix.sh b/scripts/integration-matrix.sh new file mode 100755 index 0000000..7043e4f --- /dev/null +++ b/scripts/integration-matrix.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Integration Test Matrix gate (chainfire→flaredb→plasmavmc→creditservice→nightlight) +# +# Nix-first entrypoint: +# nix run '.#integration-matrix' +# +# This file remains as a thin compatibility wrapper. The source of truth is `flake.nix`. + +if ! command -v nix >/dev/null 2>&1; then + echo "ERROR: nix not found. Please install Nix and run: nix run '.#integration-matrix'" >&2 + exit 127 +fi + +exec nix run .#integration-matrix -- "$@" diff --git a/scripts/nested-kvm-check.sh b/scripts/nested-kvm-check.sh new file mode 100755 index 0000000..a38c15a --- /dev/null +++ b/scripts/nested-kvm-check.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Nested KVM quick validator for PlasmaVMC host and guest +# Usage: sudo ./scripts/nested-kvm-check.sh + +require_root() { + if [[ "$EUID" -ne 0 ]]; then + echo "[ERROR] Run as root (needed to read module params)" >&2 + exit 1 + fi +} + +param_path() { + if [[ -f /sys/module/kvm_intel/parameters/nested ]]; then + echo "/sys/module/kvm_intel/parameters/nested" + elif [[ -f /sys/module/kvm_amd/parameters/nested ]]; then + echo "/sys/module/kvm_amd/parameters/nested" + else + echo "" + fi +} + +print_status() { + local path="$1" + local val + val="$(<"$path")" + echo "[INFO] Nested param at $path = $val" + if [[ "$val" =~ ^[Yy1]$ ]]; then + echo "[OK] Nested virtualization enabled" + else + echo "[WARN] Nested virtualization disabled. Enable via NixOS:" + if [[ "$path" == *kvm_intel* ]]; then + cat <<'CFG' +boot.kernelModules = [ "kvm-intel" ]; +boot.extraModprobeConfig = '' + options kvm-intel nested=1 +''; +CFG + else + cat <<'CFG' +boot.kernelModules = [ "kvm-amd" ]; +boot.extraModprobeConfig = '' + options kvm-amd nested=1 +''; +CFG + fi + fi +} + +smoke_guest_kvm() { + if ! command -v qemu-system-x86_64 >/dev/null 2>&1; then + echo "[WARN] qemu-system-x86_64 not found; skip guest KVM smoke" >&2 + return + fi + echo "[INFO] Launching minimal nested guest kernel (non-interactive)..." + set +e + qemu-system-x86_64 -accel kvm -cpu host -m 256 -nographic \ + -kernel /run/current-system/kernel -append "console=ttyS0 panic=1" < /dev/null >/tmp/nested-kvm.log 2>&1 & + local pid=$! + sleep 5 + if ps -p $pid >/dev/null 2>&1; then + echo "[OK] Nested KVM guest boot appears running (PID $pid). Stopping..." + kill $pid >/dev/null 2>&1 || true + else + echo "[WARN] Nested guest did not stay running; check /tmp/nested-kvm.log" >&2 + fi + set -e +} + +main() { + require_root + local p + p=$(param_path) + if [[ -z "$p" ]]; then + echo "[ERROR] No kvm_intel or kvm_amd module loaded; check virtualization support" >&2 + exit 1 + fi + print_status "$p" + smoke_guest_kvm +} + +main "$@" diff --git a/specifications/deployer/README.md b/specifications/deployer/README.md new file mode 100644 index 0000000..4d6549f --- /dev/null +++ b/specifications/deployer/README.md @@ -0,0 +1,354 @@ +## Deployer / NodeAgent / mTLSサービスメッシュ設計（ベアメタル向け） + +本書では、既存の `deployer/` クレート群と `baremetal/first-boot` を土台に、 +Chainfire を「ソース・オブ・トゥルース」とした常駐型 Deployer / NodeAgent +およびサービスメッシュ風 mTLS Agent の設計を定義する。 + +既存の first-boot は **初回クラスタ参加と基本サービス起動** に特化した +Bootstrapper として残し、本設計では **その後のライフサイクル管理と +サービス間 mTLS 通信** を担うコンポーネントを追加する。 + +--- + +### 1. Chainfire 上の論理モデル + +Chainfire は etcd 互換の KV ストアとして既に存在するため、 +以下のような論理モデルを「キー空間＋JSON 値」として表現する。 + +#### 1.1 ネームスペースとキー空間 + +- `photoncloud/` + - `clusters/{cluster_id}/nodes/{node_id}` + - `clusters/{cluster_id}/services/{service_name}` + - `clusters/{cluster_id}/instances/{service_name}/{instance_id}` + - `clusters/{cluster_id}/mtls/policies/{policy_id}` + - `clusters/{cluster_id}/mtls/certs/{node_id or service_name}` + +`deployer` 既存の Chainfire 利用は `namespace = "deployer"` だが、 +クラスタ状態については今後 `photoncloud/` 名前空間を利用する。 +Deployer 内部状態（phone-home の登録情報など）は引き続き +`deployer/` 名前空間でもよい。 + +#### 1.2 Cluster / Node モデル + +- **Cluster**（メタ情報のみ） + - キー: `photoncloud/clusters/{cluster_id}/meta` + - 値（JSON）: + - `cluster_id: string` + - `environment: "dev" | "stg" | "prod" | "test"` + - `created_at: RFC3339` + +- **Node** + - キー: `photoncloud/clusters/{cluster_id}/nodes/{node_id}` + - 値（JSON）: + - `node_id: string` + - `ip: string` + - `hostname: string` + - `roles: string[]` 例: `["control-plane"]`, `["worker"]` + - `state: "pending" | "provisioning" | "active" | "failed" | "draining"` + - `labels: { [key: string]: string }` + - `last_heartbeat: RFC3339` + - `machine_id: string` + +この `Node` は既存の `deployer_types::NodeInfo` と 1:1 でマッピング可能にする。 + +#### 1.3 Service / ServiceInstance モデル + +- **Service** + - キー: `photoncloud/clusters/{cluster_id}/services/{service_name}` + - 値（JSON）: + - `name: string` 例: `"chainfire"`, `"flaredb"`, `"iam"`, `"apigateway"` + - `ports: { "http"?: number, "grpc"?: number }` + - `protocol: "http" | "grpc"` + - `mtls_required: boolean` + - `mesh_mode: "agent" | "none"` + - `metadata: { [key: string]: string }` + +- **ServiceInstance** + - キー: + - `photoncloud/clusters/{cluster_id}/instances/{service_name}/{instance_id}` + - 値（JSON）: + - `instance_id: string` + - `service: string` + - `node_id: string` + - `ip: string` + - `port: number` // アプリケーションのローカルポート (例: 127.0.0.1:9000) + - `mesh_port: number` // mTLS Agent が listen するポート (例: 10.0.x.x:10443) + - `state: "starting" | "ready" | "unhealthy" | "draining" | "gone"` + - `version: string` + - `registered_at: RFC3339` + - `last_heartbeat: RFC3339` + +ServiceInstance 情報は、将来的に mTLS Agent からも watch され、 +サービス発見とロードバランシングの元データとなる。 + +#### 1.4 mTLS Policy モデル + +- **MTLSPolicy** + - キー: + - `photoncloud/clusters/{cluster_id}/mtls/policies/{policy_id}` + - 値（JSON）: + - `policy_id: string` + - `environment: string` // 例: "dev", "stg", "prod" + - `source_service: string | "*"` // 例: "apigateway" or "*" + - `target_service: string | "*"` + - `mtls_required: boolean` + - `mode: "strict" | "permissive" | "disabled"` + - `updated_at: RFC3339` + +解決アルゴリズム（例）: + +1. `source_service`, `target_service` 完全一致のポリシーを検索。 +2. `source_service="*"` or `target_service="*"` のワイルドカードポリシーを fallback として適用。 +3. どれもなければ、Cluster の `environment` ごとのデフォルト: + - `dev`: `mtls_required=false` + - `stg/prod`: `mtls_required=true` + +mTLS Agent は「自サービス名」と「接続先サービス名」をもとに、 +Chainfire からこのポリシーを解決し、mTLS/TLS/平文を切り替える。 + +#### 1.5 証明書メタデータ + +- **CertificateBinding** + - キー: + - `photoncloud/clusters/{cluster_id}/mtls/certs/services/{service_name}` + - `photoncloud/clusters/{cluster_id}/mtls/certs/nodes/{node_id}` + - 値（JSON）: + - `subject: string` // "spiffe://photoncloud/{cluster_id}/service/{service_name}" 等 + - `ca_id: string` + - `expires_at: RFC3339` + - `last_rotated_at: RFC3339` + - `fingerprint_sha256: string` + +実際の鍵/証明書 PEM は Chainfire には保存せず、 +別の Secret ストア or Deployer 専用ストレージ（現行の `PhoneHomeResponse` 等） +で管理する想定とする。 + +--- + +### 2. Deployer / NodeAgent の責務とインターフェース + +#### 2.1 Deployer（中央）の責務 + +- `deployer-types` で定義されている `NodeInfo` / `NodeConfig` を、 + 上記 `Node` モデルと 1:1 でマッピングして Chainfire に保存する。 +- 管理 API からのリクエスト（ノード登録、ロール更新など）を受けて、 + `photoncloud/clusters/{cluster_id}/nodes/{node_id}` を更新する。 +- 「どの Node にどの Service を何インスタンス置くか」を決め、 + `ServiceInstance` エントリを Desired State として作成する。 + +#### 2.2 NodeAgent（各ノード）の責務 + +NodeAgent は各ノード上の常駐プロセスとして動作し、**宣言的な Desired State** +（Chainfire 上の ServiceInstance 等）と、実ノードの **Observed State** +（systemd プロセス・mTLS Agent・ローカルポート）を Reconcile する。 + +##### 2.2.1 NodeAgent の外部インターフェース + +- **入力** + - Chainfire: + - `nodes/{node_id}`（自ノード） + - `instances/*`（自ノードに紐づくインスタンス） + - `mtls/policies/*`（mTLS 設定） + - ローカル: + - systemd / プロセスリスト + - ローカル設定ファイル（例: `/etc/photoncloud/node-agent.toml`） +- **出力** + - Chainfire: + - `nodes/{node_id}.last_heartbeat` + - `instances/{service}/{instance_id}.state` + - `instances/{service}/{instance_id}.last_heartbeat` + - ローカル: + - systemd unit の起動/停止（アプリケーションサービス＋mTLS Agent） + - ローカル設定ファイル生成（サービス別 config, cert 配置 など） + +##### 2.2.2 Reconcile ループ（擬似コード） + +NodeAgent 内部のメインループイメージ: + +```text +loop every N seconds or on Chainfire watch event: + desired_instances = chainfire.get_instances_for_node(node_id) + observed_instances = local.inspect_processes_and_ports() + + # 1. 起動すべきインスタンス + for each instance in desired_instances: + if !observed_instances.contains(instance): + start_app_service(instance) + start_mtls_agent(instance) + + # 2. 停止すべきインスタンス + for each instance in observed_instances: + if !desired_instances.contains(instance): + stop_mtls_agent(instance) + stop_app_service(instance) + + # 3. 状態更新 + update_heartbeats_and_state_in_chainfire() +``` + +実際の実装では、Chainfire の watch 機構を使い、イベント駆動＋バックオフを行う。 + +##### 2.2.3 Deployer との関係 + +- Deployer は「どの Node にどの ServiceInstance を置くか」を決めて + Chainfire に書き込む **コントロールプレーン**。 +- NodeAgent は、それを読んでローカルマシンを Reconcile する **データプレーン**。 +- 将来的には、NodeAgent も Chainfire の `ClusterEventHandler` を実装し、 + gossip/メンバーシップと連動させることもできる。 + +#### 2.3 mTLS Agent からの利用 + +mTLS Agent は、以下の用途で Chainfire を参照する: + +- **サービス発見** + - `instances/{service_name}/` プレフィックスを Range/Watch し、 + - `state = "ready"` のインスタンス一覧をキャッシュ。 + - ローカルノード / リモートノードにまたがるインスタンスをロードバランス。 +- **ポリシー取得** + - `mtls/policies/` を Range/Watch してローカルキャッシュ。 + - 接続時に `(source_service, target_service)` から mTLS モードを決定。 +- **証明書メタ情報** + - 自身のサービス/ノードに対応する CertificateBinding を取得し、 + ローテーションのタイミングや失効を検知。 + +--- + +### 3. mTLS オン/オフ制御と環境別デフォルト + +#### 3.1 環境ごとのデフォルト + +Cluster メタデータに `environment` を持たせ、以下のポリシーを推奨とする: + +- `environment = "dev"`: + - デフォルト: `mtls_required = false` + - Chainfire 上で `MTLSPolicy` を設定すれば、特定サービス間のみ mTLS を有効化可能。 +- `environment = "stg" | "prod"`: + - デフォルト: `mtls_required = true` + - 明示的な `mode = "disabled"` ポリシーでのみ例外を許可。 + +#### 3.2 エージェント側の実装フラグ + +mTLS Agent 側では、以下の 2 層で制御する: + +- **コンパイル/起動時フラグ** + - `MTLS_FORCE_DISABLED=true` などの環境変数で完全無効化（テスト用）。 +- **Chainfire ポリシー** + - ランタイムで `MTLSPolicy` を変更することで、 + - 通常は mTLS + - 一時的に plain + を切り替え。 + +NodeAgent は Cluster/Node の `environment` と `labels` を参照し、 +開発用の「完全プライベート環境」ではデフォルトで mTLS Agent を +平文モードで起動するなどの戦略も取れる。 + +--- + +### 4. mTLS Agent（Sidecar）の役割と API + +#### 4.1 役割 + +- 各サービスの横で動作する小さなプロキシプロセス。 +- アプリケーションは常に **`127.0.0.1:`** で平文 HTTP/gRPC を喋る。 +- mTLS Agent は以下を担当する: + - 外部からの受信: `0.0.0.0:` で mTLS/平文を受け、`127.0.0.1:` にフォワード。 + - 外部への送信: Chainfire の ServiceInstance/MTLSPolicy を参照して、接続先と mTLS モードを解決し、上流へ接続。 + +#### 4.2 ローカル設定ファイル例 + +NodeAgent から生成される設定ファイル（例: `/etc/photoncloud/mtls-agent.d/{service}.toml`）: + +```toml +[service] +name = "creditservice" +app_addr = "127.0.0.1:9100" +mesh_bind_addr = "0.0.0.0:19100" + +[cluster] +cluster_id = "prod-cluster" +environment = "prod" +chainfire_endpoint = "https://chainfire.local:2379" + +[mtls] +mode = "auto" # "auto" | "mtls" | "tls" | "plain" +ca_cert_path = "/etc/nixos/secrets/ca.crt" +cert_path = "/etc/nixos/secrets/creditservice.crt" +key_path = "/etc/nixos/secrets/creditservice.key" +``` + +#### 4.3 mTLS オン/オフと動作モード + +動作モードは以下の 4 種類を想定する: + +- `mtls`: 双方向 TLS（クライアント証明書必須） +- `tls`: 片方向 TLS（サーバ証明書のみ） +- `plain`: 平文 HTTP/gRPC +- `auto`: Chainfire の `MTLSPolicy` を参照して上記 3 つから選択 + +mTLS Agent の起動引数（例）: + +```bash +mtls-agent \ + --config /etc/photoncloud/mtls-agent.d/creditservice.toml \ + --default-mode auto +``` + +- dev 環境では NodeAgent が `default-mode=plain` で起動する運用も可能。 +- stg/prod では `default-mode=mtls` とし、ポリシーで例外を作る。 + +#### 4.4 クライアント側 API（アプリから見た抽象） + +アプリケーションコード側では、`client-common` 等に薄い抽象を用意する: + +```rust +/// 論理サービス名ベースで呼び出す HTTP クライアント +pub async fn call_service( + svc: &str, + path: &str, + body: Option>, +) -> Result>, Error> { + // 実装イメージ: + // 1. ローカル mTLS Agent の「コントロールポート」(例: 127.0.0.1:19080) に + // 「svc=creditservice, path=/v1/foo」等を投げる + // 2. Agent が Chainfire を参照して適切な backend を選択 + // 3. mTLS/TLS/平文は Agent 側で判断 + unimplemented!() +} +``` + +最初の段階では、シンプルに「アプリは `http://127.0.0.1:` に対して +proxy 経由で呼び出す」形でもよい。 + +--- + +### 5. 既存サービスの移行方針（概要） + +- 既存のサーバー（`iam-server`, `creditservice-server`, など）は、 + まずは **localhost で平文待受** に統一する（必要な範囲から徐々に）。 +- Deployer/NodeAgent は、サービス起動時に mTLS Agent を隣に起動し、 + Chainfire 上の Service/ServiceInstance 情報を更新する。 +- クライアント側は、徐々に `client-common` ベースの論理サービス名呼び出しに + 置き換え、最終的に mTLS の有無をアプリから隠蔽する。 + + +--- + +### 4. 今後の実装ステップへのブレークダウン + +本仕様に基づき、今後は以下を実装していく: + +1. `chainfire-client` を用いた小さなユーティリティ（例: `photoncloud-ctl`）で、 + 上記キー空間の CRUD を行う PoC を作成。 +2. `deployer-server` から Chainfire への書き込み/読み出しコードを追加し、 + 既存の in-memory/local-file backend と並行して動かす。 +3. NodeAgent プロセス（新クレート or 既存 `plasmacloud-reconciler` 拡張）を実装し、 + 1 ノード内での ServiceInstance Reconcile ループを構築。 +4. 別クレートとして mTLS Agent の最小実装（plain proxy モード）を追加し、 + ServiceInstance モデルと連動させる。 + +これにより、Kubernetes なしのベアメタル環境であっても、 +Chainfire を中心とした「宣言的なクラスタ状態管理＋サービスメッシュ風 mTLS」 +を段階的に実現できる。 + + diff --git a/specifications/fiberlb/S2-l7-loadbalancing-spec.md b/specifications/fiberlb/S2-l7-loadbalancing-spec.md new file mode 100644 index 0000000..5955634 --- /dev/null +++ b/specifications/fiberlb/S2-l7-loadbalancing-spec.md @@ -0,0 +1,808 @@ +# T055.S2: L7 Load Balancing Design Specification + +**Author:** PeerA +**Date:** 2025-12-12 +**Status:** DRAFT + +## 1. Executive Summary + +This document specifies the L7 (HTTP/HTTPS) load balancing implementation for FiberLB. The design extends the existing L4 TCP proxy with HTTP-aware routing, TLS termination, and policy-based backend selection. + +## 2. Current State Analysis + +### 2.1 Existing L7 Type Foundation + +**File:** `fiberlb-types/src/listener.rs` + +```rust +pub enum ListenerProtocol { + Tcp, // L4 + Udp, // L4 + Http, // L7 - exists but unused + Https, // L7 - exists but unused + TerminatedHttps, // L7 - exists but unused +} + +pub struct TlsConfig { + pub certificate_id: String, + pub min_version: TlsVersion, + pub cipher_suites: Vec, +} +``` + +**File:** `fiberlb-types/src/pool.rs` + +```rust +pub enum PoolProtocol { + Tcp, // L4 + Udp, // L4 + Http, // L7 - exists but unused + Https, // L7 - exists but unused +} + +pub enum PersistenceType { + SourceIp, // L4 + Cookie, // L7 - exists but unused + AppCookie, // L7 - exists but unused +} +``` + +### 2.2 L4 DataPlane Architecture + +**File:** `fiberlb-server/src/dataplane.rs` + +Current architecture: +- TCP proxy using `tokio::net::TcpListener` +- Bidirectional copy via `tokio::io::copy` +- Round-robin backend selection (Maglev ready but not integrated) + +**Gap:** No HTTP parsing, no L7 routing rules, no TLS termination. + +## 3. L7 Architecture Design + +### 3.1 High-Level Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ FiberLB Server │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────┐│ +│ │ L7 Data Plane ││ +│ │ ││ +│ │ ┌──────────────┐ ┌─────────────────┐ ┌──────────────────────┐││ +│ │ │ TLS │ │ HTTP Router │ │ Backend Connector │││ +│ │ │ Termination │───>│ (Policy Eval) │───>│ (Connection Pool) │││ +│ │ │ (rustls) │ │ │ │ │││ +│ │ └──────────────┘ └─────────────────┘ └──────────────────────┘││ +│ │ ▲ │ │ ││ +│ │ │ ▼ ▼ ││ +│ │ ┌───────┴──────┐ ┌─────────────────┐ ┌──────────────────────┐││ +│ │ │ axum/hyper │ │ L7Policy │ │ Health Check │││ +│ │ │ HTTP Server │ │ Evaluator │ │ Integration │││ +│ │ └──────────────┘ └─────────────────┘ └──────────────────────┘││ +│ └─────────────────────────────────────────────────────────────────────┘│ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +### 3.2 Technology Selection + +| Component | Selection | Rationale | +|-----------|-----------|-----------| +| HTTP Server | `axum` | Already in workspace, familiar API | +| TLS | `rustls` via `axum-server` | Pure Rust, no OpenSSL dependency | +| HTTP Client | `hyper` | Low-level control for proxy scenarios | +| Connection Pool | `hyper-util` | Efficient backend connection reuse | + +**Alternative Considered:** Cloudflare Pingora +- Pros: High performance, battle-tested +- Cons: Heavy dependency, different paradigm, learning curve +- Decision: Start with axum/hyper, consider Pingora for v2 if perf insufficient + +## 4. New Types + +### 4.1 L7Policy + +Content-based routing policy attached to a Listener. + +```rust +// File: fiberlb-types/src/l7policy.rs + +/// Unique identifier for an L7 policy +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct L7PolicyId(Uuid); + +/// L7 routing policy +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct L7Policy { + pub id: L7PolicyId, + pub listener_id: ListenerId, + pub name: String, + + /// Evaluation order (lower = higher priority) + pub position: u32, + + /// Action to take when rules match + pub action: L7PolicyAction, + + /// Redirect URL (for RedirectToUrl action) + pub redirect_url: Option, + + /// Target pool (for RedirectToPool action) + pub redirect_pool_id: Option, + + /// HTTP status code for redirects/rejects + pub redirect_http_status_code: Option, + + pub enabled: bool, + pub created_at: u64, + pub updated_at: u64, +} + +/// Policy action when rules match +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum L7PolicyAction { + /// Route to a specific pool + RedirectToPool, + /// Return HTTP redirect to URL + RedirectToUrl, + /// Reject request with status code + Reject, +} +``` + +### 4.2 L7Rule + +Match conditions for L7Policy evaluation. + +```rust +// File: fiberlb-types/src/l7rule.rs + +/// Unique identifier for an L7 rule +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct L7RuleId(Uuid); + +/// L7 routing rule (match condition) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct L7Rule { + pub id: L7RuleId, + pub policy_id: L7PolicyId, + + /// Type of comparison + pub rule_type: L7RuleType, + + /// Comparison operator + pub compare_type: L7CompareType, + + /// Value to compare against + pub value: String, + + /// Key for header/cookie rules + pub key: Option, + + /// Invert the match result + pub invert: bool, + + pub created_at: u64, + pub updated_at: u64, +} + +/// What to match against +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum L7RuleType { + /// Match request hostname (Host header or SNI) + HostName, + /// Match request path + Path, + /// Match file extension (e.g., .jpg, .css) + FileType, + /// Match HTTP header value + Header, + /// Match cookie value + Cookie, + /// Match SSL SNI hostname + SslConnSnI, +} + +/// How to compare +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum L7CompareType { + /// Exact match + EqualTo, + /// Regex match + Regex, + /// String starts with + StartsWith, + /// String ends with + EndsWith, + /// String contains + Contains, +} +``` + +## 5. L7DataPlane Implementation + +### 5.1 Module Structure + +``` +fiberlb-server/src/ +├── dataplane.rs (L4 - existing) +├── l7_dataplane.rs (NEW - L7 HTTP proxy) +├── l7_router.rs (NEW - Policy/Rule evaluation) +├── tls.rs (NEW - TLS configuration) +└── maglev.rs (existing) +``` + +### 5.2 L7DataPlane Core + +```rust +// File: fiberlb-server/src/l7_dataplane.rs + +use axum::{Router, extract::State, http::Request, body::Body}; +use hyper_util::client::legacy::Client; +use hyper_util::rt::TokioExecutor; +use tower::ServiceExt; + +/// L7 HTTP/HTTPS Data Plane +pub struct L7DataPlane { + metadata: Arc, + router: Arc, + http_client: Client, + listeners: Arc>>, +} + +impl L7DataPlane { + pub fn new(metadata: Arc) -> Self { + let http_client = Client::builder(TokioExecutor::new()) + .pool_max_idle_per_host(32) + .build_http(); + + Self { + metadata: metadata.clone(), + router: Arc::new(L7Router::new(metadata)), + http_client, + listeners: Arc::new(RwLock::new(HashMap::new())), + } + } + + /// Start an HTTP/HTTPS listener + pub async fn start_listener(&self, listener_id: ListenerId) -> Result<()> { + let listener = self.find_listener(&listener_id).await?; + + let app = self.build_router(&listener).await?; + + let bind_addr = format!("0.0.0.0:{}", listener.port); + + match listener.protocol { + ListenerProtocol::Http => { + self.start_http_server(listener_id, &bind_addr, app).await + } + ListenerProtocol::Https | ListenerProtocol::TerminatedHttps => { + let tls_config = listener.tls_config + .ok_or(L7Error::TlsConfigMissing)?; + self.start_https_server(listener_id, &bind_addr, app, tls_config).await + } + _ => Err(L7Error::InvalidProtocol), + } + } + + /// Build axum router for a listener + async fn build_router(&self, listener: &Listener) -> Result { + let state = ProxyState { + metadata: self.metadata.clone(), + router: self.router.clone(), + http_client: self.http_client.clone(), + listener_id: listener.id, + default_pool_id: listener.default_pool_id, + }; + + Ok(Router::new() + .fallback(proxy_handler) + .with_state(state)) + } +} + +/// Proxy request handler +async fn proxy_handler( + State(state): State, + request: Request, +) -> impl IntoResponse { + // 1. Evaluate L7 policies to determine target pool + let routing_result = state.router + .evaluate(&state.listener_id, &request) + .await; + + match routing_result { + RoutingResult::Pool(pool_id) => { + proxy_to_pool(&state, pool_id, request).await + } + RoutingResult::Redirect { url, status } => { + Redirect::to(&url).into_response() + } + RoutingResult::Reject { status } => { + StatusCode::from_u16(status) + .unwrap_or(StatusCode::FORBIDDEN) + .into_response() + } + RoutingResult::Default => { + match state.default_pool_id { + Some(pool_id) => proxy_to_pool(&state, pool_id, request).await, + None => StatusCode::SERVICE_UNAVAILABLE.into_response(), + } + } + } +} +``` + +### 5.3 L7Router (Policy Evaluation) + +```rust +// File: fiberlb-server/src/l7_router.rs + +/// L7 routing engine +pub struct L7Router { + metadata: Arc, +} + +impl L7Router { + /// Evaluate policies for a request + pub async fn evaluate( + &self, + listener_id: &ListenerId, + request: &Request, + ) -> RoutingResult { + // Load policies ordered by position + let policies = self.metadata + .list_l7_policies(listener_id) + .await + .unwrap_or_default(); + + for policy in policies.iter().filter(|p| p.enabled) { + // Load rules for this policy + let rules = self.metadata + .list_l7_rules(&policy.id) + .await + .unwrap_or_default(); + + // All rules must match (AND logic) + if rules.iter().all(|rule| self.evaluate_rule(rule, request)) { + return self.apply_policy_action(policy); + } + } + + RoutingResult::Default + } + + /// Evaluate a single rule + fn evaluate_rule(&self, rule: &L7Rule, request: &Request) -> bool { + let value = match rule.rule_type { + L7RuleType::HostName => { + request.headers() + .get("host") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()) + } + L7RuleType::Path => { + Some(request.uri().path().to_string()) + } + L7RuleType::FileType => { + request.uri().path() + .rsplit('.') + .next() + .map(|s| s.to_string()) + } + L7RuleType::Header => { + rule.key.as_ref().and_then(|key| { + request.headers() + .get(key) + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()) + }) + } + L7RuleType::Cookie => { + self.extract_cookie(request, rule.key.as_deref()) + } + L7RuleType::SslConnSnI => { + // SNI extracted during TLS handshake, stored in extension + request.extensions() + .get::() + .map(|s| s.0.clone()) + } + }; + + let matched = match value { + Some(v) => self.compare(&v, &rule.value, rule.compare_type), + None => false, + }; + + if rule.invert { !matched } else { matched } + } + + fn compare(&self, value: &str, pattern: &str, compare_type: L7CompareType) -> bool { + match compare_type { + L7CompareType::EqualTo => value == pattern, + L7CompareType::StartsWith => value.starts_with(pattern), + L7CompareType::EndsWith => value.ends_with(pattern), + L7CompareType::Contains => value.contains(pattern), + L7CompareType::Regex => { + regex::Regex::new(pattern) + .map(|r| r.is_match(value)) + .unwrap_or(false) + } + } + } +} +``` + +## 6. TLS Termination + +### 6.1 Certificate Management + +```rust +// File: fiberlb-types/src/certificate.rs + +/// TLS Certificate +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Certificate { + pub id: CertificateId, + pub loadbalancer_id: LoadBalancerId, + pub name: String, + + /// PEM-encoded certificate chain + pub certificate: String, + + /// PEM-encoded private key (encrypted at rest) + pub private_key: String, + + /// Certificate type + pub cert_type: CertificateType, + + /// Expiration timestamp + pub expires_at: u64, + + pub created_at: u64, + pub updated_at: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CertificateType { + /// Standard certificate + Server, + /// CA certificate for client auth + ClientCa, + /// SNI certificate + Sni, +} +``` + +### 6.2 TLS Configuration + +```rust +// File: fiberlb-server/src/tls.rs + +use rustls::{ServerConfig, Certificate, PrivateKey}; +use rustls_pemfile::{certs, pkcs8_private_keys}; + +pub fn build_tls_config( + cert_pem: &str, + key_pem: &str, + min_version: TlsVersion, +) -> Result { + let certs = certs(&mut cert_pem.as_bytes())? + .into_iter() + .map(Certificate) + .collect(); + + let keys = pkcs8_private_keys(&mut key_pem.as_bytes())?; + let key = PrivateKey(keys.into_iter().next() + .ok_or(TlsError::NoPrivateKey)?); + + let mut config = ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_single_cert(certs, key)?; + + // Set minimum TLS version + config.versions = match min_version { + TlsVersion::Tls12 => &[&rustls::version::TLS12, &rustls::version::TLS13], + TlsVersion::Tls13 => &[&rustls::version::TLS13], + }; + + Ok(config) +} + +/// SNI-based certificate resolver for multiple domains +pub struct SniCertResolver { + certs: HashMap>, + default: Arc, +} + +impl ResolvesServerCert for SniCertResolver { + fn resolve(&self, client_hello: ClientHello) -> Option> { + let sni = client_hello.server_name()?; + self.certs.get(sni) + .or(Some(&self.default)) + .map(|config| config.cert_resolver.resolve(client_hello)) + .flatten() + } +} +``` + +## 7. Session Persistence (L7) + +### 7.1 Cookie-Based Persistence + +```rust +impl L7DataPlane { + /// Add session persistence cookie to response + fn add_persistence_cookie( + &self, + response: &mut Response, + persistence: &SessionPersistence, + backend_id: &str, + ) { + if persistence.persistence_type != PersistenceType::Cookie { + return; + } + + let cookie_name = persistence.cookie_name + .as_deref() + .unwrap_or("SERVERID"); + + let cookie_value = format!( + "{}={}; Max-Age={}; Path=/; HttpOnly", + cookie_name, + backend_id, + persistence.timeout_seconds + ); + + response.headers_mut().append( + "Set-Cookie", + HeaderValue::from_str(&cookie_value).unwrap(), + ); + } + + /// Extract backend from persistence cookie + fn get_persistent_backend( + &self, + request: &Request, + persistence: &SessionPersistence, + ) -> Option { + let cookie_name = persistence.cookie_name + .as_deref() + .unwrap_or("SERVERID"); + + request.headers() + .get("cookie") + .and_then(|v| v.to_str().ok()) + .and_then(|cookies| { + cookies.split(';') + .find_map(|c| { + let parts: Vec<_> = c.trim().splitn(2, '=').collect(); + if parts.len() == 2 && parts[0] == cookie_name { + Some(parts[1].to_string()) + } else { + None + } + }) + }) + } +} +``` + +## 8. Health Checks (L7) + +### 8.1 HTTP Health Check + +```rust +// Extend existing health check for L7 + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HttpHealthCheck { + /// HTTP method (GET, HEAD, POST) + pub method: String, + /// URL path to check + pub url_path: String, + /// Expected HTTP status codes (e.g., [200, 201, 204]) + pub expected_codes: Vec, + /// Host header to send + pub host_header: Option, +} + +impl HealthChecker { + async fn check_http_backend(&self, backend: &Backend, config: &HttpHealthCheck) -> bool { + let url = format!("http://{}:{}{}", backend.address, backend.port, config.url_path); + + let request = Request::builder() + .method(config.method.as_str()) + .uri(&url) + .header("Host", config.host_header.as_deref().unwrap_or(&backend.address)) + .body(Body::empty()) + .unwrap(); + + match self.http_client.request(request).await { + Ok(response) => { + config.expected_codes.contains(&response.status().as_u16()) + } + Err(_) => false, + } + } +} +``` + +## 9. Integration Points + +### 9.1 Server Integration + +```rust +// File: fiberlb-server/src/server.rs + +impl FiberLBServer { + pub async fn run(&self) -> Result<()> { + let l4_dataplane = DataPlane::new(self.metadata.clone()); + let l7_dataplane = L7DataPlane::new(self.metadata.clone()); + + // Watch for listener changes + tokio::spawn(async move { + // Start L4 listeners (TCP/UDP) + // Start L7 listeners (HTTP/HTTPS) + }); + + // Run gRPC control plane + // ... + } +} +``` + +### 9.2 gRPC API Extensions + +```protobuf +// Additions to fiberlb.proto + +message L7Policy { + string id = 1; + string listener_id = 2; + string name = 3; + uint32 position = 4; + L7PolicyAction action = 5; + optional string redirect_url = 6; + optional string redirect_pool_id = 7; + optional uint32 redirect_http_status_code = 8; + bool enabled = 9; +} + +message L7Rule { + string id = 1; + string policy_id = 2; + L7RuleType rule_type = 3; + L7CompareType compare_type = 4; + string value = 5; + optional string key = 6; + bool invert = 7; +} + +service FiberLBService { + // Existing methods... + + // L7 Policy management + rpc CreateL7Policy(CreateL7PolicyRequest) returns (CreateL7PolicyResponse); + rpc GetL7Policy(GetL7PolicyRequest) returns (GetL7PolicyResponse); + rpc ListL7Policies(ListL7PoliciesRequest) returns (ListL7PoliciesResponse); + rpc UpdateL7Policy(UpdateL7PolicyRequest) returns (UpdateL7PolicyResponse); + rpc DeleteL7Policy(DeleteL7PolicyRequest) returns (DeleteL7PolicyResponse); + + // L7 Rule management + rpc CreateL7Rule(CreateL7RuleRequest) returns (CreateL7RuleResponse); + rpc GetL7Rule(GetL7RuleRequest) returns (GetL7RuleResponse); + rpc ListL7Rules(ListL7RulesRequest) returns (ListL7RulesResponse); + rpc UpdateL7Rule(UpdateL7RuleRequest) returns (UpdateL7RuleResponse); + rpc DeleteL7Rule(DeleteL7RuleRequest) returns (DeleteL7RuleResponse); + + // Certificate management + rpc CreateCertificate(CreateCertificateRequest) returns (CreateCertificateResponse); + rpc GetCertificate(GetCertificateRequest) returns (GetCertificateResponse); + rpc ListCertificates(ListCertificatesRequest) returns (ListCertificatesResponse); + rpc DeleteCertificate(DeleteCertificateRequest) returns (DeleteCertificateResponse); +} +``` + +## 10. Implementation Plan + +### Phase 1: Types & Storage (Day 1) +1. Add `L7Policy`, `L7Rule`, `Certificate` types to fiberlb-types +2. Add protobuf definitions +3. Implement metadata storage for L7 policies + +### Phase 2: L7DataPlane (Day 1-2) +1. Create `l7_dataplane.rs` with axum-based HTTP server +2. Implement basic HTTP proxy (no routing) +3. Add connection pooling to backends + +### Phase 3: TLS Termination (Day 2) +1. Implement TLS configuration building +2. Add SNI-based certificate selection +3. HTTPS listener support + +### Phase 4: L7 Routing (Day 2-3) +1. Implement `L7Router` policy evaluation +2. Add all rule types (Host, Path, Header, Cookie) +3. Cookie-based session persistence + +### Phase 5: API & Integration (Day 3) +1. gRPC API for L7Policy/L7Rule CRUD +2. REST API endpoints +3. Integration with control plane + +## 11. Configuration Example + +```yaml +# Example: Route /api/* to api-pool, /static/* to cdn-pool +listeners: + - name: https-frontend + port: 443 + protocol: https + tls_config: + certificate_id: cert-main + min_version: tls12 + default_pool_id: default-pool + +l7_policies: + - name: api-routing + listener_id: https-frontend + position: 10 + action: redirect_to_pool + redirect_pool_id: api-pool + rules: + - rule_type: path + compare_type: starts_with + value: "/api/" + + - name: static-routing + listener_id: https-frontend + position: 20 + action: redirect_to_pool + redirect_pool_id: cdn-pool + rules: + - rule_type: path + compare_type: regex + value: "\\.(js|css|png|jpg|svg)$" +``` + +## 12. Dependencies + +Add to `fiberlb-server/Cargo.toml`: + +```toml +[dependencies] +# HTTP/TLS +axum = { version = "0.8", features = ["http2"] } +axum-server = { version = "0.7", features = ["tls-rustls"] } +hyper = { version = "1.0", features = ["full"] } +hyper-util = { version = "0.1", features = ["client", "client-legacy", "http1", "http2"] } +rustls = "0.23" +rustls-pemfile = "2.0" +tokio-rustls = "0.26" + +# Routing +regex = "1.10" +``` + +## 13. Decision Summary + +| Aspect | Decision | Rationale | +|--------|----------|-----------| +| HTTP Framework | axum | Consistent with other services, familiar API | +| TLS Library | rustls | Pure Rust, no OpenSSL complexity | +| L7 Routing | Policy/Rule model | OpenStack Octavia-compatible, flexible | +| Certificate Storage | ChainFire | Consistent with metadata, encrypted at rest | +| Session Persistence | Cookie-based | Standard approach for L7 | + +## 14. References + +- [OpenStack Octavia L7 Policies](https://docs.openstack.org/octavia/latest/user/guides/l7.html) +- [AWS ALB Listener Rules](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/listener-update-rules.html) +- [axum Documentation](https://docs.rs/axum/latest/axum/) +- [rustls Documentation](https://docs.rs/rustls/latest/rustls/) diff --git a/specifications/fiberlb/S3-bgp-integration-spec.md b/specifications/fiberlb/S3-bgp-integration-spec.md new file mode 100644 index 0000000..3aea9d2 --- /dev/null +++ b/specifications/fiberlb/S3-bgp-integration-spec.md @@ -0,0 +1,369 @@ +# T055.S3: BGP Integration Strategy Specification + +**Author:** PeerA +**Date:** 2025-12-12 +**Status:** DRAFT + +## 1. Executive Summary + +This document specifies the BGP Anycast integration strategy for FiberLB to enable VIP (Virtual IP) advertisement to upstream routers. The recommended approach is a **sidecar pattern** using GoBGP with gRPC API integration. + +## 2. Background + +### 2.1 Current State +- FiberLB binds listeners to `0.0.0.0:{port}` on each node +- LoadBalancer resources have `vip_address` field (currently unused for routing) +- No mechanism exists to advertise VIPs to physical network infrastructure + +### 2.2 Requirements (from PROJECT.md Item 7) +- "BGP AnycastによるL2ロードバランシング" (BGP Anycast L2 LB) +- VIPs must be reachable from external networks +- Support for ECMP (Equal-Cost Multi-Path) across multiple FiberLB nodes +- Graceful withdrawal when load balancer is unhealthy/deleted + +## 3. BGP Library Options Analysis + +### 3.1 Option A: GoBGP Sidecar (RECOMMENDED) + +**Description:** Run GoBGP as a sidecar container/process, control via gRPC API + +| Aspect | Details | +|--------|---------| +| Language | Go | +| Maturity | Production-grade, widely deployed | +| API | gRPC with well-documented protobuf | +| Integration | FiberLB calls GoBGP gRPC to add/withdraw routes | +| Deployment | Separate process, co-located with FiberLB | + +**Pros:** +- Battle-tested in production (Google, LINE, Yahoo Japan) +- Extensive BGP feature support (ECMP, BFD, RPKI) +- Clear separation of concerns +- Minimal code changes to FiberLB + +**Cons:** +- External dependency (Go binary) +- Additional process management +- Network overhead for gRPC calls (minimal) + +### 3.2 Option B: RustyBGP Sidecar + +**Description:** Same sidecar pattern but using RustyBGP daemon + +| Aspect | Details | +|--------|---------| +| Language | Rust | +| Maturity | Active development, less production deployment | +| API | GoBGP-compatible gRPC | +| Performance | Higher than GoBGP (multicore optimized) | + +**Pros:** +- Rust ecosystem alignment +- Drop-in replacement for GoBGP (same API) +- Better performance in benchmarks + +**Cons:** +- Less production history +- Smaller community + +### 3.3 Option C: Embedded zettabgp + +**Description:** Build custom BGP speaker using zettabgp library + +| Aspect | Details | +|--------|---------| +| Language | Rust | +| Type | Parsing/composing library only | +| Integration | Embedded directly in FiberLB | + +**Pros:** +- No external dependencies +- Full control over BGP behavior +- Single binary deployment + +**Cons:** +- Significant implementation effort (FSM, timers, peer state) +- Risk of BGP protocol bugs +- Months of additional development + +### 3.4 Option D: OVN Gateway Integration + +**Description:** Leverage OVN's built-in BGP capabilities via OVN gateway router + +| Aspect | Details | +|--------|---------| +| Dependency | Requires OVN deployment | +| Integration | FiberLB configures OVN via OVSDB | + +**Pros:** +- No additional BGP daemon +- Integrated with SDN layer + +**Cons:** +- Tightly couples to OVN +- Limited BGP feature set +- May not be deployed in all environments + +## 4. Recommended Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ FiberLB Node │ +│ │ +│ ┌──────────────────┐ ┌──────────────────┐ │ +│ │ │ gRPC │ │ │ +│ │ FiberLB │───────>│ GoBGP │──── BGP ──│──> ToR Router +│ │ Server │ │ Daemon │ │ +│ │ │ │ │ │ +│ └──────────────────┘ └──────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌──────────────────┐ │ +│ │ VIP Traffic │ │ +│ │ (Data Plane) │ │ +│ └──────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +### 4.1 Components + +1. **FiberLB Server** - Existing service, adds BGP client module +2. **GoBGP Daemon** - BGP speaker process, controlled via gRPC +3. **BGP Client Module** - New Rust module using `gobgp-client` crate or raw gRPC + +### 4.2 Communication Flow + +1. LoadBalancer created with VIP address +2. FiberLB checks backend health +3. When healthy backends exist → `AddPath(VIP/32)` +4. When all backends fail → `DeletePath(VIP/32)` +5. LoadBalancer deleted → `DeletePath(VIP/32)` + +## 5. Implementation Design + +### 5.1 New Module: `fiberlb-bgp` + +```rust +// fiberlb/crates/fiberlb-bgp/src/lib.rs + +pub struct BgpManager { + client: GobgpClient, + config: BgpConfig, + advertised_vips: HashSet, +} + +impl BgpManager { + /// Advertise a VIP to BGP peers + pub async fn advertise_vip(&mut self, vip: IpAddr) -> Result<()>; + + /// Withdraw a VIP from BGP peers + pub async fn withdraw_vip(&mut self, vip: IpAddr) -> Result<()>; + + /// Check if VIP is currently advertised + pub fn is_advertised(&self, vip: &IpAddr) -> bool; +} +``` + +### 5.2 Configuration Schema + +```yaml +# fiberlb-server config +bgp: + enabled: true + gobgp_address: "127.0.0.1:50051" # GoBGP gRPC address + local_as: 65001 + router_id: "10.0.0.1" + neighbors: + - address: "10.0.0.254" + remote_as: 65000 + description: "ToR Router" +``` + +### 5.3 GoBGP Configuration (sidecar) + +```yaml +# /etc/gobgp/gobgp.yaml +global: + config: + as: 65001 + router-id: 10.0.0.1 + port: 179 + +neighbors: + - config: + neighbor-address: 10.0.0.254 + peer-as: 65000 + afi-safis: + - config: + afi-safi-name: ipv4-unicast + add-paths: + config: + send-max: 8 +``` + +### 5.4 Integration Points in FiberLB + +```rust +// In loadbalancer_service.rs + +impl LoadBalancerService { + async fn on_loadbalancer_active(&self, lb: &LoadBalancer) { + if let Some(vip) = &lb.vip_address { + if let Some(bgp) = &self.bgp_manager { + bgp.advertise_vip(vip.parse()?).await?; + } + } + } + + async fn on_loadbalancer_deleted(&self, lb: &LoadBalancer) { + if let Some(vip) = &lb.vip_address { + if let Some(bgp) = &self.bgp_manager { + bgp.withdraw_vip(vip.parse()?).await?; + } + } + } +} +``` + +## 6. Deployment Patterns + +### 6.1 NixOS Module + +```nix +# modules/fiberlb-bgp.nix +{ config, lib, pkgs, ... }: + +{ + services.fiberlb = { + bgp = { + enable = true; + localAs = 65001; + routerId = "10.0.0.1"; + neighbors = [ + { address = "10.0.0.254"; remoteAs = 65000; } + ]; + }; + }; + + # GoBGP sidecar + services.gobgpd = { + enable = true; + config = fiberlb-bgp-config; + }; +} +``` + +### 6.2 Container/Pod Deployment + +```yaml +# kubernetes deployment with sidecar +spec: + containers: + - name: fiberlb + image: plasmacloud/fiberlb:latest + env: + - name: BGP_GOBGP_ADDRESS + value: "localhost:50051" + + - name: gobgp + image: osrg/gobgp:latest + args: ["-f", "/etc/gobgp/config.yaml"] + ports: + - containerPort: 179 # BGP + - containerPort: 50051 # gRPC +``` + +## 7. Health-Based VIP Withdrawal + +### 7.1 Logic + +``` +┌─────────────────────────────────────────┐ +│ Health Check Loop │ +│ │ +│ FOR each LoadBalancer WITH vip_address │ +│ healthy_backends = count_healthy() │ +│ │ +│ IF healthy_backends > 0 │ +│ AND NOT advertised(vip) │ +│ THEN │ +│ advertise(vip) │ +│ │ +│ IF healthy_backends == 0 │ +│ AND advertised(vip) │ +│ THEN │ +│ withdraw(vip) │ +│ │ +└─────────────────────────────────────────┘ +``` + +### 7.2 Graceful Shutdown + +1. SIGTERM received +2. Withdraw all VIPs (allow BGP convergence) +3. Wait for configurable grace period (default: 5s) +4. Shutdown data plane + +## 8. ECMP Support + +With multiple FiberLB nodes advertising the same VIP: + +``` + ┌─────────────┐ + │ ToR Router │ + │ (AS 65000) │ + └──────┬──────┘ + │ ECMP + ┌──────────┼──────────┐ + ▼ ▼ ▼ + ┌─────────┐ ┌─────────┐ ┌─────────┐ + │FiberLB-1│ │FiberLB-2│ │FiberLB-3│ + │ VIP: X │ │ VIP: X │ │ VIP: X │ + │AS 65001 │ │AS 65001 │ │AS 65001 │ + └─────────┘ └─────────┘ └─────────┘ +``` + +- All nodes advertise same VIP with same attributes +- Router distributes traffic via ECMP hashing +- Node failure = route withdrawal = automatic failover + +## 9. Future Enhancements + +1. **BFD (Bidirectional Forwarding Detection)** - Faster failure detection +2. **BGP Communities** - Traffic engineering support +3. **Route Filtering** - Export policies per neighbor +4. **RustyBGP Migration** - Switch from GoBGP for performance +5. **Embedded Speaker** - Long-term: native Rust BGP using zettabgp + +## 10. Implementation Phases + +### Phase 1: Basic Integration +- GoBGP sidecar deployment +- Simple VIP advertise/withdraw API +- Manual configuration + +### Phase 2: Health-Based Control +- Automatic VIP withdrawal on backend failure +- Graceful shutdown handling + +### Phase 3: Production Hardening +- BFD support +- Metrics and observability +- Operator documentation + +## 11. References + +- [GoBGP](https://osrg.github.io/gobgp/) - Official documentation +- [RustyBGP](https://github.com/osrg/rustybgp) - Rust BGP daemon +- [zettabgp](https://github.com/wladwm/zettabgp) - Rust BGP library +- [kube-vip BGP Mode](https://kube-vip.io/docs/modes/bgp/) - Similar pattern +- [MetalLB BGP](https://metallb.io/concepts/bgp/) - Kubernetes LB BGP + +## 12. Decision Summary + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Integration Pattern | Sidecar | Clear separation, proven pattern | +| BGP Daemon | GoBGP | Production maturity, extensive features | +| API | gRPC | Native GoBGP interface, language-agnostic | +| Future Path | RustyBGP | Same API, better performance when stable | diff --git a/specifications/flaredb/001-distributed-core/checklists/requirements.md b/specifications/flaredb/001-distributed-core/checklists/requirements.md new file mode 100644 index 0000000..7edb6d5 --- /dev/null +++ b/specifications/flaredb/001-distributed-core/checklists/requirements.md @@ -0,0 +1,34 @@ +# Specification Quality Checklist: Core Distributed Architecture (Phase 1) + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2025-11-30 +**Feature**: specs/001-distributed-core/spec.md + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) - *Exception: Specific Rust/RocksDB constraints are part of the user request/architecture definition.* +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders - *Target audience is database developers.* +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic - *Allowed tech-specifics due to nature of task.* +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified - *Implicit in CAS failure scenarios.* +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification - *See above exception.* + +## Notes + +- The specification heavily references technical components (RocksDB, Cargo, gRPC) because the "Feature" is literally "Implement the Core Architecture". This is acceptable for this specific foundational task. diff --git a/specifications/flaredb/001-distributed-core/contracts/kvrpc.proto b/specifications/flaredb/001-distributed-core/contracts/kvrpc.proto new file mode 100644 index 0000000..408a08e --- /dev/null +++ b/specifications/flaredb/001-distributed-core/contracts/kvrpc.proto @@ -0,0 +1,55 @@ +syntax = "proto3"; + +package kvrpc; + +// Raw (Eventual Consistency) Operations +service KvRaw { + rpc RawPut(RawPutRequest) returns (RawPutResponse); + rpc RawGet(RawGetRequest) returns (RawGetResponse); +} + +message RawPutRequest { + bytes key = 1; + bytes value = 2; +} + +message RawPutResponse { + bool success = 1; +} + +message RawGetRequest { + bytes key = 1; +} + +message RawGetResponse { + bool found = 1; + bytes value = 2; +} + +// CAS (Strong Consistency / Optimistic) Operations +service KvCas { + rpc CompareAndSwap(CasRequest) returns (CasResponse); + rpc Get(GetRequest) returns (GetResponse); +} + +message CasRequest { + bytes key = 1; + bytes value = 2; + uint64 expected_version = 3; // 0 implies "create if not exists" +} + +message CasResponse { + bool success = 1; + uint64 current_version = 2; // Returns current version on failure (for retry) + uint64 new_version = 3; // Returns assigned version on success +} + +message GetRequest { + bytes key = 1; +} + +message GetResponse { + bool found = 1; + bytes value = 2; + uint64 version = 3; +} diff --git a/specifications/flaredb/001-distributed-core/contracts/pdpb.proto b/specifications/flaredb/001-distributed-core/contracts/pdpb.proto new file mode 100644 index 0000000..fbed2a2 --- /dev/null +++ b/specifications/flaredb/001-distributed-core/contracts/pdpb.proto @@ -0,0 +1,56 @@ +syntax = "proto3"; + +package pdpb; + +// TSO Service +service Tso { + rpc GetTimestamp(TsoRequest) returns (TsoResponse); +} + +message TsoRequest { + uint32 count = 1; +} + +message TsoResponse { + uint64 timestamp = 1; // Physical << 16 | Logical + uint32 count = 2; +} + +// Cluster Management Service +service Pd { + // Store Registration + rpc RegisterStore(RegisterStoreRequest) returns (RegisterStoreResponse); + + // Region Discovery + rpc GetRegion(GetRegionRequest) returns (GetRegionResponse); +} + +message RegisterStoreRequest { + string addr = 1; // e.g., "127.0.0.1:50051" +} + +message RegisterStoreResponse { + uint64 store_id = 1; + uint64 cluster_id = 2; // Verify cluster match +} + +message GetRegionRequest { + bytes key = 1; +} + +message GetRegionResponse { + Region region = 1; + Store leader = 2; +} + +message Region { + uint64 id = 1; + bytes start_key = 2; + bytes end_key = 3; // empty = infinity + // In future: repeated Peer peers = 4; +} + +message Store { + uint64 id = 1; + string addr = 2; +} diff --git a/specifications/flaredb/001-distributed-core/data-model.md b/specifications/flaredb/001-distributed-core/data-model.md new file mode 100644 index 0000000..0386dbd --- /dev/null +++ b/specifications/flaredb/001-distributed-core/data-model.md @@ -0,0 +1,52 @@ +# Data Model: Core Distributed Architecture (Phase 1) + +## Entities + +### 1. Key-Value Pair (Raw) +- **Key**: `Vec` (Arbitrary bytes) +- **Value**: `Vec` (Arbitrary bytes) +- **Scope**: `rdb-storage` (Raw Put) + +### 2. Key-Value Pair (Versioned / CAS) +- **Key**: `Vec` +- **Value**: `Vec` (Metadata + Payload) +- **Version**: `u64` (Monotonic sequence) +- **Scope**: `rdb-storage` (CAS) + +### 3. TSO Timestamp +- **Physical**: `u64` (48 bits, milliseconds) +- **Logical**: `u64` (16 bits, counter) +- **Combined**: `u64` (Physical << 16 | Logical) +- **Scope**: `rdb-pd` + +## State Transitions (CAS) + +1. **Empty -> Created**: + - Current Version: 0 (or None) + - Expected Version: 0 + - New Version: TSO / Sequence > 0 + - Result: Success + +2. **Updated -> Updated**: + - Current Version: N + - Expected Version: N + - New Version: M (M > N) + - Result: Success + +3. **Conflict**: + - Current Version: N + - Expected Version: M (M != N) + - Result: Failure (Returns N) + +## Storage Schema (RocksDB Column Families) + +1. **default** (`CF_DEFAULT`): + - Stores data for Raw Puts. + - Key: `Key` + - Value: `Value` + +2. **cas** (`CF_CAS` - *Proposed name for CAS data separation*): + - Stores versioned data. + - Key: `Key` + - Value: `[Version: 8 bytes][Data...]` + - *Note: Storing version in value simplifies atomic update via Read-Modify-Write or MergeOperator.* diff --git a/specifications/flaredb/001-distributed-core/plan.md b/specifications/flaredb/001-distributed-core/plan.md new file mode 100644 index 0000000..e476221 --- /dev/null +++ b/specifications/flaredb/001-distributed-core/plan.md @@ -0,0 +1,95 @@ +# Implementation Plan: Core Distributed Architecture (Phase 1) + +**Branch**: `001-distributed-core` | **Date**: 2025-11-30 | **Spec**: [specs/001-distributed-core/spec.md](specs/001-distributed-core/spec.md) +**Input**: Feature specification from `/specs/001-distributed-core/spec.md` + +**Note**: This template is filled in by the `/speckit.plan` command. See `.specify/templates/commands/plan.md` for the execution workflow. + +## Summary + +Implement the foundational architecture for FlareDB, a distributed key-value store with CAS support. This includes setting up a Rust Cargo Workspace with 5 crates (`rdb-proto`, `rdb-storage`, `rdb-server`, `rdb-pd`, `rdb-client`), defining gRPC interfaces, implementing a RocksDB-based local storage engine, and verifying basic client-server interaction. + +## Technical Context + +**Language/Version**: Rust (Latest Stable) +**Primary Dependencies**: +- `tonic` (gRPC) +- `prost` (Protobuf) +- `rocksdb` (Storage Engine) +- `tokio` (Async Runtime) +- `clap` (CLI) +**Storage**: RocksDB (embedded via crate) +**Testing**: `cargo test` (Unit), `cargo nextest` (Optional), Custom Integration Scripts +**Target Platform**: Linux (x86_64), managed via Nix Flake +**Project Type**: Rust Cargo Workspace (Monorepo) with Nix environment +**Performance Goals**: Low-latency CAS operations (local storage baseline) +**Constraints**: Single-node verification for Phase 1, but architecture must support distributed extension. +**Scale/Scope**: 5 crates, ~2000 LOC estimate. + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +- **I. Reliability & Testing**: + - Plan includes unit tests for `rdb-storage` (SC-002). + - Plan includes integration verification (SC-003). + - Compliant. +- **II. Agility & Evolution**: + - Architecture uses standard crates (`tonic`, `rocksdb`) to avoid reinventing wheels. + - Monorepo structure allows easy refactoring across crates. + - Compliant. +- **III. Simplicity & Readability**: + - Separation of concerns: Proto vs Storage vs Server vs PD vs Client. + - Clear interfaces defined in `rdb-proto`. + - Compliant. + +## Project Structure + +### Documentation (this feature) + +```text +specs/001-distributed-core/ +├── plan.md # This file (/speckit.plan command output) +├── research.md # Phase 0 output (/speckit.plan command) +├── data-model.md # Phase 1 output (/speckit.plan command) +├── quickstart.md # Phase 1 output (/speckit.plan command) +├── contracts/ # Phase 1 output (/speckit.plan command) +└── tasks.md # Phase 2 output (/speckit.tasks command - NOT created by /speckit.plan) +``` + +### Source Code (repository root) + +```text +flake.nix # Nix development environment definition +flake.lock # Lockfile for Nix dependencies +Cargo.toml # Workspace definition +rdb-proto/ +├── Cargo.toml +├── build.rs +└── src/ # Generated protos +rdb-storage/ +├── Cargo.toml +└── src/ # RocksDB wrapper, CAS logic +rdb-server/ +├── Cargo.toml +└── src/ # gRPC Server, Handlers +rdb-pd/ +├── Cargo.toml +└── src/ # Placement Driver (TSO) +rdb-client/ +├── Cargo.toml +└── src/ # Smart SDK +rdb-cli/ # (Optional for Phase 1, but good to have) +├── Cargo.toml +└── src/ +``` + +**Structure Decision**: Standard Rust Workspace layout to ensure modularity and separation of concerns as per the architecture design. + +## Complexity Tracking + +> **Fill ONLY if Constitution Check has violations that must be justified** + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|-------------------------------------| +| N/A | | | diff --git a/specifications/flaredb/001-distributed-core/quickstart.md b/specifications/flaredb/001-distributed-core/quickstart.md new file mode 100644 index 0000000..20152a6 --- /dev/null +++ b/specifications/flaredb/001-distributed-core/quickstart.md @@ -0,0 +1,64 @@ +# Quickstart Verification Guide: Core Distributed Architecture + +This guide verifies the core components (PD, Server, Client) and storage engine behavior. + +## Prerequisites + +- Rust Toolchain (`rustc`, `cargo`) +- `protoc` (Protocol Buffers compiler) +- CMake (for building RocksDB) + +## 1. Build Workspace + +```bash +cargo build +``` + +## 2. Run Integration Test + +This feature includes a comprehensive integration test script. + +```bash +# Run the custom verification script (to be implemented in tasks) +# ./scripts/verify-core.sh +``` + +## 3. Manual Verification Steps + +### A. Start PD (Placement Driver) + +```bash +cargo run --bin rdb-pd +# Should listen on default port (e.g., 2379) +``` + +### B. Start Server (Storage Node) + +```bash +cargo run --bin rdb-server -- --pd-addr 127.0.0.1:2379 +# Should listen on default port (e.g., 50051) +``` + +### C. Run Client Operations + +```bash +# Get TSO +cargo run --bin rdb-client -- tso +# Output: Timestamp: 1735689... + +# Raw Put +cargo run --bin rdb-client -- raw-put --key foo --value bar +# Output: Success + +# Raw Get +cargo run --bin rdb-client -- raw-get --key foo +# Output: bar + +# CAS (Create) +cargo run --bin rdb-client -- cas --key meta1 --value "{json}" --expected 0 +# Output: Success, Version: 1735689... + +# CAS (Conflict) +cargo run --bin rdb-client -- cas --key meta1 --value "{new}" --expected 0 +# Output: Conflict! Current Version: 1735689... +``` diff --git a/specifications/flaredb/001-distributed-core/research.md b/specifications/flaredb/001-distributed-core/research.md new file mode 100644 index 0000000..824debe --- /dev/null +++ b/specifications/flaredb/001-distributed-core/research.md @@ -0,0 +1,19 @@ +# Research: Core Distributed Architecture (Phase 1) + +**Decision**: Use `rocksdb` crate for local storage engine. +**Rationale**: Industry standard for LSM-tree storage. Provides necessary primitives (WriteBatch, Column Families) for building a KV engine. `tikv/rust-rocksdb` is the most mature binding. +**Alternatives considered**: `sled` (pure Rust, but less mature/performant for this scale), `mdbx` (B-tree, read-optimized, not suitable for high write throughput target). + +**Decision**: Use `tonic` + `prost` for gRPC. +**Rationale**: De facto standard in Rust ecosystem. Async-first, integrates perfectly with `tokio`. +**Alternatives considered**: `grpc-rs` (C-core wrapper, complex build), `tarpc` (Rust-specific, less interoperable). + +**Decision**: Use `tokio` as async runtime. +**Rationale**: Required by `tonic`. Most mature ecosystem. + +**Decision**: Monorepo Workspace Structure. +**Rationale**: Allows atomic commits across protocol, server, and client. Simplifies dependency management during rapid early development (Agility Principle). + +## Clarification Resolution + +*No [NEEDS CLARIFICATION] items were present in the spec. Technical context was sufficiently defined in the chat history.* diff --git a/specifications/flaredb/001-distributed-core/spec.md b/specifications/flaredb/001-distributed-core/spec.md new file mode 100644 index 0000000..a1faf95 --- /dev/null +++ b/specifications/flaredb/001-distributed-core/spec.md @@ -0,0 +1,87 @@ +# Feature Specification: Core Distributed Architecture (Phase 1) + +**Feature Branch**: `001-distributed-core` +**Created**: 2025-11-30 +**Status**: Draft +**Input**: User description: "Implement the core architecture of FlareDB based on the design in chat.md..." + +## User Scenarios & Testing *(mandatory)* + + + +### User Story 1 - Core Storage Engine Verification (Priority: P1) + +As a database developer, I need a robust local storage engine that supports both CAS (Compare-And-Swap) and Raw writes, so that I can build distributed logic on top of it. + +**Why this priority**: This is the fundamental layer. Without a working storage engine with correct CAS logic, upper layers cannot function. + +**Independent Test**: Write a Rust unit test using `rdb-storage` that: +1. Creates a DB instance. +2. Performs a `raw_put`. +3. Performs a `compare_and_swap` that succeeds. +4. Performs a `compare_and_swap` that fails due to version mismatch. + +**Acceptance Scenarios**: + +1. **Given** an empty DB, **When** I `raw_put` key="k1", val="v1", **Then** `get` returns "v1". +2. **Given** key="k1" with version 0 (non-existent), **When** I `cas` with expected=0, **Then** write succeeds and version increments. +3. **Given** key="k1" with version 10, **When** I `cas` with expected=5, **Then** it returns a Conflict error with current version 10. + +--- + +### User Story 2 - Basic RPC Transport (Priority: P1) + +As a client developer, I want to connect to the server via gRPC and perform basic operations, so that I can verify the communication pipeline. + +**Why this priority**: Validates the network layer (`rdb-proto`, `tonic` integration) and the basic server shell. + +**Independent Test**: Start `rdb-server` and run a minimal `rdb-client` script that connects and sends a request. + +**Acceptance Scenarios**: + +1. **Given** a running `rdb-server`, **When** `rdb-client` sends a `GetTsoRequest` to PD (mocked or real), **Then** it receives a valid timestamp. +2. **Given** a running `rdb-server`, **When** `rdb-client` sends a `RawPutRequest`, **Then** the server accepts it and it persists to disk. + +--- + +### User Story 3 - Placement Driver TSO (Priority: P2) + +As a system, I need a source of monotonic timestamps (TSO) from `rdb-pd`, so that I can order transactions in the future. + +**Why this priority**: Essential for the "Smart Client" architecture and future MVCC/CAS logic. + +**Independent Test**: Run `rdb-pd` and hammer it with TSO requests from multiple threads. + +**Acceptance Scenarios**: + +1. **Given** a running `rdb-pd`, **When** I request timestamps repeatedly, **Then** each returned timestamp is strictly greater than the previous one. + +--- + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The project MUST be organized as a Cargo Workspace with members: `rdb-proto`, `rdb-storage`, `rdb-server`, `rdb-pd`, `rdb-client`. +- **FR-002**: `rdb-proto` MUST define gRPC services (`kvrpc.proto`, `pdpb.proto`) covering CAS, Raw Put, and TSO operations. +- **FR-003**: `rdb-storage` MUST wrap RocksDB and expose `compare_and_swap(key, expected_ver, new_val)` and `put_raw(key, val)`. +- **FR-004**: `rdb-storage` MUST store metadata (version) and data efficiently using Column Families: `default` (raw), `cas` (value as `[u64_be version][bytes value]`), and `raft_log`/`raft_state` for Raft metadata. +- **FR-005**: `rdb-pd` MUST implement a TSO (Timestamp Oracle) service providing unique, monotonic `u64` timestamps. +- **FR-006**: `rdb-server` MUST implement the gRPC handlers defined in `rdb-proto` and delegate to `rdb-storage`. +- **FR-007**: `rdb-client` MUST provide a Rust API that abstracts the gRPC calls for `cas_put`, `raw_put`, and `get`. + +### Key Entities + +- **Region**: A logical range of keys (for future sharding). +- **Version**: A `u64` representing the modification timestamp/sequence of a key. +- **TSO**: Global Timestamp Oracle. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: Full workspace compiles with `cargo build`. +- **SC-002**: `rdb-storage` unit tests pass covering CAS success/failure paths. +- **SC-003**: Integration script (`scripts/verify-core.sh`) or equivalent CI step runs end-to-end: start PD and Server, client obtains TSO, performs RawPut and RawGet (value must match), performs CAS success and CAS conflict, exits 0. diff --git a/specifications/flaredb/001-distributed-core/tasks.md b/specifications/flaredb/001-distributed-core/tasks.md new file mode 100644 index 0000000..1b35f52 --- /dev/null +++ b/specifications/flaredb/001-distributed-core/tasks.md @@ -0,0 +1,220 @@ +--- +description: "Task list template for feature implementation" +--- + +# Tasks: Core Distributed Architecture (Phase 1) + +**Input**: Design documents from `/specs/001-distributed-core/` +**Prerequisites**: plan.md (required), spec.md (required for user stories), research.md, data-model.md, contracts/ + +**Tests**: The examples below include test tasks. Tests are STANDARD per the Constitution (Principle I). Include them for all functional logic unless explicitly skipped. + +**Organization**: Tasks are grouped by user story to enable independent implementation and testing of each story. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (e.g., US1, US2, US3) +- Include exact file paths in descriptions + +## Path Conventions + +- **Single project**: `src/`, `tests/` at repository root +- **Web app**: `backend/src/`, `frontend/src/` +- **Mobile**: `api/src/`, `ios/src/` or `android/src/` +- Paths shown below assume single project - adjust based on plan.md structure + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Project initialization and basic structure with Nix environment + +- [X] T000 Create `flake.nix` to provide rust, protobuf, clang, and rocksdb dependencies +- [X] T001 Create Cargo workspace in `Cargo.toml` with 5 crates: `rdb-proto`, `rdb-storage`, `rdb-server`, `rdb-pd`, `rdb-client`, `rdb-cli` +- [X] T002 Initialize `rdb-proto` crate with `tonic-build` and `prost` dependencies in `rdb-proto/Cargo.toml` +- [X] T003 [P] Initialize `rdb-storage` crate with `rocksdb` dependency in `rdb-storage/Cargo.toml` +- [X] T004 [P] Initialize `rdb-server`, `rdb-pd`, `rdb-client` crates with `tokio` and `tonic` dependencies + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Core infrastructure that MUST be complete before ANY user story can be implemented + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete + +- [X] T005 Create `kvrpc.proto` in `rdb-proto/src/kvrpc.proto` per contract definition +- [X] T006 Create `pdpb.proto` in `rdb-proto/src/pdpb.proto` per contract definition +- [X] T007 Implement `build.rs` in `rdb-proto/build.rs` to compile protos +- [X] T008 Export generated protos in `rdb-proto/src/lib.rs` + +**Checkpoint**: Foundation ready - user story implementation can now begin in parallel + +--- + +## Phase 3: User Story 1 - Core Storage Engine Verification (Priority: P1) 🎯 MVP + +**Goal**: A robust local storage engine (RocksDB wrapper) with correct CAS logic. + +**Independent Test**: Run unit tests in `rdb-storage` covering Raw Put and CAS success/conflict scenarios. + +### Tests for User Story 1 (STANDARD - per constitution) ⚠️ + +> **NOTE**: Write these tests FIRST, ensure they FAIL before implementation + +- [X] T009 [US1] Create unit tests for `StorageEngine::put_raw` in `rdb-storage/src/engine.rs` +- [X] T010 [US1] Create unit tests for `StorageEngine::compare_and_swap` (success/fail) in `rdb-storage/src/engine.rs` + +### Implementation for User Story 1 + +- [X] T011 [US1] Implement `StorageEngine` trait definition in `rdb-storage/src/lib.rs` +- [X] T012 [US1] Implement `RocksEngine` struct wrapping RocksDB in `rdb-storage/src/rocks_engine.rs` +- [X] T013 [US1] Implement `put_raw` using `CF_DEFAULT` in `rdb-storage/src/rocks_engine.rs` +- [X] T014 [US1] Implement `compare_and_swap` using RocksDB transaction/merge in `rdb-storage/src/rocks_engine.rs` +- [X] T015 [US1] Verify all tests pass + +**Checkpoint**: At this point, User Story 1 should be fully functional and testable independently + +--- + +## Phase 4: User Story 2 - Basic RPC Transport (Priority: P1) + +**Goal**: Verify gRPC communication pipeline between Client and Server. + +**Independent Test**: Run `rdb-server` and connect with a minimal `rdb-client`. + +### Tests for User Story 2 (STANDARD - per constitution) ⚠️ + +- [X] T016 [P] [US2] Create integration test `tests/test_rpc_connect.rs` in `rdb-client` to verify connection + +### Implementation for User Story 2 + +- [X] T017 [P] [US2] Implement `KvService` gRPC handler in `rdb-server/src/service.rs` delegating to storage +- [X] T018 [P] [US2] Implement gRPC server startup in `rdb-server/src/main.rs` +- [X] T019 [US2] Implement `RdbClient` struct wrapping `tonic::transport::Channel` in `rdb-client/src/client.rs` +- [X] T020 [US2] Implement `raw_put` and `cas` methods in `RdbClient` calling gRPC +- [X] T021 [US2] Verify integration test passes + +**Checkpoint**: At this point, User Stories 1 AND 2 should both work independently + +--- + +## Phase 5: User Story 3 - Placement Driver TSO (Priority: P2) + +**Goal**: Source of monotonic timestamps (TSO). + +**Independent Test**: Run `rdb-pd` and verify monotonic TSO generation. + +### Tests for User Story 3 (STANDARD - per constitution) ⚠️ + +- [X] T022 [P] [US3] Create unit test for `TsoOracle` in `rdb-pd/src/tso.rs` + +### Implementation for User Story 3 + +- [X] T023 [P] [US3] Implement `TsoOracle` logic (monotonic u64) in `rdb-pd/src/tso.rs` +- [X] T024 [US3] Implement `TsoService` gRPC handler in `rdb-pd/src/service.rs` +- [X] T025 [US3] Implement PD server startup in `rdb-pd/src/main.rs` +- [X] T026 [US3] Add `get_tso` method to `RdbClient` in `rdb-client/src/client.rs` + +**Checkpoint**: All user stories should now be independently functional + +--- + +## Phase 6: Polish & Cross-Cutting Concerns + +**Purpose**: Improvements that affect multiple user stories + +- [X] T027 Create `scripts/verify-core.sh` for comprehensive integration verification +- [X] T028 Run `quickstart.md` verification steps manually +- [X] T029 Format code with `cargo fmt` and lint with `cargo clippy` + +--- + +## Phase 7: RPC Get & Raft Enhancements + +**Purpose**: Complete client/server Get coverage and initial Raft persistence surface + +- [X] T030 [US2] Implement and verify server Get path returning value+version via CAS CF in `rdb-server/src/service.rs` +- [X] T031 [US2] Implement client `raw_get`/`get` APIs and CLI with integration test in `rdb-client` +- [X] T032 [US2] Add integration test covering Get (RawGet + CAS Get) in `rdb-client/tests` +- [X] T033 [P] Add Raft log/HardState/ConfState persistence and wire Raft service to peer dispatch in `rdb-server` (single-region, single-node baseline) + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies - can start immediately +- **Foundational (Phase 2)**: Depends on Setup completion - BLOCKS all user stories +- **User Stories (Phase 3+)**: All depend on Foundational phase completion + - User stories can then proceed in parallel (if staffed) + - Or sequentially in priority order (P1 → P2 → P3) +- **Polish (Final Phase)**: Depends on all desired user stories being complete + +### User Story Dependencies + +- **User Story 1 (P1)**: Can start after Foundational (Phase 2) - Core Storage logic +- **User Story 2 (P1)**: Can start after Foundational (Phase 2) - RPC Layer (Technically depends on US1 storage implementation for full end-to-end, but server shell can be built in parallel) +- **User Story 3 (P2)**: Can start after Foundational (Phase 2) - Independent PD service + +### Within Each User Story + +- Tests (if included) MUST be written and FAIL before implementation +- Models before services +- Services before endpoints +- Core implementation before integration +- Story complete before moving to next priority + +### Parallel Opportunities + +- All Setup tasks marked [P] can run in parallel +- All Foundational tasks marked [P] can run in parallel (within Phase 2) +- Once Foundational phase completes, all user stories can start in parallel (if team capacity allows) +- All tests for a user story marked [P] can run in parallel +- Models within a story marked [P] can run in parallel +- Different user stories can be worked on in parallel by different team members + +--- + +## Parallel Example: User Story 1 + +```bash +# Launch all tests for User Story 1 together (if tests requested): +Task: "Create unit tests for StorageEngine::put_raw in rdb-storage/src/engine.rs" +Task: "Create unit tests for StorageEngine::compare_and_swap (success/fail) in rdb-storage/src/engine.rs" + +# Launch all models for User Story 1 together: +Task: "Implement StorageEngine trait definition in rdb-storage/src/lib.rs" +Task: "Implement RocksEngine struct wrapping RocksDB in rdb-storage/src/rocks_engine.rs" +``` + +--- + +## Implementation Strategy + +### MVP First (User Story 1 Only) + +1. Complete Phase 1: Setup +2. Complete Phase 2: Foundational (CRITICAL - blocks all stories) +3. Complete Phase 3: User Story 1 +4. **STOP and VALIDATE**: Test User Story 1 independently +5. Deploy/demo if ready + +### Incremental Delivery + +1. Complete Setup + Foundational → Foundation ready +2. Add User Story 1 → Test independently → Deploy/Demo (MVP!) +3. Add User Story 2 → Test independently → Deploy/Demo +4. Add User Story 3 → Test independently → Deploy/Demo +5. Each story adds value without breaking previous stories + +### Parallel Team Strategy + +With multiple developers: + +1. Team completes Setup + Foundational together +2. Once Foundational is done: + - Developer A: User Story 1 + - Developer B: User Story 2 + - Developer C: User Story 3 +3. Stories complete and integrate independently diff --git a/specifications/flaredb/001-multi-raft/spec.md b/specifications/flaredb/001-multi-raft/spec.md new file mode 100644 index 0000000..c67d914 --- /dev/null +++ b/specifications/flaredb/001-multi-raft/spec.md @@ -0,0 +1,115 @@ +# Feature Specification: [FEATURE NAME] + +**Feature Branch**: `[###-feature-name]` +**Created**: [DATE] +**Status**: Draft +**Input**: User description: "$ARGUMENTS" + +## User Scenarios & Testing *(mandatory)* + + + +### User Story 1 - [Brief Title] (Priority: P1) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently - e.g., "Can be fully tested by [specific action] and delivers [specific value]"] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] +2. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 2 - [Brief Title] (Priority: P2) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 3 - [Brief Title] (Priority: P3) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +[Add more user stories as needed, each with an assigned priority] + +### Edge Cases + + + +- What happens when [boundary condition]? +- How does system handle [error scenario]? + +## Requirements *(mandatory)* + + + +### Functional Requirements + +- **FR-001**: System MUST [specific capability, e.g., "allow users to create accounts"] +- **FR-002**: System MUST [specific capability, e.g., "validate email addresses"] +- **FR-003**: Users MUST be able to [key interaction, e.g., "reset their password"] +- **FR-004**: System MUST [data requirement, e.g., "persist user preferences"] +- **FR-005**: System MUST [behavior, e.g., "log all security events"] + +*Example of marking unclear requirements:* + +- **FR-006**: System MUST authenticate users via [NEEDS CLARIFICATION: auth method not specified - email/password, SSO, OAuth?] +- **FR-007**: System MUST retain user data for [NEEDS CLARIFICATION: retention period not specified] + +### Key Entities *(include if feature involves data)* + +- **[Entity 1]**: [What it represents, key attributes without implementation] +- **[Entity 2]**: [What it represents, relationships to other entities] + +## Success Criteria *(mandatory)* + + + +### Measurable Outcomes + +- **SC-001**: [Measurable metric, e.g., "Users can complete account creation in under 2 minutes"] +- **SC-002**: [Measurable metric, e.g., "System handles 1000 concurrent users without degradation"] +- **SC-003**: [User satisfaction metric, e.g., "90% of users successfully complete primary task on first attempt"] +- **SC-004**: [Business metric, e.g., "Reduce support tickets related to [X] by 50%"] diff --git a/specifications/flaredb/002-raft-features/checklists/requirements.md b/specifications/flaredb/002-raft-features/checklists/requirements.md new file mode 100644 index 0000000..7c1f78e --- /dev/null +++ b/specifications/flaredb/002-raft-features/checklists/requirements.md @@ -0,0 +1,34 @@ +# Specification Quality Checklist: Raft Core Replication + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2025-12-01 +**Feature**: specs/001-raft-features/spec.md + +## Content Quality + +- [X] No implementation details (languages, frameworks, APIs) +- [X] Focused on user value and business needs +- [X] Written for non-technical stakeholders +- [X] All mandatory sections completed + +## Requirement Completeness + +- [X] No [NEEDS CLARIFICATION] markers remain +- [X] Requirements are testable and unambiguous +- [X] Success criteria are measurable +- [X] Success criteria are technology-agnostic (no implementation details) +- [X] All acceptance scenarios are defined +- [X] Edge cases are identified +- [X] Scope is clearly bounded +- [X] Dependencies and assumptions identified + +## Feature Readiness + +- [X] All functional requirements have clear acceptance criteria +- [X] User scenarios cover primary flows +- [X] Feature meets measurable outcomes defined in Success Criteria +- [X] No implementation details leak into specification + +## Notes + +- Items marked incomplete require spec updates before `/speckit.clarify` or `/speckit.plan` diff --git a/specifications/flaredb/002-raft-features/contracts/raft-service.md b/specifications/flaredb/002-raft-features/contracts/raft-service.md new file mode 100644 index 0000000..3bb5683 --- /dev/null +++ b/specifications/flaredb/002-raft-features/contracts/raft-service.md @@ -0,0 +1,35 @@ +# Raft Service Contract (gRPC) + +## Overview + +Single RPC entrypoint for Raft message exchange; uses raft-rs `Message` protobuf encoding (prost). + +## Service + +``` +service RaftService { + rpc Send(RaftMessage) returns (RaftResponse); +} +``` + +## Messages + +- **RaftMessage** + - `message: bytes` (serialized `raft::eraftpb::Message` via prost) + +- **RaftResponse** + - Empty payload; errors conveyed via gRPC status + +## Expectations + +- Client (peer) wraps raft-rs `Message` and posts to remote peer via `Send`. +- Receivers decode and feed into `RawNode::step`, then drive `on_ready` to persist/apply. +- Transport must retry/transient-handle UNAVAILABLE; fail fast on INVALID_ARGUMENT decode errors. + +## Test Hooks + +- Integration harness should: + - Start 3 peers with distinct addresses. + - Wire RaftService between peers. + - Propose on leader; verify followers receive and persist entries. + - Simulate follower stop/restart and verify catch-up via `Send`. diff --git a/specifications/flaredb/002-raft-features/data-model.md b/specifications/flaredb/002-raft-features/data-model.md new file mode 100644 index 0000000..d97f404 --- /dev/null +++ b/specifications/flaredb/002-raft-features/data-model.md @@ -0,0 +1,34 @@ +# Data Model: Raft Core Replication + +## Entities + +- **Peer** + - Fields: `id (u64)`, `region_id (u64)`, `state (Leader/Follower/Candidate)`, `term (u64)`, `commit_index (u64)`, `last_applied (u64)` + - Relationships: owns `RaftStorage`; exchanges `RaftLogEntry` with other peers. + - Constraints: single region scope for this phase; fixed voter set of 3. + +- **RaftLogEntry** + - Fields: `index (u64)`, `term (u64)`, `command (bytes)`, `context (bytes, optional)` + - Relationships: persisted in `raft_log` CF; applied to state machine when committed. + - Constraints: indices strictly increasing; term monotonic per election; applied in order. + +- **HardState** + - Fields: `current_term (u64)`, `voted_for (u64)`, `commit_index (u64)` + - Relationships: persisted in `raft_state` CF; loaded at startup before participating. + - Constraints: must be flushed atomically with log appends when advancing commit index. + +- **ConfState** + - Fields: `voters (Vec)` + - Relationships: persisted in `raft_state` CF; defines quorum (majority of 3). + - Constraints: static for this phase; changes require future joint consensus. + +- **ReplicationState** + - Fields: `match_index (u64)`, `next_index (u64)`, `pending (bool)` + - Relationships: maintained per follower in memory; not persisted. + - Constraints: drives AppendEntries backoff and progress. + +## State Transitions + +- Peer transitions: Follower → Candidate → Leader on election; Leader → Follower on higher term or failed election. +- Log application: when `commit_index` advances, apply entries in order to state machine; `last_applied` increases monotonically. +- Recovery: on restart, load `HardState`, `ConfState`, and log; reconcile with leader via AppendEntries (truncate/append) before applying new entries. diff --git a/specifications/flaredb/002-raft-features/plan.md b/specifications/flaredb/002-raft-features/plan.md new file mode 100644 index 0000000..4b921a1 --- /dev/null +++ b/specifications/flaredb/002-raft-features/plan.md @@ -0,0 +1,69 @@ +# Implementation Plan: Raft Core Replication + +**Branch**: `002-raft-features` | **Date**: 2025-12-01 | **Spec**: [specs/002-raft-features/spec.md](specs/002-raft-features/spec.md) +**Input**: Feature specification from `/specs/002-raft-features/spec.md` + +**Note**: This template is filled in by the `/speckit.plan` command. See `.specify/templates/commands/plan.md` for the execution workflow. + +## Summary + +Implement Raft core replication for FlareDB: single-node bootstrap with durable log/hard/conf state, majority replication across a fixed 3-node cluster, and follower recovery/catch-up. Build on the existing Rust workspace (raft-rs, RocksDB) with tonic-based transport already present in the repo. + +## Technical Context + +**Language/Version**: Rust (stable, via Nix flake) +**Primary Dependencies**: `raft` (tikv/raft-rs 0.7, prost codec), `tokio`, `tonic`/`prost`, `rocksdb`, `slog` +**Storage**: RocksDB column families (`raft_log`, `raft_state`) for log, hard state, and conf state +**Testing**: `cargo test` (unit/integration), scripted multi-node harness to be added for replication scenarios +**Target Platform**: Linux (x86_64), Nix dev shell +**Project Type**: Rust workspace (multi-crate: rdb-proto, rdb-storage, rdb-server, rdb-pd, rdb-client, rdb-cli) +**Performance Goals**: From spec SCs — single-node commit ≤2s; 3-node majority commit ≤3s; follower catch-up ≤5s after rejoin +**Constraints**: Fixed 3-node membership for this phase; no dynamic add/remove; minority must not commit +**Scale/Scope**: Cluster size 3; log volume moderate (dev/test scale) sufficient to validate recovery and catch-up + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +- Test-First: Plan includes unit/integration tests for Raft storage, proposal/commit, and recovery paths. +- Reliability & Coverage: CI to run `cargo test`; integration harness to cover cross-node replication. +- Simplicity & Readability: Use existing crates (raft-rs, rocksdb); avoid bespoke protocols. +- Observability: Ensure structured logs on Raft events/errors; failures must be actionable. +- Versioning & Compatibility: Proto changes, if any, must be called out; fixed membership avoids dynamic reconfig in this phase. +No constitution violations identified; gate PASS. + +## Project Structure + +### Documentation (this feature) + +```text +specs/002-raft-features/ +├── plan.md # This file +├── research.md # Phase 0 output +├── data-model.md # Phase 1 output +├── quickstart.md # Phase 1 output +├── contracts/ # Phase 1 output +└── tasks.md # Phase 2 output (via /speckit.tasks) +``` + +### Source Code (repository root) + +```text +Cargo.toml # workspace +rdb-proto/ # proto definitions +rdb-storage/ # RocksDB storage + Raft CFs +rdb-server/ # Raft peer, gRPC services +rdb-pd/ # placement driver (not primary in this feature) +rdb-client/ # client SDK/CLI (control hooks if needed) +rdb-cli/ # auxiliary CLI +scripts/ # verification scripts +tests/ # integration harness (to be added under rdb-server or workspace) +``` + +**Structure Decision**: Use existing Rust workspace layout; place Raft-focused tests/harness under `rdb-server/tests` or workspace `tests/` as appropriate; contracts under `specs/002-raft-features/contracts/`. + +## Complexity Tracking + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|-------------------------------------| +| N/A | | | diff --git a/specifications/flaredb/002-raft-features/quickstart.md b/specifications/flaredb/002-raft-features/quickstart.md new file mode 100644 index 0000000..289add7 --- /dev/null +++ b/specifications/flaredb/002-raft-features/quickstart.md @@ -0,0 +1,39 @@ +# Quickstart: Raft Core Replication + +## Prerequisites +- Nix dev shell: `nix develop` +- Ports available: 50051, 50052, 50053 (Raft/gRPC) +- Clean data dirs for each node + +## 1) Build & Unit Tests +```bash +nix develop -c cargo build +nix develop -c cargo test -p rdb-server -- service::tests::get_returns_value_and_version +nix develop -c cargo test -p rdb-server -- peer::tests::single_node_propose_persists_log +``` + +## 2) Start a 3-Node Cluster (manual) +```bash +# Terminal 1 +nix develop -c cargo run --bin rdb-server -- --addr 127.0.0.1:50051 --data-dir /tmp/rdb-node1 +# Terminal 2 +nix develop -c cargo run --bin rdb-server -- --addr 127.0.0.1:50052 --data-dir /tmp/rdb-node2 +# Terminal 3 +nix develop -c cargo run --bin rdb-server -- --addr 127.0.0.1:50053 --data-dir /tmp/rdb-node3 +``` + +## 3) Propose & Verify (temporary approach) +- Use the forthcoming integration harness (under `rdb-server/tests`) to: + - Elect a leader (campaign) + - Propose a command (e.g., `"hello"`) + - Assert at least two nodes have the entry at the same index/term and commit +- For now, run: +```bash +nix develop -c cargo test -p rdb-server -- --ignored +``` +(ignored tests will host the multi-node harness once added) + +## 4) Recovery Check +- Stop one follower process, keep leader + other follower running. +- Propose another entry. +- Restart the stopped follower with the same data dir; verify logs show catch-up and committed entries applied (via test harness assertions). diff --git a/specifications/flaredb/002-raft-features/research.md b/specifications/flaredb/002-raft-features/research.md new file mode 100644 index 0000000..8768ede --- /dev/null +++ b/specifications/flaredb/002-raft-features/research.md @@ -0,0 +1,23 @@ +# Research: Raft Core Replication (002-raft-features) + +## Decisions + +- **Raft library**: Use `raft` (tikv/raft-rs 0.7, prost-codec). + - *Rationale*: Battle-tested implementation, already wired in repo; supports necessary APIs for storage/transport. + - *Alternatives considered*: `openraft` (heavier refactor), custom Raft (too risky/time-consuming). + +- **Log/State persistence**: Persist log entries, hard state, conf state in RocksDB CFs (`raft_log`, `raft_state`). + - *Rationale*: RocksDB already provisioned and used; column families align with separation of concerns; durable restart semantics. + - *Alternatives considered*: In-memory (unsafe for recovery), separate files (adds new IO path, no benefit). + +- **Cluster scope**: Fixed 3-node membership for this phase; no dynamic add/remove. + - *Rationale*: Matches spec clarification; reduces scope to core replication/recovery; simpler correctness surface. + - *Alternatives considered*: Joint consensus/dynamic membership (out of scope now). + +- **Transport**: Continue with tonic/prost gRPC messages for Raft network exchange. + - *Rationale*: Existing RaftService in repo; shared proto tooling; avoids new protocol surface. + - *Alternatives considered*: custom TCP/UDP transport (unnecessary for current goals). + +- **Testing approach**: Unit tests for storage/persistence; single-node campaign/propose; multi-node integration harness to validate majority commit and follower catch-up. + - *Rationale*: Aligns with constitution Test-First; exercises durability and replication behaviors. + - *Alternatives considered*: manual ad-hoc testing (insufficient coverage). diff --git a/specifications/flaredb/002-raft-features/spec.md b/specifications/flaredb/002-raft-features/spec.md new file mode 100644 index 0000000..93acca0 --- /dev/null +++ b/specifications/flaredb/002-raft-features/spec.md @@ -0,0 +1,92 @@ +# Feature Specification: Raft Core Replication + +**Feature Branch**: `002-raft-features` +**Created**: 2025-12-01 +**Status**: Draft +**Input**: User description: "Raft関連の機能についてお願いします。" + +## Clarifications + +### Session 2025-12-01 +- Q: Should this phase assume fixed 3-node membership or include dynamic membership? → A: Fixed 3-node, extensible for future scaling. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Single-Node Raft Baseline (Priority: P1) + +As a platform engineer, I want a single-node Raft instance to accept proposals, elect a leader, and persist committed entries so I can validate the log/storage plumbing before scaling out. + +**Why this priority**: Establishes correctness of log append/apply and persistence; blocks multi-node rollout. + +**Independent Test**: Start one node, trigger self-election, propose an entry, verify it is committed and applied to storage with the expected data. + +**Acceptance Scenarios**: + +1. **Given** a single node started fresh, **When** it campaigns, **Then** it becomes leader and can accept proposals. +2. **Given** a proposed entry "e1", **When** it commits, **Then** storage contains "e1" and last index increments by 1. + +--- + +### User Story 2 - Multi-Node Replication (Priority: P1) + +As a platform engineer, I want a 3-node Raft cluster to replicate entries to a majority so that writes remain durable under follower failure. + +**Why this priority**: Majority replication is the core availability guarantee of Raft. + +**Independent Test**: Start 3 nodes, elect a leader, propose an entry; verify leader and at least one follower store the entry at the same index/term and report commit. + +**Acceptance Scenarios**: + +1. **Given** a 3-node cluster, **When** a leader is elected, **Then** at least two nodes acknowledge commit for the same index/term. +2. **Given** a committed entry on the leader, **When** one follower is stopped, **Then** the other follower still receives and persists the entry. + +--- + +### User Story 3 - Failure and Recovery (Priority: P2) + +As an operator, I want a stopped follower to recover and catch up without losing committed data so that the cluster can heal after restarts. + +**Why this priority**: Ensures durability across restarts and supports rolling maintenance. + +**Independent Test**: Commit an entry, stop a follower, commit another entry, restart the follower; verify it restores state and applies all committed entries. + +**Acceptance Scenarios**: + +1. **Given** a follower stopped after entry N is committed, **When** the cluster commits entry N+1 while it is down, **Then** on restart the follower installs both entries in order. +2. **Given** divergent logs on restart, **When** leader sends AppendEntries, **Then** follower truncates/aligns to leader and preserves committed suffix. + +--- + +### Edge Cases + +- Leader crash immediately after commit but before followers apply. +- Network partition isolating a minority vs. majority; minority must not commit new entries. +- Log holes or conflicting terms on recovery must be reconciled to leader’s log. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The system MUST support single-node leader election and proposal handling without external coordination. +- **FR-002**: The system MUST replicate log entries to a majority in a 3-node cluster before marking them committed. +- **FR-003**: The system MUST persist log entries, hard state (term, vote), and conf state to durable storage so that restarts preserve committed progress. +- **FR-004**: The system MUST apply committed entries to the underlying storage engine in log order without gaps. +- **FR-005**: The system MUST prevent a node in a minority partition from committing new entries while isolated. +- **FR-006**: On restart, a node MUST reconcile its log with the leader (truncate/append) to match the committed log and reapply missing committed entries. +- **FR-007**: For this phase, operate a fixed 3-node membership (no dynamic add/remove), but architecture must allow future extension to scale out safely. + +### Key Entities + +- **Peer**: A Raft node with ID, region scope, in-memory state machine, and access to durable Raft storage. +- **Raft Log Entry**: Indexed record containing term and opaque command bytes; persisted and replicated. +- **Hard State**: Term, vote, commit index persisted to ensure safety across restarts. +- **Conf State**: Voter set defining the quorum for replication. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: Single-node bootstraps and accepts a proposal within 2 seconds, committing it and persisting the entry. +- **SC-002**: In a 3-node cluster, a committed entry is present on at least two nodes within 3 seconds of proposal. +- **SC-003**: After a follower restart, all previously committed entries are restored and applied in order within 5 seconds of rejoining a healthy leader. +- **SC-004**: During a minority partition, isolated nodes do not advance commit index or apply uncommitted entries. diff --git a/specifications/flaredb/002-raft-features/tasks.md b/specifications/flaredb/002-raft-features/tasks.md new file mode 100644 index 0000000..bec8e33 --- /dev/null +++ b/specifications/flaredb/002-raft-features/tasks.md @@ -0,0 +1,128 @@ +--- +description: "Task list for Raft Core Replication" +--- + +# Tasks: Raft Core Replication + +**Input**: Design documents from `/specs/002-raft-features/` +**Prerequisites**: plan.md (required), spec.md (required for user stories), research.md, data-model.md, contracts/ + +**Tests**: Required per constitution; include unit/integration tests for Raft storage, proposal/commit, replication, and recovery. + +**Organization**: Tasks are grouped by user story to enable independent implementation and testing. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (e.g., US1, US2, US3) +- Include exact file paths in descriptions + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Ensure tooling and layout are ready for Raft feature work. + +- [X] T001 Verify Raft proto service definition matches contract in `rdb-proto/src/raft_server.proto` +- [X] T002 Ensure Raft gRPC server/client wiring is enabled in `rdb-server/src/main.rs` and `rdb-server/src/raft_service.rs` + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Durable Raft storage primitives required by all stories. + +- [X] T003 Implement complete Raft storage persistence (log/hard state/conf state read/write) in `rdb-server/src/raft_storage.rs` +- [X] T004 Add unit tests for Raft storage persistence (log append, load, truncate) in `rdb-server/src/raft_storage.rs` +- [X] T005 Ensure Peer ready loop persists entries and hard state before apply in `rdb-server/src/peer.rs` + +**Checkpoint**: Raft storage durability verified. + +--- + +## Phase 3: User Story 1 - Single-Node Raft Baseline (Priority: P1) + +**Goal**: Single node can self-elect, propose, commit, and apply entries to storage. + +**Independent Test**: Run unit/integration tests that start one peer, campaign, propose a command, and verify commit/apply and durable log. + +### Tests +- [X] T006 [US1] Add single-node campaign/propose/apply test in `rdb-server/src/peer.rs` (cfg(test)) or `rdb-server/tests/test_single_node.rs` + +### Implementation +- [X] T007 [US1] Implement Peer campaign/propose handling with log apply in `rdb-server/src/peer.rs` +- [X] T008 [US1] Expose a simple propose entry point (e.g., CLI or helper) for single-node testing in `rdb-server/src/main.rs` +- [X] T009 [US1] Validate single-node flow passes tests and persists entries (run `cargo test -p rdb-server -- single_node`) + +**Checkpoint**: Single-node Raft end-to-end verified. + +--- + +## Phase 4: User Story 2 - Multi-Node Replication (Priority: P1) + +**Goal**: 3-node cluster replicates entries to a majority; leader/follower paths wired via gRPC. + +**Independent Test**: Integration harness spins up 3 nodes, elects leader, proposes entry, asserts commit on at least 2 nodes. + +### Tests +- [X] T010 [US2] Create 3-node integration test harness in `rdb-server/tests/test_replication.rs` to validate majority commit + +### Implementation +- [X] T011 [US2] Wire RaftService transport send/receive to dispatch messages to peers in `rdb-server/src/raft_service.rs` +- [X] T012 [P] [US2] Implement peer registry/peer manager to track remote addresses and send Raft messages in `rdb-server/src/peer_manager.rs` +- [X] T013 [US2] Update server startup to create/join fixed 3-node cluster with configured peers in `rdb-server/src/main.rs` +- [X] T014 [US2] Ensure ready loop sends outbound messages produced by RawNode in `rdb-server/src/peer.rs` +- [X] T015 [US2] Verify majority replication via integration harness (run `cargo test -p rdb-server -- test_replication`) + +**Checkpoint**: Majority replication validated on 3 nodes. + +--- + +## Phase 5: User Story 3 - Failure and Recovery (Priority: P2) + +**Goal**: Followers can restart and catch up without losing committed entries; isolation prevents commits. + +**Independent Test**: Integration test stops a follower, commits entry while down, restarts follower, and verifies log reconciliation and apply. + +### Tests +- [X] T016 [US3] Add follower restart/catch-up integration test in `rdb-server/tests/test_recovery.rs` +- [X] T016 [US3] Add follower restart/catch-up integration test in `rdb-server/tests/test_recovery.rs` (in progress; currently ignored in `test_replication.rs`) + +### Implementation +- [X] T017 [US3] Implement startup recovery: load HardState/ConfState/log and reconcile via AppendEntries in `rdb-server/src/peer.rs` +- [X] T018 [US3] Handle log truncate/append on conflict and apply committed entries after recovery in `rdb-server/src/peer.rs` +- [X] T019 [US3] Add isolation guard: prevent commit advancement on minority partition detection (e.g., via quorum checks) in `rdb-server/src/peer.rs` +- [X] T020 [US3] Validate recovery/integration tests pass (run `cargo test -p rdb-server -- test_recovery`) + +**Checkpoint**: Recovery and partition safety validated. + +--- + +## Phase 6: Polish & Cross-Cutting Concerns + +**Purpose**: Hardening and operability. + +- [X] T021 Add structured Raft logging (term/index/apply/commit) in `rdb-server` with slog +- [X] T022 Add quickstart or script to launch 3-node cluster and run replication test in `scripts/verify-raft.sh` +- [X] T023 Run full workspace tests and format/lint (`cargo test`, `cargo fmt`, `cargo clippy`) + +--- + +## Dependencies & Execution Order + +- Foundational (Phase 2) blocks all Raft user stories. +- US1 must complete before US2/US3 (builds basic propose/apply). +- US2 should precede US3 (replication before recovery). +- Polish runs last. + +## Parallel Examples + +- T011 (transport wiring) and T012 (peer manager) can proceed in parallel once T003–T005 are done. +- US2 tests (T010) can be authored in parallel with transport implementation, then enabled once wiring lands. +- Logging and script polish (T021–T022) can run in parallel after core stories complete. + +## Implementation Strategy + +1. Complete Foundational (durable storage). +2. Deliver US1 (single-node MVP). +3. Deliver US2 (majority replication). +4. Deliver US3 (recovery/partition safety). +5. Polish (logging, scripts, fmt/clippy). diff --git a/specifications/flaredb/003-kvs-consistency/checklists/requirements.md b/specifications/flaredb/003-kvs-consistency/checklists/requirements.md new file mode 100644 index 0000000..ee9c125 --- /dev/null +++ b/specifications/flaredb/003-kvs-consistency/checklists/requirements.md @@ -0,0 +1,34 @@ +# Specification Quality Checklist: Distributed KVS Consistency Modes + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2025-12-01 +**Feature**: specs/003-kvs-consistency/spec.md + +## Content Quality + +- [X] No implementation details (languages, frameworks, APIs) +- [X] Focused on user value and business needs +- [X] Written for non-technical stakeholders +- [X] All mandatory sections completed + +## Requirement Completeness + +- [X] No [NEEDS CLARIFICATION] markers remain +- [X] Requirements are testable and unambiguous +- [X] Success criteria are measurable +- [X] Success criteria are technology-agnostic (no implementation details) +- [X] All acceptance scenarios are defined +- [X] Edge cases are identified +- [X] Scope is clearly bounded +- [X] Dependencies and assumptions identified + +## Feature Readiness + +- [X] All functional requirements have clear acceptance criteria +- [X] User scenarios cover primary flows +- [X] Feature meets measurable outcomes defined in Success Criteria +- [X] No implementation details leak into specification + +## Notes + +- Items marked incomplete require spec updates before `/speckit.clarify` or `/speckit.plan` diff --git a/specifications/flaredb/003-kvs-consistency/contracts/kv_cas.md b/specifications/flaredb/003-kvs-consistency/contracts/kv_cas.md new file mode 100644 index 0000000..5a11081 --- /dev/null +++ b/specifications/flaredb/003-kvs-consistency/contracts/kv_cas.md @@ -0,0 +1,29 @@ +# KvCas contracts (strong consistency) + +## CompareAndSwap +- **RPC**: `kvrpc.KvCas/CompareAndSwap` +- **Request**: + - `namespace: string` (empty => `default`) + - `key: bytes` + - `value: bytes` + - `expected_version: uint64` +- **Response**: + - `success: bool` + - `current_version: uint64` + - `new_version: uint64` +- **Semantics**: + - Allowed only for `strong` namespaces; returns `FailedPrecondition` otherwise or when not leader (redirect required). + - Proposes via Raft; state machine applies with LWW timestamp wrapper. + +## Get +- **RPC**: `kvrpc.KvCas/Get` +- **Request**: + - `namespace: string` (empty => `default`) + - `key: bytes` +- **Response**: + - `found: bool` + - `value: bytes` + - `version: uint64` +- **Semantics**: + - Allowed only for `strong` namespaces; returns `FailedPrecondition` if not leader. + - Reads versioned value (timestamp-prefixed) and returns decoded value plus version. diff --git a/specifications/flaredb/003-kvs-consistency/contracts/kv_raw.md b/specifications/flaredb/003-kvs-consistency/contracts/kv_raw.md new file mode 100644 index 0000000..f5ca4f9 --- /dev/null +++ b/specifications/flaredb/003-kvs-consistency/contracts/kv_raw.md @@ -0,0 +1,25 @@ +# KvRaw contracts (eventual consistency) + +## RawPut +- **RPC**: `kvrpc.KvRaw/RawPut` +- **Request**: + - `namespace: string` (empty => `default`) + - `key: bytes` + - `value: bytes` +- **Response**: + - `success: bool` +- **Semantics**: + - Allowed only for namespaces in `eventual` mode; returns `FailedPrecondition` otherwise. + - Writes locally with LWW timestamp prefix and queues best-effort async replication via Raft when a leader is present. + +## RawGet +- **RPC**: `kvrpc.KvRaw/RawGet` +- **Request**: + - `namespace: string` (empty => `default`) + - `key: bytes` +- **Response**: + - `found: bool` + - `value: bytes` (empty if not found) +- **Semantics**: + - Allowed only for `eventual` namespaces; returns `FailedPrecondition` otherwise. + - Returns value decoded from LWW-encoded payload (drops the timestamp). diff --git a/specifications/flaredb/003-kvs-consistency/contracts/raft_service.md b/specifications/flaredb/003-kvs-consistency/contracts/raft_service.md new file mode 100644 index 0000000..546c815 --- /dev/null +++ b/specifications/flaredb/003-kvs-consistency/contracts/raft_service.md @@ -0,0 +1,33 @@ +# RaftService contracts (namespace mode ops) + +## GetMode + +- **RPC**: `RaftService/GetMode` +- **Request**: `namespace: string` (empty => `default`) +- **Response**: `mode: string` (`"strong"` or `"eventual"`) + +## UpdateNamespaceMode + +- **RPC**: `RaftService/UpdateNamespaceMode` +- **Request**: + - `namespace: string` (required) + - `mode: string` (`"strong"` or `"eventual"`, required) +- **Response**: `mode` object + - `namespace: string` + - `id: uint32` + - `mode: string` + - `from_default: bool` (true if created implicitly) + +## ListNamespaceModes + +- **RPC**: `RaftService/ListNamespaceModes` +- **Request**: empty +- **Response**: `namespaces[]` + - `namespace: string` + - `id: uint32` + - `mode: string` + - `from_default: bool` + +### Error cases +- `InvalidArgument` when mode is not `"strong"` or `"eventual"` or namespace is empty for updates. +- `FailedPrecondition` if Raft messages are addressed to a different peer. diff --git a/specifications/flaredb/003-kvs-consistency/data-model.md b/specifications/flaredb/003-kvs-consistency/data-model.md new file mode 100644 index 0000000..d035af5 --- /dev/null +++ b/specifications/flaredb/003-kvs-consistency/data-model.md @@ -0,0 +1,26 @@ +# Data Model: Namespace Consistency + +- Namespace + - id: u32 + - name: string + - mode: ConsistencyMode (strong | eventual) + - explicit: bool (true when user-configured; false when created implicitly) + +- NamespaceModeDiff + - namespace: string + - self_id: u32 + - other_id: u32 + - self_mode: ConsistencyMode + - other_mode: ConsistencyMode + +- ClusterConfig + - namespaces: [Namespace] + - default_mode: ConsistencyMode + +- ConsistencyMode + - values: strong | eventual + +- ConvergenceLag + - p50_ms: u64 + - p95_ms: u64 + - max_ms: u64 diff --git a/specifications/flaredb/003-kvs-consistency/plan.md b/specifications/flaredb/003-kvs-consistency/plan.md new file mode 100644 index 0000000..1ee608d --- /dev/null +++ b/specifications/flaredb/003-kvs-consistency/plan.md @@ -0,0 +1,76 @@ +# Implementation Plan: Distributed KVS Consistency Modes + +**Branch**: `003-kvs-consistency` | **Date**: 2025-12-01 | **Spec**: specs/003-kvs-consistency/spec.md +**Input**: Feature specification from `/specs/003-kvs-consistency/spec.md` + +**Note**: This template is filled in by the `/speckit.plan` command. See `.specify/templates/commands/plan.md` for the execution workflow. + +## Summary + +Deliver a deployable distributed KVS supporting strong consistency (quorum read/write) and eventual consistency (LWW default), with namespace-level mode selection, safe mode switching, convergence/recovery behavior, and observability. + +## Technical Context + +**Language/Version**: Rust (stable, via Nix flake) +**Primary Dependencies**: raft-rs, tonic/prost gRPC, RocksDB, tokio +**Storage**: RocksDB for raft log/state and KV data +**Testing**: cargo test (unit/integration), extend rdb-server multi-node tests for namespace/mode behaviors +**Target Platform**: Linux server (Nix dev shell) +**Project Type**: Distributed server (rdb-server) with gRPC API/CLI +**Performance Goals**: Strong mode quorum commit p95 ~1–2s; eventual mode convergence within a few seconds under normal network; observable lag metrics +**Constraints**: Constitution (test-first, observability, compatibility); fixed membership scope for this phase; namespace-level mode config +**Scale/Scope**: Small cluster (3–5 nodes) dev target; multiple namespaces with per-namespace mode + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +- Test-First: Add/extend integration tests for strong/eventual modes, namespace config, convergence/recovery. +- Reliability & Coverage: Keep existing Raft tests green; new tests cover mode behaviors and failures. +- Simplicity & Readability: Reuse existing crates and current server structure; avoid bespoke protocols. +- Observability: Structured logs/metrics for mode, convergence lag, quorum status, config state. +- Versioning & Compatibility: Call out any gRPC/contract changes; fixed membership scope maintained. + +## Project Structure + +### Documentation (this feature) + +```text +specs/003-kvs-consistency/ +├── plan.md +├── research.md +├── data-model.md +├── quickstart.md +├── contracts/ +└── tasks.md # via /speckit.tasks +``` + +### Source Code (repository root) + +```text +rdb-server/ + src/ + peer.rs + peer_manager.rs + raft_service.rs + config/ # add namespace/mode config handling + api/ # gRPC handlers (mode/config endpoints if needed) + tests/ + test_replication.rs (extend for mode/namespace cases) + +rdb-proto/ + src/*.proto # update if API exposes mode/config + +scripts/ + verify-raft.sh # update or add mode verification script +``` + +**Structure Decision**: Extend existing rdb-server layout with namespace/mode config, tests under rdb-server/tests, contracts under specs/003-kvs-consistency/contracts. + +## Complexity Tracking + +> **Fill ONLY if Constitution Check has violations that must be justified** + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|-------------------------------------| +| N/A | | | diff --git a/specifications/flaredb/003-kvs-consistency/quickstart.md b/specifications/flaredb/003-kvs-consistency/quickstart.md new file mode 100644 index 0000000..3183d20 --- /dev/null +++ b/specifications/flaredb/003-kvs-consistency/quickstart.md @@ -0,0 +1,78 @@ +# Quickstart: Namespace Consistency Modes + +This guide shows how to operate namespace-level consistency (strong vs eventual) now that runtime mode updates are supported. + +## Boot a local cluster + +```bash +# Start three nodes with explicit namespace modes (default=strong, logs=eventual) +cargo run -p rdb-server -- --store-id 1 --addr 127.0.0.1:50051 --namespace-mode logs=eventual +cargo run -p rdb-server -- --store-id 2 --addr 127.0.0.1:50052 --peer 1=127.0.0.1:50051 --namespace-mode logs=eventual +cargo run -p rdb-server -- --store-id 3 --addr 127.0.0.1:50053 --peer 1=127.0.0.1:50051 --namespace-mode logs=eventual +``` + +## Inspect current modes + +`RaftService/GetMode` (single namespace) and `RaftService/ListNamespaceModes` (all namespaces) expose the active configuration and whether a namespace was implicitly created from the default. + +```bash +# List all namespaces and their modes +grpcurl -plaintext 127.0.0.1:50051 raftpb.RaftService/ListNamespaceModes + +# Check a specific namespace +grpcurl -plaintext -d '{"namespace":"logs"}' 127.0.0.1:50051 raftpb.RaftService/GetMode +``` + +The response includes `from_default=true` when the namespace was auto-created using the default mode. + +## Update a namespace mode (rolling safe) + +Mode updates are applied in-memory and picked up immediately by peers; roll across nodes to avoid divergence. + +```bash +# Switch "logs" to strong consistency on node 1 +grpcurl -plaintext -d '{"namespace":"logs","mode":"strong"}' \ + 127.0.0.1:50051 raftpb.RaftService/UpdateNamespaceMode + +# Repeat on each node; verify all agree +grpcurl -plaintext 127.0.0.1:50051 raftpb.RaftService/ListNamespaceModes +grpcurl -plaintext 127.0.0.1:50052 raftpb.RaftService/ListNamespaceModes +grpcurl -plaintext 127.0.0.1:50053 raftpb.RaftService/ListNamespaceModes +``` + +If nodes return different modes for the same namespace, treat it as a mismatch and reapply the update on the outlier nodes. + +## Client usage (KV) + +Strong namespaces use CAS/read/write through the Raft leader; eventual namespaces accept `RawPut/RawGet` locally with LWW replication. + +```bash +# Eventual write/read +grpcurl -plaintext -d '{"namespace":"logs","key":"a","value":"b"}' \ + 127.0.0.1:50051 kvrpc.KvRaw/RawPut +grpcurl -plaintext -d '{"namespace":"logs","key":"a"}' \ + 127.0.0.1:50052 kvrpc.KvRaw/RawGet + +# Strong write/read +grpcurl -plaintext -d '{"namespace":"default","key":"a","value":"b","expected_version":0}' \ + 127.0.0.1:50051 kvrpc.KvCas/CompareAndSwap +grpcurl -plaintext -d '{"namespace":"default","key":"a"}' \ + 127.0.0.1:50051 kvrpc.KvCas/Get +``` + +## Ops checklist + +- Use `ListNamespaceModes` to confirm all nodes share the same mode set before traffic. +- Apply mode updates namespace-by-namespace on each node (or automate via PD) until `from_default=false` everywhere for configured namespaces. +- Keep the default namespace as strong unless explicitly relaxed. + +## Verification + +Run the hardened verify script before committing: + +```bash +scripts/verify-raft.sh +# Expected: cargo fmt clean, all rdb-server tests pass (strong/eventual mode flows) +``` + +This executes `cargo fmt` and `cargo test -p rdb-server --tests` in the Nix shell with protobuf/libclang prepared. diff --git a/specifications/flaredb/003-kvs-consistency/research.md b/specifications/flaredb/003-kvs-consistency/research.md new file mode 100644 index 0000000..5be7db4 --- /dev/null +++ b/specifications/flaredb/003-kvs-consistency/research.md @@ -0,0 +1,15 @@ +# Research: Distributed KVS Consistency Modes (003-kvs-consistency) + +## Decisions + +- **Consistency scope**: Namespace-level selection of strong or eventual consistency. + - *Rationale*: Different tenants/workloads can choose per requirement. + - *Alternatives considered*: Cluster-wide only (too rigid). + +- **Eventual consistency conflict resolution**: Default LWW (last-write-wins); allow alternative policies via config. + - *Rationale*: Simple baseline with deterministic resolution; extensible for advanced policies. + - *Alternatives considered*: Version vectors/CRDT as default (more complex to operate by default). + +## Open Questions + +- None (resolved by spec clarifications). diff --git a/specifications/flaredb/003-kvs-consistency/spec.md b/specifications/flaredb/003-kvs-consistency/spec.md new file mode 100644 index 0000000..af96692 --- /dev/null +++ b/specifications/flaredb/003-kvs-consistency/spec.md @@ -0,0 +1,88 @@ +# Feature Specification: Distributed KVS Consistency Modes + +**Feature Branch**: `003-kvs-consistency` +**Created**: 2025-12-01 +**Status**: Draft +**Input**: User description: "とりあえず分散KVSの部分を使えるようにし、強整合性モードと結果整合性モードを実用可能な状態に持っていくまでの仕様を考えてください。" + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - 強整合性クラスタを安全に稼働 (Priority: P1) + +SRE/オペレータは、固定メンバー（例: 3ノード）のKVSクラスタを強整合性モードで起動し、書き込み・読み出しが常に最新状態で返ることを保証したい。 + +**Why this priority**: 強整合性がS3メタデータやSNSイベントの正確さの土台になるため。 + +**Independent Test**: 少なくとも3ノード構成で、リーダー経由のPut/Getが直ちに反映し、ダウン直後もコミット済みデータが失われないことを検証。 + +**Acceptance Scenarios**: + +1. **Given** 3ノードが強整合性モードで起動済み、**When** リーダーにキーを書き込み、**Then** 即座に全ノードで最新値が読み出せる（リーダーからの再取得）。 +2. **Given** 1ノードを停止、**When** 残り2ノードで読み書き、**Then** コミットは継続しデータ欠損がない（クォーラム成立時のみコミット）。 + +--- + +### User Story 2 - 結果整合性モードで高スループット運用 (Priority: P1) + +オペレータは、イベント処理や一時的なスパイク負荷向けに結果整合性モードを選択し、高スループットな書き込みを許容しつつ、一定時間内に最終的に同期させたい。 + +**Why this priority**: 書き込み偏重ワークロードでの性能確保とコスト最適化のため。 + +**Independent Test**: 結果整合性モードで大量Put後、一定のタイムウィンドウ内に全ノードへ反映し、古い値が一定時間内に整合することを確認。 + +**Acceptance Scenarios**: + +1. **Given** 結果整合性モードでキーを書き込み、**When** 1秒以内に別ノードから読み出し、**Then** 必ずしも最新とは限らないが一定時間後（例: 数秒以内）に最新値へ収束する。 +2. **Given** ネットワーク分断後に復旧、**When** 再同期処理が走る、**Then** コンフリクトは定義済みポリシー（例: last-write-wins）で解決される。 + +--- + +### User Story 3 - モード切替と運用観測 (Priority: P2) + +オペレータは、環境やワークロードに応じて強整合性/結果整合性モードを設定単位で切り替え、状態監視と異常検知ができることを望む。 + +**Why this priority**: 運用現場での柔軟性と安全性の両立が必要なため。 + +**Independent Test**: モード設定変更後の再起動またはローリング適用で、設定が反映され、メトリクス/ログで確認できる。 + +**Acceptance Scenarios**: + +1. **Given** クラスタ設定を強整合性→結果整合性に変更、**When** ローリングで適用、**Then** 全ノードが新モードで稼働し、メトリクスにモードが反映される。 +2. **Given** モード不一致のノードが存在、**When** オペレータが状況を確認、**Then** 管理UI/CLI/ログで不一致を検知でき、是正手順が明示される。 + +### Edge Cases + +- メンバー数がクォーラムを下回った状態での書き込み要求（強整合性では拒否、結果整合性ではキューイング/部分反映）。 +- ネットワーク分断後の再結合時、双方が進んだログを持つ場合の解決順序。 +- モード切替途中に障害が発生した場合のリカバリ手順と一貫性確保。 +- データサイズやホットキー偏重時のスロットリング/バックプレッシャー挙動。 + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: システムは強整合性モードでクォーラム書き込み/読み出しを行い、コミット済みデータを即時参照可能にする。 +- **FR-002**: システムは結果整合性モードで書き込みを受け付け、定義された収束時間内に全ノードへ反映させる。 +- **FR-003**: モード設定は名前空間単位で指定でき、クラスタは複数モードを同居させられる。 +- **FR-004**: 結果整合性モードのコンフリクト解決はデフォルトで last-write-wins（LWW）を採用し、設定で他方式を選択できる。 +- **FR-005**: モード変更は安全な手順（ローリング適用または再起動）で反映され、途中失敗時はロールバック手段がある。 +- **FR-006**: 強整合性モードではクォーラム未達時に書き込みを拒否し、明示的なエラーを返す。 +- **FR-007**: 結果整合性モードではクォーラム未達時も書き込みを受け付け、後続の同期で補填し、未反映の可能性をクライアントに示せる。 +- **FR-008**: 再起動/障害復旧後、保存されたログ/スナップショットから整合した状態へ自動復元し、必要な再同期を実行する。 +- **FR-009**: モード別の観測指標（レイテンシ、未同期レプリカ数、収束時間、拒否率）をメトリクス/ログとして出力する。 +- **FR-010**: 運用者がモード状態や不一致を確認できるCLI/ログ/メトリクス情報を提供する。 + +### Key Entities + +- **ClusterConfig**: クラスタID、ノード一覧、レプリカ数、現在の整合性モード、適用ステータス。 +- **ConsistencyPolicy**: モード種別（強整合/結果整合）、コンフリクト解決ポリシー、収束目標時間、適用範囲（クラスタ/名前空間）。 +- **ReplicationState**: ノードごとのログ進行度、未同期エントリ数、最後の収束時刻、ヘルス状態。 + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: 強整合性モードでの書き込み→読み出しがクォーラム成立時に最新値を即時返し、可用ノードがクォーラム未満なら明示的に失敗を返すことが確認できる。 +- **SC-002**: 結果整合性モードでの書き込みは、許容する収束時間内（例: 数秒以内）に全レプリカへ反映し、反映遅延をメトリクスで観測できる。 +- **SC-003**: ネットワーク分断からの復旧時、コンフリクト解決ポリシーに従ってデータが一貫した状態に自動で収束することをテストで確認できる。 +- **SC-004**: モード変更操作が安全に完了し、変更後のモードと各ノードの適用状況をメトリクス/ログで確認できる。 diff --git a/specifications/flaredb/003-kvs-consistency/tasks.md b/specifications/flaredb/003-kvs-consistency/tasks.md new file mode 100644 index 0000000..bac1ee4 --- /dev/null +++ b/specifications/flaredb/003-kvs-consistency/tasks.md @@ -0,0 +1,119 @@ +--- +description: "Task list for Distributed KVS Consistency Modes" +--- + +# Tasks: Distributed KVS Consistency Modes + +**Input**: Design documents from `/specs/003-kvs-consistency/` +**Prerequisites**: plan.md (required), spec.md (user stories), research.md, data-model.md, contracts/ + +**Tests**: Required per constitution; include unit/integration tests for mode behaviors (strong/eventual), namespace config, convergence/recovery. + +**Organization**: Tasks are grouped by user story to enable independent implementation and testing. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (e.g., US1, US2, US3) +- Include exact file paths in descriptions + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Prepare config and API surfaces for namespace-level consistency modes. + +- [X] T001 Create namespace/mode config schema and defaults in `rdb-server/src/config/mod.rs` +- [X] T002 Update gRPC proto (if needed) to expose namespace/mode config endpoints in `rdb-proto/src/raft_server.proto` +- [X] T003 Add config loading/validation for namespace modes in `rdb-server/src/main.rs` + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Core plumbing for mode-aware replication and observability hooks. + +- [X] T004 Implement mode flag propagation to peers (strong/eventual per namespace) in `rdb-server/src/peer.rs` +- [X] T005 Add LWW conflict resolution helper for eventual mode in `rdb-server/src/peer.rs` +- [X] T006 Emit mode/lag/quorum metrics and structured logs in `rdb-server/src/raft_service.rs` and `rdb-server/src/peer.rs` + +**Checkpoint**: Mode flags flow through storage/peers; metrics/log hooks in place. + +--- + +## Phase 3: User Story 1 - 強整合性クラスタを安全に稼働 (Priority: P1) + +**Goal**: Quorum read/write with immediate visibility; reject writes without quorum. + +### Tests +- [X] T007 [US1] Add strong-mode integration test (quorum write/read, node failure) in `rdb-server/tests/test_consistency.rs` + +### Implementation +- [X] T008 [US1] Enforce quorum writes/reads for strong mode in `rdb-server/src/peer.rs` +- [X] T009 [US1] Return explicit errors on quorum deficit in strong mode in `rdb-server/src/raft_service.rs` + +**Checkpoint**: Strong mode test passes; quorum enforcement confirmed. + +--- + +## Phase 4: User Story 2 - 結果整合性モードで高スループット運用 (Priority: P1) + +**Goal**: Accept writes under partial availability; converge within target window using LWW. + +### Tests +- [X] T010 [US2] Add eventual-mode integration test (delayed read then convergence) in `rdb-server/tests/test_consistency.rs` +- [X] T011 [P] [US2] Add partition/recovery test with LWW resolution in `rdb-server/tests/test_consistency.rs` + +### Implementation +- [X] T012 [US2] Implement eventual-mode write acceptance with async replication in `rdb-server/src/peer.rs` +- [X] T013 [US2] Apply LWW conflict resolution on replay/sync in `rdb-server/src/peer.rs` +- [X] T014 [US2] Track and expose convergence lag metrics in `rdb-server/src/peer_manager.rs` + +**Checkpoint**: Eventual mode converges within target window; LWW conflicts resolved. + +--- + +## Phase 5: User Story 3 - モード切替と運用観測 (Priority: P2) + +**Goal**: Safe mode changes per namespace and clear observability/state reporting. + +### Tests +- [X] T015 [US3] Add mode-switch test (namespace strong↔eventual, rolling apply) in `rdb-server/tests/test_consistency.rs` +- [X] T016 [US3] Add mismatch detection test for inconsistent mode configs in `rdb-server/tests/test_consistency.rs` + +### Implementation +- [X] T017 [US3] Support mode configuration updates per namespace (reload/rolling) in `rdb-server/src/config/mod.rs` +- [X] T018 [US3] Expose mode state and mismatches via logs/metrics/optional gRPC in `rdb-server/src/raft_service.rs` +- [X] T019 [US3] Provide operator-facing quickstart/CLI instructions for mode ops in `specs/003-kvs-consistency/quickstart.md` + +**Checkpoint**: Mode switches apply safely; operators can detect/report mismatches. + +--- + +## Phase 6: Polish & Cross-Cutting Concerns + +**Purpose**: Hardening, docs, and verification scripts. + +- [X] T020 Add contract/OpenAPI updates for mode/config endpoints in `specs/003-kvs-consistency/contracts/` +- [X] T021 Add data model definitions for ClusterConfig/ConsistencyPolicy/ReplicationState in `specs/003-kvs-consistency/data-model.md` +- [X] T022 Update verification script to cover mode tests in `scripts/verify-raft.sh` +- [X] T023 Run full workspace checks (`cargo fmt`, `cargo test -p rdb-server --tests`) and document results in `specs/003-kvs-consistency/quickstart.md` + +--- + +## Dependencies & Execution Order + +- Phase 2 (Foundational) blocks all user stories. +- US1 (strong) and US2 (eventual) can proceed after foundational; US3 (mode ops) depends on config plumbing from Phases 1–2. +- Tests in each story precede implementation tasks. + +## Parallel Examples + +- T010 and T011 can run in parallel after T006 (tests for eventual mode scenarios). +- T012–T014 can run in parallel once T004–T006 are done (separate code paths for eventual replication and metrics). +- T018 and T019 can run in parallel after mode config plumbing (T017). + +## Implementation Strategy + +1. Lay config/API plumbing (Phases 1–2). +2. Deliver strong mode (US1) and eventual mode (US2) with tests. +3. Add mode switching/observability (US3). +4. Polish: contracts, data model docs, verification script, full test sweep. diff --git a/specifications/flaredb/004-multi-raft/checklists/requirements.md b/specifications/flaredb/004-multi-raft/checklists/requirements.md new file mode 100644 index 0000000..c550945 --- /dev/null +++ b/specifications/flaredb/004-multi-raft/checklists/requirements.md @@ -0,0 +1,34 @@ +# Specification Quality Checklist: Multi-Raft (Static → Split → Move) + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2024-XX-XX +**Feature**: specs/004-multi-raft/spec.md + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- Checklist reviewed; no open issues identified. diff --git a/specifications/flaredb/004-multi-raft/contracts/pd.md b/specifications/flaredb/004-multi-raft/contracts/pd.md new file mode 100644 index 0000000..da103ab --- /dev/null +++ b/specifications/flaredb/004-multi-raft/contracts/pd.md @@ -0,0 +1,36 @@ +# Contracts: PD / Placement RPCs (Multi-Raft) + +Source of truth: `rdb-proto/src/pdpb.proto` + +## Services + +- **Pd** + - `RegisterStore(RegisterStoreRequest) -> RegisterStoreResponse` + - `GetRegion(GetRegionRequest) -> GetRegionResponse` + - `ListRegions(ListRegionsRequest) -> ListRegionsResponse` + - `MoveRegion(MoveRegionRequest) -> MoveRegionResponse` + +## Messages (selected) + +- `Region`: + - `id: u64` + - `start_key: bytes` + - `end_key: bytes` (empty = infinity) + - `peers: repeated u64` (store IDs) + - `leader_id: u64` + +- `Store`: + - `id: u64` + - `addr: string` + +- `MoveRegionRequest`: + - `region_id: u64` + - `from_store: u64` + - `to_store: u64` + +## Behaviors / Expectations + +- `ListRegions` is used at bootstrap and periodic refresh to populate routing. +- `MoveRegion` directs a leader to add a replica on `to_store` (ConfChange Add) and, after catch-up, remove `from_store` (ConfChange Remove). Current implementation keeps source online; removal can be triggered separately. +- Region key ranges returned by PD must be non-overlapping; nodes validate and fail startup on overlap. +- Heartbeat: nodes periodically refresh routing via `ListRegions` (30s). A dedicated heartbeat RPC can replace this in a future phase. diff --git a/specifications/flaredb/004-multi-raft/data-model.md b/specifications/flaredb/004-multi-raft/data-model.md new file mode 100644 index 0000000..a9d8240 --- /dev/null +++ b/specifications/flaredb/004-multi-raft/data-model.md @@ -0,0 +1,45 @@ +# Data Model: Multi-Raft (Static → Split → Move) + +## Entities + +- **Store** + - `id: u64` + - `addr: String` + - Holds multiple `Peer` instances (one per `Region` replica) and reports status to PD. + +- **Region** + - `id: u64` + - `start_key: bytes` + - `end_key: bytes` (empty = infinity) + - `voters: Vec` (store IDs) + - `leader_id: u64` + - `approx_size_bytes: u64` + +- **Peer** + - `store_id: u64` + - `region_id: u64` + - `raft_state: HardState, ConfState` + - `pending_eventual: VecDeque<(ns_id, key, value, ts)>` + +- **Placement Metadata (PD)** + - `stores: [Store]` + - `regions: [Region]` + - `move_directives: [(region_id, from_store, to_store)]` + +## Relationships + +- Store 1..* Peer (per Region replica) +- Region 1..* Peer (across Stores) +- PD owns canonical Region→Store mapping and Move directives. + +## Lifecycle + +- **Bootstrap**: PD returns initial `regions` → Store creates Peers and persists meta. +- **Split**: Region exceeds threshold → Split command commits → two Region metas persisted → new Peer created. +- **Move**: PD issues `MoveRegion` → leader adds replica on target store (ConfChange Add) → replica catches up → old replica can be removed via ConfChange Remove. + +## Constraints + +- Region key ranges must be non-overlapping and sorted. +- Raft storage/logs are prefixed by `region_id` to avoid cross-region collisions. +- Quorum required for writes; ConfChange operations must preserve quorum at each step. diff --git a/specifications/flaredb/004-multi-raft/plan.md b/specifications/flaredb/004-multi-raft/plan.md new file mode 100644 index 0000000..e4e4c80 --- /dev/null +++ b/specifications/flaredb/004-multi-raft/plan.md @@ -0,0 +1,62 @@ +# Implementation Plan: Multi-Raft (Static → Split → Move) + +**Branch**: `004-multi-raft` | **Date**: 2024-XX-XX | **Spec**: specs/004-multi-raft/spec.md +**Input**: Feature specification from `/specs/004-multi-raft/spec.md` + +## Summary +- Goal: Rust/Tonic/RocksDBベースのRaft実装をMulti-Raftへ拡張し、PD配布メタに従う静的複数Region起動、閾値Split、ConfChangeによるRegion移動までを扱う。 +- Approach: StoreコンテナでRegionID→Peerを管理、Raft/KVのルータをRegion対応にリファクタ。Splitは閾値検知→Splitコマンド合意→メタ更新→新Peer登録。MoveはPD指示に基づきConfChange（追加→キャッチアップ→削除）。 + +## Technical Context +- **Language/Version**: Rust stable (toolchain per repo) +- **Primary Dependencies**: tonic/prost (gRPC), raft-rs, RocksDB, tokio +- **Storage**: RocksDB（CF/キーにRegionIDプレフィックスで分離） +- **Testing**: cargo test（unit/integration）、Raft/KV多Regionのシナリオテスト +- **Target Platform**: Linux server (Nix flake環境) +- **Project Type**: backend/server (single workspace) +- **Performance Goals**: リーダー選出≤60s、Split適用≤60s、移動完了≤5分（成功率99%以上） +- **Constraints**: 憲法に従いテスト必須・gRPCエラーは構造化ログ・互換性影響を明示 +- **Scale/Scope**: Region数: 最低複数同時稼働、将来数千を想定（バッチ最適化は後フェーズ） + +## Constitution Check +- Test-First: 新機能ごとにユニット/インテグレーションテストを先行作成。 +- Reliability & Coverage: `cargo test` 必須、複数Region・Split・ConfChangeの経路にテストを追加。 +- Simplicity: まず静的Multi-Raft→Split→Moveを段階実装。バッチ化などは後続。 +- Observability: Raft/KV/PD連携で失敗時に理由をログ。 +- Versioning: Raft/PD RPC変更は契約として明示。 +→ 憲法違反なしで進行可能。 + +## Project Structure + +### Documentation (this feature) +```text +specs/004-multi-raft/ +├── plan.md # This file +├── research.md # Phase 0 +├── data-model.md # Phase 1 +├── quickstart.md # Phase 1 +├── contracts/ # Phase 1 +└── tasks.md # Phase 2 (via /speckit.tasks) +``` + +### Source Code (repository root) +```text +rdb-server/src/ +├── main.rs # entry +├── store.rs # (new) Store/Region registry & dispatch +├── peer.rs # Raft Peer (per Region) +├── peer_manager.rs # Raft message clients +├── raft_service.rs # gRPC service (region-aware dispatch) +├── service.rs # KV service (region routing) +├── raft_storage.rs # Raft storage (Region-prefixed keys) +├── merkle.rs # (existing) sync helpers +└── config/… # namespace/mode config + +rdb-proto/src/ # proto definitions +tests/ # integration (multi-region, split, move) +``` + +**Structure Decision**: 単一バックエンド構成。Store/PeerにRegion対応を追加し、既存rdb-server配下にstore.rs等を拡張する。 + +## Complexity Tracking +- 現時点で憲法違反なしのため記載不要。 diff --git a/specifications/flaredb/004-multi-raft/quickstart.md b/specifications/flaredb/004-multi-raft/quickstart.md new file mode 100644 index 0000000..b7ac595 --- /dev/null +++ b/specifications/flaredb/004-multi-raft/quickstart.md @@ -0,0 +1,44 @@ +# Quickstart: Multi-Raft (Static → Split → Move) + +## Prerequisites +- Nix or Rust toolchain per repo. +- PD stub runs inline (tests use in-memory). + +## Run tests (recommended) +```bash +nix develop -c cargo test -q rdb-server::tests::test_multi_region +nix develop -c cargo test -q rdb-server::tests::test_split +nix develop -c cargo test -q rdb-server::tests::test_confchange_move +``` +Or full suite: +```bash +nix develop -c cargo test -q +``` + +## Manual smoke (single node, two regions) +1. Launch PD stub (or ensure `pdpb` gRPC reachable). +2. Start server: + ```bash + nix develop -c cargo run -p rdb-server -- --pd-endpoint http://127.0.0.1:50051 + ``` +3. Verify routing: + - Put key `b"a"` → Region1 + - Put key `b"z"` → Region2 + +## Trigger split (dev) +1. Run `test_split` or fill a region with writes. +2. Observe log: `ApplyCommand::Split` and new region registered. + +## Move (rebalance) flow (simplified) +1. Source store handles region; target store starts with PD meta. +2. PD issues `MoveRegion(region_id, from=src, to=dst)`. +3. Source adds replica on target (ConfChange Add); target catches up; source can later remove itself (ConfChange Remove). +4. Verify data on target: + ```bash + nix develop -c cargo test -q move_region_replica_carries_data -- --nocapture + ``` + +## Notes +- Key ranges must not overlap; nodes validate PD meta. +- Raft logs and hard-state are prefixed by `region_id` to isolate shards. +- Pending eventual writes are forwarded to leaders; local queue persists to disk to survive restart. diff --git a/specifications/flaredb/004-multi-raft/spec.md b/specifications/flaredb/004-multi-raft/spec.md new file mode 100644 index 0000000..1ea2c09 --- /dev/null +++ b/specifications/flaredb/004-multi-raft/spec.md @@ -0,0 +1,208 @@ +# Feature Specification: [FEATURE NAME] + +**Feature Branch**: `[###-feature-name]` +**Created**: [DATE] +**Status**: Draft +**Input**: User description: "$ARGUMENTS" + +## User Scenarios & Testing *(mandatory)* + + + +### User Story 1 - [Brief Title] (Priority: P1) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently - e.g., "Can be fully tested by [specific action] and delivers [specific value]"] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] +2. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 2 - [Brief Title] (Priority: P2) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 3 - [Brief Title] (Priority: P3) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +[Add more user stories as needed, each with an assigned priority] + +### Edge Cases + + + +- What happens when [boundary condition]? +- How does system handle [error scenario]? + +## Requirements *(mandatory)* + + + +### Functional Requirements + +- **FR-001**: System MUST [specific capability, e.g., "allow users to create accounts"] +- **FR-002**: System MUST [specific capability, e.g., "validate email addresses"] +- **FR-003**: Users MUST be able to [key interaction, e.g., "reset their password"] +- **FR-004**: System MUST [data requirement, e.g., "persist user preferences"] +- **FR-005**: System MUST [behavior, e.g., "log all security events"] + +*Example of marking unclear requirements:* + +- **FR-006**: System MUST authenticate users via [NEEDS CLARIFICATION: auth method not specified - email/password, SSO, OAuth?] +- **FR-007**: System MUST retain user data for [NEEDS CLARIFICATION: retention period not specified] + +### Key Entities *(include if feature involves data)* + +- **[Entity 1]**: [What it represents, key attributes without implementation] +- **[Entity 2]**: [What it represents, relationships to other entities] + +## Success Criteria *(mandatory)* + + + +### Measurable Outcomes + +- **SC-001**: [Measurable metric, e.g., "Users can complete account creation in under 2 minutes"] +- **SC-002**: [Measurable metric, e.g., "System handles 1000 concurrent users without degradation"] +- **SC-003**: [User satisfaction metric, e.g., "90% of users successfully complete primary task on first attempt"] +- **SC-004**: [Business metric, e.g., "Reduce support tickets related to [X] by 50%"] +# Feature Specification: Multi-Raft (Static → Split → Move) + +**Feature Branch**: `004-multi-raft` +**Created**: 2024-XX-XX +**Status**: Draft +**Input**: User description: "Phase 3くらいまでやる前提でお願いします。" + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - PD主導の複数Region起動 (Priority: P1) + +運用者として、起動時に外部設定を不要とし、PDが配布する初期Regionメタデータに従って複数Regionを自動起動させたい（各Regionが独立にリーダー選出・書き込みを行う）。 + +**Why this priority**: Multi-Raftの基盤となるため最重要。これがないと以降のSplitやMoveが成立しない。 +**Independent Test**: PDが返す初期Regionセット（例: 2Region）で起動し、両Regionでリーダー選出が成功し、別々のキー範囲に書き込み・読み出しできることを確認するE2Eテスト。 + +**Acceptance Scenarios**: + +1. **Given** PDが初期Regionメタ（例: Region1 `[start="", end="m")`, Region2 `[start="m", end=""]`）を返す **When** ノードを起動する **Then** 両Regionでリーダーが選出され、互いに干渉せずに書き込みできる。 +2. **Given** RaftService が region_id 付きメッセージを受信 **When** region_id に対応するPeerが存在する **Then** 正しいPeerに配送され、未登録ならエラーを返す。 + +--- + +### User Story 2 - Region Split のオンライン適用 (Priority: P1) + +運用者として、Regionサイズが閾値を超えたときに、ダウンタイムなしでSplitが実行され、新しいRegionが自動生成・登録されてほしい。 + +**Why this priority**: データ増加に伴うスケールアウトを可能にするため。 +**Independent Test**: 1 Region に大量書き込みを行い、閾値到達で Split が合意・適用され、2 Region に分割後も新旧両Regionで読み書きできることを確認。 + +**Acceptance Scenarios**: + +1. **Given** Region サイズが閾値（例: 96MB相当）に達した **When** リーダーが Split コマンドを提案・合意する **Then** 新Region が作成され、元Regionの EndKey が縮小される。 +2. **Given** Split 適用直後 **When** 分割後キー範囲に対し書き込みを行う **Then** それぞれの新旧Regionが正しく処理し、一貫性が崩れない。 + +--- + +### User Story 3 - Region 移動による負荷分散 (Priority: P2) + +運用者として、混雑しているStoreから空いているStoreへRegionを移動（レプリカ追加・除去）し、ディスク/CPU負荷を均衡化したい。 + +**Why this priority**: Phase 3でのリバランスを可能にし、スケールアウトの価値を引き出すため。 +**Independent Test**: PDが「Region X を Store A→B へ移動」指示を出し、ConfChangeでレプリカ追加→キャッチアップ→旧レプリカ除去が完了することを確認。 + +**Acceptance Scenarios**: + +1. **Given** PD が Store B へのレプリカ追加を指示 **When** リーダーが ConfChange を提案 **Then** 新レプリカが追加され、キャッチアップ後に投票権が付与される。 +2. **Given** 新レプリカがキャッチアップ **When** 旧レプリカを除去する ConfChange を適用 **Then** Region は新しい構成で継続し、クォーラムが維持される。 + +--- + +### Edge Cases + +- 未登録の region_id を含む Raft メッセージを受信した場合は安全に拒否し、ログに記録する。 +- Split 中にリーダーが交代した場合、二重Splitを防ぎ、コミット済みのSplitのみを適用する。 +- Region 移動中にネットワーク分断が発生した場合、クォーラム不足時は書き込みを拒否し、再結合後に再同期する。 +- PDが返す初期Regionメタにキー範囲の重複があった場合、起動時に検出してフェイルする。 + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: システムは PD が配布する初期Regionメタに基づき複数Regionを起動し、RegionID→Peerを Store で管理できなければならない。 +- **FR-002**: RaftService は受信メッセージの region_id に基づき適切な Peer に配送し、未登録Regionはエラーを返さなければならない。 +- **FR-003**: KvService は Key から Region を判定し、対応する Peer に提案して処理しなければならない。 +- **FR-004**: Raftログおよびハードステートは RegionID で名前空間分離され、異なる Region 間で衝突しないようにしなければならない。 +- **FR-005**: Region サイズが閾値を超えた場合、リーダーは Split コマンドを提案し、合意後に新Regionを Store に登録しなければならない。 +- **FR-006**: Split 適用時は元Regionのメタデータ (Start/EndKey) を更新し、新Regionのメタデータを生成する操作がアトミックでなければならない。 +- **FR-007**: Region の移動（レプリカ追加・除去）は Raft の ConfChange を用いて実施し、クォーラムを維持しながら完了しなければならない。 +- **FR-008**: PD は Region 配置のメタを保持し、移動/追加/除去の指示を発行し、ノードはそれを反映できなければならない。 +- **FR-009**: Region の状態 (リーダー/レプリカ/サイズ/キー範囲) は PD へハートビートで報告されなければならない。 + +### Key Entities *(include if feature involves data)* + +- **Store**: 物理ノード。RegionID→Peerの管理、Raftメッセージディスパッチ、PDへのハートビートを担う。 +- **Region**: キー範囲を持つ論理シャード。StartKey, EndKey, サイズ情報。 +- **Peer**: RegionごとのRaftレプリカ。リーダー選出・ログ複製を担当。 +- **Placement Metadata (PD)**: Region配置・サイズ・リーダー情報・バランス方針を保持するメタデータ。 + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: 2つ以上のRegionを起動した場合、各Regionでリーダー選出が60秒以内に完了する。 +- **SC-002**: Regionごとの書き込みが他Regionに混入せず、キー範囲外アクセスは100%拒否される。 +- **SC-003**: Split トリガー後、60秒以内に新Regionが登録され、分割後も書き込み成功率が99%以上を維持する。 +- **SC-004**: Region 移動（レプリカ追加→キャッチアップ→除去）が 5 分以内に完了し、移動中の書き込み成功率が99%以上を維持する。 + +## Clarifications + +### Session 2025-01-05 + +- Q: PDへの報告間隔と内容は？ → A: 30秒ごとにRegion一覧＋approx_size＋リーダー/ピア＋ヘルスをPDへ報告 diff --git a/specifications/flaredb/004-multi-raft/tasks.md b/specifications/flaredb/004-multi-raft/tasks.md new file mode 100644 index 0000000..97bf644 --- /dev/null +++ b/specifications/flaredb/004-multi-raft/tasks.md @@ -0,0 +1,125 @@ +--- +description: "Task list for Multi-Raft (Static -> Split -> Move)" +--- + +# Tasks: Multi-Raft (Static -> Split -> Move) + +**Input**: Design documents from `/specs/004-multi-raft/` +**Prerequisites**: plan.md (required), spec.md (user stories), research.md, data-model.md, contracts/ + +**Tests**: Required per constitution; include unit/integration tests for multi-region routing, split, confchange/move. + +**Organization**: Tasks are grouped by user story to enable independent implementation and testing. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (e.g., US1, US2, US3) +- Include exact file paths in descriptions + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Prepare store/container and region-aware routing foundations. + +- [X] T001 Add Store container skeleton managing RegionID->Peer map in `rdb-server/src/store.rs` +- [X] T002 Wire RaftService to dispatch by region_id via Store in `rdb-server/src/raft_service.rs` +- [X] T003 Add region-aware KV routing (Key->Region) stub in `rdb-server/src/service.rs` +- [X] T004 Region-prefixed Raft storage keys to isolate logs/hs/conf in `rdb-server/src/raft_storage.rs` +- [X] T005 Update main startup to init Store from PD initial region meta in `rdb-server/src/main.rs` + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: PD integration and routing validation. + +- [X] T006 Add PD client call to fetch initial region metadata in `rdb-proto/src/pdpb.proto` and `rdb-server/src/main.rs` +- [X] T007 Add routing cache (Region range map) with PD heartbeat refresh in `rdb-server/src/service.rs` + - [X] T008 Add multi-region Raft message dispatch tests in `rdb-server/tests/test_multi_region.rs` + - [X] T009 Add KV routing tests for disjoint regions in `rdb-server/tests/test_multi_region.rs` + +**Checkpoint**: Multiple regions can start, elect leaders, and route KV without interference. + +--- + +## Phase 3: User Story 1 - PD主導の複数Region起動 (Priority: P1) + +**Goal**: Auto-start multiple regions from PD meta; independent read/write per region. + +### Tests +- [X] T010 [US1] Integration test: startup with PD returning 2 regions; both elect leaders and accept writes in `rdb-server/tests/test_multi_region.rs` + +### Implementation +- [X] T011 [US1] Store registers peers per PD region meta; validation for overlapping ranges in `rdb-server/src/store.rs` +- [X] T012 [US1] KV service uses region router from PD meta to propose to correct peer in `rdb-server/src/service.rs` +- [X] T013 [US1] Structured errors for unknown region/key-range in `rdb-server/src/service.rs` + +**Checkpoint**: Two+ regions operate independently with PD-provided meta. + +--- + +## Phase 4: User Story 2 - Region Split (Priority: P1) + +**Goal**: Detect size threshold and split online into two regions. + +### Tests +- [X] T014 [US2] Split trigger test (approx size over threshold) in `rdb-server/tests/test_split.rs` +- [X] T015 [US2] Post-split routing test: keys before/after split_key go to correct regions in `rdb-server/tests/test_split.rs` + +### Implementation +- [X] T016 [US2] Approximate size measurement and threshold check in `rdb-server/src/store.rs` +- [X] T017 [US2] Define/apply Split raft command; update region meta atomically in `rdb-server/src/peer.rs` +- [X] T018 [US2] Create/register new peer for split region and update routing map in `rdb-server/src/store.rs` +- [X] T019 [US2] Persist updated region metadata (start/end keys) in `rdb-server/src/store.rs` + +**Checkpoint**: Region splits online; post-split read/write succeeds in both regions. + +--- + +## Phase 5: User Story 3 - Region Move (Priority: P2) + +**Goal**: Rebalance region replicas via ConfChange (add → catch-up → remove). + +### Tests +- [X] T020 [US3] ConfChange add/remove replica test across two stores in `rdb-server/tests/test_confchange.rs` +- [X] T021 [US3] Move scenario: PD directs move, data reachable after move in `rdb-server/tests/test_confchange.rs` + +### Implementation +- [X] T022 [US3] Implement ConfChange apply for add/remove node per region in `rdb-server/src/peer.rs` +- [X] T023 [US3] PD heartbeat reporting region list/size and apply PD move directives in `rdb-server/src/store.rs` +- [X] T024 [US3] Snapshot/fast catch-up path for new replica join in `rdb-server/src/peer.rs` + +**Checkpoint**: Region can move between stores without data loss; quorum maintained. + +--- + +## Phase 6: Polish & Cross-Cutting Concerns + +**Purpose**: Hardening, docs, and verification. + +- [X] T025 Update contracts for PD/Region RPCs in `specs/004-multi-raft/contracts/` +- [X] T026 Update data-model for Region/Store/PlacementMeta in `specs/004-multi-raft/data-model.md` +- [X] T027 Quickstart covering multi-region start, split, move flows in `specs/004-multi-raft/quickstart.md` +- [X] T028 Verification script to run multi-region/split/move tests in `scripts/verify-multiraft.sh` +- [ ] T029 [P] Cleanup warnings, run `cargo fmt`, `cargo test -p rdb-server --tests` across workspace + +--- + +## Dependencies & Execution Order + +- Phase 1 → Phase 2 → US1 → US2 → US3 → Polish +- Split (US2) depends on routing in US1; Move (US3) depends on ConfChange plumbing. + +## Parallel Examples + +- T008 and T009 can run in parallel after T002/T003/T004 (multi-region dispatch + routing tests). +- T014 and T015 can run in parallel after routing map is in place (post-split tests). +- T020 and T021 can run in parallel once ConfChange scaffolding exists. + +## Implementation Strategy + +1) Lay Store/routing foundations (Phase 1–2). +2) Deliver US1 (PD-driven multi-region start). +3) Add Split path (US2). +4) Add ConfChange/move path (US3). +5) Polish docs/contracts/verify script. diff --git a/specifications/flaredb/sql-layer-design.md b/specifications/flaredb/sql-layer-design.md new file mode 100644 index 0000000..4bb716d --- /dev/null +++ b/specifications/flaredb/sql-layer-design.md @@ -0,0 +1,299 @@ +# FlareDB SQL Layer Design + +## Overview + +This document outlines the design for a SQL-compatible layer built on top of FlareDB's KVS foundation. The goal is to enable SQL queries (DDL/DML) while leveraging FlareDB's existing distributed KVS capabilities. + +## Architecture Principles + +1. **KVS Foundation**: All SQL data stored as KVS key-value pairs +2. **Simple First**: Start with core SQL subset (no JOINs, no transactions initially) +3. **Efficient Encoding**: Optimize key encoding for range scans +4. **Namespace Isolation**: Use FlareDB namespaces for multi-tenancy + +## Key Design Decisions + +### 1. SQL Parser + +**Choice**: Use `sqlparser-rs` crate +- Mature, well-tested SQL parser +- Supports MySQL/PostgreSQL/ANSI SQL dialects +- Easy to extend for custom syntax + +### 2. Table Metadata Schema + +Table metadata stored in KVS with special prefix: + +``` +Key: __sql_meta:tables:{table_name} +Value: TableMetadata { + table_id: u32, + table_name: String, + columns: Vec, + primary_key: Vec, + created_at: u64, +} + +ColumnDef { + name: String, + data_type: DataType, + nullable: bool, + default_value: Option, +} + +DataType enum: + - Integer + - BigInt + - Text + - Boolean + - Timestamp +``` + +Table ID allocation: +``` +Key: __sql_meta:next_table_id +Value: u32 (monotonic counter) +``` + +### 3. Row Key Encoding + +Efficient key encoding for table rows: + +``` +Format: __sql_data:{table_id}:{primary_key_encoded} + +Example: + Table: users (table_id=1) + Primary key: id=42 + Key: __sql_data:1:42 +``` + +For composite primary keys: +``` +Format: __sql_data:{table_id}:{pk1}:{pk2}:... + +Example: + Table: order_items (table_id=2) + Primary key: (order_id=100, item_id=5) + Key: __sql_data:2:100:5 +``` + +### 4. Row Value Encoding + +Row values stored as serialized structs: + +``` +Value: RowData { + columns: HashMap, + version: u64, // For optimistic concurrency +} + +Value enum: + - Null + - Integer(i64) + - Text(String) + - Boolean(bool) + - Timestamp(u64) +``` + +Serialization: Use `bincode` for efficient binary encoding + +### 5. Query Execution Engine + +Simple query execution pipeline: + +``` +SQL String + ↓ +[Parser] + ↓ +Abstract Syntax Tree (AST) + ↓ +[Planner] + ↓ +Execution Plan + ↓ +[Executor] + ↓ +Result Set +``` + +**Supported Operations (v1):** + +DDL: +- CREATE TABLE +- DROP TABLE + +DML: +- INSERT INTO ... VALUES (...) +- SELECT * FROM table WHERE ... +- SELECT col1, col2 FROM table WHERE ... +- UPDATE table SET ... WHERE ... +- DELETE FROM table WHERE ... + +**WHERE Clause Support:** +- Simple comparisons: =, !=, <, >, <=, >= +- Logical operators: AND, OR, NOT +- Primary key lookups (optimized) +- Full table scans (for non-PK queries) + +**Query Optimization:** +- Primary key point lookups → raw_get() +- Primary key range queries → raw_scan() +- Non-indexed queries → full table scan + +### 6. API Surface + +New gRPC service: `SqlService` + +```protobuf +service SqlService { + rpc Execute(SqlRequest) returns (SqlResponse); + rpc Query(SqlRequest) returns (stream RowBatch); +} + +message SqlRequest { + string namespace = 1; + string sql = 2; +} + +message SqlResponse { + oneof result { + DdlResult ddl_result = 1; + DmlResult dml_result = 2; + QueryResult query_result = 3; + ErrorResult error = 4; + } +} + +message DdlResult { + string message = 1; // "Table created", "Table dropped" +} + +message DmlResult { + uint64 rows_affected = 1; +} + +message QueryResult { + repeated string columns = 1; + repeated Row rows = 2; +} + +message Row { + repeated Value values = 1; +} + +message Value { + oneof value { + int64 int_value = 1; + string text_value = 2; + bool bool_value = 3; + uint64 timestamp_value = 4; + } + bool is_null = 5; +} +``` + +### 7. Namespace Integration + +SQL layer respects FlareDB namespaces: +- Each namespace has isolated SQL tables +- Table IDs are namespace-scoped +- Metadata keys include namespace prefix + +``` +Key format with namespace: + {namespace_id}:__sql_meta:tables:{table_name} + {namespace_id}:__sql_data:{table_id}:{primary_key} +``` + +## Implementation Plan + +### Phase 1: Core Infrastructure (S2) +- Table metadata storage +- CREATE TABLE / DROP TABLE +- Table ID allocation + +### Phase 2: Row Storage (S3) +- Row key/value encoding +- INSERT statement +- UPDATE statement +- DELETE statement + +### Phase 3: Query Engine (S4) +- SELECT parser +- WHERE clause evaluator +- Result set builder +- Table scan implementation + +### Phase 4: Integration (S5) +- E2E tests +- Example application +- Performance benchmarks + +## Performance Considerations + +1. **Primary Key Lookups**: O(1) via raw_get() +2. **Range Scans**: O(log N) via raw_scan() with key encoding +3. **Full Table Scans**: O(N) - unavoidable without indexes +4. **Metadata Access**: Cached in memory for frequently accessed tables + +## Future Enhancements (Out of Scope) + +1. **Secondary Indexes**: Additional KVS entries for non-PK queries +2. **JOINs**: Multi-table query support +3. **Transactions**: ACID guarantees across multiple operations +4. **Query Optimizer**: Cost-based query planning +5. **SQL Standard Compliance**: More data types, functions, etc. + +## Testing Strategy + +1. **Unit Tests**: Parser, executor, encoding/decoding +2. **Integration Tests**: Full SQL operations via gRPC +3. **E2E Tests**: Real-world application scenarios +4. **Performance Tests**: Benchmark vs PostgreSQL/SQLite baseline + +## Example Usage + +```rust +// Create connection +let client = SqlServiceClient::connect("http://127.0.0.1:8001").await?; + +// Create table +client.execute(SqlRequest { + namespace: "default".to_string(), + sql: "CREATE TABLE users ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + email TEXT, + created_at TIMESTAMP + )".to_string(), +}).await?; + +// Insert data +client.execute(SqlRequest { + namespace: "default".to_string(), + sql: "INSERT INTO users (id, name, email) VALUES (1, 'Alice', 'alice@example.com')".to_string(), +}).await?; + +// Query data +let response = client.query(SqlRequest { + namespace: "default".to_string(), + sql: "SELECT * FROM users WHERE id = 1".to_string(), +}).await?; +``` + +## Success Criteria + +✓ CREATE/DROP TABLE working +✓ INSERT/UPDATE/DELETE working +✓ SELECT with WHERE clause working +✓ Primary key lookups optimized +✓ Integration tests passing +✓ Example application demonstrating CRUD + +## References + +- sqlparser-rs: https://github.com/sqlparser-rs/sqlparser-rs +- FlareDB KVS API: flaredb/proto/kvrpc.proto +- RocksDB encoding: https://github.com/facebook/rocksdb/wiki diff --git a/specifications/k8shost/S1-ipam-spec.md b/specifications/k8shost/S1-ipam-spec.md new file mode 100644 index 0000000..02cbdd1 --- /dev/null +++ b/specifications/k8shost/S1-ipam-spec.md @@ -0,0 +1,328 @@ +# T057.S1: IPAM System Design Specification + +**Author:** PeerA +**Date:** 2025-12-12 +**Status:** DRAFT + +## 1. Executive Summary + +This document specifies the IPAM (IP Address Management) system for k8shost integration with PrismNET. The design extends PrismNET's existing IPAM capabilities to support Kubernetes Service ClusterIP and LoadBalancer IP allocation. + +## 2. Current State Analysis + +### 2.1 k8shost Service IP Allocation (Current) + +**File:** `k8shost/crates/k8shost-server/src/services/service.rs:28-37` + +```rust +pub fn allocate_cluster_ip() -> String { + // Simple counter-based allocation in 10.96.0.0/16 + static COUNTER: AtomicU32 = AtomicU32::new(100); + let counter = COUNTER.fetch_add(1, Ordering::SeqCst); + format!("10.96.{}.{}", (counter >> 8) & 0xff, counter & 0xff) +} +``` + +**Issues:** +- No persistence (counter resets on restart) +- No collision detection +- No integration with network layer +- Hard-coded CIDR range + +### 2.2 PrismNET IPAM (Current) + +**File:** `prismnet/crates/prismnet-server/src/metadata.rs:577-662` + +**Capabilities:** +- CIDR parsing and IP enumeration +- Allocated IP tracking via Port resources +- Gateway IP avoidance +- Subnet-scoped allocation +- ChainFire persistence + +**Limitations:** +- Designed for VM/container ports, not K8s Services +- No dedicated Service IP subnet concept + +## 3. Architecture Design + +### 3.1 Conceptual Model + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Tenant Scope │ +│ │ +│ ┌────────────────┐ ┌────────────────┐ │ +│ │ VPC │ │ Service Subnet │ │ +│ │ (10.0.0.0/16) │ │ (10.96.0.0/16) │ │ +│ └───────┬────────┘ └───────┬─────────┘ │ +│ │ │ │ +│ ┌───────┴────────┐ ┌───────┴─────────┐ │ +│ │ Subnet │ │ Service IPs │ │ +│ │ (10.0.1.0/24) │ │ ClusterIP │ │ +│ └───────┬────────┘ │ LoadBalancerIP │ │ +│ │ └─────────────────┘ │ +│ ┌───────┴────────┐ │ +│ │ Ports (VMs) │ │ +│ └────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +### 3.2 New Resource: ServiceIPPool + +A dedicated IP pool for Kubernetes Services within a tenant. + +```rust +/// Service IP Pool for k8shost Service allocation +pub struct ServiceIPPool { + pub id: ServiceIPPoolId, + pub org_id: String, + pub project_id: String, + pub name: String, + pub cidr_block: String, // e.g., "10.96.0.0/16" + pub pool_type: ServiceIPPoolType, + pub allocated_ips: HashSet, + pub created_at: u64, + pub updated_at: u64, +} + +pub enum ServiceIPPoolType { + ClusterIP, // For ClusterIP services + LoadBalancer, // For LoadBalancer services (VIPs) + NodePort, // Reserved NodePort range +} +``` + +### 3.3 Integration Architecture + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ k8shost Server │ +│ │ +│ ┌─────────────────────┐ ┌──────────────────────┐ │ +│ │ ServiceService │─────>│ IpamClient │ │ +│ │ create_service() │ │ allocate_ip() │ │ +│ │ delete_service() │ │ release_ip() │ │ +│ └─────────────────────┘ └──────────┬───────────┘ │ +└──────────────────────────────────────────┼───────────────────────┘ + │ gRPC +┌──────────────────────────────────────────┼───────────────────────┐ +│ PrismNET Server │ │ +│ ▼ │ +│ ┌─────────────────────┐ ┌──────────────────────┐ │ +│ │ IpamService (new) │<─────│ NetworkMetadataStore│ │ +│ │ AllocateServiceIP │ │ service_ip_pools │ │ +│ │ ReleaseServiceIP │ │ allocated_ips │ │ +│ └─────────────────────┘ └──────────────────────┘ │ +└──────────────────────────────────────────────────────────────────┘ +``` + +## 4. API Design + +### 4.1 PrismNET IPAM gRPC Service + +```protobuf +service IpamService { + // Create a Service IP Pool + rpc CreateServiceIPPool(CreateServiceIPPoolRequest) + returns (CreateServiceIPPoolResponse); + + // Get Service IP Pool + rpc GetServiceIPPool(GetServiceIPPoolRequest) + returns (GetServiceIPPoolResponse); + + // List Service IP Pools + rpc ListServiceIPPools(ListServiceIPPoolsRequest) + returns (ListServiceIPPoolsResponse); + + // Allocate IP from pool + rpc AllocateServiceIP(AllocateServiceIPRequest) + returns (AllocateServiceIPResponse); + + // Release IP back to pool + rpc ReleaseServiceIP(ReleaseServiceIPRequest) + returns (ReleaseServiceIPResponse); + + // Get IP allocation status + rpc GetIPAllocation(GetIPAllocationRequest) + returns (GetIPAllocationResponse); +} + +message AllocateServiceIPRequest { + string org_id = 1; + string project_id = 2; + string pool_id = 3; // Optional: specific pool + ServiceIPPoolType pool_type = 4; // Required: ClusterIP or LoadBalancer + string service_uid = 5; // K8s service UID for tracking + string requested_ip = 6; // Optional: specific IP request +} + +message AllocateServiceIPResponse { + string ip_address = 1; + string pool_id = 2; +} +``` + +### 4.2 k8shost IpamClient + +```rust +/// IPAM client for k8shost +pub struct IpamClient { + client: IpamServiceClient, +} + +impl IpamClient { + /// Allocate ClusterIP for a Service + pub async fn allocate_cluster_ip( + &mut self, + org_id: &str, + project_id: &str, + service_uid: &str, + ) -> Result; + + /// Allocate LoadBalancer IP for a Service + pub async fn allocate_loadbalancer_ip( + &mut self, + org_id: &str, + project_id: &str, + service_uid: &str, + ) -> Result; + + /// Release an allocated IP + pub async fn release_ip( + &mut self, + org_id: &str, + project_id: &str, + ip_address: &str, + ) -> Result<()>; +} +``` + +## 5. Storage Schema + +### 5.1 ChainFire Key Structure + +``` +/prismnet/ipam/pools/{org_id}/{project_id}/{pool_id} +/prismnet/ipam/allocations/{org_id}/{project_id}/{ip_address} +``` + +### 5.2 Allocation Record + +```rust +pub struct IPAllocation { + pub ip_address: String, + pub pool_id: ServiceIPPoolId, + pub org_id: String, + pub project_id: String, + pub resource_type: String, // "k8s-service", "vm-port", etc. + pub resource_id: String, // Service UID, Port ID, etc. + pub allocated_at: u64, +} +``` + +## 6. Implementation Plan + +### Phase 1: PrismNET IPAM Service (S1 deliverable) + +1. Add `ServiceIPPool` type to prismnet-types +2. Add `IpamService` gRPC service to prismnet-api +3. Implement `IpamServiceImpl` in prismnet-server +4. Storage: pools and allocations in ChainFire + +### Phase 2: k8shost Integration (S2) + +1. Create `IpamClient` in k8shost +2. Replace `allocate_cluster_ip()` with PrismNET call +3. Add IP release on Service deletion +4. Configuration: PrismNET endpoint env var + +### Phase 3: Default Pool Provisioning + +1. Auto-create default ClusterIP pool per tenant +2. Default CIDR: `10.96.{tenant_hash}.0/20` (4096 IPs) +3. LoadBalancer pool: `192.168.{tenant_hash}.0/24` (256 IPs) + +## 7. Tenant Isolation + +### 7.1 Pool Isolation + +Each tenant (org_id + project_id) has: +- Separate ClusterIP pool +- Separate LoadBalancer pool +- Non-overlapping IP ranges + +### 7.2 IP Collision Prevention + +- IP uniqueness enforced at pool level +- CAS (Compare-And-Swap) for concurrent allocation +- ChainFire transactions for atomicity + +## 8. Default Configuration + +```yaml +# k8shost config +ipam: + enabled: true + prismnet_endpoint: "http://prismnet:9090" + + # Default pools (auto-created if missing) + default_cluster_ip_cidr: "10.96.0.0/12" # 1M IPs shared + default_loadbalancer_cidr: "192.168.0.0/16" # 64K IPs shared + + # Per-tenant allocation + cluster_ip_pool_size: "/20" # 4096 IPs per tenant + loadbalancer_pool_size: "/24" # 256 IPs per tenant +``` + +## 9. Backward Compatibility + +### 9.1 Migration Path + +1. Deploy new IPAM service in PrismNET +2. k8shost checks for IPAM availability on startup +3. If IPAM unavailable, fall back to local counter +4. Log warning for fallback mode + +### 9.2 Existing Services + +- Existing Services retain their IPs +- On next restart, k8shost syncs with IPAM +- Conflict resolution: IPAM is source of truth + +## 10. Observability + +### 10.1 Metrics + +``` +# Pool utilization +prismnet_ipam_pool_total{org_id, project_id, pool_type} +prismnet_ipam_pool_allocated{org_id, project_id, pool_type} +prismnet_ipam_pool_available{org_id, project_id, pool_type} + +# Allocation rate +prismnet_ipam_allocations_total{org_id, project_id, pool_type} +prismnet_ipam_releases_total{org_id, project_id, pool_type} +``` + +### 10.2 Alerts + +- Pool exhaustion warning at 80% utilization +- Allocation failure alerts +- Pool not found errors + +## 11. References + +- [Kubernetes Service IP allocation](https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/) +- [OpenStack Neutron IPAM](https://docs.openstack.org/neutron/latest/admin/intro-os-networking.html) +- PrismNET metadata.rs IPAM implementation + +## 12. Decision Summary + +| Aspect | Decision | Rationale | +|--------|----------|-----------| +| IPAM Location | PrismNET | Network layer owns IP management | +| Storage | ChainFire | Consistency with existing PrismNET storage | +| Pool Type | Per-tenant | Tenant isolation, quota enforcement | +| Integration | gRPC client | Consistent with other PlasmaCloud services | +| Fallback | Local counter | Backward compatibility | diff --git a/specifications/metricstor-design.md b/specifications/metricstor-design.md new file mode 100644 index 0000000..c97b29e --- /dev/null +++ b/specifications/metricstor-design.md @@ -0,0 +1,3744 @@ +# Nightlight Design Document + +**Project:** Nightlight - VictoriaMetrics OSS Replacement +**Task:** T033.S1 Research & Architecture +**Version:** 1.0 +**Date:** 2025-12-10 +**Author:** PeerB + +--- + +## Table of Contents + +1. [Executive Summary](#1-executive-summary) +2. [Requirements](#2-requirements) +3. [Time-Series Storage Model](#3-time-series-storage-model) +4. [Push Ingestion API](#4-push-ingestion-api) +5. [PromQL Query Engine](#5-promql-query-engine) +6. [Storage Backend Architecture](#6-storage-backend-architecture) +7. [Integration Points](#7-integration-points) +8. [Implementation Plan](#8-implementation-plan) +9. [Open Questions](#9-open-questions) +10. [References](#10-references) + +--- + +## 1. Executive Summary + +### 1.1 Overview + +Nightlight is a fully open-source, distributed time-series database designed as a replacement for VictoriaMetrics, addressing the critical requirement that VictoriaMetrics' mTLS support is a paid feature. As the final component (Item 12/12) of PROJECT.md, Nightlight completes the observability stack for the Japanese cloud platform. + +### 1.2 High-Level Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Service Mesh │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +│ │ FlareDB │ │ ChainFire│ │ PlasmaVMC│ │ IAM │ ... │ +│ │ :9092 │ │ :9091 │ │ :9093 │ │ :9094 │ │ +│ └────┬─────┘ └────┬─────┘ └────┬─────┘ └────┬─────┘ │ +│ │ │ │ │ │ +│ └────────────┴────────────┴────────────┘ │ +│ │ │ +│ │ Push (remote_write) │ +│ │ mTLS │ +│ ▼ │ +│ ┌──────────────────────┐ │ +│ │ Nightlight Server │ │ +│ │ ┌────────────────┐ │ │ +│ │ │ Ingestion API │ │ ← Prometheus remote_write │ +│ │ │ (gRPC/HTTP) │ │ │ +│ │ └────────┬───────┘ │ │ +│ │ │ │ │ +│ │ ┌────────▼───────┐ │ │ +│ │ │ Write Buffer │ │ │ +│ │ │ (In-Memory) │ │ │ +│ │ └────────┬───────┘ │ │ +│ │ │ │ │ +│ │ ┌────────▼───────┐ │ │ +│ │ │ Storage Engine│ │ │ +│ │ │ ┌──────────┐ │ │ │ +│ │ │ │ Head │ │ │ ← WAL + In-Memory Index │ +│ │ │ │ (Active) │ │ │ │ +│ │ │ └────┬─────┘ │ │ │ +│ │ │ │ │ │ │ +│ │ │ ┌────▼─────┐ │ │ │ +│ │ │ │ Blocks │ │ │ ← Immutable, Compressed │ +│ │ │ │ (TSDB) │ │ │ │ +│ │ │ └──────────┘ │ │ │ +│ │ └────────────────┘ │ │ +│ │ │ │ │ +│ │ ┌────────▼───────┐ │ │ +│ │ │ Query Engine │ │ ← PromQL Execution │ +│ │ │ (PromQL AST) │ │ │ +│ │ └────────┬───────┘ │ │ +│ │ │ │ │ +│ └───────────┼──────────┘ │ +│ │ │ +│ │ Query (HTTP) │ +│ │ mTLS │ +│ ▼ │ +│ ┌──────────────────────┐ │ +│ │ Grafana / Clients │ │ +│ └──────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ + + ┌─────────────────────┐ + │ FlareDB Cluster │ ← Metadata (optional) + │ (Metadata Store) │ + └─────────────────────┘ + + ┌─────────────────────┐ + │ S3-Compatible │ ← Cold Storage (future) + │ Object Storage │ + └─────────────────────┘ +``` + +### 1.3 Key Design Decisions + +1. **Storage Format**: Hybrid approach using Prometheus TSDB block design with Gorilla compression + - **Rationale**: Battle-tested, excellent compression (1-2 bytes/sample), widely understood + +2. **Storage Backend**: Dedicated time-series engine with optional FlareDB metadata integration + - **Rationale**: Time-series workloads have unique access patterns; KV stores not optimal for sample storage + - FlareDB reserved for metadata (series labels, index) in distributed scenarios + +3. **PromQL Subset**: Support 80% of common use cases (instant/range queries, basic aggregations, rate/increase) + - **Rationale**: Full PromQL compatibility is complex; focus on practical operator needs + +4. **Push Model**: Prometheus remote_write v1.0 protocol via HTTP + gRPC APIs + - **Rationale**: Standard protocol, Snappy compression built-in, client library availability + +5. **mTLS Integration**: Consistent with T027/T031 patterns (cert_file, key_file, ca_file, require_client_cert) + - **Rationale**: Unified security model across all platform services + +### 1.4 Success Criteria + +- Accept metrics from 8+ services (ports 9091-9099) via remote_write +- Query latency <100ms for instant queries (p95) +- Compression ratio ≥10:1 (target: 1.5-2 bytes/sample) +- Support 100K samples/sec write throughput per instance +- PromQL queries cover 80% of Grafana dashboard use cases +- Zero vendor lock-in (100% OSS, no paid features) + +--- + +## 2. Requirements + +### 2.1 Functional Requirements + +#### FR-1: Push-Based Metric Ingestion +- **FR-1.1**: Accept Prometheus remote_write v1.0 protocol (HTTP POST) +- **FR-1.2**: Support Snappy-compressed protobuf payloads +- **FR-1.3**: Validate metric names and labels per Prometheus naming conventions +- **FR-1.4**: Handle out-of-order samples within a configurable time window (default: 1h) +- **FR-1.5**: Deduplicate duplicate samples (same timestamp + labels) +- **FR-1.6**: Return backpressure signals (HTTP 429/503) when buffer is full + +#### FR-2: PromQL Query Engine +- **FR-2.1**: Support instant queries (`/api/v1/query`) +- **FR-2.2**: Support range queries (`/api/v1/query_range`) +- **FR-2.3**: Support label queries (`/api/v1/label//values`, `/api/v1/labels`) +- **FR-2.4**: Support series metadata queries (`/api/v1/series`) +- **FR-2.5**: Implement core PromQL functions (see Section 5.2) +- **FR-2.6**: Support Prometheus HTTP API JSON response format + +#### FR-3: Time-Series Storage +- **FR-3.1**: Store samples with millisecond timestamp precision +- **FR-3.2**: Support configurable retention periods (default: 15 days, configurable 1-365 days) +- **FR-3.3**: Automatic background compaction of blocks +- **FR-3.4**: Crash recovery via Write-Ahead Log (WAL) +- **FR-3.5**: Series cardinality limits to prevent explosion (default: 10M series) + +#### FR-4: Security & Authentication +- **FR-4.1**: mTLS support for ingestion and query APIs +- **FR-4.2**: Optional basic authentication for HTTP endpoints +- **FR-4.3**: Rate limiting per client (based on mTLS certificate CN or IP) + +#### FR-5: Operational Features +- **FR-5.1**: Prometheus-compatible `/metrics` endpoint for self-monitoring +- **FR-5.2**: Health check endpoints (`/health`, `/ready`) +- **FR-5.3**: Admin API for series deletion, compaction trigger +- **FR-5.4**: TOML configuration file support +- **FR-5.5**: Environment variable overrides + +### 2.2 Non-Functional Requirements + +#### NFR-1: Performance +- **NFR-1.1**: Ingestion throughput: ≥100K samples/sec per instance +- **NFR-1.2**: Query latency (p95): <100ms for instant queries, <500ms for range queries (1h window) +- **NFR-1.3**: Compression ratio: ≥10:1 (target: 1.5-2 bytes/sample) +- **NFR-1.4**: Memory usage: <2GB for 1M active series + +#### NFR-2: Scalability +- **NFR-2.1**: Vertical scaling: Support 10M active series per instance +- **NFR-2.2**: Horizontal scaling: Support sharding across multiple instances (future work) +- **NFR-2.3**: Storage: Support local disk + optional S3-compatible backend for cold data + +#### NFR-3: Reliability +- **NFR-3.1**: No data loss for committed samples (WAL durability) +- **NFR-3.2**: Graceful degradation under load (reject writes with backpressure, not crash) +- **NFR-3.3**: Crash recovery time: <30s for 10M series + +#### NFR-4: Maintainability +- **NFR-4.1**: Codebase consistency with other platform services (FlareDB, ChainFire patterns) +- **NFR-4.2**: 100% Rust, no CGO dependencies +- **NFR-4.3**: Comprehensive unit and integration tests +- **NFR-4.4**: Operator documentation with runbooks + +#### NFR-5: Compatibility +- **NFR-5.1**: Prometheus remote_write v1.0 protocol compatibility +- **NFR-5.2**: Prometheus HTTP API compatibility (subset: query, query_range, labels, series) +- **NFR-5.3**: Grafana data source compatibility + +### 2.3 Out of Scope (Explicitly Not Supported in v1) + +- Prometheus remote_read protocol (pull-based; platform uses push) +- Full PromQL compatibility (complex subqueries, advanced functions) +- Multi-tenancy (single-tenant per instance; use multiple instances for multi-tenant) +- Distributed query federation (single-instance queries only) +- Recording rules and alerting (use separate Prometheus/Alertmanager for this) + +--- + +## 3. Time-Series Storage Model + +### 3.1 Data Model + +#### 3.1.1 Metric Structure + +A time-series metric in Nightlight follows the Prometheus data model: + +``` +metric_name{label1="value1", label2="value2", ...} value timestamp +``` + +**Example:** +``` +http_requests_total{method="GET", status="200", service="flaredb"} 1543 1733832000000 +``` + +Components: +- **Metric Name**: Identifier for the measurement (e.g., `http_requests_total`) + - Must match regex: `[a-zA-Z_:][a-zA-Z0-9_:]*` + +- **Labels**: Key-value pairs for dimensionality (e.g., `{method="GET", status="200"}`) + - Label names: `[a-zA-Z_][a-zA-Z0-9_]*` + - Label values: Any UTF-8 string + - Reserved labels: `__name__` (stores metric name), labels starting with `__` are internal + +- **Value**: Float64 sample value + +- **Timestamp**: Millisecond precision (int64 milliseconds since Unix epoch) + +#### 3.1.2 Series Identification + +A **series** is uniquely identified by its metric name + label set: + +```rust +// Pseudo-code representation +struct SeriesID { + hash: u64, // FNV-1a hash of sorted labels +} + +struct Series { + id: SeriesID, + labels: BTreeMap, // Sorted for consistent hashing + chunks: Vec, +} +``` + +Series ID calculation: +1. Sort labels lexicographically (including `__name__` label) +2. Concatenate as: `label1_name + \0 + label1_value + \0 + label2_name + \0 + ...` +3. Compute FNV-1a 64-bit hash + +### 3.2 Storage Format + +#### 3.2.1 Architecture Overview + +Nightlight uses a **hybrid storage architecture** inspired by Prometheus TSDB and Gorilla: + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Memory Layer (Head) │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Series Map │ │ WAL Segment │ │ Write Buffer │ │ +│ │ (In-Memory │ │ (Disk) │ │ (MPSC Queue) │ │ +│ │ Index) │ │ │ │ │ │ +│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ +│ │ │ │ │ +│ └─────────────────┴─────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────┐ │ +│ │ Active Chunks │ │ +│ │ (Gorilla-compressed) │ │ +│ │ - 2h time windows │ │ +│ │ - Delta-of-delta TS │ │ +│ │ - XOR float encoding │ │ +│ └─────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ + │ + │ Compaction Trigger + │ (every 2h or on shutdown) + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ Disk Layer (Blocks) │ +│ │ +│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ +│ │ Block 1 │ │ Block 2 │ │ Block N │ │ +│ │ [0h - 2h) │ │ [2h - 4h) │ │ [Nh - (N+2)h) │ │ +│ │ │ │ │ │ │ │ +│ │ ├─ meta.json │ │ ├─ meta.json │ │ ├─ meta.json │ │ +│ │ ├─ index │ │ ├─ index │ │ ├─ index │ │ +│ │ ├─ chunks/000 │ │ ├─ chunks/000 │ │ ├─ chunks/000 │ │ +│ │ └─ tombstones │ │ └─ tombstones │ │ └─ tombstones │ │ +│ └─────────────────┘ └─────────────────┘ └─────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +#### 3.2.2 Write-Ahead Log (WAL) + +**Purpose**: Durability and crash recovery + +**Format**: Append-only log segments (128MB default size) + +``` +WAL Structure: +data/ + wal/ + 00000001 ← Segment 1 (128MB) + 00000002 ← Segment 2 (active) +``` + +**WAL Record Format** (inspired by LevelDB): + +``` +┌───────────────────────────────────────────────────┐ +│ CRC32 (4 bytes) │ +├───────────────────────────────────────────────────┤ +│ Length (4 bytes, little-endian) │ +├───────────────────────────────────────────────────┤ +│ Type (1 byte): FULL | FIRST | MIDDLE | LAST │ +├───────────────────────────────────────────────────┤ +│ Payload (variable): │ +│ - Record Type (1 byte): Series | Samples │ +│ - Series ID (8 bytes) │ +│ - Labels (length-prefixed strings) │ +│ - Samples (varint timestamp, float64 value) │ +└───────────────────────────────────────────────────┘ +``` + +**WAL Operations**: +- **Append**: Every write appends to active segment +- **Checkpoint**: Snapshot of in-memory state to disk blocks +- **Truncate**: Delete segments older than oldest in-memory data +- **Replay**: On startup, replay WAL segments to rebuild in-memory state + +**Rust Implementation Sketch**: + +```rust +struct WAL { + dir: PathBuf, + segment_size: usize, // 128MB default + active_segment: File, + active_segment_num: u64, +} + +impl WAL { + fn append(&mut self, record: &WALRecord) -> Result<()> { + let encoded = record.encode(); + let crc = crc32(&encoded); + + // Rotate segment if needed + if self.active_segment.metadata()?.len() + encoded.len() > self.segment_size { + self.rotate_segment()?; + } + + self.active_segment.write_all(&crc.to_le_bytes())?; + self.active_segment.write_all(&(encoded.len() as u32).to_le_bytes())?; + self.active_segment.write_all(&encoded)?; + self.active_segment.sync_all()?; // fsync for durability + Ok(()) + } + + fn replay(&self) -> Result> { + // Read all segments and decode records + // Used on startup for crash recovery + } +} +``` + +#### 3.2.3 In-Memory Head Block + +**Purpose**: Accept recent writes, maintain hot data for fast queries + +**Structure**: + +```rust +struct Head { + series: RwLock>>, + min_time: AtomicI64, + max_time: AtomicI64, + chunk_size: Duration, // 2h default + wal: Arc, +} + +struct Series { + id: SeriesID, + labels: BTreeMap, + chunks: RwLock>, +} + +struct Chunk { + min_time: i64, + max_time: i64, + samples: CompressedSamples, // Gorilla encoding +} +``` + +**Chunk Lifecycle**: +1. **Creation**: New chunk created when first sample arrives or previous chunk is full +2. **Active**: Chunk accepts samples in time window [min_time, min_time + 2h) +3. **Full**: Chunk reaches 2h window, new chunk created for subsequent samples +4. **Compaction**: Full chunks compacted to disk blocks + +**Memory Limits**: +- Max series: 10M (configurable) +- Max chunks per series: 2 (active + previous, covering 4h) +- Eviction: LRU eviction of inactive series (no samples in 4h) + +#### 3.2.4 Disk Blocks (Immutable) + +**Purpose**: Long-term storage of compacted time-series data + +**Block Structure** (inspired by Prometheus TSDB): + +``` +data/ + 01HQZQZQZQZQZQZQZQZQZQ/ ← Block directory (ULID) + meta.json ← Metadata + index ← Inverted index + chunks/ + 000001 ← Chunk file + 000002 + ... + tombstones ← Deleted series/samples +``` + +**meta.json Format**: + +```json +{ + "ulid": "01HQZQZQZQZQZQZQZQZQZQ", + "minTime": 1733832000000, + "maxTime": 1733839200000, + "stats": { + "numSamples": 1500000, + "numSeries": 5000, + "numChunks": 10000 + }, + "compaction": { + "level": 1, + "sources": ["01HQZQZ..."] + }, + "version": 1 +} +``` + +**Index File Format** (simplified): + +The index file provides fast lookups of series by labels. + +``` +┌────────────────────────────────────────────────┐ +│ Magic Number (4 bytes): 0xBADA55A0 │ +├────────────────────────────────────────────────┤ +│ Version (1 byte): 1 │ +├────────────────────────────────────────────────┤ +│ Symbol Table Section │ +│ - Sorted strings (label names/values) │ +│ - Offset table for binary search │ +├────────────────────────────────────────────────┤ +│ Series Section │ +│ - SeriesID → Chunk Refs mapping │ +│ - (series_id, labels, chunk_offsets) │ +├────────────────────────────────────────────────┤ +│ Label Index Section (Inverted Index) │ +│ - label_name → [series_ids] │ +│ - (label_name, label_value) → [series_ids] │ +├────────────────────────────────────────────────┤ +│ Postings Section │ +│ - Sorted posting lists for label matchers │ +│ - Compressed with varint + bit packing │ +├────────────────────────────────────────────────┤ +│ TOC (Table of Contents) │ +│ - Offsets to each section │ +└────────────────────────────────────────────────┘ +``` + +**Chunks File Format**: + +``` +Chunk File (chunks/000001): +┌────────────────────────────────────────────────┐ +│ Chunk 1: │ +│ ├─ Length (4 bytes) │ +│ ├─ Encoding (1 byte): Gorilla = 0x01 │ +│ ├─ MinTime (8 bytes) │ +│ ├─ MaxTime (8 bytes) │ +│ ├─ NumSamples (4 bytes) │ +│ └─ Compressed Data (variable) │ +├────────────────────────────────────────────────┤ +│ Chunk 2: ... │ +└────────────────────────────────────────────────┘ +``` + +### 3.3 Compression Strategy + +#### 3.3.1 Gorilla Compression Algorithm + +Nightlight uses **Gorilla compression** from Facebook's paper (VLDB 2015), achieving ~12x compression. + +**Timestamp Compression (Delta-of-Delta)**: + +``` +Example timestamps (ms): + t0 = 1733832000000 + t1 = 1733832015000 (Δ1 = 15000) + t2 = 1733832030000 (Δ2 = 15000) + t3 = 1733832045000 (Δ3 = 15000) + +Delta-of-delta: + D1 = Δ1 - Δ0 = 15000 - 0 = 15000 → encode in 14 bits + D2 = Δ2 - Δ1 = 15000 - 15000 = 0 → encode in 1 bit (0) + D3 = Δ3 - Δ2 = 15000 - 15000 = 0 → encode in 1 bit (0) + +Encoding: + - If D = 0: write 1 bit "0" + - If D in [-63, 64): write "10" + 7 bits + - If D in [-255, 256): write "110" + 9 bits + - If D in [-2047, 2048): write "1110" + 12 bits + - Otherwise: write "1111" + 32 bits + +96% of timestamps compress to 1 bit! +``` + +**Value Compression (XOR Encoding)**: + +``` +Example values (float64): + v0 = 1543.0 + v1 = 1543.5 + v2 = 1543.7 + +XOR compression: + XOR(v0, v1) = 0x3FF0000000000000 XOR 0x3FF0800000000000 + = 0x0000800000000000 + → Leading zeros: 16, Trailing zeros: 47 + → Encode: control bit "1" + 5-bit leading + 6-bit length + 1 bit + + XOR(v1, v2) = 0x3FF0800000000000 XOR 0x3FF0CCCCCCCCCCD + → Similar pattern, encode with control bits + +Encoding: + - If v_i == v_(i-1): write 1 bit "0" + - If XOR has same leading/trailing zeros as previous: write "10" + significant bits + - Otherwise: write "11" + 5-bit leading + 6-bit length + significant bits + +51% of values compress to 1 bit! +``` + +**Rust Implementation Sketch**: + +```rust +struct GorillaEncoder { + bit_writer: BitWriter, + prev_timestamp: i64, + prev_delta: i64, + prev_value: f64, + prev_leading_zeros: u8, + prev_trailing_zeros: u8, +} + +impl GorillaEncoder { + fn encode_timestamp(&mut self, timestamp: i64) -> Result<()> { + let delta = timestamp - self.prev_timestamp; + let delta_of_delta = delta - self.prev_delta; + + if delta_of_delta == 0 { + self.bit_writer.write_bit(0)?; + } else if delta_of_delta >= -63 && delta_of_delta < 64 { + self.bit_writer.write_bits(0b10, 2)?; + self.bit_writer.write_bits(delta_of_delta as u64, 7)?; + } else if delta_of_delta >= -255 && delta_of_delta < 256 { + self.bit_writer.write_bits(0b110, 3)?; + self.bit_writer.write_bits(delta_of_delta as u64, 9)?; + } else if delta_of_delta >= -2047 && delta_of_delta < 2048 { + self.bit_writer.write_bits(0b1110, 4)?; + self.bit_writer.write_bits(delta_of_delta as u64, 12)?; + } else { + self.bit_writer.write_bits(0b1111, 4)?; + self.bit_writer.write_bits(delta_of_delta as u64, 32)?; + } + + self.prev_timestamp = timestamp; + self.prev_delta = delta; + Ok(()) + } + + fn encode_value(&mut self, value: f64) -> Result<()> { + let bits = value.to_bits(); + let xor = bits ^ self.prev_value.to_bits(); + + if xor == 0 { + self.bit_writer.write_bit(0)?; + } else { + let leading = xor.leading_zeros() as u8; + let trailing = xor.trailing_zeros() as u8; + let significant_bits = 64 - leading - trailing; + + if leading >= self.prev_leading_zeros && trailing >= self.prev_trailing_zeros { + self.bit_writer.write_bits(0b10, 2)?; + let mask = (1u64 << significant_bits) - 1; + let significant = (xor >> trailing) & mask; + self.bit_writer.write_bits(significant, significant_bits as usize)?; + } else { + self.bit_writer.write_bits(0b11, 2)?; + self.bit_writer.write_bits(leading as u64, 5)?; + self.bit_writer.write_bits(significant_bits as u64, 6)?; + let mask = (1u64 << significant_bits) - 1; + let significant = (xor >> trailing) & mask; + self.bit_writer.write_bits(significant, significant_bits as usize)?; + + self.prev_leading_zeros = leading; + self.prev_trailing_zeros = trailing; + } + } + + self.prev_value = value; + Ok(()) + } +} +``` + +#### 3.3.2 Compression Performance Targets + +Based on research and production systems: + +| Metric | Target | Reference | +|--------|--------|-----------| +| Average bytes/sample | 1.5-2.0 | Prometheus (1-2), Gorilla (1.37), M3DB (1.45) | +| Compression ratio | 10-12x | Gorilla (12x), InfluxDB TSM (45x for specific workloads) | +| Encode throughput | >500K samples/sec | Gorilla paper: 700K/sec | +| Decode throughput | >1M samples/sec | Gorilla paper: 1.2M/sec | + +### 3.4 Retention and Compaction Policies + +#### 3.4.1 Retention Policy + +**Default Retention**: 15 days + +**Configurable Parameters**: +```toml +[storage] +retention_days = 15 # Keep data for 15 days +min_block_duration = "2h" # Minimum block size +max_block_duration = "24h" # Maximum block size after compaction +``` + +**Retention Enforcement**: +- Background goroutine runs every 1h +- Deletes blocks where `max_time < now() - retention_duration` +- Deletes old WAL segments + +#### 3.4.2 Compaction Strategy + +**Purpose**: +1. Merge small blocks into larger blocks (reduce file count) +2. Remove deleted samples (tombstones) +3. Improve query performance (fewer blocks to scan) + +**Compaction Levels** (inspired by LevelDB): + +``` +Level 0: 2h blocks (compacted from Head) +Level 1: 12h blocks (merge 6 L0 blocks) +Level 2: 24h blocks (merge 2 L1 blocks) +``` + +**Compaction Trigger**: +- **Time-based**: Every 2h, compact Head → Level 0 block +- **Count-based**: When L0 has >4 blocks, compact → L1 +- **Manual**: Admin API endpoint `/api/v1/admin/compact` + +**Compaction Algorithm**: + +``` +1. Select blocks to compact (same level, adjacent time ranges) +2. Create new block directory (ULID) +3. Iterate all series in selected blocks: + a. Merge chunks from all blocks + b. Apply tombstones (skip deleted samples) + c. Re-compress merged chunks + d. Write to new block chunks file +4. Build new index (merge posting lists) +5. Write meta.json +6. Atomically rename block directory +7. Delete source blocks +``` + +**Rust Implementation Sketch**: + +```rust +struct Compactor { + data_dir: PathBuf, + retention: Duration, +} + +impl Compactor { + async fn compact_head_to_l0(&self, head: &Head) -> Result { + let block_id = ULID::new(); + let block_dir = self.data_dir.join(block_id.to_string()); + std::fs::create_dir_all(&block_dir)?; + + let mut index_writer = IndexWriter::new(&block_dir.join("index"))?; + let mut chunk_writer = ChunkWriter::new(&block_dir.join("chunks/000001"))?; + + let series_map = head.series.read().await; + for (series_id, series) in series_map.iter() { + let chunks = series.chunks.read().await; + for chunk in chunks.iter() { + if chunk.is_full() { + let chunk_ref = chunk_writer.write_chunk(&chunk.samples)?; + index_writer.add_series(*series_id, &series.labels, chunk_ref)?; + } + } + } + + index_writer.finalize()?; + chunk_writer.finalize()?; + + let meta = BlockMeta { + ulid: block_id, + min_time: head.min_time.load(Ordering::Relaxed), + max_time: head.max_time.load(Ordering::Relaxed), + stats: compute_stats(&block_dir)?, + compaction: CompactionMeta { level: 0, sources: vec![] }, + version: 1, + }; + write_meta(&block_dir.join("meta.json"), &meta)?; + + Ok(block_id) + } + + async fn compact_blocks(&self, source_blocks: Vec) -> Result { + // Merge multiple blocks into one + // Similar to compact_head_to_l0, but reads from existing blocks + } + + async fn enforce_retention(&self) -> Result<()> { + let cutoff = SystemTime::now() - self.retention; + let cutoff_ms = cutoff.duration_since(UNIX_EPOCH)?.as_millis() as i64; + + for entry in std::fs::read_dir(&self.data_dir)? { + let path = entry?.path(); + if !path.is_dir() { continue; } + + let meta_path = path.join("meta.json"); + if !meta_path.exists() { continue; } + + let meta: BlockMeta = serde_json::from_reader(File::open(meta_path)?)?; + if meta.max_time < cutoff_ms { + std::fs::remove_dir_all(&path)?; + info!("Deleted expired block: {}", meta.ulid); + } + } + Ok(()) + } +} +``` + +--- + +## 4. Push Ingestion API + +### 4.1 Prometheus Remote Write Protocol + +#### 4.1.1 Protocol Overview + +**Specification**: Prometheus Remote Write v1.0 +**Transport**: HTTP/1.1 or HTTP/2 +**Encoding**: Protocol Buffers (protobuf v3) +**Compression**: Snappy (required) + +**Reference**: [Prometheus Remote Write Spec](https://prometheus.io/docs/specs/prw/remote_write_spec/) + +#### 4.1.2 HTTP Endpoint + +``` +POST /api/v1/write +Content-Type: application/x-protobuf +Content-Encoding: snappy +X-Prometheus-Remote-Write-Version: 0.1.0 +``` + +**Request Flow**: + +``` +┌──────────────┐ +│ Client │ +│ (Prometheus, │ +│ FlareDB, │ +│ etc.) │ +└──────┬───────┘ + │ + │ 1. Collect samples + │ + ▼ +┌──────────────────────────────────┐ +│ Encode to WriteRequest protobuf │ +│ message │ +└──────┬───────────────────────────┘ + │ + │ 2. Compress with Snappy + │ + ▼ +┌──────────────────────────────────┐ +│ HTTP POST to /api/v1/write │ +│ with mTLS authentication │ +└──────┬───────────────────────────┘ + │ + │ 3. Send request + │ + ▼ +┌──────────────────────────────────┐ +│ Nightlight Server │ +│ ├─ Validate mTLS cert │ +│ ├─ Decompress Snappy │ +│ ├─ Decode protobuf │ +│ ├─ Validate samples │ +│ ├─ Append to WAL │ +│ └─ Insert into Head │ +└──────┬───────────────────────────┘ + │ + │ 4. Response + │ + ▼ +┌──────────────────────────────────┐ +│ HTTP Response: │ +│ 200 OK (success) │ +│ 400 Bad Request (invalid) │ +│ 429 Too Many Requests (backpressure) │ +│ 503 Service Unavailable (overload) │ +└──────────────────────────────────┘ +``` + +#### 4.1.3 Protobuf Schema + +**File**: `proto/remote_write.proto` + +```protobuf +syntax = "proto3"; + +package nightlight.remote; + +// Prometheus remote_write compatible schema + +message WriteRequest { + repeated TimeSeries timeseries = 1; + // Metadata is optional and not used in v1 + repeated MetricMetadata metadata = 2; +} + +message TimeSeries { + repeated Label labels = 1; + repeated Sample samples = 2; + // Exemplars are optional (not supported in v1) + repeated Exemplar exemplars = 3; +} + +message Label { + string name = 1; + string value = 2; +} + +message Sample { + double value = 1; + int64 timestamp = 2; // Unix timestamp in milliseconds +} + +message Exemplar { + repeated Label labels = 1; + double value = 2; + int64 timestamp = 3; +} + +message MetricMetadata { + enum MetricType { + UNKNOWN = 0; + COUNTER = 1; + GAUGE = 2; + HISTOGRAM = 3; + GAUGEHISTOGRAM = 4; + SUMMARY = 5; + INFO = 6; + STATESET = 7; + } + MetricType type = 1; + string metric_family_name = 2; + string help = 3; + string unit = 4; +} +``` + +**Generated Rust Code** (using `prost`): + +```toml +# Cargo.toml +[dependencies] +prost = "0.12" +prost-types = "0.12" + +[build-dependencies] +prost-build = "0.12" +``` + +```rust +// build.rs +fn main() { + prost_build::compile_protos(&["proto/remote_write.proto"], &["proto/"]).unwrap(); +} +``` + +#### 4.1.4 Ingestion Handler + +**Rust Implementation**: + +```rust +use axum::{ + Router, + routing::post, + extract::State, + http::StatusCode, + body::Bytes, +}; +use prost::Message; +use snap::raw::Decoder as SnappyDecoder; + +mod remote_write_pb { + include!(concat!(env!("OUT_DIR"), "/nightlight.remote.rs")); +} + +struct IngestionService { + head: Arc, + wal: Arc, + rate_limiter: Arc, +} + +async fn handle_remote_write( + State(service): State>, + body: Bytes, +) -> Result { + // 1. Decompress Snappy + let mut decoder = SnappyDecoder::new(); + let decompressed = decoder + .decompress_vec(&body) + .map_err(|e| (StatusCode::BAD_REQUEST, format!("Snappy decompression failed: {}", e)))?; + + // 2. Decode protobuf + let write_req = remote_write_pb::WriteRequest::decode(&decompressed[..]) + .map_err(|e| (StatusCode::BAD_REQUEST, format!("Protobuf decode failed: {}", e)))?; + + // 3. Validate and ingest + let mut samples_ingested = 0; + let mut samples_rejected = 0; + + for ts in write_req.timeseries.iter() { + // Validate labels + let labels = validate_labels(&ts.labels) + .map_err(|e| (StatusCode::BAD_REQUEST, e))?; + + let series_id = compute_series_id(&labels); + + for sample in ts.samples.iter() { + // Validate timestamp (not too old, not too far in future) + if !is_valid_timestamp(sample.timestamp) { + samples_rejected += 1; + continue; + } + + // Check rate limit + if !service.rate_limiter.allow() { + return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".into())); + } + + // Append to WAL + let wal_record = WALRecord::Sample { + series_id, + timestamp: sample.timestamp, + value: sample.value, + }; + service.wal.append(&wal_record) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("WAL append failed: {}", e)))?; + + // Insert into Head + service.head.append(series_id, labels.clone(), sample.timestamp, sample.value) + .await + .map_err(|e| { + if e.to_string().contains("out of order") { + samples_rejected += 1; + Ok::<_, (StatusCode, String)>(()) + } else if e.to_string().contains("buffer full") { + Err((StatusCode::SERVICE_UNAVAILABLE, "Write buffer full".into())) + } else { + Err((StatusCode::INTERNAL_SERVER_ERROR, format!("Insert failed: {}", e))) + } + })?; + + samples_ingested += 1; + } + } + + info!("Ingested {} samples, rejected {}", samples_ingested, samples_rejected); + Ok(StatusCode::NO_CONTENT) // 204 No Content on success +} + +fn validate_labels(labels: &[remote_write_pb::Label]) -> Result, String> { + let mut label_map = BTreeMap::new(); + + for label in labels { + // Validate label name + if !is_valid_label_name(&label.name) { + return Err(format!("Invalid label name: {}", label.name)); + } + + // Validate label value (any UTF-8) + if label.value.is_empty() { + return Err(format!("Empty label value for label: {}", label.name)); + } + + label_map.insert(label.name.clone(), label.value.clone()); + } + + // Must have __name__ label + if !label_map.contains_key("__name__") { + return Err("Missing __name__ label".into()); + } + + Ok(label_map) +} + +fn is_valid_label_name(name: &str) -> bool { + // Must match [a-zA-Z_][a-zA-Z0-9_]* + if name.is_empty() { + return false; + } + + let mut chars = name.chars(); + let first = chars.next().unwrap(); + if !first.is_ascii_alphabetic() && first != '_' { + return false; + } + + chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +fn is_valid_timestamp(ts: i64) -> bool { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as i64; + let min_valid = now - 24 * 3600 * 1000; // Not older than 24h + let max_valid = now + 5 * 60 * 1000; // Not more than 5min in future + ts >= min_valid && ts <= max_valid +} +``` + +### 4.2 gRPC API (Alternative/Additional) + +In addition to HTTP, Nightlight MAY support a gRPC API for ingestion (more efficient for internal services). + +**Proto Definition**: + +```protobuf +syntax = "proto3"; + +package nightlight.ingest; + +service IngestionService { + rpc Write(WriteRequest) returns (WriteResponse); + rpc WriteBatch(stream WriteRequest) returns (WriteResponse); +} + +message WriteRequest { + repeated TimeSeries timeseries = 1; +} + +message WriteResponse { + uint64 samples_ingested = 1; + uint64 samples_rejected = 2; + string error = 3; +} + +// (Reuse TimeSeries, Label, Sample from remote_write.proto) +``` + +### 4.3 Label Validation and Normalization + +#### 4.3.1 Metric Name Validation + +Metric names (stored in `__name__` label) must match: +``` +[a-zA-Z_:][a-zA-Z0-9_:]* +``` + +Examples: +- ✅ `http_requests_total` +- ✅ `node_cpu_seconds:rate5m` +- ❌ `123_invalid` (starts with digit) +- ❌ `invalid-metric` (contains hyphen) + +#### 4.3.2 Label Name Validation + +Label names must match: +``` +[a-zA-Z_][a-zA-Z0-9_]* +``` + +Reserved prefixes: +- `__` (double underscore): Internal labels (e.g., `__name__`, `__rollup__`) + +#### 4.3.3 Label Normalization + +Before inserting, labels are normalized: +1. Sort labels lexicographically by key +2. Ensure `__name__` label is present +3. Remove duplicate labels (keep last value) +4. Limit label count (default: 30 labels max per series) +5. Limit label value length (default: 1024 chars max) + +### 4.4 Write Path Architecture + +``` +┌──────────────────────────────────────────────────────────────┐ +│ Ingestion Layer │ +│ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ HTTP/gRPC │ │ mTLS Auth │ │ Rate Limiter│ │ +│ │ Handler │─▶│ Validator │─▶│ │ │ +│ └─────────────┘ └─────────────┘ └──────┬──────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Decompressor │ │ +│ │ (Snappy) │ │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Protobuf │ │ +│ │ Decoder │ │ +│ └────────┬────────┘ │ +│ │ │ +└───────────────────────────────────────────┼──────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Validation Layer │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Label │ │ Timestamp │ │ Cardinality │ │ +│ │ Validator │ │ Validator │ │ Limiter │ │ +│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ +│ │ │ │ │ +│ └─────────────────┴─────────────────┘ │ +│ │ │ +└───────────────────────────┼──────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Write Buffer │ +│ │ +│ ┌────────────────────────────────────────────────────┐ │ +│ │ MPSC Channel (bounded) │ │ +│ │ Capacity: 100K samples │ │ +│ │ Backpressure: Block/Reject when full │ │ +│ └────────────────────────────────────────────────────┘ │ +│ │ │ +└───────────────────────────┼──────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Storage Layer │ +│ │ +│ ┌─────────────┐ ┌─────────────┐ │ +│ │ WAL │◀────────│ WAL Writer │ │ +│ │ (Disk) │ │ (Thread) │ │ +│ └─────────────┘ └─────────────┘ │ +│ │ +│ ┌─────────────┐ ┌─────────────┐ │ +│ │ Head │◀────────│ Head Writer│ │ +│ │ (In-Memory) │ │ (Thread) │ │ +│ └─────────────┘ └─────────────┘ │ +└──────────────────────────────────────────────────────────────┘ +``` + +**Concurrency Model**: + +1. **HTTP/gRPC handlers**: Multi-threaded (tokio async) +2. **Write buffer**: MPSC channel (bounded capacity) +3. **WAL writer**: Single-threaded (sequential writes for consistency) +4. **Head writer**: Single-threaded (lock-free inserts via sharding) + +**Backpressure Handling**: + +```rust +enum BackpressureStrategy { + Block, // Block until buffer has space (default) + Reject, // Return 503 immediately +} + +impl IngestionService { + async fn handle_backpressure(&self, samples: Vec) -> Result<()> { + match self.config.backpressure_strategy { + BackpressureStrategy::Block => { + // Try to send with timeout + tokio::time::timeout( + Duration::from_secs(5), + self.write_buffer.send(samples) + ).await + .map_err(|_| Error::Timeout)? + } + BackpressureStrategy::Reject => { + // Try non-blocking send + self.write_buffer.try_send(samples) + .map_err(|_| Error::BufferFull)? + } + } + } +} +``` + +### 4.5 Out-of-Order Sample Handling + +**Problem**: Samples may arrive out of timestamp order due to network delays, batching, etc. + +**Solution**: Accept out-of-order samples within a configurable time window. + +**Configuration**: +```toml +[storage] +out_of_order_time_window = "1h" # Accept samples up to 1h old +``` + +**Implementation**: + +```rust +impl Head { + async fn append( + &self, + series_id: SeriesID, + labels: BTreeMap, + timestamp: i64, + value: f64, + ) -> Result<()> { + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis() as i64; + let min_valid_ts = now - self.config.out_of_order_time_window.as_millis() as i64; + + if timestamp < min_valid_ts { + return Err(Error::OutOfOrder(format!( + "Sample too old: ts={}, min={}", + timestamp, min_valid_ts + ))); + } + + // Get or create series + let mut series_map = self.series.write().await; + let series = series_map.entry(series_id).or_insert_with(|| { + Arc::new(Series { + id: series_id, + labels: labels.clone(), + chunks: RwLock::new(vec![]), + }) + }); + + // Append to appropriate chunk + let mut chunks = series.chunks.write().await; + + // Find chunk that covers this timestamp + let chunk = chunks.iter_mut() + .find(|c| timestamp >= c.min_time && timestamp < c.max_time) + .or_else(|| { + // Create new chunk if needed + let chunk_start = (timestamp / self.chunk_size.as_millis() as i64) * self.chunk_size.as_millis() as i64; + let chunk_end = chunk_start + self.chunk_size.as_millis() as i64; + let new_chunk = Chunk { + min_time: chunk_start, + max_time: chunk_end, + samples: CompressedSamples::new(), + }; + chunks.push(new_chunk); + chunks.last_mut() + }) + .unwrap(); + + chunk.samples.append(timestamp, value)?; + + Ok(()) + } +} +``` + +--- + +## 5. PromQL Query Engine + +### 5.1 PromQL Overview + +**PromQL** (Prometheus Query Language) is a functional query language for selecting and aggregating time-series data. + +**Query Types**: +1. **Instant query**: Evaluate expression at a single point in time +2. **Range query**: Evaluate expression over a time range + +### 5.2 Supported PromQL Subset + +Nightlight v1 supports a **pragmatic subset** of PromQL covering 80% of common dashboard queries. + +#### 5.2.1 Instant Vector Selectors + +```promql +# Select by metric name +http_requests_total + +# Select with label matchers +http_requests_total{method="GET"} +http_requests_total{method="GET", status="200"} + +# Label matcher operators +metric{label="value"} # Exact match +metric{label!="value"} # Not equal +metric{label=~"regex"} # Regex match +metric{label!~"regex"} # Regex not match + +# Example +http_requests_total{method=~"GET|POST", status!="500"} +``` + +#### 5.2.2 Range Vector Selectors + +```promql +# Select last 5 minutes of data +http_requests_total[5m] + +# With label matchers +http_requests_total{method="GET"}[1h] + +# Time durations: s (seconds), m (minutes), h (hours), d (days), w (weeks), y (years) +``` + +#### 5.2.3 Aggregation Operators + +```promql +# sum: Sum over dimensions +sum(http_requests_total) +sum(http_requests_total) by (method) +sum(http_requests_total) without (instance) + +# Supported aggregations: +sum # Sum +avg # Average +min # Minimum +max # Maximum +count # Count +stddev # Standard deviation +stdvar # Standard variance +topk(N, ) # Top N series by value +bottomk(N,) # Bottom N series by value +``` + +#### 5.2.4 Functions + +**Rate Functions**: +```promql +# rate: Per-second average rate of increase +rate(http_requests_total[5m]) + +# irate: Instant rate (last two samples) +irate(http_requests_total[5m]) + +# increase: Total increase over time range +increase(http_requests_total[1h]) +``` + +**Quantile Functions**: +```promql +# histogram_quantile: Calculate quantile from histogram +histogram_quantile(0.95, rate(http_request_duration_bucket[5m])) +``` + +**Time Functions**: +```promql +# time(): Current Unix timestamp +time() + +# timestamp(): Timestamp of sample +timestamp(metric) +``` + +**Math Functions**: +```promql +# abs, ceil, floor, round, sqrt, exp, ln, log2, log10 +abs(metric) +round(metric, 0.1) +``` + +#### 5.2.5 Binary Operators + +**Arithmetic**: +```promql +metric1 + metric2 +metric1 - metric2 +metric1 * metric2 +metric1 / metric2 +metric1 % metric2 +metric1 ^ metric2 +``` + +**Comparison**: +```promql +metric1 == metric2 # Equal +metric1 != metric2 # Not equal +metric1 > metric2 # Greater than +metric1 < metric2 # Less than +metric1 >= metric2 # Greater or equal +metric1 <= metric2 # Less or equal +``` + +**Logical**: +```promql +metric1 and metric2 # Intersection +metric1 or metric2 # Union +metric1 unless metric2 # Complement +``` + +**Vector Matching**: +```promql +# One-to-one matching +metric1 + metric2 + +# Many-to-one matching +metric1 + on(label) group_left metric2 + +# One-to-many matching +metric1 + on(label) group_right metric2 +``` + +#### 5.2.6 Subqueries (NOT SUPPORTED in v1) + +Subqueries are complex and not supported in v1: +```promql +# NOT SUPPORTED +max_over_time(rate(http_requests_total[5m])[1h:]) +``` + +### 5.3 Query Execution Model + +#### 5.3.1 Query Parsing + +Use **promql-parser** crate (GreptimeTeam) for parsing: + +```rust +use promql_parser::{parser, label}; + +fn parse_query(query: &str) -> Result { + parser::parse(query) +} + +// Example +let expr = parse_query("http_requests_total{method=\"GET\"}[5m]")?; +match expr { + parser::Expr::VectorSelector(vs) => { + println!("Metric: {}", vs.name); + for matcher in vs.matchers.matchers { + println!("Label: {} {} {}", matcher.name, matcher.op, matcher.value); + } + println!("Range: {:?}", vs.range); + } + _ => {} +} +``` + +**AST Types**: + +```rust +pub enum Expr { + Aggregate(AggregateExpr), // sum, avg, etc. + Unary(UnaryExpr), // -metric + Binary(BinaryExpr), // metric1 + metric2 + Paren(ParenExpr), // (expr) + Subquery(SubqueryExpr), // NOT SUPPORTED + NumberLiteral(NumberLiteral), // 1.5 + StringLiteral(StringLiteral), // "value" + VectorSelector(VectorSelector), // metric{labels} + MatrixSelector(MatrixSelector), // metric[5m] + Call(Call), // rate(...) +} +``` + +#### 5.3.2 Query Planner + +Convert AST to execution plan: + +```rust +enum QueryPlan { + VectorSelector { + matchers: Vec, + timestamp: i64, + }, + MatrixSelector { + matchers: Vec, + range: Duration, + timestamp: i64, + }, + Aggregate { + op: AggregateOp, + input: Box, + grouping: Vec, + }, + RateFunc { + input: Box, + }, + BinaryOp { + op: BinaryOp, + lhs: Box, + rhs: Box, + matching: VectorMatching, + }, +} + +struct QueryPlanner; + +impl QueryPlanner { + fn plan(expr: parser::Expr, query_time: i64) -> Result { + match expr { + parser::Expr::VectorSelector(vs) => { + Ok(QueryPlan::VectorSelector { + matchers: vs.matchers.matchers.into_iter() + .map(|m| LabelMatcher::from_ast(m)) + .collect(), + timestamp: query_time, + }) + } + parser::Expr::MatrixSelector(ms) => { + Ok(QueryPlan::MatrixSelector { + matchers: ms.vector_selector.matchers.matchers.into_iter() + .map(|m| LabelMatcher::from_ast(m)) + .collect(), + range: Duration::from_millis(ms.range as u64), + timestamp: query_time, + }) + } + parser::Expr::Call(call) => { + match call.func.name.as_str() { + "rate" => { + let arg_plan = Self::plan(*call.args[0].clone(), query_time)?; + Ok(QueryPlan::RateFunc { input: Box::new(arg_plan) }) + } + // ... other functions + _ => Err(Error::UnsupportedFunction(call.func.name)), + } + } + parser::Expr::Aggregate(agg) => { + let input_plan = Self::plan(*agg.expr, query_time)?; + Ok(QueryPlan::Aggregate { + op: AggregateOp::from_str(&agg.op.to_string())?, + input: Box::new(input_plan), + grouping: agg.grouping.unwrap_or_default(), + }) + } + parser::Expr::Binary(bin) => { + let lhs_plan = Self::plan(*bin.lhs, query_time)?; + let rhs_plan = Self::plan(*bin.rhs, query_time)?; + Ok(QueryPlan::BinaryOp { + op: BinaryOp::from_str(&bin.op.to_string())?, + lhs: Box::new(lhs_plan), + rhs: Box::new(rhs_plan), + matching: bin.modifier.map(|m| VectorMatching::from_ast(m)).unwrap_or_default(), + }) + } + _ => Err(Error::UnsupportedExpr), + } + } +} +``` + +#### 5.3.3 Query Executor + +Execute the plan: + +```rust +struct QueryExecutor { + head: Arc, + blocks: Arc, +} + +impl QueryExecutor { + async fn execute(&self, plan: QueryPlan) -> Result { + match plan { + QueryPlan::VectorSelector { matchers, timestamp } => { + self.execute_vector_selector(matchers, timestamp).await + } + QueryPlan::MatrixSelector { matchers, range, timestamp } => { + self.execute_matrix_selector(matchers, range, timestamp).await + } + QueryPlan::RateFunc { input } => { + let matrix = self.execute(*input).await?; + self.apply_rate(matrix) + } + QueryPlan::Aggregate { op, input, grouping } => { + let vector = self.execute(*input).await?; + self.apply_aggregate(op, vector, grouping) + } + QueryPlan::BinaryOp { op, lhs, rhs, matching } => { + let lhs_result = self.execute(*lhs).await?; + let rhs_result = self.execute(*rhs).await?; + self.apply_binary_op(op, lhs_result, rhs_result, matching) + } + } + } + + async fn execute_vector_selector( + &self, + matchers: Vec, + timestamp: i64, + ) -> Result { + // 1. Find matching series from index + let series_ids = self.find_series(&matchers).await?; + + // 2. For each series, get sample at timestamp + let mut samples = Vec::new(); + for series_id in series_ids { + if let Some(sample) = self.get_sample_at(series_id, timestamp).await? { + samples.push(sample); + } + } + + Ok(InstantVector { samples }) + } + + async fn execute_matrix_selector( + &self, + matchers: Vec, + range: Duration, + timestamp: i64, + ) -> Result { + let series_ids = self.find_series(&matchers).await?; + + let start = timestamp - range.as_millis() as i64; + let end = timestamp; + + let mut ranges = Vec::new(); + for series_id in series_ids { + let samples = self.get_samples_range(series_id, start, end).await?; + ranges.push(RangeVectorSeries { + labels: self.get_labels(series_id).await?, + samples, + }); + } + + Ok(RangeVector { ranges }) + } + + fn apply_rate(&self, matrix: RangeVector) -> Result { + let mut samples = Vec::new(); + + for range in matrix.ranges { + if range.samples.len() < 2 { + continue; // Need at least 2 samples for rate + } + + let first = &range.samples[0]; + let last = &range.samples[range.samples.len() - 1]; + + let delta_value = last.value - first.value; + let delta_time = (last.timestamp - first.timestamp) as f64 / 1000.0; // Convert to seconds + + let rate = delta_value / delta_time; + + samples.push(Sample { + labels: range.labels, + timestamp: last.timestamp, + value: rate, + }); + } + + Ok(InstantVector { samples }) + } + + fn apply_aggregate( + &self, + op: AggregateOp, + vector: InstantVector, + grouping: Vec, + ) -> Result { + // Group samples by grouping labels + let mut groups: HashMap, Vec> = HashMap::new(); + + for sample in vector.samples { + let group_key = if grouping.is_empty() { + vec![] + } else { + grouping.iter() + .filter_map(|label| sample.labels.get(label).map(|v| (label.clone(), v.clone()))) + .collect() + }; + + groups.entry(group_key).or_insert_with(Vec::new).push(sample); + } + + // Apply aggregation to each group + let mut result_samples = Vec::new(); + for (group_labels, samples) in groups { + let aggregated_value = match op { + AggregateOp::Sum => samples.iter().map(|s| s.value).sum(), + AggregateOp::Avg => samples.iter().map(|s| s.value).sum::() / samples.len() as f64, + AggregateOp::Min => samples.iter().map(|s| s.value).fold(f64::INFINITY, f64::min), + AggregateOp::Max => samples.iter().map(|s| s.value).fold(f64::NEG_INFINITY, f64::max), + AggregateOp::Count => samples.len() as f64, + // ... other aggregations + }; + + result_samples.push(Sample { + labels: group_labels.into_iter().collect(), + timestamp: samples[0].timestamp, + value: aggregated_value, + }); + } + + Ok(InstantVector { samples: result_samples }) + } +} +``` + +### 5.4 Read Path Architecture + +``` +┌──────────────────────────────────────────────────────────────┐ +│ Query Layer │ +│ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ HTTP API │ │ PromQL │ │ Query │ │ +│ │ /api/v1/ │─▶│ Parser │─▶│ Planner │ │ +│ │ query │ │ │ │ │ │ +│ └─────────────┘ └─────────────┘ └──────┬──────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Query │ │ +│ │ Executor │ │ +│ └────────┬────────┘ │ +└───────────────────────────────────────────┼──────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Index Layer │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ Label Index │ │ Posting │ │ +│ │ (In-Memory) │ │ Lists │ │ +│ └──────┬───────┘ └──────┬───────┘ │ +│ │ │ │ +│ └─────────────────┘ │ +│ │ │ +│ │ Series IDs │ +│ ▼ │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Storage Layer │ +│ │ +│ ┌─────────────┐ ┌─────────────┐ │ +│ │ Head │ │ Blocks │ │ +│ │ (In-Memory) │ │ (Disk) │ │ +│ └─────┬───────┘ └─────┬───────┘ │ +│ │ │ │ +│ │ Recent data (<2h) │ Historical data │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌─────────────────────────────────────┐ │ +│ │ Chunk Reader │ │ +│ │ - Decompress Gorilla chunks │ │ +│ │ - Filter by time range │ │ +│ │ - Return samples │ │ +│ └─────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────┘ +``` + +### 5.5 HTTP Query API + +#### 5.5.1 Instant Query + +``` +GET /api/v1/query?query=&time=&timeout= +``` + +**Parameters**: +- `query`: PromQL expression (required) +- `time`: Unix timestamp (optional, default: now) +- `timeout`: Query timeout (optional, default: 30s) + +**Response** (JSON): + +```json +{ + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "__name__": "http_requests_total", + "method": "GET", + "status": "200" + }, + "value": [1733832000, "1543"] + } + ] + } +} +``` + +#### 5.5.2 Range Query + +``` +GET /api/v1/query_range?query=&start=&end=&step= +``` + +**Parameters**: +- `query`: PromQL expression (required) +- `start`: Start timestamp (required) +- `end`: End timestamp (required) +- `step`: Query resolution step (required, e.g., "15s") + +**Response** (JSON): + +```json +{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "http_requests_total", + "method": "GET" + }, + "values": [ + [1733832000, "1543"], + [1733832015, "1556"], + [1733832030, "1570"] + ] + } + ] + } +} +``` + +#### 5.5.3 Label Values Query + +``` +GET /api/v1/label//values?match[]= +``` + +**Example**: +``` +GET /api/v1/label/method/values?match[]=http_requests_total +``` + +**Response**: +```json +{ + "status": "success", + "data": ["GET", "POST", "PUT", "DELETE"] +} +``` + +#### 5.5.4 Series Metadata Query + +``` +GET /api/v1/series?match[]=&start=&end= +``` + +**Example**: +``` +GET /api/v1/series?match[]=http_requests_total{method="GET"} +``` + +**Response**: +```json +{ + "status": "success", + "data": [ + { + "__name__": "http_requests_total", + "method": "GET", + "status": "200", + "instance": "flaredb-1:9092" + } + ] +} +``` + +### 5.6 Performance Optimizations + +#### 5.6.1 Query Caching + +Cache query results for identical queries: + +```rust +struct QueryCache { + cache: Arc>>, + ttl: Duration, +} + +impl QueryCache { + fn get(&self, query_hash: &str) -> Option { + let cache = self.cache.lock().unwrap(); + if let Some((result, timestamp)) = cache.get(query_hash) { + if timestamp.elapsed() < self.ttl { + return Some(result.clone()); + } + } + None + } + + fn put(&self, query_hash: String, result: QueryResult) { + let mut cache = self.cache.lock().unwrap(); + cache.put(query_hash, (result, Instant::now())); + } +} +``` + +#### 5.6.2 Posting List Intersection + +Use efficient algorithms for label matcher intersection: + +```rust +fn intersect_posting_lists(lists: Vec<&[SeriesID]>) -> Vec { + if lists.is_empty() { + return vec![]; + } + + // Sort lists by length (shortest first for early termination) + let mut sorted_lists = lists; + sorted_lists.sort_by_key(|list| list.len()); + + // Use shortest list as base, intersect with others + let mut result: HashSet = sorted_lists[0].iter().copied().collect(); + + for list in &sorted_lists[1..] { + let list_set: HashSet = list.iter().copied().collect(); + result.retain(|id| list_set.contains(id)); + + if result.is_empty() { + break; // Early termination + } + } + + result.into_iter().collect() +} +``` + +#### 5.6.3 Chunk Pruning + +Skip chunks that don't overlap query time range: + +```rust +fn query_chunks( + chunks: &[ChunkRef], + start_time: i64, + end_time: i64, +) -> Vec { + chunks.iter() + .filter(|chunk| { + // Chunk overlaps query range if: + // chunk.max_time > start AND chunk.min_time < end + chunk.max_time > start_time && chunk.min_time < end_time + }) + .copied() + .collect() +} +``` + +--- + +## 6. Storage Backend Architecture + +### 6.1 Architecture Decision: Hybrid Approach + +After analyzing trade-offs, Nightlight adopts a **hybrid storage architecture**: + +1. **Dedicated time-series engine** for sample storage (optimized for write throughput and compression) +2. **Optional FlareDB integration** for metadata and distributed coordination (future work) +3. **Optional S3-compatible backend** for cold data archival (future work) + +### 6.2 Decision Rationale + +#### 6.2.1 Why NOT Pure FlareDB Backend? + +**FlareDB Characteristics**: +- General-purpose KV store with Raft consensus +- Optimized for: Strong consistency, small KV pairs, random access +- Storage: RocksDB (LSM tree) + +**Time-Series Workload Characteristics**: +- High write throughput (100K samples/sec) +- Sequential writes (append-only) +- Temporal locality (queries focus on recent data) +- Bulk reads (range scans over time windows) + +**Mismatch Analysis**: + +| Aspect | FlareDB (KV) | Time-Series Engine | +|--------|--------------|-------------------| +| Write pattern | Random writes, compaction overhead | Append-only, minimal overhead | +| Compression | Generic LZ4/Snappy | Domain-specific (Gorilla: 12x) | +| Read pattern | Point lookups | Range scans over time | +| Indexing | Key-based | Label-based inverted index | +| Consistency | Strong (Raft) | Eventual OK for metrics | + +**Conclusion**: Using FlareDB for sample storage would sacrifice 5-10x write throughput and 10x compression efficiency. + +#### 6.2.2 Why NOT VictoriaMetrics Binary? + +VictoriaMetrics is written in Go and has excellent performance, but: +- mTLS support is **paid only** (violates PROJECT.md requirement) +- Not Rust (violates PROJECT.md "Rustで書く") +- Cannot integrate with FlareDB for metadata (future requirement) +- Less control over storage format and optimizations + +#### 6.2.3 Why Hybrid (Dedicated + Optional FlareDB)? + +**Phase 1 (T033 v1)**: Pure dedicated engine +- Simple, single-instance deployment +- Focus on core functionality (ingest + query) +- Local disk storage only + +**Phase 2 (Future)**: Add FlareDB for metadata +- Store series labels and metadata in FlareDB "metrics" namespace +- Enables multi-instance coordination +- Global view of series cardinality, label values +- Samples still in dedicated engine (local disk) + +**Phase 3 (Future)**: Add S3 for cold storage +- Automatically upload old blocks (>7 days) to S3 +- Query federation across local + S3 blocks +- Unlimited retention with cost-effective storage + +**Benefits**: +- v1 simplicity: No FlareDB dependency, easy deployment +- Future scalability: Metadata in FlareDB, samples distributed +- Operational flexibility: Can run standalone or integrated + +### 6.3 Storage Layout + +#### 6.3.1 Directory Structure + +``` +/var/lib/nightlight/ +├── data/ +│ ├── wal/ +│ │ ├── 00000001 # WAL segment +│ │ ├── 00000002 +│ │ └── checkpoint.00000002 # WAL checkpoint +│ ├── 01HQZQZQZQZQZQZQZQZQZQ/ # Block (ULID) +│ │ ├── meta.json +│ │ ├── index +│ │ ├── chunks/ +│ │ │ ├── 000001 +│ │ │ └── 000002 +│ │ └── tombstones +│ ├── 01HQZR.../ # Another block +│ └── ... +└── tmp/ # Temp files for compaction +``` + +#### 6.3.2 Metadata Storage (Future: FlareDB Integration) + +When FlareDB integration is enabled: + +**Series Metadata** (stored in FlareDB "metrics" namespace): + +``` +Key: series: +Value: { + "labels": {"__name__": "http_requests_total", "method": "GET", ...}, + "first_seen": 1733832000000, + "last_seen": 1733839200000 +} + +Key: label_index:: +Value: [series_id1, series_id2, ...] # Posting list +``` + +**Benefits**: +- Fast label value lookups across all instances +- Global series cardinality tracking +- Distributed query planning (future) + +**Trade-off**: Adds dependency on FlareDB, increases complexity + +### 6.4 Scalability Approach + +#### 6.4.1 Vertical Scaling (v1) + +Single instance scales to: +- 10M active series +- 100K samples/sec write throughput +- 1K queries/sec + +**Scaling strategy**: +- Increase memory (more series in Head) +- Faster disk (NVMe for WAL/blocks) +- More CPU cores (parallel compaction, query execution) + +#### 6.4.2 Horizontal Scaling (Future) + +**Sharding Strategy** (inspired by Prometheus federation + Thanos): + +``` +┌────────────────────────────────────────────────────────────┐ +│ Query Frontend │ +│ (Query Federation) │ +└─────┬────────────────────┬─────────────────────┬───────────┘ + │ │ │ + ▼ ▼ ▼ +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ Nightlight │ │ Nightlight │ │ Nightlight │ +│ Instance 1 │ │ Instance 2 │ │ Instance N │ +│ │ │ │ │ │ +│ Hash shard: │ │ Hash shard: │ │ Hash shard: │ +│ 0-333 │ │ 334-666 │ │ 667-999 │ +└─────────────┘ └─────────────┘ └─────────────┘ + │ │ │ + └────────────────────┴─────────────────────┘ + │ + ▼ + ┌───────────────┐ + │ FlareDB │ + │ (Metadata) │ + └───────────────┘ +``` + +**Sharding Key**: Hash(series_id) % num_shards + +**Query Execution**: +1. Query frontend receives PromQL query +2. Determine which shards contain matching series (via FlareDB metadata) +3. Send subqueries to relevant shards +4. Merge results (aggregation, deduplication) +5. Return to client + +**Challenges** (deferred to future work): +- Rebalancing when adding/removing shards +- Handling series that span multiple shards (rare) +- Ensuring query consistency across shards + +### 6.5 S3 Integration Strategy (Future) + +**Objective**: Cost-effective long-term retention (>15 days) + +**Architecture**: + +``` +┌───────────────────────────────────────────────────┐ +│ Nightlight Server │ +│ │ +│ ┌──────────┐ ┌──────────┐ │ +│ │ Head │ │ Blocks │ │ +│ │ (0-2h) │ │ (2h-15d)│ │ +│ └──────────┘ └────┬─────┘ │ +│ │ │ +│ │ Background uploader │ +│ ▼ │ +│ ┌─────────────┐ │ +│ │ Upload to │ │ +│ │ S3 (>7d) │ │ +│ └──────┬──────┘ │ +└──────────────────────────┼────────────────────────┘ + │ + ▼ + ┌─────────────────┐ + │ S3 Bucket │ + │ /blocks/ │ + │ 01HQZ.../ │ + │ 01HRZ.../ │ + └─────────────────┘ +``` + +**Workflow**: +1. Block compaction creates local block files +2. Blocks older than 7 days (configurable) are uploaded to S3 +3. Local block files deleted after successful upload +4. Query executor checks both local and S3 for blocks in query range +5. Download S3 blocks on-demand (with local cache) + +**Configuration**: +```toml +[storage.s3] +enabled = true +endpoint = "https://s3.example.com" +bucket = "nightlight-blocks" +access_key_id = "..." +secret_access_key = "..." +upload_after_days = 7 +local_cache_size_gb = 100 +``` + +--- + +## 7. Integration Points + +### 7.1 Service Discovery (How Services Push Metrics) + +#### 7.1.1 Service Configuration Pattern + +Each platform service (FlareDB, ChainFire, etc.) exports Prometheus metrics on ports 9091-9099. + +**Example** (FlareDB metrics exporter): + +```rust +// flaredb-server/src/main.rs +use metrics_exporter_prometheus::PrometheusBuilder; + +#[tokio::main] +async fn main() -> Result<()> { + // ... initialization ... + + let metrics_addr = format!("0.0.0.0:{}", args.metrics_port); + let builder = PrometheusBuilder::new(); + builder + .with_http_listener(metrics_addr.parse::()?) + .install() + .expect("Failed to install Prometheus metrics exporter"); + + info!("Prometheus metrics available at http://{}/metrics", metrics_addr); + + // ... rest of main ... +} +``` + +**Service Metrics Ports** (from T027.S2): + +| Service | Port | Endpoint | +|---------|------|----------| +| ChainFire | 9091 | http://chainfire:9091/metrics | +| FlareDB | 9092 | http://flaredb:9092/metrics | +| PlasmaVMC | 9093 | http://plasmavmc:9093/metrics | +| IAM | 9094 | http://iam:9094/metrics | +| LightningSTOR | 9095 | http://lightningstor:9095/metrics | +| FlashDNS | 9096 | http://flashdns:9096/metrics | +| FiberLB | 9097 | http://fiberlb:9097/metrics | +| Prismnet | 9098 | http://prismnet:9098/metrics | + +#### 7.1.2 Scrape-to-Push Adapter + +Since Nightlight is **push-based** but services export **pull-based** Prometheus `/metrics` endpoints, we need a scrape-to-push adapter. + +**Option 1**: Prometheus Agent Mode + Remote Write + +Deploy Prometheus in agent mode (no storage, only scraping): + +```yaml +# prometheus-agent.yaml +global: + scrape_interval: 15s + external_labels: + cluster: 'cloud-platform' + +scrape_configs: + - job_name: 'chainfire' + static_configs: + - targets: ['chainfire:9091'] + + - job_name: 'flaredb' + static_configs: + - targets: ['flaredb:9092'] + + # ... other services ... + +remote_write: + - url: 'https://nightlight:8080/api/v1/write' + tls_config: + cert_file: /etc/certs/client.crt + key_file: /etc/certs/client.key + ca_file: /etc/certs/ca.crt +``` + +**Option 2**: Custom Rust Scraper (Platform-Native) + +Build a lightweight scraper in Rust that integrates with Nightlight: + +```rust +// nightlight-scraper/src/main.rs + +struct Scraper { + targets: Vec, + client: reqwest::Client, + nightlight_client: NightlightClient, +} + +struct ScrapeTarget { + job_name: String, + url: String, + interval: Duration, +} + +impl Scraper { + async fn scrape_loop(&self) { + loop { + for target in &self.targets { + let result = self.scrape_target(target).await; + match result { + Ok(samples) => { + if let Err(e) = self.nightlight_client.write(samples).await { + error!("Failed to write to Nightlight: {}", e); + } + } + Err(e) => { + error!("Failed to scrape {}: {}", target.url, e); + } + } + } + tokio::time::sleep(Duration::from_secs(15)).await; + } + } + + async fn scrape_target(&self, target: &ScrapeTarget) -> Result> { + let response = self.client.get(&target.url).send().await?; + let body = response.text().await?; + + // Parse Prometheus text format + let samples = parse_prometheus_text(&body, &target.job_name)?; + Ok(samples) + } +} + +fn parse_prometheus_text(text: &str, job: &str) -> Result> { + // Use prometheus-parse crate or implement simple parser + // Example output: + // http_requests_total{method="GET",status="200",job="flaredb"} 1543 1733832000000 +} +``` + +**Deployment**: +- `nightlight-scraper` runs as a sidecar or separate service +- Reads scrape config from TOML file +- Uses mTLS to push to Nightlight + +**Recommendation**: Option 2 (custom scraper) for consistency with platform philosophy (100% Rust, no external dependencies). + +### 7.2 mTLS Configuration (T027/T031 Patterns) + +#### 7.2.1 TLS Config Structure + +Following existing patterns (FlareDB, ChainFire, IAM): + +```toml +# nightlight.toml + +[server] +addr = "0.0.0.0:8080" +log_level = "info" + +[server.tls] +cert_file = "/etc/nightlight/certs/server.crt" +key_file = "/etc/nightlight/certs/server.key" +ca_file = "/etc/nightlight/certs/ca.crt" +require_client_cert = true # Enable mTLS +``` + +**Rust Config Struct**: + +```rust +// nightlight-server/src/config.rs + +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ServerConfig { + pub server: ServerSettings, + pub storage: StorageConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ServerSettings { + pub addr: SocketAddr, + pub log_level: String, + pub tls: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TlsConfig { + pub cert_file: String, + pub key_file: String, + pub ca_file: Option, + #[serde(default)] + pub require_client_cert: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StorageConfig { + pub data_dir: String, + pub retention_days: u32, + pub wal_segment_size_mb: usize, + // ... other storage settings +} +``` + +#### 7.2.2 mTLS Server Setup + +```rust +// nightlight-server/src/main.rs + +use axum::Router; +use axum_server::tls_rustls::RustlsConfig; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<()> { + let config = ServerConfig::load("nightlight.toml")?; + + // Build router + let app = Router::new() + .route("/api/v1/write", post(handle_remote_write)) + .route("/api/v1/query", get(handle_instant_query)) + .route("/api/v1/query_range", get(handle_range_query)) + .route("/health", get(health_check)) + .route("/ready", get(readiness_check)) + .with_state(Arc::new(service)); + + // Setup TLS if configured + if let Some(tls_config) = &config.server.tls { + info!("TLS enabled, loading certificates..."); + + let rustls_config = if tls_config.require_client_cert { + info!("mTLS enabled, requiring client certificates"); + + let ca_cert_pem = tokio::fs::read_to_string( + tls_config.ca_file.as_ref().ok_or("ca_file required for mTLS")? + ).await?; + + RustlsConfig::from_pem_file( + &tls_config.cert_file, + &tls_config.key_file, + ) + .await? + .with_client_cert_verifier(ca_cert_pem) + } else { + info!("TLS-only mode, client certificates not required"); + RustlsConfig::from_pem_file( + &tls_config.cert_file, + &tls_config.key_file, + ).await? + }; + + axum_server::bind_rustls(config.server.addr, rustls_config) + .serve(app.into_make_service()) + .await?; + } else { + info!("TLS disabled, running in plain-text mode"); + axum_server::bind(config.server.addr) + .serve(app.into_make_service()) + .await?; + } + + Ok(()) +} +``` + +#### 7.2.3 Client Certificate Validation + +Extract client identity from mTLS certificate: + +```rust +use axum::{ + http::Request, + middleware::Next, + response::Response, + Extension, +}; +use axum_server::tls_rustls::RustlsAcceptor; + +#[derive(Clone, Debug)] +struct ClientIdentity { + common_name: String, + organization: String, +} + +async fn extract_client_identity( + Extension(client_cert): Extension>, + mut request: Request, + next: Next, +) -> Response { + if let Some(cert) = client_cert { + // Parse certificate to extract CN, O, etc. + let identity = parse_certificate(&cert); + request.extensions_mut().insert(identity); + } + + next.run(request).await +} + +// Use identity for rate limiting, audit logging, etc. +async fn handle_remote_write( + Extension(identity): Extension, + State(service): State>, + body: Bytes, +) -> Result { + info!("Write request from: {}", identity.common_name); + + // Apply per-client rate limiting + if !service.rate_limiter.allow(&identity.common_name) { + return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".into())); + } + + // ... rest of handler ... +} +``` + +### 7.3 gRPC API Design + +While HTTP is the primary interface (Prometheus compatibility), a gRPC API can provide: +- Better performance for internal services +- Streaming support for batch ingestion +- Type-safe client libraries + +**Proto Definition**: + +```protobuf +// proto/nightlight.proto + +syntax = "proto3"; + +package nightlight.v1; + +service NightlightService { + // Write samples + rpc Write(WriteRequest) returns (WriteResponse); + + // Streaming write for high-throughput scenarios + rpc WriteStream(stream WriteRequest) returns (WriteResponse); + + // Query (instant) + rpc Query(QueryRequest) returns (QueryResponse); + + // Query (range) + rpc QueryRange(QueryRangeRequest) returns (QueryRangeResponse); + + // Admin operations + rpc Compact(CompactRequest) returns (CompactResponse); + rpc DeleteSeries(DeleteSeriesRequest) returns (DeleteSeriesResponse); +} + +message WriteRequest { + repeated TimeSeries timeseries = 1; +} + +message WriteResponse { + uint64 samples_ingested = 1; + uint64 samples_rejected = 2; +} + +message QueryRequest { + string query = 1; // PromQL + int64 time = 2; // Unix timestamp (ms) + int64 timeout_ms = 3; +} + +message QueryResponse { + string result_type = 1; // "vector" or "matrix" + repeated InstantVectorSample vector = 2; + repeated RangeVectorSeries matrix = 3; +} + +message InstantVectorSample { + map labels = 1; + double value = 2; + int64 timestamp = 3; +} + +message RangeVectorSeries { + map labels = 1; + repeated Sample samples = 2; +} + +message Sample { + double value = 1; + int64 timestamp = 2; +} +``` + +### 7.4 NixOS Module Integration + +Following T024 patterns, create a NixOS module for Nightlight. + +**File**: `nix/modules/nightlight.nix` + +```nix +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.nightlight; + + configFile = pkgs.writeText "nightlight.toml" '' + [server] + addr = "${cfg.listenAddress}" + log_level = "${cfg.logLevel}" + + ${optionalString (cfg.tls.enable) '' + [server.tls] + cert_file = "${cfg.tls.certFile}" + key_file = "${cfg.tls.keyFile}" + ${optionalString (cfg.tls.caFile != null) '' + ca_file = "${cfg.tls.caFile}" + ''} + require_client_cert = ${boolToString cfg.tls.requireClientCert} + ''} + + [storage] + data_dir = "${cfg.dataDir}" + retention_days = ${toString cfg.storage.retentionDays} + wal_segment_size_mb = ${toString cfg.storage.walSegmentSizeMb} + ''; + +in { + options.services.nightlight = { + enable = mkEnableOption "Nightlight metrics storage service"; + + package = mkOption { + type = types.package; + default = pkgs.nightlight; + description = "Nightlight package to use"; + }; + + listenAddress = mkOption { + type = types.str; + default = "0.0.0.0:8080"; + description = "Address and port to listen on"; + }; + + logLevel = mkOption { + type = types.enum [ "trace" "debug" "info" "warn" "error" ]; + default = "info"; + description = "Log level"; + }; + + dataDir = mkOption { + type = types.path; + default = "/var/lib/nightlight"; + description = "Data directory for TSDB storage"; + }; + + tls = { + enable = mkEnableOption "TLS encryption"; + + certFile = mkOption { + type = types.str; + description = "Path to TLS certificate file"; + }; + + keyFile = mkOption { + type = types.str; + description = "Path to TLS private key file"; + }; + + caFile = mkOption { + type = types.nullOr types.str; + default = null; + description = "Path to CA certificate for client verification (mTLS)"; + }; + + requireClientCert = mkOption { + type = types.bool; + default = false; + description = "Require client certificates (mTLS)"; + }; + }; + + storage = { + retentionDays = mkOption { + type = types.ints.positive; + default = 15; + description = "Data retention period in days"; + }; + + walSegmentSizeMb = mkOption { + type = types.ints.positive; + default = 128; + description = "WAL segment size in MB"; + }; + }; + }; + + config = mkIf cfg.enable { + systemd.services.nightlight = { + description = "Nightlight Metrics Storage Service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + serviceConfig = { + Type = "simple"; + ExecStart = "${cfg.package}/bin/nightlight-server --config ${configFile}"; + Restart = "on-failure"; + RestartSec = "5s"; + + # Security hardening + DynamicUser = true; + StateDirectory = "nightlight"; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + NoNewPrivileges = true; + }; + }; + + # Expose metrics endpoint + networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ 8080 ]; + }; +} +``` + +**Usage Example** (in NixOS configuration): + +```nix +{ + services.nightlight = { + enable = true; + listenAddress = "0.0.0.0:8080"; + logLevel = "info"; + + tls = { + enable = true; + certFile = "/etc/certs/nightlight-server.crt"; + keyFile = "/etc/certs/nightlight-server.key"; + caFile = "/etc/certs/ca.crt"; + requireClientCert = true; + }; + + storage = { + retentionDays = 30; + }; + }; +} +``` + +--- + +## 8. Implementation Plan + +### 8.1 Step Breakdown (S1-S6) + +The implementation follows a phased approach aligned with the task.yaml steps. + +#### **S1: Research & Architecture** ✅ (Current Document) + +**Deliverable**: This design document + +**Status**: Completed + +--- + +#### **S2: Workspace Scaffold** + +**Goal**: Create nightlight workspace with skeleton structure + +**Tasks**: +1. Create workspace structure: + ``` + nightlight/ + ├── Cargo.toml + ├── crates/ + │ ├── nightlight-api/ # Client library + │ ├── nightlight-server/ # Main service + │ └── nightlight-types/ # Shared types + ├── proto/ + │ ├── remote_write.proto + │ └── nightlight.proto + └── README.md + ``` + +2. Setup proto compilation in build.rs + +3. Define core types: + ```rust + // nightlight-types/src/lib.rs + + pub type SeriesID = u64; + pub type Timestamp = i64; // Unix timestamp in milliseconds + + pub struct Sample { + pub timestamp: Timestamp, + pub value: f64, + } + + pub struct Series { + pub id: SeriesID, + pub labels: BTreeMap, + } + + pub struct LabelMatcher { + pub name: String, + pub value: String, + pub op: MatchOp, + } + + pub enum MatchOp { + Equal, + NotEqual, + RegexMatch, + RegexNotMatch, + } + ``` + +4. Add dependencies: + ```toml + [workspace.dependencies] + # Core + tokio = { version = "1.35", features = ["full"] } + anyhow = "1.0" + tracing = "0.1" + tracing-subscriber = "0.3" + + # Serialization + serde = { version = "1.0", features = ["derive"] } + serde_json = "1.0" + toml = "0.8" + + # gRPC + tonic = "0.10" + prost = "0.12" + prost-types = "0.12" + + # HTTP + axum = "0.7" + axum-server = { version = "0.6", features = ["tls-rustls"] } + + # Compression + snap = "1.1" # Snappy + + # Time-series + promql-parser = "0.4" + + # Storage + rocksdb = "0.21" # (NOT for TSDB, only for examples) + + # Crypto + rustls = "0.21" + ``` + +**Estimated Effort**: 2 days + +--- + +#### **S3: Push Ingestion** + +**Goal**: Implement Prometheus remote_write compatible ingestion endpoint + +**Tasks**: + +1. **Implement WAL**: + ```rust + // nightlight-server/src/wal.rs + + struct WAL { + dir: PathBuf, + segment_size: usize, + active_segment: RwLock, + } + + impl WAL { + fn new(dir: PathBuf, segment_size: usize) -> Result; + fn append(&self, record: WALRecord) -> Result<()>; + fn replay(&self) -> Result>; + fn checkpoint(&self, min_segment: u64) -> Result<()>; + } + ``` + +2. **Implement In-Memory Head Block**: + ```rust + // nightlight-server/src/head.rs + + struct Head { + series: DashMap>, // Concurrent HashMap + min_time: AtomicI64, + max_time: AtomicI64, + config: HeadConfig, + } + + impl Head { + async fn append(&self, series_id: SeriesID, labels: Labels, ts: Timestamp, value: f64) -> Result<()>; + async fn get(&self, series_id: SeriesID) -> Option>; + async fn series_count(&self) -> usize; + } + ``` + +3. **Implement Gorilla Compression** (basic version): + ```rust + // nightlight-server/src/compression.rs + + struct GorillaEncoder { /* ... */ } + struct GorillaDecoder { /* ... */ } + + impl GorillaEncoder { + fn encode_timestamp(&mut self, ts: i64) -> Result<()>; + fn encode_value(&mut self, value: f64) -> Result<()>; + fn finish(self) -> Vec; + } + ``` + +4. **Implement HTTP Ingestion Handler**: + ```rust + // nightlight-server/src/handlers/ingest.rs + + async fn handle_remote_write( + State(service): State>, + body: Bytes, + ) -> Result { + // 1. Decompress Snappy + // 2. Decode protobuf + // 3. Validate samples + // 4. Append to WAL + // 5. Insert into Head + // 6. Return 204 No Content + } + ``` + +5. **Add Rate Limiting**: + ```rust + struct RateLimiter { + rate: f64, // samples/sec + tokens: AtomicU64, + } + + impl RateLimiter { + fn allow(&self) -> bool; + } + ``` + +6. **Integration Test**: + ```rust + #[tokio::test] + async fn test_remote_write_ingestion() { + // Start server + // Send WriteRequest + // Verify samples stored + } + ``` + +**Estimated Effort**: 5 days + +--- + +#### **S4: PromQL Query Engine** + +**Goal**: Basic PromQL query support (instant + range queries) + +**Tasks**: + +1. **Integrate promql-parser**: + ```rust + // nightlight-server/src/query/parser.rs + + use promql_parser::parser; + + pub fn parse(query: &str) -> Result { + parser::parse(query).map_err(|e| Error::ParseError(e.to_string())) + } + ``` + +2. **Implement Query Planner**: + ```rust + // nightlight-server/src/query/planner.rs + + pub enum QueryPlan { + VectorSelector { matchers: Vec, timestamp: i64 }, + MatrixSelector { matchers: Vec, range: Duration, timestamp: i64 }, + Aggregate { op: AggregateOp, input: Box, grouping: Vec }, + RateFunc { input: Box }, + // ... other operators + } + + pub fn plan(expr: parser::Expr, query_time: i64) -> Result; + ``` + +3. **Implement Label Index**: + ```rust + // nightlight-server/src/index.rs + + struct LabelIndex { + // label_name -> label_value -> [series_ids] + inverted_index: DashMap>>, + } + + impl LabelIndex { + fn find_series(&self, matchers: &[LabelMatcher]) -> Result>; + fn add_series(&self, series_id: SeriesID, labels: &Labels); + } + ``` + +4. **Implement Query Executor**: + ```rust + // nightlight-server/src/query/executor.rs + + struct QueryExecutor { + head: Arc, + blocks: Arc, + index: Arc, + } + + impl QueryExecutor { + async fn execute(&self, plan: QueryPlan) -> Result; + + async fn execute_vector_selector(&self, matchers: Vec, ts: i64) -> Result; + async fn execute_matrix_selector(&self, matchers: Vec, range: Duration, ts: i64) -> Result; + + fn apply_rate(&self, matrix: RangeVector) -> Result; + fn apply_aggregate(&self, op: AggregateOp, vector: InstantVector, grouping: Vec) -> Result; + } + ``` + +5. **Implement HTTP Query Handlers**: + ```rust + // nightlight-server/src/handlers/query.rs + + async fn handle_instant_query( + Query(params): Query, + State(executor): State>, + ) -> Result, (StatusCode, String)> { + let expr = parse(¶ms.query)?; + let plan = plan(expr, params.time.unwrap_or_else(now))?; + let result = executor.execute(plan).await?; + Ok(Json(format_response(result))) + } + + async fn handle_range_query( + Query(params): Query, + State(executor): State>, + ) -> Result, (StatusCode, String)> { + // Similar to instant query, but iterate over [start, end] with step + } + ``` + +6. **Integration Test**: + ```rust + #[tokio::test] + async fn test_instant_query() { + // Ingest samples + // Query: http_requests_total{method="GET"} + // Verify results + } + + #[tokio::test] + async fn test_range_query_with_rate() { + // Ingest counter samples + // Query: rate(http_requests_total[5m]) + // Verify rate calculation + } + ``` + +**Estimated Effort**: 7 days + +--- + +#### **S5: Storage Layer** + +**Goal**: Time-series storage with retention and compaction + +**Tasks**: + +1. **Implement Block Writer**: + ```rust + // nightlight-server/src/block/writer.rs + + struct BlockWriter { + block_dir: PathBuf, + index_writer: IndexWriter, + chunk_writer: ChunkWriter, + } + + impl BlockWriter { + fn new(block_dir: PathBuf) -> Result; + fn write_series(&mut self, series: &Series, samples: &[Sample]) -> Result<()>; + fn finalize(self) -> Result; + } + ``` + +2. **Implement Block Reader**: + ```rust + // nightlight-server/src/block/reader.rs + + struct BlockReader { + meta: BlockMeta, + index: Index, + chunks: ChunkReader, + } + + impl BlockReader { + fn open(block_dir: PathBuf) -> Result; + fn query_samples(&self, series_id: SeriesID, start: i64, end: i64) -> Result>; + } + ``` + +3. **Implement Compaction**: + ```rust + // nightlight-server/src/compaction.rs + + struct Compactor { + data_dir: PathBuf, + config: CompactionConfig, + } + + impl Compactor { + async fn compact_head_to_l0(&self, head: &Head) -> Result; + async fn compact_blocks(&self, source_blocks: Vec) -> Result; + async fn run_compaction_loop(&self); // Background task + } + ``` + +4. **Implement Retention Enforcement**: + ```rust + impl Compactor { + async fn enforce_retention(&self, retention: Duration) -> Result<()> { + let cutoff = SystemTime::now() - retention; + // Delete blocks older than cutoff + } + } + ``` + +5. **Implement Block Manager**: + ```rust + // nightlight-server/src/block/manager.rs + + struct BlockManager { + blocks: RwLock>>, + data_dir: PathBuf, + } + + impl BlockManager { + fn load_blocks(&mut self) -> Result<()>; + fn add_block(&mut self, block: BlockReader); + fn remove_block(&mut self, block_id: &BlockID); + fn query_blocks(&self, start: i64, end: i64) -> Vec>; + } + ``` + +6. **Integration Test**: + ```rust + #[tokio::test] + async fn test_compaction() { + // Ingest data for >2h + // Trigger compaction + // Verify block created + // Query old data from block + } + + #[tokio::test] + async fn test_retention() { + // Create old blocks + // Run retention enforcement + // Verify old blocks deleted + } + ``` + +**Estimated Effort**: 8 days + +--- + +#### **S6: Integration & Documentation** + +**Goal**: NixOS module, TLS config, integration tests, operator docs + +**Tasks**: + +1. **Create NixOS Module**: + - File: `nix/modules/nightlight.nix` + - Follow T024 patterns + - Include systemd service, firewall rules + - Support TLS configuration options + +2. **Implement mTLS**: + - Load certs in server startup + - Configure Rustls with client cert verification + - Extract client identity for rate limiting + +3. **Create Nightlight Scraper**: + - Standalone scraper service + - Reads scrape config (TOML) + - Scrapes `/metrics` endpoints from services + - Pushes to Nightlight via remote_write + +4. **Integration Tests**: + ```rust + #[tokio::test] + async fn test_e2e_ingest_and_query() { + // Start Nightlight server + // Ingest samples via remote_write + // Query via /api/v1/query + // Query via /api/v1/query_range + // Verify results match + } + + #[tokio::test] + async fn test_mtls_authentication() { + // Start server with mTLS + // Connect without client cert -> rejected + // Connect with valid client cert -> accepted + } + + #[tokio::test] + async fn test_grafana_compatibility() { + // Configure Grafana to use Nightlight + // Execute sample queries + // Verify dashboards render correctly + } + ``` + +5. **Write Operator Documentation**: + - **File**: `docs/por/T033-nightlight/OPERATOR.md` + - Installation (NixOS, standalone) + - Configuration guide + - mTLS setup + - Scraper configuration + - Troubleshooting + - Performance tuning + +6. **Write Developer Documentation**: + - **File**: `nightlight/README.md` + - Architecture overview + - Building from source + - Running tests + - Contributing guidelines + +**Estimated Effort**: 5 days + +--- + +### 8.2 Dependency Ordering + +``` +S1 (Research) → S2 (Scaffold) + ↓ + S3 (Ingestion) ──┐ + ↓ │ + S4 (Query) │ + ↓ │ + S5 (Storage) ←────┘ + ↓ + S6 (Integration) +``` + +**Critical Path**: S1 → S2 → S3 → S5 → S6 +**Parallelizable**: S4 can start after S3 completes basic ingestion + +### 8.3 Total Effort Estimate + +| Step | Effort | Priority | +|------|--------|----------| +| S1: Research | 2 days | P0 | +| S2: Scaffold | 2 days | P0 | +| S3: Ingestion | 5 days | P0 | +| S4: Query Engine | 7 days | P0 | +| S5: Storage Layer | 8 days | P1 | +| S6: Integration | 5 days | P1 | +| **Total** | **29 days** | | + +**Realistic Timeline**: 6-8 weeks (accounting for testing, debugging, documentation) + +--- + +## 9. Open Questions + +### 9.1 Decisions Requiring User Input + +#### Q1: Scraper Implementation Choice + +**Question**: Should we use Prometheus in agent mode or build a custom Rust scraper? + +**Option A**: Prometheus Agent + Remote Write +- **Pros**: Battle-tested, standard tool, no implementation effort +- **Cons**: Adds Go dependency, less platform integration + +**Option B**: Custom Rust Scraper +- **Pros**: 100% Rust, platform consistency, easier integration +- **Cons**: Implementation effort, needs testing + +**Recommendation**: Option B (custom scraper) for consistency with PROJECT.md philosophy + +**Decision**: [ ] A [ ] B [ ] Defer to later + +--- + +#### Q2: gRPC vs HTTP Priority + +**Question**: Should we prioritize gRPC API or focus only on HTTP (Prometheus compatibility)? + +**Option A**: HTTP only (v1) +- **Pros**: Simpler, Prometheus/Grafana compatibility is sufficient +- **Cons**: Less efficient for internal services + +**Option B**: Both HTTP and gRPC (v1) +- **Pros**: Better performance for internal services, more flexibility +- **Cons**: More implementation effort + +**Recommendation**: Option A for v1, add gRPC in v2 if needed + +**Decision**: [ ] A [ ] B + +--- + +#### Q3: FlareDB Metadata Integration Timeline + +**Question**: When should we integrate FlareDB for metadata storage? + +**Option A**: v1 (T033) +- **Pros**: Unified metadata story from the start +- **Cons**: Increases complexity, adds dependency + +**Option B**: v2 (Future) +- **Pros**: Simpler v1, can deploy standalone +- **Cons**: Migration effort later + +**Recommendation**: Option B (defer to v2) + +**Decision**: [ ] A [ ] B + +--- + +#### Q4: S3 Cold Storage Priority + +**Question**: Should S3 cold storage be part of v1 or deferred? + +**Option A**: v1 (T033.S5) +- **Pros**: Unlimited retention from day 1 +- **Cons**: Complexity, operational overhead + +**Option B**: v2 (Future) +- **Pros**: Simpler v1, focus on core functionality +- **Cons**: Limited retention (local disk only) + +**Recommendation**: Option B (defer to v2), use local disk for v1 with 15-30 day retention + +**Decision**: [ ] A [ ] B + +--- + +### 9.2 Areas Needing Further Investigation + +#### I1: PromQL Function Coverage + +**Issue**: Need to determine exact subset of PromQL functions to support in v1. + +**Investigation Needed**: +- Survey existing Grafana dashboards in use +- Identify most common functions (rate, increase, histogram_quantile, etc.) +- Prioritize by usage frequency + +**Proposed Approach**: +- Analyze 10-20 sample dashboards +- Create coverage matrix +- Implement top 80% functions first + +--- + +#### I2: Query Performance Benchmarking + +**Issue**: Need to validate query latency targets (p95 <100ms) are achievable. + +**Investigation Needed**: +- Benchmark promql-parser crate performance +- Measure Gorilla decompression throughput +- Test index lookup performance at 10M series scale + +**Proposed Approach**: +- Create benchmark suite with synthetic data (1M, 10M series) +- Measure end-to-end query latency +- Identify bottlenecks and optimize + +--- + +#### I3: Series Cardinality Limits + +**Issue**: How to prevent series explosion (high cardinality killing performance)? + +**Investigation Needed**: +- Research cardinality estimation algorithms (HyperLogLog) +- Define cardinality limits (per metric, per label, global) +- Implement rejection strategy (reject new series beyond limit) + +**Proposed Approach**: +- Add cardinality tracking to label index +- Implement warnings at 80% limit, rejection at 100% +- Provide admin API to inspect high-cardinality series + +--- + +#### I4: Out-of-Order Sample Edge Cases + +**Issue**: How to handle out-of-order samples spanning chunk boundaries? + +**Investigation Needed**: +- Test scenarios: samples arriving 1h late, 2h late, etc. +- Determine if we need multi-chunk updates or reject old samples +- Benchmark impact of re-sorting chunks + +**Proposed Approach**: +- Implement configurable out-of-order window (default: 1h) +- Reject samples older than window +- For within-window samples, insert into correct chunk (may require chunk re-compression) + +--- + +## 10. References + +### 10.1 Research Sources + +#### Time-Series Storage Formats + +- [Gorilla: A Fast, Scalable, In-Memory Time Series Database (Facebook)](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) +- [Gorilla Compression Algorithm - The Morning Paper](https://blog.acolyer.org/2016/05/03/gorilla-a-fast-scalable-in-memory-time-series-database/) +- [Prometheus TSDB Storage Documentation](https://prometheus.io/docs/prometheus/latest/storage/) +- [Prometheus TSDB Architecture - Palark Blog](https://palark.com/blog/prometheus-architecture-tsdb/) +- [InfluxDB TSM Storage Engine](https://www.influxdata.com/blog/new-storage-engine-time-structured-merge-tree/) +- [M3DB Storage Architecture](https://m3db.io/docs/architecture/m3db/) +- [M3DB at Uber Blog](https://www.uber.com/blog/m3/) + +#### PromQL Implementation + +- [promql-parser Rust Crate (GreptimeTeam)](https://github.com/GreptimeTeam/promql-parser) +- [promql-parser Documentation](https://docs.rs/promql-parser) +- [promql Crate (vthriller)](https://github.com/vthriller/promql) + +#### Prometheus Remote Write Protocol + +- [Prometheus Remote Write 1.0 Specification](https://prometheus.io/docs/specs/prw/remote_write_spec/) +- [Prometheus Remote Write 2.0 Specification](https://prometheus.io/docs/specs/prw/remote_write_spec_2_0/) +- [Prometheus Protobuf Schema (remote.proto)](https://github.com/prometheus/prometheus/blob/main/prompb/remote.proto) + +#### Rust TSDB Implementations + +- [InfluxDB 3 Engineering with Rust - InfoQ](https://www.infoq.com/articles/timeseries-db-rust/) +- [Datadog's Rust TSDB - Datadog Blog](https://www.datadoghq.com/blog/engineering/rust-timeseries-engine/) +- [GreptimeDB Announcement](https://greptime.com/blogs/2022-11-15-this-time-for-real) +- [tstorage-rs Embedded TSDB](https://github.com/dpgil/tstorage-rs) +- [tsink High-Performance Embedded TSDB](https://dev.to/h2337/building-high-performance-time-series-applications-with-tsink-a-rust-embedded-database-5fa7) + +### 10.2 Platform References + +#### Internal Documentation + +- PROJECT.md (Item 12: Metrics Store) +- docs/por/T033-nightlight/task.yaml +- docs/por/T027-production-hardening/ (TLS patterns) +- docs/por/T024-nixos-packaging/ (NixOS module patterns) + +#### Existing Service Patterns + +- flaredb/crates/flaredb-server/src/main.rs (TLS, metrics export) +- flaredb/crates/flaredb-server/src/config/mod.rs (Config structure) +- chainfire/crates/chainfire-server/src/config.rs (TLS config) +- iam/crates/iam-server/src/config.rs (Config patterns) + +### 10.3 External Tools + +- [Grafana](https://grafana.com/) - Visualization and dashboards +- [Prometheus](https://prometheus.io/) - Reference implementation +- [VictoriaMetrics](https://victoriametrics.com/) - Replacement target (study architecture) + +--- + +## Appendix A: PromQL Function Reference (v1 Support) + +### Supported Functions + +| Function | Category | Description | Example | +|----------|----------|-------------|---------| +| `rate()` | Counter | Per-second rate of increase | `rate(http_requests_total[5m])` | +| `irate()` | Counter | Instant rate (last 2 samples) | `irate(http_requests_total[5m])` | +| `increase()` | Counter | Total increase over range | `increase(http_requests_total[1h])` | +| `histogram_quantile()` | Histogram | Calculate quantile from histogram | `histogram_quantile(0.95, rate(http_duration_bucket[5m]))` | +| `sum()` | Aggregation | Sum values | `sum(metric)` | +| `avg()` | Aggregation | Average values | `avg(metric)` | +| `min()` | Aggregation | Minimum value | `min(metric)` | +| `max()` | Aggregation | Maximum value | `max(metric)` | +| `count()` | Aggregation | Count series | `count(metric)` | +| `stddev()` | Aggregation | Standard deviation | `stddev(metric)` | +| `stdvar()` | Aggregation | Standard variance | `stdvar(metric)` | +| `topk()` | Aggregation | Top K series | `topk(5, metric)` | +| `bottomk()` | Aggregation | Bottom K series | `bottomk(5, metric)` | +| `time()` | Time | Current timestamp | `time()` | +| `timestamp()` | Time | Sample timestamp | `timestamp(metric)` | +| `abs()` | Math | Absolute value | `abs(metric)` | +| `ceil()` | Math | Round up | `ceil(metric)` | +| `floor()` | Math | Round down | `floor(metric)` | +| `round()` | Math | Round to nearest | `round(metric, 0.1)` | + +### NOT Supported in v1 + +| Function | Category | Reason | +|----------|----------|--------| +| `predict_linear()` | Prediction | Complex, low usage | +| `deriv()` | Math | Low usage | +| `holt_winters()` | Prediction | Complex | +| `resets()` | Counter | Low usage | +| `changes()` | Analysis | Low usage | +| Subqueries | Advanced | Very complex | + +--- + +## Appendix B: Configuration Reference + +### Complete Configuration Example + +```toml +# nightlight.toml - Complete configuration example + +[server] +# Listen address for HTTP/gRPC API +addr = "0.0.0.0:8080" + +# Log level: trace, debug, info, warn, error +log_level = "info" + +# Metrics port for self-monitoring (Prometheus /metrics endpoint) +metrics_port = 9099 + +[server.tls] +# Enable TLS +cert_file = "/etc/nightlight/certs/server.crt" +key_file = "/etc/nightlight/certs/server.key" + +# Enable mTLS (require client certificates) +ca_file = "/etc/nightlight/certs/ca.crt" +require_client_cert = true + +[storage] +# Data directory for TSDB blocks and WAL +data_dir = "/var/lib/nightlight/data" + +# Data retention period (days) +retention_days = 15 + +# WAL segment size (MB) +wal_segment_size_mb = 128 + +# Block duration for compaction +min_block_duration = "2h" +max_block_duration = "24h" + +# Out-of-order sample acceptance window +out_of_order_time_window = "1h" + +# Series cardinality limits +max_series = 10_000_000 +max_series_per_metric = 100_000 + +# Memory limits +max_head_chunks_per_series = 2 +max_head_size_mb = 2048 + +[query] +# Query timeout (seconds) +timeout_seconds = 30 + +# Maximum query range (hours) +max_range_hours = 24 + +# Query result cache TTL (seconds) +cache_ttl_seconds = 60 + +# Maximum concurrent queries +max_concurrent_queries = 100 + +[ingestion] +# Write buffer size (samples) +write_buffer_size = 100_000 + +# Backpressure strategy: "block" or "reject" +backpressure_strategy = "block" + +# Rate limiting (samples per second per client) +rate_limit_per_client = 50_000 + +# Maximum samples per write request +max_samples_per_request = 10_000 + +[compaction] +# Enable background compaction +enabled = true + +# Compaction interval (seconds) +interval_seconds = 7200 # 2 hours + +# Number of compaction threads +num_threads = 2 + +[s3] +# S3 cold storage (optional, future) +enabled = false +endpoint = "https://s3.example.com" +bucket = "nightlight-blocks" +access_key_id = "..." +secret_access_key = "..." +upload_after_days = 7 +local_cache_size_gb = 100 + +[flaredb] +# FlareDB metadata integration (optional, future) +enabled = false +endpoints = ["flaredb-1:50051", "flaredb-2:50051"] +namespace = "metrics" +``` + +--- + +## Appendix C: Metrics Exported by Nightlight + +Nightlight exports metrics about itself on port 9099 (configurable). + +### Ingestion Metrics + +``` +# Samples ingested +nightlight_samples_ingested_total{} counter + +# Samples rejected (out-of-order, invalid, etc.) +nightlight_samples_rejected_total{reason="out_of_order|invalid|rate_limit"} counter + +# Ingestion latency (milliseconds) +nightlight_ingestion_latency_ms{quantile="0.5|0.9|0.99"} summary + +# Active series +nightlight_active_series{} gauge + +# Head memory usage (bytes) +nightlight_head_memory_bytes{} gauge +``` + +### Query Metrics + +``` +# Queries executed +nightlight_queries_total{type="instant|range"} counter + +# Query latency (milliseconds) +nightlight_query_latency_ms{type="instant|range", quantile="0.5|0.9|0.99"} summary + +# Query errors +nightlight_query_errors_total{reason="timeout|parse_error|execution_error"} counter +``` + +### Storage Metrics + +``` +# WAL segments +nightlight_wal_segments{} gauge + +# WAL size (bytes) +nightlight_wal_size_bytes{} gauge + +# Blocks +nightlight_blocks_total{level="0|1|2"} gauge + +# Block size (bytes) +nightlight_block_size_bytes{level="0|1|2"} gauge + +# Compactions +nightlight_compactions_total{level="0|1|2"} counter + +# Compaction duration (seconds) +nightlight_compaction_duration_seconds{level="0|1|2", quantile="0.5|0.9|0.99"} summary +``` + +### System Metrics + +``` +# Go runtime metrics (if using Go for scraper) +# Rust memory metrics +nightlight_memory_allocated_bytes{} gauge + +# CPU usage +nightlight_cpu_usage_seconds_total{} counter +``` + +--- + +## Appendix D: Error Codes and Troubleshooting + +### HTTP Error Codes + +| Code | Meaning | Common Causes | +|------|---------|---------------| +| 200 | OK | Query successful | +| 204 | No Content | Write successful | +| 400 | Bad Request | Invalid PromQL, malformed protobuf | +| 401 | Unauthorized | mTLS cert validation failed | +| 429 | Too Many Requests | Rate limit exceeded | +| 500 | Internal Server Error | Storage error, WAL corruption | +| 503 | Service Unavailable | Write buffer full, server overloaded | + +### Common Issues + +#### Issue: "Samples rejected: out_of_order" + +**Cause**: Samples arriving with timestamps older than `out_of_order_time_window` + +**Solution**: +- Increase `out_of_order_time_window` in config +- Check clock sync on clients (NTP) +- Reduce scrape batch size + +#### Issue: "Rate limit exceeded" + +**Cause**: Client exceeding `rate_limit_per_client` samples/sec + +**Solution**: +- Increase rate limit in config +- Reduce scrape frequency +- Shard writes across multiple clients + +#### Issue: "Query timeout" + +**Cause**: Query exceeding `timeout_seconds` + +**Solution**: +- Increase query timeout +- Reduce query time range +- Add more specific label matchers to reduce series scanned + +#### Issue: "Series cardinality explosion" + +**Cause**: Too many unique label combinations (high cardinality) + +**Solution**: +- Review label design (avoid unbounded labels like user_id) +- Use relabeling to drop high-cardinality labels +- Increase `max_series` limit (if justified) + +--- + +**End of Design Document** + +**Total Length**: ~3,800 lines + +**Status**: Ready for review and S2-S6 implementation + +**Next Steps**: +1. Review and approve design decisions +2. Create GitHub issues for S2-S6 tasks +3. Begin S2: Workspace Scaffold diff --git a/testing/qemu-cluster/README.md b/testing/qemu-cluster/README.md new file mode 100644 index 0000000..ecaa387 --- /dev/null +++ b/testing/qemu-cluster/README.md @@ -0,0 +1,96 @@ +# PhotonCloud QEMU Test Cluster + +QEMU環境でPhotoCloud Bare-Metal Service Meshをテストするための環境構築スクリプト。 + +## 構成 + +- **VM数**: 2台（node-01, node-02） +- **ネットワーク**: 192.168.100.0/24（ブリッジモード） +- **OS**: Ubuntu 22.04 LTS（最小構成） +- **デプロイ**: Chainfire + NodeAgent + mTLS Agent + +## セットアップ + +### 1. QEMU/KVMのインストール + +```bash +sudo apt-get update +sudo apt-get install -y qemu-system-x86_64 qemu-kvm bridge-utils +``` + +### 2. ベースイメージの作成 + +```bash +./scripts/create-base-image.sh +``` + +###3. クラスタの起動 + +```bash +./scripts/start-cluster.sh +``` + +### 4. PhotonCloudコンポーネントのデプロイ + +```bash +./scripts/deploy-photoncloud.sh +``` + +## テストシナリオ + +### シナリオ1: プレーンHTTP通信 + +1. node-01でapi-serverを起動（plain mode） +2. node-02でworker-serviceを起動（plain mode） +3. worker-service → api-server へHTTPリクエスト +4. 通信成功を確認 + +### シナリオ2: mTLS有効化 + +1. cert-authorityで証明書を発行 +2. api-serverをmTLSモードで再起動 +3. worker-serviceをmTLSモードで再起動 +4. 通信成功を確認 + +### シナリオ3: 動的ポリシー変更 + +1. Chainfire上でポリシーをplain→mtlsに変更 +2. NodeAgentがポリシー変更を検知 +3. mTLS Agentがポリシーを適用 +4. 通信が継続されることを確認 + +### シナリオ4: サービス発見 + +1. api-serverを2インスタンス起動 +2. worker-serviceがServiceDiscovery経由で両インスタンスを発見 +3. ラウンドロビンで負荷分散されることを確認 + +### シナリオ5: ノード障害 + +1. node-01をシャットダウン +2. NodeAgentのハートビートが停止 +3. mTLS Agentがnode-01のインスタンスを除外 +4. node-02のインスタンスのみに通信が行くことを確認 + +## ディレクトリ構成 + +``` +testing/qemu-cluster/ +├── README.md +├── scripts/ +│ ├── create-base-image.sh # ベースイメージ作成 +│ ├── start-cluster.sh # クラスタ起動 +│ ├── stop-cluster.sh # クラスタ停止 +│ └── deploy-photoncloud.sh # PhotonCloud デプロイ +├── images/ +│ └── base.qcow2 # ベースイメージ +├── vms/ +│ ├── node-01.qcow2 # node-01のディスク +│ └── node-02.qcow2 # node-02のディスク +└── configs/ + ├── cluster-config.json # クラスタ設定 + ├── node-01-config.toml # node-01設定 + └── node-02-config.toml # node-02設定 +``` + + diff --git a/testing/qemu-cluster/configs/cluster-config.json b/testing/qemu-cluster/configs/cluster-config.json new file mode 100644 index 0000000..c8956d3 --- /dev/null +++ b/testing/qemu-cluster/configs/cluster-config.json @@ -0,0 +1,50 @@ +{ + "cluster": { + "cluster_id": "qemu-test-cluster", + "environment": "dev" + }, + "nodes": [ + { + "node_id": "node-01", + "hostname": "photon-node-01", + "ip": "10.0.2.15", + "roles": ["worker"], + "labels": { + "zone": "zone-a" + } + }, + { + "node_id": "node-02", + "hostname": "photon-node-02", + "ip": "10.0.2.15", + "roles": ["worker"], + "labels": { + "zone": "zone-b" + } + } + ], + "services": [ + { + "name": "test-api", + "ports": { + "http": 8080 + }, + "protocol": "http", + "mtls_required": false, + "mesh_mode": "agent" + } + ], + "instances": [], + "mtls_policies": [ + { + "policy_id": "default-dev", + "environment": "dev", + "source_service": "*", + "target_service": "*", + "mtls_required": false, + "mode": "plain" + } + ] +} + + diff --git a/testing/qemu-cluster/scripts/create-base-image.sh b/testing/qemu-cluster/scripts/create-base-image.sh new file mode 100755 index 0000000..4236d71 --- /dev/null +++ b/testing/qemu-cluster/scripts/create-base-image.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +IMAGES_DIR="$PROJECT_ROOT/testing/qemu-cluster/images" + +echo "Creating base image directory..." +mkdir -p "$IMAGES_DIR" + +BASE_IMAGE="$IMAGES_DIR/base.qcow2" + +# ベースイメージが既に存在する場合はスキップ +if [ -f "$BASE_IMAGE" ]; then + echo "Base image already exists: $BASE_IMAGE" + exit 0 +fi + +echo "Creating base QCOW2 image (10GB)..." +qemu-img create -f qcow2 "$BASE_IMAGE" 10G + +echo "Base image created: $BASE_IMAGE" +echo "" +echo "Next steps:" +echo " 1. Install Ubuntu 22.04 LTS manually:" +echo " qemu-system-x86_64 -enable-kvm -m 2048 -hda $BASE_IMAGE -cdrom ubuntu-22.04-server-amd64.iso" +echo " 2. Install required packages:" +echo " - openssh-server" +echo " - curl, wget" +echo " - net-tools" +echo " 3. Create your administrative user with a secure password" +echo " 4. Shutdown the VM" +echo "" +echo "Or use the automated installation script (TODO: implement)" + + diff --git a/testing/qemu-cluster/scripts/deploy-photoncloud.sh b/testing/qemu-cluster/scripts/deploy-photoncloud.sh new file mode 100755 index 0000000..1a557ad --- /dev/null +++ b/testing/qemu-cluster/scripts/deploy-photoncloud.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +echo "Deploying PhotonCloud to QEMU cluster..." + +# ビルド +echo "Building PhotonCloud components..." +cd "$PROJECT_ROOT/deployer" && cargo build --release +cd "$PROJECT_ROOT/mtls-agent" && cargo build --release + +# バイナリをVMにコピー +echo "Copying binaries to node-01..." +scp -P 2201 \ + "$PROJECT_ROOT/deployer/target/release/node-agent" \ + "$PROJECT_ROOT/deployer/target/release/deployer-ctl" \ + "$PROJECT_ROOT/mtls-agent/target/release/mtls-agent" \ + photon@localhost:/tmp/ + +echo "Copying binaries to node-02..." +scp -P 2202 \ + "$PROJECT_ROOT/deployer/target/release/node-agent" \ + "$PROJECT_ROOT/mtls-agent/target/release/mtls-agent" \ + photon@localhost:/tmp/ + +# 設定ファイルをコピー +echo "Copying configuration files..." +scp -P 2201 "$PROJECT_ROOT/testing/qemu-cluster/configs/node-01-config.toml" photon@localhost:/tmp/ +scp -P 2202 "$PROJECT_ROOT/testing/qemu-cluster/configs/node-02-config.toml" photon@localhost:/tmp/ + +# インストールスクリプトを実行 +echo "Installing on node-01..." +ssh -p 2201 photon@localhost << 'EOF' + sudo mv /tmp/node-agent /usr/local/bin/ + sudo mv /tmp/deployer-ctl /usr/local/bin/ + sudo mv /tmp/mtls-agent /usr/local/bin/ + sudo chmod +x /usr/local/bin/{node-agent,deployer-ctl,mtls-agent} + sudo mkdir -p /etc/photoncloud + sudo mv /tmp/node-01-config.toml /etc/photoncloud/config.toml +EOF + +echo "Installing on node-02..." +ssh -p 2202 photon@localhost << 'EOF' + sudo mv /tmp/node-agent /usr/local/bin/ + sudo mv /tmp/mtls-agent /usr/local/bin/ + sudo chmod +x /usr/local/bin/{node-agent,mtls-agent} + sudo mkdir -p /etc/photoncloud + sudo mv /tmp/node-02-config.toml /etc/photoncloud/config.toml +EOF + +echo "Deployment complete!" +echo "" +echo "Start services:" +echo " node-01: ssh -p 2201 photon@localhost 'sudo node-agent --config /etc/photoncloud/config.toml'" +echo " node-02: ssh -p 2202 photon@localhost 'sudo node-agent --config /etc/photoncloud/config.toml'" + + diff --git a/testing/qemu-cluster/scripts/start-cluster.sh b/testing/qemu-cluster/scripts/start-cluster.sh new file mode 100755 index 0000000..644a844 --- /dev/null +++ b/testing/qemu-cluster/scripts/start-cluster.sh @@ -0,0 +1,73 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +IMAGES_DIR="$PROJECT_ROOT/testing/qemu-cluster/images" +VMS_DIR="$PROJECT_ROOT/testing/qemu-cluster/vms" + +echo "Starting PhotonCloud QEMU Test Cluster..." +mkdir -p "$VMS_DIR" + +BASE_IMAGE="$IMAGES_DIR/base.qcow2" + +if [ ! -f "$BASE_IMAGE" ]; then + echo "Error: Base image not found: $BASE_IMAGE" + echo "Run ./scripts/create-base-image.sh first" + exit 1 +fi + +# VM設定 +NODE01_IMAGE="$VMS_DIR/node-01.qcow2" +NODE02_IMAGE="$VMS_DIR/node-02.qcow2" +NODE01_MAC="52:54:00:12:34:01" +NODE02_MAC="52:54:00:12:34:02" + +# ベースイメージからVMイメージを作成（COW） +if [ ! -f "$NODE01_IMAGE" ]; then + echo "Creating node-01 image..." + qemu-img create -f qcow2 -b "$BASE_IMAGE" -F qcow2 "$NODE01_IMAGE" 10G +fi + +if [ ! -f "$NODE02_IMAGE" ]; then + echo "Creating node-02 image..." + qemu-img create -f qcow2 -b "$BASE_IMAGE" -F qcow2 "$NODE02_IMAGE" 10G +fi + +# VMを起動 +echo "Starting node-01..." +qemu-system-x86_64 \ + -enable-kvm \ + -name node-01 \ + -m 2048 \ + -smp 2 \ + -hda "$NODE01_IMAGE" \ + -netdev user,id=net0,hostfwd=tcp::2201-:22,hostfwd=tcp::18080-:18080 \ + -device e1000,netdev=net0,mac="$NODE01_MAC" \ + -nographic \ + -daemonize \ + -pidfile "$VMS_DIR/node-01.pid" + +echo "Starting node-02..." +qemu-system-x86_64 \ + -enable-kvm \ + -name node-02 \ + -m 2048 \ + -smp 2 \ + -hda "$NODE02_IMAGE" \ + -netdev user,id=net0,hostfwd=tcp::2202-:22,hostfwd=tcp::18081-:18081 \ + -device e1000,netdev=net0,mac="$NODE02_MAC" \ + -nographic \ + -daemonize \ + -pidfile "$VMS_DIR/node-02.pid" + +echo "Cluster started successfully!" +echo "" +echo "Access VMs:" +echo " node-01: ssh -p 2201 photon@localhost" +echo " node-02: ssh -p 2202 photon@localhost" +echo "" +echo "Stop cluster:" +echo " ./scripts/stop-cluster.sh" + + diff --git a/testing/qemu-cluster/scripts/stop-cluster.sh b/testing/qemu-cluster/scripts/stop-cluster.sh new file mode 100755 index 0000000..94e6f7c --- /dev/null +++ b/testing/qemu-cluster/scripts/stop-cluster.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +VMS_DIR="$PROJECT_ROOT/testing/qemu-cluster/vms" + +echo "Stopping PhotonCloud QEMU Test Cluster..." + +# PIDファイルを使ってVMを停止 +for pidfile in "$VMS_DIR"/*.pid; do + if [ -f "$pidfile" ]; then + pid=$(cat "$pidfile") + vm_name=$(basename "$pidfile" .pid) + echo "Stopping $vm_name (PID: $pid)..." + kill "$pid" 2>/dev/null || true + rm -f "$pidfile" + fi +done + +echo "Cluster stopped." + + diff --git a/testing/run-s3-test.sh b/testing/run-s3-test.sh new file mode 100644 index 0000000..d60ea90 --- /dev/null +++ b/testing/run-s3-test.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -e + +# Configuration +CHAINFIRE_PORT=10000 +IAM_PORT=10010 +LIGHTNINGSTOR_PORT=10020 +S3_PORT=10021 +DATA_DIR=$(mktemp -d) + +echo "Building services..." +(cd chainfire && cargo build -p chainfire-server) +(cd iam && cargo build -p iam-server) +(cd lightningstor && cargo build -p lightningstor-server) + +CHAINFIRE_BIN="./chainfire/target/debug/chainfire" +IAM_BIN="./iam/target/debug/iam-server" +LIGHTNINGSTOR_BIN="./lightningstor/target/debug/lightningstor-server" + +echo "Starting Chainfire..." +$CHAINFIRE_BIN \ + --node-id 1 \ + --data-dir "$DATA_DIR/chainfire" \ + --api-addr "127.0.0.1:$CHAINFIRE_PORT" \ + --raft-addr "127.0.0.1:$((CHAINFIRE_PORT + 1))" \ + --gossip-addr "127.0.0.1:$((CHAINFIRE_PORT + 2))" \ + --initial-cluster "1=127.0.0.1:$((CHAINFIRE_PORT + 1))" \ + --metrics-port $((CHAINFIRE_PORT + 3)) & +CHAINFIRE_PID=$! + +echo "Starting IAM..." +export IAM_CRED_MASTER_KEY=$(openssl rand -base64 32) +$IAM_BIN \ + --addr "127.0.0.1:$IAM_PORT" \ + --metrics-port $((IAM_PORT + 1)) & +IAM_PID=$! + +echo "Starting LightningStor..." +export DEFAULT_ORG_ID=org1 +export DEFAULT_PROJECT_ID=proj1 +$LIGHTNINGSTOR_BIN \ + --grpc-addr "127.0.0.1:$LIGHTNINGSTOR_PORT" \ + --s3-addr "127.0.0.1:$S3_PORT" \ + --chainfire-endpoint "http://127.0.0.1:$CHAINFIRE_PORT" \ + --data-dir "$DATA_DIR/lightningstor" \ + --metrics-port $((LIGHTNINGSTOR_PORT + 2)) \ + --in-memory-metadata & +LIGHTNINGSTOR_PID=$! + +# Cleanup function +cleanup() { + echo "Cleaning up..." + kill $CHAINFIRE_PID $IAM_PID $LIGHTNINGSTOR_PID 2>/dev/null || true + rm -rf "$DATA_DIR" +} +trap cleanup EXIT + +# Wait for services to start +echo "Waiting for services to start..." +sleep 5 + +# Test S3 functionality +echo "Running S3 tests..." +# Use credentials from environment or default dummy values +export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-AKIAIOSFODNN7EXAMPLE} +export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY} +export AWS_DEFAULT_REGION=us-east-1 + +ENDPOINT_URL="http://localhost:$S3_PORT" + +echo "1. Create bucket" +aws --endpoint-url "$ENDPOINT_URL" s3 mb s3://test-bucket + +echo "2. Upload object" +echo "Hello, LightningStor!" > "$DATA_DIR/testfile.txt" +aws --endpoint-url "$ENDPOINT_URL" s3 cp "$DATA_DIR/testfile.txt" s3://test-bucket/hello.txt + +echo "3. Download object" +aws --endpoint-url "$ENDPOINT_URL" s3 cp s3://test-bucket/hello.txt "$DATA_DIR/downloaded.txt" + +echo "4. Verify content" +diff "$DATA_DIR/testfile.txt" "$DATA_DIR/downloaded.txt" + +echo "S3 tests passed successfully!" \ No newline at end of file diff --git a/testing/s3-test.nix b/testing/s3-test.nix new file mode 100644 index 0000000..09c830b --- /dev/null +++ b/testing/s3-test.nix @@ -0,0 +1,25 @@ +let + rust_overlay = import (builtins.fetchTarball "https://github.com/oxalica/rust-overlay/archive/master.tar.gz"); + pkgs = import { overlays = [ rust_overlay ]; }; + rustToolchain = pkgs.rust-bin.stable.latest.default; +in +pkgs.mkShell { + name = "s3-test-env"; + + buildInputs = with pkgs; [ + rustToolchain + awscli2 + jq + curl + protobuf + pkg-config + openssl + ]; + + # Set up environment variables if needed + shellHook = '' + export PATH=$PATH:$PWD/target/debug + echo "S3 Test Environment Loaded" + echo "Run ./testing/run-s3-test.sh to execute the tests." + ''; +} \ No newline at end of file