From e1a5d394e540d2e24d2b1dd4fd9bcc1d1aae774c Mon Sep 17 00:00:00 2001 From: centra Date: Sat, 28 Mar 2026 00:09:22 +0900 Subject: [PATCH] ci: unify workspace inventory and harden tier0 gating --- .github/workflows/nix.yml | 102 +++++------ client-common/src/lib.rs | 26 ++- coronafs/crates/coronafs-server/src/config.rs | 18 +- coronafs/crates/coronafs-server/src/main.rs | 49 +++--- mtls-agent/src/client.rs | 24 +-- mtls-agent/src/discovery.rs | 23 +-- mtls-agent/src/main.rs | 48 ++--- mtls-agent/src/policy.rs | 2 + nix/ci/flake.nix | 61 +++++-- nix/ci/workspaces.json | 166 ++++++++++++++++++ scripts/ci_changed_workspaces.py | 134 ++++++++++++++ 11 files changed, 493 insertions(+), 160 deletions(-) create mode 100644 nix/ci/workspaces.json create mode 100644 scripts/ci_changed_workspaces.py diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index c8354ae..9f87315 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -2,65 +2,71 @@ name: Nix CI on: push: - branches: [ master ] + branches: [ main, master ] pull_request: - branches: [ master ] + branches: [ main, master ] + workflow_dispatch: jobs: # Detect which workspaces have changed to save CI minutes filter: runs-on: ubuntu-latest outputs: - workspaces: ${{ steps.filter.outputs.changes }} - any_changed: ${{ steps.filter.outputs.workspaces_any_changed }} - global_changed: ${{ steps.filter.outputs.global }} - shared_crates_changed: ${{ steps.filter.outputs.shared_crates }} + workspaces: ${{ steps.detect.outputs.workspaces }} + build_targets: ${{ steps.detect.outputs.build_targets }} + any_changed: ${{ steps.detect.outputs.any_changed }} + build_changed: ${{ steps.detect.outputs.build_changed }} + global_changed: ${{ steps.detect.outputs.global_changed }} + shared_crates_changed: ${{ steps.detect.outputs.shared_crates_changed }} steps: - uses: actions/checkout@v4 - - uses: dorny/paths-filter@v3 - id: filter with: - filters: | - global: - - 'flake.nix' - - 'flake.lock' - - 'nix/**' - - 'nix-nos/**' - - '.github/workflows/nix.yml' - - 'Cargo.toml' - - 'Cargo.lock' - - 'crates/**' - - 'client-common/**' - - 'baremetal/**' - - 'scripts/**' - - 'specifications/**' - - 'docs/**' - shared_crates: 'crates/**' - chainfire: 'chainfire/**' - flaredb: 'flaredb/**' - iam: 'iam/**' - plasmavmc: 'plasmavmc/**' - prismnet: 'prismnet/**' - flashdns: 'flashdns/**' - fiberlb: 'fiberlb/**' - lightningstor: 'lightningstor/**' - nightlight: 'nightlight/**' - creditservice: 'creditservice/**' - k8shost: 'k8shost/**' - apigateway: 'apigateway/**' - deployer: 'deployer/**' + fetch-depth: 0 + + - name: Collect changed files + env: + EVENT_NAME: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} + BEFORE_SHA: ${{ github.event.before }} + HEAD_SHA: ${{ github.sha }} + run: | + set -euo pipefail + + if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then + printf 'flake.nix\n' > changed-files.txt + elif [[ "$EVENT_NAME" == "pull_request" ]]; then + git fetch --no-tags --depth=1 origin "$BASE_REF" + git diff --name-only "origin/$BASE_REF...$HEAD_SHA" > changed-files.txt + elif [[ "$BEFORE_SHA" == "0000000000000000000000000000000000000000" ]]; then + git diff-tree --no-commit-id --name-only -r "$HEAD_SHA" > changed-files.txt + else + git diff --name-only "$BEFORE_SHA" "$HEAD_SHA" > changed-files.txt + fi + + if [[ ! -f changed-files.txt ]]; then + : > changed-files.txt + fi + + sed -n '1,200p' changed-files.txt + + - name: Detect changed workspaces + id: detect + run: | + python3 scripts/ci_changed_workspaces.py \ + --config nix/ci/workspaces.json \ + --changed-files-file changed-files.txt \ + --github-output "$GITHUB_OUTPUT" # Run CI gates for changed workspaces # Uses the provider-agnostic 'photoncloud-gate' defined in nix/ci/flake.nix gate: needs: filter - if: ${{ needs.filter.outputs.any_changed == 'true' || needs.filter.outputs.global_changed == 'true' }} + if: ${{ needs.filter.outputs.any_changed == 'true' }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - # If global files changed, run all. Otherwise run only changed ones. - workspace: ${{ fromJSON(needs.filter.outputs.global_changed == 'true' && '["chainfire", "flaredb", "iam", "plasmavmc", "prismnet", "flashdns", "fiberlb", "lightningstor", "nightlight", "creditservice", "k8shost", "apigateway", "deployer"]' || needs.filter.outputs.workspaces) }} + workspace: ${{ fromJSON(needs.filter.outputs.workspaces) }} name: gate (${{ matrix.workspace }}) steps: - uses: actions/checkout@v4 @@ -88,26 +94,20 @@ jobs: # Build server packages (tier 1+) build: needs: [filter, gate] - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') && needs.filter.outputs.build_changed == 'true' }} runs-on: ubuntu-latest strategy: matrix: - workspace: ${{ fromJSON(needs.filter.outputs.global_changed == 'true' && '["chainfire", "flaredb", "iam", "plasmavmc", "prismnet", "flashdns", "fiberlb", "lightningstor", "nightlight", "creditservice", "k8shost", "apigateway", "deployer"]' || needs.filter.outputs.workspaces) }} - name: build (${{ matrix.workspace }}) + target: ${{ fromJSON(needs.filter.outputs.build_targets) }} + name: build (${{ matrix.target.package }}) steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@v11 - uses: DeterminateSystems/magic-nix-cache-action@v8 - - name: Build server + - name: Build package run: | - # Only build if the workspace has a corresponding package in flake.nix - # We check if it exists before building to avoid failure on non-package workspaces - if nix flake show --json | jq -e ".packages.\"x86_64-linux\".\"${{ matrix.workspace }}-server\"" > /dev/null; then - nix build .#${{ matrix.workspace }}-server --accept-flake-config - else - echo "No server package found for ${{ matrix.workspace }}, skipping build." - fi + nix build .#${{ matrix.target.package }} --accept-flake-config # Summary job for PR status checks ci-status: diff --git a/client-common/src/lib.rs b/client-common/src/lib.rs index 9458b42..8ca690f 100644 --- a/client-common/src/lib.rs +++ b/client-common/src/lib.rs @@ -1,10 +1,10 @@ //! Shared client config types (endpoint/auth/retry) for PhotonCloud SDKs. -//! +//! //! Lightweight, type-only helpers to keep SDK crates consistent without //! forcing a unified SDK dependency tree. -use std::time::Duration; use backoff::ExponentialBackoffBuilder; +use std::time::Duration; use thiserror::Error; use tonic::codegen::InterceptedService; use tonic::service::Interceptor; @@ -70,7 +70,9 @@ impl EndpointConfig { key.clone(), )); } - ep = ep.tls_config(cfg).map_err(|e| ClientError::TlsConfig(e.to_string()))?; + ep = ep + .tls_config(cfg) + .map_err(|e| ClientError::TlsConfig(e.to_string()))?; } Ok(ep) @@ -96,7 +98,9 @@ pub enum AuthConfig { impl AuthConfig { pub fn bearer(token: impl Into) -> Self { - Self::Bearer { token: token.into() } + Self::Bearer { + token: token.into(), + } } } @@ -157,12 +161,13 @@ impl Interceptor for AuthInterceptor { match &self.0 { AuthConfig::None => {} AuthConfig::Bearer { token } => { - req.metadata_mut() - .insert("authorization", format!("Bearer {}", token).parse().unwrap()); + req.metadata_mut().insert( + "authorization", + format!("Bearer {}", token).parse().unwrap(), + ); } AuthConfig::AccessKey { id, secret } => { - req.metadata_mut() - .insert("x-api-key", id.parse().unwrap()); + req.metadata_mut().insert("x-api-key", id.parse().unwrap()); req.metadata_mut() .insert("x-api-secret", secret.parse().unwrap()); } @@ -180,7 +185,10 @@ pub fn auth_interceptor(auth: &AuthConfig) -> Option { } /// Helper to wrap a tonic client with an interceptor when auth is provided. -pub fn with_auth(channel: Channel, auth: &AuthConfig) -> InterceptedService { +pub fn with_auth( + channel: Channel, + auth: &AuthConfig, +) -> InterceptedService { let interceptor = auth_interceptor(auth).unwrap_or(AuthInterceptor(AuthConfig::None)); InterceptedService::new(channel, interceptor) } diff --git a/coronafs/crates/coronafs-server/src/config.rs b/coronafs/crates/coronafs-server/src/config.rs index fb98da7..7746f83 100644 --- a/coronafs/crates/coronafs-server/src/config.rs +++ b/coronafs/crates/coronafs-server/src/config.rs @@ -2,33 +2,23 @@ use serde::{Deserialize, Serialize}; use std::net::SocketAddr; use std::path::PathBuf; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ServerMode { + #[default] Combined, Controller, Node, } -impl Default for ServerMode { - fn default() -> Self { - Self::Combined - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum MetadataBackend { + #[default] Filesystem, Chainfire, } -impl Default for MetadataBackend { - fn default() -> Self { - Self::Filesystem - } -} - #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct ServerConfig { diff --git a/coronafs/crates/coronafs-server/src/main.rs b/coronafs/crates/coronafs-server/src/main.rs index d819866..6b29928 100644 --- a/coronafs/crates/coronafs-server/src/main.rs +++ b/coronafs/crates/coronafs-server/src/main.rs @@ -123,19 +123,14 @@ struct AppState { reserved_ports: Arc>>, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] enum VolumeFileFormat { + #[default] Raw, Qcow2, } -impl Default for VolumeFileFormat { - fn default() -> Self { - Self::Raw - } -} - impl VolumeFileFormat { fn as_qemu_arg(self) -> &'static str { match self { @@ -871,13 +866,12 @@ async fn materialize_impl( return load_response_required(state, id).await; } - let format = req.format.unwrap_or_else(|| { - if req.lazy { - VolumeFileFormat::Qcow2 - } else { - VolumeFileFormat::Raw - } - }); + let default_format = if req.lazy { + VolumeFileFormat::Qcow2 + } else { + VolumeFileFormat::Raw + }; + let format = req.format.unwrap_or(default_format); let temp_path = temp_create_path(&state.config, id); if fs::try_exists(&temp_path).await.unwrap_or(false) { let _ = fs::remove_file(&temp_path).await; @@ -1103,10 +1097,7 @@ async fn delete_impl(state: &AppState, id: &str) -> Result<()> { } async fn load_response(state: &AppState, id: &str) -> Option { - match load_response_required(state, id).await { - Ok(response) => Some(response), - Err(_) => None, - } + load_response_required(state, id).await.ok() } async fn list_volume_responses(state: &AppState) -> Result> { @@ -1517,8 +1508,10 @@ mod tests { #[test] fn export_probe_host_prefers_loopback_for_wildcard_bind() { - let mut config = ServerConfig::default(); - config.export_bind_addr = "0.0.0.0".to_string(); + let mut config = ServerConfig { + export_bind_addr: "0.0.0.0".to_string(), + ..ServerConfig::default() + }; assert_eq!(export_probe_host(&config), "127.0.0.1"); config.export_bind_addr = "10.100.0.11".to_string(); assert_eq!(export_probe_host(&config), "10.100.0.11"); @@ -1650,6 +1643,13 @@ mod tests { assert_ne!(next, preferred); release_export_port(&state, Some(port)).await; + release_export_port(&state, Some(next)).await; + for _ in 0..10 { + if port_is_usable(&state.config, &HashSet::new(), preferred).await { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } let reused = reserve_export_port(&state, Some(preferred)).await.unwrap(); assert_eq!(reused, preferred); } @@ -1760,7 +1760,7 @@ mod tests { &state, "vol-a", CreateVolumeRequest { - size_bytes: 1 * 1024 * 1024, + size_bytes: 1024 * 1024, format: Some(VolumeFileFormat::Raw), backing_file: None, backing_format: None, @@ -2275,10 +2275,11 @@ mod tests { #[test] fn volume_create_api_is_available_in_node_mode() { - let mut config = ServerConfig::default(); - config.mode = ServerMode::Node; let state = AppState { - config: Arc::new(config), + config: Arc::new(ServerConfig { + mode: ServerMode::Node, + ..ServerConfig::default() + }), metadata_store: MetadataStore::Filesystem, volume_guards: Arc::new(Mutex::new(HashMap::new())), reserved_ports: Arc::new(Mutex::new(HashSet::new())), diff --git a/mtls-agent/src/client.rs b/mtls-agent/src/client.rs index 1d2d4b9..65f99e8 100644 --- a/mtls-agent/src/client.rs +++ b/mtls-agent/src/client.rs @@ -1,7 +1,12 @@ +#![allow(dead_code)] + use std::sync::Arc; use anyhow::{Context, Result}; -use rustls::{pki_types::{PrivateKeyDer, ServerName}, ClientConfig, RootCertStore}; +use rustls::{ + pki_types::{PrivateKeyDer, ServerName}, + ClientConfig, RootCertStore, +}; use rustls_pemfile::certs; use std::fs; use std::io::BufReader; @@ -15,7 +20,7 @@ use crate::discovery::ServiceDiscovery; pub enum MtlsStream { Plain(TcpStream), - Tls(tokio_rustls::client::TlsStream), + Tls(Box>), } impl AsyncRead for MtlsStream { @@ -26,7 +31,7 @@ impl AsyncRead for MtlsStream { ) -> Poll> { match self.get_mut() { MtlsStream::Plain(stream) => Pin::new(stream).poll_read(cx, buf), - MtlsStream::Tls(stream) => Pin::new(stream).poll_read(cx, buf), + MtlsStream::Tls(stream) => Pin::new(stream.as_mut()).poll_read(cx, buf), } } } @@ -39,24 +44,21 @@ impl AsyncWrite for MtlsStream { ) -> Poll> { match self.get_mut() { MtlsStream::Plain(stream) => Pin::new(stream).poll_write(cx, data), - MtlsStream::Tls(stream) => Pin::new(stream).poll_write(cx, data), + MtlsStream::Tls(stream) => Pin::new(stream.as_mut()).poll_write(cx, data), } } fn poll_flush(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { match self.get_mut() { MtlsStream::Plain(stream) => Pin::new(stream).poll_flush(cx), - MtlsStream::Tls(stream) => Pin::new(stream).poll_flush(cx), + MtlsStream::Tls(stream) => Pin::new(stream.as_mut()).poll_flush(cx), } } - fn poll_shutdown( - self: Pin<&mut Self>, - cx: &mut TaskContext<'_>, - ) -> Poll> { + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { match self.get_mut() { MtlsStream::Plain(stream) => Pin::new(stream).poll_shutdown(cx), - MtlsStream::Tls(stream) => Pin::new(stream).poll_shutdown(cx), + MtlsStream::Tls(stream) => Pin::new(stream.as_mut()).poll_shutdown(cx), } } } @@ -109,7 +111,7 @@ impl MtlsClient { let server_name = ServerName::try_from(service_name.to_string()) .context("invalid server name for TLS")?; let tls_stream = connector.connect(server_name, stream).await?; - return Ok(MtlsStream::Tls(tls_stream)); + return Ok(MtlsStream::Tls(Box::new(tls_stream))); } Ok(MtlsStream::Plain(stream)) diff --git a/mtls-agent/src/discovery.rs b/mtls-agent/src/discovery.rs index 902a51a..fbbd6d6 100644 --- a/mtls-agent/src/discovery.rs +++ b/mtls-agent/src/discovery.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] + use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -96,7 +98,10 @@ impl ServiceDiscovery { Ok(instances) } - async fn fetch_instances_from_chainfire(&self, service_name: &str) -> Result> { + async fn fetch_instances_from_chainfire( + &self, + service_name: &str, + ) -> Result> { let mut client = Client::connect(self.chainfire_endpoint.clone()).await?; let prefix = format!( "{}instances/{}/", @@ -136,10 +141,7 @@ impl ServiceDiscovery { source_service: &str, target_service: &str, ) -> Result> { - let policy_key = format!( - "{}-{}", - source_service, target_service - ); + let policy_key = format!("{}-{}", source_service, target_service); // キャッシュをチェック { @@ -153,10 +155,7 @@ impl ServiceDiscovery { // Chainfireから取得 let mut client = Client::connect(self.chainfire_endpoint.clone()).await?; - let prefix = format!( - "{}mtls/policies/", - cluster_prefix(&self.cluster_id) - ); + let prefix = format!("{}mtls/policies/", cluster_prefix(&self.cluster_id)); let prefix_bytes = prefix.as_bytes(); let (kvs, _) = client.scan_prefix(prefix_bytes, 0).await?; @@ -164,7 +163,9 @@ impl ServiceDiscovery { for (_, value, _) in kvs { match serde_json::from_slice::(&value) { Ok(policy) => { - if policy.source_service == source_service && policy.target_service == target_service { + if policy.source_service == source_service + && policy.target_service == target_service + { // キャッシュに保存 let mut cache = self.policy_cache.write().await; cache.insert( @@ -207,7 +208,7 @@ impl ServiceDiscovery { if inst.state.as_deref().unwrap_or("healthy") == "healthy" { service_map .entry(inst.service.clone()) - .or_insert_with(Vec::new) + .or_default() .push(inst); } } diff --git a/mtls-agent/src/main.rs b/mtls-agent/src/main.rs index 3eb08fd..20ce002 100644 --- a/mtls-agent/src/main.rs +++ b/mtls-agent/src/main.rs @@ -120,10 +120,7 @@ async fn main() -> Result<()> { if let Some(disc) = &discovery { // デフォルトポリシーを確認（簡易実装） // 実際には、自身のサービス名とターゲットサービス名でポリシーを検索 - if let Ok(Some(policy)) = disc - .get_mtls_policy(&cfg.service.name, "default") - .await - { + if let Ok(Some(policy)) = disc.get_mtls_policy(&cfg.service.name, "default").await { if policy.mtls_required.unwrap_or(false) { "mtls" } else { @@ -168,15 +165,16 @@ async fn main() -> Result<()> { } fn load_config(path: &PathBuf) -> Result { - let contents = fs::read_to_string(path) - .with_context(|| format!("failed to read {}", path.display()))?; + let contents = + fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; let cfg: Config = toml::from_str(&contents).with_context(|| format!("failed to parse {}", path.display()))?; Ok(cfg) } fn load_certs(path: &str) -> Result>> { - let file = fs::File::open(path).with_context(|| format!("failed to open cert file {}", path))?; + let file = + fs::File::open(path).with_context(|| format!("failed to open cert file {}", path))?; let mut reader = BufReader::new(file); let certs = certs(&mut reader) .collect::, _>>() @@ -246,7 +244,9 @@ fn build_server_config(cfg: &Config, mode: &str) -> Result { let client_certs = load_certs(ca_path)?; let mut roots = rustls::RootCertStore::empty(); for c in client_certs { - roots.add(c).map_err(|e| anyhow!("adding CA failed: {:?}", e))?; + roots + .add(c) + .map_err(|e| anyhow!("adding CA failed: {:?}", e))?; } let verifier = rustls::server::WebPkiClientVerifier::builder(std::sync::Arc::new(roots)).build()?; @@ -266,7 +266,10 @@ fn build_server_config(cfg: &Config, mode: &str) -> Result { async fn run_plain_proxy(listen_addr: &str, app_addr: &str) -> Result<()> { let listener = TcpListener::bind(listen_addr).await?; - info!("listening on {} and forwarding to {}", listen_addr, app_addr); + info!( + "listening on {} and forwarding to {}", + listen_addr, app_addr + ); loop { let (inbound, peer) = listener.accept().await?; @@ -347,8 +350,8 @@ async fn handle_connection(mut inbound: TcpStream, app_addr: &str) -> Result<()> #[cfg(test)] mod tests { use super::*; - use rustls::{ClientConfig, RootCertStore}; use rustls::pki_types::ServerName; + use rustls::{ClientConfig, RootCertStore}; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::OnceLock; @@ -361,14 +364,20 @@ mod tests { .args(args) .status() .unwrap_or_else(|error| panic!("failed to spawn openssl {:?}: {}", args, error)); - assert!(status.success(), "openssl {:?} failed with status {}", args, status); + assert!( + status.success(), + "openssl {:?} failed with status {}", + args, + status + ); } fn ensure_test_certs() -> &'static Path { static CERT_DIR: OnceLock = OnceLock::new(); CERT_DIR.get_or_init(|| { - let dir = std::env::temp_dir().join(format!("mtls-agent-test-certs-{}", std::process::id())); + let dir = + std::env::temp_dir().join(format!("mtls-agent-test-certs-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let ca_key = dir.join("ca.key"); @@ -384,12 +393,7 @@ mod tests { let client_pem = dir.join("client.pem"); if !ca_pem.exists() { - run_openssl(&[ - "genrsa", - "-out", - ca_key.to_string_lossy().as_ref(), - "2048", - ]); + run_openssl(&["genrsa", "-out", ca_key.to_string_lossy().as_ref(), "2048"]); run_openssl(&[ "req", "-x509", @@ -490,10 +494,7 @@ mod tests { } fn test_cert_path(name: &str) -> String { - ensure_test_certs() - .join(name) - .display() - .to_string() + ensure_test_certs().join(name).display().to_string() } fn unused_loopback_addr() -> String { @@ -648,7 +649,8 @@ mod tests { let write_result = tls_stream.write_all(b"blocked").await; if write_result.is_ok() { let mut buf = [0u8; 1]; - let read_result = tokio::time::timeout(Duration::from_secs(1), tls_stream.read(&mut buf)).await; + let read_result = + tokio::time::timeout(Duration::from_secs(1), tls_stream.read(&mut buf)).await; match read_result { Ok(Ok(0)) | Ok(Err(_)) | Err(_) => {} Ok(Ok(_)) => panic!("mTLS mode accepted traffic without a client certificate"), diff --git a/mtls-agent/src/policy.rs b/mtls-agent/src/policy.rs index b7f71c6..a91f8c2 100644 --- a/mtls-agent/src/policy.rs +++ b/mtls-agent/src/policy.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] + use std::sync::Arc; use std::time::{Duration, Instant}; diff --git a/nix/ci/flake.nix b/nix/ci/flake.nix index 607eb2f..197d60f 100644 --- a/nix/ci/flake.nix +++ b/nix/ci/flake.nix @@ -27,21 +27,9 @@ rustfmtComponent = pkgs.rust-bin.stable.latest.rustfmt; clippyComponent = pkgs.rust-bin.stable.latest.clippy; - wsList = [ - "chainfire" - "flaredb" - "iam" - "plasmavmc" - "prismnet" - "flashdns" - "fiberlb" - "lightningstor" - "nightlight" - "creditservice" - "k8shost" - "apigateway" - "deployer" - ]; + ciInventory = builtins.fromJSON (builtins.readFile ./workspaces.json); + wsList = map (workspace: workspace.name) ciInventory.workspaces; + supportedWorkspaces = pkgs.lib.concatStringsSep ", " wsList; gate = pkgs.writeShellApplication { name = "photoncloud-gate"; @@ -53,6 +41,7 @@ gnugrep gawk git + jq rustToolchain rustfmtComponent clippyComponent @@ -61,6 +50,7 @@ llvmPackages.clang pkg-config openssl + qemu rocksdb ]; @@ -82,6 +72,7 @@ Notes: - Requires running inside a git checkout (uses `git rev-parse`). - Logs are written to ./work/ci// by default (NOT .cccc/). + - Supported workspaces: ${supportedWorkspaces} USAGE } @@ -143,6 +134,7 @@ pkgs.gnugrep pkgs.gawk pkgs.git + pkgs.jq rustToolchain rustfmtComponent clippyComponent @@ -150,6 +142,8 @@ pkgs.llvmPackages.libclang pkgs.llvmPackages.clang pkgs.pkg-config + pkgs.openssl + pkgs.qemu ]}" CARGO="${rustToolchain}/bin/cargo" @@ -166,6 +160,14 @@ export PROTOC="${pkgs.protobuf}/bin/protoc" export ROCKSDB_LIB_DIR="${pkgs.rocksdb}/lib" + manifest_has_target_kind() { + local manifest="$1"; shift + local kind="$1"; shift + + "$CARGO" metadata --format-version 1 --no-deps --manifest-path "$manifest" \ + | jq -e --arg kind "$kind" 'any(.packages[]?.targets[]?.kind[]?; . == $kind)' > /dev/null + } + run_cmd() { local ws="$1"; shift local title="$1"; shift @@ -222,10 +224,22 @@ for manifest in "''${manifests[@]}"; do local crate + local ran_unit_tests crate="$(basename "$(dirname "$manifest")")" run_shared_crate_cmd "$crate" "$manifest" "fmt" "$CARGO_FMT fmt --manifest-path \"$manifest\" $fmt_rustfmt_args" run_shared_crate_cmd "$crate" "$manifest" "clippy" "$CARGO_CLIPPY clippy --manifest-path \"$manifest\" --all-targets -- -D warnings" - run_shared_crate_cmd "$crate" "$manifest" "test (tier0 unit)" "$CARGO test --manifest-path \"$manifest\" --lib" + ran_unit_tests="0" + if manifest_has_target_kind "$manifest" "lib"; then + run_shared_crate_cmd "$crate" "$manifest" "test (tier0 unit lib)" "$CARGO test --manifest-path \"$manifest\" --lib" + ran_unit_tests="1" + fi + if manifest_has_target_kind "$manifest" "bin"; then + run_shared_crate_cmd "$crate" "$manifest" "test (tier0 unit bin)" "$CARGO test --manifest-path \"$manifest\" --bins" + ran_unit_tests="1" + fi + if [[ "$ran_unit_tests" == "0" ]]; then + echo "[gate][shared:$crate] WARN: no lib/bin unit test targets" + fi if [[ "$tier" == "1" || "$tier" == "2" ]]; then run_shared_crate_cmd "$crate" "$manifest" "test (tier1 integration)" "$CARGO test --manifest-path \"$manifest\" --tests" @@ -254,13 +268,26 @@ continue fi + workspace_manifest="$repo_root/$ws/Cargo.toml" + # Format gate: call Nix-provided `cargo-fmt` directly (avoid resolving ~/.cargo/bin/cargo-fmt). # # NOTE: Avoid `--all` here; with path-dependencies it may traverse outside the workspace directory. run_cmd "$ws" "fmt" "$CARGO_FMT fmt $fmt_rustfmt_args" # Lint gate: call Nix-provided `cargo-clippy` directly (avoid resolving ~/.cargo/bin/cargo-clippy). run_cmd "$ws" "clippy" "$CARGO_CLIPPY clippy --workspace --all-targets -- -D warnings" - run_cmd "$ws" "test (tier0 unit)" "$CARGO test --workspace --lib" + ran_unit_tests="0" + if manifest_has_target_kind "$workspace_manifest" "lib"; then + run_cmd "$ws" "test (tier0 unit lib)" "$CARGO test --workspace --lib" + ran_unit_tests="1" + fi + if manifest_has_target_kind "$workspace_manifest" "bin"; then + run_cmd "$ws" "test (tier0 unit bin)" "$CARGO test --workspace --bins" + ran_unit_tests="1" + fi + if [[ "$ran_unit_tests" == "0" ]]; then + echo "[gate][$ws] WARN: no lib/bin unit test targets" + fi if [[ "$tier" == "1" || "$tier" == "2" ]]; then run_cmd "$ws" "test (tier1 integration)" "$CARGO test --workspace --tests" diff --git a/nix/ci/workspaces.json b/nix/ci/workspaces.json new file mode 100644 index 0000000..e9ccafe --- /dev/null +++ b/nix/ci/workspaces.json @@ -0,0 +1,166 @@ +{ + "global_paths": [ + "flake.nix", + "flake.lock", + "shell.nix", + "nix/**", + "nix-nos/**", + ".github/workflows/nix.yml", + "Cargo.toml", + "Cargo.lock", + "crates/**", + "baremetal/**", + "scripts/**", + "specifications/**", + "docs/**" + ], + "shared_crates_paths": [ + "crates/**" + ], + "workspaces": [ + { + "name": "chainfire", + "paths": [ + "chainfire/**" + ], + "build_packages": [ + "chainfire-server" + ] + }, + { + "name": "flaredb", + "paths": [ + "flaredb/**" + ], + "build_packages": [ + "flaredb-server" + ] + }, + { + "name": "iam", + "paths": [ + "iam/**" + ], + "build_packages": [ + "iam-server" + ] + }, + { + "name": "coronafs", + "paths": [ + "coronafs/**" + ], + "build_packages": [ + "coronafs-server" + ] + }, + { + "name": "plasmavmc", + "paths": [ + "plasmavmc/**" + ], + "build_packages": [ + "plasmavmc-server" + ] + }, + { + "name": "prismnet", + "paths": [ + "prismnet/**" + ], + "build_packages": [ + "prismnet-server" + ] + }, + { + "name": "flashdns", + "paths": [ + "flashdns/**" + ], + "build_packages": [ + "flashdns-server" + ] + }, + { + "name": "fiberlb", + "paths": [ + "fiberlb/**" + ], + "build_packages": [ + "fiberlb-server" + ] + }, + { + "name": "lightningstor", + "paths": [ + "lightningstor/**" + ], + "build_packages": [ + "lightningstor-server", + "lightningstor-node" + ] + }, + { + "name": "nightlight", + "paths": [ + "nightlight/**" + ], + "build_packages": [ + "nightlight-server" + ] + }, + { + "name": "creditservice", + "paths": [ + "creditservice/**" + ], + "build_packages": [ + "creditservice-server" + ] + }, + { + "name": "k8shost", + "paths": [ + "k8shost/**" + ], + "build_packages": [ + "k8shost-server" + ] + }, + { + "name": "apigateway", + "paths": [ + "apigateway/**" + ], + "build_packages": [ + "apigateway-server" + ] + }, + { + "name": "deployer", + "paths": [ + "deployer/**" + ], + "build_packages": [ + "deployer-server", + "deployer-ctl", + "node-agent", + "nix-agent", + "plasmacloud-reconciler", + "fleet-scheduler" + ] + }, + { + "name": "client-common", + "paths": [ + "client-common/**" + ] + }, + { + "name": "mtls-agent", + "paths": [ + "mtls-agent/**" + ] + } + ] +} diff --git a/scripts/ci_changed_workspaces.py b/scripts/ci_changed_workspaces.py new file mode 100644 index 0000000..c3e5bb4 --- /dev/null +++ b/scripts/ci_changed_workspaces.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +import argparse +import fnmatch +import json +from pathlib import Path +from typing import Any + + +def load_changed_files(args: argparse.Namespace) -> list[str]: + changed_files: list[str] = [] + + for path in args.changed_files_file: + for line in Path(path).read_text().splitlines(): + candidate = line.strip() + if candidate: + changed_files.append(candidate) + + changed_files.extend(path.strip() for path in args.changed_file if path.strip()) + return changed_files + + +def matches_any(path: str, patterns: list[str]) -> bool: + return any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns) + + +def detect_changes(config: dict[str, Any], changed_files: list[str]) -> dict[str, Any]: + workspaces: list[dict[str, Any]] = config["workspaces"] + all_workspace_names = [workspace["name"] for workspace in workspaces] + + global_changed = any( + matches_any(path, config["global_paths"]) + for path in changed_files + ) + shared_crates_changed = any( + matches_any(path, config["shared_crates_paths"]) + for path in changed_files + ) + + if global_changed: + changed_workspaces = all_workspace_names + else: + changed_workspaces = [ + workspace["name"] + for workspace in workspaces + if any(matches_any(path, workspace["paths"]) for path in changed_files) + ] + + selected_workspaces = set(changed_workspaces) + build_targets: list[dict[str, str]] = [] + seen_build_targets: set[tuple[str, str]] = set() + + for workspace in workspaces: + if workspace["name"] not in selected_workspaces: + continue + + for package in workspace.get("build_packages", []): + key = (workspace["name"], package) + if key in seen_build_targets: + continue + seen_build_targets.add(key) + build_targets.append({ + "workspace": workspace["name"], + "package": package, + }) + + return { + "workspaces": changed_workspaces, + "build_targets": build_targets, + "any_changed": global_changed or bool(changed_workspaces), + "build_changed": bool(build_targets), + "global_changed": global_changed, + "shared_crates_changed": shared_crates_changed, + } + + +def write_github_output(path: Path, result: dict[str, Any]) -> None: + serialized = { + "workspaces": json.dumps(result["workspaces"], separators=(",", ":")), + "build_targets": json.dumps(result["build_targets"], separators=(",", ":")), + "any_changed": str(result["any_changed"]).lower(), + "build_changed": str(result["build_changed"]).lower(), + "global_changed": str(result["global_changed"]).lower(), + "shared_crates_changed": str(result["shared_crates_changed"]).lower(), + } + + with path.open("a", encoding="utf-8") as handle: + for key, value in serialized.items(): + handle.write(f"{key}={value}\n") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Map changed files to PhotonCloud CI workspaces." + ) + parser.add_argument( + "--config", + required=True, + help="Path to the JSON CI workspace inventory.", + ) + parser.add_argument( + "--changed-files-file", + action="append", + default=[], + help="File containing newline-separated changed paths.", + ) + parser.add_argument( + "--changed-file", + action="append", + default=[], + help="Single changed path. Can be repeated.", + ) + parser.add_argument( + "--github-output", + help="Optional path to append GitHub Actions step outputs.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + config = json.loads(Path(args.config).read_text()) + changed_files = load_changed_files(args) + result = detect_changes(config, changed_files) + + if args.github_output: + write_github_output(Path(args.github_output), result) + + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())