use std::collections::HashSet;
use std::future::Future;
use std::path::Path;

use anyhow::{Context, Result};
use chainfire_client::{Client, ClientError};
use chrono::Utc;
use deployer_types::{
    BootstrapPlan, ClusterNodeRecord, ClusterStateSpec, CommissionState, DesiredSystemSpec,
    HostDeploymentSpec, HostDeploymentStatus, InstallPlan, InstallState, NodeAssignment,
    NodeConfig, NodeSpec, ObservedSystemState, PowerState, ServiceInstanceSpec,
    ServicePublicationState, ServiceSpec, ServiceStatusRecord,
};
use serde::de::DeserializeOwned;
use serde_json::{json, Value};
use tokio::fs;
use tokio::time::{sleep, Duration};
use tracing::{info, warn};

const CHAINFIRE_RETRY_LIMIT: usize = 8;
const CHAINFIRE_RETRY_BASE_DELAY_MS: u64 = 250;
const CHAINFIRE_RETRY_MAX_DELAY_MS: u64 = 2_000;

fn cluster_prefix(cluster_namespace: &str, cluster_id: &str) -> String {
    format!("{}/clusters/{}/", cluster_namespace, cluster_id)
}

fn key_cluster_meta(cluster_namespace: &str, cluster_id: &str) -> Vec<u8> {
    format!("{}meta", cluster_prefix(cluster_namespace, cluster_id)).into_bytes()
}

fn deployer_node_config_key(deployer_namespace: &str, machine_id: &str) -> Vec<u8> {
    format!("{}/nodes/config/{}", deployer_namespace, machine_id).into_bytes()
}

fn key_node(cluster_namespace: &str, cluster_id: &str, node_id: &str) -> Vec<u8> {
    format!(
        "{}nodes/{}",
        cluster_prefix(cluster_namespace, cluster_id),
        node_id
    )
    .into_bytes()
}

fn key_desired_system(cluster_namespace: &str, cluster_id: &str, node_id: &str) -> Vec<u8> {
    format!(
        "{}nodes/{}/desired-system",
        cluster_prefix(cluster_namespace, cluster_id),
        node_id
    )
    .into_bytes()
}

fn key_observed_system(cluster_namespace: &str, cluster_id: &str, node_id: &str) -> Vec<u8> {
    format!(
        "{}nodes/{}/observed-system",
        cluster_prefix(cluster_namespace, cluster_id),
        node_id
    )
    .into_bytes()
}

fn key_host_deployment_spec(
    cluster_namespace: &str,
    cluster_id: &str,
    deployment_name: &str,
) -> Vec<u8> {
    format!(
        "{}deployments/hosts/{}/spec",
        cluster_prefix(cluster_namespace, cluster_id),
        deployment_name
    )
    .into_bytes()
}

fn key_host_deployment_status(
    cluster_namespace: &str,
    cluster_id: &str,
    deployment_name: &str,
) -> Vec<u8> {
    format!(
        "{}deployments/hosts/{}/status",
        cluster_prefix(cluster_namespace, cluster_id),
        deployment_name
    )
    .into_bytes()
}

fn parse_commission_state(value: &str) -> Result<CommissionState> {
    serde_json::from_str(&format!("\"{value}\""))
        .with_context(|| format!("invalid commission state {value}"))
}

fn parse_install_state(value: &str) -> Result<InstallState> {
    serde_json::from_str(&format!("\"{value}\""))
        .with_context(|| format!("invalid install state {value}"))
}

fn parse_power_state(value: &str) -> Result<PowerState> {
    serde_json::from_str(&format!("\"{value}\""))
        .with_context(|| format!("invalid power state {value}"))
}

fn key_node_class(cluster_namespace: &str, cluster_id: &str, node_class: &str) -> Vec<u8> {
    format!(
        "{}node-classes/{}",
        cluster_prefix(cluster_namespace, cluster_id),
        node_class
    )
    .into_bytes()
}

fn key_pool(cluster_namespace: &str, cluster_id: &str, pool: &str) -> Vec<u8> {
    format!(
        "{}pools/{}",
        cluster_prefix(cluster_namespace, cluster_id),
        pool
    )
    .into_bytes()
}

fn key_enrollment_rule(cluster_namespace: &str, cluster_id: &str, rule: &str) -> Vec<u8> {
    format!(
        "{}enrollment-rules/{}",
        cluster_prefix(cluster_namespace, cluster_id),
        rule
    )
    .into_bytes()
}

fn key_service(cluster_namespace: &str, cluster_id: &str, svc: &str) -> Vec<u8> {
    format!(
        "{}services/{}",
        cluster_prefix(cluster_namespace, cluster_id),
        svc
    )
    .into_bytes()
}

fn key_service_status(cluster_namespace: &str, cluster_id: &str, service: &str) -> Vec<u8> {
    format!(
        "{}service-statuses/{}",
        cluster_prefix(cluster_namespace, cluster_id),
        service
    )
    .into_bytes()
}

fn key_publication(cluster_namespace: &str, cluster_id: &str, service: &str) -> Vec<u8> {
    format!(
        "{}publications/{}",
        cluster_prefix(cluster_namespace, cluster_id),
        service
    )
    .into_bytes()
}

fn key_instance(cluster_namespace: &str, cluster_id: &str, svc: &str, inst: &str) -> Vec<u8> {
    format!(
        "{}instances/{}/{}",
        cluster_prefix(cluster_namespace, cluster_id),
        svc,
        inst
    )
    .into_bytes()
}

fn key_mtls_policy(cluster_namespace: &str, cluster_id: &str, policy_id: &str) -> Vec<u8> {
    format!(
        "{}mtls/policies/{}",
        cluster_prefix(cluster_namespace, cluster_id),
        policy_id
    )
    .into_bytes()
}

async fn read_config_file<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> Result<T> {
    let contents = fs::read_to_string(&path)
        .await
        .with_context(|| format!("failed to read {}", path.as_ref().display()))?;

    let json_result = serde_json::from_str(&contents);
    if let Ok(value) = json_result {
        return Ok(value);
    }

    let yaml_result = serde_yaml::from_str(&contents);
    if let Ok(value) = yaml_result {
        return Ok(value);
    }

    let path_display = path.as_ref().display();
    Err(anyhow::anyhow!(
        "failed to parse {} as JSON or YAML",
        path_display
    ))
}

fn merge_unique_strings(base: &[String], extra: &[String]) -> Vec<String> {
    let mut merged = base.to_vec();
    for value in extra {
        if !merged.iter().any(|existing| existing == value) {
            merged.push(value.clone());
        }
    }
    merged
}

fn merge_install_plan(
    preferred: Option<&InstallPlan>,
    fallback: Option<&InstallPlan>,
) -> Option<InstallPlan> {
    InstallPlan::from_layers(preferred, fallback)
}

fn node_config_from_spec(node: &NodeSpec) -> NodeConfig {
    let assignment = NodeAssignment {
        node_id: node.node_id.clone(),
        hostname: node.hostname.clone(),
        role: node
            .roles
            .first()
            .cloned()
            .unwrap_or_else(|| "worker".to_string()),
        ip: node.ip.clone(),
        labels: node.labels.clone(),
        pool: node.pool.clone(),
        node_class: node.node_class.clone(),
        failure_domain: node.failure_domain.clone(),
    };
    let bootstrap_plan = BootstrapPlan {
        services: Vec::new(),
        nix_profile: node.nix_profile.clone(),
        install_plan: node.install_plan.clone(),
    };
    NodeConfig::from_parts(
        assignment,
        bootstrap_plan,
        deployer_types::BootstrapSecrets::default(),
    )
}

fn desired_system_from_spec(node: &NodeSpec) -> Option<DesiredSystemSpec> {
    let mut desired = node.desired_system.clone().unwrap_or_default();
    desired.node_id = node.node_id.clone();
    if desired.nixos_configuration.is_none() {
        desired.nixos_configuration = node
            .install_plan
            .as_ref()
            .and_then(|plan| plan.nixos_configuration.clone());
    }
    if desired.switch_action.is_none() {
        desired.switch_action = Some("switch".to_string());
    }
    if desired.rollback_on_failure.is_none() {
        desired.rollback_on_failure = Some(true);
    }
    if desired.drain_before_apply.is_none() {
        desired.drain_before_apply = Some(false);
    }
    if desired.nixos_configuration.is_some() || desired.target_system.is_some() {
        Some(desired)
    } else {
        None
    }
}

fn resolve_nodes(spec: &ClusterStateSpec) -> Result<Vec<NodeSpec>> {
    let node_classes = spec
        .node_classes
        .iter()
        .map(|node_class| (node_class.name.as_str(), node_class))
        .collect::<std::collections::HashMap<_, _>>();
    let pools = spec
        .pools
        .iter()
        .map(|pool| (pool.name.as_str(), pool))
        .collect::<std::collections::HashMap<_, _>>();

    let mut resolved_nodes = Vec::with_capacity(spec.nodes.len());

    for node in &spec.nodes {
        let mut resolved = node.clone();

        let pool_spec = match resolved.pool.as_deref() {
            Some(pool_name) => Some(pools.get(pool_name).copied().with_context(|| {
                format!(
                    "node {} references unknown pool {}",
                    node.node_id, pool_name
                )
            })?),
            None => None,
        };

        if resolved.node_class.is_none() {
            resolved.node_class = pool_spec.and_then(|pool| pool.node_class.clone());
        }

        let node_class_spec = match resolved.node_class.as_deref() {
            Some(node_class_name) => Some(
                node_classes
                    .get(node_class_name)
                    .copied()
                    .with_context(|| {
                        format!(
                            "node {} references unknown node class {}",
                            node.node_id, node_class_name
                        )
                    })?,
            ),
            None => None,
        };

        if let Some(node_class) = node_class_spec {
            resolved.roles = merge_unique_strings(&node_class.roles, &resolved.roles);
            for (key, value) in &node_class.labels {
                resolved
                    .labels
                    .entry(key.clone())
                    .or_insert_with(|| value.clone());
            }
            if resolved.nix_profile.is_none() {
                resolved.nix_profile = node_class.nix_profile.clone();
            }
            resolved.install_plan = merge_install_plan(
                resolved.install_plan.as_ref(),
                node_class.install_plan.as_ref(),
            );
        }

        if let Some(pool) = pool_spec {
            for (key, value) in &pool.labels {
                resolved
                    .labels
                    .entry(key.clone())
                    .or_insert_with(|| value.clone());
            }
        }

        if let Some(pool) = resolved.pool.as_deref() {
            resolved
                .labels
                .entry("pool".to_string())
                .or_insert_with(|| pool.to_string());
            resolved
                .labels
                .entry("pool.ultracloud.io/name".to_string())
                .or_insert_with(|| pool.to_string());
        }

        if let Some(node_class) = resolved.node_class.as_deref() {
            resolved
                .labels
                .entry("node_class".to_string())
                .or_insert_with(|| node_class.to_string());
            resolved
                .labels
                .entry("nodeclass.ultracloud.io/name".to_string())
                .or_insert_with(|| node_class.to_string());
        }

        if let Some(failure_domain) = resolved.failure_domain.as_deref() {
            resolved
                .labels
                .entry("failure_domain".to_string())
                .or_insert_with(|| failure_domain.to_string());
            resolved
                .labels
                .entry("topology.kubernetes.io/zone".to_string())
                .or_insert_with(|| failure_domain.to_string());
        }

        resolved_nodes.push(resolved);
    }

    Ok(resolved_nodes)
}

async fn merge_existing_node_observed_fields(
    client: &mut Client,
    key: &[u8],
    node: &NodeSpec,
) -> Result<NodeSpec> {
    let mut merged = node.clone();

    let Some(existing) = client.get(key).await? else {
        return Ok(merged);
    };

    let existing_node = match serde_json::from_slice::<NodeSpec>(&existing) {
        Ok(existing_node) => existing_node,
        Err(error) => {
            warn!(
                error = %error,
                key = %String::from_utf8_lossy(key),
                "failed to decode existing node while preserving observed fields"
            );
            return Ok(merged);
        }
    };

    if merged.state.is_none() {
        merged.state = existing_node.state;
    }
    if merged.machine_id.is_none() {
        merged.machine_id = existing_node.machine_id;
    }
    if merged.hardware_facts.is_none() {
        merged.hardware_facts = existing_node.hardware_facts;
    }
    if merged.commission_state.is_none() {
        merged.commission_state = existing_node.commission_state;
    }
    if merged.install_state.is_none() {
        merged.install_state = existing_node.install_state;
    }
    if merged.commissioned_at.is_none() {
        merged.commissioned_at = existing_node.commissioned_at;
    }
    if merged.last_inventory_hash.is_none() {
        merged.last_inventory_hash = existing_node.last_inventory_hash;
    }
    if merged.power_state.is_none() {
        merged.power_state = existing_node.power_state;
    }
    if merged.bmc_ref.is_none() {
        merged.bmc_ref = existing_node.bmc_ref;
    }
    if merged.last_heartbeat.is_none() {
        merged.last_heartbeat = existing_node.last_heartbeat;
    }

    Ok(merged)
}

async fn with_chainfire_endpoint_failover<T, F, Fut>(
    endpoints: &[String],
    operation: &'static str,
    mut action: F,
) -> Result<T>
where
    F: FnMut(&str) -> Fut,
    Fut: Future<Output = Result<T>>,
{
    let endpoints = if endpoints.is_empty() {
        vec!["http://127.0.0.1:7000".to_string()]
    } else {
        endpoints.to_vec()
    };
    let mut last_retryable_error: Option<anyhow::Error> = None;

    for attempt in 0..=CHAINFIRE_RETRY_LIMIT {
        for endpoint in &endpoints {
            match action(endpoint).await {
                Ok(value) => return Ok(value),
                Err(error) => {
                    if !is_retryable_chainfire_error(&error) {
                        return Err(error);
                    }

                    warn!(
                        attempt = attempt + 1,
                        endpoint,
                        error = %error,
                        operation,
                        "retrying Chainfire operation on alternate endpoint"
                    );
                    last_retryable_error = Some(error);
                }
            }
        }

        if attempt == CHAINFIRE_RETRY_LIMIT {
            break;
        }

        let delay = retry_delay(attempt);
        warn!(
            attempt = attempt + 1,
            retry_after_ms = delay.as_millis() as u64,
            endpoints = ?endpoints,
            operation,
            "retrying transient Chainfire operation across endpoints"
        );
        sleep(delay).await;
    }

    Err(last_retryable_error
        .unwrap_or_else(|| anyhow::anyhow!("no Chainfire endpoints configured")))
}

fn chainfire_endpoints(raw: &str) -> Vec<String> {
    raw.split(',')
        .map(str::trim)
        .filter(|endpoint| !endpoint.is_empty())
        .map(ToOwned::to_owned)
        .collect()
}

fn retry_delay(attempt: usize) -> Duration {
    let factor = 1u64 << attempt.min(3);
    Duration::from_millis(
        (CHAINFIRE_RETRY_BASE_DELAY_MS.saturating_mul(factor)).min(CHAINFIRE_RETRY_MAX_DELAY_MS),
    )
}

fn is_retryable_chainfire_error(error: &anyhow::Error) -> bool {
    if let Some(client_error) = error.downcast_ref::<ClientError>() {
        return match client_error {
            ClientError::Connection(_) | ClientError::Transport(_) => true,
            ClientError::Rpc(status) => {
                matches!(
                    format!("{:?}", status.code()).as_str(),
                    "Cancelled" | "Unavailable" | "DeadlineExceeded" | "Unknown"
                ) || retryable_chainfire_message(status.message())
            }
            _ => false,
        };
    }

    error
        .chain()
        .any(|cause| retryable_chainfire_message(&cause.to_string()))
}

fn retryable_chainfire_message(message: &str) -> bool {
    [
        "connection was not ready",
        "operation was canceled",
        "transport error",
        "connection reset by peer",
        "broken pipe",
        "deadline has elapsed",
        "timed out",
        "server closed the connection",
        "NotLeader",
        "leader_id: None",
        "leader unavailable",
    ]
    .iter()
    .any(|pattern| message.contains(pattern))
}

/// 初回ブートストラップ:
/// - Cluster メタを作り
/// - 少なくとも 1 台分の Node / 必要なら Service/Instance を作成
pub async fn bootstrap_cluster(
    endpoint: &str,
    cluster_namespace: &str,
    deployer_namespace: &str,
    cli_cluster_id: Option<&str>,
    config_path: &Path,
) -> Result<()> {
    let endpoints = chainfire_endpoints(endpoint);
    with_chainfire_endpoint_failover(&endpoints, "bootstrap cluster state", |endpoint| {
        let endpoint = endpoint.to_string();
        async move {
        let spec: ClusterStateSpec = read_config_file(config_path).await?;
        let resolved_nodes = resolve_nodes(&spec)?;
        let cluster_id = cli_cluster_id.unwrap_or(&spec.cluster.cluster_id);

        info!(cluster_id, "connecting to Chainfire at {}", endpoint);
        let mut client = Client::connect(endpoint.to_string()).await?;

        // 1. Cluster メタ
        let meta_key = key_cluster_meta(cluster_namespace, cluster_id);
        let meta_value = serde_json::to_vec(&spec.cluster)?;
        client.put(&meta_key, &meta_value).await?;
        info!("upserted cluster meta for {}", cluster_id);

        // 2. Node
        for node in &resolved_nodes {
            let key = key_node(cluster_namespace, cluster_id, &node.node_id);
            let merged = merge_existing_node_observed_fields(&mut client, &key, node).await?;
            let value = serde_json::to_vec(&merged)?;
            client.put(&key, &value).await?;
            info!(node_id = %node.node_id, "upserted node");

            if let Some(desired_system) = desired_system_from_spec(node) {
                client
                    .put(
                        &key_desired_system(cluster_namespace, cluster_id, &node.node_id),
                        &serde_json::to_vec(&desired_system)?,
                    )
                    .await?;
                info!(node_id = %node.node_id, "upserted desired system");
            }

            if let Some(machine_id) = node.machine_id.as_deref() {
                let config = node_config_from_spec(node);
                client
                    .put(
                        &deployer_node_config_key(deployer_namespace, machine_id),
                        serde_json::to_vec(&config)?,
                    )
                    .await?;
                info!(node_id = %node.node_id, machine_id = %machine_id, "seeded deployer bootstrap config");
            }
        }

        for node_class in &spec.node_classes {
            let key = key_node_class(cluster_namespace, cluster_id, &node_class.name);
            let value = serde_json::to_vec(node_class)?;
            client.put(&key, &value).await?;
            info!(node_class = %node_class.name, "upserted node class");
        }

        for pool in &spec.pools {
            let key = key_pool(cluster_namespace, cluster_id, &pool.name);
            let value = serde_json::to_vec(pool)?;
            client.put(&key, &value).await?;
            info!(pool = %pool.name, "upserted pool");
        }

        for rule in &spec.enrollment_rules {
            let key = key_enrollment_rule(cluster_namespace, cluster_id, &rule.name);
            let value = serde_json::to_vec(rule)?;
            client.put(&key, &value).await?;
            info!(enrollment_rule = %rule.name, "upserted enrollment rule");
        }

        for deployment in &spec.host_deployments {
            let key = key_host_deployment_spec(cluster_namespace, cluster_id, &deployment.name);
            let value = serde_json::to_vec(deployment)?;
            client.put(&key, &value).await?;
            info!(deployment = %deployment.name, "upserted host deployment");
        }

        // 3. Service / Instance (必要であれば)
        for svc in &spec.services {
            let key = key_service(cluster_namespace, cluster_id, &svc.name);
            let value = serde_json::to_vec(svc)?;
            client.put(&key, &value).await?;
            info!(service = %svc.name, "upserted service");
        }

        for inst in &spec.instances {
            let key = key_instance(cluster_namespace, cluster_id, &inst.service, &inst.instance_id);
            let value = serde_json::to_vec(inst)?;
            client.put(&key, &value).await?;
            info!(instance = %inst.instance_id, service = %inst.service, "upserted instance");
        }

        // 4. mTLS Policy
        for policy in &spec.mtls_policies {
            let key = key_mtls_policy(cluster_namespace, cluster_id, &policy.policy_id);
            let value = serde_json::to_vec(policy)?;
            client.put(&key, &value).await?;
            info!(policy_id = %policy.policy_id, "upserted mTLS policy");
        }

        Ok(())
        }
    })
    .await
}

/// GitOps 的に、「クラスタ全体の宣言」を Chainfire に apply する。
/// prune=true の場合、指定 prefix から外れたキーを削除する方向にも拡張可能。
pub async fn apply_cluster_state(
    endpoint: &str,
    cluster_namespace: &str,
    deployer_namespace: &str,
    cli_cluster_id: Option<&str>,
    config_path: &Path,
    prune: bool,
) -> Result<()> {
    let endpoints = chainfire_endpoints(endpoint);
    with_chainfire_endpoint_failover(&endpoints, "apply cluster state", |endpoint| {
        let endpoint = endpoint.to_string();
        async move {
            let spec: ClusterStateSpec = read_config_file(config_path).await?;
            let resolved_nodes = resolve_nodes(&spec)?;
            let cluster_id = cli_cluster_id.unwrap_or(&spec.cluster.cluster_id);

            info!(
                cluster_id,
                "applying cluster state to Chainfire at {}", endpoint
            );
            let mut client = Client::connect(endpoint.to_string()).await?;

            // MVP としては bootstrap と同じく upsert のみ行う。
            // 将来的に、既存一覧を取得して差分削除 (prune) を実装できる構造にしておく。
            let meta_key = key_cluster_meta(cluster_namespace, cluster_id);
            let meta_value = serde_json::to_vec(&spec.cluster)?;
            client.put(&meta_key, &meta_value).await?;

            for node in &resolved_nodes {
                let key = key_node(cluster_namespace, cluster_id, &node.node_id);
                let merged = merge_existing_node_observed_fields(&mut client, &key, node).await?;
                let value = serde_json::to_vec(&merged)?;
                client.put(&key, &value).await?;

                if let Some(desired_system) = desired_system_from_spec(node) {
                    client
                        .put(
                            &key_desired_system(cluster_namespace, cluster_id, &node.node_id),
                            &serde_json::to_vec(&desired_system)?,
                        )
                        .await?;
                }

                if let Some(machine_id) = node.machine_id.as_deref() {
                    let config = node_config_from_spec(node);
                    client
                        .put(
                            &deployer_node_config_key(deployer_namespace, machine_id),
                            serde_json::to_vec(&config)?,
                        )
                        .await?;
                }
            }

            for node_class in &spec.node_classes {
                let key = key_node_class(cluster_namespace, cluster_id, &node_class.name);
                let value = serde_json::to_vec(node_class)?;
                client.put(&key, &value).await?;
            }
            for pool in &spec.pools {
                let key = key_pool(cluster_namespace, cluster_id, &pool.name);
                let value = serde_json::to_vec(pool)?;
                client.put(&key, &value).await?;
            }
            for rule in &spec.enrollment_rules {
                let key = key_enrollment_rule(cluster_namespace, cluster_id, &rule.name);
                let value = serde_json::to_vec(rule)?;
                client.put(&key, &value).await?;
            }
            for deployment in &spec.host_deployments {
                let key = key_host_deployment_spec(cluster_namespace, cluster_id, &deployment.name);
                let value = serde_json::to_vec(deployment)?;
                client.put(&key, &value).await?;
            }
            for svc in &spec.services {
                let key = key_service(cluster_namespace, cluster_id, &svc.name);
                let value = serde_json::to_vec(svc)?;
                client.put(&key, &value).await?;
            }
            for inst in &spec.instances {
                let key = key_instance(
                    cluster_namespace,
                    cluster_id,
                    &inst.service,
                    &inst.instance_id,
                );
                let value = serde_json::to_vec(inst)?;
                client.put(&key, &value).await?;
            }
            for policy in &spec.mtls_policies {
                let key = key_mtls_policy(cluster_namespace, cluster_id, &policy.policy_id);
                let value = serde_json::to_vec(policy)?;
                client.put(&key, &value).await?;
            }

            if prune {
                prune_cluster_state(
                    &mut client,
                    cluster_namespace,
                    deployer_namespace,
                    cluster_id,
                    &spec,
                )
                .await?;
            }

            Ok(())
        }
    })
    .await
}

/// 指定 prefix 以下のキーをダンプする（デバッグ・手動修復用）。
pub async fn dump_prefix(endpoint: &str, prefix: &str, json_output: bool) -> Result<()> {
    let endpoints = chainfire_endpoints(endpoint);
    with_chainfire_endpoint_failover(&endpoints, "dump Chainfire prefix", |endpoint| {
        let endpoint = endpoint.to_string();
        async move {
            let mut client = Client::connect(endpoint.to_string()).await?;
            let start = prefix.as_bytes();

            info!("dumping keys with prefix {:?}", prefix);
            let (kvs, _next) = client.scan_prefix(start, 0).await?;
            if kvs.is_empty() {
                warn!("no keys found under prefix {:?}", prefix);
            }

            for (key, value, rev) in kvs {
                let k = String::from_utf8_lossy(&key);
                if json_output {
                    let value = serde_json::from_slice::<Value>(&value).unwrap_or_else(|_| {
                        Value::String(String::from_utf8_lossy(&value).into_owned())
                    });
                    println!(
                        "{}",
                        serde_json::to_string(&json!({
                            "revision": rev,
                            "key": k.as_ref(),
                            "value": value,
                        }))?
                    );
                } else {
                    let v = String::from_utf8_lossy(&value);
                    println!("rev={} key={} value={}", rev, k, v);
                }
            }

            Ok(())
        }
    })
    .await
}

async fn get_json_key<T: DeserializeOwned>(client: &mut Client, key: &[u8]) -> Result<Option<T>> {
    client
        .get(key)
        .await?
        .map(|bytes| serde_json::from_slice::<T>(&bytes))
        .transpose()
        .with_context(|| format!("failed to decode key {}", String::from_utf8_lossy(key)))
}

pub async fn inspect_node(
    endpoint: &str,
    cluster_namespace: &str,
    cluster_id: &str,
    node_id: &str,
    include_desired_system: bool,
    include_observed_system: bool,
    json_output: bool,
) -> Result<()> {
    let endpoints = chainfire_endpoints(endpoint);
    with_chainfire_endpoint_failover(&endpoints, "inspect node", |endpoint| {
        let endpoint = endpoint.to_string();
        let cluster_namespace = cluster_namespace.to_string();
        let cluster_id = cluster_id.to_string();
        let node_id = node_id.to_string();
        async move {
            let mut client = Client::connect(endpoint).await?;
            let node = get_json_key::<ClusterNodeRecord>(
                &mut client,
                &key_node(&cluster_namespace, &cluster_id, &node_id),
            )
            .await?
            .with_context(|| format!("node {} not found", node_id))?;

            let desired_system = if include_desired_system {
                get_json_key::<DesiredSystemSpec>(
                    &mut client,
                    &key_desired_system(&cluster_namespace, &cluster_id, &node_id),
                )
                .await?
            } else {
                None
            };

            let observed_system = if include_observed_system {
                get_json_key::<ObservedSystemState>(
                    &mut client,
                    &key_observed_system(&cluster_namespace, &cluster_id, &node_id),
                )
                .await?
            } else {
                None
            };

            if json_output {
                println!(
                    "{}",
                    serde_json::to_string_pretty(&json!({
                        "node": node,
                        "desired_system": desired_system,
                        "observed_system": observed_system,
                    }))?
                );
            } else {
                println!("node_id={}", node.node_id);
                println!("hostname={}", node.hostname);
                println!("ip={}", node.ip);
                println!("state={}", node.state.as_deref().unwrap_or("unknown"));
                println!(
                    "commission_state={}",
                    node.commission_state
                        .map(|value| serde_json::to_string(&value).unwrap_or_default())
                        .unwrap_or_else(|| "\"unknown\"".to_string())
                );
                println!(
                    "install_state={}",
                    node.install_state
                        .map(|value| serde_json::to_string(&value).unwrap_or_default())
                        .unwrap_or_else(|| "\"unknown\"".to_string())
                );
                if let Some(observed_system) = observed_system {
                    println!(
                        "observed_status={}",
                        observed_system
                            .status
                            .unwrap_or_else(|| "unknown".to_string())
                    );
                }
            }

            Ok(())
        }
    })
    .await
}

pub async fn set_node_states(
    endpoint: &str,
    cluster_namespace: &str,
    cluster_id: &str,
    node_id: &str,
    state: Option<String>,
    commission_state: Option<String>,
    install_state: Option<String>,
    power_state: Option<String>,
    bmc_ref: Option<String>,
) -> Result<()> {
    let endpoints = chainfire_endpoints(endpoint);
    with_chainfire_endpoint_failover(&endpoints, "set node state", |endpoint| {
        let endpoint = endpoint.to_string();
        let cluster_namespace = cluster_namespace.to_string();
        let cluster_id = cluster_id.to_string();
        let node_id = node_id.to_string();
        let state = state.clone();
        let commission_state = commission_state.clone();
        let install_state = install_state.clone();
        let power_state = power_state.clone();
        let bmc_ref = bmc_ref.clone();
        async move {
            let mut client = Client::connect(endpoint).await?;
            let key = key_node(&cluster_namespace, &cluster_id, &node_id);
            let mut node = get_json_key::<ClusterNodeRecord>(&mut client, &key)
                .await?
                .with_context(|| format!("node {} not found", node_id))?;

            if let Some(state) = state {
                node.state = Some(state);
            }
            if let Some(commission_state) = commission_state {
                let parsed = parse_commission_state(&commission_state)?;
                if matches!(parsed, CommissionState::Commissioned) && node.commissioned_at.is_none()
                {
                    node.commissioned_at = Some(Utc::now());
                }
                node.commission_state = Some(parsed);
            }
            if let Some(install_state) = install_state {
                node.install_state = Some(parse_install_state(&install_state)?);
            }
            if let Some(power_state) = power_state {
                node.power_state = Some(parse_power_state(&power_state)?);
            }
            if let Some(bmc_ref) = bmc_ref {
                node.bmc_ref = Some(bmc_ref);
            }

            client.put(&key, &serde_json::to_vec(&node)?).await?;
            println!("{}", serde_json::to_string_pretty(&node)?);
            Ok(())
        }
    })
    .await
}

#[allow(clippy::too_many_arguments)]
pub async fn set_observed_system(
    endpoint: &str,
    cluster_namespace: &str,
    cluster_id: &str,
    node_id: &str,
    status: Option<String>,
    nixos_configuration: Option<String>,
    target_system: Option<String>,
    current_system: Option<String>,
    configured_system: Option<String>,
    booted_system: Option<String>,
    rollback_system: Option<String>,
) -> Result<()> {
    let endpoints = chainfire_endpoints(endpoint);
    with_chainfire_endpoint_failover(&endpoints, "set observed system", |endpoint| {
        let endpoint = endpoint.to_string();
        let cluster_namespace = cluster_namespace.to_string();
        let cluster_id = cluster_id.to_string();
        let node_id = node_id.to_string();
        let status = status.clone();
        let nixos_configuration = nixos_configuration.clone();
        let target_system = target_system.clone();
        let current_system = current_system.clone();
        let configured_system = configured_system.clone();
        let booted_system = booted_system.clone();
        let rollback_system = rollback_system.clone();
        async move {
            let mut client = Client::connect(endpoint).await?;
            let key = key_observed_system(&cluster_namespace, &cluster_id, &node_id);
            let mut observed = get_json_key::<ObservedSystemState>(&mut client, &key)
                .await?
                .unwrap_or_else(|| ObservedSystemState {
                    node_id: node_id.clone(),
                    ..ObservedSystemState::default()
                });

            observed.node_id = node_id.clone();
            if let Some(status) = status {
                observed.status = Some(status);
            }
            if let Some(nixos_configuration) = nixos_configuration {
                observed.nixos_configuration = Some(nixos_configuration);
            }
            if let Some(target_system) = target_system {
                observed.target_system = Some(target_system);
            }
            if let Some(current_system) = current_system {
                observed.current_system = Some(current_system);
            }
            if let Some(configured_system) = configured_system {
                observed.configured_system = Some(configured_system);
            }
            if let Some(booted_system) = booted_system {
                observed.booted_system = Some(booted_system);
            }
            if let Some(rollback_system) = rollback_system {
                observed.rollback_system = Some(rollback_system);
            }

            client.put(&key, &serde_json::to_vec(&observed)?).await?;
            println!("{}", serde_json::to_string_pretty(&observed)?);
            Ok(())
        }
    })
    .await
}

pub async fn inspect_host_deployment(
    endpoint: &str,
    cluster_namespace: &str,
    cluster_id: &str,
    deployment_name: &str,
    json_output: bool,
) -> Result<()> {
    let endpoints = chainfire_endpoints(endpoint);
    with_chainfire_endpoint_failover(&endpoints, "inspect host deployment", |endpoint| {
        let endpoint = endpoint.to_string();
        let cluster_namespace = cluster_namespace.to_string();
        let cluster_id = cluster_id.to_string();
        let deployment_name = deployment_name.to_string();
        async move {
            let mut client = Client::connect(endpoint).await?;
            let spec = get_json_key::<HostDeploymentSpec>(
                &mut client,
                &key_host_deployment_spec(&cluster_namespace, &cluster_id, &deployment_name),
            )
            .await?
            .with_context(|| format!("host deployment {} not found", deployment_name))?;
            let status = get_json_key::<HostDeploymentStatus>(
                &mut client,
                &key_host_deployment_status(&cluster_namespace, &cluster_id, &deployment_name),
            )
            .await?;

            if json_output {
                println!(
                    "{}",
                    serde_json::to_string_pretty(&json!({
                        "spec": spec,
                        "status": status,
                    }))?
                );
            } else {
                println!("name={}", spec.name);
                println!(
                    "nixos_configuration={}",
                    spec.nixos_configuration.as_deref().unwrap_or("unknown")
                );
                if let Some(status) = status {
                    println!("phase={}", status.phase.as_deref().unwrap_or("unknown"));
                    println!("paused={}", status.paused);
                    println!("selected_nodes={}", status.selected_nodes.join(","));
                    println!("completed_nodes={}", status.completed_nodes.join(","));
                    println!("failed_nodes={}", status.failed_nodes.join(","));
                }
            }

            Ok(())
        }
    })
    .await
}

pub async fn inspect_service(
    endpoint: &str,
    cluster_namespace: &str,
    cluster_id: &str,
    service_name: &str,
    include_instances: bool,
    json_output: bool,
) -> Result<()> {
    let endpoints = chainfire_endpoints(endpoint);
    with_chainfire_endpoint_failover(&endpoints, "inspect service", |endpoint| {
        let endpoint = endpoint.to_string();
        let cluster_namespace = cluster_namespace.to_string();
        let cluster_id = cluster_id.to_string();
        let service_name = service_name.to_string();
        async move {
            let mut client = Client::connect(endpoint).await?;
            let spec = get_json_key::<ServiceSpec>(
                &mut client,
                &key_service(&cluster_namespace, &cluster_id, &service_name),
            )
            .await?
            .with_context(|| format!("service {} not found", service_name))?;
            let status = get_json_key::<ServiceStatusRecord>(
                &mut client,
                &key_service_status(&cluster_namespace, &cluster_id, &service_name),
            )
            .await?;
            let publication = get_json_key::<ServicePublicationState>(
                &mut client,
                &key_publication(&cluster_namespace, &cluster_id, &service_name),
            )
            .await?;

            let instances = if include_instances {
                let prefix = format!(
                    "{}instances/{}/",
                    cluster_prefix(&cluster_namespace, &cluster_id),
                    service_name
                );
                let kvs = client.get_prefix(prefix.as_bytes()).await?;
                let mut instances = Vec::new();
                for (_key, value) in kvs {
                    match serde_json::from_slice::<ServiceInstanceSpec>(&value) {
                        Ok(instance) => instances.push(instance),
                        Err(error) => warn!(
                            error = %error,
                            service = %service_name,
                            "failed to decode service instance"
                        ),
                    }
                }
                instances.sort_by(|lhs, rhs| lhs.instance_id.cmp(&rhs.instance_id));
                Some(instances)
            } else {
                None
            };

            if json_output {
                println!(
                    "{}",
                    serde_json::to_string_pretty(&json!({
                        "spec": spec,
                        "status": status,
                        "publication": publication,
                        "instances": instances,
                    }))?
                );
            } else {
                println!("name={}", spec.name);
                println!(
                    "schedule_mode={}",
                    spec.schedule
                        .as_ref()
                        .map(|schedule| serde_json::to_string(&schedule.mode).unwrap_or_default())
                        .unwrap_or_else(|| "\"none\"".to_string())
                );
                println!("depends_on={}", spec.depends_on.len());
                if let Some(status) = status {
                    println!("phase={}", status.phase);
                    println!("healthy_instances={}", status.healthy_instances);
                    println!("scheduled_instances={}", status.scheduled_instances);
                    println!("desired_instances={}", status.desired_instances);
                    println!("dependencies_ready={}", status.dependencies_ready);
                    println!("publish_ready={}", status.publish_ready);
                    if !status.blockers.is_empty() {
                        println!("blockers={}", status.blockers.join(";"));
                    }
                }
                if let Some(publication) = publication {
                    println!(
                        "published={}",
                        publication.dns.is_some() || publication.load_balancer.is_some()
                    );
                }
                if let Some(instances) = instances {
                    println!("instances={}", instances.len());
                }
            }

            Ok(())
        }
    })
    .await
}

pub async fn set_host_deployment_paused(
    endpoint: &str,
    cluster_namespace: &str,
    cluster_id: &str,
    deployment_name: &str,
    paused: bool,
) -> Result<()> {
    let endpoints = chainfire_endpoints(endpoint);
    with_chainfire_endpoint_failover(&endpoints, "set host deployment pause state", |endpoint| {
        let endpoint = endpoint.to_string();
        let cluster_namespace = cluster_namespace.to_string();
        let cluster_id = cluster_id.to_string();
        let deployment_name = deployment_name.to_string();
        async move {
            let mut client = Client::connect(endpoint).await?;
            let spec_key =
                key_host_deployment_spec(&cluster_namespace, &cluster_id, &deployment_name);
            if client.get(&spec_key).await?.is_none() {
                return Err(anyhow::anyhow!(
                    "host deployment {} not found",
                    deployment_name
                ));
            }

            let status_key =
                key_host_deployment_status(&cluster_namespace, &cluster_id, &deployment_name);
            let mut status = get_json_key::<HostDeploymentStatus>(&mut client, &status_key)
                .await?
                .unwrap_or_else(|| HostDeploymentStatus {
                    name: deployment_name.clone(),
                    ..HostDeploymentStatus::default()
                });
            status.name = deployment_name.clone();
            status.paused_by_operator = paused;
            status.paused = paused;
            status.phase = Some(if paused { "paused" } else { "ready" }.to_string());
            status.message = Some(if paused {
                "paused by operator".to_string()
            } else {
                "resumed by operator".to_string()
            });
            status.updated_at = Some(Utc::now());
            client
                .put(&status_key, &serde_json::to_vec(&status)?)
                .await?;
            println!("{}", serde_json::to_string_pretty(&status)?);
            Ok(())
        }
    })
    .await
}

pub async fn abort_host_deployment(
    endpoint: &str,
    cluster_namespace: &str,
    cluster_id: &str,
    deployment_name: &str,
) -> Result<()> {
    let endpoints = chainfire_endpoints(endpoint);
    with_chainfire_endpoint_failover(&endpoints, "abort host deployment", |endpoint| {
        let endpoint = endpoint.to_string();
        let cluster_namespace = cluster_namespace.to_string();
        let cluster_id = cluster_id.to_string();
        let deployment_name = deployment_name.to_string();
        async move {
            let mut client = Client::connect(endpoint).await?;
            let spec_key =
                key_host_deployment_spec(&cluster_namespace, &cluster_id, &deployment_name);
            if client.get(&spec_key).await?.is_none() {
                return Err(anyhow::anyhow!(
                    "host deployment {} not found",
                    deployment_name
                ));
            }

            let node_prefix = format!("{}nodes/", cluster_prefix(&cluster_namespace, &cluster_id));
            let existing = client.get_prefix(node_prefix.as_bytes()).await?;
            let mut cleared_nodes = Vec::new();

            for (key, value) in &existing {
                let key_str = String::from_utf8_lossy(&key);
                if key_str.ends_with("/desired-system") {
                    let Ok(desired) = serde_json::from_slice::<DesiredSystemSpec>(value) else {
                        continue;
                    };
                    if desired.deployment_id.as_deref() == Some(deployment_name.as_str()) {
                        client.delete(&key).await?;
                        cleared_nodes.push(desired.node_id.clone());
                    }
                }
            }

            for (key, value) in existing {
                let key_str = String::from_utf8_lossy(&key);
                if key_str.ends_with("/desired-system") {
                    continue;
                }

                let node_suffix = key_str
                    .strip_prefix(&node_prefix)
                    .filter(|suffix| !suffix.contains('/'));
                let Some(node_id) = node_suffix else {
                    continue;
                };
                let mut node = match serde_json::from_slice::<ClusterNodeRecord>(&value) {
                    Ok(node) => node,
                    Err(_) => continue,
                };
                if cleared_nodes.iter().any(|cleared| cleared == node_id)
                    && node.state.as_deref() == Some("draining")
                {
                    node.state = Some("active".to_string());
                    client.put(&key, &serde_json::to_vec(&node)?).await?;
                }
            }

            let status = HostDeploymentStatus {
                name: deployment_name.clone(),
                phase: Some("aborted".to_string()),
                paused: true,
                paused_by_operator: true,
                selected_nodes: Vec::new(),
                completed_nodes: Vec::new(),
                in_progress_nodes: Vec::new(),
                failed_nodes: Vec::new(),
                message: Some(format!(
                    "aborted by operator; cleared desired-system from {} node(s)",
                    cleared_nodes.len()
                )),
                updated_at: Some(Utc::now()),
            };
            client
                .put(
                    &key_host_deployment_status(&cluster_namespace, &cluster_id, &deployment_name),
                    &serde_json::to_vec(&status)?,
                )
                .await?;
            println!("{}", serde_json::to_string_pretty(&status)?);
            Ok(())
        }
    })
    .await
}

async fn prune_cluster_state(
    client: &mut Client,
    cluster_namespace: &str,
    deployer_namespace: &str,
    cluster_id: &str,
    spec: &ClusterStateSpec,
) -> Result<()> {
    let mut desired_keys = HashSet::new();
    desired_keys.insert(
        String::from_utf8_lossy(&key_cluster_meta(cluster_namespace, cluster_id)).to_string(),
    );

    let resolved_nodes = resolve_nodes(spec)?;

    for node in &resolved_nodes {
        desired_keys.insert(
            String::from_utf8_lossy(&key_node(cluster_namespace, cluster_id, &node.node_id))
                .to_string(),
        );
        if desired_system_from_spec(node).is_some() {
            desired_keys.insert(
                String::from_utf8_lossy(&key_desired_system(
                    cluster_namespace,
                    cluster_id,
                    &node.node_id,
                ))
                .to_string(),
            );
        }
    }
    for node_class in &spec.node_classes {
        desired_keys.insert(
            String::from_utf8_lossy(&key_node_class(
                cluster_namespace,
                cluster_id,
                &node_class.name,
            ))
            .to_string(),
        );
    }
    for pool in &spec.pools {
        desired_keys.insert(
            String::from_utf8_lossy(&key_pool(cluster_namespace, cluster_id, &pool.name))
                .to_string(),
        );
    }
    for rule in &spec.enrollment_rules {
        desired_keys.insert(
            String::from_utf8_lossy(&key_enrollment_rule(
                cluster_namespace,
                cluster_id,
                &rule.name,
            ))
            .to_string(),
        );
    }
    for deployment in &spec.host_deployments {
        desired_keys.insert(
            String::from_utf8_lossy(&key_host_deployment_spec(
                cluster_namespace,
                cluster_id,
                &deployment.name,
            ))
            .to_string(),
        );
    }
    for svc in &spec.services {
        desired_keys.insert(
            String::from_utf8_lossy(&key_service(cluster_namespace, cluster_id, &svc.name))
                .to_string(),
        );
    }
    for inst in &spec.instances {
        desired_keys.insert(
            String::from_utf8_lossy(&key_instance(
                cluster_namespace,
                cluster_id,
                &inst.service,
                &inst.instance_id,
            ))
            .to_string(),
        );
    }
    for policy in &spec.mtls_policies {
        desired_keys.insert(
            String::from_utf8_lossy(&key_mtls_policy(
                cluster_namespace,
                cluster_id,
                &policy.policy_id,
            ))
            .to_string(),
        );
    }

    let prefix = cluster_prefix(cluster_namespace, cluster_id);
    let existing = client.get_prefix(prefix.as_bytes()).await?;

    for (key, _) in existing {
        let key_str = String::from_utf8_lossy(&key);
        if !is_prunable_key(&key_str, &prefix) {
            continue;
        }
        if desired_keys.contains(key_str.as_ref()) {
            continue;
        }
        warn!(key = %key_str, "pruning stale cluster key");
        client.delete(&key).await?;
    }

    let mut desired_deployer_keys = HashSet::new();
    for node in &resolved_nodes {
        if let Some(machine_id) = node.machine_id.as_deref() {
            desired_deployer_keys.insert(
                String::from_utf8_lossy(&deployer_node_config_key(deployer_namespace, machine_id))
                    .to_string(),
            );
        }
    }

    let deployer_prefix = format!("{}/nodes/", deployer_namespace);
    let existing = client.get_prefix(deployer_prefix.as_bytes()).await?;
    for (key, _) in existing {
        let key_str = String::from_utf8_lossy(&key);
        if !is_prunable_deployer_key(&key_str, deployer_namespace) {
            continue;
        }
        if desired_deployer_keys.contains(key_str.as_ref()) {
            continue;
        }
        warn!(key = %key_str, "pruning stale deployer bootstrap key");
        client.delete(&key).await?;
    }

    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use deployer_types::{ClusterSpec, NodeState};
    use std::collections::HashMap;

    #[test]
    fn retryable_chainfire_messages_cover_connection_readiness() {
        assert!(retryable_chainfire_message("connection was not ready"));
        assert!(retryable_chainfire_message("transport error"));
        assert!(retryable_chainfire_message(
            "Raft write failed: NotLeader { leader_id: None }"
        ));
        assert!(!retryable_chainfire_message("permission denied"));
    }

    #[test]
    fn retry_delay_is_capped() {
        assert_eq!(retry_delay(0), Duration::from_millis(250));
        assert_eq!(retry_delay(1), Duration::from_millis(500));
        assert_eq!(retry_delay(2), Duration::from_millis(1000));
        assert_eq!(retry_delay(3), Duration::from_millis(2000));
        assert_eq!(retry_delay(7), Duration::from_millis(2000));
    }

    #[test]
    fn chainfire_endpoints_support_comma_separated_failover() {
        assert_eq!(
            chainfire_endpoints(
                "http://127.0.0.1:12379, http://127.0.0.1:12380 ,,http://127.0.0.1:12381"
            ),
            vec![
                "http://127.0.0.1:12379".to_string(),
                "http://127.0.0.1:12380".to_string(),
                "http://127.0.0.1:12381".to_string(),
            ]
        );
    }

    fn test_spec() -> ClusterStateSpec {
        ClusterStateSpec {
            cluster: ClusterSpec {
                cluster_id: "test-cluster".to_string(),
                environment: Some("dev".to_string()),
            },
            nodes: vec![NodeSpec {
                node_id: "node01".to_string(),
                machine_id: None,
                hostname: "node01".to_string(),
                ip: "10.0.0.11".to_string(),
                roles: vec!["custom-role".to_string()],
                labels: HashMap::from([("rack".to_string(), "r1".to_string())]),
                pool: Some("general".to_string()),
                node_class: None,
                failure_domain: Some("rack-a".to_string()),
                nix_profile: None,
                install_plan: None,
                hardware_facts: None,
                desired_system: None,
                state: Some(match NodeState::Pending {
                    NodeState::Pending => "pending".to_string(),
                    _ => unreachable!(),
                }),
                commission_state: None,
                install_state: None,
                commissioned_at: None,
                last_inventory_hash: None,
                power_state: None,
                bmc_ref: None,
                last_heartbeat: None,
            }],
            node_classes: vec![deployer_types::NodeClassSpec {
                name: "worker-linux".to_string(),
                description: None,
                nix_profile: Some("profiles/worker-linux".to_string()),
                install_plan: Some(InstallPlan {
                    nixos_configuration: Some("worker-golden".to_string()),
                    disko_config_path: Some("profiles/worker-linux/disko.nix".to_string()),
                    target_disk: Some("/dev/disk/by-id/worker-golden".to_string()),
                    target_disk_by_id: None,
                }),
                roles: vec!["worker".to_string()],
                labels: HashMap::from([("tier".to_string(), "general".to_string())]),
            }],
            pools: vec![deployer_types::NodePoolSpec {
                name: "general".to_string(),
                description: None,
                node_class: Some("worker-linux".to_string()),
                min_size: Some(1),
                max_size: Some(10),
                labels: HashMap::from([("env".to_string(), "dev".to_string())]),
            }],
            enrollment_rules: vec![],
            host_deployments: vec![],
            services: vec![],
            instances: vec![],
            mtls_policies: vec![],
        }
    }

    #[test]
    fn test_resolve_nodes_applies_node_class_and_pool_defaults() {
        let spec = test_spec();
        let resolved = resolve_nodes(&spec).unwrap();
        let node = &resolved[0];

        assert_eq!(node.node_class.as_deref(), Some("worker-linux"));
        assert_eq!(node.nix_profile.as_deref(), Some("profiles/worker-linux"));
        let install_plan = node
            .install_plan
            .as_ref()
            .expect("install plan should inherit");
        assert_eq!(
            install_plan.nixos_configuration.as_deref(),
            Some("worker-golden")
        );
        assert!(node.roles.iter().any(|role| role == "worker"));
        assert!(node.roles.iter().any(|role| role == "custom-role"));
        assert_eq!(node.labels.get("tier").map(String::as_str), Some("general"));
        assert_eq!(node.labels.get("env").map(String::as_str), Some("dev"));
        assert_eq!(node.labels.get("pool").map(String::as_str), Some("general"));
        assert_eq!(
            node.labels
                .get("nodeclass.ultracloud.io/name")
                .map(String::as_str),
            Some("worker-linux")
        );
        assert_eq!(
            node.labels
                .get("topology.kubernetes.io/zone")
                .map(String::as_str),
            Some("rack-a")
        );
    }

    #[test]
    fn test_desired_system_is_derived_from_install_plan() {
        let spec = test_spec();
        let resolved = resolve_nodes(&spec).unwrap();
        let desired = desired_system_from_spec(&resolved[0]).expect("desired system should exist");

        assert_eq!(desired.node_id, "node01");
        assert_eq!(
            desired.nixos_configuration.as_deref(),
            Some("worker-golden")
        );
        assert_eq!(desired.switch_action.as_deref(), Some("switch"));
        assert_eq!(desired.rollback_on_failure, Some(true));
    }

    #[test]
    fn test_desired_system_keeps_explicit_node_overrides() {
        let mut spec = test_spec();
        spec.nodes[0].desired_system = Some(DesiredSystemSpec {
            node_id: String::new(),
            deployment_id: None,
            nixos_configuration: Some("node01-next".to_string()),
            target_system: Some("/nix/store/node01-next".to_string()),
            flake_ref: Some("github:centra/cloud".to_string()),
            switch_action: Some("boot".to_string()),
            health_check_command: vec!["true".to_string()],
            rollback_on_failure: Some(false),
            drain_before_apply: Some(false),
        });

        let resolved = resolve_nodes(&spec).unwrap();
        let desired = desired_system_from_spec(&resolved[0]).expect("desired system should exist");
        assert_eq!(desired.node_id, "node01");
        assert_eq!(desired.nixos_configuration.as_deref(), Some("node01-next"));
        assert_eq!(
            desired.target_system.as_deref(),
            Some("/nix/store/node01-next")
        );
        assert_eq!(desired.flake_ref.as_deref(), Some("github:centra/cloud"));
        assert_eq!(desired.switch_action.as_deref(), Some("boot"));
        assert_eq!(desired.health_check_command, vec!["true".to_string()]);
        assert_eq!(desired.rollback_on_failure, Some(false));
    }

    #[test]
    fn test_is_prunable_key_keeps_observed_system() {
        let prefix = cluster_prefix("ultracloud", "test-cluster");
        assert!(is_prunable_key(&format!("{}nodes/node01", prefix), &prefix));
        assert!(is_prunable_key(
            &format!("{}nodes/node01/desired-system", prefix),
            &prefix
        ));
        assert!(!is_prunable_key(
            &format!("{}nodes/node01/observed-system", prefix),
            &prefix
        ));
        assert!(is_prunable_key(
            &format!("{}deployments/hosts/worker-rollout/spec", prefix),
            &prefix
        ));
        assert!(!is_prunable_key(
            &format!("{}deployments/hosts/worker-rollout/status", prefix),
            &prefix
        ));
    }
}

fn is_prunable_key(key: &str, prefix: &str) -> bool {
    if key == format!("{}meta", prefix) {
        return true;
    }

    if let Some(node_suffix) = key.strip_prefix(&format!("{}nodes/", prefix)) {
        return !node_suffix.is_empty()
            && (!node_suffix.contains('/') || node_suffix.ends_with("/desired-system"));
    }

    key.starts_with(&format!("{}node-classes/", prefix))
        || key.starts_with(&format!("{}pools/", prefix))
        || key.starts_with(&format!("{}enrollment-rules/", prefix))
        || key.starts_with(&format!("{}deployments/hosts/", prefix)) && key.ends_with("/spec")
        || key.starts_with(&format!("{}services/", prefix))
        || key.starts_with(&format!("{}instances/", prefix))
        || key.starts_with(&format!("{}mtls/policies/", prefix))
}

fn is_prunable_deployer_key(key: &str, deployer_namespace: &str) -> bool {
    key.starts_with(&format!("{}/nodes/config/", deployer_namespace))
}
