use std::collections::{HashMap, HashSet};
use std::fs;
use std::path::PathBuf;
use std::process::Stdio;
use std::time::Duration;

use anyhow::{Context, Result};
use chainfire_client::{Client, ClientError};
use chrono::{DateTime, Utc};
use deployer_types::{
    build_service_status_record, compute_service_dependency_cycles, ClusterNodeRecord,
    ContainerSpec, HealthCheckSpec, ProcessSpec, ServiceInstanceSpec, ServicePublicationState,
    ServiceSpec,
};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::process::Command;
use tokio::sync::{mpsc, oneshot};
use tokio::time::MissedTickBehavior;
use tracing::{info, warn};

use crate::process::{render_container_process_spec, ProcessManager};
use crate::watcher::{ChainfireWatcher, WatchChange};

fn cluster_prefix(cluster_namespace: &str, cluster_id: &str) -> String {
    format!("{}/clusters/{}/", cluster_namespace, cluster_id)
}

fn key_node(cluster_namespace: &str, cluster_id: &str, node_id: &str) -> Vec<u8> {
    format!(
        "{}nodes/{}",
        cluster_prefix(cluster_namespace, cluster_id),
        node_id
    )
    .into_bytes()
}

fn key_instance(
    cluster_namespace: &str,
    cluster_id: &str,
    service: &str,
    instance_id: &str,
) -> Vec<u8> {
    format!(
        "{}instances/{}/{}",
        cluster_prefix(cluster_namespace, cluster_id),
        service,
        instance_id
    )
    .into_bytes()
}

fn instances_prefix(cluster_namespace: &str, cluster_id: &str) -> Vec<u8> {
    format!("{}instances/", cluster_prefix(cluster_namespace, cluster_id)).into_bytes()
}

fn service_status_prefix(cluster_namespace: &str, cluster_id: &str) -> String {
    format!(
        "{}/clusters/{}/service-statuses/",
        cluster_namespace, cluster_id
    )
}

fn key_service_status(cluster_namespace: &str, cluster_id: &str, service: &str) -> Vec<u8> {
    format!(
        "{}{}",
        service_status_prefix(cluster_namespace, cluster_id),
        service
    )
    .into_bytes()
}

#[derive(Debug, Deserialize, Serialize)]
pub struct NodeState {
    pub node_id: String,
    pub ip: String,
    pub hostname: String,
    #[serde(default)]
    pub roles: Vec<String>,
    #[serde(default)]
    pub labels: std::collections::HashMap<String, String>,
    #[serde(default)]
    pub state: Option<String>,
    #[serde(default)]
    pub last_heartbeat: Option<DateTime<Utc>>,
    #[serde(flatten)]
    pub extra: std::collections::HashMap<String, Value>,
}

pub struct Agent {
    endpoint: String,
    cluster_namespace: String,
    cluster_id: String,
    node_id: String,
    interval: Duration,
    heartbeat_timeout_secs: u64,
    apply: bool,
    allow_local_instance_upsert: bool,
    process_manager: ProcessManager,
    next_health_checks: HashMap<String, DateTime<Utc>>,
    watched_node_signature: Option<Vec<u8>>,
    watched_instance_signatures: HashMap<String, Vec<u8>>,
}

#[derive(Debug, Deserialize, Serialize, Clone)]
struct LocalInstanceSpec {
    service: String,
    instance_id: String,
    #[serde(default)]
    node_id: Option<String>,
    ip: String,
    port: u16,
    #[serde(default)]
    mesh_port: Option<u16>,
    #[serde(default)]
    health_check: Option<HealthCheckSpec>,
    #[serde(default)]
    process: Option<ProcessSpec>,
    #[serde(default)]
    container: Option<ContainerSpec>,
}

async fn spawn_watch_task(
    watcher: ChainfireWatcher,
    tx: mpsc::Sender<WatchChange>,
    ready: oneshot::Sender<()>,
) {
    let mut ready = Some(ready);
    if let Err(error) = watcher
        .watch_with_ready(
            move || {
                if let Some(ready) = ready.take() {
                    let _ = ready.send(());
                }
                Ok(())
            },
            move |change| {
                tx.try_send(change)
                    .map_err(|send_error| anyhow::anyhow!("failed to enqueue watch change: {}", send_error))
            },
        )
        .await
    {
        warn!(error = %error, "node-agent watch task exited");
    }
}

async fn wait_for_watchers_ready(receivers: Vec<oneshot::Receiver<()>>) {
    for receiver in receivers {
        match tokio::time::timeout(std::time::Duration::from_secs(10), receiver).await {
            Ok(Ok(())) => {}
            Ok(Err(_)) => warn!("node-agent watch task ended before signaling readiness"),
            Err(_) => warn!("timed out waiting for node-agent watch readiness"),
        }
    }
}

fn canonical_json_signature(bytes: &[u8], drop_keys: &[&str]) -> Option<Vec<u8>> {
    let mut value: Value = serde_json::from_slice(bytes).ok()?;
    strip_observed_fields(&mut value, drop_keys);
    serde_json::to_vec(&sort_json_value(value)).ok()
}

fn strip_observed_fields(value: &mut Value, drop_keys: &[&str]) {
    match value {
        Value::Object(map) => {
            for key in drop_keys {
                map.remove(*key);
            }
            for nested in map.values_mut() {
                strip_observed_fields(nested, drop_keys);
            }
        }
        Value::Array(items) => {
            for item in items {
                strip_observed_fields(item, drop_keys);
            }
        }
        _ => {}
    }
}

fn sort_json_value(value: Value) -> Value {
    match value {
        Value::Array(items) => Value::Array(items.into_iter().map(sort_json_value).collect()),
        Value::Object(map) => {
            let mut keys = map.keys().cloned().collect::<Vec<_>>();
            keys.sort();
            let mut normalized = serde_json::Map::new();
            for key in keys {
                if let Some(value) = map.get(&key) {
                    normalized.insert(key, sort_json_value(value.clone()));
                }
            }
            Value::Object(normalized)
        }
        other => other,
    }
}

impl Agent {
    pub fn new(
        endpoint: String,
        cluster_namespace: String,
        cluster_id: String,
        node_id: String,
        interval: Duration,
        heartbeat_timeout_secs: u64,
        apply: bool,
        allow_local_instance_upsert: bool,
        pid_dir: PathBuf,
    ) -> Self {
        Self {
            endpoint,
            cluster_namespace,
            cluster_id,
            node_id,
            interval,
            heartbeat_timeout_secs,
            apply,
            allow_local_instance_upsert,
            process_manager: ProcessManager::new(pid_dir),
            next_health_checks: HashMap::new(),
            watched_node_signature: None,
            watched_instance_signatures: HashMap::new(),
        }
    }

    pub async fn run_loop(&mut self) -> Result<()> {
        self.tick().await?;

        let (watch_tx, mut watch_rx) = mpsc::channel::<WatchChange>(64);
        let ready_receivers = self.spawn_watchers(watch_tx);
        wait_for_watchers_ready(ready_receivers).await;
        self.tick().await?;
        let mut interval = tokio::time::interval(self.interval);
        interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
        interval.tick().await;

        loop {
            tokio::select! {
                _ = interval.tick() => {
                    if let Err(error) = self.tick().await {
                        warn!(error = %error, "node-agent tick failed");
                    }
                }
                maybe_change = watch_rx.recv() => {
                    let Some(change) = maybe_change else {
                        continue;
                    };

                    let mut should_reconcile = self.apply_watch_change(&change);
                    while let Ok(extra) = watch_rx.try_recv() {
                        should_reconcile |= self.apply_watch_change(&extra);
                    }

                    if should_reconcile {
                        if let Err(error) = self.tick().await {
                            warn!(error = %error, "node-agent watch-triggered tick failed");
                        }
                    }
                }
            }
        }
    }

    pub async fn tick(&mut self) -> Result<()> {
        let mut client = Client::connect(self.endpoint.clone()).await?;

        // Node 情報
        let node_key = key_node(&self.cluster_namespace, &self.cluster_id, &self.node_id);
        let node_raw = client.get_with_revision(&node_key).await?;
        let Some((node_bytes, node_revision)) = node_raw else {
            warn!(
                "node definition not found in Chainfire for cluster_id={}, node_id={}",
                self.cluster_id, self.node_id
            );
            return Ok(());
        };

        let mut node: NodeState = match serde_json::from_slice(&node_bytes) {
            Ok(n) => n,
            Err(e) => {
                warn!(error = %e, "failed to parse node JSON");
                return Ok(());
            }
        };

        // Heartbeat を更新し、Chainfire 上の Node を upsert
        node.last_heartbeat = Some(Utc::now());
        if self.apply {
            let should_activate = match node.state.as_deref() {
                Some("failed") | Some("draining") => false,
                Some("active") => false,
                Some("provisioning") | Some("pending") | None => true,
                Some(other) => {
                    warn!(state = %other, "unknown node state; leaving unchanged");
                    false
                }
            };
            if should_activate {
                node.state = Some("active".to_string());
            }
        }
        let updated = serde_json::to_vec(&node)?;
        if let Err(e) = client
            .put_if_revision(&node_key, &updated, node_revision)
            .await
        {
            warn!(error = %e, "failed to update node heartbeat");
        }

        // ローカル定義された ServiceInstance を Chainfire に登録
        if self.allow_local_instance_upsert {
            if let Err(e) = self.sync_local_instances(&mut client).await {
                warn!(error = %e, "failed to sync local service instances");
            }
        } else {
            info!("local instance upsert disabled; skipping /etc/ultracloud/instances.json");
        }

        if self.apply {
            // プロセスの起動/停止をReconcile
            if let Err(e) = self.reconcile_processes(&mut client).await {
                warn!(error = %e, "failed to reconcile processes");
            }

            // ヘルスチェックを実行して状態を更新
            if let Err(e) = self.update_health_status(&mut client).await {
                warn!(error = %e, "failed to update health status");
            }
        } else {
            info!("apply=false; skipping process reconciliation and health checks");
        }

        if let Err(e) = self.persist_service_statuses(&mut client).await {
            warn!(error = %e, "failed to persist aggregated service statuses");
        }

        self.log_node_only(&node);

        Ok(())
    }

    fn log_node_only(&self, node: &NodeState) {
        info!(
            node_id = %node.node_id,
            hostname = %node.hostname,
            roles = ?node.roles,
            "observed desired state for node"
        );
    }

    fn spawn_watchers(&self, tx: mpsc::Sender<WatchChange>) -> Vec<oneshot::Receiver<()>> {
        let mut ready_receivers = Vec::new();

        let node_watcher = ChainfireWatcher::key(
            self.endpoint.clone(),
            key_node(&self.cluster_namespace, &self.cluster_id, &self.node_id),
        );
        let (node_ready_tx, node_ready_rx) = oneshot::channel();
        tokio::spawn(spawn_watch_task(node_watcher, tx.clone(), node_ready_tx));
        ready_receivers.push(node_ready_rx);

        let instances_watcher = ChainfireWatcher::prefix(
            self.endpoint.clone(),
            instances_prefix(&self.cluster_namespace, &self.cluster_id),
        );
        let (instances_ready_tx, instances_ready_rx) = oneshot::channel();
        tokio::spawn(spawn_watch_task(instances_watcher, tx, instances_ready_tx));
        ready_receivers.push(instances_ready_rx);

        ready_receivers
    }

    fn apply_watch_change(&mut self, change: &WatchChange) -> bool {
        let key = String::from_utf8_lossy(&change.key).to_string();
        let node_key =
            String::from_utf8_lossy(&key_node(&self.cluster_namespace, &self.cluster_id, &self.node_id))
                .to_string();
        let instances_prefix =
            String::from_utf8_lossy(&instances_prefix(&self.cluster_namespace, &self.cluster_id))
                .to_string();

        if key == node_key {
            return self.apply_node_watch_change(change);
        }

        if key.starts_with(&instances_prefix) {
            return self.apply_instance_watch_change(&key, change);
        }

        false
    }

    fn apply_node_watch_change(&mut self, change: &WatchChange) -> bool {
        if matches!(change.event_type, chainfire_client::EventType::Delete) {
            self.watched_node_signature = None;
            return true;
        }
        let Some(signature) = canonical_json_signature(&change.value, &["last_heartbeat"]) else {
            return true;
        };
        if self.watched_node_signature.as_ref() == Some(&signature) {
            return false;
        }
        self.watched_node_signature = Some(signature);
        true
    }

    fn apply_instance_watch_change(&mut self, key: &str, change: &WatchChange) -> bool {
        if matches!(change.event_type, chainfire_client::EventType::Delete) {
            self.watched_instance_signatures.remove(key);
            return true;
        }
        let Some(signature) =
            canonical_json_signature(&change.value, &["state", "last_heartbeat", "observed_at"])
        else {
            self.watched_instance_signatures.remove(key);
            return true;
        };

        if self.watched_instance_signatures.get(key) == Some(&signature) {
            return false;
        }

        self.watched_instance_signatures
            .insert(key.to_string(), signature);
        true
    }

    fn is_local_instance(&self, inst: &LocalInstanceSpec) -> bool {
        matches!(inst.node_id.as_deref(), Some(node_id) if node_id == self.node_id)
    }

    fn is_local_managed_instance(&self, inst: &ServiceInstanceSpec) -> bool {
        inst.node_id == self.node_id
    }

    fn render_template_value(&self, value: &str, inst: &ServiceInstanceSpec) -> String {
        let mesh_port = inst
            .mesh_port
            .map(|port| port.to_string())
            .unwrap_or_default();
        [
            ("${SERVICE}", inst.service.as_str()),
            ("${INSTANCE_ID}", inst.instance_id.as_str()),
            ("${NODE_ID}", inst.node_id.as_str()),
            ("${INSTANCE_IP}", inst.ip.as_str()),
            ("${INSTANCE_PORT}", &inst.port.to_string()),
            ("${MESH_PORT}", mesh_port.as_str()),
        ]
        .into_iter()
        .fold(value.to_string(), |acc, (needle, replacement)| {
            acc.replace(needle, replacement)
        })
    }

    fn render_process_spec(&self, spec: &ProcessSpec, inst: &ServiceInstanceSpec) -> ProcessSpec {
        ProcessSpec {
            command: self.render_template_value(&spec.command, inst),
            args: spec
                .args
                .iter()
                .map(|arg| self.render_template_value(arg, inst))
                .collect(),
            working_dir: spec
                .working_dir
                .as_deref()
                .map(|value| self.render_template_value(value, inst)),
            env: spec
                .env
                .iter()
                .map(|(key, value)| (key.clone(), self.render_template_value(value, inst)))
                .collect(),
        }
    }

    fn render_container_spec(
        &self,
        spec: &ContainerSpec,
        inst: &ServiceInstanceSpec,
    ) -> ContainerSpec {
        let mut rendered = spec.clone();
        rendered.image = self.render_template_value(&rendered.image, inst);
        rendered.command = rendered
            .command
            .iter()
            .map(|arg| self.render_template_value(arg, inst))
            .collect();
        rendered.args = rendered
            .args
            .iter()
            .map(|arg| self.render_template_value(arg, inst))
            .collect();
        rendered.working_dir = rendered
            .working_dir
            .as_deref()
            .map(|value| self.render_template_value(value, inst));
        rendered.env = rendered
            .env
            .iter()
            .map(|(key, value)| (key.clone(), self.render_template_value(value, inst)))
            .collect();
        for volume in &mut rendered.volumes {
            volume.source = self.render_template_value(&volume.source, inst);
            volume.target = self.render_template_value(&volume.target, inst);
        }
        rendered
    }

    fn desired_process_spec(&self, inst: &ServiceInstanceSpec) -> Option<ProcessSpec> {
        match (&inst.container, &inst.process) {
            (Some(container), maybe_process) => {
                if maybe_process.is_some() {
                    warn!(
                        service = %inst.service,
                        instance_id = %inst.instance_id,
                        "instance defines both process and container; container takes precedence"
                    );
                }
                let rendered = self.render_container_spec(container, inst);
                Some(render_container_process_spec(
                    &inst.service,
                    &inst.instance_id,
                    inst.port,
                    &rendered,
                ))
            }
            (None, Some(process)) => Some(self.render_process_spec(process, inst)),
            (None, None) => None,
        }
    }

    fn apply_instance_health_fields(
        inst_value: &mut Value,
        started_at: &DateTime<Utc>,
        heartbeat_at: &DateTime<Utc>,
        health_status: &str,
    ) {
        let Some(obj) = inst_value.as_object_mut() else {
            return;
        };

        let observed_at = Value::String(started_at.to_rfc3339());
        match obj.get_mut("observed_at") {
            Some(slot) if slot.is_null() => *slot = observed_at,
            Some(_) => {}
            None => {
                obj.insert("observed_at".to_string(), observed_at);
            }
        }

        obj.insert(
            "state".to_string(),
            Value::String(health_status.to_string()),
        );
        obj.insert(
            "last_heartbeat".to_string(),
            Value::String(heartbeat_at.to_rfc3339()),
        );
    }

    async fn persist_instance_health(
        &self,
        client: &mut Client,
        key: &[u8],
        inst: &ServiceInstanceSpec,
        mut inst_value: Value,
        mut mod_revision: u64,
        started_at: &DateTime<Utc>,
        health_status: &str,
    ) -> Result<()> {
        for attempt in 0..3 {
            let heartbeat_at = Utc::now();
            Self::apply_instance_health_fields(
                &mut inst_value,
                started_at,
                &heartbeat_at,
                health_status,
            );

            let updated = serde_json::to_vec(&inst_value)?;
            match client.put_if_revision(key, &updated, mod_revision).await {
                Ok(()) => return Ok(()),
                Err(ClientError::Conflict(error)) if attempt < 2 => {
                    warn!(
                        service = %inst.service,
                        instance_id = %inst.instance_id,
                        mod_revision,
                        attempt = attempt + 1,
                        error = %error,
                        "instance health update raced with another writer; retrying with fresh revision"
                    );

                    let Some((latest_bytes, latest_revision)) =
                        client.get_with_revision(key).await?
                    else {
                        warn!(
                            service = %inst.service,
                            instance_id = %inst.instance_id,
                            "instance disappeared while retrying health update"
                        );
                        return Ok(());
                    };

                    inst_value = serde_json::from_slice(&latest_bytes).with_context(|| {
                        format!(
                            "failed to parse refreshed instance JSON for {}",
                            inst.instance_id
                        )
                    })?;
                    mod_revision = latest_revision;
                }
                Err(error) => return Err(error.into()),
            }
        }

        Ok(())
    }

    /// ローカルファイル (/etc/ultracloud/instances.json) から ServiceInstance 定義を読み、
    /// Chainfire 上の `ultracloud/clusters/{cluster_id}/instances/{service}/{instance_id}` に upsert する。
    async fn sync_local_instances(&self, client: &mut Client) -> Result<()> {
        let path = PathBuf::from("/etc/ultracloud/instances.json");
        let contents = match fs::read_to_string(&path) {
            Ok(c) => c,
            Err(e) => {
                warn!(error = %e, "no local instances file found, skipping");
                return Ok(());
            }
        };

        let instances: Vec<LocalInstanceSpec> = serde_json::from_str(&contents)
            .with_context(|| format!("failed to parse {}", path.display()))?;

        for inst in &instances {
            let mut inst = inst.clone();
            if inst.node_id.is_none() {
                inst.node_id = Some(self.node_id.clone());
            }
            if !self.is_local_instance(&inst) {
                continue;
            }

            let key = key_instance(
                &self.cluster_namespace,
                &self.cluster_id,
                &inst.service,
                &inst.instance_id,
            );
            let mut desired_value = serde_json::to_value(&inst)?;

            if let Some(existing) = client.get(&key).await? {
                if let Ok(existing_value) = serde_json::from_slice::<Value>(&existing) {
                    if let (Some(existing_obj), Some(desired_obj)) =
                        (existing_value.as_object(), desired_value.as_object_mut())
                    {
                        for preserve_key in ["state", "last_heartbeat", "observed_at"] {
                            if let Some(value) = existing_obj.get(preserve_key) {
                                desired_obj
                                    .entry(preserve_key.to_string())
                                    .or_insert(value.clone());
                            }
                        }
                    }
                }
            }

            let value = serde_json::to_vec(&desired_value)?;
            client.put(&key, &value).await?;
            info!(
                service = %inst.service,
                instance_id = %inst.instance_id,
                "synced local ServiceInstance to Chainfire"
            );
        }

        Ok(())
    }

    async fn load_cluster_nodes(&self, client: &mut Client) -> Result<Vec<ClusterNodeRecord>> {
        let prefix = format!(
            "{}nodes/",
            cluster_prefix(&self.cluster_namespace, &self.cluster_id)
        );
        let kvs = client.get_prefix(prefix.as_bytes()).await?;
        let mut nodes = Vec::with_capacity(kvs.len());

        for (_key, value) in kvs {
            match serde_json::from_slice::<ClusterNodeRecord>(&value) {
                Ok(node) => nodes.push(node),
                Err(error) => warn!(error = %error, "failed to decode cluster node"),
            }
        }

        nodes.sort_by(|lhs, rhs| lhs.node_id.cmp(&rhs.node_id));
        Ok(nodes)
    }

    async fn load_services(&self, client: &mut Client) -> Result<Vec<ServiceSpec>> {
        let prefix = format!(
            "{}services/",
            cluster_prefix(&self.cluster_namespace, &self.cluster_id)
        );
        let kvs = client.get_prefix(prefix.as_bytes()).await?;
        let mut services = Vec::with_capacity(kvs.len());

        for (_key, value) in kvs {
            match serde_json::from_slice::<ServiceSpec>(&value) {
                Ok(service) => services.push(service),
                Err(error) => warn!(error = %error, "failed to decode service spec"),
            }
        }

        services.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name));
        Ok(services)
    }

    async fn load_instances_by_service(
        &self,
        client: &mut Client,
    ) -> Result<HashMap<String, Vec<ServiceInstanceSpec>>> {
        let prefix = format!(
            "{}instances/",
            cluster_prefix(&self.cluster_namespace, &self.cluster_id)
        );
        let kvs = client.get_prefix(prefix.as_bytes()).await?;
        let mut instances = HashMap::<String, Vec<ServiceInstanceSpec>>::new();

        for (_key, value) in kvs {
            match serde_json::from_slice::<ServiceInstanceSpec>(&value) {
                Ok(instance) => {
                    instances
                        .entry(instance.service.clone())
                        .or_default()
                        .push(instance);
                }
                Err(error) => warn!(error = %error, "failed to decode service instance"),
            }
        }

        for service_instances in instances.values_mut() {
            service_instances.sort_by(|lhs, rhs| lhs.instance_id.cmp(&rhs.instance_id));
        }

        Ok(instances)
    }

    async fn load_publications(
        &self,
        client: &mut Client,
    ) -> Result<HashMap<String, ServicePublicationState>> {
        let prefix = format!(
            "{}publications/",
            cluster_prefix(&self.cluster_namespace, &self.cluster_id)
        );
        let kvs = client.get_prefix(prefix.as_bytes()).await?;
        let mut publications = HashMap::with_capacity(kvs.len());

        for (_key, value) in kvs {
            match serde_json::from_slice::<ServicePublicationState>(&value) {
                Ok(publication) => {
                    publications.insert(publication.service.clone(), publication);
                }
                Err(error) => warn!(error = %error, "failed to decode service publication"),
            }
        }

        Ok(publications)
    }

    async fn persist_service_statuses(&self, client: &mut Client) -> Result<()> {
        let nodes = self.load_cluster_nodes(client).await?;
        let services = self.load_services(client).await?;
        let instances_by_service = self.load_instances_by_service(client).await?;
        let publications = self.load_publications(client).await?;
        let dependency_cycles = compute_service_dependency_cycles(&services);
        let services_by_name = services
            .iter()
            .map(|service| (service.name.as_str(), service))
            .collect::<HashMap<_, _>>();

        for service in &services {
            let status = build_service_status_record(
                service,
                &nodes,
                &services_by_name,
                &instances_by_service,
                &publications,
                &dependency_cycles,
                self.heartbeat_timeout_secs,
            );
            let key = key_service_status(&self.cluster_namespace, &self.cluster_id, &service.name);
            client.put(&key, serde_json::to_vec(&status)?).await?;
        }

        self.cleanup_stale_service_statuses(client, &services)
            .await?;
        Ok(())
    }

    async fn cleanup_stale_service_statuses(
        &self,
        client: &mut Client,
        services: &[ServiceSpec],
    ) -> Result<()> {
        let desired = services
            .iter()
            .map(|service| service.name.as_str())
            .collect::<HashSet<_>>();
        let prefix = service_status_prefix(&self.cluster_namespace, &self.cluster_id);
        let existing = client.get_prefix(prefix.as_bytes()).await?;

        for (key, _) in existing {
            let key_str = String::from_utf8_lossy(&key);
            let Some(service_name) = key_str.strip_prefix(&prefix) else {
                continue;
            };
            if service_name.is_empty()
                || service_name.contains('/')
                || desired.contains(service_name)
            {
                continue;
            }
            let _ = client.delete(&key).await?;
        }

        Ok(())
    }

    /// Desired Stateに基づいてプロセスを起動/停止する
    async fn reconcile_processes(&mut self, client: &mut Client) -> Result<()> {
        let prefix = format!(
            "{}instances/",
            cluster_prefix(&self.cluster_namespace, &self.cluster_id)
        );
        let (kvs, _) = client.scan_prefix(prefix.as_bytes(), 0).await?;

        let mut desired_instances = Vec::new();
        let mut desired_keys = HashSet::new();
        for (_key, value, _) in kvs {
            let inst: ServiceInstanceSpec = match serde_json::from_slice(&value) {
                Ok(i) => i,
                Err(e) => {
                    warn!(error = %e, "failed to parse instance");
                    continue;
                }
            };

            if !self.is_local_managed_instance(&inst) {
                continue;
            }

            if let Some(proc_spec) = self.desired_process_spec(&inst) {
                desired_keys.insert(format!("{}/{}", inst.service, inst.instance_id));
                desired_instances.push((inst.service.clone(), inst.instance_id.clone(), proc_spec));
            }
        }

        // Desired Stateに基づいてプロセスを管理
        for (service, instance_id, proc_spec) in desired_instances {
            if let Some(process) = self.process_manager.get_mut(&service, &instance_id) {
                if process.spec != proc_spec {
                    process.spec = proc_spec;
                    info!(
                        service = %service,
                        instance_id = %instance_id,
                        "process spec changed, restarting"
                    );
                    if let Err(e) = process.restart().await {
                        warn!(
                            service = %service,
                            instance_id = %instance_id,
                            error = ?e,
                            "failed to restart process after spec update"
                        );
                    }
                }
            } else {
                // 新しいプロセスを追加
                self.process_manager
                    .add(service.clone(), instance_id.clone(), proc_spec);
                info!(
                    service = %service,
                    instance_id = %instance_id,
                    "added new process to manager"
                );
            }
        }

        self.process_manager.stop_unmanaged(&desired_keys).await?;

        // Desired Stateにないプロセスは停止して削除
        for (service, instance_id) in self.process_manager.list_instances() {
            let key = format!("{}/{}", service, instance_id);
            if !desired_keys.contains(&key) {
                if let Err(e) = self
                    .process_manager
                    .stop_and_remove(&service, &instance_id)
                    .await
                {
                    warn!(
                        service = %service,
                        instance_id = %instance_id,
                        error = %e,
                        "failed to stop/remove process"
                    );
                }
            }
        }

        // Reconcile: 停止しているプロセスを再起動
        self.process_manager.reconcile().await?;

        Ok(())
    }

    /// 各ServiceInstanceのヘルスチェックを実行し、Chainfire上の状態を更新
    fn health_check_key(inst: &ServiceInstanceSpec) -> String {
        format!("{}/{}", inst.service, inst.instance_id)
    }

    fn health_check_due(&self, inst: &ServiceInstanceSpec, _spec: &HealthCheckSpec) -> bool {
        let key = Self::health_check_key(inst);
        match self.next_health_checks.get(&key) {
            Some(next_due) => Utc::now() >= *next_due,
            None => true,
        }
    }

    fn schedule_next_health_check(&mut self, inst: &ServiceInstanceSpec, spec: &HealthCheckSpec) {
        let interval_secs = spec.interval_secs.unwrap_or(self.interval.as_secs()).max(1);
        let key = Self::health_check_key(inst);
        self.next_health_checks.insert(
            key,
            Utc::now() + chrono::Duration::seconds(interval_secs as i64),
        );
    }

    fn startup_grace_secs(&self, inst: &ServiceInstanceSpec, spec: &HealthCheckSpec) -> u64 {
        spec.startup_grace_secs.unwrap_or_else(|| {
            let interval = spec.interval_secs.unwrap_or(self.interval.as_secs()).max(1);
            let timeout = spec.timeout_secs.unwrap_or(5).max(1);
            if inst.container.is_some() {
                std::cmp::max(60, std::cmp::max(interval * 6, timeout * 6))
            } else {
                std::cmp::max(5, std::cmp::max(interval * 3, timeout * 3))
            }
        })
    }

    async fn process_running(&mut self, inst: &ServiceInstanceSpec) -> bool {
        self.process_manager
            .is_running(&inst.service, &inst.instance_id)
            .await
            .unwrap_or(false)
    }

    fn starting_within_grace(
        &self,
        inst: &ServiceInstanceSpec,
        spec: &HealthCheckSpec,
        process_running: bool,
        started_at: DateTime<Utc>,
        now: DateTime<Utc>,
        health_status: &str,
    ) -> bool {
        if health_status == "healthy" || !process_running {
            return false;
        }

        let startup_grace_secs = self.startup_grace_secs(inst, spec);
        if startup_grace_secs == 0 {
            return false;
        }

        now.signed_duration_since(started_at).num_seconds() < startup_grace_secs as i64
    }

    async fn update_health_status(&mut self, client: &mut Client) -> Result<()> {
        let prefix = format!(
            "{}instances/",
            cluster_prefix(&self.cluster_namespace, &self.cluster_id)
        );
        let (kvs, _) = client.scan_prefix(prefix.as_bytes(), 0).await?;
        let mut seen = HashSet::new();

        for (key, value, mod_revision) in kvs {
            let inst_value: Value = match serde_json::from_slice(&value) {
                Ok(v) => v,
                Err(e) => {
                    warn!(error = %e, "failed to parse instance json");
                    continue;
                }
            };

            let inst: ServiceInstanceSpec = match serde_json::from_value(inst_value.clone()) {
                Ok(i) => i,
                Err(e) => {
                    warn!(error = %e, "failed to parse instance");
                    continue;
                }
            };

            if !self.is_local_managed_instance(&inst) {
                continue;
            }

            let health_key = Self::health_check_key(&inst);
            seen.insert(health_key.clone());

            if let Some(ref health_check) = inst.health_check {
                if !self.health_check_due(&inst, health_check) {
                    continue;
                }
            }

            let now = Utc::now();
            let process_running = self.process_running(&inst).await;
            let started_at = self
                .process_manager
                .started_at(&inst.service, &inst.instance_id)
                .or(inst.observed_at)
                .unwrap_or(now);

            let health_status = if let Some(ref health_check) = inst.health_check {
                let status = self.check_health(&inst, health_check).await;
                self.schedule_next_health_check(&inst, health_check);
                if self.starting_within_grace(
                    &inst,
                    health_check,
                    process_running,
                    started_at,
                    now,
                    &status,
                ) {
                    "starting".to_string()
                } else {
                    status
                }
            } else {
                "healthy".to_string() // デフォルトはhealthy
            };

            if let Err(e) = self
                .persist_instance_health(
                    client,
                    &key,
                    &inst,
                    inst_value,
                    mod_revision,
                    &started_at,
                    &health_status,
                )
                .await
            {
                warn!(
                    service = %inst.service,
                    instance_id = %inst.instance_id,
                    mod_revision,
                    error = ?e,
                    "failed to update instance health status"
                );
            }

            info!(
                service = %inst.service,
                instance_id = %inst.instance_id,
                status = %health_status,
                "health check completed"
            );
        }

        self.next_health_checks.retain(|key, _| seen.contains(key));

        Ok(())
    }

    async fn check_health(&self, inst: &ServiceInstanceSpec, spec: &HealthCheckSpec) -> String {
        match spec.check_type.as_str() {
            "http" => {
                if let Some(ref path) = spec.path {
                    let url = format!("http://{}:{}{}", inst.ip, inst.port, path);
                    let client = reqwest::Client::builder()
                        .timeout(Duration::from_secs(spec.timeout_secs.unwrap_or(5)))
                        .build();
                    if let Ok(c) = client {
                        if let Ok(resp) = c.get(&url).send().await {
                            if resp.status().is_success() {
                                return "healthy".to_string();
                            }
                        }
                    }
                }
                "unhealthy".to_string()
            }
            "tcp" => {
                use tokio::net::TcpStream;
                let addr = format!("{}:{}", inst.ip, inst.port);
                match tokio::time::timeout(
                    Duration::from_secs(spec.timeout_secs.unwrap_or(5)),
                    TcpStream::connect(&addr),
                )
                .await
                {
                    Ok(Ok(_)) => "healthy".to_string(),
                    _ => "unhealthy".to_string(),
                }
            }
            "command" => {
                if let Some(ref cmd) = spec.path {
                    match Command::new("sh")
                        .arg("-c")
                        .arg(cmd)
                        .stdout(Stdio::null())
                        .stderr(Stdio::null())
                        .status()
                        .await
                    {
                        Ok(status) if status.success() => "healthy".to_string(),
                        _ => "unhealthy".to_string(),
                    }
                } else {
                    "unknown".to_string()
                }
            }
            _ => "unknown".to_string(),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::collections::HashMap;
    use std::path::PathBuf;

    fn test_agent() -> Agent {
        Agent::new(
            "http://127.0.0.1:7000".to_string(),
            "ultracloud".to_string(),
            "test-cluster".to_string(),
            "node01".to_string(),
            Duration::from_secs(1),
            300,
            false,
            false,
            PathBuf::from("/tmp/ultracloud-node-agent-tests"),
        )
    }

    fn test_instance() -> ServiceInstanceSpec {
        ServiceInstanceSpec {
            instance_id: "api-node01".to_string(),
            service: "api".to_string(),
            node_id: "node01".to_string(),
            ip: "127.0.0.2".to_string(),
            port: 18080,
            mesh_port: Some(28080),
            version: None,
            health_check: None,
            process: Some(ProcessSpec {
                command: "python3".to_string(),
                args: vec![
                    "-m".to_string(),
                    "http.server".to_string(),
                    "${INSTANCE_PORT}".to_string(),
                    "--bind".to_string(),
                    "${INSTANCE_IP}".to_string(),
                ],
                working_dir: Some("/srv/${SERVICE}".to_string()),
                env: HashMap::from([("INSTANCE".to_string(), "${INSTANCE_ID}".to_string())]),
            }),
            container: None,
            managed_by: None,
            state: None,
            last_heartbeat: None,
            observed_at: None,
        }
    }

    #[test]
    fn test_render_process_spec_templates() {
        let agent = test_agent();
        let instance = test_instance();
        let rendered = agent.render_process_spec(instance.process.as_ref().unwrap(), &instance);

        assert_eq!(rendered.args[2], "18080");
        assert_eq!(rendered.args[4], "127.0.0.2");
        assert_eq!(rendered.working_dir.as_deref(), Some("/srv/api"));
        assert_eq!(
            rendered.env.get("INSTANCE").map(String::as_str),
            Some("api-node01")
        );
    }

    #[test]
    fn test_container_takes_precedence_when_present() {
        let agent = test_agent();
        let mut instance = test_instance();
        instance.container = Some(ContainerSpec {
            image: "nginx:latest".to_string(),
            runtime: Some("docker".to_string()),
            command: vec![],
            args: vec![],
            env: HashMap::new(),
            ports: vec![],
            volumes: vec![],
            network_mode: None,
            pull_policy: None,
            working_dir: None,
        });

        let rendered = agent.desired_process_spec(&instance).unwrap();
        assert_eq!(rendered.command, "docker");
    }

    #[test]
    fn test_health_check_interval_is_tracked_per_instance() {
        let mut agent = test_agent();
        let mut instance = test_instance();
        let health_check = HealthCheckSpec {
            check_type: "http".to_string(),
            path: Some("/health".to_string()),
            interval_secs: Some(30),
            timeout_secs: Some(5),
            startup_grace_secs: Some(30),
        };
        instance.health_check = Some(health_check.clone());

        assert!(agent.health_check_due(&instance, &health_check));
        agent.schedule_next_health_check(&instance, &health_check);
        assert!(!agent.health_check_due(&instance, &health_check));

        let key = Agent::health_check_key(&instance);
        agent
            .next_health_checks
            .insert(key, Utc::now() - chrono::Duration::seconds(1));
        assert!(agent.health_check_due(&instance, &health_check));
    }

    #[test]
    fn test_process_instances_get_default_startup_grace() {
        let agent = test_agent();
        let mut instance = test_instance();
        instance.container = None;
        let health_check = HealthCheckSpec {
            check_type: "http".to_string(),
            path: Some("/health".to_string()),
            interval_secs: Some(1),
            timeout_secs: Some(2),
            startup_grace_secs: None,
        };

        assert_eq!(agent.startup_grace_secs(&instance, &health_check), 6);
    }

    #[test]
    fn test_apply_instance_health_fields_replaces_nulls_and_preserves_observed_at() {
        let started_at = DateTime::parse_from_rfc3339("2026-03-31T03:00:00Z")
            .unwrap()
            .with_timezone(&Utc);
        let heartbeat_at = DateTime::parse_from_rfc3339("2026-03-31T03:00:05Z")
            .unwrap()
            .with_timezone(&Utc);
        let original_observed_at = "2026-03-31T02:59:59Z";

        let mut null_observed = serde_json::json!({
            "observed_at": null,
            "state": null,
            "last_heartbeat": null
        });
        Agent::apply_instance_health_fields(
            &mut null_observed,
            &started_at,
            &heartbeat_at,
            "healthy",
        );
        assert_eq!(
            null_observed.get("observed_at").and_then(Value::as_str),
            Some("2026-03-31T03:00:00+00:00")
        );
        assert_eq!(
            null_observed.get("state").and_then(Value::as_str),
            Some("healthy")
        );
        assert_eq!(
            null_observed.get("last_heartbeat").and_then(Value::as_str),
            Some("2026-03-31T03:00:05+00:00")
        );

        let mut existing_observed = serde_json::json!({
            "observed_at": original_observed_at,
            "state": "starting",
            "last_heartbeat": null
        });
        Agent::apply_instance_health_fields(
            &mut existing_observed,
            &started_at,
            &heartbeat_at,
            "healthy",
        );
        assert_eq!(
            existing_observed.get("observed_at").and_then(Value::as_str),
            Some(original_observed_at)
        );
        assert_eq!(
            existing_observed.get("state").and_then(Value::as_str),
            Some("healthy")
        );
    }

    #[test]
    fn watch_signature_ignores_node_heartbeat_only_changes() {
        let mut agent = test_agent();
        let node_key = key_node(&agent.cluster_namespace, &agent.cluster_id, &agent.node_id);

        let first = WatchChange {
            event_type: chainfire_client::EventType::Put,
            key: node_key.clone(),
            value: serde_json::to_vec(&serde_json::json!({
                "node_id": "node01",
                "hostname": "node01",
                "ip": "10.0.0.11",
                "roles": ["worker"],
                "labels": {"tier": "general"},
                "state": "active",
                "last_heartbeat": "2026-04-01T00:00:00Z"
            }))
            .unwrap(),
            revision: 1,
        };
        assert!(agent.apply_watch_change(&first));

        let heartbeat_only = WatchChange {
            revision: 2,
            value: serde_json::to_vec(&serde_json::json!({
                "node_id": "node01",
                "hostname": "node01",
                "ip": "10.0.0.11",
                "roles": ["worker"],
                "labels": {"tier": "general"},
                "state": "active",
                "last_heartbeat": "2026-04-01T00:00:05Z"
            }))
            .unwrap(),
            ..first.clone()
        };
        assert!(!agent.apply_watch_change(&heartbeat_only));

        let drain = WatchChange {
            revision: 3,
            value: serde_json::to_vec(&serde_json::json!({
                "node_id": "node01",
                "hostname": "node01",
                "ip": "10.0.0.11",
                "roles": ["worker"],
                "labels": {"tier": "general"},
                "state": "draining",
                "last_heartbeat": "2026-04-01T00:00:10Z"
            }))
            .unwrap(),
            ..first
        };
        assert!(agent.apply_watch_change(&drain));
    }

    #[test]
    fn watch_signature_ignores_instance_observed_fields_only_changes() {
        let mut agent = test_agent();
        let instance_key = key_instance(
            &agent.cluster_namespace,
            &agent.cluster_id,
            "api",
            "api-node01",
        );

        let first = WatchChange {
            event_type: chainfire_client::EventType::Put,
            key: instance_key.clone(),
            value: serde_json::to_vec(&serde_json::json!({
                "instance_id": "api-node01",
                "service": "api",
                "node_id": "node01",
                "ip": "127.0.0.2",
                "port": 18080,
                "managed_by": "fleet-scheduler",
                "state": null,
                "last_heartbeat": null,
                "observed_at": null
            }))
            .unwrap(),
            revision: 1,
        };
        assert!(agent.apply_watch_change(&first));

        let observed_only = WatchChange {
            revision: 2,
            value: serde_json::to_vec(&serde_json::json!({
                "instance_id": "api-node01",
                "service": "api",
                "node_id": "node01",
                "ip": "127.0.0.2",
                "port": 18080,
                "managed_by": "fleet-scheduler",
                "state": "healthy",
                "last_heartbeat": "2026-04-01T00:00:05Z",
                "observed_at": "2026-04-01T00:00:05Z"
            }))
            .unwrap(),
            ..first.clone()
        };
        assert!(!agent.apply_watch_change(&observed_only));

        let desired_change = WatchChange {
            revision: 3,
            value: serde_json::to_vec(&serde_json::json!({
                "instance_id": "api-node01",
                "service": "api",
                "node_id": "node01",
                "ip": "127.0.0.2",
                "port": 18081,
                "managed_by": "fleet-scheduler",
                "state": "healthy",
                "last_heartbeat": "2026-04-01T00:00:10Z",
                "observed_at": "2026-04-01T00:00:10Z"
            }))
            .unwrap(),
            ..first
        };
        assert!(agent.apply_watch_change(&desired_change));
    }
}
