//! Replicated distributed storage backend
//!
//! Implements StorageBackend using N-way replication for
//! performance-oriented redundancy with read scaling.

use crate::chunk::ChunkManager;
use crate::config::DistributedConfig;
use crate::node::{NodeClientTrait, NodeError, NodeRegistry, NodeResult};
use crate::placement::{ConsistentHashSelector, NodeSelector};
use crate::repair::{RepairQueue, ReplicatedRepairTask};
use async_trait::async_trait;
use bytes::{Bytes, BytesMut};
use futures::stream::{FuturesUnordered, StreamExt};
use lightningstor_storage::{StorageBackend, StorageError, StorageResult};
use lightningstor_types::ObjectId;
use std::net::IpAddr;
use std::sync::Arc;
use std::time::Duration;
use tracing::{debug, error, warn};

const BEST_EFFORT_DELETE_TIMEOUT: Duration = Duration::from_secs(5);
const MAX_CONCURRENT_CHUNK_WRITES: usize = 16;
const MAX_CONCURRENT_CHUNK_READS: usize = 32;
const TARGET_CONCURRENT_CHUNK_WRITES_PER_REPLICA: usize = 4;
const TARGET_CONCURRENT_CHUNK_READS_PER_REPLICA: usize = 8;

#[derive(Debug, Clone)]
struct ReplicatedObjectMetadata {
    chunk_count: usize,
    original_size: usize,
    chunk_size: usize,
}

impl ReplicatedObjectMetadata {
    fn new(original_size: usize, chunk_count: usize, chunk_size: usize) -> Self {
        Self {
            chunk_count,
            original_size,
            chunk_size,
        }
    }

    fn to_bytes(&self) -> Bytes {
        let mut bytes = BytesMut::with_capacity(24);
        bytes.extend_from_slice(&(self.chunk_count as u64).to_le_bytes());
        bytes.extend_from_slice(&(self.original_size as u64).to_le_bytes());
        bytes.extend_from_slice(&(self.chunk_size as u64).to_le_bytes());
        bytes.freeze()
    }

    fn from_bytes(bytes: &[u8]) -> Option<Self> {
        if bytes.len() < 24 {
            return None;
        }

        Some(Self {
            chunk_count: u64::from_le_bytes(bytes[0..8].try_into().ok()?) as usize,
            original_size: u64::from_le_bytes(bytes[8..16].try_into().ok()?) as usize,
            chunk_size: u64::from_le_bytes(bytes[16..24].try_into().ok()?) as usize,
        })
    }
}

/// Replicated storage backend with N-way replication
///
/// Stores objects by replicating them to N nodes. Provides:
/// - Fast reads (any replica can serve)
/// - Simple failure handling (no reconstruction needed)
/// - Higher storage overhead than erasure coding
pub struct ReplicatedBackend {
    /// Node registry for discovering storage nodes
    node_registry: Arc<dyn NodeRegistry>,
    /// Node selector for placement decisions
    node_selector: Arc<dyn NodeSelector>,
    /// Configuration (kept for future use)
    #[allow(dead_code)]
    config: DistributedConfig,
    /// Chunk manager for splitting large objects into smaller replica RPCs.
    chunk_manager: Arc<ChunkManager>,
    /// Number of replicas
    replica_count: usize,
    /// Read quorum (minimum replicas for successful read)
    read_quorum: usize,
    /// Write quorum (minimum replicas for successful write)
    write_quorum: usize,
    /// Durable queue for repairing under-replicated chunks.
    repair_queue: Option<Arc<dyn RepairQueue>>,
}

impl ReplicatedBackend {
    /// Create a new replicated backend
    ///
    /// # Arguments
    /// * `config` - Distributed storage configuration (must have Replicated redundancy mode)
    /// * `node_registry` - Registry for discovering storage nodes
    pub async fn new(
        config: DistributedConfig,
        node_registry: Arc<dyn NodeRegistry>,
    ) -> StorageResult<Self> {
        Self::new_with_repair_queue(config, node_registry, None).await
    }

    /// Create a replicated backend with an optional durable repair queue.
    pub async fn new_with_repair_queue(
        config: DistributedConfig,
        node_registry: Arc<dyn NodeRegistry>,
        repair_queue: Option<Arc<dyn RepairQueue>>,
    ) -> StorageResult<Self> {
        let (replica_count, read_quorum, write_quorum) = match &config.redundancy {
            crate::config::RedundancyMode::Replicated {
                replica_count,
                read_quorum,
                write_quorum,
            } => (*replica_count, *read_quorum, *write_quorum),
            _ => {
                return Err(StorageError::Backend(
                    "ReplicatedBackend requires Replicated redundancy mode".into(),
                ))
            }
        };

        let node_selector = Arc::new(ConsistentHashSelector::new());

        Ok(Self {
            node_registry,
            node_selector,
            chunk_manager: Arc::new(ChunkManager::new(config.chunk.clone())),
            config,
            replica_count,
            read_quorum,
            write_quorum,
            repair_queue,
        })
    }

    /// Get the number of replicas
    pub fn replica_count(&self) -> usize {
        self.replica_count
    }

    /// Get the read quorum
    pub fn read_quorum(&self) -> usize {
        self.read_quorum
    }

    /// Get the write quorum
    pub fn write_quorum(&self) -> usize {
        self.write_quorum
    }

    async fn finalize_pending_replica_writes(
        repair_queue: Option<Arc<dyn RepairQueue>>,
        mut pending_writes: FuturesUnordered<tokio::task::JoinHandle<(String, NodeResult<()>)>>,
        key: String,
        shard_index: u32,
        mut success_count: usize,
        total_replicas: usize,
        reason: String,
    ) {
        let mut errors = Vec::new();

        while let Some(result) = pending_writes.next().await {
            match result {
                Ok((_, Ok(()))) => success_count += 1,
                Ok((node_id, Err(err))) => errors.push(format!("{node_id}: {err}")),
                Err(join_err) => errors.push(format!("join error: {join_err}")),
            }
        }

        if success_count >= total_replicas {
            return;
        }

        if let Some(queue) = repair_queue {
            queue
                .enqueue_repair(ReplicatedRepairTask::new(key.clone(), shard_index, reason))
                .await;
        }

        warn!(
            chunk_key = %key,
            shard_index,
            success_count,
            total_replicas,
            errors = ?errors,
            "Replica write completed below desired replication; repair task queued"
        );
    }

    async fn finalize_pending_chunked_write_repairs(
        repair_queue: Option<Arc<dyn RepairQueue>>,
        mut pending_writes: FuturesUnordered<tokio::task::JoinHandle<(String, NodeResult<()>)>>,
        repair_targets: Vec<(String, u32)>,
        object_id: String,
        mut success_count: usize,
        total_replicas: usize,
        reason: String,
    ) {
        let mut errors = Vec::new();

        while let Some(result) = pending_writes.next().await {
            match result {
                Ok((_, Ok(()))) => success_count += 1,
                Ok((node_id, Err(err))) => errors.push(format!("{node_id}: {err}")),
                Err(join_err) => errors.push(format!("join error: {join_err}")),
            }
        }

        if success_count >= total_replicas {
            return;
        }

        if let Some(queue) = repair_queue {
            for (chunk_key, shard_index) in repair_targets {
                queue
                    .enqueue_repair(ReplicatedRepairTask::new(
                        chunk_key,
                        shard_index,
                        reason.clone(),
                    ))
                    .await;
            }
        }

        warn!(
            object_id = %object_id,
            success_count,
            total_replicas,
            errors = ?errors,
            "Chunked replica write completed below desired replication; repair tasks queued"
        );
    }

    fn chunk_write_parallelism(&self, chunk_count: usize) -> usize {
        chunk_count
            .min(
                self.replica_count
                    .saturating_mul(TARGET_CONCURRENT_CHUNK_WRITES_PER_REPLICA)
                    .min(MAX_CONCURRENT_CHUNK_WRITES),
            )
            .max(1)
    }

    fn chunk_read_parallelism(&self, chunk_count: usize) -> usize {
        chunk_count
            .min(
                self.replica_count
                    .saturating_mul(TARGET_CONCURRENT_CHUNK_READS_PER_REPLICA)
                    .min(MAX_CONCURRENT_CHUNK_READS),
            )
            .max(1)
    }

    /// Select nodes for writing replicas
    async fn select_replica_nodes_for_key(
        &self,
        placement_key: &str,
    ) -> StorageResult<Vec<Arc<dyn NodeClientTrait>>> {
        let nodes = self
            .node_registry
            .get_healthy_nodes()
            .await
            .map_err(|e| StorageError::Backend(e.to_string()))?;

        if nodes.len() < self.replica_count {
            return Err(StorageError::Backend(format!(
                "Not enough healthy nodes: need {}, have {}",
                self.replica_count,
                nodes.len()
            )));
        }

        self.node_selector
            .select_nodes_for_key(&nodes, self.replica_count, placement_key)
            .await
            .map_err(|e| StorageError::Backend(e.to_string()))
    }

    /// Generate the chunk key for an object
    fn object_key(object_id: &ObjectId) -> String {
        format!("obj_{}", object_id)
    }

    /// Generate the chunk key for a part
    fn part_key(upload_id: &str, part_number: u32) -> String {
        format!("part_{}_{}", upload_id, part_number)
    }

    fn object_metadata_key(object_id: &ObjectId) -> String {
        format!("objmeta_{}", object_id)
    }

    fn object_chunk_key(object_id: &ObjectId, chunk_index: usize) -> String {
        format!("obj_{}_chunk_{}", object_id, chunk_index)
    }

    async fn read_replicas(&self, key: &str, shard_index: u32) -> StorageResult<Option<Bytes>> {
        let nodes = self
            .node_registry
            .get_healthy_nodes()
            .await
            .map_err(|e| StorageError::Backend(e.to_string()))?;

        self.read_replicas_from_nodes(&nodes, key, shard_index).await
    }

    async fn read_replicas_from_nodes(
        &self,
        nodes: &[Arc<dyn NodeClientTrait>],
        key: &str,
        shard_index: u32,
    ) -> StorageResult<Option<Bytes>> {

        if nodes.is_empty() {
            return Err(StorageError::Backend(
                "No healthy storage nodes available".to_string(),
            ));
        }

        let mut ordered_nodes = Self::ordered_read_nodes(nodes, self
            .node_selector
            .select_for_read(nodes, key)
            .await
            .ok());

        if let Some(preferred) = ordered_nodes.first() {
            match preferred.get_chunk(key, shard_index, false).await {
                Ok(data) => return Ok(Some(data)),
                Err(NodeError::NotFound(_)) => {}
                Err(e) => {
                    warn!(
                        chunk_key = key,
                        node_id = preferred.node_id(),
                        error = ?e,
                        "Failed to read from preferred node, trying others"
                    );
                }
            }
        }

        for node in ordered_nodes.drain(1..) {
            match node.get_chunk(key, shard_index, false).await {
                Ok(data) => return Ok(Some(data)),
                Err(NodeError::NotFound(_)) => continue,
                Err(_) => continue,
            }
        }

        Ok(None)
    }

    async fn load_object_metadata(
        &self,
        object_id: &ObjectId,
    ) -> StorageResult<Option<ReplicatedObjectMetadata>> {
        match self
            .read_replicas(&Self::object_metadata_key(object_id), 0)
            .await
        {
            Ok(Some(bytes)) => ReplicatedObjectMetadata::from_bytes(bytes.as_ref())
                .map(Some)
                .ok_or_else(|| {
                    StorageError::Backend(format!(
                        "Invalid replicated metadata for object {}",
                        object_id
                    ))
                }),
            Ok(None) => Ok(None),
            Err(err) => Err(err),
        }
    }

    async fn delete_key_best_effort(&self, chunk_key: String) -> StorageResult<()> {
        let nodes = self
            .node_registry
            .get_all_nodes()
            .await
            .map_err(|e| StorageError::Backend(e.to_string()))?;

        if nodes.is_empty() {
            return Err(StorageError::Backend(
                "No storage nodes available for delete".into(),
            ));
        }

        let delete_quorum = self.write_quorum.min(nodes.len());
        let total_replicas = nodes.len();
        let mut success_count = 0usize;
        let mut completed_count = 0usize;
        let mut delete_futures = FuturesUnordered::new();

        for node in nodes {
            let node_id = node.node_id().to_string();
            let key = chunk_key.clone();
            delete_futures.push(tokio::spawn(async move {
                (
                    node_id,
                    key.clone(),
                    tokio::time::timeout(BEST_EFFORT_DELETE_TIMEOUT, node.delete_chunk(&key)).await,
                )
            }));
        }

        while let Some(result) = delete_futures.next().await {
            completed_count += 1;

            match result {
                Ok((node_id, key, Ok(Ok(())))) => {
                    success_count += 1;
                    debug!(
                        node_id,
                        chunk_key = key,
                        success_count,
                        delete_quorum,
                        "Deleted replica"
                    );

                    if success_count >= delete_quorum {
                        debug!(
                            chunk_key,
                            success_count, delete_quorum, total_replicas, "Delete quorum satisfied"
                        );
                        return Ok(());
                    }
                }
                Ok((node_id, key, Ok(Err(e)))) => {
                    warn!(
                        node_id,
                        chunk_key = key,
                        error = ?e,
                        "Failed to delete replica"
                    );
                }
                Ok((node_id, key, Err(_))) => {
                    warn!(
                        node_id,
                        chunk_key = key,
                        timeout_secs = BEST_EFFORT_DELETE_TIMEOUT.as_secs(),
                        "Timed out deleting replica"
                    );
                }
                Err(e) => {
                    warn!(error = ?e, chunk_key, "Delete task join failed");
                }
            }

            let remaining = total_replicas.saturating_sub(completed_count);
            if success_count + remaining < delete_quorum {
                break;
            }
        }

        Err(StorageError::Backend(format!(
            "Failed to reach delete quorum: deleted {} of {} required replicas",
            success_count, delete_quorum
        )))
    }

    async fn write_replicas(
        &self,
        key: String,
        shard_index: u32,
        data: Bytes,
    ) -> StorageResult<()> {
        let nodes = self.select_replica_nodes_for_key(&key).await?;
        let total_replicas = nodes.len();
        let mut write_futures = FuturesUnordered::new();

        for node in nodes {
            let node = node.clone();
            let chunk_key = key.clone();
            let data = data.clone();
            write_futures.push(tokio::spawn(async move {
                let node_id = node.node_id().to_string();
                let result = node.put_chunk(&chunk_key, shard_index, false, data).await;
                (node_id, result)
            }));
        }

        let mut success_count = 0usize;
        let mut error_count = 0usize;
        let mut errors = Vec::new();

        while let Some(result) = write_futures.next().await {
            match result {
                Ok((_, Ok(()))) => {
                    success_count += 1;
                    if success_count >= self.write_quorum {
                        if success_count < total_replicas {
                            let pending_writes =
                                std::mem::replace(&mut write_futures, FuturesUnordered::new());
                            tokio::spawn(Self::finalize_pending_replica_writes(
                                self.repair_queue.clone(),
                                pending_writes,
                                key.clone(),
                                shard_index,
                                success_count,
                                total_replicas,
                                format!(
                                    "replica write completed below desired replication after quorum ({success_count}/{total_replicas})"
                                ),
                            ));
                        }
                        debug!(
                            chunk_key = %key,
                            success_count,
                            error_count,
                            write_quorum = self.write_quorum,
                            total_replicas,
                            "Write quorum satisfied"
                        );
                        return Ok(());
                    }
                }
                Ok((node_id, Err(err))) => {
                    error_count += 1;
                    errors.push(format!("{node_id}: {err}"));
                    let remaining = total_replicas.saturating_sub(success_count + error_count);
                    if success_count + remaining < self.write_quorum {
                        break;
                    }
                }
                Err(join_err) => {
                    error_count += 1;
                    errors.push(format!("join error: {join_err}"));
                    let remaining = total_replicas.saturating_sub(success_count + error_count);
                    if success_count + remaining < self.write_quorum {
                        break;
                    }
                }
            }
        }

        error!(
            chunk_key = %key,
            success_count,
            write_quorum = self.write_quorum,
            errors = ?errors,
            "Failed to write quorum"
        );
        Err(StorageError::Backend(format!(
            "Failed to write quorum: {} of {} required succeeded",
            success_count, self.write_quorum
        )))
    }

    async fn write_chunked_object(&self, object_id: &ObjectId, data: Bytes) -> StorageResult<()> {
        let chunk_size = self.chunk_manager.effective_chunk_size(data.len());
        let chunk_count = ChunkManager::chunk_count_for_size(data.len(), chunk_size);
        let metadata = ReplicatedObjectMetadata::new(data.len(), chunk_count, chunk_size);
        let mut requests = Vec::with_capacity(chunk_count + 1);
        for chunk_index in 0..chunk_count {
            let chunk_key = Self::object_chunk_key(object_id, chunk_index);
            let (start, len) = ChunkManager::chunk_range_for_size(data.len(), chunk_index, chunk_size);
            let chunk_bytes = data.slice(start..start + len);
            requests.push((chunk_key, chunk_index as u32, false, chunk_bytes));
        }
        requests.push((Self::object_metadata_key(object_id), 0, false, metadata.to_bytes()));

        let nodes = self
            .select_replica_nodes_for_key(&Self::object_chunk_key(object_id, 0))
            .await?;
        let total_replicas = nodes.len();
        let mut write_futures = FuturesUnordered::new();

        for node in nodes {
            let node = node.clone();
            let node_id = node.node_id().to_string();
            let batch = requests.clone();
            write_futures.push(tokio::spawn(async move {
                let result = node.batch_put_chunks(batch).await;
                (node_id, result)
            }));
        }

        let mut success_count = 0usize;
        let mut error_count = 0usize;
        let mut errors = Vec::new();

        while let Some(result) = write_futures.next().await {
            match result {
                Ok((_, Ok(()))) => {
                    success_count += 1;
                    if success_count >= self.write_quorum {
                        if success_count < total_replicas {
                            let repair_targets = requests
                                .iter()
                                .map(|(chunk_key, shard_index, _, _)| {
                                    (chunk_key.clone(), *shard_index)
                                })
                                .collect::<Vec<_>>();
                            let pending_writes =
                                std::mem::replace(&mut write_futures, FuturesUnordered::new());
                            tokio::spawn(Self::finalize_pending_chunked_write_repairs(
                                self.repair_queue.clone(),
                                pending_writes,
                                repair_targets,
                                object_id.to_string(),
                                success_count,
                                total_replicas,
                                format!(
                                    "chunked object write completed below desired replication after quorum ({success_count}/{total_replicas})"
                                ),
                            ));
                        }
                        debug!(
                            object_id = %object_id,
                            chunk_count,
                            success_count,
                            error_count,
                            write_quorum = self.write_quorum,
                            total_replicas,
                            "Chunked object write quorum satisfied via batch replica writes"
                        );
                        return Ok(());
                    }
                }
                Ok((node_id, Err(err))) => {
                    error_count += 1;
                    errors.push(format!("{node_id}: {err}"));
                    let remaining = total_replicas.saturating_sub(success_count + error_count);
                    if success_count + remaining < self.write_quorum {
                        break;
                    }
                }
                Err(join_err) => {
                    error_count += 1;
                    errors.push(format!("join error: {join_err}"));
                    let remaining = total_replicas.saturating_sub(success_count + error_count);
                    if success_count + remaining < self.write_quorum {
                        break;
                    }
                }
            }
        }

        error!(
            object_id = %object_id,
            chunk_count,
            success_count,
            write_quorum = self.write_quorum,
            errors = ?errors,
            "Failed to write chunked object quorum"
        );
        Err(StorageError::Backend(format!(
            "Failed to write chunked object quorum: {} of {} required succeeded",
            success_count, self.write_quorum
        )))
    }

    pub async fn repair_chunk(&self, task: &ReplicatedRepairTask) -> StorageResult<()> {
        let healthy_nodes = self
            .node_registry
            .get_healthy_nodes()
            .await
            .map_err(|e| StorageError::Backend(e.to_string()))?;
        if healthy_nodes.is_empty() {
            return Err(StorageError::Backend(
                "No healthy storage nodes available for repair".to_string(),
            ));
        }
        let desired_nodes = self
            .node_selector
            .select_nodes_for_key(&healthy_nodes, self.replica_count, &task.key)
            .await
            .map_err(|e| StorageError::Backend(e.to_string()))?;

        let mut present_nodes = Vec::new();
        let mut missing_nodes = Vec::new();
        for node in desired_nodes {
            match node.chunk_exists(&task.key).await {
                Ok(true) => present_nodes.push(node),
                Ok(false) => missing_nodes.push(node),
                Err(err) => {
                    warn!(
                        chunk_key = task.key,
                        node_id = node.node_id(),
                        error = ?err,
                        "Failed to inspect chunk during repair; treating replica as missing"
                    );
                    missing_nodes.push(node);
                }
            }
        }

        if missing_nodes.is_empty() {
            return Ok(());
        }

        if present_nodes.is_empty() {
            let desired_node_ids = missing_nodes
                .iter()
                .map(|node| node.node_id().to_string())
                .collect::<std::collections::HashSet<_>>();
            for node in healthy_nodes {
                if desired_node_ids.contains(node.node_id()) {
                    continue;
                }
                match node.chunk_exists(&task.key).await {
                    Ok(true) => {
                        present_nodes.push(node);
                        break;
                    }
                    Ok(false) => {}
                    Err(err) => {
                        warn!(
                            chunk_key = task.key,
                            node_id = node.node_id(),
                            error = ?err,
                            "Failed to inspect off-placement chunk during repair"
                        );
                    }
                }
            }
        }

        let source = present_nodes.first().ok_or_else(|| {
            StorageError::Backend(format!(
                "Cannot repair {} because no healthy source replica is available",
                task.key
            ))
        })?;

        let data = source
            .get_chunk(&task.key, task.shard_index, false)
            .await
            .map_err(|err| {
                StorageError::Backend(format!(
                    "Failed to load repair source for {} from {}: {}",
                    task.key,
                    source.node_id(),
                    err
                ))
            })?;

        let mut repair_futures = FuturesUnordered::new();
        for node in missing_nodes {
            let node_id = node.node_id().to_string();
            let key = task.key.clone();
            let chunk = data.clone();
            let shard_index = task.shard_index;
            repair_futures.push(tokio::spawn(async move {
                let result = node.put_chunk(&key, shard_index, false, chunk).await;
                (node_id, result)
            }));
        }

        let mut repaired = 0usize;
        let mut errors = Vec::new();
        while let Some(result) = repair_futures.next().await {
            match result {
                Ok((_, Ok(()))) => repaired += 1,
                Ok((node_id, Err(err))) => errors.push(format!("{node_id}: {err}")),
                Err(join_err) => errors.push(format!("join error: {join_err}")),
            }
        }

        if errors.is_empty() {
            return Ok(());
        }

        Err(StorageError::Backend(format!(
            "Repair for {} only restored {} replicas: {}",
            task.key,
            repaired,
            errors.join(", ")
        )))
    }

    pub async fn chunk_exists_anywhere(&self, key: &str) -> StorageResult<bool> {
        let nodes = self
            .node_registry
            .get_all_nodes()
            .await
            .map_err(|e| StorageError::Backend(e.to_string()))?;

        for node in nodes {
            match node.chunk_exists(key).await {
                Ok(true) => return Ok(true),
                Ok(false) => {}
                Err(err) => {
                    warn!(
                        chunk_key = key,
                        node_id = node.node_id(),
                        error = ?err,
                        "Failed to inspect chunk while probing global existence"
                    );
                }
            }
        }

        Ok(false)
    }

    async fn read_chunked_object(
        &self,
        object_id: &ObjectId,
        metadata: &ReplicatedObjectMetadata,
    ) -> StorageResult<Bytes> {
        let nodes = self
            .node_registry
            .get_healthy_nodes()
            .await
            .map_err(|e| StorageError::Backend(e.to_string()))?;

        if !nodes.is_empty() {
            let preferred = self
                .node_selector
                .select_for_read(&nodes, &Self::object_chunk_key(object_id, 0))
                .await
                .ok();
            let ordered_nodes = Self::ordered_read_nodes(&nodes, preferred);

            if metadata.chunk_count > 1 {
                if let Some(local_node) = ordered_nodes.iter().find(|node| Self::is_local_node(node))
                {
                    let batch_requests: Vec<(String, u32, bool)> = (0..metadata.chunk_count)
                        .map(|chunk_index| {
                            (
                                Self::object_chunk_key(object_id, chunk_index),
                                chunk_index as u32,
                                false,
                            )
                        })
                        .collect();
                    match local_node.batch_get_chunks(batch_requests).await {
                        Ok(chunks) => {
                            return Self::assemble_chunked_bytes(
                                object_id,
                                metadata.original_size,
                                chunks,
                            );
                        }
                        Err(err) => {
                            warn!(
                                object_id = %object_id,
                                node_id = local_node.node_id(),
                                error = ?err,
                                "Local replica batch read failed, falling back to distributed reads"
                            );
                        }
                    }
                }
            }

            if ordered_nodes.len() > 1 && metadata.chunk_count > 1 && !Self::has_local_node(&ordered_nodes)
            {
                match self
                    .read_chunked_object_from_distributed_batches(
                        object_id,
                        metadata,
                        &ordered_nodes,
                    )
                    .await
                {
                    Ok(bytes) => return Ok(bytes),
                    Err(err) => {
                        warn!(
                            object_id = %object_id,
                            error = ?err,
                            "Distributed batch chunk read failed, falling back to single-replica batch reads"
                        );
                    }
                }
            }

            let batch_requests: Vec<(String, u32, bool)> = (0..metadata.chunk_count)
                .map(|chunk_index| {
                    (
                        Self::object_chunk_key(object_id, chunk_index),
                        chunk_index as u32,
                        false,
                    )
                })
                .collect();

            for node in ordered_nodes {
                match node.batch_get_chunks(batch_requests.clone()).await {
                    Ok(chunks) => {
                        return Self::assemble_chunked_bytes(object_id, metadata.original_size, chunks);
                    }
                    Err(err) => {
                        warn!(
                            object_id = %object_id,
                            node_id = node.node_id(),
                            error = ?err,
                            "Batch chunk read failed, falling back to per-chunk replica reads"
                        );
                    }
                }
            }
        }

        let parallelism = self.chunk_read_parallelism(metadata.chunk_count);
        let healthy_nodes = self
            .node_registry
            .get_healthy_nodes()
            .await
            .map_err(|e| StorageError::Backend(e.to_string()))?;
        let mut chunks: Vec<Option<Bytes>> = vec![None; metadata.chunk_count];
        let mut in_flight = FuturesUnordered::new();
        let mut next_chunk = 0usize;

        let push_chunk_read = |reads: &mut FuturesUnordered<_>, chunk_index: usize| {
            let chunk_key = Self::object_chunk_key(object_id, chunk_index);
            let nodes = healthy_nodes.clone();
            reads.push(async move {
                self.read_replicas_from_nodes(&nodes, &chunk_key, chunk_index as u32)
                    .await
                    .and_then(|chunk| {
                        chunk.ok_or_else(|| {
                            StorageError::Backend(format!(
                                "Chunk {} for object {} not found",
                                chunk_index, object_id
                            ))
                        })
                    })
                    .map(|chunk| (chunk_index, chunk))
            });
        };

        while next_chunk < parallelism {
            push_chunk_read(&mut in_flight, next_chunk);
            next_chunk += 1;
        }

        while let Some(result) = in_flight.next().await {
            let (chunk_index, chunk) = result?;
            chunks[chunk_index] = Some(chunk);

            if next_chunk < metadata.chunk_count {
                push_chunk_read(&mut in_flight, next_chunk);
                next_chunk += 1;
            }
        }

        let ordered_chunks = chunks
            .into_iter()
            .map(|chunk| {
                chunk.ok_or_else(|| {
                    StorageError::Backend(format!(
                        "Missing reassembled chunk for object {}",
                        object_id
                    ))
                })
            })
            .collect::<StorageResult<Vec<_>>>()?;

        Self::assemble_chunked_bytes(object_id, metadata.original_size, ordered_chunks)
    }

    async fn read_chunked_object_from_distributed_batches(
        &self,
        object_id: &ObjectId,
        metadata: &ReplicatedObjectMetadata,
        nodes: &[Arc<dyn NodeClientTrait>],
    ) -> StorageResult<Bytes> {
        let worker_count = nodes.len().min(metadata.chunk_count);
        if worker_count <= 1 {
            return Err(StorageError::Backend(
                "Distributed chunk batch read requires multiple replicas".into(),
            ));
        }

        let mut assignments: Vec<Vec<(usize, (String, u32, bool))>> =
            (0..worker_count).map(|_| Vec::new()).collect();
        for chunk_index in 0..metadata.chunk_count {
            let slot = chunk_index % worker_count;
            assignments[slot].push((
                chunk_index,
                (
                    Self::object_chunk_key(object_id, chunk_index),
                    chunk_index as u32,
                    false,
                ),
            ));
        }

        let mut batch_reads = FuturesUnordered::new();
        for (node, assignment) in nodes
            .iter()
            .take(worker_count)
            .cloned()
            .zip(assignments.into_iter())
        {
            if assignment.is_empty() {
                continue;
            }

            let chunk_indexes: Vec<usize> = assignment.iter().map(|(index, _)| *index).collect();
            let batch_requests: Vec<(String, u32, bool)> =
                assignment.into_iter().map(|(_, request)| request).collect();

            batch_reads.push(async move {
                let chunks = node.batch_get_chunks(batch_requests).await.map_err(|err| {
                    StorageError::Backend(format!(
                        "Replica {} batch_get_chunks failed: {}",
                        node.node_id(),
                        err
                    ))
                })?;

                if chunks.len() != chunk_indexes.len() {
                    return Err(StorageError::Backend(format!(
                        "Replica {} returned {} chunks for {} requested chunks",
                        node.node_id(),
                        chunks.len(),
                        chunk_indexes.len()
                    )));
                }

                Ok::<Vec<(usize, Bytes)>, StorageError>(
                    chunk_indexes.into_iter().zip(chunks.into_iter()).collect(),
                )
            });
        }

        let mut chunk_slots: Vec<Option<Bytes>> = vec![None; metadata.chunk_count];
        while let Some(result) = batch_reads.next().await {
            for (chunk_index, chunk) in result? {
                chunk_slots[chunk_index] = Some(chunk);
            }
        }

        let ordered_chunks = chunk_slots
            .into_iter()
            .map(|chunk| {
                chunk.ok_or_else(|| {
                    StorageError::Backend(format!(
                        "Missing distributed batch chunk for object {}",
                        object_id
                    ))
                })
            })
            .collect::<StorageResult<Vec<_>>>()?;

        Self::assemble_chunked_bytes(object_id, metadata.original_size, ordered_chunks)
    }

    async fn delete_chunked_object(
        &self,
        object_id: &ObjectId,
        metadata: &ReplicatedObjectMetadata,
    ) -> StorageResult<()> {
        let parallelism = self.chunk_write_parallelism(metadata.chunk_count);
        let mut in_flight = FuturesUnordered::new();
        let mut next_chunk = 0usize;

        let push_chunk_delete = |deletes: &mut FuturesUnordered<_>, chunk_index: usize| {
            let chunk_key = Self::object_chunk_key(object_id, chunk_index);
            deletes.push(async move { self.delete_key_best_effort(chunk_key).await });
        };

        while next_chunk < parallelism {
            push_chunk_delete(&mut in_flight, next_chunk);
            next_chunk += 1;
        }

        while let Some(result) = in_flight.next().await {
            result?;

            if next_chunk < metadata.chunk_count {
                push_chunk_delete(&mut in_flight, next_chunk);
                next_chunk += 1;
            }
        }

        self.delete_key_best_effort(Self::object_metadata_key(object_id))
            .await
    }

    fn assemble_chunked_bytes(
        object_id: &ObjectId,
        original_size: usize,
        chunks: Vec<Bytes>,
    ) -> StorageResult<Bytes> {
        let mut combined = BytesMut::with_capacity(original_size);
        for chunk in chunks {
            combined.extend_from_slice(chunk.as_ref());
        }
        if combined.len() < original_size {
            return Err(StorageError::Backend(format!(
                "Reassembled object {} is shorter than expected: {} < {}",
                object_id,
                combined.len(),
                original_size
            )));
        }
        combined.truncate(original_size);
        Ok(combined.freeze())
    }

    fn ordered_read_nodes(
        nodes: &[Arc<dyn NodeClientTrait>],
        preferred: Option<Arc<dyn NodeClientTrait>>,
    ) -> Vec<Arc<dyn NodeClientTrait>> {
        let mut ordered = Vec::with_capacity(nodes.len());

        if let Some(local) = nodes.iter().find(|node| Self::is_local_node(node)) {
            ordered.push(local.clone());
        }

        if let Some(preferred) = preferred {
            if ordered
                .iter()
                .all(|existing| existing.node_id() != preferred.node_id())
            {
                ordered.push(preferred);
            }
        }

        for node in nodes {
            if ordered
                .iter()
                .all(|existing| existing.node_id() != node.node_id())
            {
                ordered.push(node.clone());
            }
        }

        ordered
    }

    fn has_local_node(nodes: &[Arc<dyn NodeClientTrait>]) -> bool {
        nodes.iter().any(Self::is_local_node)
    }

    fn is_local_node(node: &Arc<dyn NodeClientTrait>) -> bool {
        Self::endpoint_is_local(node.endpoint())
    }

    fn endpoint_is_local(endpoint: &str) -> bool {
        let authority = endpoint
            .split_once("://")
            .map(|(_, rest)| rest)
            .unwrap_or(endpoint)
            .split('/')
            .next()
            .unwrap_or(endpoint);
        let host = if authority.starts_with('[') {
            authority
                .split_once(']')
                .map(|(host, _)| host.trim_start_matches('['))
                .unwrap_or(authority.trim_matches(['[', ']']))
        } else {
            authority
                .rsplit_once(':')
                .map(|(host, _)| host)
                .unwrap_or(authority)
        };

        if host.eq_ignore_ascii_case("localhost") {
            return true;
        }

        host.parse::<IpAddr>()
            .map(|ip| ip.is_loopback())
            .unwrap_or(false)
    }
}

#[async_trait]
impl StorageBackend for ReplicatedBackend {
    async fn put_object(&self, object_id: &ObjectId, data: Bytes) -> StorageResult<()> {
        debug!(
            object_id = %object_id,
            size = data.len(),
            replicas = self.replica_count,
            "Putting object with replication"
        );

        if data.len() > self.chunk_manager.chunk_size() {
            self.write_chunked_object(object_id, data).await
        } else {
            let chunk_key = Self::object_key(object_id);
            self.write_replicas(chunk_key, 0, data).await
        }
    }

    async fn get_object(&self, object_id: &ObjectId) -> StorageResult<Bytes> {
        debug!(object_id = %object_id, "Getting object from replicas");

        let chunk_key = Self::object_key(object_id);
        if let Some(data) = self.read_replicas(&chunk_key, 0).await? {
            return Ok(data);
        }

        if let Some(metadata) = self.load_object_metadata(object_id).await? {
            return self.read_chunked_object(object_id, &metadata).await;
        }

        Err(StorageError::NotFound(*object_id))
    }

    async fn delete_object(&self, object_id: &ObjectId) -> StorageResult<()> {
        debug!(object_id = %object_id, "Deleting object from all replicas");

        if let Some(metadata) = self.load_object_metadata(object_id).await? {
            self.delete_chunked_object(object_id, &metadata).await
        } else {
            self.delete_key_best_effort(Self::object_key(object_id))
                .await
        }
    }

    async fn object_exists(&self, object_id: &ObjectId) -> StorageResult<bool> {
        if self.load_object_metadata(object_id).await?.is_some() {
            return Ok(true);
        }

        match self.read_replicas(&Self::object_key(object_id), 0).await? {
            Some(_) => Ok(true),
            None => Ok(false),
        }
    }

    async fn object_size(&self, object_id: &ObjectId) -> StorageResult<u64> {
        if let Some(metadata) = self.load_object_metadata(object_id).await? {
            return Ok(metadata.original_size as u64);
        }

        let nodes = self
            .node_registry
            .get_healthy_nodes()
            .await
            .map_err(|e| StorageError::Backend(e.to_string()))?;

        let chunk_key = Self::object_key(object_id);

        for node in &nodes {
            if let Ok(Some(size)) = node.chunk_size(&chunk_key).await {
                return Ok(size);
            }
        }

        Err(StorageError::NotFound(*object_id))
    }

    async fn put_part(&self, upload_id: &str, part_number: u32, data: Bytes) -> StorageResult<()> {
        debug!(
            upload_id,
            part_number,
            size = data.len(),
            "Putting multipart part with replication"
        );

        let chunk_key = Self::part_key(upload_id, part_number);
        self.write_replicas(chunk_key, part_number, data).await
    }

    async fn get_part(&self, upload_id: &str, part_number: u32) -> StorageResult<Bytes> {
        let chunk_key = Self::part_key(upload_id, part_number);

        if let Some(data) = self.read_replicas(&chunk_key, part_number).await? {
            return Ok(data);
        }

        Err(StorageError::Backend(format!(
            "Part {}:{} not found on any node",
            upload_id, part_number
        )))
    }

    async fn delete_part(&self, upload_id: &str, part_number: u32) -> StorageResult<()> {
        self.delete_key_best_effort(Self::part_key(upload_id, part_number))
            .await
    }

    async fn delete_upload_parts(&self, upload_id: &str) -> StorageResult<()> {
        // Would need to track part numbers in metadata to delete all parts
        // For now, just log and return success
        debug!(
            upload_id,
            "delete_upload_parts called (no-op without metadata tracking)"
        );
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::config::RedundancyMode;
    use crate::node::{MockNodeRegistry, NodeError, NodeResult};
    use crate::repair::RepairQueue;
    use async_trait::async_trait;
    use dashmap::DashMap;
    use std::sync::Arc;
    use std::time::{Duration, Instant};
    use tokio::time::sleep;

    #[derive(Default)]
    struct CapturingRepairQueue {
        tasks: DashMap<String, ReplicatedRepairTask>,
    }

    #[async_trait]
    impl RepairQueue for CapturingRepairQueue {
        async fn enqueue_repair(&self, task: ReplicatedRepairTask) {
            self.tasks.insert(task.id.clone(), task);
        }
    }

    struct SlowNodeClient {
        node_id: String,
        endpoint: String,
        delay: Duration,
        chunks: DashMap<String, Vec<u8>>,
    }

    impl SlowNodeClient {
        fn new(node_id: impl Into<String>, endpoint: impl Into<String>, delay: Duration) -> Self {
            Self {
                node_id: node_id.into(),
                endpoint: endpoint.into(),
                delay,
                chunks: DashMap::new(),
            }
        }

        fn chunk(&self, chunk_id: &str) -> Option<Vec<u8>> {
            self.chunks.get(chunk_id).map(|value| value.value().clone())
        }
    }

    #[async_trait]
    impl NodeClientTrait for SlowNodeClient {
        fn node_id(&self) -> &str {
            &self.node_id
        }

        fn endpoint(&self) -> &str {
            &self.endpoint
        }

        async fn is_healthy(&self) -> bool {
            true
        }

        async fn put_chunk(
            &self,
            chunk_id: &str,
            _shard_index: u32,
            _is_parity: bool,
            data: Bytes,
        ) -> NodeResult<()> {
            sleep(self.delay).await;
            self.chunks.insert(chunk_id.to_string(), data.to_vec());
            Ok(())
        }

        async fn get_chunk(
            &self,
            chunk_id: &str,
            _shard_index: u32,
            _is_parity: bool,
        ) -> NodeResult<Bytes> {
            self.chunks
                .get(chunk_id)
                .map(|value| Bytes::from(value.value().clone()))
                .ok_or_else(|| NodeError::NotFound(chunk_id.to_string()))
        }

        async fn delete_chunk(&self, chunk_id: &str) -> NodeResult<()> {
            sleep(self.delay).await;
            self.chunks.remove(chunk_id);
            Ok(())
        }

        async fn chunk_exists(&self, chunk_id: &str) -> NodeResult<bool> {
            Ok(self.chunks.contains_key(chunk_id))
        }

        async fn chunk_size(&self, chunk_id: &str) -> NodeResult<Option<u64>> {
            Ok(self
                .chunks
                .get(chunk_id)
                .map(|value| value.value().len() as u64))
        }

        async fn ping(&self) -> NodeResult<Duration> {
            Ok(Duration::from_millis(1))
        }
    }

    struct FixedNodeRegistry {
        nodes: Vec<Arc<dyn NodeClientTrait>>,
    }

    #[async_trait]
    impl NodeRegistry for FixedNodeRegistry {
        async fn get_all_nodes(&self) -> NodeResult<Vec<Arc<dyn NodeClientTrait>>> {
            Ok(self.nodes.clone())
        }

        async fn get_healthy_nodes(&self) -> NodeResult<Vec<Arc<dyn NodeClientTrait>>> {
            Ok(self.nodes.clone())
        }

        async fn register_node(&self, _info: crate::node::NodeInfo) -> NodeResult<()> {
            Ok(())
        }

        async fn deregister_node(&self, _node_id: &str) -> NodeResult<()> {
            Ok(())
        }

        async fn update_health(&self, _node_id: &str, _healthy: bool) -> NodeResult<()> {
            Ok(())
        }

        async fn get_node(&self, node_id: &str) -> NodeResult<Option<Arc<dyn NodeClientTrait>>> {
            Ok(self
                .nodes
                .iter()
                .find(|node| node.node_id() == node_id)
                .cloned())
        }

        async fn node_count(&self) -> usize {
            self.nodes.len()
        }

        async fn healthy_node_count(&self) -> usize {
            self.nodes.len()
        }
    }

    fn create_replicated_config(replica_count: usize) -> DistributedConfig {
        DistributedConfig {
            redundancy: RedundancyMode::Replicated {
                replica_count,
                read_quorum: 1,
                write_quorum: (replica_count / 2) + 1,
            },
            ..Default::default()
        }
    }

    #[tokio::test]
    async fn test_replicated_backend_creation() {
        let config = create_replicated_config(3);
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));

        let backend = ReplicatedBackend::new(config, registry).await.unwrap();

        assert_eq!(backend.replica_count(), 3);
        assert_eq!(backend.read_quorum(), 1);
        assert_eq!(backend.write_quorum(), 2);
    }

    #[tokio::test]
    async fn test_chunk_parallelism_scales_with_replica_count_and_chunk_count() {
        let config = create_replicated_config(3);
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));
        let backend = ReplicatedBackend::new(config, registry).await.unwrap();

        assert_eq!(backend.chunk_write_parallelism(1), 1);
        assert_eq!(backend.chunk_write_parallelism(2), 2);
        assert_eq!(backend.chunk_write_parallelism(64), 12);
        assert_eq!(backend.chunk_read_parallelism(1), 1);
        assert_eq!(backend.chunk_read_parallelism(4), 4);
        assert_eq!(backend.chunk_read_parallelism(64), 24);
    }

    #[tokio::test]
    async fn test_chunk_parallelism_respects_global_caps() {
        let config = create_replicated_config(8);
        let registry = Arc::new(MockNodeRegistry::with_nodes(8));
        let backend = ReplicatedBackend::new(config, registry).await.unwrap();

        assert_eq!(
            backend.chunk_write_parallelism(64),
            MAX_CONCURRENT_CHUNK_WRITES
        );
        assert_eq!(
            backend.chunk_read_parallelism(64),
            MAX_CONCURRENT_CHUNK_READS
        );
    }

    #[tokio::test]
    async fn test_replicated_backend_put_get() {
        let config = create_replicated_config(3);
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));

        let backend = ReplicatedBackend::new(config, registry.clone())
            .await
            .unwrap();

        let object_id = ObjectId::new();
        let data = Bytes::from(vec![42u8; 1024]);

        backend.put_object(&object_id, data.clone()).await.unwrap();

        // Verify data was written to nodes
        let nodes = registry.all_mock_nodes();
        let total_chunks: usize = nodes.iter().map(|n| n.chunk_count()).sum();
        assert!(total_chunks >= 2); // At least write_quorum nodes

        // Get the object back
        let retrieved = backend.get_object(&object_id).await.unwrap();
        assert_eq!(retrieved, data);
    }

    #[tokio::test]
    async fn test_replicated_backend_chunks_large_objects() {
        let mut config = create_replicated_config(3);
        config.chunk.chunk_size = 64;
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));

        let backend = ReplicatedBackend::new(config, registry.clone())
            .await
            .unwrap();

        let object_id = ObjectId::new();
        let data = Bytes::from(vec![7u8; 256]);

        backend.put_object(&object_id, data.clone()).await.unwrap();

        let nodes = registry.all_mock_nodes();
        let total_chunks: usize = nodes.iter().map(|n| n.chunk_count()).sum();
        assert!(total_chunks >= 10);

        let chunk_key_prefix = format!("obj_{}_chunk_", object_id);
        let metadata_key = format!("objmeta_{}", object_id);
        let all_chunk_ids: Vec<String> = nodes.iter().flat_map(|n| n.chunk_ids()).collect();
        assert!(all_chunk_ids
            .iter()
            .any(|key| key.starts_with(&chunk_key_prefix)));
        assert!(all_chunk_ids.iter().any(|key| key == &metadata_key));

        let retrieved = backend.get_object(&object_id).await.unwrap();
        assert_eq!(retrieved, data);
        assert_eq!(backend.object_size(&object_id).await.unwrap(), 256);

        backend.delete_object(&object_id).await.unwrap();
        assert!(!backend.object_exists(&object_id).await.unwrap());
    }

    #[tokio::test]
    async fn test_replicated_backend_tolerates_minority_failure() {
        let config = create_replicated_config(3);
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));

        let backend = ReplicatedBackend::new(config, registry.clone())
            .await
            .unwrap();

        let object_id = ObjectId::new();
        let data = Bytes::from(vec![42u8; 512]);

        backend.put_object(&object_id, data.clone()).await.unwrap();

        // Fail 1 node (minority)
        let nodes = registry.all_mock_nodes();
        nodes[0].set_healthy(false);

        // Should still be able to read from healthy nodes
        let retrieved = backend.get_object(&object_id).await.unwrap();
        assert_eq!(retrieved, data);
    }

    #[tokio::test]
    async fn test_replicated_backend_write_quorum_failure() {
        let config = create_replicated_config(3);
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));

        let backend = ReplicatedBackend::new(config, registry.clone())
            .await
            .unwrap();

        // Fail 2 nodes (below write quorum of 2)
        let nodes = registry.all_mock_nodes();
        nodes[0].set_healthy(false);
        nodes[1].set_healthy(false);

        let object_id = ObjectId::new();
        let data = Bytes::from(vec![42u8; 256]);

        // Write should fail due to insufficient healthy nodes
        let result = backend.put_object(&object_id, data).await;
        assert!(result.is_err());
    }

    #[tokio::test]
    async fn test_under_replicated_write_enqueues_repair_task() {
        let config = create_replicated_config(3);
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));
        let nodes = registry.all_mock_nodes();
        nodes[2].set_fail_puts(true);

        let repair_queue = Arc::new(CapturingRepairQueue::default());
        let backend = ReplicatedBackend::new_with_repair_queue(
            config,
            registry,
            Some(repair_queue.clone()),
        )
        .await
        .unwrap();

        let object_id = ObjectId::new();
        backend
            .put_object(&object_id, Bytes::from_static(b"repair-me"))
            .await
            .unwrap();

        let mut task = None;
        for _ in 0..20 {
            task = repair_queue
                .tasks
                .iter()
                .next()
                .map(|entry| entry.value().clone());
            if task.is_some() {
                break;
            }
            sleep(Duration::from_millis(10)).await;
        }
        let task = task.expect("repair task should be queued");
        assert_eq!(task.key, ReplicatedBackend::object_key(&object_id));
        assert_eq!(task.shard_index, 0);
    }

    #[tokio::test]
    async fn test_repair_chunk_restores_missing_replica() {
        let config = create_replicated_config(3);
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));
        let nodes = registry.all_mock_nodes();
        let backend = ReplicatedBackend::new(config, registry.clone())
            .await
            .unwrap();

        let object_id = ObjectId::new();
        let data = Bytes::from(vec![11u8; 128]);
        backend.put_object(&object_id, data.clone()).await.unwrap();

        let key = ReplicatedBackend::object_key(&object_id);
        let mut missing = None;
        for node in &nodes {
            if node.chunk_exists(&key).await.unwrap() {
                missing = Some(node.clone());
                break;
            }
        }
        let missing = missing.expect("at least one replica should exist");
        missing.delete_chunk(&key).await.unwrap();
        assert!(!missing.chunk_exists(&key).await.unwrap());

        let task = ReplicatedRepairTask::new(key.clone(), 0, "test");
        backend.repair_chunk(&task).await.unwrap();
        assert!(missing.chunk_exists(&key).await.unwrap());
    }

    #[tokio::test]
    async fn test_repair_chunk_can_source_from_off_placement_replica() {
        let config = create_replicated_config(2);
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));
        let nodes = registry.all_mock_nodes();
        let backend = ReplicatedBackend::new(config, registry.clone())
            .await
            .unwrap();

        let object_id = ObjectId::new();
        let data = Bytes::from(vec![23u8; 128]);
        backend.put_object(&object_id, data.clone()).await.unwrap();

        let key = ReplicatedBackend::object_key(&object_id);
        let desired_nodes = backend.select_replica_nodes_for_key(&key).await.unwrap();
        assert_eq!(desired_nodes.len(), 2);
        let off_placement = nodes
            .iter()
            .find(|node| {
                desired_nodes
                    .iter()
                    .all(|desired| desired.node_id() != node.node_id())
            })
            .cloned()
            .expect("off-placement node should exist");

        let source_bytes = desired_nodes[0].get_chunk(&key, 0, false).await.unwrap();
        off_placement.put_chunk(&key, 0, false, source_bytes).await.unwrap();
        for node in &desired_nodes {
            node.delete_chunk(&key).await.unwrap();
            assert!(!node.chunk_exists(&key).await.unwrap());
        }

        let task = ReplicatedRepairTask::new(key.clone(), 0, "off-placement-source");
        backend.repair_chunk(&task).await.unwrap();
        for node in &desired_nodes {
            assert!(node.chunk_exists(&key).await.unwrap());
        }
    }

    #[tokio::test]
    async fn test_replicated_backend_returns_after_quorum_without_waiting_for_slow_replica() {
        let config = create_replicated_config(3);
        let fast_a = Arc::new(crate::node::MockNodeClient::new(
            "fast-a",
            "http://fast-a:9002",
        ));
        let fast_b = Arc::new(crate::node::MockNodeClient::new(
            "fast-b",
            "http://fast-b:9002",
        ));
        let slow = Arc::new(SlowNodeClient::new(
            "slow-c",
            "http://slow-c:9002",
            Duration::from_millis(250),
        ));
        let registry = Arc::new(FixedNodeRegistry {
            nodes: vec![fast_a.clone(), fast_b.clone(), slow.clone()],
        });

        let backend = ReplicatedBackend::new(config, registry).await.unwrap();
        let object_id = ObjectId::new();
        let data = Bytes::from(vec![7u8; 128]);

        let started = Instant::now();
        backend.put_object(&object_id, data.clone()).await.unwrap();
        let elapsed = started.elapsed();

        assert!(elapsed < Duration::from_millis(200), "elapsed={elapsed:?}");
        assert_eq!(fast_a.put_count(), 1);
        assert_eq!(fast_b.put_count(), 1);

        sleep(Duration::from_millis(350)).await;
        assert_eq!(
            slow.chunk(&ReplicatedBackend::object_key(&object_id))
                .as_deref(),
            Some(data.as_ref())
        );
    }

    #[tokio::test]
    async fn test_replicated_backend_multipart_replication_continues_after_quorum() {
        let config = create_replicated_config(3);
        let fast_a = Arc::new(crate::node::MockNodeClient::new(
            "fast-a",
            "http://fast-a:9002",
        ));
        let fast_b = Arc::new(crate::node::MockNodeClient::new(
            "fast-b",
            "http://fast-b:9002",
        ));
        let slow = Arc::new(SlowNodeClient::new(
            "slow-c",
            "http://slow-c:9002",
            Duration::from_millis(250),
        ));
        let registry = Arc::new(FixedNodeRegistry {
            nodes: vec![fast_a, fast_b, slow.clone()],
        });

        let backend = ReplicatedBackend::new(config, registry).await.unwrap();
        let upload_id = "upload-123";
        let data = Bytes::from(vec![9u8; 4096]);

        let started = Instant::now();
        backend.put_part(upload_id, 3, data.clone()).await.unwrap();
        let elapsed = started.elapsed();

        assert!(elapsed < Duration::from_millis(200), "elapsed={elapsed:?}");
        sleep(Duration::from_millis(350)).await;
        assert_eq!(
            slow.chunk(&ReplicatedBackend::part_key(upload_id, 3))
                .as_deref(),
            Some(data.as_ref())
        );
    }

    #[tokio::test]
    async fn test_replicated_backend_delete() {
        let config = create_replicated_config(3);
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));

        let backend = ReplicatedBackend::new(config, registry.clone())
            .await
            .unwrap();

        let object_id = ObjectId::new();
        let data = Bytes::from(vec![42u8; 256]);

        backend.put_object(&object_id, data).await.unwrap();
        assert!(backend.object_exists(&object_id).await.unwrap());

        backend.delete_object(&object_id).await.unwrap();
        assert!(!backend.object_exists(&object_id).await.unwrap());
    }

    #[tokio::test]
    async fn test_replicated_backend_delete_returns_after_quorum_without_waiting_for_slow_replica()
    {
        let config = create_replicated_config(3);
        let fast_a = Arc::new(crate::node::MockNodeClient::new(
            "fast-a",
            "http://fast-a:9002",
        ));
        let fast_b = Arc::new(crate::node::MockNodeClient::new(
            "fast-b",
            "http://fast-b:9002",
        ));
        let slow = Arc::new(SlowNodeClient::new(
            "slow-c",
            "http://slow-c:9002",
            Duration::from_millis(250),
        ));
        let registry = Arc::new(FixedNodeRegistry {
            nodes: vec![fast_a.clone(), fast_b.clone(), slow.clone()],
        });

        let backend = ReplicatedBackend::new(config, registry).await.unwrap();
        let object_id = ObjectId::new();
        let data = Bytes::from(vec![3u8; 128]);

        backend.put_object(&object_id, data).await.unwrap();

        let started = Instant::now();
        backend.delete_object(&object_id).await.unwrap();
        let elapsed = started.elapsed();

        assert!(elapsed < Duration::from_millis(200), "elapsed={elapsed:?}");
        assert_eq!(fast_a.delete_count(), 1);
        assert_eq!(fast_b.delete_count(), 1);

        sleep(Duration::from_millis(350)).await;
        assert!(slow
            .chunk(&ReplicatedBackend::object_key(&object_id))
            .is_none());
    }

    #[tokio::test]
    async fn test_replicated_backend_prefers_local_replica_for_chunked_reads() {
        let mut config = create_replicated_config(3);
        config.chunk.chunk_size = 64;
        let local = Arc::new(crate::node::MockNodeClient::new(
            "local",
            "http://127.0.0.1:9002",
        ));
        let slow_a = Arc::new(SlowNodeClient::new(
            "slow-a",
            "http://slow-a:9002",
            Duration::from_millis(250),
        ));
        let slow_b = Arc::new(SlowNodeClient::new(
            "slow-b",
            "http://slow-b:9002",
            Duration::from_millis(250),
        ));
        let registry = Arc::new(FixedNodeRegistry {
            nodes: vec![slow_a.clone(), slow_b.clone(), local.clone()],
        });

        let backend = ReplicatedBackend::new(config, registry).await.unwrap();
        let object_id = ObjectId::new();
        let data = Bytes::from(vec![5u8; 256]);

        backend.put_object(&object_id, data.clone()).await.unwrap();

        let started = Instant::now();
        let retrieved = backend.get_object(&object_id).await.unwrap();
        let elapsed = started.elapsed();

        assert_eq!(retrieved, data);
        assert!(elapsed < Duration::from_millis(150), "elapsed={elapsed:?}");
        assert!(local.get_count() >= 4);
    }

    #[tokio::test]
    async fn test_replicated_backend_object_size() {
        let config = create_replicated_config(3);
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));

        let backend = ReplicatedBackend::new(config, registry).await.unwrap();

        let object_id = ObjectId::new();
        let data = Bytes::from(vec![42u8; 1234]);

        backend.put_object(&object_id, data).await.unwrap();

        let size = backend.object_size(&object_id).await.unwrap();
        assert_eq!(size, 1234);
    }

    #[tokio::test]
    async fn test_replicated_backend_multipart() {
        let config = create_replicated_config(3);
        let registry = Arc::new(MockNodeRegistry::with_nodes(3));

        let backend = ReplicatedBackend::new(config, registry).await.unwrap();

        let upload_id = "test-upload-123";
        let part1 = Bytes::from(vec![1u8; 1024]);
        let part2 = Bytes::from(vec![2u8; 1024]);

        backend.put_part(upload_id, 1, part1.clone()).await.unwrap();
        backend.put_part(upload_id, 2, part2.clone()).await.unwrap();

        let retrieved1 = backend.get_part(upload_id, 1).await.unwrap();
        let retrieved2 = backend.get_part(upload_id, 2).await.unwrap();

        assert_eq!(retrieved1, part1);
        assert_eq!(retrieved2, part2);

        backend.delete_part(upload_id, 1).await.unwrap();
        let result = backend.get_part(upload_id, 1).await;
        assert!(result.is_err());
    }
}
