From 63c72517566d802c34c80a35ba12a5534b9767bf Mon Sep 17 00:00:00 2001 From: centra Date: Mon, 30 Mar 2026 16:46:35 +0900 Subject: [PATCH] Harden FiberLB native BGP control plane --- fiberlb/crates/fiberlb-server/src/config.rs | 30 ++ fiberlb/crates/fiberlb-server/src/lib.rs | 2 + fiberlb/crates/fiberlb-server/src/main.rs | 42 ++- .../crates/fiberlb-server/src/vip_manager.rs | 272 ++++++++++++++---- .../crates/fiberlb-server/src/vip_owner.rs | 161 +++++++++++ nix/modules/fiberlb.nix | 21 ++ nix/modules/plasmacloud-network.nix | 2 + nix/tests/fiberlb-native-bgp-vm-smoke.nix | 84 +++++- 8 files changed, 551 insertions(+), 63 deletions(-) create mode 100644 fiberlb/crates/fiberlb-server/src/vip_owner.rs diff --git a/fiberlb/crates/fiberlb-server/src/config.rs b/fiberlb/crates/fiberlb-server/src/config.rs index cbc40f4..d658256 100644 --- a/fiberlb/crates/fiberlb-server/src/config.rs +++ b/fiberlb/crates/fiberlb-server/src/config.rs @@ -79,6 +79,10 @@ pub struct ServerConfig { #[serde(default)] pub vip_advertisement: VipAdvertisementConfig, + /// Local VIP ownership configuration. + #[serde(default)] + pub vip_ownership: VipOwnershipConfig, + /// Native BGP speaker configuration #[serde(default)] pub bgp: BgpConfig, @@ -145,6 +149,31 @@ impl Default for VipAdvertisementConfig { } } +/// Local VIP ownership runtime configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VipOwnershipConfig { + /// Whether FiberLB should claim VIP /32 addresses on the local node. + #[serde(default)] + pub enabled: bool, + + /// Interface used for local VIP ownership. + #[serde(default = "default_vip_ownership_interface")] + pub interface: String, +} + +fn default_vip_ownership_interface() -> String { + "lo".to_string() +} + +impl Default for VipOwnershipConfig { + fn default() -> Self { + Self { + enabled: false, + interface: default_vip_ownership_interface(), + } + } +} + /// Static BGP peer configuration. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct BgpPeerConfig { @@ -272,6 +301,7 @@ impl Default for ServerConfig { auth: AuthConfig::default(), health: HealthRuntimeConfig::default(), vip_advertisement: VipAdvertisementConfig::default(), + vip_ownership: VipOwnershipConfig::default(), bgp: BgpConfig::default(), } } diff --git a/fiberlb/crates/fiberlb-server/src/lib.rs b/fiberlb/crates/fiberlb-server/src/lib.rs index 20b253f..17dc2f5 100644 --- a/fiberlb/crates/fiberlb-server/src/lib.rs +++ b/fiberlb/crates/fiberlb-server/src/lib.rs @@ -11,6 +11,7 @@ pub mod metadata; pub mod services; pub mod tls; pub mod vip_manager; +pub mod vip_owner; pub use bgp_client::{create_bgp_client, BgpClient, BgpError, NativeBgpSpeaker}; pub use config::ServerConfig; @@ -23,3 +24,4 @@ pub use metadata::LbMetadataStore; pub use services::*; pub use tls::{build_tls_config, CertificateStore, SniCertResolver}; pub use vip_manager::VipManager; +pub use vip_owner::{KernelVipAddressOwner, VipAddressOwner, VipOwnershipError}; diff --git a/fiberlb/crates/fiberlb-server/src/main.rs b/fiberlb/crates/fiberlb-server/src/main.rs index f38442d..48b7de1 100644 --- a/fiberlb/crates/fiberlb-server/src/main.rs +++ b/fiberlb/crates/fiberlb-server/src/main.rs @@ -15,9 +15,9 @@ use fiberlb_api::{ }; use fiberlb_server::{ config::MetadataBackend, create_bgp_client, spawn_health_checker, BackendServiceImpl, - CertificateServiceImpl, DataPlane, HealthCheckServiceImpl, L7DataPlane, L7PolicyServiceImpl, - L7RuleServiceImpl, LbMetadataStore, ListenerServiceImpl, LoadBalancerServiceImpl, - PoolServiceImpl, ServerConfig, VipManager, + CertificateServiceImpl, DataPlane, HealthCheckServiceImpl, KernelVipAddressOwner, L7DataPlane, + L7PolicyServiceImpl, L7RuleServiceImpl, LbMetadataStore, ListenerServiceImpl, + LoadBalancerServiceImpl, PoolServiceImpl, ServerConfig, VipAddressOwner, VipManager, }; use iam_service_auth::AuthService; use metrics_exporter_prometheus::PrometheusBuilder; @@ -248,7 +248,14 @@ async fn main() -> Result<(), Box> { ) })?; let bgp = create_bgp_client(config.bgp.clone()).await?; - let manager = Arc::new(VipManager::new(bgp, metadata.clone(), next_hop)); + let vip_owner: Option> = if config.vip_ownership.enabled { + Some(Arc::new(KernelVipAddressOwner::new( + config.vip_ownership.interface.clone(), + ))) + } else { + None + }; + let manager = Arc::new(VipManager::new(bgp, metadata.clone(), next_hop, vip_owner)); let _vip_task = manager.clone().spawn(Duration::from_secs( config.vip_advertisement.interval_secs.max(1), )); @@ -352,7 +359,11 @@ async fn main() -> Result<(), Box> { CertificateServiceServer::new(certificate_service), make_interceptor(auth_service.clone()), )) - .serve(grpc_addr) + .serve_with_shutdown(grpc_addr, async { + if let Err(error) = wait_for_shutdown_signal().await { + tracing::warn!(error = %error, "FiberLB shutdown signal handler failed"); + } + }) .await; let _ = health_shutdown_tx.send(true); @@ -367,6 +378,27 @@ async fn main() -> Result<(), Box> { Ok(()) } +async fn wait_for_shutdown_signal() -> Result<(), Box> { + #[cfg(unix)] + { + use tokio::signal::unix::{signal, SignalKind}; + + let mut terminate = signal(SignalKind::terminate())?; + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = terminate.recv() => {} + } + } + + #[cfg(not(unix))] + { + tokio::signal::ctrl_c().await?; + } + + tracing::info!("FiberLB shutdown signal received"); + Ok(()) +} + fn parse_metadata_backend(value: &str) -> Result> { match value.trim().to_ascii_lowercase().as_str() { "flaredb" => Ok(MetadataBackend::FlareDb), diff --git a/fiberlb/crates/fiberlb-server/src/vip_manager.rs b/fiberlb/crates/fiberlb-server/src/vip_manager.rs index a43a02f..e95c24a 100644 --- a/fiberlb/crates/fiberlb-server/src/vip_manager.rs +++ b/fiberlb/crates/fiberlb-server/src/vip_manager.rs @@ -14,15 +14,22 @@ use tracing::{debug, error, info, warn}; use crate::bgp_client::BgpClient; use crate::metadata::LbMetadataStore; +use crate::vip_owner::VipAddressOwner; use fiberlb_types::LoadBalancerId; -/// VIP advertisement state -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum VipState { - /// VIP is advertised to BGP peers - Advertised, - /// VIP is withdrawn from BGP peers - Withdrawn, +/// Current local control-plane state for a VIP. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct VipState { + /// The VIP is configured on the local node. + owned: bool, + /// The VIP is advertised to BGP peers. + advertised: bool, +} + +impl VipState { + fn is_idle(self) -> bool { + !self.owned && !self.advertised + } } /// VIP Manager @@ -37,17 +44,25 @@ pub struct VipManager { metadata: Arc, /// Current VIP advertisement state vip_state: Arc>>, + /// Optional local VIP owner. + vip_owner: Option>, /// Router's own IP address (used as BGP next hop) next_hop: IpAddr, } impl VipManager { /// Create a new VIP manager - pub fn new(bgp: Arc, metadata: Arc, next_hop: IpAddr) -> Self { + pub fn new( + bgp: Arc, + metadata: Arc, + next_hop: IpAddr, + vip_owner: Option>, + ) -> Self { Self { bgp, metadata, vip_state: Arc::new(RwLock::new(HashMap::new())), + vip_owner, next_hop, } } @@ -148,39 +163,72 @@ impl VipManager { ) -> Result<(), Box> { let mut state = self.vip_state.write().await; - // Find VIPs to announce (active but not yet advertised) for vip in active_vips { - match state.get(vip) { - Some(VipState::Advertised) => { - // Already advertised, nothing to do - debug!("VIP {} already advertised", vip); - } - Some(VipState::Withdrawn) | None => { - // Need to announce - info!("Announcing VIP {} (healthy backends available)", vip); - if let Err(e) = self.bgp.announce_route(*vip, self.next_hop).await { - error!("Failed to announce VIP {}: {}", vip, e); - } else { - state.insert(*vip, VipState::Advertised); + let mut vip_state = state.get(vip).copied().unwrap_or_default(); + let mut changed = false; + + if !vip_state.owned { + if let Some(vip_owner) = &self.vip_owner { + info!("Claiming local VIP {} on this node", vip); + if let Err(error) = vip_owner.ensure_present(*vip).await { + error!("Failed to claim local VIP {}: {}", vip, error); + continue; } } + vip_state.owned = true; + changed = true; + } else { + debug!("VIP {} already locally owned", vip); + } + + if !vip_state.advertised { + info!("Announcing VIP {} (healthy backends available)", vip); + if let Err(error) = self.bgp.announce_route(*vip, self.next_hop).await { + error!("Failed to announce VIP {}: {}", vip, error); + } else { + vip_state.advertised = true; + changed = true; + } + } else { + debug!("VIP {} already advertised", vip); + } + + if changed { + state.insert(*vip, vip_state); } } - // Find VIPs to withdraw (advertised but no longer active) - let advertised_vips: Vec = state - .iter() - .filter(|(_, &state)| state == VipState::Advertised) - .map(|(vip, _)| *vip) - .collect(); - - for vip in advertised_vips { + let managed_vips: Vec = state.keys().copied().collect(); + for vip in managed_vips { if !active_vips.contains(&vip) { - info!("Withdrawing VIP {} (no healthy backends)", vip); - if let Err(e) = self.bgp.withdraw_route(vip).await { - error!("Failed to withdraw VIP {}: {}", vip, e); + let mut vip_state = state.get(&vip).copied().unwrap_or_default(); + + if vip_state.owned { + if let Some(vip_owner) = &self.vip_owner { + info!("Releasing local VIP {} (no healthy backends)", vip); + if let Err(error) = vip_owner.ensure_absent(vip).await { + error!("Failed to release local VIP {}: {}", vip, error); + } else { + vip_state.owned = false; + } + } else { + vip_state.owned = false; + } + } + + if vip_state.advertised { + info!("Withdrawing VIP {} (no healthy backends)", vip); + if let Err(error) = self.bgp.withdraw_route(vip).await { + error!("Failed to withdraw VIP {}: {}", vip, error); + } else { + vip_state.advertised = false; + } + } + + if vip_state.is_idle() { + state.remove(&vip); } else { - state.insert(vip, VipState::Withdrawn); + state.insert(vip, vip_state); } } } @@ -190,22 +238,52 @@ impl VipManager { /// Manually advertise a VIP (for testing or manual control) pub async fn advertise_vip(&self, vip: IpAddr) -> Result<(), Box> { - info!("Manually advertising VIP {}", vip); - self.bgp.announce_route(vip, self.next_hop).await?; - let mut state = self.vip_state.write().await; - state.insert(vip, VipState::Advertised); + let mut vip_state = state.get(&vip).copied().unwrap_or_default(); + + if !vip_state.owned { + if let Some(vip_owner) = &self.vip_owner { + info!("Claiming local VIP {} on this node", vip); + vip_owner.ensure_present(vip).await?; + } + vip_state.owned = true; + } + + if !vip_state.advertised { + info!("Manually advertising VIP {}", vip); + self.bgp.announce_route(vip, self.next_hop).await?; + vip_state.advertised = true; + } + + state.insert(vip, vip_state); Ok(()) } /// Manually withdraw a VIP (for testing or manual control) pub async fn withdraw_vip(&self, vip: IpAddr) -> Result<(), Box> { - info!("Manually withdrawing VIP {}", vip); - self.bgp.withdraw_route(vip).await?; - let mut state = self.vip_state.write().await; - state.insert(vip, VipState::Withdrawn); + let mut vip_state = state.get(&vip).copied().unwrap_or_default(); + + if vip_state.owned { + if let Some(vip_owner) = &self.vip_owner { + info!("Releasing local VIP {} on this node", vip); + vip_owner.ensure_absent(vip).await?; + } + vip_state.owned = false; + } + + if vip_state.advertised { + info!("Manually withdrawing VIP {}", vip); + self.bgp.withdraw_route(vip).await?; + vip_state.advertised = false; + } + + if vip_state.is_idle() { + state.remove(&vip); + } else { + state.insert(vip, vip_state); + } Ok(()) } @@ -216,19 +294,41 @@ impl VipManager { pub async fn shutdown(&self) -> Result<(), Box> { info!("VIP manager shutting down, withdrawing all VIPs..."); - let state = self.vip_state.read().await; - let advertised_vips: Vec = state - .iter() - .filter(|(_, &state)| state == VipState::Advertised) - .map(|(vip, _)| *vip) - .collect(); + let mut state = self.vip_state.write().await; + let managed_vips: Vec = state.keys().copied().collect(); - drop(state); // Release read lock before withdrawing + for vip in managed_vips { + let mut vip_state = state.get(&vip).copied().unwrap_or_default(); - for vip in advertised_vips { - info!("Withdrawing VIP {} for shutdown", vip); - if let Err(e) = self.bgp.withdraw_route(vip).await { - error!("Failed to withdraw VIP {} during shutdown: {}", vip, e); + if vip_state.owned { + info!("Releasing local VIP {} for shutdown", vip); + if let Some(vip_owner) = &self.vip_owner { + if let Err(error) = vip_owner.ensure_absent(vip).await { + error!( + "Failed to release local VIP {} during shutdown: {}", + vip, error + ); + } else { + vip_state.owned = false; + } + } else { + vip_state.owned = false; + } + } + + if vip_state.advertised { + info!("Withdrawing VIP {} for shutdown", vip); + if let Err(error) = self.bgp.withdraw_route(vip).await { + error!("Failed to withdraw VIP {} during shutdown: {}", vip, error); + } else { + vip_state.advertised = false; + } + } + + if vip_state.is_idle() { + state.remove(&vip); + } else { + state.insert(vip, vip_state); } } @@ -241,7 +341,7 @@ impl VipManager { let state = self.vip_state.read().await; state .iter() - .filter(|(_, &state)| state == VipState::Advertised) + .filter(|(_, state)| state.advertised) .map(|(vip, _)| *vip) .collect() } @@ -251,19 +351,22 @@ impl VipManager { mod tests { use super::*; use crate::bgp_client::{BgpClient, Result}; + use crate::vip_owner::VipOwnershipError; use std::sync::Mutex; /// Mock BGP client for testing struct MockBgpClient { announced: Arc>>, withdrawn: Arc>>, + events: Arc>>, } impl MockBgpClient { - fn new() -> Self { + fn new(events: Arc>>) -> Self { Self { announced: Arc::new(Mutex::new(HashSet::new())), withdrawn: Arc::new(Mutex::new(HashSet::new())), + events, } } } @@ -271,11 +374,19 @@ mod tests { #[tonic::async_trait] impl BgpClient for MockBgpClient { async fn announce_route(&self, prefix: IpAddr, _next_hop: IpAddr) -> Result<()> { + self.events + .lock() + .unwrap() + .push(format!("announce:{prefix}")); self.announced.lock().unwrap().insert(prefix); Ok(()) } async fn withdraw_route(&self, prefix: IpAddr) -> Result<()> { + self.events + .lock() + .unwrap() + .push(format!("withdraw:{prefix}")); self.withdrawn.lock().unwrap().insert(prefix); Ok(()) } @@ -285,13 +396,51 @@ mod tests { } } + struct MockVipOwner { + owned: Arc>>, + released: Arc>>, + events: Arc>>, + } + + impl MockVipOwner { + fn new(events: Arc>>) -> Self { + Self { + owned: Arc::new(Mutex::new(HashSet::new())), + released: Arc::new(Mutex::new(HashSet::new())), + events, + } + } + } + + #[tonic::async_trait] + impl VipAddressOwner for MockVipOwner { + async fn ensure_present(&self, vip: IpAddr) -> std::result::Result<(), VipOwnershipError> { + self.events.lock().unwrap().push(format!("own:{vip}")); + self.owned.lock().unwrap().insert(vip); + Ok(()) + } + + async fn ensure_absent(&self, vip: IpAddr) -> std::result::Result<(), VipOwnershipError> { + self.events.lock().unwrap().push(format!("unown:{vip}")); + self.released.lock().unwrap().insert(vip); + Ok(()) + } + } + #[tokio::test] async fn test_vip_advertisement_tracking() { - let mock_bgp = Arc::new(MockBgpClient::new()); + let events = Arc::new(Mutex::new(Vec::new())); + let mock_bgp = Arc::new(MockBgpClient::new(events.clone())); + let mock_owner = Arc::new(MockVipOwner::new(events.clone())); let metadata = Arc::new(LbMetadataStore::new_in_memory()); let next_hop = "10.0.0.1".parse().unwrap(); - let manager = VipManager::new(mock_bgp.clone(), metadata, next_hop); + let manager = VipManager::new( + mock_bgp.clone(), + metadata, + next_hop, + Some(mock_owner.clone()), + ); let vip: IpAddr = "10.0.1.100".parse().unwrap(); @@ -304,5 +453,16 @@ mod tests { manager.withdraw_vip(vip).await.unwrap(); assert!(manager.get_advertised_vips().await.is_empty()); assert!(mock_bgp.withdrawn.lock().unwrap().contains(&vip)); + assert!(mock_owner.released.lock().unwrap().contains(&vip)); + + assert_eq!( + events.lock().unwrap().clone(), + vec![ + format!("own:{vip}"), + format!("announce:{vip}"), + format!("unown:{vip}"), + format!("withdraw:{vip}"), + ] + ); } } diff --git a/fiberlb/crates/fiberlb-server/src/vip_owner.rs b/fiberlb/crates/fiberlb-server/src/vip_owner.rs new file mode 100644 index 0000000..c5b7a14 --- /dev/null +++ b/fiberlb/crates/fiberlb-server/src/vip_owner.rs @@ -0,0 +1,161 @@ +//! Local VIP ownership management for FiberLB. +//! +//! This module claims and releases VIP `/32` addresses on a local interface so +//! the kernel actually treats the advertised VIP as a local destination. + +use std::net::{IpAddr, Ipv4Addr}; +use std::path::Path; + +use tokio::process::Command; + +/// Result type for VIP ownership operations. +pub type Result = std::result::Result; + +/// VIP ownership errors. +#[derive(Debug, thiserror::Error)] +pub enum VipOwnershipError { + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + #[error("kernel VIP ownership command failed: {0}")] + CommandFailed(String), + #[error("unsupported VIP ownership operation: {0}")] + Unsupported(String), +} + +/// Trait for local VIP ownership. +#[tonic::async_trait] +pub trait VipAddressOwner: Send + Sync { + /// Ensure a VIP is locally owned on this node. + async fn ensure_present(&self, vip: IpAddr) -> Result<()>; + + /// Ensure a VIP is no longer locally owned on this node. + async fn ensure_absent(&self, vip: IpAddr) -> Result<()>; +} + +/// Linux kernel-backed VIP owner using `ip addr add/del`. +pub struct KernelVipAddressOwner { + interface: String, + ip_command: String, +} + +impl KernelVipAddressOwner { + /// Create a kernel-backed VIP owner for the given interface. + pub fn new(interface: impl Into) -> Self { + Self { + interface: interface.into(), + ip_command: resolve_ip_command(), + } + } + + async fn run_ip(&self, args: &[String]) -> Result<()> { + let output = Command::new(&self.ip_command).args(args).output().await?; + if output.status.success() { + return Ok(()); + } + + Err(VipOwnershipError::CommandFailed(format!( + "{} {}: {}", + self.ip_command, + args.join(" "), + render_command_output(&output) + ))) + } + + async fn add_v4(&self, vip: Ipv4Addr) -> Result<()> { + let args = vec![ + "-4".to_string(), + "addr".to_string(), + "add".to_string(), + format!("{vip}/32"), + "dev".to_string(), + self.interface.clone(), + ]; + + match self.run_ip(&args).await { + Ok(()) => Ok(()), + Err(VipOwnershipError::CommandFailed(message)) if message.contains("File exists") => { + Ok(()) + } + Err(error) => Err(error), + } + } + + async fn del_v4(&self, vip: Ipv4Addr) -> Result<()> { + let args = vec![ + "-4".to_string(), + "addr".to_string(), + "del".to_string(), + format!("{vip}/32"), + "dev".to_string(), + self.interface.clone(), + ]; + + match self.run_ip(&args).await { + Ok(()) => Ok(()), + Err(VipOwnershipError::CommandFailed(message)) + if message.contains("Cannot assign requested address") => + { + Ok(()) + } + Err(error) => Err(error), + } + } +} + +#[tonic::async_trait] +impl VipAddressOwner for KernelVipAddressOwner { + async fn ensure_present(&self, vip: IpAddr) -> Result<()> { + match vip { + IpAddr::V4(vip) => self.add_v4(vip).await, + IpAddr::V6(_) => Err(VipOwnershipError::Unsupported(format!( + "VIP ownership currently supports IPv4 only (got {vip})" + ))), + } + } + + async fn ensure_absent(&self, vip: IpAddr) -> Result<()> { + match vip { + IpAddr::V4(vip) => self.del_v4(vip).await, + IpAddr::V6(_) => Err(VipOwnershipError::Unsupported(format!( + "VIP ownership currently supports IPv4 only (got {vip})" + ))), + } + } +} + +fn resolve_ip_command() -> String { + if let Ok(path) = std::env::var("FIBERLB_IP_COMMAND") { + let trimmed = path.trim(); + if !trimmed.is_empty() { + return trimmed.to_string(); + } + } + + for candidate in [ + "/run/current-system/sw/bin/ip", + "/usr/sbin/ip", + "/sbin/ip", + "/usr/bin/ip", + "/bin/ip", + ] { + if Path::new(candidate).exists() { + return candidate.to_string(); + } + } + + "ip".to_string() +} + +fn render_command_output(output: &std::process::Output) -> String { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if !stderr.is_empty() { + return stderr; + } + + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if !stdout.is_empty() { + return stdout; + } + + format!("exit status {}", output.status) +} diff --git a/nix/modules/fiberlb.nix b/nix/modules/fiberlb.nix index ff1d769..3fdac82 100644 --- a/nix/modules/fiberlb.nix +++ b/nix/modules/fiberlb.nix @@ -47,6 +47,10 @@ let vip_advertisement = { interval_secs = cfg.vipCheckIntervalSecs; }; + vip_ownership = { + enabled = cfg.vipOwnership.enable; + interface = cfg.vipOwnership.interface; + }; } // lib.optionalAttrs cfg.bgp.enable { bgp = { @@ -155,6 +159,16 @@ in description = "Interval between FiberLB VIP-to-BGP reconciliation sweeps."; }; + vipOwnership = { + enable = lib.mkEnableOption "FiberLB local VIP ownership"; + + interface = lib.mkOption { + type = lib.types.str; + default = "lo"; + description = "Interface where FiberLB should claim VIP /32 addresses."; + }; + }; + bgp = { enable = lib.mkEnableOption "FiberLB native BGP VIP advertisement"; @@ -240,6 +254,10 @@ in assertion = (!cfg.bgp.enable) || ((builtins.length cfg.bgp.peers) > 0); message = "services.fiberlb.bgp.peers must contain at least one peer when native BGP is enabled"; } + { + assertion = (!cfg.vipOwnership.enable) || cfg.bgp.enable; + message = "services.fiberlb.vipOwnership.enable requires services.fiberlb.bgp.enable"; + } ]; # Create system user @@ -258,6 +276,7 @@ in wantedBy = [ "multi-user.target" ]; after = [ "network.target" "iam.service" ] ++ flaredbDependencies; requires = [ "iam.service" ] ++ flaredbDependencies; + path = lib.optionals cfg.vipOwnership.enable [ pkgs.iproute2 ]; serviceConfig = { Type = "simple"; @@ -276,6 +295,8 @@ in ProtectSystem = "strict"; ProtectHome = true; ReadWritePaths = [ cfg.dataDir ]; + AmbientCapabilities = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ]; + CapabilityBoundingSet = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ]; # Environment variables for service endpoints Environment = [ diff --git a/nix/modules/plasmacloud-network.nix b/nix/modules/plasmacloud-network.nix index 0b767ba..49fb666 100644 --- a/nix/modules/plasmacloud-network.nix +++ b/nix/modules/plasmacloud-network.nix @@ -99,6 +99,8 @@ in { if node != null then node.ip else "127.0.0.1"; peers = cfg.fiberlbBgp.peers; }; + + services.fiberlb.vipOwnership.enable = mkDefault true; }) # PrismNET OVN integration diff --git a/nix/tests/fiberlb-native-bgp-vm-smoke.nix b/nix/tests/fiberlb-native-bgp-vm-smoke.nix index d238fe2..15d9035 100644 --- a/nix/tests/fiberlb-native-bgp-vm-smoke.nix +++ b/nix/tests/fiberlb-native-bgp-vm-smoke.nix @@ -29,6 +29,25 @@ let iamProto = "iam.proto"; fiberlbProtoDir = ../../fiberlb/crates/fiberlb-api/proto; fiberlbProto = "fiberlb.proto"; + backendScript = pkgs.writeText "fiberlb-smoke-backend.py" '' + from http.server import BaseHTTPRequestHandler, HTTPServer + + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"fiberlb smoke backend\n" + self.send_response(200) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + return + + + HTTPServer(("127.0.0.1", 18081), Handler).serve_forever() + ''; in { name = "fiberlb-native-bgp-vm-smoke"; @@ -49,6 +68,7 @@ in ]; environment.systemPackages = with pkgs; [ + curl gobgp gobgpd jq @@ -117,6 +137,10 @@ in healthCheckIntervalSecs = 1; healthCheckTimeoutSecs = 1; vipCheckIntervalSecs = 1; + vipOwnership = { + enable = true; + interface = "lo"; + }; bgp = { enable = true; localAs = 65010; @@ -141,7 +165,7 @@ in after = [ "network.target" ]; serviceConfig = { Type = "simple"; - ExecStart = "${pkgs.python3}/bin/python -m http.server 18081 --bind 127.0.0.1"; + ExecStart = "${pkgs.python3}/bin/python ${backendScript}"; Restart = "always"; RestartSec = "1s"; }; @@ -276,6 +300,26 @@ in time.sleep(1) raise AssertionError(f"route {prefix} still present in GoBGP RIB") + def wait_for_local_vip(vip, present): + pattern = f"inet {vip}/32" + if present: + lb.wait_until_succeeds( + f"ip -4 addr show dev lo | grep -F {shlex.quote(pattern)}" + ) + else: + deadline = time.time() + 60 + while time.time() < deadline: + output = lb.succeed("ip -4 addr show dev lo || true") + if pattern not in output: + return + time.sleep(1) + raise AssertionError(f"VIP {vip} still present on loopback") + + def wait_for_http_success(url): + router.wait_until_succeeds( + f"curl -fsS --max-time 5 {shlex.quote(url)} | grep -F 'fiberlb smoke backend'" + ) + start_all() serial_stdout_off() @@ -307,7 +351,9 @@ in ) loadbalancer = lb_response["loadbalancer"] lb_id = loadbalancer["id"] - vip_prefix = f"{loadbalancer['vipAddress']}/32" + vip = loadbalancer["vipAddress"] + vip_prefix = f"{vip}/32" + listener_url = f"http://{vip}:18080/" pool_id = grpcurl_json( lb, @@ -363,16 +409,50 @@ in headers=[f"authorization: Bearer {token}"], ) + grpcurl_json( + lb, + "127.0.0.1:50085", + FIBERLB_PROTO_DIR, + FIBERLB_PROTO, + "fiberlb.v1.ListenerService/CreateListener", + { + "name": "bgp-smoke-listener", + "loadbalancerId": lb_id, + "protocol": "LISTENER_PROTOCOL_TCP", + "port": 18080, + "defaultPoolId": pool_id, + }, + headers=[f"authorization: Bearer {token}"], + ) + wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token) + wait_for_local_vip(vip, True) wait_for_route(vip_prefix, True) + router.succeed(f"ip route replace {shlex.quote(vip_prefix)} via 192.168.100.2 dev eth1") + wait_for_http_success(listener_url) + + router.succeed("systemctl restart gobgpd-peer.service") + router.wait_for_unit("gobgpd-peer.service") + router.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2") + wait_for_route(vip_prefix, True) + wait_for_http_success(listener_url) lb.succeed("systemctl stop mock-backend.service") wait_for_backend_status("BACKEND_STATUS_OFFLINE", backend_id, token) wait_for_route(vip_prefix, False) + wait_for_local_vip(vip, False) + router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}") lb.succeed("systemctl start mock-backend.service") lb.wait_for_unit("mock-backend.service") wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token) + wait_for_local_vip(vip, True) wait_for_route(vip_prefix, True) + wait_for_http_success(listener_url) + + lb.succeed("systemctl stop fiberlb.service") + wait_for_local_vip(vip, False) + wait_for_route(vip_prefix, False) + router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}") ''; }