Harden FiberLB native BGP control plane
Some checks failed
Nix CI / filter (push) Failing after 1s
Nix CI / gate () (push) Has been skipped
Nix CI / gate (shared crates) (push) Has been skipped
Nix CI / build () (push) Has been skipped
Nix CI / ci-status (push) Failing after 1s

This commit is contained in:
centra 2026-03-30 16:46:35 +09:00
parent ce4bab07d6
commit 63c7251756
Signed by: centra
GPG key ID: 0C09689D20B25ACA
8 changed files with 551 additions and 63 deletions

View file

@ -79,6 +79,10 @@ pub struct ServerConfig {
#[serde(default)] #[serde(default)]
pub vip_advertisement: VipAdvertisementConfig, pub vip_advertisement: VipAdvertisementConfig,
/// Local VIP ownership configuration.
#[serde(default)]
pub vip_ownership: VipOwnershipConfig,
/// Native BGP speaker configuration /// Native BGP speaker configuration
#[serde(default)] #[serde(default)]
pub bgp: BgpConfig, pub bgp: BgpConfig,
@ -145,6 +149,31 @@ impl Default for VipAdvertisementConfig {
} }
} }
/// Local VIP ownership runtime configuration.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VipOwnershipConfig {
/// Whether FiberLB should claim VIP /32 addresses on the local node.
#[serde(default)]
pub enabled: bool,
/// Interface used for local VIP ownership.
#[serde(default = "default_vip_ownership_interface")]
pub interface: String,
}
fn default_vip_ownership_interface() -> String {
"lo".to_string()
}
impl Default for VipOwnershipConfig {
fn default() -> Self {
Self {
enabled: false,
interface: default_vip_ownership_interface(),
}
}
}
/// Static BGP peer configuration. /// Static BGP peer configuration.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct BgpPeerConfig { pub struct BgpPeerConfig {
@ -272,6 +301,7 @@ impl Default for ServerConfig {
auth: AuthConfig::default(), auth: AuthConfig::default(),
health: HealthRuntimeConfig::default(), health: HealthRuntimeConfig::default(),
vip_advertisement: VipAdvertisementConfig::default(), vip_advertisement: VipAdvertisementConfig::default(),
vip_ownership: VipOwnershipConfig::default(),
bgp: BgpConfig::default(), bgp: BgpConfig::default(),
} }
} }

View file

@ -11,6 +11,7 @@ pub mod metadata;
pub mod services; pub mod services;
pub mod tls; pub mod tls;
pub mod vip_manager; pub mod vip_manager;
pub mod vip_owner;
pub use bgp_client::{create_bgp_client, BgpClient, BgpError, NativeBgpSpeaker}; pub use bgp_client::{create_bgp_client, BgpClient, BgpError, NativeBgpSpeaker};
pub use config::ServerConfig; pub use config::ServerConfig;
@ -23,3 +24,4 @@ pub use metadata::LbMetadataStore;
pub use services::*; pub use services::*;
pub use tls::{build_tls_config, CertificateStore, SniCertResolver}; pub use tls::{build_tls_config, CertificateStore, SniCertResolver};
pub use vip_manager::VipManager; pub use vip_manager::VipManager;
pub use vip_owner::{KernelVipAddressOwner, VipAddressOwner, VipOwnershipError};

View file

@ -15,9 +15,9 @@ use fiberlb_api::{
}; };
use fiberlb_server::{ use fiberlb_server::{
config::MetadataBackend, create_bgp_client, spawn_health_checker, BackendServiceImpl, config::MetadataBackend, create_bgp_client, spawn_health_checker, BackendServiceImpl,
CertificateServiceImpl, DataPlane, HealthCheckServiceImpl, L7DataPlane, L7PolicyServiceImpl, CertificateServiceImpl, DataPlane, HealthCheckServiceImpl, KernelVipAddressOwner, L7DataPlane,
L7RuleServiceImpl, LbMetadataStore, ListenerServiceImpl, LoadBalancerServiceImpl, L7PolicyServiceImpl, L7RuleServiceImpl, LbMetadataStore, ListenerServiceImpl,
PoolServiceImpl, ServerConfig, VipManager, LoadBalancerServiceImpl, PoolServiceImpl, ServerConfig, VipAddressOwner, VipManager,
}; };
use iam_service_auth::AuthService; use iam_service_auth::AuthService;
use metrics_exporter_prometheus::PrometheusBuilder; use metrics_exporter_prometheus::PrometheusBuilder;
@ -248,7 +248,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
) )
})?; })?;
let bgp = create_bgp_client(config.bgp.clone()).await?; let bgp = create_bgp_client(config.bgp.clone()).await?;
let manager = Arc::new(VipManager::new(bgp, metadata.clone(), next_hop)); let vip_owner: Option<Arc<dyn VipAddressOwner>> = if config.vip_ownership.enabled {
Some(Arc::new(KernelVipAddressOwner::new(
config.vip_ownership.interface.clone(),
)))
} else {
None
};
let manager = Arc::new(VipManager::new(bgp, metadata.clone(), next_hop, vip_owner));
let _vip_task = manager.clone().spawn(Duration::from_secs( let _vip_task = manager.clone().spawn(Duration::from_secs(
config.vip_advertisement.interval_secs.max(1), config.vip_advertisement.interval_secs.max(1),
)); ));
@ -352,7 +359,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
CertificateServiceServer::new(certificate_service), CertificateServiceServer::new(certificate_service),
make_interceptor(auth_service.clone()), make_interceptor(auth_service.clone()),
)) ))
.serve(grpc_addr) .serve_with_shutdown(grpc_addr, async {
if let Err(error) = wait_for_shutdown_signal().await {
tracing::warn!(error = %error, "FiberLB shutdown signal handler failed");
}
})
.await; .await;
let _ = health_shutdown_tx.send(true); let _ = health_shutdown_tx.send(true);
@ -367,6 +378,27 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(()) Ok(())
} }
async fn wait_for_shutdown_signal() -> Result<(), Box<dyn std::error::Error>> {
#[cfg(unix)]
{
use tokio::signal::unix::{signal, SignalKind};
let mut terminate = signal(SignalKind::terminate())?;
tokio::select! {
_ = tokio::signal::ctrl_c() => {}
_ = terminate.recv() => {}
}
}
#[cfg(not(unix))]
{
tokio::signal::ctrl_c().await?;
}
tracing::info!("FiberLB shutdown signal received");
Ok(())
}
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> { fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
match value.trim().to_ascii_lowercase().as_str() { match value.trim().to_ascii_lowercase().as_str() {
"flaredb" => Ok(MetadataBackend::FlareDb), "flaredb" => Ok(MetadataBackend::FlareDb),

View file

@ -14,15 +14,22 @@ use tracing::{debug, error, info, warn};
use crate::bgp_client::BgpClient; use crate::bgp_client::BgpClient;
use crate::metadata::LbMetadataStore; use crate::metadata::LbMetadataStore;
use crate::vip_owner::VipAddressOwner;
use fiberlb_types::LoadBalancerId; use fiberlb_types::LoadBalancerId;
/// VIP advertisement state /// Current local control-plane state for a VIP.
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
enum VipState { struct VipState {
/// VIP is advertised to BGP peers /// The VIP is configured on the local node.
Advertised, owned: bool,
/// VIP is withdrawn from BGP peers /// The VIP is advertised to BGP peers.
Withdrawn, advertised: bool,
}
impl VipState {
fn is_idle(self) -> bool {
!self.owned && !self.advertised
}
} }
/// VIP Manager /// VIP Manager
@ -37,17 +44,25 @@ pub struct VipManager {
metadata: Arc<LbMetadataStore>, metadata: Arc<LbMetadataStore>,
/// Current VIP advertisement state /// Current VIP advertisement state
vip_state: Arc<RwLock<HashMap<IpAddr, VipState>>>, vip_state: Arc<RwLock<HashMap<IpAddr, VipState>>>,
/// Optional local VIP owner.
vip_owner: Option<Arc<dyn VipAddressOwner>>,
/// Router's own IP address (used as BGP next hop) /// Router's own IP address (used as BGP next hop)
next_hop: IpAddr, next_hop: IpAddr,
} }
impl VipManager { impl VipManager {
/// Create a new VIP manager /// Create a new VIP manager
pub fn new(bgp: Arc<dyn BgpClient>, metadata: Arc<LbMetadataStore>, next_hop: IpAddr) -> Self { pub fn new(
bgp: Arc<dyn BgpClient>,
metadata: Arc<LbMetadataStore>,
next_hop: IpAddr,
vip_owner: Option<Arc<dyn VipAddressOwner>>,
) -> Self {
Self { Self {
bgp, bgp,
metadata, metadata,
vip_state: Arc::new(RwLock::new(HashMap::new())), vip_state: Arc::new(RwLock::new(HashMap::new())),
vip_owner,
next_hop, next_hop,
} }
} }
@ -148,39 +163,72 @@ impl VipManager {
) -> Result<(), Box<dyn std::error::Error>> { ) -> Result<(), Box<dyn std::error::Error>> {
let mut state = self.vip_state.write().await; let mut state = self.vip_state.write().await;
// Find VIPs to announce (active but not yet advertised)
for vip in active_vips { for vip in active_vips {
match state.get(vip) { let mut vip_state = state.get(vip).copied().unwrap_or_default();
Some(VipState::Advertised) => { let mut changed = false;
// Already advertised, nothing to do
debug!("VIP {} already advertised", vip); if !vip_state.owned {
} if let Some(vip_owner) = &self.vip_owner {
Some(VipState::Withdrawn) | None => { info!("Claiming local VIP {} on this node", vip);
// Need to announce if let Err(error) = vip_owner.ensure_present(*vip).await {
info!("Announcing VIP {} (healthy backends available)", vip); error!("Failed to claim local VIP {}: {}", vip, error);
if let Err(e) = self.bgp.announce_route(*vip, self.next_hop).await { continue;
error!("Failed to announce VIP {}: {}", vip, e);
} else {
state.insert(*vip, VipState::Advertised);
} }
} }
vip_state.owned = true;
changed = true;
} else {
debug!("VIP {} already locally owned", vip);
}
if !vip_state.advertised {
info!("Announcing VIP {} (healthy backends available)", vip);
if let Err(error) = self.bgp.announce_route(*vip, self.next_hop).await {
error!("Failed to announce VIP {}: {}", vip, error);
} else {
vip_state.advertised = true;
changed = true;
}
} else {
debug!("VIP {} already advertised", vip);
}
if changed {
state.insert(*vip, vip_state);
} }
} }
// Find VIPs to withdraw (advertised but no longer active) let managed_vips: Vec<IpAddr> = state.keys().copied().collect();
let advertised_vips: Vec<IpAddr> = state for vip in managed_vips {
.iter()
.filter(|(_, &state)| state == VipState::Advertised)
.map(|(vip, _)| *vip)
.collect();
for vip in advertised_vips {
if !active_vips.contains(&vip) { if !active_vips.contains(&vip) {
info!("Withdrawing VIP {} (no healthy backends)", vip); let mut vip_state = state.get(&vip).copied().unwrap_or_default();
if let Err(e) = self.bgp.withdraw_route(vip).await {
error!("Failed to withdraw VIP {}: {}", vip, e); if vip_state.owned {
if let Some(vip_owner) = &self.vip_owner {
info!("Releasing local VIP {} (no healthy backends)", vip);
if let Err(error) = vip_owner.ensure_absent(vip).await {
error!("Failed to release local VIP {}: {}", vip, error);
} else {
vip_state.owned = false;
}
} else {
vip_state.owned = false;
}
}
if vip_state.advertised {
info!("Withdrawing VIP {} (no healthy backends)", vip);
if let Err(error) = self.bgp.withdraw_route(vip).await {
error!("Failed to withdraw VIP {}: {}", vip, error);
} else {
vip_state.advertised = false;
}
}
if vip_state.is_idle() {
state.remove(&vip);
} else { } else {
state.insert(vip, VipState::Withdrawn); state.insert(vip, vip_state);
} }
} }
} }
@ -190,22 +238,52 @@ impl VipManager {
/// Manually advertise a VIP (for testing or manual control) /// Manually advertise a VIP (for testing or manual control)
pub async fn advertise_vip(&self, vip: IpAddr) -> Result<(), Box<dyn std::error::Error>> { pub async fn advertise_vip(&self, vip: IpAddr) -> Result<(), Box<dyn std::error::Error>> {
info!("Manually advertising VIP {}", vip);
self.bgp.announce_route(vip, self.next_hop).await?;
let mut state = self.vip_state.write().await; let mut state = self.vip_state.write().await;
state.insert(vip, VipState::Advertised); let mut vip_state = state.get(&vip).copied().unwrap_or_default();
if !vip_state.owned {
if let Some(vip_owner) = &self.vip_owner {
info!("Claiming local VIP {} on this node", vip);
vip_owner.ensure_present(vip).await?;
}
vip_state.owned = true;
}
if !vip_state.advertised {
info!("Manually advertising VIP {}", vip);
self.bgp.announce_route(vip, self.next_hop).await?;
vip_state.advertised = true;
}
state.insert(vip, vip_state);
Ok(()) Ok(())
} }
/// Manually withdraw a VIP (for testing or manual control) /// Manually withdraw a VIP (for testing or manual control)
pub async fn withdraw_vip(&self, vip: IpAddr) -> Result<(), Box<dyn std::error::Error>> { pub async fn withdraw_vip(&self, vip: IpAddr) -> Result<(), Box<dyn std::error::Error>> {
info!("Manually withdrawing VIP {}", vip);
self.bgp.withdraw_route(vip).await?;
let mut state = self.vip_state.write().await; let mut state = self.vip_state.write().await;
state.insert(vip, VipState::Withdrawn); let mut vip_state = state.get(&vip).copied().unwrap_or_default();
if vip_state.owned {
if let Some(vip_owner) = &self.vip_owner {
info!("Releasing local VIP {} on this node", vip);
vip_owner.ensure_absent(vip).await?;
}
vip_state.owned = false;
}
if vip_state.advertised {
info!("Manually withdrawing VIP {}", vip);
self.bgp.withdraw_route(vip).await?;
vip_state.advertised = false;
}
if vip_state.is_idle() {
state.remove(&vip);
} else {
state.insert(vip, vip_state);
}
Ok(()) Ok(())
} }
@ -216,19 +294,41 @@ impl VipManager {
pub async fn shutdown(&self) -> Result<(), Box<dyn std::error::Error>> { pub async fn shutdown(&self) -> Result<(), Box<dyn std::error::Error>> {
info!("VIP manager shutting down, withdrawing all VIPs..."); info!("VIP manager shutting down, withdrawing all VIPs...");
let state = self.vip_state.read().await; let mut state = self.vip_state.write().await;
let advertised_vips: Vec<IpAddr> = state let managed_vips: Vec<IpAddr> = state.keys().copied().collect();
.iter()
.filter(|(_, &state)| state == VipState::Advertised)
.map(|(vip, _)| *vip)
.collect();
drop(state); // Release read lock before withdrawing for vip in managed_vips {
let mut vip_state = state.get(&vip).copied().unwrap_or_default();
for vip in advertised_vips { if vip_state.owned {
info!("Withdrawing VIP {} for shutdown", vip); info!("Releasing local VIP {} for shutdown", vip);
if let Err(e) = self.bgp.withdraw_route(vip).await { if let Some(vip_owner) = &self.vip_owner {
error!("Failed to withdraw VIP {} during shutdown: {}", vip, e); if let Err(error) = vip_owner.ensure_absent(vip).await {
error!(
"Failed to release local VIP {} during shutdown: {}",
vip, error
);
} else {
vip_state.owned = false;
}
} else {
vip_state.owned = false;
}
}
if vip_state.advertised {
info!("Withdrawing VIP {} for shutdown", vip);
if let Err(error) = self.bgp.withdraw_route(vip).await {
error!("Failed to withdraw VIP {} during shutdown: {}", vip, error);
} else {
vip_state.advertised = false;
}
}
if vip_state.is_idle() {
state.remove(&vip);
} else {
state.insert(vip, vip_state);
} }
} }
@ -241,7 +341,7 @@ impl VipManager {
let state = self.vip_state.read().await; let state = self.vip_state.read().await;
state state
.iter() .iter()
.filter(|(_, &state)| state == VipState::Advertised) .filter(|(_, state)| state.advertised)
.map(|(vip, _)| *vip) .map(|(vip, _)| *vip)
.collect() .collect()
} }
@ -251,19 +351,22 @@ impl VipManager {
mod tests { mod tests {
use super::*; use super::*;
use crate::bgp_client::{BgpClient, Result}; use crate::bgp_client::{BgpClient, Result};
use crate::vip_owner::VipOwnershipError;
use std::sync::Mutex; use std::sync::Mutex;
/// Mock BGP client for testing /// Mock BGP client for testing
struct MockBgpClient { struct MockBgpClient {
announced: Arc<Mutex<HashSet<IpAddr>>>, announced: Arc<Mutex<HashSet<IpAddr>>>,
withdrawn: Arc<Mutex<HashSet<IpAddr>>>, withdrawn: Arc<Mutex<HashSet<IpAddr>>>,
events: Arc<Mutex<Vec<String>>>,
} }
impl MockBgpClient { impl MockBgpClient {
fn new() -> Self { fn new(events: Arc<Mutex<Vec<String>>>) -> Self {
Self { Self {
announced: Arc::new(Mutex::new(HashSet::new())), announced: Arc::new(Mutex::new(HashSet::new())),
withdrawn: Arc::new(Mutex::new(HashSet::new())), withdrawn: Arc::new(Mutex::new(HashSet::new())),
events,
} }
} }
} }
@ -271,11 +374,19 @@ mod tests {
#[tonic::async_trait] #[tonic::async_trait]
impl BgpClient for MockBgpClient { impl BgpClient for MockBgpClient {
async fn announce_route(&self, prefix: IpAddr, _next_hop: IpAddr) -> Result<()> { async fn announce_route(&self, prefix: IpAddr, _next_hop: IpAddr) -> Result<()> {
self.events
.lock()
.unwrap()
.push(format!("announce:{prefix}"));
self.announced.lock().unwrap().insert(prefix); self.announced.lock().unwrap().insert(prefix);
Ok(()) Ok(())
} }
async fn withdraw_route(&self, prefix: IpAddr) -> Result<()> { async fn withdraw_route(&self, prefix: IpAddr) -> Result<()> {
self.events
.lock()
.unwrap()
.push(format!("withdraw:{prefix}"));
self.withdrawn.lock().unwrap().insert(prefix); self.withdrawn.lock().unwrap().insert(prefix);
Ok(()) Ok(())
} }
@ -285,13 +396,51 @@ mod tests {
} }
} }
struct MockVipOwner {
owned: Arc<Mutex<HashSet<IpAddr>>>,
released: Arc<Mutex<HashSet<IpAddr>>>,
events: Arc<Mutex<Vec<String>>>,
}
impl MockVipOwner {
fn new(events: Arc<Mutex<Vec<String>>>) -> Self {
Self {
owned: Arc::new(Mutex::new(HashSet::new())),
released: Arc::new(Mutex::new(HashSet::new())),
events,
}
}
}
#[tonic::async_trait]
impl VipAddressOwner for MockVipOwner {
async fn ensure_present(&self, vip: IpAddr) -> std::result::Result<(), VipOwnershipError> {
self.events.lock().unwrap().push(format!("own:{vip}"));
self.owned.lock().unwrap().insert(vip);
Ok(())
}
async fn ensure_absent(&self, vip: IpAddr) -> std::result::Result<(), VipOwnershipError> {
self.events.lock().unwrap().push(format!("unown:{vip}"));
self.released.lock().unwrap().insert(vip);
Ok(())
}
}
#[tokio::test] #[tokio::test]
async fn test_vip_advertisement_tracking() { async fn test_vip_advertisement_tracking() {
let mock_bgp = Arc::new(MockBgpClient::new()); let events = Arc::new(Mutex::new(Vec::new()));
let mock_bgp = Arc::new(MockBgpClient::new(events.clone()));
let mock_owner = Arc::new(MockVipOwner::new(events.clone()));
let metadata = Arc::new(LbMetadataStore::new_in_memory()); let metadata = Arc::new(LbMetadataStore::new_in_memory());
let next_hop = "10.0.0.1".parse().unwrap(); let next_hop = "10.0.0.1".parse().unwrap();
let manager = VipManager::new(mock_bgp.clone(), metadata, next_hop); let manager = VipManager::new(
mock_bgp.clone(),
metadata,
next_hop,
Some(mock_owner.clone()),
);
let vip: IpAddr = "10.0.1.100".parse().unwrap(); let vip: IpAddr = "10.0.1.100".parse().unwrap();
@ -304,5 +453,16 @@ mod tests {
manager.withdraw_vip(vip).await.unwrap(); manager.withdraw_vip(vip).await.unwrap();
assert!(manager.get_advertised_vips().await.is_empty()); assert!(manager.get_advertised_vips().await.is_empty());
assert!(mock_bgp.withdrawn.lock().unwrap().contains(&vip)); assert!(mock_bgp.withdrawn.lock().unwrap().contains(&vip));
assert!(mock_owner.released.lock().unwrap().contains(&vip));
assert_eq!(
events.lock().unwrap().clone(),
vec![
format!("own:{vip}"),
format!("announce:{vip}"),
format!("unown:{vip}"),
format!("withdraw:{vip}"),
]
);
} }
} }

View file

@ -0,0 +1,161 @@
//! Local VIP ownership management for FiberLB.
//!
//! This module claims and releases VIP `/32` addresses on a local interface so
//! the kernel actually treats the advertised VIP as a local destination.
use std::net::{IpAddr, Ipv4Addr};
use std::path::Path;
use tokio::process::Command;
/// Result type for VIP ownership operations.
pub type Result<T> = std::result::Result<T, VipOwnershipError>;
/// VIP ownership errors.
#[derive(Debug, thiserror::Error)]
pub enum VipOwnershipError {
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("kernel VIP ownership command failed: {0}")]
CommandFailed(String),
#[error("unsupported VIP ownership operation: {0}")]
Unsupported(String),
}
/// Trait for local VIP ownership.
#[tonic::async_trait]
pub trait VipAddressOwner: Send + Sync {
/// Ensure a VIP is locally owned on this node.
async fn ensure_present(&self, vip: IpAddr) -> Result<()>;
/// Ensure a VIP is no longer locally owned on this node.
async fn ensure_absent(&self, vip: IpAddr) -> Result<()>;
}
/// Linux kernel-backed VIP owner using `ip addr add/del`.
pub struct KernelVipAddressOwner {
interface: String,
ip_command: String,
}
impl KernelVipAddressOwner {
/// Create a kernel-backed VIP owner for the given interface.
pub fn new(interface: impl Into<String>) -> Self {
Self {
interface: interface.into(),
ip_command: resolve_ip_command(),
}
}
async fn run_ip(&self, args: &[String]) -> Result<()> {
let output = Command::new(&self.ip_command).args(args).output().await?;
if output.status.success() {
return Ok(());
}
Err(VipOwnershipError::CommandFailed(format!(
"{} {}: {}",
self.ip_command,
args.join(" "),
render_command_output(&output)
)))
}
async fn add_v4(&self, vip: Ipv4Addr) -> Result<()> {
let args = vec![
"-4".to_string(),
"addr".to_string(),
"add".to_string(),
format!("{vip}/32"),
"dev".to_string(),
self.interface.clone(),
];
match self.run_ip(&args).await {
Ok(()) => Ok(()),
Err(VipOwnershipError::CommandFailed(message)) if message.contains("File exists") => {
Ok(())
}
Err(error) => Err(error),
}
}
async fn del_v4(&self, vip: Ipv4Addr) -> Result<()> {
let args = vec![
"-4".to_string(),
"addr".to_string(),
"del".to_string(),
format!("{vip}/32"),
"dev".to_string(),
self.interface.clone(),
];
match self.run_ip(&args).await {
Ok(()) => Ok(()),
Err(VipOwnershipError::CommandFailed(message))
if message.contains("Cannot assign requested address") =>
{
Ok(())
}
Err(error) => Err(error),
}
}
}
#[tonic::async_trait]
impl VipAddressOwner for KernelVipAddressOwner {
async fn ensure_present(&self, vip: IpAddr) -> Result<()> {
match vip {
IpAddr::V4(vip) => self.add_v4(vip).await,
IpAddr::V6(_) => Err(VipOwnershipError::Unsupported(format!(
"VIP ownership currently supports IPv4 only (got {vip})"
))),
}
}
async fn ensure_absent(&self, vip: IpAddr) -> Result<()> {
match vip {
IpAddr::V4(vip) => self.del_v4(vip).await,
IpAddr::V6(_) => Err(VipOwnershipError::Unsupported(format!(
"VIP ownership currently supports IPv4 only (got {vip})"
))),
}
}
}
fn resolve_ip_command() -> String {
if let Ok(path) = std::env::var("FIBERLB_IP_COMMAND") {
let trimmed = path.trim();
if !trimmed.is_empty() {
return trimmed.to_string();
}
}
for candidate in [
"/run/current-system/sw/bin/ip",
"/usr/sbin/ip",
"/sbin/ip",
"/usr/bin/ip",
"/bin/ip",
] {
if Path::new(candidate).exists() {
return candidate.to_string();
}
}
"ip".to_string()
}
fn render_command_output(output: &std::process::Output) -> String {
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
if !stderr.is_empty() {
return stderr;
}
let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string();
if !stdout.is_empty() {
return stdout;
}
format!("exit status {}", output.status)
}

View file

@ -47,6 +47,10 @@ let
vip_advertisement = { vip_advertisement = {
interval_secs = cfg.vipCheckIntervalSecs; interval_secs = cfg.vipCheckIntervalSecs;
}; };
vip_ownership = {
enabled = cfg.vipOwnership.enable;
interface = cfg.vipOwnership.interface;
};
} // lib.optionalAttrs cfg.bgp.enable { } // lib.optionalAttrs cfg.bgp.enable {
bgp = bgp =
{ {
@ -155,6 +159,16 @@ in
description = "Interval between FiberLB VIP-to-BGP reconciliation sweeps."; description = "Interval between FiberLB VIP-to-BGP reconciliation sweeps.";
}; };
vipOwnership = {
enable = lib.mkEnableOption "FiberLB local VIP ownership";
interface = lib.mkOption {
type = lib.types.str;
default = "lo";
description = "Interface where FiberLB should claim VIP /32 addresses.";
};
};
bgp = { bgp = {
enable = lib.mkEnableOption "FiberLB native BGP VIP advertisement"; enable = lib.mkEnableOption "FiberLB native BGP VIP advertisement";
@ -240,6 +254,10 @@ in
assertion = (!cfg.bgp.enable) || ((builtins.length cfg.bgp.peers) > 0); assertion = (!cfg.bgp.enable) || ((builtins.length cfg.bgp.peers) > 0);
message = "services.fiberlb.bgp.peers must contain at least one peer when native BGP is enabled"; message = "services.fiberlb.bgp.peers must contain at least one peer when native BGP is enabled";
} }
{
assertion = (!cfg.vipOwnership.enable) || cfg.bgp.enable;
message = "services.fiberlb.vipOwnership.enable requires services.fiberlb.bgp.enable";
}
]; ];
# Create system user # Create system user
@ -258,6 +276,7 @@ in
wantedBy = [ "multi-user.target" ]; wantedBy = [ "multi-user.target" ];
after = [ "network.target" "iam.service" ] ++ flaredbDependencies; after = [ "network.target" "iam.service" ] ++ flaredbDependencies;
requires = [ "iam.service" ] ++ flaredbDependencies; requires = [ "iam.service" ] ++ flaredbDependencies;
path = lib.optionals cfg.vipOwnership.enable [ pkgs.iproute2 ];
serviceConfig = { serviceConfig = {
Type = "simple"; Type = "simple";
@ -276,6 +295,8 @@ in
ProtectSystem = "strict"; ProtectSystem = "strict";
ProtectHome = true; ProtectHome = true;
ReadWritePaths = [ cfg.dataDir ]; ReadWritePaths = [ cfg.dataDir ];
AmbientCapabilities = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
CapabilityBoundingSet = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
# Environment variables for service endpoints # Environment variables for service endpoints
Environment = [ Environment = [

View file

@ -99,6 +99,8 @@ in {
if node != null then node.ip else "127.0.0.1"; if node != null then node.ip else "127.0.0.1";
peers = cfg.fiberlbBgp.peers; peers = cfg.fiberlbBgp.peers;
}; };
services.fiberlb.vipOwnership.enable = mkDefault true;
}) })
# PrismNET OVN integration # PrismNET OVN integration

View file

@ -29,6 +29,25 @@ let
iamProto = "iam.proto"; iamProto = "iam.proto";
fiberlbProtoDir = ../../fiberlb/crates/fiberlb-api/proto; fiberlbProtoDir = ../../fiberlb/crates/fiberlb-api/proto;
fiberlbProto = "fiberlb.proto"; fiberlbProto = "fiberlb.proto";
backendScript = pkgs.writeText "fiberlb-smoke-backend.py" ''
from http.server import BaseHTTPRequestHandler, HTTPServer
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
body = b"fiberlb smoke backend\n"
self.send_response(200)
self.send_header("Content-Type", "text/plain; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, format, *args):
return
HTTPServer(("127.0.0.1", 18081), Handler).serve_forever()
'';
in in
{ {
name = "fiberlb-native-bgp-vm-smoke"; name = "fiberlb-native-bgp-vm-smoke";
@ -49,6 +68,7 @@ in
]; ];
environment.systemPackages = with pkgs; [ environment.systemPackages = with pkgs; [
curl
gobgp gobgp
gobgpd gobgpd
jq jq
@ -117,6 +137,10 @@ in
healthCheckIntervalSecs = 1; healthCheckIntervalSecs = 1;
healthCheckTimeoutSecs = 1; healthCheckTimeoutSecs = 1;
vipCheckIntervalSecs = 1; vipCheckIntervalSecs = 1;
vipOwnership = {
enable = true;
interface = "lo";
};
bgp = { bgp = {
enable = true; enable = true;
localAs = 65010; localAs = 65010;
@ -141,7 +165,7 @@ in
after = [ "network.target" ]; after = [ "network.target" ];
serviceConfig = { serviceConfig = {
Type = "simple"; Type = "simple";
ExecStart = "${pkgs.python3}/bin/python -m http.server 18081 --bind 127.0.0.1"; ExecStart = "${pkgs.python3}/bin/python ${backendScript}";
Restart = "always"; Restart = "always";
RestartSec = "1s"; RestartSec = "1s";
}; };
@ -276,6 +300,26 @@ in
time.sleep(1) time.sleep(1)
raise AssertionError(f"route {prefix} still present in GoBGP RIB") raise AssertionError(f"route {prefix} still present in GoBGP RIB")
def wait_for_local_vip(vip, present):
pattern = f"inet {vip}/32"
if present:
lb.wait_until_succeeds(
f"ip -4 addr show dev lo | grep -F {shlex.quote(pattern)}"
)
else:
deadline = time.time() + 60
while time.time() < deadline:
output = lb.succeed("ip -4 addr show dev lo || true")
if pattern not in output:
return
time.sleep(1)
raise AssertionError(f"VIP {vip} still present on loopback")
def wait_for_http_success(url):
router.wait_until_succeeds(
f"curl -fsS --max-time 5 {shlex.quote(url)} | grep -F 'fiberlb smoke backend'"
)
start_all() start_all()
serial_stdout_off() serial_stdout_off()
@ -307,7 +351,9 @@ in
) )
loadbalancer = lb_response["loadbalancer"] loadbalancer = lb_response["loadbalancer"]
lb_id = loadbalancer["id"] lb_id = loadbalancer["id"]
vip_prefix = f"{loadbalancer['vipAddress']}/32" vip = loadbalancer["vipAddress"]
vip_prefix = f"{vip}/32"
listener_url = f"http://{vip}:18080/"
pool_id = grpcurl_json( pool_id = grpcurl_json(
lb, lb,
@ -363,16 +409,50 @@ in
headers=[f"authorization: Bearer {token}"], headers=[f"authorization: Bearer {token}"],
) )
grpcurl_json(
lb,
"127.0.0.1:50085",
FIBERLB_PROTO_DIR,
FIBERLB_PROTO,
"fiberlb.v1.ListenerService/CreateListener",
{
"name": "bgp-smoke-listener",
"loadbalancerId": lb_id,
"protocol": "LISTENER_PROTOCOL_TCP",
"port": 18080,
"defaultPoolId": pool_id,
},
headers=[f"authorization: Bearer {token}"],
)
wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token) wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
wait_for_local_vip(vip, True)
wait_for_route(vip_prefix, True) wait_for_route(vip_prefix, True)
router.succeed(f"ip route replace {shlex.quote(vip_prefix)} via 192.168.100.2 dev eth1")
wait_for_http_success(listener_url)
router.succeed("systemctl restart gobgpd-peer.service")
router.wait_for_unit("gobgpd-peer.service")
router.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2")
wait_for_route(vip_prefix, True)
wait_for_http_success(listener_url)
lb.succeed("systemctl stop mock-backend.service") lb.succeed("systemctl stop mock-backend.service")
wait_for_backend_status("BACKEND_STATUS_OFFLINE", backend_id, token) wait_for_backend_status("BACKEND_STATUS_OFFLINE", backend_id, token)
wait_for_route(vip_prefix, False) wait_for_route(vip_prefix, False)
wait_for_local_vip(vip, False)
router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")
lb.succeed("systemctl start mock-backend.service") lb.succeed("systemctl start mock-backend.service")
lb.wait_for_unit("mock-backend.service") lb.wait_for_unit("mock-backend.service")
wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token) wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
wait_for_local_vip(vip, True)
wait_for_route(vip_prefix, True) wait_for_route(vip_prefix, True)
wait_for_http_success(listener_url)
lb.succeed("systemctl stop fiberlb.service")
wait_for_local_vip(vip, False)
wait_for_route(vip_prefix, False)
router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")
''; '';
} }