Harden FiberLB native BGP control plane
This commit is contained in:
parent
ce4bab07d6
commit
63c7251756
8 changed files with 551 additions and 63 deletions
|
|
@ -79,6 +79,10 @@ pub struct ServerConfig {
|
|||
#[serde(default)]
|
||||
pub vip_advertisement: VipAdvertisementConfig,
|
||||
|
||||
/// Local VIP ownership configuration.
|
||||
#[serde(default)]
|
||||
pub vip_ownership: VipOwnershipConfig,
|
||||
|
||||
/// Native BGP speaker configuration
|
||||
#[serde(default)]
|
||||
pub bgp: BgpConfig,
|
||||
|
|
@ -145,6 +149,31 @@ impl Default for VipAdvertisementConfig {
|
|||
}
|
||||
}
|
||||
|
||||
/// Local VIP ownership runtime configuration.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct VipOwnershipConfig {
|
||||
/// Whether FiberLB should claim VIP /32 addresses on the local node.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
|
||||
/// Interface used for local VIP ownership.
|
||||
#[serde(default = "default_vip_ownership_interface")]
|
||||
pub interface: String,
|
||||
}
|
||||
|
||||
fn default_vip_ownership_interface() -> String {
|
||||
"lo".to_string()
|
||||
}
|
||||
|
||||
impl Default for VipOwnershipConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
interface: default_vip_ownership_interface(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Static BGP peer configuration.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BgpPeerConfig {
|
||||
|
|
@ -272,6 +301,7 @@ impl Default for ServerConfig {
|
|||
auth: AuthConfig::default(),
|
||||
health: HealthRuntimeConfig::default(),
|
||||
vip_advertisement: VipAdvertisementConfig::default(),
|
||||
vip_ownership: VipOwnershipConfig::default(),
|
||||
bgp: BgpConfig::default(),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ pub mod metadata;
|
|||
pub mod services;
|
||||
pub mod tls;
|
||||
pub mod vip_manager;
|
||||
pub mod vip_owner;
|
||||
|
||||
pub use bgp_client::{create_bgp_client, BgpClient, BgpError, NativeBgpSpeaker};
|
||||
pub use config::ServerConfig;
|
||||
|
|
@ -23,3 +24,4 @@ pub use metadata::LbMetadataStore;
|
|||
pub use services::*;
|
||||
pub use tls::{build_tls_config, CertificateStore, SniCertResolver};
|
||||
pub use vip_manager::VipManager;
|
||||
pub use vip_owner::{KernelVipAddressOwner, VipAddressOwner, VipOwnershipError};
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@ use fiberlb_api::{
|
|||
};
|
||||
use fiberlb_server::{
|
||||
config::MetadataBackend, create_bgp_client, spawn_health_checker, BackendServiceImpl,
|
||||
CertificateServiceImpl, DataPlane, HealthCheckServiceImpl, L7DataPlane, L7PolicyServiceImpl,
|
||||
L7RuleServiceImpl, LbMetadataStore, ListenerServiceImpl, LoadBalancerServiceImpl,
|
||||
PoolServiceImpl, ServerConfig, VipManager,
|
||||
CertificateServiceImpl, DataPlane, HealthCheckServiceImpl, KernelVipAddressOwner, L7DataPlane,
|
||||
L7PolicyServiceImpl, L7RuleServiceImpl, LbMetadataStore, ListenerServiceImpl,
|
||||
LoadBalancerServiceImpl, PoolServiceImpl, ServerConfig, VipAddressOwner, VipManager,
|
||||
};
|
||||
use iam_service_auth::AuthService;
|
||||
use metrics_exporter_prometheus::PrometheusBuilder;
|
||||
|
|
@ -248,7 +248,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
)
|
||||
})?;
|
||||
let bgp = create_bgp_client(config.bgp.clone()).await?;
|
||||
let manager = Arc::new(VipManager::new(bgp, metadata.clone(), next_hop));
|
||||
let vip_owner: Option<Arc<dyn VipAddressOwner>> = if config.vip_ownership.enabled {
|
||||
Some(Arc::new(KernelVipAddressOwner::new(
|
||||
config.vip_ownership.interface.clone(),
|
||||
)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let manager = Arc::new(VipManager::new(bgp, metadata.clone(), next_hop, vip_owner));
|
||||
let _vip_task = manager.clone().spawn(Duration::from_secs(
|
||||
config.vip_advertisement.interval_secs.max(1),
|
||||
));
|
||||
|
|
@ -352,7 +359,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
CertificateServiceServer::new(certificate_service),
|
||||
make_interceptor(auth_service.clone()),
|
||||
))
|
||||
.serve(grpc_addr)
|
||||
.serve_with_shutdown(grpc_addr, async {
|
||||
if let Err(error) = wait_for_shutdown_signal().await {
|
||||
tracing::warn!(error = %error, "FiberLB shutdown signal handler failed");
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
let _ = health_shutdown_tx.send(true);
|
||||
|
|
@ -367,6 +378,27 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_shutdown_signal() -> Result<(), Box<dyn std::error::Error>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use tokio::signal::unix::{signal, SignalKind};
|
||||
|
||||
let mut terminate = signal(SignalKind::terminate())?;
|
||||
tokio::select! {
|
||||
_ = tokio::signal::ctrl_c() => {}
|
||||
_ = terminate.recv() => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
tokio::signal::ctrl_c().await?;
|
||||
}
|
||||
|
||||
tracing::info!("FiberLB shutdown signal received");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"flaredb" => Ok(MetadataBackend::FlareDb),
|
||||
|
|
|
|||
|
|
@ -14,15 +14,22 @@ use tracing::{debug, error, info, warn};
|
|||
|
||||
use crate::bgp_client::BgpClient;
|
||||
use crate::metadata::LbMetadataStore;
|
||||
use crate::vip_owner::VipAddressOwner;
|
||||
use fiberlb_types::LoadBalancerId;
|
||||
|
||||
/// VIP advertisement state
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum VipState {
|
||||
/// VIP is advertised to BGP peers
|
||||
Advertised,
|
||||
/// VIP is withdrawn from BGP peers
|
||||
Withdrawn,
|
||||
/// Current local control-plane state for a VIP.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
struct VipState {
|
||||
/// The VIP is configured on the local node.
|
||||
owned: bool,
|
||||
/// The VIP is advertised to BGP peers.
|
||||
advertised: bool,
|
||||
}
|
||||
|
||||
impl VipState {
|
||||
fn is_idle(self) -> bool {
|
||||
!self.owned && !self.advertised
|
||||
}
|
||||
}
|
||||
|
||||
/// VIP Manager
|
||||
|
|
@ -37,17 +44,25 @@ pub struct VipManager {
|
|||
metadata: Arc<LbMetadataStore>,
|
||||
/// Current VIP advertisement state
|
||||
vip_state: Arc<RwLock<HashMap<IpAddr, VipState>>>,
|
||||
/// Optional local VIP owner.
|
||||
vip_owner: Option<Arc<dyn VipAddressOwner>>,
|
||||
/// Router's own IP address (used as BGP next hop)
|
||||
next_hop: IpAddr,
|
||||
}
|
||||
|
||||
impl VipManager {
|
||||
/// Create a new VIP manager
|
||||
pub fn new(bgp: Arc<dyn BgpClient>, metadata: Arc<LbMetadataStore>, next_hop: IpAddr) -> Self {
|
||||
pub fn new(
|
||||
bgp: Arc<dyn BgpClient>,
|
||||
metadata: Arc<LbMetadataStore>,
|
||||
next_hop: IpAddr,
|
||||
vip_owner: Option<Arc<dyn VipAddressOwner>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
bgp,
|
||||
metadata,
|
||||
vip_state: Arc::new(RwLock::new(HashMap::new())),
|
||||
vip_owner,
|
||||
next_hop,
|
||||
}
|
||||
}
|
||||
|
|
@ -148,39 +163,72 @@ impl VipManager {
|
|||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut state = self.vip_state.write().await;
|
||||
|
||||
// Find VIPs to announce (active but not yet advertised)
|
||||
for vip in active_vips {
|
||||
match state.get(vip) {
|
||||
Some(VipState::Advertised) => {
|
||||
// Already advertised, nothing to do
|
||||
debug!("VIP {} already advertised", vip);
|
||||
}
|
||||
Some(VipState::Withdrawn) | None => {
|
||||
// Need to announce
|
||||
info!("Announcing VIP {} (healthy backends available)", vip);
|
||||
if let Err(e) = self.bgp.announce_route(*vip, self.next_hop).await {
|
||||
error!("Failed to announce VIP {}: {}", vip, e);
|
||||
} else {
|
||||
state.insert(*vip, VipState::Advertised);
|
||||
let mut vip_state = state.get(vip).copied().unwrap_or_default();
|
||||
let mut changed = false;
|
||||
|
||||
if !vip_state.owned {
|
||||
if let Some(vip_owner) = &self.vip_owner {
|
||||
info!("Claiming local VIP {} on this node", vip);
|
||||
if let Err(error) = vip_owner.ensure_present(*vip).await {
|
||||
error!("Failed to claim local VIP {}: {}", vip, error);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
vip_state.owned = true;
|
||||
changed = true;
|
||||
} else {
|
||||
debug!("VIP {} already locally owned", vip);
|
||||
}
|
||||
|
||||
if !vip_state.advertised {
|
||||
info!("Announcing VIP {} (healthy backends available)", vip);
|
||||
if let Err(error) = self.bgp.announce_route(*vip, self.next_hop).await {
|
||||
error!("Failed to announce VIP {}: {}", vip, error);
|
||||
} else {
|
||||
vip_state.advertised = true;
|
||||
changed = true;
|
||||
}
|
||||
} else {
|
||||
debug!("VIP {} already advertised", vip);
|
||||
}
|
||||
|
||||
if changed {
|
||||
state.insert(*vip, vip_state);
|
||||
}
|
||||
}
|
||||
|
||||
// Find VIPs to withdraw (advertised but no longer active)
|
||||
let advertised_vips: Vec<IpAddr> = state
|
||||
.iter()
|
||||
.filter(|(_, &state)| state == VipState::Advertised)
|
||||
.map(|(vip, _)| *vip)
|
||||
.collect();
|
||||
|
||||
for vip in advertised_vips {
|
||||
let managed_vips: Vec<IpAddr> = state.keys().copied().collect();
|
||||
for vip in managed_vips {
|
||||
if !active_vips.contains(&vip) {
|
||||
info!("Withdrawing VIP {} (no healthy backends)", vip);
|
||||
if let Err(e) = self.bgp.withdraw_route(vip).await {
|
||||
error!("Failed to withdraw VIP {}: {}", vip, e);
|
||||
let mut vip_state = state.get(&vip).copied().unwrap_or_default();
|
||||
|
||||
if vip_state.owned {
|
||||
if let Some(vip_owner) = &self.vip_owner {
|
||||
info!("Releasing local VIP {} (no healthy backends)", vip);
|
||||
if let Err(error) = vip_owner.ensure_absent(vip).await {
|
||||
error!("Failed to release local VIP {}: {}", vip, error);
|
||||
} else {
|
||||
vip_state.owned = false;
|
||||
}
|
||||
} else {
|
||||
vip_state.owned = false;
|
||||
}
|
||||
}
|
||||
|
||||
if vip_state.advertised {
|
||||
info!("Withdrawing VIP {} (no healthy backends)", vip);
|
||||
if let Err(error) = self.bgp.withdraw_route(vip).await {
|
||||
error!("Failed to withdraw VIP {}: {}", vip, error);
|
||||
} else {
|
||||
vip_state.advertised = false;
|
||||
}
|
||||
}
|
||||
|
||||
if vip_state.is_idle() {
|
||||
state.remove(&vip);
|
||||
} else {
|
||||
state.insert(vip, VipState::Withdrawn);
|
||||
state.insert(vip, vip_state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -190,22 +238,52 @@ impl VipManager {
|
|||
|
||||
/// Manually advertise a VIP (for testing or manual control)
|
||||
pub async fn advertise_vip(&self, vip: IpAddr) -> Result<(), Box<dyn std::error::Error>> {
|
||||
info!("Manually advertising VIP {}", vip);
|
||||
self.bgp.announce_route(vip, self.next_hop).await?;
|
||||
|
||||
let mut state = self.vip_state.write().await;
|
||||
state.insert(vip, VipState::Advertised);
|
||||
let mut vip_state = state.get(&vip).copied().unwrap_or_default();
|
||||
|
||||
if !vip_state.owned {
|
||||
if let Some(vip_owner) = &self.vip_owner {
|
||||
info!("Claiming local VIP {} on this node", vip);
|
||||
vip_owner.ensure_present(vip).await?;
|
||||
}
|
||||
vip_state.owned = true;
|
||||
}
|
||||
|
||||
if !vip_state.advertised {
|
||||
info!("Manually advertising VIP {}", vip);
|
||||
self.bgp.announce_route(vip, self.next_hop).await?;
|
||||
vip_state.advertised = true;
|
||||
}
|
||||
|
||||
state.insert(vip, vip_state);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Manually withdraw a VIP (for testing or manual control)
|
||||
pub async fn withdraw_vip(&self, vip: IpAddr) -> Result<(), Box<dyn std::error::Error>> {
|
||||
info!("Manually withdrawing VIP {}", vip);
|
||||
self.bgp.withdraw_route(vip).await?;
|
||||
|
||||
let mut state = self.vip_state.write().await;
|
||||
state.insert(vip, VipState::Withdrawn);
|
||||
let mut vip_state = state.get(&vip).copied().unwrap_or_default();
|
||||
|
||||
if vip_state.owned {
|
||||
if let Some(vip_owner) = &self.vip_owner {
|
||||
info!("Releasing local VIP {} on this node", vip);
|
||||
vip_owner.ensure_absent(vip).await?;
|
||||
}
|
||||
vip_state.owned = false;
|
||||
}
|
||||
|
||||
if vip_state.advertised {
|
||||
info!("Manually withdrawing VIP {}", vip);
|
||||
self.bgp.withdraw_route(vip).await?;
|
||||
vip_state.advertised = false;
|
||||
}
|
||||
|
||||
if vip_state.is_idle() {
|
||||
state.remove(&vip);
|
||||
} else {
|
||||
state.insert(vip, vip_state);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -216,19 +294,41 @@ impl VipManager {
|
|||
pub async fn shutdown(&self) -> Result<(), Box<dyn std::error::Error>> {
|
||||
info!("VIP manager shutting down, withdrawing all VIPs...");
|
||||
|
||||
let state = self.vip_state.read().await;
|
||||
let advertised_vips: Vec<IpAddr> = state
|
||||
.iter()
|
||||
.filter(|(_, &state)| state == VipState::Advertised)
|
||||
.map(|(vip, _)| *vip)
|
||||
.collect();
|
||||
let mut state = self.vip_state.write().await;
|
||||
let managed_vips: Vec<IpAddr> = state.keys().copied().collect();
|
||||
|
||||
drop(state); // Release read lock before withdrawing
|
||||
for vip in managed_vips {
|
||||
let mut vip_state = state.get(&vip).copied().unwrap_or_default();
|
||||
|
||||
for vip in advertised_vips {
|
||||
info!("Withdrawing VIP {} for shutdown", vip);
|
||||
if let Err(e) = self.bgp.withdraw_route(vip).await {
|
||||
error!("Failed to withdraw VIP {} during shutdown: {}", vip, e);
|
||||
if vip_state.owned {
|
||||
info!("Releasing local VIP {} for shutdown", vip);
|
||||
if let Some(vip_owner) = &self.vip_owner {
|
||||
if let Err(error) = vip_owner.ensure_absent(vip).await {
|
||||
error!(
|
||||
"Failed to release local VIP {} during shutdown: {}",
|
||||
vip, error
|
||||
);
|
||||
} else {
|
||||
vip_state.owned = false;
|
||||
}
|
||||
} else {
|
||||
vip_state.owned = false;
|
||||
}
|
||||
}
|
||||
|
||||
if vip_state.advertised {
|
||||
info!("Withdrawing VIP {} for shutdown", vip);
|
||||
if let Err(error) = self.bgp.withdraw_route(vip).await {
|
||||
error!("Failed to withdraw VIP {} during shutdown: {}", vip, error);
|
||||
} else {
|
||||
vip_state.advertised = false;
|
||||
}
|
||||
}
|
||||
|
||||
if vip_state.is_idle() {
|
||||
state.remove(&vip);
|
||||
} else {
|
||||
state.insert(vip, vip_state);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -241,7 +341,7 @@ impl VipManager {
|
|||
let state = self.vip_state.read().await;
|
||||
state
|
||||
.iter()
|
||||
.filter(|(_, &state)| state == VipState::Advertised)
|
||||
.filter(|(_, state)| state.advertised)
|
||||
.map(|(vip, _)| *vip)
|
||||
.collect()
|
||||
}
|
||||
|
|
@ -251,19 +351,22 @@ impl VipManager {
|
|||
mod tests {
|
||||
use super::*;
|
||||
use crate::bgp_client::{BgpClient, Result};
|
||||
use crate::vip_owner::VipOwnershipError;
|
||||
use std::sync::Mutex;
|
||||
|
||||
/// Mock BGP client for testing
|
||||
struct MockBgpClient {
|
||||
announced: Arc<Mutex<HashSet<IpAddr>>>,
|
||||
withdrawn: Arc<Mutex<HashSet<IpAddr>>>,
|
||||
events: Arc<Mutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
impl MockBgpClient {
|
||||
fn new() -> Self {
|
||||
fn new(events: Arc<Mutex<Vec<String>>>) -> Self {
|
||||
Self {
|
||||
announced: Arc::new(Mutex::new(HashSet::new())),
|
||||
withdrawn: Arc::new(Mutex::new(HashSet::new())),
|
||||
events,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -271,11 +374,19 @@ mod tests {
|
|||
#[tonic::async_trait]
|
||||
impl BgpClient for MockBgpClient {
|
||||
async fn announce_route(&self, prefix: IpAddr, _next_hop: IpAddr) -> Result<()> {
|
||||
self.events
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(format!("announce:{prefix}"));
|
||||
self.announced.lock().unwrap().insert(prefix);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn withdraw_route(&self, prefix: IpAddr) -> Result<()> {
|
||||
self.events
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(format!("withdraw:{prefix}"));
|
||||
self.withdrawn.lock().unwrap().insert(prefix);
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -285,13 +396,51 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
struct MockVipOwner {
|
||||
owned: Arc<Mutex<HashSet<IpAddr>>>,
|
||||
released: Arc<Mutex<HashSet<IpAddr>>>,
|
||||
events: Arc<Mutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
impl MockVipOwner {
|
||||
fn new(events: Arc<Mutex<Vec<String>>>) -> Self {
|
||||
Self {
|
||||
owned: Arc::new(Mutex::new(HashSet::new())),
|
||||
released: Arc::new(Mutex::new(HashSet::new())),
|
||||
events,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl VipAddressOwner for MockVipOwner {
|
||||
async fn ensure_present(&self, vip: IpAddr) -> std::result::Result<(), VipOwnershipError> {
|
||||
self.events.lock().unwrap().push(format!("own:{vip}"));
|
||||
self.owned.lock().unwrap().insert(vip);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ensure_absent(&self, vip: IpAddr) -> std::result::Result<(), VipOwnershipError> {
|
||||
self.events.lock().unwrap().push(format!("unown:{vip}"));
|
||||
self.released.lock().unwrap().insert(vip);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_vip_advertisement_tracking() {
|
||||
let mock_bgp = Arc::new(MockBgpClient::new());
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let mock_bgp = Arc::new(MockBgpClient::new(events.clone()));
|
||||
let mock_owner = Arc::new(MockVipOwner::new(events.clone()));
|
||||
let metadata = Arc::new(LbMetadataStore::new_in_memory());
|
||||
let next_hop = "10.0.0.1".parse().unwrap();
|
||||
|
||||
let manager = VipManager::new(mock_bgp.clone(), metadata, next_hop);
|
||||
let manager = VipManager::new(
|
||||
mock_bgp.clone(),
|
||||
metadata,
|
||||
next_hop,
|
||||
Some(mock_owner.clone()),
|
||||
);
|
||||
|
||||
let vip: IpAddr = "10.0.1.100".parse().unwrap();
|
||||
|
||||
|
|
@ -304,5 +453,16 @@ mod tests {
|
|||
manager.withdraw_vip(vip).await.unwrap();
|
||||
assert!(manager.get_advertised_vips().await.is_empty());
|
||||
assert!(mock_bgp.withdrawn.lock().unwrap().contains(&vip));
|
||||
assert!(mock_owner.released.lock().unwrap().contains(&vip));
|
||||
|
||||
assert_eq!(
|
||||
events.lock().unwrap().clone(),
|
||||
vec![
|
||||
format!("own:{vip}"),
|
||||
format!("announce:{vip}"),
|
||||
format!("unown:{vip}"),
|
||||
format!("withdraw:{vip}"),
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
161
fiberlb/crates/fiberlb-server/src/vip_owner.rs
Normal file
161
fiberlb/crates/fiberlb-server/src/vip_owner.rs
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
//! Local VIP ownership management for FiberLB.
|
||||
//!
|
||||
//! This module claims and releases VIP `/32` addresses on a local interface so
|
||||
//! the kernel actually treats the advertised VIP as a local destination.
|
||||
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
use std::path::Path;
|
||||
|
||||
use tokio::process::Command;
|
||||
|
||||
/// Result type for VIP ownership operations.
|
||||
pub type Result<T> = std::result::Result<T, VipOwnershipError>;
|
||||
|
||||
/// VIP ownership errors.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum VipOwnershipError {
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("kernel VIP ownership command failed: {0}")]
|
||||
CommandFailed(String),
|
||||
#[error("unsupported VIP ownership operation: {0}")]
|
||||
Unsupported(String),
|
||||
}
|
||||
|
||||
/// Trait for local VIP ownership.
|
||||
#[tonic::async_trait]
|
||||
pub trait VipAddressOwner: Send + Sync {
|
||||
/// Ensure a VIP is locally owned on this node.
|
||||
async fn ensure_present(&self, vip: IpAddr) -> Result<()>;
|
||||
|
||||
/// Ensure a VIP is no longer locally owned on this node.
|
||||
async fn ensure_absent(&self, vip: IpAddr) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Linux kernel-backed VIP owner using `ip addr add/del`.
|
||||
pub struct KernelVipAddressOwner {
|
||||
interface: String,
|
||||
ip_command: String,
|
||||
}
|
||||
|
||||
impl KernelVipAddressOwner {
|
||||
/// Create a kernel-backed VIP owner for the given interface.
|
||||
pub fn new(interface: impl Into<String>) -> Self {
|
||||
Self {
|
||||
interface: interface.into(),
|
||||
ip_command: resolve_ip_command(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_ip(&self, args: &[String]) -> Result<()> {
|
||||
let output = Command::new(&self.ip_command).args(args).output().await?;
|
||||
if output.status.success() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
Err(VipOwnershipError::CommandFailed(format!(
|
||||
"{} {}: {}",
|
||||
self.ip_command,
|
||||
args.join(" "),
|
||||
render_command_output(&output)
|
||||
)))
|
||||
}
|
||||
|
||||
async fn add_v4(&self, vip: Ipv4Addr) -> Result<()> {
|
||||
let args = vec![
|
||||
"-4".to_string(),
|
||||
"addr".to_string(),
|
||||
"add".to_string(),
|
||||
format!("{vip}/32"),
|
||||
"dev".to_string(),
|
||||
self.interface.clone(),
|
||||
];
|
||||
|
||||
match self.run_ip(&args).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(VipOwnershipError::CommandFailed(message)) if message.contains("File exists") => {
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
async fn del_v4(&self, vip: Ipv4Addr) -> Result<()> {
|
||||
let args = vec![
|
||||
"-4".to_string(),
|
||||
"addr".to_string(),
|
||||
"del".to_string(),
|
||||
format!("{vip}/32"),
|
||||
"dev".to_string(),
|
||||
self.interface.clone(),
|
||||
];
|
||||
|
||||
match self.run_ip(&args).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(VipOwnershipError::CommandFailed(message))
|
||||
if message.contains("Cannot assign requested address") =>
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl VipAddressOwner for KernelVipAddressOwner {
|
||||
async fn ensure_present(&self, vip: IpAddr) -> Result<()> {
|
||||
match vip {
|
||||
IpAddr::V4(vip) => self.add_v4(vip).await,
|
||||
IpAddr::V6(_) => Err(VipOwnershipError::Unsupported(format!(
|
||||
"VIP ownership currently supports IPv4 only (got {vip})"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn ensure_absent(&self, vip: IpAddr) -> Result<()> {
|
||||
match vip {
|
||||
IpAddr::V4(vip) => self.del_v4(vip).await,
|
||||
IpAddr::V6(_) => Err(VipOwnershipError::Unsupported(format!(
|
||||
"VIP ownership currently supports IPv4 only (got {vip})"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_ip_command() -> String {
|
||||
if let Ok(path) = std::env::var("FIBERLB_IP_COMMAND") {
|
||||
let trimmed = path.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return trimmed.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
for candidate in [
|
||||
"/run/current-system/sw/bin/ip",
|
||||
"/usr/sbin/ip",
|
||||
"/sbin/ip",
|
||||
"/usr/bin/ip",
|
||||
"/bin/ip",
|
||||
] {
|
||||
if Path::new(candidate).exists() {
|
||||
return candidate.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
"ip".to_string()
|
||||
}
|
||||
|
||||
fn render_command_output(output: &std::process::Output) -> String {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
|
||||
if !stderr.is_empty() {
|
||||
return stderr;
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||
if !stdout.is_empty() {
|
||||
return stdout;
|
||||
}
|
||||
|
||||
format!("exit status {}", output.status)
|
||||
}
|
||||
|
|
@ -47,6 +47,10 @@ let
|
|||
vip_advertisement = {
|
||||
interval_secs = cfg.vipCheckIntervalSecs;
|
||||
};
|
||||
vip_ownership = {
|
||||
enabled = cfg.vipOwnership.enable;
|
||||
interface = cfg.vipOwnership.interface;
|
||||
};
|
||||
} // lib.optionalAttrs cfg.bgp.enable {
|
||||
bgp =
|
||||
{
|
||||
|
|
@ -155,6 +159,16 @@ in
|
|||
description = "Interval between FiberLB VIP-to-BGP reconciliation sweeps.";
|
||||
};
|
||||
|
||||
vipOwnership = {
|
||||
enable = lib.mkEnableOption "FiberLB local VIP ownership";
|
||||
|
||||
interface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "lo";
|
||||
description = "Interface where FiberLB should claim VIP /32 addresses.";
|
||||
};
|
||||
};
|
||||
|
||||
bgp = {
|
||||
enable = lib.mkEnableOption "FiberLB native BGP VIP advertisement";
|
||||
|
||||
|
|
@ -240,6 +254,10 @@ in
|
|||
assertion = (!cfg.bgp.enable) || ((builtins.length cfg.bgp.peers) > 0);
|
||||
message = "services.fiberlb.bgp.peers must contain at least one peer when native BGP is enabled";
|
||||
}
|
||||
{
|
||||
assertion = (!cfg.vipOwnership.enable) || cfg.bgp.enable;
|
||||
message = "services.fiberlb.vipOwnership.enable requires services.fiberlb.bgp.enable";
|
||||
}
|
||||
];
|
||||
|
||||
# Create system user
|
||||
|
|
@ -258,6 +276,7 @@ in
|
|||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" "iam.service" ] ++ flaredbDependencies;
|
||||
requires = [ "iam.service" ] ++ flaredbDependencies;
|
||||
path = lib.optionals cfg.vipOwnership.enable [ pkgs.iproute2 ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
|
|
@ -276,6 +295,8 @@ in
|
|||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
ReadWritePaths = [ cfg.dataDir ];
|
||||
AmbientCapabilities = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
|
||||
CapabilityBoundingSet = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
|
||||
|
||||
# Environment variables for service endpoints
|
||||
Environment = [
|
||||
|
|
|
|||
|
|
@ -99,6 +99,8 @@ in {
|
|||
if node != null then node.ip else "127.0.0.1";
|
||||
peers = cfg.fiberlbBgp.peers;
|
||||
};
|
||||
|
||||
services.fiberlb.vipOwnership.enable = mkDefault true;
|
||||
})
|
||||
|
||||
# PrismNET OVN integration
|
||||
|
|
|
|||
|
|
@ -29,6 +29,25 @@ let
|
|||
iamProto = "iam.proto";
|
||||
fiberlbProtoDir = ../../fiberlb/crates/fiberlb-api/proto;
|
||||
fiberlbProto = "fiberlb.proto";
|
||||
backendScript = pkgs.writeText "fiberlb-smoke-backend.py" ''
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
body = b"fiberlb smoke backend\n"
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format, *args):
|
||||
return
|
||||
|
||||
|
||||
HTTPServer(("127.0.0.1", 18081), Handler).serve_forever()
|
||||
'';
|
||||
in
|
||||
{
|
||||
name = "fiberlb-native-bgp-vm-smoke";
|
||||
|
|
@ -49,6 +68,7 @@ in
|
|||
];
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
curl
|
||||
gobgp
|
||||
gobgpd
|
||||
jq
|
||||
|
|
@ -117,6 +137,10 @@ in
|
|||
healthCheckIntervalSecs = 1;
|
||||
healthCheckTimeoutSecs = 1;
|
||||
vipCheckIntervalSecs = 1;
|
||||
vipOwnership = {
|
||||
enable = true;
|
||||
interface = "lo";
|
||||
};
|
||||
bgp = {
|
||||
enable = true;
|
||||
localAs = 65010;
|
||||
|
|
@ -141,7 +165,7 @@ in
|
|||
after = [ "network.target" ];
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
ExecStart = "${pkgs.python3}/bin/python -m http.server 18081 --bind 127.0.0.1";
|
||||
ExecStart = "${pkgs.python3}/bin/python ${backendScript}";
|
||||
Restart = "always";
|
||||
RestartSec = "1s";
|
||||
};
|
||||
|
|
@ -276,6 +300,26 @@ in
|
|||
time.sleep(1)
|
||||
raise AssertionError(f"route {prefix} still present in GoBGP RIB")
|
||||
|
||||
def wait_for_local_vip(vip, present):
|
||||
pattern = f"inet {vip}/32"
|
||||
if present:
|
||||
lb.wait_until_succeeds(
|
||||
f"ip -4 addr show dev lo | grep -F {shlex.quote(pattern)}"
|
||||
)
|
||||
else:
|
||||
deadline = time.time() + 60
|
||||
while time.time() < deadline:
|
||||
output = lb.succeed("ip -4 addr show dev lo || true")
|
||||
if pattern not in output:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise AssertionError(f"VIP {vip} still present on loopback")
|
||||
|
||||
def wait_for_http_success(url):
|
||||
router.wait_until_succeeds(
|
||||
f"curl -fsS --max-time 5 {shlex.quote(url)} | grep -F 'fiberlb smoke backend'"
|
||||
)
|
||||
|
||||
start_all()
|
||||
serial_stdout_off()
|
||||
|
||||
|
|
@ -307,7 +351,9 @@ in
|
|||
)
|
||||
loadbalancer = lb_response["loadbalancer"]
|
||||
lb_id = loadbalancer["id"]
|
||||
vip_prefix = f"{loadbalancer['vipAddress']}/32"
|
||||
vip = loadbalancer["vipAddress"]
|
||||
vip_prefix = f"{vip}/32"
|
||||
listener_url = f"http://{vip}:18080/"
|
||||
|
||||
pool_id = grpcurl_json(
|
||||
lb,
|
||||
|
|
@ -363,16 +409,50 @@ in
|
|||
headers=[f"authorization: Bearer {token}"],
|
||||
)
|
||||
|
||||
grpcurl_json(
|
||||
lb,
|
||||
"127.0.0.1:50085",
|
||||
FIBERLB_PROTO_DIR,
|
||||
FIBERLB_PROTO,
|
||||
"fiberlb.v1.ListenerService/CreateListener",
|
||||
{
|
||||
"name": "bgp-smoke-listener",
|
||||
"loadbalancerId": lb_id,
|
||||
"protocol": "LISTENER_PROTOCOL_TCP",
|
||||
"port": 18080,
|
||||
"defaultPoolId": pool_id,
|
||||
},
|
||||
headers=[f"authorization: Bearer {token}"],
|
||||
)
|
||||
|
||||
wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
|
||||
wait_for_local_vip(vip, True)
|
||||
wait_for_route(vip_prefix, True)
|
||||
router.succeed(f"ip route replace {shlex.quote(vip_prefix)} via 192.168.100.2 dev eth1")
|
||||
wait_for_http_success(listener_url)
|
||||
|
||||
router.succeed("systemctl restart gobgpd-peer.service")
|
||||
router.wait_for_unit("gobgpd-peer.service")
|
||||
router.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2")
|
||||
wait_for_route(vip_prefix, True)
|
||||
wait_for_http_success(listener_url)
|
||||
|
||||
lb.succeed("systemctl stop mock-backend.service")
|
||||
wait_for_backend_status("BACKEND_STATUS_OFFLINE", backend_id, token)
|
||||
wait_for_route(vip_prefix, False)
|
||||
wait_for_local_vip(vip, False)
|
||||
router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")
|
||||
|
||||
lb.succeed("systemctl start mock-backend.service")
|
||||
lb.wait_for_unit("mock-backend.service")
|
||||
wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
|
||||
wait_for_local_vip(vip, True)
|
||||
wait_for_route(vip_prefix, True)
|
||||
wait_for_http_success(listener_url)
|
||||
|
||||
lb.succeed("systemctl stop fiberlb.service")
|
||||
wait_for_local_vip(vip, False)
|
||||
wait_for_route(vip_prefix, False)
|
||||
router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")
|
||||
'';
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue