Harden FiberLB native BGP control plane
This commit is contained in:
parent
ce4bab07d6
commit
63c7251756
8 changed files with 551 additions and 63 deletions
|
|
@ -79,6 +79,10 @@ pub struct ServerConfig {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub vip_advertisement: VipAdvertisementConfig,
|
pub vip_advertisement: VipAdvertisementConfig,
|
||||||
|
|
||||||
|
/// Local VIP ownership configuration.
|
||||||
|
#[serde(default)]
|
||||||
|
pub vip_ownership: VipOwnershipConfig,
|
||||||
|
|
||||||
/// Native BGP speaker configuration
|
/// Native BGP speaker configuration
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub bgp: BgpConfig,
|
pub bgp: BgpConfig,
|
||||||
|
|
@ -145,6 +149,31 @@ impl Default for VipAdvertisementConfig {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Local VIP ownership runtime configuration.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct VipOwnershipConfig {
|
||||||
|
/// Whether FiberLB should claim VIP /32 addresses on the local node.
|
||||||
|
#[serde(default)]
|
||||||
|
pub enabled: bool,
|
||||||
|
|
||||||
|
/// Interface used for local VIP ownership.
|
||||||
|
#[serde(default = "default_vip_ownership_interface")]
|
||||||
|
pub interface: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_vip_ownership_interface() -> String {
|
||||||
|
"lo".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for VipOwnershipConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
enabled: false,
|
||||||
|
interface: default_vip_ownership_interface(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Static BGP peer configuration.
|
/// Static BGP peer configuration.
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct BgpPeerConfig {
|
pub struct BgpPeerConfig {
|
||||||
|
|
@ -272,6 +301,7 @@ impl Default for ServerConfig {
|
||||||
auth: AuthConfig::default(),
|
auth: AuthConfig::default(),
|
||||||
health: HealthRuntimeConfig::default(),
|
health: HealthRuntimeConfig::default(),
|
||||||
vip_advertisement: VipAdvertisementConfig::default(),
|
vip_advertisement: VipAdvertisementConfig::default(),
|
||||||
|
vip_ownership: VipOwnershipConfig::default(),
|
||||||
bgp: BgpConfig::default(),
|
bgp: BgpConfig::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@ pub mod metadata;
|
||||||
pub mod services;
|
pub mod services;
|
||||||
pub mod tls;
|
pub mod tls;
|
||||||
pub mod vip_manager;
|
pub mod vip_manager;
|
||||||
|
pub mod vip_owner;
|
||||||
|
|
||||||
pub use bgp_client::{create_bgp_client, BgpClient, BgpError, NativeBgpSpeaker};
|
pub use bgp_client::{create_bgp_client, BgpClient, BgpError, NativeBgpSpeaker};
|
||||||
pub use config::ServerConfig;
|
pub use config::ServerConfig;
|
||||||
|
|
@ -23,3 +24,4 @@ pub use metadata::LbMetadataStore;
|
||||||
pub use services::*;
|
pub use services::*;
|
||||||
pub use tls::{build_tls_config, CertificateStore, SniCertResolver};
|
pub use tls::{build_tls_config, CertificateStore, SniCertResolver};
|
||||||
pub use vip_manager::VipManager;
|
pub use vip_manager::VipManager;
|
||||||
|
pub use vip_owner::{KernelVipAddressOwner, VipAddressOwner, VipOwnershipError};
|
||||||
|
|
|
||||||
|
|
@ -15,9 +15,9 @@ use fiberlb_api::{
|
||||||
};
|
};
|
||||||
use fiberlb_server::{
|
use fiberlb_server::{
|
||||||
config::MetadataBackend, create_bgp_client, spawn_health_checker, BackendServiceImpl,
|
config::MetadataBackend, create_bgp_client, spawn_health_checker, BackendServiceImpl,
|
||||||
CertificateServiceImpl, DataPlane, HealthCheckServiceImpl, L7DataPlane, L7PolicyServiceImpl,
|
CertificateServiceImpl, DataPlane, HealthCheckServiceImpl, KernelVipAddressOwner, L7DataPlane,
|
||||||
L7RuleServiceImpl, LbMetadataStore, ListenerServiceImpl, LoadBalancerServiceImpl,
|
L7PolicyServiceImpl, L7RuleServiceImpl, LbMetadataStore, ListenerServiceImpl,
|
||||||
PoolServiceImpl, ServerConfig, VipManager,
|
LoadBalancerServiceImpl, PoolServiceImpl, ServerConfig, VipAddressOwner, VipManager,
|
||||||
};
|
};
|
||||||
use iam_service_auth::AuthService;
|
use iam_service_auth::AuthService;
|
||||||
use metrics_exporter_prometheus::PrometheusBuilder;
|
use metrics_exporter_prometheus::PrometheusBuilder;
|
||||||
|
|
@ -248,7 +248,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let bgp = create_bgp_client(config.bgp.clone()).await?;
|
let bgp = create_bgp_client(config.bgp.clone()).await?;
|
||||||
let manager = Arc::new(VipManager::new(bgp, metadata.clone(), next_hop));
|
let vip_owner: Option<Arc<dyn VipAddressOwner>> = if config.vip_ownership.enabled {
|
||||||
|
Some(Arc::new(KernelVipAddressOwner::new(
|
||||||
|
config.vip_ownership.interface.clone(),
|
||||||
|
)))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let manager = Arc::new(VipManager::new(bgp, metadata.clone(), next_hop, vip_owner));
|
||||||
let _vip_task = manager.clone().spawn(Duration::from_secs(
|
let _vip_task = manager.clone().spawn(Duration::from_secs(
|
||||||
config.vip_advertisement.interval_secs.max(1),
|
config.vip_advertisement.interval_secs.max(1),
|
||||||
));
|
));
|
||||||
|
|
@ -352,7 +359,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
CertificateServiceServer::new(certificate_service),
|
CertificateServiceServer::new(certificate_service),
|
||||||
make_interceptor(auth_service.clone()),
|
make_interceptor(auth_service.clone()),
|
||||||
))
|
))
|
||||||
.serve(grpc_addr)
|
.serve_with_shutdown(grpc_addr, async {
|
||||||
|
if let Err(error) = wait_for_shutdown_signal().await {
|
||||||
|
tracing::warn!(error = %error, "FiberLB shutdown signal handler failed");
|
||||||
|
}
|
||||||
|
})
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let _ = health_shutdown_tx.send(true);
|
let _ = health_shutdown_tx.send(true);
|
||||||
|
|
@ -367,6 +378,27 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn wait_for_shutdown_signal() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
use tokio::signal::unix::{signal, SignalKind};
|
||||||
|
|
||||||
|
let mut terminate = signal(SignalKind::terminate())?;
|
||||||
|
tokio::select! {
|
||||||
|
_ = tokio::signal::ctrl_c() => {}
|
||||||
|
_ = terminate.recv() => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
{
|
||||||
|
tokio::signal::ctrl_c().await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
tracing::info!("FiberLB shutdown signal received");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
|
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
|
||||||
match value.trim().to_ascii_lowercase().as_str() {
|
match value.trim().to_ascii_lowercase().as_str() {
|
||||||
"flaredb" => Ok(MetadataBackend::FlareDb),
|
"flaredb" => Ok(MetadataBackend::FlareDb),
|
||||||
|
|
|
||||||
|
|
@ -14,15 +14,22 @@ use tracing::{debug, error, info, warn};
|
||||||
|
|
||||||
use crate::bgp_client::BgpClient;
|
use crate::bgp_client::BgpClient;
|
||||||
use crate::metadata::LbMetadataStore;
|
use crate::metadata::LbMetadataStore;
|
||||||
|
use crate::vip_owner::VipAddressOwner;
|
||||||
use fiberlb_types::LoadBalancerId;
|
use fiberlb_types::LoadBalancerId;
|
||||||
|
|
||||||
/// VIP advertisement state
|
/// Current local control-plane state for a VIP.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||||
enum VipState {
|
struct VipState {
|
||||||
/// VIP is advertised to BGP peers
|
/// The VIP is configured on the local node.
|
||||||
Advertised,
|
owned: bool,
|
||||||
/// VIP is withdrawn from BGP peers
|
/// The VIP is advertised to BGP peers.
|
||||||
Withdrawn,
|
advertised: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VipState {
|
||||||
|
fn is_idle(self) -> bool {
|
||||||
|
!self.owned && !self.advertised
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// VIP Manager
|
/// VIP Manager
|
||||||
|
|
@ -37,17 +44,25 @@ pub struct VipManager {
|
||||||
metadata: Arc<LbMetadataStore>,
|
metadata: Arc<LbMetadataStore>,
|
||||||
/// Current VIP advertisement state
|
/// Current VIP advertisement state
|
||||||
vip_state: Arc<RwLock<HashMap<IpAddr, VipState>>>,
|
vip_state: Arc<RwLock<HashMap<IpAddr, VipState>>>,
|
||||||
|
/// Optional local VIP owner.
|
||||||
|
vip_owner: Option<Arc<dyn VipAddressOwner>>,
|
||||||
/// Router's own IP address (used as BGP next hop)
|
/// Router's own IP address (used as BGP next hop)
|
||||||
next_hop: IpAddr,
|
next_hop: IpAddr,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl VipManager {
|
impl VipManager {
|
||||||
/// Create a new VIP manager
|
/// Create a new VIP manager
|
||||||
pub fn new(bgp: Arc<dyn BgpClient>, metadata: Arc<LbMetadataStore>, next_hop: IpAddr) -> Self {
|
pub fn new(
|
||||||
|
bgp: Arc<dyn BgpClient>,
|
||||||
|
metadata: Arc<LbMetadataStore>,
|
||||||
|
next_hop: IpAddr,
|
||||||
|
vip_owner: Option<Arc<dyn VipAddressOwner>>,
|
||||||
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
bgp,
|
bgp,
|
||||||
metadata,
|
metadata,
|
||||||
vip_state: Arc::new(RwLock::new(HashMap::new())),
|
vip_state: Arc::new(RwLock::new(HashMap::new())),
|
||||||
|
vip_owner,
|
||||||
next_hop,
|
next_hop,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -148,39 +163,72 @@ impl VipManager {
|
||||||
) -> Result<(), Box<dyn std::error::Error>> {
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let mut state = self.vip_state.write().await;
|
let mut state = self.vip_state.write().await;
|
||||||
|
|
||||||
// Find VIPs to announce (active but not yet advertised)
|
|
||||||
for vip in active_vips {
|
for vip in active_vips {
|
||||||
match state.get(vip) {
|
let mut vip_state = state.get(vip).copied().unwrap_or_default();
|
||||||
Some(VipState::Advertised) => {
|
let mut changed = false;
|
||||||
// Already advertised, nothing to do
|
|
||||||
|
if !vip_state.owned {
|
||||||
|
if let Some(vip_owner) = &self.vip_owner {
|
||||||
|
info!("Claiming local VIP {} on this node", vip);
|
||||||
|
if let Err(error) = vip_owner.ensure_present(*vip).await {
|
||||||
|
error!("Failed to claim local VIP {}: {}", vip, error);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
vip_state.owned = true;
|
||||||
|
changed = true;
|
||||||
|
} else {
|
||||||
|
debug!("VIP {} already locally owned", vip);
|
||||||
|
}
|
||||||
|
|
||||||
|
if !vip_state.advertised {
|
||||||
|
info!("Announcing VIP {} (healthy backends available)", vip);
|
||||||
|
if let Err(error) = self.bgp.announce_route(*vip, self.next_hop).await {
|
||||||
|
error!("Failed to announce VIP {}: {}", vip, error);
|
||||||
|
} else {
|
||||||
|
vip_state.advertised = true;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
debug!("VIP {} already advertised", vip);
|
debug!("VIP {} already advertised", vip);
|
||||||
}
|
}
|
||||||
Some(VipState::Withdrawn) | None => {
|
|
||||||
// Need to announce
|
if changed {
|
||||||
info!("Announcing VIP {} (healthy backends available)", vip);
|
state.insert(*vip, vip_state);
|
||||||
if let Err(e) = self.bgp.announce_route(*vip, self.next_hop).await {
|
|
||||||
error!("Failed to announce VIP {}: {}", vip, e);
|
|
||||||
} else {
|
|
||||||
state.insert(*vip, VipState::Advertised);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find VIPs to withdraw (advertised but no longer active)
|
let managed_vips: Vec<IpAddr> = state.keys().copied().collect();
|
||||||
let advertised_vips: Vec<IpAddr> = state
|
for vip in managed_vips {
|
||||||
.iter()
|
|
||||||
.filter(|(_, &state)| state == VipState::Advertised)
|
|
||||||
.map(|(vip, _)| *vip)
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
for vip in advertised_vips {
|
|
||||||
if !active_vips.contains(&vip) {
|
if !active_vips.contains(&vip) {
|
||||||
info!("Withdrawing VIP {} (no healthy backends)", vip);
|
let mut vip_state = state.get(&vip).copied().unwrap_or_default();
|
||||||
if let Err(e) = self.bgp.withdraw_route(vip).await {
|
|
||||||
error!("Failed to withdraw VIP {}: {}", vip, e);
|
if vip_state.owned {
|
||||||
|
if let Some(vip_owner) = &self.vip_owner {
|
||||||
|
info!("Releasing local VIP {} (no healthy backends)", vip);
|
||||||
|
if let Err(error) = vip_owner.ensure_absent(vip).await {
|
||||||
|
error!("Failed to release local VIP {}: {}", vip, error);
|
||||||
} else {
|
} else {
|
||||||
state.insert(vip, VipState::Withdrawn);
|
vip_state.owned = false;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
vip_state.owned = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if vip_state.advertised {
|
||||||
|
info!("Withdrawing VIP {} (no healthy backends)", vip);
|
||||||
|
if let Err(error) = self.bgp.withdraw_route(vip).await {
|
||||||
|
error!("Failed to withdraw VIP {}: {}", vip, error);
|
||||||
|
} else {
|
||||||
|
vip_state.advertised = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if vip_state.is_idle() {
|
||||||
|
state.remove(&vip);
|
||||||
|
} else {
|
||||||
|
state.insert(vip, vip_state);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -190,22 +238,52 @@ impl VipManager {
|
||||||
|
|
||||||
/// Manually advertise a VIP (for testing or manual control)
|
/// Manually advertise a VIP (for testing or manual control)
|
||||||
pub async fn advertise_vip(&self, vip: IpAddr) -> Result<(), Box<dyn std::error::Error>> {
|
pub async fn advertise_vip(&self, vip: IpAddr) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let mut state = self.vip_state.write().await;
|
||||||
|
let mut vip_state = state.get(&vip).copied().unwrap_or_default();
|
||||||
|
|
||||||
|
if !vip_state.owned {
|
||||||
|
if let Some(vip_owner) = &self.vip_owner {
|
||||||
|
info!("Claiming local VIP {} on this node", vip);
|
||||||
|
vip_owner.ensure_present(vip).await?;
|
||||||
|
}
|
||||||
|
vip_state.owned = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if !vip_state.advertised {
|
||||||
info!("Manually advertising VIP {}", vip);
|
info!("Manually advertising VIP {}", vip);
|
||||||
self.bgp.announce_route(vip, self.next_hop).await?;
|
self.bgp.announce_route(vip, self.next_hop).await?;
|
||||||
|
vip_state.advertised = true;
|
||||||
|
}
|
||||||
|
|
||||||
let mut state = self.vip_state.write().await;
|
state.insert(vip, vip_state);
|
||||||
state.insert(vip, VipState::Advertised);
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Manually withdraw a VIP (for testing or manual control)
|
/// Manually withdraw a VIP (for testing or manual control)
|
||||||
pub async fn withdraw_vip(&self, vip: IpAddr) -> Result<(), Box<dyn std::error::Error>> {
|
pub async fn withdraw_vip(&self, vip: IpAddr) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let mut state = self.vip_state.write().await;
|
||||||
|
let mut vip_state = state.get(&vip).copied().unwrap_or_default();
|
||||||
|
|
||||||
|
if vip_state.owned {
|
||||||
|
if let Some(vip_owner) = &self.vip_owner {
|
||||||
|
info!("Releasing local VIP {} on this node", vip);
|
||||||
|
vip_owner.ensure_absent(vip).await?;
|
||||||
|
}
|
||||||
|
vip_state.owned = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if vip_state.advertised {
|
||||||
info!("Manually withdrawing VIP {}", vip);
|
info!("Manually withdrawing VIP {}", vip);
|
||||||
self.bgp.withdraw_route(vip).await?;
|
self.bgp.withdraw_route(vip).await?;
|
||||||
|
vip_state.advertised = false;
|
||||||
|
}
|
||||||
|
|
||||||
let mut state = self.vip_state.write().await;
|
if vip_state.is_idle() {
|
||||||
state.insert(vip, VipState::Withdrawn);
|
state.remove(&vip);
|
||||||
|
} else {
|
||||||
|
state.insert(vip, vip_state);
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
@ -216,19 +294,41 @@ impl VipManager {
|
||||||
pub async fn shutdown(&self) -> Result<(), Box<dyn std::error::Error>> {
|
pub async fn shutdown(&self) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
info!("VIP manager shutting down, withdrawing all VIPs...");
|
info!("VIP manager shutting down, withdrawing all VIPs...");
|
||||||
|
|
||||||
let state = self.vip_state.read().await;
|
let mut state = self.vip_state.write().await;
|
||||||
let advertised_vips: Vec<IpAddr> = state
|
let managed_vips: Vec<IpAddr> = state.keys().copied().collect();
|
||||||
.iter()
|
|
||||||
.filter(|(_, &state)| state == VipState::Advertised)
|
|
||||||
.map(|(vip, _)| *vip)
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
drop(state); // Release read lock before withdrawing
|
for vip in managed_vips {
|
||||||
|
let mut vip_state = state.get(&vip).copied().unwrap_or_default();
|
||||||
|
|
||||||
for vip in advertised_vips {
|
if vip_state.owned {
|
||||||
|
info!("Releasing local VIP {} for shutdown", vip);
|
||||||
|
if let Some(vip_owner) = &self.vip_owner {
|
||||||
|
if let Err(error) = vip_owner.ensure_absent(vip).await {
|
||||||
|
error!(
|
||||||
|
"Failed to release local VIP {} during shutdown: {}",
|
||||||
|
vip, error
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
vip_state.owned = false;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
vip_state.owned = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if vip_state.advertised {
|
||||||
info!("Withdrawing VIP {} for shutdown", vip);
|
info!("Withdrawing VIP {} for shutdown", vip);
|
||||||
if let Err(e) = self.bgp.withdraw_route(vip).await {
|
if let Err(error) = self.bgp.withdraw_route(vip).await {
|
||||||
error!("Failed to withdraw VIP {} during shutdown: {}", vip, e);
|
error!("Failed to withdraw VIP {} during shutdown: {}", vip, error);
|
||||||
|
} else {
|
||||||
|
vip_state.advertised = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if vip_state.is_idle() {
|
||||||
|
state.remove(&vip);
|
||||||
|
} else {
|
||||||
|
state.insert(vip, vip_state);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -241,7 +341,7 @@ impl VipManager {
|
||||||
let state = self.vip_state.read().await;
|
let state = self.vip_state.read().await;
|
||||||
state
|
state
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|(_, &state)| state == VipState::Advertised)
|
.filter(|(_, state)| state.advertised)
|
||||||
.map(|(vip, _)| *vip)
|
.map(|(vip, _)| *vip)
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
@ -251,19 +351,22 @@ impl VipManager {
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::bgp_client::{BgpClient, Result};
|
use crate::bgp_client::{BgpClient, Result};
|
||||||
|
use crate::vip_owner::VipOwnershipError;
|
||||||
use std::sync::Mutex;
|
use std::sync::Mutex;
|
||||||
|
|
||||||
/// Mock BGP client for testing
|
/// Mock BGP client for testing
|
||||||
struct MockBgpClient {
|
struct MockBgpClient {
|
||||||
announced: Arc<Mutex<HashSet<IpAddr>>>,
|
announced: Arc<Mutex<HashSet<IpAddr>>>,
|
||||||
withdrawn: Arc<Mutex<HashSet<IpAddr>>>,
|
withdrawn: Arc<Mutex<HashSet<IpAddr>>>,
|
||||||
|
events: Arc<Mutex<Vec<String>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MockBgpClient {
|
impl MockBgpClient {
|
||||||
fn new() -> Self {
|
fn new(events: Arc<Mutex<Vec<String>>>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
announced: Arc::new(Mutex::new(HashSet::new())),
|
announced: Arc::new(Mutex::new(HashSet::new())),
|
||||||
withdrawn: Arc::new(Mutex::new(HashSet::new())),
|
withdrawn: Arc::new(Mutex::new(HashSet::new())),
|
||||||
|
events,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -271,11 +374,19 @@ mod tests {
|
||||||
#[tonic::async_trait]
|
#[tonic::async_trait]
|
||||||
impl BgpClient for MockBgpClient {
|
impl BgpClient for MockBgpClient {
|
||||||
async fn announce_route(&self, prefix: IpAddr, _next_hop: IpAddr) -> Result<()> {
|
async fn announce_route(&self, prefix: IpAddr, _next_hop: IpAddr) -> Result<()> {
|
||||||
|
self.events
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push(format!("announce:{prefix}"));
|
||||||
self.announced.lock().unwrap().insert(prefix);
|
self.announced.lock().unwrap().insert(prefix);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn withdraw_route(&self, prefix: IpAddr) -> Result<()> {
|
async fn withdraw_route(&self, prefix: IpAddr) -> Result<()> {
|
||||||
|
self.events
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push(format!("withdraw:{prefix}"));
|
||||||
self.withdrawn.lock().unwrap().insert(prefix);
|
self.withdrawn.lock().unwrap().insert(prefix);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
@ -285,13 +396,51 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct MockVipOwner {
|
||||||
|
owned: Arc<Mutex<HashSet<IpAddr>>>,
|
||||||
|
released: Arc<Mutex<HashSet<IpAddr>>>,
|
||||||
|
events: Arc<Mutex<Vec<String>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MockVipOwner {
|
||||||
|
fn new(events: Arc<Mutex<Vec<String>>>) -> Self {
|
||||||
|
Self {
|
||||||
|
owned: Arc::new(Mutex::new(HashSet::new())),
|
||||||
|
released: Arc::new(Mutex::new(HashSet::new())),
|
||||||
|
events,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tonic::async_trait]
|
||||||
|
impl VipAddressOwner for MockVipOwner {
|
||||||
|
async fn ensure_present(&self, vip: IpAddr) -> std::result::Result<(), VipOwnershipError> {
|
||||||
|
self.events.lock().unwrap().push(format!("own:{vip}"));
|
||||||
|
self.owned.lock().unwrap().insert(vip);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ensure_absent(&self, vip: IpAddr) -> std::result::Result<(), VipOwnershipError> {
|
||||||
|
self.events.lock().unwrap().push(format!("unown:{vip}"));
|
||||||
|
self.released.lock().unwrap().insert(vip);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_vip_advertisement_tracking() {
|
async fn test_vip_advertisement_tracking() {
|
||||||
let mock_bgp = Arc::new(MockBgpClient::new());
|
let events = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
let mock_bgp = Arc::new(MockBgpClient::new(events.clone()));
|
||||||
|
let mock_owner = Arc::new(MockVipOwner::new(events.clone()));
|
||||||
let metadata = Arc::new(LbMetadataStore::new_in_memory());
|
let metadata = Arc::new(LbMetadataStore::new_in_memory());
|
||||||
let next_hop = "10.0.0.1".parse().unwrap();
|
let next_hop = "10.0.0.1".parse().unwrap();
|
||||||
|
|
||||||
let manager = VipManager::new(mock_bgp.clone(), metadata, next_hop);
|
let manager = VipManager::new(
|
||||||
|
mock_bgp.clone(),
|
||||||
|
metadata,
|
||||||
|
next_hop,
|
||||||
|
Some(mock_owner.clone()),
|
||||||
|
);
|
||||||
|
|
||||||
let vip: IpAddr = "10.0.1.100".parse().unwrap();
|
let vip: IpAddr = "10.0.1.100".parse().unwrap();
|
||||||
|
|
||||||
|
|
@ -304,5 +453,16 @@ mod tests {
|
||||||
manager.withdraw_vip(vip).await.unwrap();
|
manager.withdraw_vip(vip).await.unwrap();
|
||||||
assert!(manager.get_advertised_vips().await.is_empty());
|
assert!(manager.get_advertised_vips().await.is_empty());
|
||||||
assert!(mock_bgp.withdrawn.lock().unwrap().contains(&vip));
|
assert!(mock_bgp.withdrawn.lock().unwrap().contains(&vip));
|
||||||
|
assert!(mock_owner.released.lock().unwrap().contains(&vip));
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
events.lock().unwrap().clone(),
|
||||||
|
vec![
|
||||||
|
format!("own:{vip}"),
|
||||||
|
format!("announce:{vip}"),
|
||||||
|
format!("unown:{vip}"),
|
||||||
|
format!("withdraw:{vip}"),
|
||||||
|
]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
161
fiberlb/crates/fiberlb-server/src/vip_owner.rs
Normal file
161
fiberlb/crates/fiberlb-server/src/vip_owner.rs
Normal file
|
|
@ -0,0 +1,161 @@
|
||||||
|
//! Local VIP ownership management for FiberLB.
|
||||||
|
//!
|
||||||
|
//! This module claims and releases VIP `/32` addresses on a local interface so
|
||||||
|
//! the kernel actually treats the advertised VIP as a local destination.
|
||||||
|
|
||||||
|
use std::net::{IpAddr, Ipv4Addr};
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use tokio::process::Command;
|
||||||
|
|
||||||
|
/// Result type for VIP ownership operations.
|
||||||
|
pub type Result<T> = std::result::Result<T, VipOwnershipError>;
|
||||||
|
|
||||||
|
/// VIP ownership errors.
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum VipOwnershipError {
|
||||||
|
#[error("I/O error: {0}")]
|
||||||
|
Io(#[from] std::io::Error),
|
||||||
|
#[error("kernel VIP ownership command failed: {0}")]
|
||||||
|
CommandFailed(String),
|
||||||
|
#[error("unsupported VIP ownership operation: {0}")]
|
||||||
|
Unsupported(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Trait for local VIP ownership.
|
||||||
|
#[tonic::async_trait]
|
||||||
|
pub trait VipAddressOwner: Send + Sync {
|
||||||
|
/// Ensure a VIP is locally owned on this node.
|
||||||
|
async fn ensure_present(&self, vip: IpAddr) -> Result<()>;
|
||||||
|
|
||||||
|
/// Ensure a VIP is no longer locally owned on this node.
|
||||||
|
async fn ensure_absent(&self, vip: IpAddr) -> Result<()>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Linux kernel-backed VIP owner using `ip addr add/del`.
|
||||||
|
pub struct KernelVipAddressOwner {
|
||||||
|
interface: String,
|
||||||
|
ip_command: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl KernelVipAddressOwner {
|
||||||
|
/// Create a kernel-backed VIP owner for the given interface.
|
||||||
|
pub fn new(interface: impl Into<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
interface: interface.into(),
|
||||||
|
ip_command: resolve_ip_command(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_ip(&self, args: &[String]) -> Result<()> {
|
||||||
|
let output = Command::new(&self.ip_command).args(args).output().await?;
|
||||||
|
if output.status.success() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(VipOwnershipError::CommandFailed(format!(
|
||||||
|
"{} {}: {}",
|
||||||
|
self.ip_command,
|
||||||
|
args.join(" "),
|
||||||
|
render_command_output(&output)
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn add_v4(&self, vip: Ipv4Addr) -> Result<()> {
|
||||||
|
let args = vec![
|
||||||
|
"-4".to_string(),
|
||||||
|
"addr".to_string(),
|
||||||
|
"add".to_string(),
|
||||||
|
format!("{vip}/32"),
|
||||||
|
"dev".to_string(),
|
||||||
|
self.interface.clone(),
|
||||||
|
];
|
||||||
|
|
||||||
|
match self.run_ip(&args).await {
|
||||||
|
Ok(()) => Ok(()),
|
||||||
|
Err(VipOwnershipError::CommandFailed(message)) if message.contains("File exists") => {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Err(error) => Err(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn del_v4(&self, vip: Ipv4Addr) -> Result<()> {
|
||||||
|
let args = vec![
|
||||||
|
"-4".to_string(),
|
||||||
|
"addr".to_string(),
|
||||||
|
"del".to_string(),
|
||||||
|
format!("{vip}/32"),
|
||||||
|
"dev".to_string(),
|
||||||
|
self.interface.clone(),
|
||||||
|
];
|
||||||
|
|
||||||
|
match self.run_ip(&args).await {
|
||||||
|
Ok(()) => Ok(()),
|
||||||
|
Err(VipOwnershipError::CommandFailed(message))
|
||||||
|
if message.contains("Cannot assign requested address") =>
|
||||||
|
{
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Err(error) => Err(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tonic::async_trait]
|
||||||
|
impl VipAddressOwner for KernelVipAddressOwner {
|
||||||
|
async fn ensure_present(&self, vip: IpAddr) -> Result<()> {
|
||||||
|
match vip {
|
||||||
|
IpAddr::V4(vip) => self.add_v4(vip).await,
|
||||||
|
IpAddr::V6(_) => Err(VipOwnershipError::Unsupported(format!(
|
||||||
|
"VIP ownership currently supports IPv4 only (got {vip})"
|
||||||
|
))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ensure_absent(&self, vip: IpAddr) -> Result<()> {
|
||||||
|
match vip {
|
||||||
|
IpAddr::V4(vip) => self.del_v4(vip).await,
|
||||||
|
IpAddr::V6(_) => Err(VipOwnershipError::Unsupported(format!(
|
||||||
|
"VIP ownership currently supports IPv4 only (got {vip})"
|
||||||
|
))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_ip_command() -> String {
|
||||||
|
if let Ok(path) = std::env::var("FIBERLB_IP_COMMAND") {
|
||||||
|
let trimmed = path.trim();
|
||||||
|
if !trimmed.is_empty() {
|
||||||
|
return trimmed.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for candidate in [
|
||||||
|
"/run/current-system/sw/bin/ip",
|
||||||
|
"/usr/sbin/ip",
|
||||||
|
"/sbin/ip",
|
||||||
|
"/usr/bin/ip",
|
||||||
|
"/bin/ip",
|
||||||
|
] {
|
||||||
|
if Path::new(candidate).exists() {
|
||||||
|
return candidate.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
"ip".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn render_command_output(output: &std::process::Output) -> String {
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
|
||||||
|
if !stderr.is_empty() {
|
||||||
|
return stderr;
|
||||||
|
}
|
||||||
|
|
||||||
|
let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||||
|
if !stdout.is_empty() {
|
||||||
|
return stdout;
|
||||||
|
}
|
||||||
|
|
||||||
|
format!("exit status {}", output.status)
|
||||||
|
}
|
||||||
|
|
@ -47,6 +47,10 @@ let
|
||||||
vip_advertisement = {
|
vip_advertisement = {
|
||||||
interval_secs = cfg.vipCheckIntervalSecs;
|
interval_secs = cfg.vipCheckIntervalSecs;
|
||||||
};
|
};
|
||||||
|
vip_ownership = {
|
||||||
|
enabled = cfg.vipOwnership.enable;
|
||||||
|
interface = cfg.vipOwnership.interface;
|
||||||
|
};
|
||||||
} // lib.optionalAttrs cfg.bgp.enable {
|
} // lib.optionalAttrs cfg.bgp.enable {
|
||||||
bgp =
|
bgp =
|
||||||
{
|
{
|
||||||
|
|
@ -155,6 +159,16 @@ in
|
||||||
description = "Interval between FiberLB VIP-to-BGP reconciliation sweeps.";
|
description = "Interval between FiberLB VIP-to-BGP reconciliation sweeps.";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
vipOwnership = {
|
||||||
|
enable = lib.mkEnableOption "FiberLB local VIP ownership";
|
||||||
|
|
||||||
|
interface = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "lo";
|
||||||
|
description = "Interface where FiberLB should claim VIP /32 addresses.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
bgp = {
|
bgp = {
|
||||||
enable = lib.mkEnableOption "FiberLB native BGP VIP advertisement";
|
enable = lib.mkEnableOption "FiberLB native BGP VIP advertisement";
|
||||||
|
|
||||||
|
|
@ -240,6 +254,10 @@ in
|
||||||
assertion = (!cfg.bgp.enable) || ((builtins.length cfg.bgp.peers) > 0);
|
assertion = (!cfg.bgp.enable) || ((builtins.length cfg.bgp.peers) > 0);
|
||||||
message = "services.fiberlb.bgp.peers must contain at least one peer when native BGP is enabled";
|
message = "services.fiberlb.bgp.peers must contain at least one peer when native BGP is enabled";
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
assertion = (!cfg.vipOwnership.enable) || cfg.bgp.enable;
|
||||||
|
message = "services.fiberlb.vipOwnership.enable requires services.fiberlb.bgp.enable";
|
||||||
|
}
|
||||||
];
|
];
|
||||||
|
|
||||||
# Create system user
|
# Create system user
|
||||||
|
|
@ -258,6 +276,7 @@ in
|
||||||
wantedBy = [ "multi-user.target" ];
|
wantedBy = [ "multi-user.target" ];
|
||||||
after = [ "network.target" "iam.service" ] ++ flaredbDependencies;
|
after = [ "network.target" "iam.service" ] ++ flaredbDependencies;
|
||||||
requires = [ "iam.service" ] ++ flaredbDependencies;
|
requires = [ "iam.service" ] ++ flaredbDependencies;
|
||||||
|
path = lib.optionals cfg.vipOwnership.enable [ pkgs.iproute2 ];
|
||||||
|
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
Type = "simple";
|
Type = "simple";
|
||||||
|
|
@ -276,6 +295,8 @@ in
|
||||||
ProtectSystem = "strict";
|
ProtectSystem = "strict";
|
||||||
ProtectHome = true;
|
ProtectHome = true;
|
||||||
ReadWritePaths = [ cfg.dataDir ];
|
ReadWritePaths = [ cfg.dataDir ];
|
||||||
|
AmbientCapabilities = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
|
||||||
|
CapabilityBoundingSet = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
|
||||||
|
|
||||||
# Environment variables for service endpoints
|
# Environment variables for service endpoints
|
||||||
Environment = [
|
Environment = [
|
||||||
|
|
|
||||||
|
|
@ -99,6 +99,8 @@ in {
|
||||||
if node != null then node.ip else "127.0.0.1";
|
if node != null then node.ip else "127.0.0.1";
|
||||||
peers = cfg.fiberlbBgp.peers;
|
peers = cfg.fiberlbBgp.peers;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
services.fiberlb.vipOwnership.enable = mkDefault true;
|
||||||
})
|
})
|
||||||
|
|
||||||
# PrismNET OVN integration
|
# PrismNET OVN integration
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,25 @@ let
|
||||||
iamProto = "iam.proto";
|
iamProto = "iam.proto";
|
||||||
fiberlbProtoDir = ../../fiberlb/crates/fiberlb-api/proto;
|
fiberlbProtoDir = ../../fiberlb/crates/fiberlb-api/proto;
|
||||||
fiberlbProto = "fiberlb.proto";
|
fiberlbProto = "fiberlb.proto";
|
||||||
|
backendScript = pkgs.writeText "fiberlb-smoke-backend.py" ''
|
||||||
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self):
|
||||||
|
body = b"fiberlb smoke backend\n"
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def log_message(self, format, *args):
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
HTTPServer(("127.0.0.1", 18081), Handler).serve_forever()
|
||||||
|
'';
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
name = "fiberlb-native-bgp-vm-smoke";
|
name = "fiberlb-native-bgp-vm-smoke";
|
||||||
|
|
@ -49,6 +68,7 @@ in
|
||||||
];
|
];
|
||||||
|
|
||||||
environment.systemPackages = with pkgs; [
|
environment.systemPackages = with pkgs; [
|
||||||
|
curl
|
||||||
gobgp
|
gobgp
|
||||||
gobgpd
|
gobgpd
|
||||||
jq
|
jq
|
||||||
|
|
@ -117,6 +137,10 @@ in
|
||||||
healthCheckIntervalSecs = 1;
|
healthCheckIntervalSecs = 1;
|
||||||
healthCheckTimeoutSecs = 1;
|
healthCheckTimeoutSecs = 1;
|
||||||
vipCheckIntervalSecs = 1;
|
vipCheckIntervalSecs = 1;
|
||||||
|
vipOwnership = {
|
||||||
|
enable = true;
|
||||||
|
interface = "lo";
|
||||||
|
};
|
||||||
bgp = {
|
bgp = {
|
||||||
enable = true;
|
enable = true;
|
||||||
localAs = 65010;
|
localAs = 65010;
|
||||||
|
|
@ -141,7 +165,7 @@ in
|
||||||
after = [ "network.target" ];
|
after = [ "network.target" ];
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
Type = "simple";
|
Type = "simple";
|
||||||
ExecStart = "${pkgs.python3}/bin/python -m http.server 18081 --bind 127.0.0.1";
|
ExecStart = "${pkgs.python3}/bin/python ${backendScript}";
|
||||||
Restart = "always";
|
Restart = "always";
|
||||||
RestartSec = "1s";
|
RestartSec = "1s";
|
||||||
};
|
};
|
||||||
|
|
@ -276,6 +300,26 @@ in
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
raise AssertionError(f"route {prefix} still present in GoBGP RIB")
|
raise AssertionError(f"route {prefix} still present in GoBGP RIB")
|
||||||
|
|
||||||
|
def wait_for_local_vip(vip, present):
|
||||||
|
pattern = f"inet {vip}/32"
|
||||||
|
if present:
|
||||||
|
lb.wait_until_succeeds(
|
||||||
|
f"ip -4 addr show dev lo | grep -F {shlex.quote(pattern)}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
deadline = time.time() + 60
|
||||||
|
while time.time() < deadline:
|
||||||
|
output = lb.succeed("ip -4 addr show dev lo || true")
|
||||||
|
if pattern not in output:
|
||||||
|
return
|
||||||
|
time.sleep(1)
|
||||||
|
raise AssertionError(f"VIP {vip} still present on loopback")
|
||||||
|
|
||||||
|
def wait_for_http_success(url):
|
||||||
|
router.wait_until_succeeds(
|
||||||
|
f"curl -fsS --max-time 5 {shlex.quote(url)} | grep -F 'fiberlb smoke backend'"
|
||||||
|
)
|
||||||
|
|
||||||
start_all()
|
start_all()
|
||||||
serial_stdout_off()
|
serial_stdout_off()
|
||||||
|
|
||||||
|
|
@ -307,7 +351,9 @@ in
|
||||||
)
|
)
|
||||||
loadbalancer = lb_response["loadbalancer"]
|
loadbalancer = lb_response["loadbalancer"]
|
||||||
lb_id = loadbalancer["id"]
|
lb_id = loadbalancer["id"]
|
||||||
vip_prefix = f"{loadbalancer['vipAddress']}/32"
|
vip = loadbalancer["vipAddress"]
|
||||||
|
vip_prefix = f"{vip}/32"
|
||||||
|
listener_url = f"http://{vip}:18080/"
|
||||||
|
|
||||||
pool_id = grpcurl_json(
|
pool_id = grpcurl_json(
|
||||||
lb,
|
lb,
|
||||||
|
|
@ -363,16 +409,50 @@ in
|
||||||
headers=[f"authorization: Bearer {token}"],
|
headers=[f"authorization: Bearer {token}"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
grpcurl_json(
|
||||||
|
lb,
|
||||||
|
"127.0.0.1:50085",
|
||||||
|
FIBERLB_PROTO_DIR,
|
||||||
|
FIBERLB_PROTO,
|
||||||
|
"fiberlb.v1.ListenerService/CreateListener",
|
||||||
|
{
|
||||||
|
"name": "bgp-smoke-listener",
|
||||||
|
"loadbalancerId": lb_id,
|
||||||
|
"protocol": "LISTENER_PROTOCOL_TCP",
|
||||||
|
"port": 18080,
|
||||||
|
"defaultPoolId": pool_id,
|
||||||
|
},
|
||||||
|
headers=[f"authorization: Bearer {token}"],
|
||||||
|
)
|
||||||
|
|
||||||
wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
|
wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
|
||||||
|
wait_for_local_vip(vip, True)
|
||||||
wait_for_route(vip_prefix, True)
|
wait_for_route(vip_prefix, True)
|
||||||
|
router.succeed(f"ip route replace {shlex.quote(vip_prefix)} via 192.168.100.2 dev eth1")
|
||||||
|
wait_for_http_success(listener_url)
|
||||||
|
|
||||||
|
router.succeed("systemctl restart gobgpd-peer.service")
|
||||||
|
router.wait_for_unit("gobgpd-peer.service")
|
||||||
|
router.wait_until_succeeds("gobgp -u 127.0.0.1 -p 50051 neighbor | grep -F 192.168.100.2")
|
||||||
|
wait_for_route(vip_prefix, True)
|
||||||
|
wait_for_http_success(listener_url)
|
||||||
|
|
||||||
lb.succeed("systemctl stop mock-backend.service")
|
lb.succeed("systemctl stop mock-backend.service")
|
||||||
wait_for_backend_status("BACKEND_STATUS_OFFLINE", backend_id, token)
|
wait_for_backend_status("BACKEND_STATUS_OFFLINE", backend_id, token)
|
||||||
wait_for_route(vip_prefix, False)
|
wait_for_route(vip_prefix, False)
|
||||||
|
wait_for_local_vip(vip, False)
|
||||||
|
router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")
|
||||||
|
|
||||||
lb.succeed("systemctl start mock-backend.service")
|
lb.succeed("systemctl start mock-backend.service")
|
||||||
lb.wait_for_unit("mock-backend.service")
|
lb.wait_for_unit("mock-backend.service")
|
||||||
wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
|
wait_for_backend_status("BACKEND_STATUS_ONLINE", backend_id, token)
|
||||||
|
wait_for_local_vip(vip, True)
|
||||||
wait_for_route(vip_prefix, True)
|
wait_for_route(vip_prefix, True)
|
||||||
|
wait_for_http_success(listener_url)
|
||||||
|
|
||||||
|
lb.succeed("systemctl stop fiberlb.service")
|
||||||
|
wait_for_local_vip(vip, False)
|
||||||
|
wait_for_route(vip_prefix, False)
|
||||||
|
router.fail(f"curl -fsS --max-time 3 {shlex.quote(listener_url)}")
|
||||||
'';
|
'';
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue