harden plasmavmc image ingestion and internal execution paths

This commit is contained in:
centra 2026-04-02 07:57:25 +09:00
parent 260fb4c576
commit 0745216107
Signed by: centra
GPG key ID: 0C09689D20B25ACA
64 changed files with 3531 additions and 1434 deletions

View file

@ -161,7 +161,9 @@ impl RedfishTarget {
match action { match action {
PowerAction::Cycle => Ok(PowerState::Cycling), PowerAction::Cycle => Ok(PowerState::Cycling),
PowerAction::On | PowerAction::Off | PowerAction::Refresh => self.refresh(&client).await, PowerAction::On | PowerAction::Off | PowerAction::Refresh => {
self.refresh(&client).await
}
} }
} }
@ -295,7 +297,12 @@ pub async fn request_reinstall(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use axum::{extract::State, http::StatusCode, routing::{get, post}, Json, Router}; use axum::{
extract::State,
http::StatusCode,
routing::{get, post},
Json, Router,
};
use serde_json::Value; use serde_json::Value;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use tokio::net::TcpListener; use tokio::net::TcpListener;
@ -303,7 +310,10 @@ mod tests {
#[test] #[test]
fn parse_redfish_short_reference_defaults_to_https() { fn parse_redfish_short_reference_defaults_to_https() {
let parsed = RedfishTarget::parse("redfish://lab-bmc/node01").unwrap(); let parsed = RedfishTarget::parse("redfish://lab-bmc/node01").unwrap();
assert_eq!(parsed.resource_url.as_str(), "https://lab-bmc/redfish/v1/Systems/node01"); assert_eq!(
parsed.resource_url.as_str(),
"https://lab-bmc/redfish/v1/Systems/node01"
);
} }
#[test] #[test]
@ -361,8 +371,14 @@ mod tests {
addr addr
)) ))
.unwrap(); .unwrap();
assert_eq!(target.perform(PowerAction::Refresh).await.unwrap(), PowerState::On); assert_eq!(
assert_eq!(target.perform(PowerAction::Off).await.unwrap(), PowerState::On); target.perform(PowerAction::Refresh).await.unwrap(),
PowerState::On
);
assert_eq!(
target.perform(PowerAction::Off).await.unwrap(),
PowerState::On
);
let payloads = state.seen_payloads.lock().unwrap().clone(); let payloads = state.seen_payloads.lock().unwrap().clone();
assert_eq!(payloads, vec![r#"{"ResetType":"ForceOff"}"#.to_string()]); assert_eq!(payloads, vec![r#"{"ResetType":"ForceOff"}"#.to_string()]);

View file

@ -31,5 +31,3 @@ pub async fn run_deployer_command(endpoint: &str, action: &str) -> Result<()> {
Ok(()) Ok(())
} }

View file

@ -177,6 +177,10 @@ pub struct VipOwnershipConfig {
/// Interface used for local VIP ownership. /// Interface used for local VIP ownership.
#[serde(default = "default_vip_ownership_interface")] #[serde(default = "default_vip_ownership_interface")]
pub interface: String, pub interface: String,
/// Optional explicit `ip` command path used for local VIP ownership.
#[serde(default)]
pub ip_command: Option<String>,
} }
fn default_vip_ownership_interface() -> String { fn default_vip_ownership_interface() -> String {
@ -188,6 +192,7 @@ impl Default for VipOwnershipConfig {
Self { Self {
enabled: false, enabled: false,
interface: default_vip_ownership_interface(), interface: default_vip_ownership_interface(),
ip_command: None,
} }
} }
} }

View file

@ -41,26 +41,6 @@ struct Args {
#[arg(long)] #[arg(long)]
grpc_addr: Option<String>, grpc_addr: Option<String>,
/// ChainFire endpoint for cluster coordination
#[arg(long, env = "FIBERLB_CHAINFIRE_ENDPOINT")]
chainfire_endpoint: Option<String>,
/// FlareDB endpoint for metadata and tenant data storage
#[arg(long, env = "FIBERLB_FLAREDB_ENDPOINT")]
flaredb_endpoint: Option<String>,
/// Metadata backend (flaredb, postgres, sqlite)
#[arg(long, env = "FIBERLB_METADATA_BACKEND")]
metadata_backend: Option<String>,
/// SQL database URL for metadata (required for postgres/sqlite backend)
#[arg(long, env = "FIBERLB_METADATA_DATABASE_URL")]
metadata_database_url: Option<String>,
/// Run in single-node mode (required when metadata backend is SQLite)
#[arg(long, env = "FIBERLB_SINGLE_NODE")]
single_node: bool,
/// Log level (overrides config) /// Log level (overrides config)
#[arg(short, long)] #[arg(short, long)]
log_level: Option<String>, log_level: Option<String>,
@ -93,21 +73,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
if let Some(log_level) = args.log_level { if let Some(log_level) = args.log_level {
config.log_level = log_level; config.log_level = log_level;
} }
if let Some(chainfire_endpoint) = args.chainfire_endpoint {
config.chainfire_endpoint = Some(chainfire_endpoint);
}
if let Some(flaredb_endpoint) = args.flaredb_endpoint {
config.flaredb_endpoint = Some(flaredb_endpoint);
}
if let Some(metadata_backend) = args.metadata_backend {
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
}
if let Some(metadata_database_url) = args.metadata_database_url {
config.metadata_database_url = Some(metadata_database_url);
}
if args.single_node {
config.single_node = true;
}
// Initialize tracing // Initialize tracing
tracing_subscriber::fmt() tracing_subscriber::fmt()
@ -194,12 +159,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
) )
} }
MetadataBackend::Postgres | MetadataBackend::Sqlite => { MetadataBackend::Postgres | MetadataBackend::Sqlite => {
let database_url = config let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
.metadata_database_url
.as_deref()
.ok_or_else(|| {
format!( format!(
"metadata_database_url is required when metadata_backend={} (env: FIBERLB_METADATA_DATABASE_URL)", "metadata_database_url is required when metadata_backend={}",
metadata_backend_name(config.metadata_backend) metadata_backend_name(config.metadata_backend)
) )
})?; })?;
@ -282,8 +244,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
})?; })?;
let bgp = create_bgp_client(config.bgp.clone()).await?; let bgp = create_bgp_client(config.bgp.clone()).await?;
let vip_owner: Option<Arc<dyn VipAddressOwner>> = if config.vip_ownership.enabled { let vip_owner: Option<Arc<dyn VipAddressOwner>> = if config.vip_ownership.enabled {
Some(Arc::new(KernelVipAddressOwner::new( Some(Arc::new(KernelVipAddressOwner::with_ip_command(
config.vip_ownership.interface.clone(), config.vip_ownership.interface.clone(),
config.vip_ownership.ip_command.clone(),
))) )))
} else { } else {
None None
@ -439,19 +402,6 @@ async fn wait_for_shutdown_signal() -> Result<(), Box<dyn std::error::Error>> {
Ok(()) Ok(())
} }
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
match value.trim().to_ascii_lowercase().as_str() {
"flaredb" => Ok(MetadataBackend::FlareDb),
"postgres" => Ok(MetadataBackend::Postgres),
"sqlite" => Ok(MetadataBackend::Sqlite),
other => Err(format!(
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
other
)
.into()),
}
}
fn metadata_backend_name(backend: MetadataBackend) -> &'static str { fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
match backend { match backend {
MetadataBackend::FlareDb => "flaredb", MetadataBackend::FlareDb => "flaredb",

View file

@ -61,12 +61,12 @@ impl LbMetadataStore {
endpoint: Option<String>, endpoint: Option<String>,
pd_endpoint: Option<String>, pd_endpoint: Option<String>,
) -> Result<Self> { ) -> Result<Self> {
let endpoint = endpoint.unwrap_or_else(|| { let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
std::env::var("FIBERLB_FLAREDB_ENDPOINT") Self::connect_flaredb(endpoint, pd_endpoint).await
.unwrap_or_else(|_| "127.0.0.1:2479".to_string()) }
});
async fn connect_flaredb(endpoint: String, pd_endpoint: Option<String>) -> Result<Self> {
let pd_endpoint = pd_endpoint let pd_endpoint = pd_endpoint
.or_else(|| std::env::var("FIBERLB_CHAINFIRE_ENDPOINT").ok())
.map(|value| normalize_transport_addr(&value)) .map(|value| normalize_transport_addr(&value))
.unwrap_or_else(|| endpoint.clone()); .unwrap_or_else(|| endpoint.clone());

View file

@ -41,9 +41,17 @@ pub struct KernelVipAddressOwner {
impl KernelVipAddressOwner { impl KernelVipAddressOwner {
/// Create a kernel-backed VIP owner for the given interface. /// Create a kernel-backed VIP owner for the given interface.
pub fn new(interface: impl Into<String>) -> Self { pub fn new(interface: impl Into<String>) -> Self {
Self::with_ip_command(interface, None::<String>)
}
/// Create a kernel-backed VIP owner with an optional explicit `ip` command path.
pub fn with_ip_command(
interface: impl Into<String>,
ip_command: Option<impl Into<String>>,
) -> Self {
Self { Self {
interface: interface.into(), interface: interface.into(),
ip_command: resolve_ip_command(), ip_command: resolve_ip_command(ip_command.map(Into::into)),
} }
} }
@ -123,7 +131,14 @@ impl VipAddressOwner for KernelVipAddressOwner {
} }
} }
fn resolve_ip_command() -> String { fn resolve_ip_command(configured: Option<String>) -> String {
if let Some(path) = configured {
let trimmed = path.trim();
if !trimmed.is_empty() {
return trimmed.to_string();
}
}
if let Ok(path) = std::env::var("FIBERLB_IP_COMMAND") { if let Ok(path) = std::env::var("FIBERLB_IP_COMMAND") {
let trimmed = path.trim(); let trimmed = path.trim();
if !trimmed.is_empty() { if !trimmed.is_empty() {
@ -159,3 +174,37 @@ fn render_command_output(output: &std::process::Output) -> String {
format!("exit status {}", output.status) format!("exit status {}", output.status)
} }
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{Mutex, OnceLock};
fn env_lock() -> &'static Mutex<()> {
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
LOCK.get_or_init(|| Mutex::new(()))
}
#[test]
fn configured_ip_command_wins_over_env() {
let _guard = env_lock().lock().unwrap();
std::env::set_var("FIBERLB_IP_COMMAND", "/env/ip");
assert_eq!(
resolve_ip_command(Some(" /configured/ip ".to_string())),
"/configured/ip"
);
std::env::remove_var("FIBERLB_IP_COMMAND");
}
#[test]
fn env_ip_command_is_used_when_config_is_absent() {
let _guard = env_lock().lock().unwrap();
std::env::set_var("FIBERLB_IP_COMMAND", " /env/ip ");
assert_eq!(resolve_ip_command(None), "/env/ip");
std::env::remove_var("FIBERLB_IP_COMMAND");
}
}

View file

@ -1,27 +1,24 @@
//! FlashDNS authoritative DNS server binary //! FlashDNS authoritative DNS server binary
use anyhow::Result;
use chainfire_client::Client as ChainFireClient;
use clap::Parser;
use flashdns_api::{RecordServiceServer, ZoneServiceServer}; use flashdns_api::{RecordServiceServer, ZoneServiceServer};
use flashdns_server::{ use flashdns_server::{
config::{MetadataBackend, ServerConfig}, config::{MetadataBackend, ServerConfig},
dns::DnsHandler, dns::DnsHandler,
metadata::DnsMetadataStore, metadata::DnsMetadataStore,
RecordServiceImpl, RecordServiceImpl, ZoneServiceImpl,
ZoneServiceImpl,
}; };
use chainfire_client::Client as ChainFireClient;
use iam_service_auth::AuthService; use iam_service_auth::AuthService;
use metrics_exporter_prometheus::PrometheusBuilder; use metrics_exporter_prometheus::PrometheusBuilder;
use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tonic::transport::{Certificate, Identity, Server, ServerTlsConfig}; use tonic::transport::{Certificate, Identity, Server, ServerTlsConfig};
use tonic::{Request, Status}; use tonic::{Request, Status};
use tonic_health::server::health_reporter; use tonic_health::server::health_reporter;
use tracing_subscriber::EnvFilter; use tracing_subscriber::EnvFilter;
use anyhow::Result;
use clap::Parser;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use config::{Config as Cfg, Environment, File, FileFormat};
/// Command-line arguments for FlashDNS server. /// Command-line arguments for FlashDNS server.
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
@ -39,26 +36,6 @@ struct CliArgs {
#[arg(long)] #[arg(long)]
dns_addr: Option<String>, dns_addr: Option<String>,
/// ChainFire endpoint for cluster coordination (overrides config)
#[arg(long, env = "FLASHDNS_CHAINFIRE_ENDPOINT")]
chainfire_endpoint: Option<String>,
/// FlareDB endpoint for metadata and tenant data storage (overrides config)
#[arg(long, env = "FLASHDNS_FLAREDB_ENDPOINT")]
flaredb_endpoint: Option<String>,
/// Metadata backend (flaredb, postgres, sqlite)
#[arg(long, env = "FLASHDNS_METADATA_BACKEND")]
metadata_backend: Option<String>,
/// SQL database URL for metadata (required for postgres/sqlite backend)
#[arg(long, env = "FLASHDNS_METADATA_DATABASE_URL")]
metadata_database_url: Option<String>,
/// Run in single-node mode (required when metadata backend is SQLite)
#[arg(long, env = "FLASHDNS_SINGLE_NODE")]
single_node: bool,
/// Log level (overrides config) /// Log level (overrides config)
#[arg(short, long)] #[arg(short, long)]
log_level: Option<String>, log_level: Option<String>,
@ -72,54 +49,22 @@ struct CliArgs {
async fn main() -> Result<(), Box<dyn std::error::Error>> { async fn main() -> Result<(), Box<dyn std::error::Error>> {
let cli_args = CliArgs::parse(); let cli_args = CliArgs::parse();
// Load configuration using config-rs let mut config = if cli_args.config.exists() {
let mut settings = Cfg::builder()
// Layer 1: Application defaults. Serialize ServerConfig::default() into TOML.
.add_source(File::from_str(
toml::to_string(&ServerConfig::default())?.as_str(),
FileFormat::Toml,
))
// Layer 2: Environment variables (e.g., FLASHDNS_GRPC_ADDR, FLASHDNS_LOG_LEVEL)
.add_source(
Environment::with_prefix("FLASHDNS")
.separator("__") // Use double underscore for nested fields
);
// Layer 3: Configuration file (if specified)
if cli_args.config.exists() {
tracing::info!("Loading config from file: {}", cli_args.config.display()); tracing::info!("Loading config from file: {}", cli_args.config.display());
settings = settings.add_source(File::from(cli_args.config.as_path())); let contents = tokio::fs::read_to_string(&cli_args.config).await?;
toml::from_str(&contents)?
} else { } else {
tracing::info!("Config file not found, using defaults and environment variables."); tracing::info!("Config file not found, using defaults.");
} ServerConfig::default()
};
let mut config: ServerConfig = settings // Apply command line overrides
.build()?
.try_deserialize()
.map_err(|e| anyhow::anyhow!("Failed to load configuration: {}", e))?;
// Apply command line overrides (Layer 4: highest precedence)
if let Some(grpc_addr_str) = cli_args.grpc_addr { if let Some(grpc_addr_str) = cli_args.grpc_addr {
config.grpc_addr = grpc_addr_str.parse()?; config.grpc_addr = grpc_addr_str.parse()?;
} }
if let Some(dns_addr_str) = cli_args.dns_addr { if let Some(dns_addr_str) = cli_args.dns_addr {
config.dns_addr = dns_addr_str.parse()?; config.dns_addr = dns_addr_str.parse()?;
} }
if let Some(chainfire_endpoint) = cli_args.chainfire_endpoint {
config.chainfire_endpoint = Some(chainfire_endpoint);
}
if let Some(flaredb_endpoint) = cli_args.flaredb_endpoint {
config.flaredb_endpoint = Some(flaredb_endpoint);
}
if let Some(metadata_backend) = cli_args.metadata_backend {
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
}
if let Some(metadata_database_url) = cli_args.metadata_database_url {
config.metadata_database_url = Some(metadata_database_url);
}
if cli_args.single_node {
config.single_node = true;
}
if let Some(log_level) = cli_args.log_level { if let Some(log_level) = cli_args.log_level {
config.log_level = log_level; config.log_level = log_level;
} }
@ -173,16 +118,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
config.chainfire_endpoint.clone(), config.chainfire_endpoint.clone(),
) )
.await .await
.map_err(|e| anyhow::anyhow!("Failed to initialize FlareDB metadata store: {}", e))?, .map_err(|e| {
anyhow::anyhow!("Failed to initialize FlareDB metadata store: {}", e)
})?,
) )
} }
MetadataBackend::Postgres | MetadataBackend::Sqlite => { MetadataBackend::Postgres | MetadataBackend::Sqlite => {
let database_url = config let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
.metadata_database_url
.as_deref()
.ok_or_else(|| {
anyhow::anyhow!( anyhow::anyhow!(
"metadata_database_url is required when metadata_backend={} (env: FLASHDNS_METADATA_DATABASE_URL)", "metadata_database_url is required when metadata_backend={}",
metadata_backend_name(config.metadata_backend) metadata_backend_name(config.metadata_backend)
) )
})?; })?;
@ -195,13 +139,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Arc::new( Arc::new(
DnsMetadataStore::new_sql(database_url, config.single_node) DnsMetadataStore::new_sql(database_url, config.single_node)
.await .await
.map_err(|e| anyhow::anyhow!("Failed to initialize SQL metadata store: {}", e))?, .map_err(|e| {
anyhow::anyhow!("Failed to initialize SQL metadata store: {}", e)
})?,
) )
} }
}; };
// Initialize IAM authentication service // Initialize IAM authentication service
tracing::info!("Connecting to IAM server at {}", config.auth.iam_server_addr); tracing::info!(
"Connecting to IAM server at {}",
config.auth.iam_server_addr
);
let auth_service = AuthService::new(&config.auth.iam_server_addr) let auth_service = AuthService::new(&config.auth.iam_server_addr)
.await .await
.map_err(|e| anyhow::anyhow!("Failed to connect to IAM server: {}", e))?; .map_err(|e| anyhow::anyhow!("Failed to connect to IAM server: {}", e))?;
@ -300,18 +249,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(()) Ok(())
} }
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend> {
match value.trim().to_ascii_lowercase().as_str() {
"flaredb" => Ok(MetadataBackend::FlareDb),
"postgres" => Ok(MetadataBackend::Postgres),
"sqlite" => Ok(MetadataBackend::Sqlite),
other => Err(anyhow::anyhow!(
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
other
)),
}
}
fn metadata_backend_name(backend: MetadataBackend) -> &'static str { fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
match backend { match backend {
MetadataBackend::FlareDb => "flaredb", MetadataBackend::FlareDb => "flaredb",
@ -345,11 +282,7 @@ fn ensure_sql_backend_matches_url(backend: MetadataBackend, database_url: &str)
} }
} }
async fn register_chainfire_membership( async fn register_chainfire_membership(endpoint: &str, service: &str, addr: String) -> Result<()> {
endpoint: &str,
service: &str,
addr: String,
) -> Result<()> {
let node_id = let node_id =
std::env::var("HOSTNAME").unwrap_or_else(|_| format!("{}-{}", service, std::process::id())); std::env::var("HOSTNAME").unwrap_or_else(|_| format!("{}-{}", service, std::process::id()));
let ts = SystemTime::now() let ts = SystemTime::now()

View file

@ -57,12 +57,8 @@ impl DnsMetadataStore {
endpoint: Option<String>, endpoint: Option<String>,
pd_endpoint: Option<String>, pd_endpoint: Option<String>,
) -> Result<Self> { ) -> Result<Self> {
let endpoint = endpoint.unwrap_or_else(|| { let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
std::env::var("FLASHDNS_FLAREDB_ENDPOINT")
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
});
let pd_endpoint = pd_endpoint let pd_endpoint = pd_endpoint
.or_else(|| std::env::var("FLASHDNS_CHAINFIRE_ENDPOINT").ok())
.map(|value| normalize_transport_addr(&value)) .map(|value| normalize_transport_addr(&value))
.unwrap_or_else(|| endpoint.clone()); .unwrap_or_else(|| endpoint.clone());

View file

@ -27,6 +27,14 @@ pub struct ServerConfig {
/// Logging configuration /// Logging configuration
#[serde(default)] #[serde(default)]
pub logging: LoggingConfig, pub logging: LoggingConfig,
/// Admin API policy configuration
#[serde(default)]
pub admin: AdminConfig,
/// Development-only safety valves
#[serde(default)]
pub dev: DevConfig,
} }
impl ServerConfig { impl ServerConfig {
@ -102,6 +110,29 @@ impl ServerConfig {
} }
} }
if let Ok(value) = std::env::var("IAM_ALLOW_UNAUTHENTICATED_ADMIN")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_UNAUTHENTICATED_ADMIN"))
{
let value = value.trim().to_ascii_lowercase();
config.admin.allow_unauthenticated =
matches!(value.as_str(), "1" | "true" | "yes" | "on");
}
if let Ok(value) = std::env::var("IAM_ALLOW_RANDOM_SIGNING_KEY")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_RANDOM_SIGNING_KEY"))
{
let value = value.trim().to_ascii_lowercase();
config.dev.allow_random_signing_key =
matches!(value.as_str(), "1" | "true" | "yes" | "on");
}
if let Ok(value) = std::env::var("IAM_ALLOW_MEMORY_BACKEND")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_MEMORY_BACKEND"))
{
let value = value.trim().to_ascii_lowercase();
config.dev.allow_memory_backend = matches!(value.as_str(), "1" | "true" | "yes" | "on");
}
Ok(config) Ok(config)
} }
@ -132,6 +163,8 @@ impl ServerConfig {
}, },
}, },
logging: LoggingConfig::default(), logging: LoggingConfig::default(),
admin: AdminConfig::default(),
dev: DevConfig::default(),
} }
} }
} }
@ -226,6 +259,26 @@ pub struct ClusterConfig {
pub chainfire_endpoint: Option<String>, pub chainfire_endpoint: Option<String>,
} }
/// Admin API policy configuration
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct AdminConfig {
/// Allow admin APIs to run without an explicit admin token (dev only)
#[serde(default)]
pub allow_unauthenticated: bool,
}
/// Development-only safety valves
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct DevConfig {
/// Allow generating a random signing key when none is configured
#[serde(default)]
pub allow_random_signing_key: bool,
/// Allow the in-memory backend
#[serde(default)]
pub allow_memory_backend: bool,
}
/// Backend type /// Backend type
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] #[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")] #[serde(rename_all = "lowercase")]

View file

@ -64,8 +64,9 @@ fn load_admin_token() -> Option<String> {
.filter(|value| !value.is_empty()) .filter(|value| !value.is_empty())
} }
fn allow_unauthenticated_admin() -> bool { fn allow_unauthenticated_admin(config: &ServerConfig) -> bool {
std::env::var("IAM_ALLOW_UNAUTHENTICATED_ADMIN") config.admin.allow_unauthenticated
|| std::env::var("IAM_ALLOW_UNAUTHENTICATED_ADMIN")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_UNAUTHENTICATED_ADMIN")) .or_else(|_| std::env::var("PHOTON_IAM_ALLOW_UNAUTHENTICATED_ADMIN"))
.ok() .ok()
.map(|value| { .map(|value| {
@ -334,7 +335,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Create token service // Create token service
let signing_key = if config.authn.internal_token.signing_key.is_empty() { let signing_key = if config.authn.internal_token.signing_key.is_empty() {
let allow_random = std::env::var("IAM_ALLOW_RANDOM_SIGNING_KEY") let allow_random = config.dev.allow_random_signing_key
|| std::env::var("IAM_ALLOW_RANDOM_SIGNING_KEY")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_RANDOM_SIGNING_KEY")) .or_else(|_| std::env::var("PHOTON_IAM_ALLOW_RANDOM_SIGNING_KEY"))
.ok() .ok()
.map(|value| { .map(|value| {
@ -346,7 +348,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.unwrap_or(false); .unwrap_or(false);
if !allow_random { if !allow_random {
return Err("No signing key configured. Set IAM_ALLOW_RANDOM_SIGNING_KEY=true for dev or configure authn.internal_token.signing_key.".into()); return Err("No signing key configured. Set dev.allow_random_signing_key=true (or IAM_ALLOW_RANDOM_SIGNING_KEY=true for legacy dev mode) or configure authn.internal_token.signing_key.".into());
} }
warn!("No signing key configured, generating random key (dev-only)"); warn!("No signing key configured, generating random key (dev-only)");
@ -369,9 +371,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let token_service = Arc::new(InternalTokenService::new(token_config)); let token_service = Arc::new(InternalTokenService::new(token_config));
let admin_token = load_admin_token(); let admin_token = load_admin_token();
if admin_token.is_none() && !allow_unauthenticated_admin() { if admin_token.is_none() && !allow_unauthenticated_admin(&config) {
return Err( return Err(
"IAM admin token not configured. Set IAM_ADMIN_TOKEN or explicitly allow dev mode with IAM_ALLOW_UNAUTHENTICATED_ADMIN=true." "IAM admin token not configured. Set IAM_ADMIN_TOKEN or explicitly allow dev mode with admin.allow_unauthenticated=true."
.into(), .into(),
); );
} }
@ -550,7 +552,8 @@ async fn create_backend(
) -> Result<Backend, Box<dyn std::error::Error>> { ) -> Result<Backend, Box<dyn std::error::Error>> {
match config.store.backend { match config.store.backend {
BackendKind::Memory => { BackendKind::Memory => {
let allow_memory = std::env::var("IAM_ALLOW_MEMORY_BACKEND") let allow_memory = config.dev.allow_memory_backend
|| std::env::var("IAM_ALLOW_MEMORY_BACKEND")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_MEMORY_BACKEND")) .or_else(|_| std::env::var("PHOTON_IAM_ALLOW_MEMORY_BACKEND"))
.ok() .ok()
.map(|value| { .map(|value| {
@ -562,7 +565,7 @@ async fn create_backend(
.unwrap_or(false); .unwrap_or(false);
if !allow_memory { if !allow_memory {
return Err( return Err(
"In-memory IAM backend is disabled. Use FlareDB backend, or set IAM_ALLOW_MEMORY_BACKEND=true for tests/dev only." "In-memory IAM backend is disabled. Use FlareDB backend, or set dev.allow_memory_backend=true for tests/dev only."
.into(), .into(),
); );
} }

View file

@ -91,6 +91,18 @@ impl Default for PrismNetConfig {
} }
} }
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CreditServiceConfig {
#[serde(default)]
pub server_addr: Option<String>,
}
impl Default for CreditServiceConfig {
fn default() -> Self {
Self { server_addr: None }
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ChainFireConfig { pub struct ChainFireConfig {
pub endpoint: Option<String>, pub endpoint: Option<String>,
@ -110,6 +122,8 @@ pub struct Config {
pub flaredb: FlareDbConfig, pub flaredb: FlareDbConfig,
pub chainfire: ChainFireConfig, pub chainfire: ChainFireConfig,
pub iam: IamConfig, pub iam: IamConfig,
#[serde(default)]
pub creditservice: CreditServiceConfig,
pub fiberlb: FiberLbConfig, pub fiberlb: FiberLbConfig,
pub flashdns: FlashDnsConfig, pub flashdns: FlashDnsConfig,
pub prismnet: PrismNetConfig, pub prismnet: PrismNetConfig,

View file

@ -38,8 +38,8 @@ use tracing_subscriber::EnvFilter;
#[command(about = "Kubernetes API server for PlasmaCloud's k8shost component")] #[command(about = "Kubernetes API server for PlasmaCloud's k8shost component")]
struct Args { struct Args {
/// Configuration file path /// Configuration file path
#[arg(short, long)] #[arg(short, long, default_value = "k8shost.toml")]
config: Option<PathBuf>, config: PathBuf,
/// Listen address for gRPC server (e.g., "[::]:6443") /// Listen address for gRPC server (e.g., "[::]:6443")
#[arg(long)] #[arg(long)]
@ -91,17 +91,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let args = Args::parse(); let args = Args::parse();
// Load configuration // Load configuration
let mut settings = ::config::Config::builder() let mut settings = ::config::Config::builder().add_source(::config::File::from_str(
.add_source(::config::File::from_str(
toml::to_string(&Config::default())?.as_str(), toml::to_string(&Config::default())?.as_str(),
::config::FileFormat::Toml, ::config::FileFormat::Toml,
)) ));
.add_source(::config::Environment::with_prefix("K8SHOST").separator("_"));
// Add config file if specified if args.config.exists() {
if let Some(config_path) = &args.config { info!("Loading config from file: {}", args.config.display());
info!("Loading config from file: {}", config_path.display()); settings = settings.add_source(::config::File::from(args.config.as_path()));
settings = settings.add_source(::config::File::from(config_path.as_path())); } else {
info!(
"Config file not found: {}, using defaults",
args.config.display()
);
} }
let loaded_config: Config = settings let loaded_config: Config = settings
@ -136,6 +138,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.iam_server_addr .iam_server_addr
.unwrap_or(loaded_config.iam.server_addr), .unwrap_or(loaded_config.iam.server_addr),
}, },
creditservice: config::CreditServiceConfig {
server_addr: loaded_config.creditservice.server_addr,
},
fiberlb: config::FiberLbConfig { fiberlb: config::FiberLbConfig {
server_addr: args server_addr: args
.fiberlb_server_addr .fiberlb_server_addr
@ -275,8 +280,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let ipam_client = Arc::new(IpamClient::new(config.prismnet.server_addr.clone())); let ipam_client = Arc::new(IpamClient::new(config.prismnet.server_addr.clone()));
// Create service implementations with storage // Create service implementations with storage
let creditservice_endpoint = config.creditservice.server_addr.as_deref();
let pod_service = Arc::new( let pod_service = Arc::new(
PodServiceImpl::new_with_credit_service(storage.clone(), auth_service.clone()).await, PodServiceImpl::new_with_credit_service(
storage.clone(),
auth_service.clone(),
creditservice_endpoint,
)
.await,
); );
let service_service = Arc::new(ServiceServiceImpl::new( let service_service = Arc::new(ServiceServiceImpl::new(
storage.clone(), storage.clone(),
@ -290,7 +301,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
)); ));
// Start scheduler in background with CreditService integration // Start scheduler in background with CreditService integration
let scheduler = Arc::new(scheduler::Scheduler::new_with_credit_service(storage.clone()).await); let scheduler = Arc::new(
scheduler::Scheduler::new_with_credit_service(storage.clone(), creditservice_endpoint)
.await,
);
tokio::spawn(async move { tokio::spawn(async move {
scheduler.run().await; scheduler.run().await;
}); });

View file

@ -33,10 +33,11 @@ impl Scheduler {
} }
/// Create a new scheduler with CreditService quota enforcement /// Create a new scheduler with CreditService quota enforcement
pub async fn new_with_credit_service(storage: Arc<Storage>) -> Self { pub async fn new_with_credit_service(storage: Arc<Storage>, endpoint: Option<&str>) -> Self {
// Initialize CreditService client if endpoint is configured // Initialize CreditService client if endpoint is configured
let credit_service = match std::env::var("CREDITSERVICE_ENDPOINT") { let credit_service = match endpoint {
Ok(endpoint) => match CreditServiceClient::connect(&endpoint).await { Some(endpoint) if !endpoint.trim().is_empty() => {
match CreditServiceClient::connect(endpoint).await {
Ok(client) => { Ok(client) => {
info!( info!(
"Scheduler: CreditService quota enforcement enabled: {}", "Scheduler: CreditService quota enforcement enabled: {}",
@ -51,9 +52,12 @@ impl Scheduler {
); );
None None
} }
}, }
Err(_) => { }
info!("Scheduler: CREDITSERVICE_ENDPOINT not set, quota enforcement disabled"); _ => {
info!(
"Scheduler: CreditService endpoint not configured, quota enforcement disabled"
);
None None
} }
}; };

View file

@ -45,10 +45,15 @@ impl PodServiceImpl {
} }
} }
pub async fn new_with_credit_service(storage: Arc<Storage>, auth: Arc<AuthService>) -> Self { pub async fn new_with_credit_service(
storage: Arc<Storage>,
auth: Arc<AuthService>,
endpoint: Option<&str>,
) -> Self {
// Initialize CreditService client if endpoint is configured // Initialize CreditService client if endpoint is configured
let credit_service = match std::env::var("CREDITSERVICE_ENDPOINT") { let credit_service = match endpoint {
Ok(endpoint) => match CreditServiceClient::connect(&endpoint).await { Some(endpoint) if !endpoint.trim().is_empty() => {
match CreditServiceClient::connect(endpoint).await {
Ok(client) => { Ok(client) => {
tracing::info!("CreditService admission control enabled: {}", endpoint); tracing::info!("CreditService admission control enabled: {}", endpoint);
Some(Arc::new(RwLock::new(client))) Some(Arc::new(RwLock::new(client)))
@ -60,9 +65,10 @@ impl PodServiceImpl {
); );
None None
} }
}, }
Err(_) => { }
tracing::info!("CREDITSERVICE_ENDPOINT not set, admission control disabled"); _ => {
tracing::info!("CreditService endpoint not configured, admission control disabled");
None None
} }
}; };

View file

@ -50,6 +50,75 @@ pub enum ObjectStorageBackend {
Distributed, Distributed,
} }
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct S3AuthConfig {
#[serde(default = "default_s3_auth_enabled")]
pub enabled: bool,
#[serde(default = "default_s3_aws_region")]
pub aws_region: String,
#[serde(default = "default_s3_iam_cache_ttl_secs")]
pub iam_cache_ttl_secs: u64,
#[serde(default = "default_s3_default_org_id")]
pub default_org_id: Option<String>,
#[serde(default = "default_s3_default_project_id")]
pub default_project_id: Option<String>,
#[serde(default = "default_s3_max_auth_body_bytes")]
pub max_auth_body_bytes: usize,
}
impl Default for S3AuthConfig {
fn default() -> Self {
Self {
enabled: default_s3_auth_enabled(),
aws_region: default_s3_aws_region(),
iam_cache_ttl_secs: default_s3_iam_cache_ttl_secs(),
default_org_id: default_s3_default_org_id(),
default_project_id: default_s3_default_project_id(),
max_auth_body_bytes: default_s3_max_auth_body_bytes(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct S3PerformanceConfig {
#[serde(default = "default_s3_streaming_put_threshold_bytes")]
pub streaming_put_threshold_bytes: usize,
#[serde(default = "default_s3_inline_put_max_bytes")]
pub inline_put_max_bytes: usize,
#[serde(default = "default_s3_multipart_put_concurrency")]
pub multipart_put_concurrency: usize,
#[serde(default = "default_s3_multipart_fetch_concurrency")]
pub multipart_fetch_concurrency: usize,
}
impl Default for S3PerformanceConfig {
fn default() -> Self {
Self {
streaming_put_threshold_bytes: default_s3_streaming_put_threshold_bytes(),
inline_put_max_bytes: default_s3_inline_put_max_bytes(),
multipart_put_concurrency: default_s3_multipart_put_concurrency(),
multipart_fetch_concurrency: default_s3_multipart_fetch_concurrency(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct S3Config {
#[serde(default)]
pub auth: S3AuthConfig,
#[serde(default)]
pub performance: S3PerformanceConfig,
}
/// Server configuration /// Server configuration
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ServerConfig { pub struct ServerConfig {
@ -100,6 +169,10 @@ pub struct ServerConfig {
/// Authentication configuration /// Authentication configuration
#[serde(default)] #[serde(default)]
pub auth: AuthConfig, pub auth: AuthConfig,
/// S3 API runtime settings
#[serde(default)]
pub s3: S3Config,
} }
/// Authentication configuration /// Authentication configuration
@ -114,6 +187,46 @@ fn default_iam_server_addr() -> String {
"127.0.0.1:50051".to_string() "127.0.0.1:50051".to_string()
} }
fn default_s3_auth_enabled() -> bool {
true
}
fn default_s3_aws_region() -> String {
"us-east-1".to_string()
}
fn default_s3_iam_cache_ttl_secs() -> u64 {
30
}
fn default_s3_default_org_id() -> Option<String> {
Some("default".to_string())
}
fn default_s3_default_project_id() -> Option<String> {
Some("default".to_string())
}
fn default_s3_max_auth_body_bytes() -> usize {
1024 * 1024 * 1024
}
fn default_s3_streaming_put_threshold_bytes() -> usize {
16 * 1024 * 1024
}
fn default_s3_inline_put_max_bytes() -> usize {
128 * 1024 * 1024
}
fn default_s3_multipart_put_concurrency() -> usize {
4
}
fn default_s3_multipart_fetch_concurrency() -> usize {
4
}
impl Default for AuthConfig { impl Default for AuthConfig {
fn default() -> Self { fn default() -> Self {
Self { Self {
@ -139,6 +252,7 @@ impl Default for ServerConfig {
sync_on_write: false, sync_on_write: false,
tls: None, tls: None,
auth: AuthConfig::default(), auth: AuthConfig::default(),
s3: S3Config::default(),
} }
} }
} }

View file

@ -5,7 +5,7 @@ use clap::Parser;
use iam_service_auth::AuthService; use iam_service_auth::AuthService;
use lightningstor_api::{BucketServiceServer, ObjectServiceServer}; use lightningstor_api::{BucketServiceServer, ObjectServiceServer};
use lightningstor_distributed::{ use lightningstor_distributed::{
DistributedConfig, ErasureCodedBackend, RedundancyMode, ReplicatedBackend, RepairQueue, DistributedConfig, ErasureCodedBackend, RedundancyMode, RepairQueue, ReplicatedBackend,
StaticNodeRegistry, StaticNodeRegistry,
}; };
use lightningstor_server::{ use lightningstor_server::{
@ -57,26 +57,6 @@ struct Args {
#[arg(short, long)] #[arg(short, long)]
log_level: Option<String>, log_level: Option<String>,
/// ChainFire endpoint for cluster coordination (overrides config)
#[arg(long, env = "LIGHTNINGSTOR_CHAINFIRE_ENDPOINT")]
chainfire_endpoint: Option<String>,
/// FlareDB endpoint for metadata and tenant data storage (overrides config)
#[arg(long, env = "LIGHTNINGSTOR_FLAREDB_ENDPOINT")]
flaredb_endpoint: Option<String>,
/// Metadata backend (flaredb, postgres, sqlite)
#[arg(long, env = "LIGHTNINGSTOR_METADATA_BACKEND")]
metadata_backend: Option<String>,
/// SQL database URL for metadata (required for postgres/sqlite backend)
#[arg(long, env = "LIGHTNINGSTOR_METADATA_DATABASE_URL")]
metadata_database_url: Option<String>,
/// Run in single-node mode (required when metadata backend is SQLite)
#[arg(long, env = "LIGHTNINGSTOR_SINGLE_NODE")]
single_node: bool,
/// Data directory for object storage (overrides config) /// Data directory for object storage (overrides config)
#[arg(long)] #[arg(long)]
data_dir: Option<String>, data_dir: Option<String>,
@ -112,21 +92,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
if let Some(log_level) = args.log_level { if let Some(log_level) = args.log_level {
config.log_level = log_level; config.log_level = log_level;
} }
if let Some(chainfire_endpoint) = args.chainfire_endpoint {
config.chainfire_endpoint = Some(chainfire_endpoint);
}
if let Some(flaredb_endpoint) = args.flaredb_endpoint {
config.flaredb_endpoint = Some(flaredb_endpoint);
}
if let Some(metadata_backend) = args.metadata_backend {
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
}
if let Some(metadata_database_url) = args.metadata_database_url {
config.metadata_database_url = Some(metadata_database_url);
}
if args.single_node {
config.single_node = true;
}
if let Some(data_dir) = args.data_dir { if let Some(data_dir) = args.data_dir {
config.data_dir = data_dir; config.data_dir = data_dir;
} }
@ -187,12 +152,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
) )
} }
MetadataBackend::Postgres | MetadataBackend::Sqlite => { MetadataBackend::Postgres | MetadataBackend::Sqlite => {
let database_url = config let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
.metadata_database_url
.as_deref()
.ok_or_else(|| {
format!( format!(
"metadata_database_url is required when metadata_backend={} (env: LIGHTNINGSTOR_METADATA_DATABASE_URL)", "metadata_database_url is required when metadata_backend={}",
metadata_backend_name(config.metadata_backend) metadata_backend_name(config.metadata_backend)
) )
})?; })?;
@ -263,10 +225,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let s3_addr: SocketAddr = config.s3_addr; let s3_addr: SocketAddr = config.s3_addr;
// Start S3 HTTP server with shared state // Start S3 HTTP server with shared state
let s3_router = s3::create_router_with_auth( let s3_router = s3::create_router_with_auth_config(
storage.clone(), storage.clone(),
metadata.clone(), metadata.clone(),
Some(config.auth.iam_server_addr.clone()), Some(config.auth.iam_server_addr.clone()),
config.s3.clone(),
); );
let s3_server = tokio::spawn(async move { let s3_server = tokio::spawn(async move {
tracing::info!("S3 HTTP server listening on {}", s3_addr); tracing::info!("S3 HTTP server listening on {}", s3_addr);
@ -341,19 +304,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(()) Ok(())
} }
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
match value.trim().to_ascii_lowercase().as_str() {
"flaredb" => Ok(MetadataBackend::FlareDb),
"postgres" => Ok(MetadataBackend::Postgres),
"sqlite" => Ok(MetadataBackend::Sqlite),
other => Err(format!(
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
other
)
.into()),
}
}
fn metadata_backend_name(backend: MetadataBackend) -> &'static str { fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
match backend { match backend {
MetadataBackend::FlareDb => "flaredb", MetadataBackend::FlareDb => "flaredb",
@ -442,7 +392,9 @@ async fn create_storage_backend(
ObjectStorageBackend::LocalFs => { ObjectStorageBackend::LocalFs => {
tracing::info!("Object storage backend: local_fs"); tracing::info!("Object storage backend: local_fs");
Ok(StorageRuntime { Ok(StorageRuntime {
backend: Arc::new(LocalFsBackend::new(&config.data_dir, config.sync_on_write).await?), backend: Arc::new(
LocalFsBackend::new(&config.data_dir, config.sync_on_write).await?,
),
repair_worker: None, repair_worker: None,
}) })
} }

View file

@ -55,19 +55,18 @@ impl MetadataStore {
endpoint: Option<String>, endpoint: Option<String>,
pd_endpoint: Option<String>, pd_endpoint: Option<String>,
) -> Result<Self> { ) -> Result<Self> {
let endpoint = endpoint.unwrap_or_else(|| { let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
std::env::var("LIGHTNINGSTOR_FLAREDB_ENDPOINT")
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
});
let pd_endpoint = pd_endpoint let pd_endpoint = pd_endpoint
.or_else(|| std::env::var("LIGHTNINGSTOR_CHAINFIRE_ENDPOINT").ok())
.map(|value| normalize_transport_addr(&value)) .map(|value| normalize_transport_addr(&value))
.unwrap_or_else(|| endpoint.clone()); .unwrap_or_else(|| endpoint.clone());
let mut clients = Vec::with_capacity(FLAREDB_CLIENT_POOL_SIZE); let mut clients = Vec::with_capacity(FLAREDB_CLIENT_POOL_SIZE);
for _ in 0..FLAREDB_CLIENT_POOL_SIZE { for _ in 0..FLAREDB_CLIENT_POOL_SIZE {
let client = let client = RdbClient::connect_with_pd_namespace(
RdbClient::connect_with_pd_namespace(endpoint.clone(), pd_endpoint.clone(), "lightningstor") endpoint.clone(),
pd_endpoint.clone(),
"lightningstor",
)
.await .await
.map_err(|e| { .map_err(|e| {
lightningstor_types::Error::StorageError(format!( lightningstor_types::Error::StorageError(format!(
@ -321,7 +320,11 @@ impl MetadataStore {
Ok((results, next)) Ok((results, next))
} }
async fn flaredb_put(clients: &[Arc<Mutex<RdbClient>>], key: &[u8], value: &[u8]) -> Result<()> { async fn flaredb_put(
clients: &[Arc<Mutex<RdbClient>>],
key: &[u8],
value: &[u8],
) -> Result<()> {
let client = Self::flaredb_client_for_key(clients, key); let client = Self::flaredb_client_for_key(clients, key);
let raw_result = { let raw_result = {
let mut c = client.lock().await; let mut c = client.lock().await;
@ -443,7 +446,8 @@ impl MetadataStore {
let client = Self::flaredb_scan_client(clients); let client = Self::flaredb_scan_client(clients);
let (mut items, next) = match { let (mut items, next) = match {
let mut c = client.lock().await; let mut c = client.lock().await;
c.raw_scan(start_key.clone(), end_key.clone(), fetch_limit).await c.raw_scan(start_key.clone(), end_key.clone(), fetch_limit)
.await
} { } {
Ok((keys, values, next)) => { Ok((keys, values, next)) => {
let items = keys let items = keys
@ -697,7 +701,8 @@ impl MetadataStore {
.await .await
} }
StorageBackend::Sql(sql) => { StorageBackend::Sql(sql) => {
let prefix_end = String::from_utf8(Self::prefix_end(prefix.as_bytes())).map_err(|e| { let prefix_end =
String::from_utf8(Self::prefix_end(prefix.as_bytes())).map_err(|e| {
lightningstor_types::Error::StorageError(format!( lightningstor_types::Error::StorageError(format!(
"Failed to encode prefix end: {}", "Failed to encode prefix end: {}",
e e
@ -908,7 +913,10 @@ impl MetadataStore {
} }
fn multipart_bucket_prefix(bucket_id: &BucketId, prefix: &str) -> String { fn multipart_bucket_prefix(bucket_id: &BucketId, prefix: &str) -> String {
format!("/lightningstor/multipart/by-bucket/{}/{}", bucket_id, prefix) format!(
"/lightningstor/multipart/by-bucket/{}/{}",
bucket_id, prefix
)
} }
fn multipart_object_key(object_id: &ObjectId) -> String { fn multipart_object_key(object_id: &ObjectId) -> String {
@ -955,7 +963,8 @@ impl MetadataStore {
} }
pub async fn delete_replicated_repair_task(&self, task_id: &str) -> Result<()> { pub async fn delete_replicated_repair_task(&self, task_id: &str) -> Result<()> {
self.delete_key(&Self::replicated_repair_task_key(task_id)).await self.delete_key(&Self::replicated_repair_task_key(task_id))
.await
} }
/// Save bucket metadata /// Save bucket metadata
@ -1246,7 +1255,8 @@ impl MetadataStore {
)) ))
.await?; .await?;
} }
self.delete_key(&Self::multipart_upload_key(upload_id)).await self.delete_key(&Self::multipart_upload_key(upload_id))
.await
} }
pub async fn list_multipart_uploads( pub async fn list_multipart_uploads(
@ -1331,7 +1341,8 @@ impl MetadataStore {
} }
pub async fn delete_object_multipart_upload(&self, object_id: &ObjectId) -> Result<()> { pub async fn delete_object_multipart_upload(&self, object_id: &ObjectId) -> Result<()> {
self.delete_key(&Self::multipart_object_key(object_id)).await self.delete_key(&Self::multipart_object_key(object_id))
.await
} }
} }
@ -1380,13 +1391,11 @@ mod tests {
store.delete_bucket(&bucket).await.unwrap(); store.delete_bucket(&bucket).await.unwrap();
assert!(!store.bucket_cache.contains_key(&cache_key)); assert!(!store.bucket_cache.contains_key(&cache_key));
assert!(!store.bucket_cache.contains_key(&cache_id_key)); assert!(!store.bucket_cache.contains_key(&cache_id_key));
assert!( assert!(store
store
.load_bucket("org-a", "project-a", "bench-bucket") .load_bucket("org-a", "project-a", "bench-bucket")
.await .await
.unwrap() .unwrap()
.is_none() .is_none());
);
} }
#[tokio::test] #[tokio::test]
@ -1426,13 +1435,11 @@ mod tests {
.await .await
.unwrap(); .unwrap();
assert!(!store.object_cache.contains_key(&cache_key)); assert!(!store.object_cache.contains_key(&cache_key));
assert!( assert!(store
store
.load_object(&bucket.id, object.key.as_str(), None) .load_object(&bucket.id, object.key.as_str(), None)
.await .await
.unwrap() .unwrap()
.is_none() .is_none());
);
} }
#[tokio::test] #[tokio::test]
@ -1496,8 +1503,10 @@ mod tests {
); );
store.save_bucket(&bucket).await.unwrap(); store.save_bucket(&bucket).await.unwrap();
let upload_a = MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/one.bin").unwrap()); let upload_a =
let upload_b = MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/two.bin").unwrap()); MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/one.bin").unwrap());
let upload_b =
MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/two.bin").unwrap());
let other_bucket = Bucket::new( let other_bucket = Bucket::new(
BucketName::new("other-bucket").unwrap(), BucketName::new("other-bucket").unwrap(),
"org-a", "org-a",
@ -1505,8 +1514,10 @@ mod tests {
"default", "default",
); );
store.save_bucket(&other_bucket).await.unwrap(); store.save_bucket(&other_bucket).await.unwrap();
let upload_other = let upload_other = MultipartUpload::new(
MultipartUpload::new(other_bucket.id.to_string(), ObjectKey::new("a/three.bin").unwrap()); other_bucket.id.to_string(),
ObjectKey::new("a/three.bin").unwrap(),
);
store.save_multipart_upload(&upload_a).await.unwrap(); store.save_multipart_upload(&upload_a).await.unwrap();
store.save_multipart_upload(&upload_b).await.unwrap(); store.save_multipart_upload(&upload_b).await.unwrap();
@ -1543,10 +1554,7 @@ mod tests {
assert_eq!(tasks[0].attempt_count, 1); assert_eq!(tasks[0].attempt_count, 1);
assert_eq!(tasks[0].last_error.as_deref(), Some("transient failure")); assert_eq!(tasks[0].last_error.as_deref(), Some("transient failure"));
store store.delete_replicated_repair_task(&task.id).await.unwrap();
.delete_replicated_repair_task(&task.id)
.await
.unwrap();
assert!(store assert!(store
.list_replicated_repair_tasks(10) .list_replicated_repair_tasks(10)
.await .await

View file

@ -3,6 +3,7 @@
//! Implements simplified SigV4 authentication compatible with AWS S3 SDKs and aws-cli. //! Implements simplified SigV4 authentication compatible with AWS S3 SDKs and aws-cli.
//! Integrates with IAM for access key validation. //! Integrates with IAM for access key validation.
use crate::config::S3AuthConfig;
use crate::tenant::TenantContext; use crate::tenant::TenantContext;
use axum::{ use axum::{
body::{Body, Bytes}, body::{Body, Bytes},
@ -23,8 +24,6 @@ use tracing::{debug, warn};
use url::form_urlencoded; use url::form_urlencoded;
type HmacSha256 = Hmac<Sha256>; type HmacSha256 = Hmac<Sha256>;
const DEFAULT_MAX_AUTH_BODY_BYTES: usize = 1024 * 1024 * 1024;
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub(crate) struct VerifiedBodyBytes(pub Bytes); pub(crate) struct VerifiedBodyBytes(pub Bytes);
@ -49,6 +48,8 @@ pub struct AuthState {
aws_region: String, aws_region: String,
/// AWS service name for SigV4 (e.g., s3) /// AWS service name for SigV4 (e.g., s3)
aws_service: String, aws_service: String,
/// Maximum request body size to buffer during auth verification
max_auth_body_bytes: usize,
} }
pub struct IamClient { pub struct IamClient {
@ -60,6 +61,8 @@ pub struct IamClient {
enum IamClientMode { enum IamClientMode {
Env { Env {
credentials: std::collections::HashMap<String, String>, credentials: std::collections::HashMap<String, String>,
default_org_id: Option<String>,
default_project_id: Option<String>,
}, },
Grpc { Grpc {
endpoint: String, endpoint: String,
@ -83,11 +86,11 @@ struct CachedCredential {
impl IamClient { impl IamClient {
/// Create a new IAM client. If an endpoint is supplied, use the IAM gRPC API. /// Create a new IAM client. If an endpoint is supplied, use the IAM gRPC API.
pub fn new(iam_endpoint: Option<String>) -> Self { pub fn new(iam_endpoint: Option<String>) -> Self {
let cache_ttl = std::env::var("LIGHTNINGSTOR_S3_IAM_CACHE_TTL_SECS") Self::new_with_config(iam_endpoint, &S3AuthConfig::default())
.ok() }
.and_then(|value| value.parse::<u64>().ok())
.map(StdDuration::from_secs) pub fn new_with_config(iam_endpoint: Option<String>, config: &S3AuthConfig) -> Self {
.unwrap_or_else(|| StdDuration::from_secs(30)); let cache_ttl = StdDuration::from_secs(config.iam_cache_ttl_secs);
if let Some(endpoint) = iam_endpoint if let Some(endpoint) = iam_endpoint
.map(|value| normalize_iam_endpoint(&value)) .map(|value| normalize_iam_endpoint(&value))
@ -106,6 +109,8 @@ impl IamClient {
Self { Self {
mode: IamClientMode::Env { mode: IamClientMode::Env {
credentials: Self::load_env_credentials(), credentials: Self::load_env_credentials(),
default_org_id: config.default_org_id.clone(),
default_project_id: config.default_project_id.clone(),
}, },
credential_cache: Arc::new(RwLock::new(HashMap::new())), credential_cache: Arc::new(RwLock::new(HashMap::new())),
cache_ttl, cache_ttl,
@ -160,32 +165,32 @@ impl IamClient {
#[cfg(test)] #[cfg(test)]
fn env_credentials(&self) -> Option<&std::collections::HashMap<String, String>> { fn env_credentials(&self) -> Option<&std::collections::HashMap<String, String>> {
match &self.mode { match &self.mode {
IamClientMode::Env { credentials } => Some(credentials), IamClientMode::Env { credentials, .. } => Some(credentials),
IamClientMode::Grpc { .. } => None, IamClientMode::Grpc { .. } => None,
} }
} }
fn env_default_tenant() -> (Option<String>, Option<String>) { fn env_default_tenant(
let org_id = std::env::var("S3_TENANT_ORG_ID") default_org_id: Option<String>,
.ok() default_project_id: Option<String>,
.or_else(|| std::env::var("S3_ORG_ID").ok()) ) -> (Option<String>, Option<String>) {
.or_else(|| Some("default".to_string())); (default_org_id, default_project_id)
let project_id = std::env::var("S3_TENANT_PROJECT_ID")
.ok()
.or_else(|| std::env::var("S3_PROJECT_ID").ok())
.or_else(|| Some("default".to_string()));
(org_id, project_id)
} }
/// Validate access key and resolve the credential context. /// Validate access key and resolve the credential context.
pub async fn get_credential(&self, access_key_id: &str) -> Result<ResolvedCredential, String> { pub async fn get_credential(&self, access_key_id: &str) -> Result<ResolvedCredential, String> {
match &self.mode { match &self.mode {
IamClientMode::Env { credentials } => { IamClientMode::Env {
credentials,
default_org_id,
default_project_id,
} => {
let secret_key = credentials let secret_key = credentials
.get(access_key_id) .get(access_key_id)
.cloned() .cloned()
.ok_or_else(|| "Access key ID not found".to_string())?; .ok_or_else(|| "Access key ID not found".to_string())?;
let (org_id, project_id) = Self::env_default_tenant(); let (org_id, project_id) =
Self::env_default_tenant(default_org_id.clone(), default_project_id.clone());
Ok(ResolvedCredential { Ok(ResolvedCredential {
secret_key, secret_key,
principal_id: access_key_id.to_string(), principal_id: access_key_id.to_string(),
@ -318,16 +323,21 @@ fn iam_admin_token() -> Option<String> {
impl AuthState { impl AuthState {
/// Create new auth state with IAM integration /// Create new auth state with IAM integration
pub fn new(iam_endpoint: Option<String>) -> Self { pub fn new(iam_endpoint: Option<String>) -> Self {
let iam_client = Some(Arc::new(RwLock::new(IamClient::new(iam_endpoint)))); Self::new_with_config(iam_endpoint, &S3AuthConfig::default())
}
pub fn new_with_config(iam_endpoint: Option<String>, config: &S3AuthConfig) -> Self {
let iam_client = Some(Arc::new(RwLock::new(IamClient::new_with_config(
iam_endpoint,
config,
))));
Self { Self {
iam_client, iam_client,
enabled: std::env::var("S3_AUTH_ENABLED") enabled: config.enabled,
.unwrap_or_else(|_| "true".to_string()) aws_region: config.aws_region.clone(),
.parse()
.unwrap_or(true),
aws_region: std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string()), // Default S3 region
aws_service: "s3".to_string(), aws_service: "s3".to_string(),
max_auth_body_bytes: config.max_auth_body_bytes,
} }
} }
@ -338,6 +348,7 @@ impl AuthState {
enabled: false, enabled: false,
aws_region: "us-east-1".to_string(), aws_region: "us-east-1".to_string(),
aws_service: "s3".to_string(), aws_service: "s3".to_string(),
max_auth_body_bytes: S3AuthConfig::default().max_auth_body_bytes,
} }
} }
} }
@ -438,12 +449,8 @@ pub async fn sigv4_auth_middleware(
let should_buffer_body = should_buffer_auth_body(payload_hash_header.as_deref()); let should_buffer_body = should_buffer_auth_body(payload_hash_header.as_deref());
let body_bytes = if should_buffer_body { let body_bytes = if should_buffer_body {
let max_body_bytes = std::env::var("S3_MAX_AUTH_BODY_BYTES")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(DEFAULT_MAX_AUTH_BODY_BYTES);
let (parts, body) = request.into_parts(); let (parts, body) = request.into_parts();
let body_bytes = match axum::body::to_bytes(body, max_body_bytes).await { let body_bytes = match axum::body::to_bytes(body, auth_state.max_auth_body_bytes).await {
Ok(b) => b, Ok(b) => b,
Err(e) => { Err(e) => {
return error_response( return error_response(

View file

@ -6,5 +6,8 @@ mod auth;
mod router; mod router;
mod xml; mod xml;
pub use auth::{AuthState, sigv4_auth_middleware}; pub use auth::{sigv4_auth_middleware, AuthState};
pub use router::{create_router, create_router_with_auth, create_router_with_state}; pub use router::{
create_router, create_router_with_auth, create_router_with_auth_config,
create_router_with_state,
};

View file

@ -1,5 +1,6 @@
//! S3 API router using Axum //! S3 API router using Axum
use crate::config::{S3Config, S3PerformanceConfig};
use axum::{ use axum::{
body::{Body, Bytes}, body::{Body, Bytes},
extract::{Request, State}, extract::{Request, State},
@ -14,8 +15,8 @@ use futures::{stream, stream::FuturesUnordered, StreamExt};
use http_body_util::BodyExt; use http_body_util::BodyExt;
use md5::{Digest, Md5}; use md5::{Digest, Md5};
use serde::Deserialize; use serde::Deserialize;
use std::io;
use sha2::Sha256; use sha2::Sha256;
use std::io;
use std::sync::Arc; use std::sync::Arc;
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
@ -38,17 +39,24 @@ use super::xml::{
pub struct S3State { pub struct S3State {
pub storage: Arc<dyn StorageBackend>, pub storage: Arc<dyn StorageBackend>,
pub metadata: Arc<MetadataStore>, pub metadata: Arc<MetadataStore>,
pub performance: S3PerformanceConfig,
} }
// Keep streamed single-PUT parts aligned with the distributed backend's
// large-object chunking so GET does not degrade into many small serial reads.
const DEFAULT_STREAMING_PUT_THRESHOLD_BYTES: usize = 16 * 1024 * 1024;
const DEFAULT_INLINE_PUT_MAX_BYTES: usize = 128 * 1024 * 1024;
const DEFAULT_MULTIPART_PUT_CONCURRENCY: usize = 4;
impl S3State { impl S3State {
pub fn new(storage: Arc<dyn StorageBackend>, metadata: Arc<MetadataStore>) -> Self { pub fn new(storage: Arc<dyn StorageBackend>, metadata: Arc<MetadataStore>) -> Self {
Self { storage, metadata } Self::new_with_config(storage, metadata, S3PerformanceConfig::default())
}
pub fn new_with_config(
storage: Arc<dyn StorageBackend>,
metadata: Arc<MetadataStore>,
performance: S3PerformanceConfig,
) -> Self {
Self {
storage,
metadata,
performance,
}
} }
} }
@ -57,7 +65,12 @@ pub fn create_router_with_state(
storage: Arc<dyn StorageBackend>, storage: Arc<dyn StorageBackend>,
metadata: Arc<MetadataStore>, metadata: Arc<MetadataStore>,
) -> Router { ) -> Router {
create_router_with_auth_state(storage, metadata, Arc::new(AuthState::new(None))) create_router_with_auth_state(
storage,
metadata,
Arc::new(AuthState::new(None)),
S3PerformanceConfig::default(),
)
} }
/// Create the S3-compatible HTTP router with auth and storage backends /// Create the S3-compatible HTTP router with auth and storage backends
@ -66,15 +79,30 @@ pub fn create_router_with_auth(
metadata: Arc<MetadataStore>, metadata: Arc<MetadataStore>,
iam_endpoint: Option<String>, iam_endpoint: Option<String>,
) -> Router { ) -> Router {
create_router_with_auth_state(storage, metadata, Arc::new(AuthState::new(iam_endpoint))) create_router_with_auth_config(storage, metadata, iam_endpoint, S3Config::default())
}
pub fn create_router_with_auth_config(
storage: Arc<dyn StorageBackend>,
metadata: Arc<MetadataStore>,
iam_endpoint: Option<String>,
config: S3Config,
) -> Router {
create_router_with_auth_state(
storage,
metadata,
Arc::new(AuthState::new_with_config(iam_endpoint, &config.auth)),
config.performance,
)
} }
fn create_router_with_auth_state( fn create_router_with_auth_state(
storage: Arc<dyn StorageBackend>, storage: Arc<dyn StorageBackend>,
metadata: Arc<MetadataStore>, metadata: Arc<MetadataStore>,
auth_state: Arc<AuthState>, auth_state: Arc<AuthState>,
performance: S3PerformanceConfig,
) -> Router { ) -> Router {
let state = Arc::new(S3State::new(storage, metadata)); let state = Arc::new(S3State::new_with_config(storage, metadata, performance));
Router::new() Router::new()
// Catch-all route for ALL operations (including root /) // Catch-all route for ALL operations (including root /)
@ -126,28 +154,16 @@ fn request_tenant(extensions: &axum::http::Extensions) -> TenantContext {
}) })
} }
fn streaming_put_threshold_bytes() -> usize { fn streaming_put_threshold_bytes(state: &Arc<S3State>) -> usize {
std::env::var("LIGHTNINGSTOR_S3_STREAMING_PUT_THRESHOLD_BYTES") state.performance.streaming_put_threshold_bytes
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_STREAMING_PUT_THRESHOLD_BYTES)
} }
fn inline_put_max_bytes() -> usize { fn inline_put_max_bytes(state: &Arc<S3State>) -> usize {
std::env::var("LIGHTNINGSTOR_S3_INLINE_PUT_MAX_BYTES") state.performance.inline_put_max_bytes
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_INLINE_PUT_MAX_BYTES)
} }
fn multipart_put_concurrency() -> usize { fn multipart_put_concurrency(state: &Arc<S3State>) -> usize {
std::env::var("LIGHTNINGSTOR_S3_MULTIPART_PUT_CONCURRENCY") state.performance.multipart_put_concurrency
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_MULTIPART_PUT_CONCURRENCY)
} }
fn request_content_length(headers: &HeaderMap) -> Option<usize> { fn request_content_length(headers: &HeaderMap) -> Option<usize> {
@ -751,13 +767,13 @@ async fn put_object(
(body_len, etag, None, Some(body_bytes)) (body_len, etag, None, Some(body_bytes))
} }
None => { None => {
let prepared = if let Some(content_length) = let prepared = if let Some(content_length) = content_length
content_length.filter(|content_length| *content_length <= inline_put_max_bytes()) .filter(|content_length| *content_length <= inline_put_max_bytes(&state))
{ {
read_inline_put_body( read_inline_put_body(
body, body,
verified_payload_hash.as_deref(), verified_payload_hash.as_deref(),
inline_put_max_bytes(), inline_put_max_bytes(&state),
Some(content_length), Some(content_length),
) )
.await .await
@ -948,7 +964,7 @@ async fn stream_put_body(
) -> Result<PreparedPutBody, Response<Body>> { ) -> Result<PreparedPutBody, Response<Body>> {
let verify_payload_hash = expected_payload_hash let verify_payload_hash = expected_payload_hash
.filter(|expected_payload_hash| *expected_payload_hash != "UNSIGNED-PAYLOAD"); .filter(|expected_payload_hash| *expected_payload_hash != "UNSIGNED-PAYLOAD");
let threshold = streaming_put_threshold_bytes(); let threshold = streaming_put_threshold_bytes(state);
let mut buffered = BytesMut::with_capacity(threshold); let mut buffered = BytesMut::with_capacity(threshold);
let mut full_md5 = Some(Md5::new()); let mut full_md5 = Some(Md5::new());
let mut full_sha256 = verify_payload_hash.map(|_| Sha256::new()); let mut full_sha256 = verify_payload_hash.map(|_| Sha256::new());
@ -958,7 +974,7 @@ async fn stream_put_body(
let mut scheduled_part_numbers = Vec::new(); let mut scheduled_part_numbers = Vec::new();
let mut completed_parts = Vec::new(); let mut completed_parts = Vec::new();
let mut in_flight_uploads = FuturesUnordered::new(); let mut in_flight_uploads = FuturesUnordered::new();
let max_in_flight_uploads = multipart_put_concurrency(); let max_in_flight_uploads = multipart_put_concurrency(state);
while let Some(frame) = body.frame().await { while let Some(frame) = body.frame().await {
let frame = match frame { let frame = match frame {
@ -1282,7 +1298,10 @@ fn multipart_object_body(state: Arc<S3State>, object: &Object, upload: Multipart
))); )));
} }
return Ok(Some((bytes.slice(0..body_end), (storage, upload, idx, offset)))); return Ok(Some((
bytes.slice(0..body_end),
(storage, upload, idx, offset),
)));
} }
Ok(None) Ok(None)
@ -1374,7 +1393,11 @@ async fn get_object(
); );
} }
let multipart_upload = match state.metadata.load_object_multipart_upload(&object.id).await { let multipart_upload = match state
.metadata
.load_object_multipart_upload(&object.id)
.await
{
Ok(upload) => upload, Ok(upload) => upload,
Err(e) => { Err(e) => {
return error_response( return error_response(
@ -1385,7 +1408,10 @@ async fn get_object(
} }
}; };
let (body, content_length) = if let Some(upload) = multipart_upload { let (body, content_length) = if let Some(upload) = multipart_upload {
(multipart_object_body(Arc::clone(&state), &object, upload), object.size as usize) (
multipart_object_body(Arc::clone(&state), &object, upload),
object.size as usize,
)
} else { } else {
let data = match state.storage.get_object(&object.id).await { let data = match state.storage.get_object(&object.id).await {
Ok(data) => data, Ok(data) => data,
@ -1653,6 +1679,7 @@ mod tests {
storage, storage,
metadata.clone(), metadata.clone(),
Arc::new(AuthState::disabled()), Arc::new(AuthState::disabled()),
S3PerformanceConfig::default(),
); );
std::mem::forget(tempdir); std::mem::forget(tempdir);
(router, metadata) (router, metadata)
@ -1982,7 +2009,8 @@ mod tests {
.unwrap(); .unwrap();
assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.status(), StatusCode::OK);
let body = vec![b'x'; DEFAULT_STREAMING_PUT_THRESHOLD_BYTES + 1024]; let threshold = S3PerformanceConfig::default().streaming_put_threshold_bytes;
let body = vec![b'x'; threshold + 1024];
let response = router let response = router
.clone() .clone()
.oneshot( .oneshot(
@ -2197,10 +2225,12 @@ mod tests {
async fn large_put_streams_multipart_parts_with_parallel_uploads() { async fn large_put_streams_multipart_parts_with_parallel_uploads() {
let storage = Arc::new(DelayedMultipartStorage::new(Duration::from_millis(25))); let storage = Arc::new(DelayedMultipartStorage::new(Duration::from_millis(25)));
let metadata = Arc::new(MetadataStore::new_in_memory()); let metadata = Arc::new(MetadataStore::new_in_memory());
let performance = S3PerformanceConfig::default();
let router = create_router_with_auth_state( let router = create_router_with_auth_state(
storage.clone(), storage.clone(),
metadata, metadata,
Arc::new(AuthState::disabled()), Arc::new(AuthState::disabled()),
performance.clone(),
); );
let response = router let response = router
@ -2216,7 +2246,7 @@ mod tests {
.unwrap(); .unwrap();
assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.status(), StatusCode::OK);
let body = vec![b'x'; (DEFAULT_STREAMING_PUT_THRESHOLD_BYTES * 2) + 4096]; let body = vec![b'x'; (performance.streaming_put_threshold_bytes * 2) + 4096];
let response = router let response = router
.clone() .clone()
.oneshot( .oneshot(
@ -2250,10 +2280,12 @@ mod tests {
async fn moderate_put_with_content_length_stays_inline() { async fn moderate_put_with_content_length_stays_inline() {
let storage = Arc::new(DelayedMultipartStorage::new(Duration::from_millis(25))); let storage = Arc::new(DelayedMultipartStorage::new(Duration::from_millis(25)));
let metadata = Arc::new(MetadataStore::new_in_memory()); let metadata = Arc::new(MetadataStore::new_in_memory());
let performance = S3PerformanceConfig::default();
let router = create_router_with_auth_state( let router = create_router_with_auth_state(
storage.clone(), storage.clone(),
metadata.clone(), metadata.clone(),
Arc::new(AuthState::disabled()), Arc::new(AuthState::disabled()),
performance.clone(),
); );
let response = router let response = router
@ -2269,7 +2301,7 @@ mod tests {
.unwrap(); .unwrap();
assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.status(), StatusCode::OK);
let body = vec![b'y'; DEFAULT_STREAMING_PUT_THRESHOLD_BYTES + 4096]; let body = vec![b'y'; performance.streaming_put_threshold_bytes + 4096];
let response = router let response = router
.clone() .clone()
.oneshot( .oneshot(

View file

@ -170,5 +170,8 @@ pub struct CompleteMultipartUploadResult {
/// Convert to XML with declaration /// Convert to XML with declaration
pub fn to_xml<T: Serialize>(value: &T) -> Result<String, quick_xml::DeError> { pub fn to_xml<T: Serialize>(value: &T) -> Result<String, quick_xml::DeError> {
let xml = xml_to_string(value)?; let xml = xml_to_string(value)?;
Ok(format!("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n{}", xml)) Ok(format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n{}",
xml
))
} }

View file

@ -6,6 +6,7 @@
use anyhow::Result; use anyhow::Result;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::fs; use std::fs;
use std::path::Path;
/// Main server configuration /// Main server configuration
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
@ -123,7 +124,7 @@ pub struct TlsConfig {
impl Config { impl Config {
/// Load configuration from a YAML file /// Load configuration from a YAML file
pub fn from_file(path: &str) -> Result<Self> { pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
let content = fs::read_to_string(path)?; let content = fs::read_to_string(path)?;
let config = serde_yaml::from_str(&content)?; let config = serde_yaml::from_str(&content)?;
Ok(config) Ok(config)
@ -136,27 +137,6 @@ impl Config {
fs::write(path, content)?; fs::write(path, content)?;
Ok(()) Ok(())
} }
/// Apply environment variable overrides
///
/// This allows NixOS service module to override configuration via environment variables.
/// Environment variables take precedence over configuration file values.
pub fn apply_env_overrides(&mut self) {
if let Ok(val) = std::env::var("NIGHTLIGHT_HTTP_ADDR") {
self.server.http_addr = val;
}
if let Ok(val) = std::env::var("NIGHTLIGHT_GRPC_ADDR") {
self.server.grpc_addr = val;
}
if let Ok(val) = std::env::var("NIGHTLIGHT_DATA_DIR") {
self.storage.data_dir = val;
}
if let Ok(val) = std::env::var("NIGHTLIGHT_RETENTION_DAYS") {
if let Ok(days) = val.parse() {
self.storage.retention_days = days;
}
}
}
} }
impl Default for Config { impl Default for Config {

View file

@ -183,7 +183,11 @@ impl Admin for AdminServiceImpl {
_request: Request<HealthRequest>, _request: Request<HealthRequest>,
) -> Result<Response<HealthResponse>, Status> { ) -> Result<Response<HealthResponse>, Status> {
let storage_result = self.storage.stats().await; let storage_result = self.storage.stats().await;
let status = if storage_result.is_ok() { "ok" } else { "degraded" }; let status = if storage_result.is_ok() {
"ok"
} else {
"degraded"
};
let storage_message = match &storage_result { let storage_message = match &storage_result {
Ok(_) => "storage ready".to_string(), Ok(_) => "storage ready".to_string(),
Err(error) => error.to_string(), Err(error) => error.to_string(),
@ -253,7 +257,9 @@ impl Admin for AdminServiceImpl {
version: env!("CARGO_PKG_VERSION").to_string(), version: env!("CARGO_PKG_VERSION").to_string(),
commit: option_env!("GIT_COMMIT").unwrap_or("unknown").to_string(), commit: option_env!("GIT_COMMIT").unwrap_or("unknown").to_string(),
build_time: option_env!("BUILD_TIME").unwrap_or("unknown").to_string(), build_time: option_env!("BUILD_TIME").unwrap_or("unknown").to_string(),
rust_version: option_env!("RUSTC_VERSION").unwrap_or("unknown").to_string(), rust_version: option_env!("RUSTC_VERSION")
.unwrap_or("unknown")
.to_string(),
target: format!("{}-{}", std::env::consts::ARCH, std::env::consts::OS), target: format!("{}-{}", std::env::consts::ARCH, std::env::consts::OS),
})) }))
} }
@ -330,9 +336,7 @@ mod tests {
use super::*; use super::*;
use crate::ingestion::IngestionService; use crate::ingestion::IngestionService;
use crate::storage::Storage; use crate::storage::Storage;
use nightlight_api::nightlight::{ use nightlight_api::nightlight::{InstantQueryRequest, LabelValuesRequest, SeriesQueryRequest};
InstantQueryRequest, LabelValuesRequest, SeriesQueryRequest,
};
use nightlight_api::prometheus::{Label, Sample, TimeSeries, WriteRequest}; use nightlight_api::prometheus::{Label, Sample, TimeSeries, WriteRequest};
#[tokio::test] #[tokio::test]
@ -380,7 +384,10 @@ mod tests {
data.result[0].metric.get("__name__").map(String::as_str), data.result[0].metric.get("__name__").map(String::as_str),
Some("grpc_metric") Some("grpc_metric")
); );
assert_eq!(data.result[0].value.as_ref().map(|value| value.value), Some(12.5)); assert_eq!(
data.result[0].value.as_ref().map(|value| value.value),
Some(12.5)
);
} }
#[tokio::test] #[tokio::test]
@ -452,7 +459,10 @@ mod tests {
.unwrap() .unwrap()
.into_inner(); .into_inner();
assert_eq!(label_values.status, "success"); assert_eq!(label_values.status, "success");
assert_eq!(label_values.data, vec!["api".to_string(), "worker".to_string()]); assert_eq!(
label_values.data,
vec!["api".to_string(), "worker".to_string()]
);
} }
#[tokio::test] #[tokio::test]
@ -477,26 +487,33 @@ mod tests {
.unwrap(); .unwrap();
let query = QueryService::from_storage(storage.queryable()); let query = QueryService::from_storage(storage.queryable());
query.execute_instant_query("admin_metric", 2_000).await.unwrap(); query
.execute_instant_query("admin_metric", 2_000)
.await
.unwrap();
let admin = AdminServiceImpl::new( let admin =
Arc::clone(&storage), AdminServiceImpl::new(Arc::clone(&storage), ingestion.metrics(), query.metrics());
ingestion.metrics(),
query.metrics(),
);
let stats = admin let stats = admin
.stats(Request::new(StatsRequest {})) .stats(Request::new(StatsRequest {}))
.await .await
.unwrap() .unwrap()
.into_inner(); .into_inner();
assert_eq!(stats.storage.as_ref().map(|value| value.total_samples), Some(1));
assert_eq!( assert_eq!(
stats.ingestion stats.storage.as_ref().map(|value| value.total_samples),
Some(1)
);
assert_eq!(
stats
.ingestion
.as_ref() .as_ref()
.map(|value| value.samples_ingested_total), .map(|value| value.samples_ingested_total),
Some(1) Some(1)
); );
assert_eq!(stats.query.as_ref().map(|value| value.queries_total), Some(1)); assert_eq!(
stats.query.as_ref().map(|value| value.queries_total),
Some(1)
);
} }
} }

View file

@ -15,8 +15,8 @@ use nightlight_api::prometheus::{Label, WriteRequest};
use nightlight_types::Error; use nightlight_types::Error;
use prost::Message; use prost::Message;
use snap::raw::Decoder as SnappyDecoder; use snap::raw::Decoder as SnappyDecoder;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Instant; use std::time::Instant;
use tracing::{debug, error, info, warn}; use tracing::{debug, error, info, warn};
@ -113,9 +113,8 @@ impl IngestionService {
} }
// Store series with samples in shared storage // Store series with samples in shared storage
let series_id = nightlight_types::SeriesId( let series_id =
compute_series_fingerprint(&internal_labels) nightlight_types::SeriesId(compute_series_fingerprint(&internal_labels));
);
let time_series = nightlight_types::TimeSeries { let time_series = nightlight_types::TimeSeries {
id: series_id, id: series_id,
@ -178,11 +177,11 @@ impl IngestionMetrics {
} }
/// Axum handler for /api/v1/write endpoint /// Axum handler for /api/v1/write endpoint
async fn handle_remote_write( async fn handle_remote_write(State(service): State<IngestionService>, body: Bytes) -> Response {
State(service): State<IngestionService>, service
body: Bytes, .metrics
) -> Response { .requests_total
service.metrics.requests_total.fetch_add(1, Ordering::Relaxed); .fetch_add(1, Ordering::Relaxed);
debug!("Received remote_write request, size: {} bytes", body.len()); debug!("Received remote_write request, size: {} bytes", body.len());
@ -225,17 +224,26 @@ async fn handle_remote_write(
} }
Err(Error::Storage(msg)) if msg.contains("buffer full") => { Err(Error::Storage(msg)) if msg.contains("buffer full") => {
warn!("Write buffer full, returning 429"); warn!("Write buffer full, returning 429");
service.metrics.requests_failed.fetch_add(1, Ordering::Relaxed); service
.metrics
.requests_failed
.fetch_add(1, Ordering::Relaxed);
IngestionError::Backpressure.into_response() IngestionError::Backpressure.into_response()
} }
Err(Error::InvalidLabel(msg)) => { Err(Error::InvalidLabel(msg)) => {
warn!("Invalid labels: {}", msg); warn!("Invalid labels: {}", msg);
service.metrics.requests_failed.fetch_add(1, Ordering::Relaxed); service
.metrics
.requests_failed
.fetch_add(1, Ordering::Relaxed);
IngestionError::InvalidLabels.into_response() IngestionError::InvalidLabels.into_response()
} }
Err(e) => { Err(e) => {
error!("Failed to process write request: {}", e); error!("Failed to process write request: {}", e);
service.metrics.requests_failed.fetch_add(1, Ordering::Relaxed); service
.metrics
.requests_failed
.fetch_add(1, Ordering::Relaxed);
IngestionError::StorageError.into_response() IngestionError::StorageError.into_response()
} }
} }
@ -285,7 +293,11 @@ fn validate_labels(labels: Vec<Label>) -> Result<Vec<Label>, Error> {
} }
// Label names must contain only [a-zA-Z0-9_] // Label names must contain only [a-zA-Z0-9_]
if !label.name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { if !label
.name
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_')
{
return Err(Error::InvalidLabel(format!( return Err(Error::InvalidLabel(format!(
"Invalid label name '{}': must contain only [a-zA-Z0-9_]", "Invalid label name '{}': must contain only [a-zA-Z0-9_]",
label.name label.name
@ -337,15 +349,12 @@ impl IntoResponse for IngestionError {
IngestionError::InvalidProtobuf => { IngestionError::InvalidProtobuf => {
(StatusCode::BAD_REQUEST, "Invalid protobuf encoding") (StatusCode::BAD_REQUEST, "Invalid protobuf encoding")
} }
IngestionError::InvalidLabels => { IngestionError::InvalidLabels => (StatusCode::BAD_REQUEST, "Invalid metric labels"),
(StatusCode::BAD_REQUEST, "Invalid metric labels") IngestionError::StorageError => (StatusCode::INTERNAL_SERVER_ERROR, "Storage error"),
} IngestionError::Backpressure => (
IngestionError::StorageError => { StatusCode::TOO_MANY_REQUESTS,
(StatusCode::INTERNAL_SERVER_ERROR, "Storage error") "Write buffer full, retry later",
} ),
IngestionError::Backpressure => {
(StatusCode::TOO_MANY_REQUESTS, "Write buffer full, retry later")
}
}; };
(status, message).into_response() (status, message).into_response()

View file

@ -12,6 +12,7 @@ use std::time::Duration;
use anyhow::Result; use anyhow::Result;
use axum::{routing::get, Router}; use axum::{routing::get, Router};
use clap::Parser;
use nightlight_api::nightlight::admin_server::AdminServer; use nightlight_api::nightlight::admin_server::AdminServer;
use nightlight_api::nightlight::metric_query_server::MetricQueryServer; use nightlight_api::nightlight::metric_query_server::MetricQueryServer;
use tokio::time::MissedTickBehavior; use tokio::time::MissedTickBehavior;
@ -33,8 +34,18 @@ use storage::Storage;
const DEFAULT_SNAPSHOT_INTERVAL_SECS: u64 = 30; const DEFAULT_SNAPSHOT_INTERVAL_SECS: u64 = 30;
#[derive(Parser, Debug)]
#[command(name = "nightlight-server")]
struct Args {
/// Configuration file path
#[arg(short, long, default_value = "nightlight.yaml")]
config: std::path::PathBuf,
}
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
let args = Args::parse();
tracing_subscriber::fmt() tracing_subscriber::fmt()
.with_max_level(Level::INFO) .with_max_level(Level::INFO)
.with_target(false) .with_target(false)
@ -44,17 +55,20 @@ async fn main() -> Result<()> {
info!("Nightlight server starting"); info!("Nightlight server starting");
info!("Version: {}", env!("CARGO_PKG_VERSION")); info!("Version: {}", env!("CARGO_PKG_VERSION"));
let mut config = match Config::from_file("config.yaml") { let config = match Config::from_file(&args.config) {
Ok(config) => { Ok(config) => {
info!("Configuration loaded from config.yaml"); info!("Configuration loaded from {}", args.config.display());
config config
} }
Err(error) => { Err(error) => {
info!("Failed to load config.yaml: {}, using defaults", error); info!(
"Failed to load {}: {}, using defaults",
args.config.display(),
error
);
Config::default() Config::default()
} }
}; };
config.apply_env_overrides();
if config.tls.is_some() { if config.tls.is_some() {
warn!("Nightlight TLS configuration is currently ignored; starting plaintext listeners"); warn!("Nightlight TLS configuration is currently ignored; starting plaintext listeners");
@ -133,7 +147,9 @@ async fn main() -> Result<()> {
info!(" - Admin.Health / Stats / BuildInfo"); info!(" - Admin.Health / Stats / BuildInfo");
let shutdown = async { let shutdown = async {
tokio::signal::ctrl_c().await.expect("failed to install Ctrl+C handler"); tokio::signal::ctrl_c()
.await
.expect("failed to install Ctrl+C handler");
}; };
tokio::pin!(shutdown); tokio::pin!(shutdown);
@ -163,8 +179,9 @@ async fn maintenance_loop(
config: StorageConfig, config: StorageConfig,
mut shutdown: tokio::sync::broadcast::Receiver<()>, mut shutdown: tokio::sync::broadcast::Receiver<()>,
) { ) {
let snapshot_interval_secs = let snapshot_interval_secs = config
config.compaction_interval_seconds.clamp(5, DEFAULT_SNAPSHOT_INTERVAL_SECS); .compaction_interval_seconds
.clamp(5, DEFAULT_SNAPSHOT_INTERVAL_SECS);
let mut snapshot_interval = tokio::time::interval(Duration::from_secs(snapshot_interval_secs)); let mut snapshot_interval = tokio::time::interval(Duration::from_secs(snapshot_interval_secs));
snapshot_interval.set_missed_tick_behavior(MissedTickBehavior::Delay); snapshot_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);

View file

@ -49,7 +49,7 @@ let
raft_addr = raftAddr; raft_addr = raftAddr;
}) })
cfg.initialPeers; cfg.initialPeers;
chainfireConfigFile = tomlFormat.generate "chainfire.toml" { generatedConfig = {
node = { node = {
id = numericId cfg.nodeId; id = numericId cfg.nodeId;
name = cfg.nodeId; name = cfg.nodeId;
@ -73,6 +73,7 @@ let
role = cfg.raftRole; role = cfg.raftRole;
}; };
}; };
chainfireConfigFile = tomlFormat.generate "chainfire.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
in in
{ {
options.services.chainfire = { options.services.chainfire = {

View file

@ -75,6 +75,8 @@ let
fiberlbBaseConfig = { fiberlbBaseConfig = {
grpc_addr = "0.0.0.0:${toString cfg.port}"; grpc_addr = "0.0.0.0:${toString cfg.port}";
log_level = "info"; log_level = "info";
metadata_backend = cfg.metadataBackend;
single_node = cfg.singleNode;
auth = { auth = {
iam_server_addr = iam_server_addr =
if cfg.iamAddr != null if cfg.iamAddr != null
@ -93,6 +95,8 @@ let
vip_ownership = { vip_ownership = {
enabled = cfg.vipOwnership.enable; enabled = cfg.vipOwnership.enable;
interface = cfg.vipOwnership.interface; interface = cfg.vipOwnership.interface;
} // lib.optionalAttrs (cfg.vipOwnership.ipCommand != null) {
ip_command = cfg.vipOwnership.ipCommand;
}; };
} // lib.optionalAttrs cfg.bgp.enable { } // lib.optionalAttrs cfg.bgp.enable {
bgp = bgp =
@ -127,6 +131,15 @@ let
// lib.optionalAttrs (cfg.bgp.nextHop != null) { // lib.optionalAttrs (cfg.bgp.nextHop != null) {
next_hop = cfg.bgp.nextHop; next_hop = cfg.bgp.nextHop;
}; };
}
// lib.optionalAttrs (cfg.flaredbAddr != null) {
flaredb_endpoint = cfg.flaredbAddr;
}
// lib.optionalAttrs (normalizedDatabaseUrl != null) {
metadata_database_url = normalizedDatabaseUrl;
}
// lib.optionalAttrs (cfg.chainfireAddr != null) {
chainfire_endpoint = "http://${cfg.chainfireAddr}";
}; };
fiberlbConfigFile = tomlFormat.generate "fiberlb.toml" (lib.recursiveUpdate fiberlbBaseConfig cfg.settings); fiberlbConfigFile = tomlFormat.generate "fiberlb.toml" (lib.recursiveUpdate fiberlbBaseConfig cfg.settings);
flaredbDependencies = lib.optional (cfg.metadataBackend == "flaredb") "flaredb.service"; flaredbDependencies = lib.optional (cfg.metadataBackend == "flaredb") "flaredb.service";
@ -222,6 +235,13 @@ in
default = "lo"; default = "lo";
description = "Interface where FiberLB should claim VIP /32 addresses."; description = "Interface where FiberLB should claim VIP /32 addresses.";
}; };
ipCommand = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Optional explicit path to the `ip` command used for VIP ownership.";
example = "/run/current-system/sw/bin/ip";
};
}; };
vipDrain = { vipDrain = {
@ -367,22 +387,7 @@ in
AmbientCapabilities = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ]; AmbientCapabilities = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
CapabilityBoundingSet = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ]; CapabilityBoundingSet = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
# Environment variables for service endpoints ExecStart = "${cfg.package}/bin/fiberlb --config ${fiberlbConfigFile}";
Environment = [
"RUST_LOG=info"
"FIBERLB_FLAREDB_ENDPOINT=${if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479"}"
"FIBERLB_METADATA_BACKEND=${cfg.metadataBackend}"
] ++ lib.optional (normalizedDatabaseUrl != null) "FIBERLB_METADATA_DATABASE_URL=${normalizedDatabaseUrl}"
++ lib.optional cfg.singleNode "FIBERLB_SINGLE_NODE=true"
++ lib.optional (cfg.chainfireAddr != null) "FIBERLB_CHAINFIRE_ENDPOINT=http://${cfg.chainfireAddr}";
# Start command
ExecStart = lib.concatStringsSep " " ([
"${cfg.package}/bin/fiberlb"
"--config ${fiberlbConfigFile}"
"--grpc-addr 0.0.0.0:${toString cfg.port}"
"--flaredb-endpoint ${if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479"}"
]);
}; };
}; };
}; };

View file

@ -25,7 +25,7 @@ let
// lib.optionalAttrs (cfg.databaseUrl != null) { // lib.optionalAttrs (cfg.databaseUrl != null) {
metadata_database_url = cfg.databaseUrl; metadata_database_url = cfg.databaseUrl;
}; };
configFile = tomlFormat.generate "flashdns.toml" generatedConfig; configFile = tomlFormat.generate "flashdns.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
in in
{ {
options.services.flashdns = { options.services.flashdns = {

View file

@ -3,12 +3,19 @@
let let
cfg = config.services.iam; cfg = config.services.iam;
tomlFormat = pkgs.formats.toml { }; tomlFormat = pkgs.formats.toml { };
iamConfigFile = tomlFormat.generate "iam.toml" { generatedConfig = {
server = { server = {
addr = "0.0.0.0:${toString cfg.port}"; addr = "0.0.0.0:${toString cfg.port}";
http_addr = "0.0.0.0:${toString cfg.httpPort}"; http_addr = "0.0.0.0:${toString cfg.httpPort}";
}; };
logging.level = "info"; logging.level = "info";
admin = {
allow_unauthenticated = cfg.allowUnauthenticatedAdmin;
};
dev = {
allow_random_signing_key = cfg.allowRandomSigningKey;
allow_memory_backend = cfg.storeBackend == "memory";
};
store = { store = {
backend = cfg.storeBackend; backend = cfg.storeBackend;
flaredb_endpoint = flaredb_endpoint =
@ -25,6 +32,7 @@ let
chainfire_endpoint = cfg.chainfireAddr; chainfire_endpoint = cfg.chainfireAddr;
}; };
}; };
iamConfigFile = tomlFormat.generate "iam.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
in in
{ {
options.services.iam = { options.services.iam = {
@ -87,6 +95,18 @@ in
description = "Admin token injected as IAM_ADMIN_TOKEN for privileged IAM APIs."; description = "Admin token injected as IAM_ADMIN_TOKEN for privileged IAM APIs.";
}; };
allowRandomSigningKey = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Allow IAM to generate a random signing key when authn.internal_token.signing_key is unset (dev only).";
};
allowUnauthenticatedAdmin = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Allow privileged admin APIs without an admin token (dev only).";
};
settings = lib.mkOption { settings = lib.mkOption {
type = lib.types.attrs; type = lib.types.attrs;
default = {}; default = {};
@ -119,20 +139,6 @@ in
wants = [ "chainfire.service" "flaredb.service" ]; wants = [ "chainfire.service" "flaredb.service" ];
environment = lib.mkMerge [ environment = lib.mkMerge [
{
CHAINFIRE_ENDPOINT = if cfg.chainfireAddr != null then cfg.chainfireAddr else "127.0.0.1:2379";
FLAREDB_ENDPOINT = if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479";
IAM_STORE_BACKEND = cfg.storeBackend;
}
(lib.mkIf (cfg.databaseUrl != null) {
IAM_DATABASE_URL = cfg.databaseUrl;
})
(lib.mkIf cfg.singleNode {
IAM_SINGLE_NODE = "true";
})
(lib.mkIf (cfg.storeBackend == "memory") {
IAM_ALLOW_MEMORY_BACKEND = "1";
})
(lib.mkIf (cfg.adminToken != null) { (lib.mkIf (cfg.adminToken != null) {
IAM_ADMIN_TOKEN = cfg.adminToken; IAM_ADMIN_TOKEN = cfg.adminToken;
}) })

View file

@ -2,6 +2,59 @@
let let
cfg = config.services.k8shost; cfg = config.services.k8shost;
tomlFormat = pkgs.formats.toml { };
generatedConfig = {
server = {
addr = "0.0.0.0:${toString cfg.port}";
http_addr = "127.0.0.1:${toString cfg.httpPort}";
log_level = "info";
};
flaredb = {
pd_addr = cfg.flaredbPdAddr;
direct_addr = cfg.flaredbDirectAddr;
};
chainfire = {
endpoint = cfg.chainfireAddr;
};
iam = {
server_addr =
if cfg.iamAddr != null
then cfg.iamAddr
else "http://127.0.0.1:50080";
};
creditservice =
lib.optionalAttrs
(
cfg.creditserviceAddr != null
|| ((config.services ? creditservice) && config.services.creditservice.enable)
)
{
server_addr =
if cfg.creditserviceAddr != null
then cfg.creditserviceAddr
else "http://127.0.0.1:${toString config.services.creditservice.grpcPort}";
};
fiberlb = {
server_addr =
if cfg.fiberlbAddr != null
then cfg.fiberlbAddr
else "http://127.0.0.1:50085";
};
flashdns = {
server_addr =
if cfg.flashdnsAddr != null
then cfg.flashdnsAddr
else "http://127.0.0.1:50084";
};
prismnet = {
server_addr =
if cfg.prismnetAddr != null
then cfg.prismnetAddr
else "http://127.0.0.1:50081";
};
};
configFile = tomlFormat.generate "k8shost.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
configPath = "/etc/k8shost/k8shost.toml";
in in
{ {
options.services.k8shost = { options.services.k8shost = {
@ -26,6 +79,13 @@ in
example = "http://10.0.0.1:50080"; example = "http://10.0.0.1:50080";
}; };
creditserviceAddr = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "CreditService endpoint address (http://host:port) for pod admission and scheduler quota enforcement.";
example = "http://10.0.0.1:3010";
};
chainfireAddr = lib.mkOption { chainfireAddr = lib.mkOption {
type = lib.types.nullOr lib.types.str; type = lib.types.nullOr lib.types.str;
default = null; default = null;
@ -123,24 +183,10 @@ in
ProtectHome = true; ProtectHome = true;
ReadWritePaths = [ cfg.dataDir ]; ReadWritePaths = [ cfg.dataDir ];
# Environment variables for service endpoints ExecStart = "${cfg.package}/bin/k8shost-server --config ${configPath}";
Environment = [ };
"RUST_LOG=info" };
];
# Start command environment.etc."k8shost/k8shost.toml".source = configFile;
ExecStart = lib.concatStringsSep " " ([
"${cfg.package}/bin/k8shost-server"
"--addr 0.0.0.0:${toString cfg.port}"
"--http-addr 127.0.0.1:${toString cfg.httpPort}"
] ++ lib.optional (cfg.iamAddr != null) "--iam-server-addr ${cfg.iamAddr}"
++ lib.optional (cfg.chainfireAddr != null) "--chainfire-endpoint ${cfg.chainfireAddr}"
++ lib.optional (cfg.prismnetAddr != null) "--prismnet-server-addr ${cfg.prismnetAddr}"
++ lib.optional (cfg.flaredbPdAddr != null) "--flaredb-pd-addr ${cfg.flaredbPdAddr}"
++ lib.optional (cfg.flaredbDirectAddr != null) "--flaredb-direct-addr ${cfg.flaredbDirectAddr}"
++ lib.optional (cfg.fiberlbAddr != null) "--fiberlb-server-addr ${cfg.fiberlbAddr}"
++ lib.optional (cfg.flashdnsAddr != null) "--flashdns-server-addr ${cfg.flashdnsAddr}");
};
};
}; };
} }

View file

@ -73,6 +73,22 @@ let
// lib.optionalAttrs (cfg.distributedRegistryEndpoint != null) { // lib.optionalAttrs (cfg.distributedRegistryEndpoint != null) {
registry_endpoint = cfg.distributedRegistryEndpoint; registry_endpoint = cfg.distributedRegistryEndpoint;
}; };
s3 = {
auth = {
enabled = cfg.s3AuthEnabled;
aws_region = cfg.s3AwsRegion;
iam_cache_ttl_secs = cfg.s3IamCacheTtlSecs;
default_org_id = cfg.s3DefaultOrgId;
default_project_id = cfg.s3DefaultProjectId;
max_auth_body_bytes = cfg.s3MaxAuthBodyBytes;
};
performance = {
streaming_put_threshold_bytes = cfg.s3StreamingPutThresholdBytes;
inline_put_max_bytes = cfg.s3InlinePutMaxBytes;
multipart_put_concurrency = cfg.s3MultipartPutConcurrency;
multipart_fetch_concurrency = cfg.s3MultipartFetchConcurrency;
};
};
} }
// lib.optionalAttrs (cfg.flaredbAddr != null) { // lib.optionalAttrs (cfg.flaredbAddr != null) {
flaredb_endpoint = cfg.flaredbAddr; flaredb_endpoint = cfg.flaredbAddr;
@ -321,6 +337,54 @@ in
description = "Maximum concurrent multipart GET part fetches."; description = "Maximum concurrent multipart GET part fetches.";
}; };
s3AuthEnabled = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Enable S3 Signature V4 authentication.";
};
s3AwsRegion = lib.mkOption {
type = lib.types.str;
default = "us-east-1";
description = "AWS region name exposed by the S3 compatibility layer.";
};
s3IamCacheTtlSecs = lib.mkOption {
type = lib.types.int;
default = 30;
description = "Seconds to cache IAM-backed S3 credential lookups.";
};
s3DefaultOrgId = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "default";
description = "Default org ID used for static S3 credentials.";
};
s3DefaultProjectId = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "default";
description = "Default project ID used for static S3 credentials.";
};
s3MaxAuthBodyBytes = lib.mkOption {
type = lib.types.int;
default = 1024 * 1024 * 1024;
description = "Maximum request body size buffered during S3 auth verification.";
};
s3AccessKeyId = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Optional static S3 access key ID injected via environment for dev/test.";
};
s3SecretKey = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Optional static S3 secret key injected via environment for dev/test.";
};
databaseUrl = lib.mkOption { databaseUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str; type = lib.types.nullOr lib.types.str;
default = null; default = null;
@ -360,6 +424,13 @@ in
}; };
config = lib.mkIf cfg.enable { config = lib.mkIf cfg.enable {
assertions = [
{
assertion = (cfg.s3AccessKeyId == null) == (cfg.s3SecretKey == null);
message = "services.lightningstor.s3AccessKeyId and services.lightningstor.s3SecretKey must be set together";
}
];
users.users.lightningstor = { users.users.lightningstor = {
isSystemUser = true; isSystemUser = true;
group = "lightningstor"; group = "lightningstor";
@ -391,17 +462,12 @@ in
ExecStart = execStart; ExecStart = execStart;
}; };
environment = { environment = lib.mkMerge [
RUST_LOG = "info"; (lib.mkIf (cfg.s3AccessKeyId != null) {
LIGHTNINGSTOR_S3_STREAMING_PUT_THRESHOLD_BYTES = S3_ACCESS_KEY_ID = cfg.s3AccessKeyId;
toString cfg.s3StreamingPutThresholdBytes; S3_SECRET_KEY = cfg.s3SecretKey;
LIGHTNINGSTOR_S3_INLINE_PUT_MAX_BYTES = })
toString cfg.s3InlinePutMaxBytes; ];
LIGHTNINGSTOR_S3_MULTIPART_PUT_CONCURRENCY =
toString cfg.s3MultipartPutConcurrency;
LIGHTNINGSTOR_S3_MULTIPART_FETCH_CONCURRENCY =
toString cfg.s3MultipartFetchConcurrency;
};
}; };
}; };
} }

View file

@ -2,6 +2,19 @@
let let
cfg = config.services.nightlight; cfg = config.services.nightlight;
yamlFormat = pkgs.formats.yaml { };
generatedConfig = {
server = {
grpc_addr = "0.0.0.0:${toString cfg.grpcPort}";
http_addr = "0.0.0.0:${toString cfg.httpPort}";
};
storage = {
data_dir = toString cfg.dataDir;
retention_days = cfg.retentionDays;
};
};
configFile = yamlFormat.generate "nightlight.yaml" (lib.recursiveUpdate generatedConfig cfg.settings);
configPath = "/etc/nightlight/nightlight.yaml";
in in
{ {
options.services.nightlight = { options.services.nightlight = {
@ -79,19 +92,10 @@ in
ProtectHome = true; ProtectHome = true;
ReadWritePaths = [ cfg.dataDir ]; ReadWritePaths = [ cfg.dataDir ];
# Start command ExecStart = "${cfg.package}/bin/nightlight-server --config ${configPath}";
# Note: nightlight-server uses a config file, so we'll pass data-dir directly };
# The config will be auto-generated or use defaults from Config::default() };
ExecStart = "${cfg.package}/bin/nightlight-server";
# Environment variables for configuration environment.etc."nightlight/nightlight.yaml".source = configFile;
Environment = [
"NIGHTLIGHT_HTTP_ADDR=0.0.0.0:${toString cfg.httpPort}"
"NIGHTLIGHT_GRPC_ADDR=0.0.0.0:${toString cfg.grpcPort}"
"NIGHTLIGHT_DATA_DIR=${cfg.dataDir}"
"NIGHTLIGHT_RETENTION_DAYS=${toString cfg.retentionDays}"
];
};
};
}; };
} }

View file

@ -27,7 +27,7 @@ let
else if cfg.coronafsEndpoint != null then cfg.coronafsEndpoint else if cfg.coronafsEndpoint != null then cfg.coronafsEndpoint
else null; else null;
tomlFormat = pkgs.formats.toml { }; tomlFormat = pkgs.formats.toml { };
plasmavmcConfigFile = tomlFormat.generate "plasmavmc.toml" { generatedConfig = {
addr = "0.0.0.0:${toString cfg.port}"; addr = "0.0.0.0:${toString cfg.port}";
http_addr = "0.0.0.0:${toString cfg.httpPort}"; http_addr = "0.0.0.0:${toString cfg.httpPort}";
log_level = "info"; log_level = "info";
@ -37,7 +37,78 @@ let
then cfg.iamAddr then cfg.iamAddr
else "127.0.0.1:50080"; else "127.0.0.1:50080";
}; };
kvm = {
qemu_path = "${pkgs.qemu}/bin/qemu-system-x86_64";
runtime_dir = "/run/libvirt/plasmavmc";
}; };
storage = {
backend = "flaredb";
flaredb_endpoint =
if cfg.flaredbAddr != null
then cfg.flaredbAddr
else "127.0.0.1:2479";
} // lib.optionalAttrs (cfg.chainfireAddr != null) {
chainfire_endpoint = "http://${cfg.chainfireAddr}";
};
agent =
{
shared_live_migration = cfg.sharedLiveMigration;
}
// lib.optionalAttrs (cfg.mode != "server") {
node_id = cfg.nodeId;
node_name = cfg.nodeName;
heartbeat_interval_secs = cfg.heartbeatIntervalSeconds;
}
// lib.optionalAttrs (cfg.controlPlaneAddr != null) {
control_plane_addr = cfg.controlPlaneAddr;
}
// lib.optionalAttrs (cfg.advertiseAddr != null) {
advertise_endpoint = cfg.advertiseAddr;
};
integrations =
lib.optionalAttrs (cfg.prismnetAddr != null) {
prismnet_endpoint = "http://${cfg.prismnetAddr}";
};
watcher =
lib.optionalAttrs (cfg.chainfireAddr != null) {
enabled = true;
};
health =
lib.optionalAttrs (cfg.mode == "server") {
node_monitor_interval_secs = 5;
node_heartbeat_timeout_secs = 30;
};
artifacts =
{
image_cache_dir = "${toString cfg.dataDir}/images";
}
// lib.optionalAttrs (cfg.lightningstorAddr != null) {
lightningstor_endpoint = cfg.lightningstorAddr;
};
volumes = {
managed_volume_root = toString cfg.managedVolumeRoot;
coronafs =
(lib.optionalAttrs (effectiveCoronafsControllerEndpoint != null) {
controller_endpoint = effectiveCoronafsControllerEndpoint;
})
// (lib.optionalAttrs (effectiveCoronafsNodeEndpoint != null) {
node_endpoint = effectiveCoronafsNodeEndpoint;
})
// (lib.optionalAttrs (cfg.coronafsEndpoint != null) {
endpoint = cfg.coronafsEndpoint;
})
// {
node_local_attach = cfg.coronafsNodeLocalAttach || cfg.experimentalCoronafsNodeLocalAttach;
};
ceph =
(lib.optionalAttrs (cfg.cephMonitors != [ ]) {
monitors = cfg.cephMonitors;
cluster_id = cfg.cephClusterId;
user = cfg.cephUser;
});
};
};
plasmavmcConfigFile = tomlFormat.generate "plasmavmc.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
in in
{ {
options.services.plasmavmc = { options.services.plasmavmc = {
@ -286,64 +357,9 @@ in
exit 1 exit 1
''; '';
environment = lib.mkMerge [ environment = lib.optionalAttrs (cfg.cephSecret != null) {
{
PLASMAVMC_MODE = cfg.mode;
PLASMAVMC_STORAGE_BACKEND = "flaredb";
PLASMAVMC_FLAREDB_ENDPOINT = if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479";
PLASMAVMC_QEMU_PATH = "${pkgs.qemu}/bin/qemu-system-x86_64";
PLASMAVMC_RUNTIME_DIR = "/run/libvirt/plasmavmc";
PLASMAVMC_IMAGE_CACHE_DIR = "${toString cfg.dataDir}/images";
PLASMAVMC_MANAGED_VOLUME_ROOT = toString cfg.managedVolumeRoot;
PLASMAVMC_SHARED_LIVE_MIGRATION = lib.boolToString cfg.sharedLiveMigration;
}
(lib.mkIf (cfg.prismnetAddr != null) {
PRISMNET_ENDPOINT = "http://${cfg.prismnetAddr}";
})
(lib.mkIf (cfg.chainfireAddr != null) {
PLASMAVMC_CHAINFIRE_ENDPOINT = "http://${cfg.chainfireAddr}";
PLASMAVMC_STATE_WATCHER = "1";
})
(lib.mkIf (cfg.lightningstorAddr != null) {
PLASMAVMC_LIGHTNINGSTOR_ENDPOINT = cfg.lightningstorAddr;
})
(lib.mkIf (effectiveCoronafsControllerEndpoint != null) {
PLASMAVMC_CORONAFS_CONTROLLER_ENDPOINT = effectiveCoronafsControllerEndpoint;
})
(lib.mkIf (effectiveCoronafsNodeEndpoint != null) {
PLASMAVMC_CORONAFS_NODE_ENDPOINT = effectiveCoronafsNodeEndpoint;
})
(lib.mkIf (cfg.coronafsNodeLocalAttach || cfg.experimentalCoronafsNodeLocalAttach) {
PLASMAVMC_CORONAFS_NODE_LOCAL_ATTACH = "1";
PLASMAVMC_CORONAFS_ENABLE_EXPERIMENTAL_NODE_LOCAL_ATTACH = "1";
})
(lib.mkIf (cfg.coronafsEndpoint != null) {
PLASMAVMC_CORONAFS_ENDPOINT = cfg.coronafsEndpoint;
})
(lib.mkIf (cfg.cephMonitors != [ ]) {
PLASMAVMC_CEPH_MONITORS = lib.concatStringsSep "," cfg.cephMonitors;
PLASMAVMC_CEPH_CLUSTER_ID = cfg.cephClusterId;
PLASMAVMC_CEPH_USER = cfg.cephUser;
})
(lib.mkIf (cfg.cephSecret != null) {
PLASMAVMC_CEPH_SECRET = cfg.cephSecret; PLASMAVMC_CEPH_SECRET = cfg.cephSecret;
}) };
(lib.mkIf (cfg.mode != "server") {
PLASMAVMC_NODE_ID = cfg.nodeId;
PLASMAVMC_NODE_NAME = cfg.nodeName;
PLASMAVMC_NODE_HEARTBEAT_INTERVAL_SECS = toString cfg.heartbeatIntervalSeconds;
})
(lib.mkIf (cfg.controlPlaneAddr != null) {
PLASMAVMC_CONTROL_PLANE_ADDR = cfg.controlPlaneAddr;
})
(lib.mkIf (cfg.advertiseAddr != null) {
PLASMAVMC_ENDPOINT_ADVERTISE = cfg.advertiseAddr;
})
(lib.mkIf (cfg.mode == "server") {
PLASMAVMC_NODE_HEALTH_MONITOR_INTERVAL_SECS = "5";
PLASMAVMC_NODE_HEARTBEAT_TIMEOUT_SECS = "30";
})
];
serviceConfig = { serviceConfig = {
Type = "simple"; Type = "simple";

View file

@ -3,17 +3,29 @@
let let
cfg = config.services.prismnet; cfg = config.services.prismnet;
tomlFormat = pkgs.formats.toml { }; tomlFormat = pkgs.formats.toml { };
prismnetConfigFile = tomlFormat.generate "prismnet.toml" { generatedConfig = {
grpc_addr = "0.0.0.0:${toString cfg.port}"; grpc_addr = "0.0.0.0:${toString cfg.port}";
http_addr = "0.0.0.0:${toString cfg.httpPort}"; http_addr = "0.0.0.0:${toString cfg.httpPort}";
log_level = "info"; log_level = "info";
metadata_backend = cfg.metadataBackend;
single_node = cfg.singleNode;
auth = { auth = {
iam_server_addr = iam_server_addr =
if cfg.iamAddr != null if cfg.iamAddr != null
then cfg.iamAddr then cfg.iamAddr
else "127.0.0.1:50080"; else "127.0.0.1:50080";
}; };
}
// lib.optionalAttrs (cfg.chainfireAddr != null) {
chainfire_endpoint = "http://${cfg.chainfireAddr}";
}
// lib.optionalAttrs (cfg.flaredbAddr != null) {
flaredb_endpoint = cfg.flaredbAddr;
}
// lib.optionalAttrs (cfg.databaseUrl != null) {
metadata_database_url = cfg.databaseUrl;
}; };
prismnetConfigFile = tomlFormat.generate "prismnet.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
in in
{ {
options.services.prismnet = { options.services.prismnet = {
@ -108,22 +120,6 @@ in
after = [ "network.target" "iam.service" "flaredb.service" "chainfire.service" ]; after = [ "network.target" "iam.service" "flaredb.service" "chainfire.service" ];
wants = [ "iam.service" "flaredb.service" "chainfire.service" ]; wants = [ "iam.service" "flaredb.service" "chainfire.service" ];
environment = lib.mkMerge [
{
PRISMNET_FLAREDB_ENDPOINT = if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479";
PRISMNET_METADATA_BACKEND = cfg.metadataBackend;
}
(lib.mkIf (cfg.databaseUrl != null) {
PRISMNET_METADATA_DATABASE_URL = cfg.databaseUrl;
})
(lib.mkIf cfg.singleNode {
PRISMNET_SINGLE_NODE = "1";
})
(lib.mkIf (cfg.chainfireAddr != null) {
PRISMNET_CHAINFIRE_ENDPOINT = "http://${cfg.chainfireAddr}";
})
];
serviceConfig = { serviceConfig = {
Type = "simple"; Type = "simple";
User = "prismnet"; User = "prismnet";
@ -143,12 +139,7 @@ in
ReadWritePaths = [ cfg.dataDir ]; ReadWritePaths = [ cfg.dataDir ];
# Start command # Start command
ExecStart = lib.concatStringsSep " " [ ExecStart = "${cfg.package}/bin/prismnet-server --config ${prismnetConfigFile}";
"${cfg.package}/bin/prismnet-server"
"--config ${prismnetConfigFile}"
"--grpc-addr 0.0.0.0:${toString cfg.port}"
"--flaredb-endpoint ${if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479"}"
];
}; };
}; };
}; };

View file

@ -15,17 +15,13 @@
services.flaredb = { services.flaredb = {
enable = true; enable = true;
dataDir = "/var/lib/flaredb"; dataDir = "/var/lib/flaredb";
settings = { pdAddr = "127.0.0.1:${toString config.services.chainfire.port}";
chainfire_endpoint = "http://127.0.0.1:${toString config.services.chainfire.port}";
};
}; };
services.iam = { services.iam = {
enable = true; enable = true;
dataDir = "/var/lib/iam"; dataDir = "/var/lib/iam";
settings = { flaredbAddr = "127.0.0.1:${toString config.services.flaredb.port}";
flaredb_endpoint = "http://127.0.0.1:${toString config.services.flaredb.port}";
};
}; };
services.plasmavmc.enable = true; services.plasmavmc.enable = true;

View file

@ -71,6 +71,8 @@
port = 50080; port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs; chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs; flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
}; };
services.prismnet = { services.prismnet = {
@ -162,13 +164,6 @@
flashdnsAddr = "http://10.100.0.11:50084"; flashdnsAddr = "http://10.100.0.11:50084";
}; };
systemd.services.iam.environment = { services.lightningstor.s3AccessKeyId = "photoncloud-test";
IAM_ALLOW_RANDOM_SIGNING_KEY = "1"; services.lightningstor.s3SecretKey = "photoncloud-test-secret";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
};
systemd.services.lightningstor.environment = {
S3_ACCESS_KEY_ID = "photoncloud-test";
S3_SECRET_KEY = "photoncloud-test-secret";
};
} }

View file

@ -64,10 +64,7 @@
port = 50080; port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs; chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs; flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
}; allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
}; };
} }

View file

@ -64,10 +64,7 @@
port = 50080; port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs; chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs; flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
}; allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
}; };
} }

View file

@ -69,6 +69,8 @@
port = 50080; port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs; chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs; flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
}; };
services.plasmavmc = { services.plasmavmc = {
@ -124,13 +126,6 @@
region = "test"; region = "test";
}; };
systemd.services.iam.environment = { services.lightningstor.s3AccessKeyId = "photoncloud-test";
IAM_ALLOW_RANDOM_SIGNING_KEY = "1"; services.lightningstor.s3SecretKey = "photoncloud-test-secret";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
};
systemd.services.lightningstor.environment = {
S3_ACCESS_KEY_ID = "photoncloud-test";
S3_SECRET_KEY = "photoncloud-test-secret";
};
} }

View file

@ -66,10 +66,7 @@
port = 50080; port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs; chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs; flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
}; allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
}; };
} }

View file

@ -66,10 +66,7 @@
port = 50080; port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs; chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs; flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
}; allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
}; };
} }

View file

@ -173,10 +173,7 @@ in
port = 50080; port = 50080;
httpPort = 8083; httpPort = 8083;
storeBackend = "memory"; storeBackend = "memory";
}; allowRandomSigningKey = true;
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
}; };
services.fiberlb = { services.fiberlb = {
@ -260,10 +257,7 @@ in
port = 50080; port = 50080;
httpPort = 8083; httpPort = 8083;
storeBackend = "memory"; storeBackend = "memory";
}; allowRandomSigningKey = true;
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
}; };
services.fiberlb = { services.fiberlb = {

View file

@ -260,10 +260,7 @@ in
port = 50080; port = 50080;
httpPort = 8083; httpPort = 8083;
storeBackend = "memory"; storeBackend = "memory";
}; allowRandomSigningKey = true;
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
}; };
services.fiberlb = { services.fiberlb = {

View file

@ -165,10 +165,7 @@ in
port = 50080; port = 50080;
httpPort = 8083; httpPort = 8083;
storeBackend = "memory"; storeBackend = "memory";
}; allowRandomSigningKey = true;
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
}; };
services.fiberlb = { services.fiberlb = {

View file

@ -120,10 +120,7 @@ in
port = 50080; port = 50080;
httpPort = 8083; httpPort = 8083;
storeBackend = "memory"; storeBackend = "memory";
}; allowRandomSigningKey = true;
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
}; };
services.fiberlb = { services.fiberlb = {

14
plasmavmc/Cargo.lock generated
View file

@ -2458,6 +2458,18 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
[[package]]
name = "nix"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
dependencies = [
"bitflags 2.11.0",
"cfg-if",
"cfg_aliases",
"libc",
]
[[package]] [[package]]
name = "nom" name = "nom"
version = "7.1.3" version = "7.1.3"
@ -2807,6 +2819,7 @@ name = "plasmavmc-kvm"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"nix",
"plasmavmc-hypervisor", "plasmavmc-hypervisor",
"plasmavmc-types", "plasmavmc-types",
"serde", "serde",
@ -2853,6 +2866,7 @@ dependencies = [
"reqwest 0.12.28", "reqwest 0.12.28",
"serde", "serde",
"serde_json", "serde_json",
"sha2",
"tempfile", "tempfile",
"thiserror 1.0.69", "thiserror 1.0.69",
"tokio", "tokio",

View file

@ -15,6 +15,7 @@ tracing = { workspace = true }
thiserror = { workspace = true } thiserror = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
nix = { version = "0.29", features = ["signal"] }
[lints] [lints]
workspace = true workspace = true

View file

@ -115,14 +115,20 @@ mod tests {
fn resolve_runtime_dir_defaults() { fn resolve_runtime_dir_defaults() {
let _guard = env_test_lock().lock().unwrap(); let _guard = env_test_lock().lock().unwrap();
std::env::remove_var(ENV_RUNTIME_DIR); std::env::remove_var(ENV_RUNTIME_DIR);
assert_eq!(resolve_runtime_dir(), PathBuf::from("/run/libvirt/plasmavmc")); assert_eq!(
resolve_runtime_dir(),
PathBuf::from("/run/libvirt/plasmavmc")
);
} }
#[test] #[test]
fn resolve_runtime_dir_from_env() { fn resolve_runtime_dir_from_env() {
let _guard = env_test_lock().lock().unwrap(); let _guard = env_test_lock().lock().unwrap();
std::env::set_var(ENV_RUNTIME_DIR, "/tmp/plasmavmc-runtime"); std::env::set_var(ENV_RUNTIME_DIR, "/tmp/plasmavmc-runtime");
assert_eq!(resolve_runtime_dir(), PathBuf::from("/tmp/plasmavmc-runtime")); assert_eq!(
resolve_runtime_dir(),
PathBuf::from("/tmp/plasmavmc-runtime")
);
std::env::remove_var(ENV_RUNTIME_DIR); std::env::remove_var(ENV_RUNTIME_DIR);
} }

View file

@ -8,18 +8,19 @@ mod qmp;
use async_trait::async_trait; use async_trait::async_trait;
use env::{ use env::{
resolve_kernel_initrd, resolve_nbd_aio_mode, resolve_nbd_max_queues, resolve_qcow2_path, resolve_qemu_path, resolve_kernel_initrd, resolve_nbd_aio_mode, resolve_nbd_max_queues, resolve_qcow2_path,
resolve_qmp_timeout_secs, resolve_runtime_dir, ENV_QCOW2_PATH, resolve_qemu_path, resolve_qmp_timeout_secs, resolve_runtime_dir, ENV_QCOW2_PATH,
}; };
use nix::sys::signal::{kill as nix_kill, Signal};
use nix::unistd::Pid;
use plasmavmc_hypervisor::{BackendCapabilities, HypervisorBackend, UnsupportedReason}; use plasmavmc_hypervisor::{BackendCapabilities, HypervisorBackend, UnsupportedReason};
use plasmavmc_types::{ use plasmavmc_types::{
AttachedDisk, DiskAttachment, DiskBus, DiskCache, Error, HypervisorType, NetworkSpec, AttachedDisk, DiskAttachment, DiskBus, DiskCache, Error, HypervisorType, NetworkSpec, NicModel,
NicModel, Result, VirtualMachine, VmHandle, VmSpec, VmState, VmStatus, VolumeFormat, Result, VirtualMachine, VmHandle, VmSpec, VmState, VmStatus, VolumeFormat,
}; };
use qmp::QmpClient; use qmp::QmpClient;
use serde_json::{json, Value}; use serde_json::{json, Value};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::time::Duration; use std::time::Duration;
use tokio::process::Command; use tokio::process::Command;
use tokio::{net::UnixStream, time::Instant}; use tokio::{net::UnixStream, time::Instant};
@ -62,31 +63,6 @@ fn volume_format_name(format: VolumeFormat) -> &'static str {
} }
} }
fn build_rbd_uri(pool: &str, image: &str, monitors: &[String], user: &str) -> String {
let mut uri = format!("rbd:{pool}/{image}");
if !user.is_empty() {
uri.push_str(&format!(":id={user}"));
}
if !monitors.is_empty() {
uri.push_str(&format!(":mon_host={}", monitors.join(";")));
}
uri
}
fn disk_source_arg(disk: &AttachedDisk) -> Result<(String, &'static str)> {
match &disk.attachment {
DiskAttachment::File { path, format } => Ok((path.clone(), volume_format_name(*format))),
DiskAttachment::Nbd { uri, format } => Ok((uri.clone(), volume_format_name(*format))),
DiskAttachment::CephRbd {
pool,
image,
monitors,
user,
..
} => Ok((build_rbd_uri(pool, image, monitors, user), "raw")),
}
}
fn effective_disk_cache(disk: &AttachedDisk) -> DiskCache { fn effective_disk_cache(disk: &AttachedDisk) -> DiskCache {
match (&disk.attachment, disk.cache) { match (&disk.attachment, disk.cache) {
// Shared NBD-backed volumes perform better and behave more predictably // Shared NBD-backed volumes perform better and behave more predictably
@ -96,14 +72,6 @@ fn effective_disk_cache(disk: &AttachedDisk) -> DiskCache {
} }
} }
fn disk_cache_mode(cache: DiskCache) -> &'static str {
match cache {
DiskCache::None => "none",
DiskCache::Writeback => "writeback",
DiskCache::Writethrough => "writethrough",
}
}
fn disk_aio_mode(disk: &AttachedDisk) -> Option<&'static str> { fn disk_aio_mode(disk: &AttachedDisk) -> Option<&'static str> {
match (&disk.attachment, disk.cache) { match (&disk.attachment, disk.cache) {
(DiskAttachment::File { .. }, DiskCache::None) => Some("native"), (DiskAttachment::File { .. }, DiskCache::None) => Some("native"),
@ -125,7 +93,8 @@ fn disk_queue_count(vm: &VirtualMachine, disk: &AttachedDisk) -> u16 {
return 1; return 1;
} }
vm.spec.cpu vm.spec
.cpu
.vcpus .vcpus
.clamp(1, resolve_nbd_max_queues().max(1) as u32) as u16 .clamp(1, resolve_nbd_max_queues().max(1) as u32) as u16
} }
@ -153,6 +122,169 @@ fn qmp_timeout() -> Duration {
Duration::from_secs(resolve_qmp_timeout_secs()) Duration::from_secs(resolve_qmp_timeout_secs())
} }
fn disk_cache_json(cache: DiskCache) -> Value {
json!({
"direct": matches!(cache, DiskCache::None),
"no-flush": false
})
}
fn validate_ceph_component(field_name: &str, value: &str) -> Result<()> {
if value.is_empty() {
return Err(Error::HypervisorError(format!("{field_name} is required")));
}
let mut chars = value.chars();
let Some(first) = chars.next() else {
return Err(Error::HypervisorError(format!("{field_name} is required")));
};
if !first.is_ascii_alphanumeric() {
return Err(Error::HypervisorError(format!(
"{field_name} must start with an ASCII alphanumeric character"
)));
}
if chars.any(|ch| !(ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-'))) {
return Err(Error::HypervisorError(format!(
"{field_name} contains unsupported characters"
)));
}
Ok(())
}
fn parse_host_port(authority: &str, default_port: u16) -> Result<(String, u16)> {
if let Some(rest) = authority.strip_prefix('[') {
let (host, tail) = rest
.split_once(']')
.ok_or_else(|| Error::HypervisorError("invalid IPv6 authority".into()))?;
let port = tail
.strip_prefix(':')
.and_then(|value| value.parse::<u16>().ok())
.unwrap_or(default_port);
return Ok((host.to_string(), port));
}
if let Some((host, port)) = authority.rsplit_once(':') {
if !host.is_empty() {
if let Ok(port) = port.parse::<u16>() {
return Ok((host.to_string(), port));
}
}
}
Ok((authority.to_string(), default_port))
}
fn parse_nbd_uri(uri: &str) -> Result<(String, u16, Option<String>)> {
let remainder = uri
.strip_prefix("nbd://")
.ok_or_else(|| Error::HypervisorError(format!("unsupported NBD URI: {uri}")))?;
if remainder.contains('@') || remainder.contains('?') || remainder.contains('#') {
return Err(Error::HypervisorError(format!(
"unsupported NBD URI components: {uri}"
)));
}
let (authority, path) = remainder.split_once('/').unwrap_or((remainder, ""));
let (host, port) = parse_host_port(authority, 10809)?;
if host.is_empty() {
return Err(Error::HypervisorError(format!(
"missing NBD host in URI: {uri}"
)));
}
let export = (!path.is_empty()).then(|| path.to_string());
Ok((host, port, export))
}
fn parse_ceph_monitor(monitor: &str) -> Result<Value> {
let (host, port) = parse_host_port(monitor, 6789)?;
if host.is_empty() {
return Err(Error::HypervisorError(format!(
"invalid Ceph monitor address: {monitor}"
)));
}
Ok(json!({
"type": "inet",
"host": host,
"port": port.to_string()
}))
}
fn disk_blockdev_arg(disk: &AttachedDisk, disk_id: &str) -> Result<String> {
let effective_cache = effective_disk_cache(disk);
let aio_mode = disk_aio_mode(disk);
let file = match &disk.attachment {
DiskAttachment::File { path, .. } => {
let mut file = json!({
"driver": "file",
"filename": path
});
if let Some(aio_mode) = aio_mode {
file["aio"] = json!(aio_mode);
}
file
}
DiskAttachment::Nbd { uri, .. } => {
let (host, port, export) = parse_nbd_uri(uri)?;
let mut nbd = json!({
"driver": "nbd",
"server": {
"type": "inet",
"host": host,
"port": port.to_string()
}
});
if let Some(export) = export {
nbd["export"] = json!(export);
}
if let Some(aio_mode) = aio_mode {
nbd["aio"] = json!(aio_mode);
}
nbd
}
DiskAttachment::CephRbd {
pool,
image,
monitors,
user,
..
} => {
validate_ceph_component("ceph pool", pool)?;
validate_ceph_component("ceph image", image)?;
if !user.is_empty() {
validate_ceph_component("ceph user", user)?;
}
let servers: Vec<Value> = monitors
.iter()
.map(|monitor| parse_ceph_monitor(monitor))
.collect::<Result<Vec<_>>>()?;
let mut rbd = json!({
"driver": "rbd",
"pool": pool,
"image": image,
"server": servers
});
if !user.is_empty() {
rbd["user"] = json!(user);
}
rbd
}
};
let format_driver = match &disk.attachment {
DiskAttachment::File { format, .. } | DiskAttachment::Nbd { format, .. } => {
volume_format_name(*format)
}
DiskAttachment::CephRbd { .. } => "raw",
};
Ok(json!({
"node-name": format!("drive-{disk_id}"),
"driver": format_driver,
"read-only": disk.read_only,
"cache": disk_cache_json(effective_cache),
"file": file
})
.to_string())
}
fn build_disk_args(vm: &VirtualMachine, disks: &[AttachedDisk]) -> Result<Vec<String>> { fn build_disk_args(vm: &VirtualMachine, disks: &[AttachedDisk]) -> Result<Vec<String>> {
if disks.is_empty() && vm.spec.disks.is_empty() { if disks.is_empty() && vm.spec.disks.is_empty() {
let qcow_path = resolve_qcow2_path().ok_or_else(|| { let qcow_path = resolve_qcow2_path().ok_or_else(|| {
@ -167,8 +299,20 @@ fn build_disk_args(vm: &VirtualMachine, disks: &[AttachedDisk]) -> Result<Vec<St
))); )));
} }
return Ok(vec![ return Ok(vec![
"-drive".into(), "-blockdev".into(),
format!("file={},if=virtio,format=qcow2", qcow_path.display()), json!({
"node-name": "drive-root",
"driver": "qcow2",
"read-only": false,
"cache": disk_cache_json(DiskCache::Writeback),
"file": {
"driver": "file",
"filename": qcow_path.display().to_string()
}
})
.to_string(),
"-device".into(),
"virtio-blk-pci,drive=drive-root,id=disk-root".into(),
]); ]);
} }
@ -205,21 +349,12 @@ fn build_disk_args(vm: &VirtualMachine, disks: &[AttachedDisk]) -> Result<Vec<St
for (index, disk) in disks.into_iter().enumerate() { for (index, disk) in disks.into_iter().enumerate() {
let disk_id = sanitize_device_component(&disk.id, index); let disk_id = sanitize_device_component(&disk.id, index);
let (source, format_name) = disk_source_arg(disk)?;
if disk_uses_dedicated_iothread(disk) { if disk_uses_dedicated_iothread(disk) {
args.push("-object".into()); args.push("-object".into());
args.push(format!("iothread,id=iothread-{disk_id}")); args.push(format!("iothread,id=iothread-{disk_id}"));
} }
let effective_cache = effective_disk_cache(disk); args.push("-blockdev".into());
let mut drive_arg = format!( args.push(disk_blockdev_arg(disk, &disk_id)?);
"file={source},if=none,format={format_name},id=drive-{disk_id},cache={}",
disk_cache_mode(effective_cache)
);
if let Some(aio_mode) = disk_aio_mode(disk) {
drive_arg.push_str(&format!(",aio={aio_mode}"));
}
args.push("-drive".into());
args.push(drive_arg);
let bootindex = bootindex_suffix(disk.boot_index); let bootindex = bootindex_suffix(disk.boot_index);
let device_arg = match disk.bus { let device_arg = match disk.bus {
@ -359,33 +494,24 @@ async fn wait_for_qmp(qmp_socket: &Path, timeout: Duration) -> Result<()> {
} }
fn kill_pid(pid: u32) -> Result<()> { fn kill_pid(pid: u32) -> Result<()> {
let status = std::process::Command::new("kill") let pid = Pid::from_raw(pid as i32);
.arg("-9") match nix_kill(pid, Signal::SIGKILL) {
.arg(pid.to_string()) Ok(()) => Ok(()),
.stdout(Stdio::null()) Err(nix::errno::Errno::ESRCH) => Ok(()),
.stderr(Stdio::null()) Err(error) => Err(Error::HypervisorError(format!(
.status() "failed to send SIGKILL to pid {}: {error}",
.map_err(|e| Error::HypervisorError(format!("Failed to invoke kill -9: {e}")))?; pid.as_raw()
if status.success() { ))),
Ok(())
} else if !pid_running(pid) {
Ok(())
} else {
Err(Error::HypervisorError(format!(
"kill -9 exited with status: {status}"
)))
} }
} }
fn pid_running(pid: u32) -> bool { fn pid_running(pid: u32) -> bool {
std::process::Command::new("kill") match nix_kill(Pid::from_raw(pid as i32), None::<Signal>) {
.arg("-0") Ok(()) => true,
.arg(pid.to_string()) Err(nix::errno::Errno::EPERM) => true,
.stdout(Stdio::null()) Err(nix::errno::Errno::ESRCH) => false,
.stderr(Stdio::null()) Err(_) => false,
.status() }
.map(|status| status.success())
.unwrap_or(false)
} }
fn vm_stopped_out_of_band(handle: &VmHandle, qmp_socket: &Path) -> bool { fn vm_stopped_out_of_band(handle: &VmHandle, qmp_socket: &Path) -> bool {
@ -569,7 +695,9 @@ impl HypervisorBackend for KvmBackend {
match QmpClient::connect(&qmp_socket).await { match QmpClient::connect(&qmp_socket).await {
Ok(mut client) => match client.query_status().await { Ok(mut client) => match client.query_status().await {
Ok(status) if matches!(status.actual_state, VmState::Stopped | VmState::Failed) => { Ok(status)
if matches!(status.actual_state, VmState::Stopped | VmState::Failed) =>
{
break; break;
} }
Ok(_) => {} Ok(_) => {}
@ -1109,10 +1237,12 @@ mod tests {
let args_joined = args.join(" "); let args_joined = args.join(" ");
assert!(args_joined.contains("vm-root.qcow2")); assert!(args_joined.contains("vm-root.qcow2"));
assert!(args_joined.contains("vm-data.qcow2")); assert!(args_joined.contains("vm-data.qcow2"));
assert!(args_joined.contains("-blockdev"));
assert!(args_joined.contains("bootindex=1")); assert!(args_joined.contains("bootindex=1"));
assert!(args_joined.contains("cache=writeback")); assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
assert!(args_joined.contains("cache=none,aio=native")); assert!(args_joined.contains("\"cache\":{\"direct\":false,\"no-flush\":false}"));
assert!(args_joined.contains("cache=writeback,aio=threads")); assert!(args_joined.contains("\"aio\":\"native\""));
assert!(args_joined.contains("\"aio\":\"threads\""));
} }
#[test] #[test]
@ -1135,6 +1265,7 @@ mod tests {
let console = PathBuf::from("/tmp/console.log"); let console = PathBuf::from("/tmp/console.log");
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap(); let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
let args_joined = args.join(" "); let args_joined = args.join(" ");
assert!(args_joined.contains("\"driver\":\"nbd\""));
assert!(args_joined.contains("-object iothread,id=iothread-root")); assert!(args_joined.contains("-object iothread,id=iothread-root"));
assert!(args_joined.contains("virtio-blk-pci,drive=drive-root,id=disk-root,iothread=iothread-root,num-queues=4,queue-size=1024,bootindex=1")); assert!(args_joined.contains("virtio-blk-pci,drive=drive-root,id=disk-root,iothread=iothread-root,num-queues=4,queue-size=1024,bootindex=1"));
} }
@ -1159,7 +1290,8 @@ mod tests {
let console = PathBuf::from("/tmp/console.log"); let console = PathBuf::from("/tmp/console.log");
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap(); let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
let args_joined = args.join(" "); let args_joined = args.join(" ");
assert!(args_joined.contains("cache=none,aio=io_uring")); assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
assert!(args_joined.contains("\"aio\":\"io_uring\""));
} }
#[test] #[test]
@ -1182,7 +1314,8 @@ mod tests {
let console = PathBuf::from("/tmp/console.log"); let console = PathBuf::from("/tmp/console.log");
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap(); let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
let args_joined = args.join(" "); let args_joined = args.join(" ");
assert!(args_joined.contains("cache=none,aio=io_uring")); assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
assert!(args_joined.contains("\"aio\":\"io_uring\""));
} }
#[test] #[test]
@ -1205,10 +1338,34 @@ mod tests {
let console = PathBuf::from("/tmp/console.log"); let console = PathBuf::from("/tmp/console.log");
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap(); let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
let args_joined = args.join(" "); let args_joined = args.join(" ");
assert!(args_joined.contains("cache=none,aio=threads")); assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
assert!(args_joined.contains("\"aio\":\"threads\""));
std::env::remove_var(crate::env::ENV_NBD_AIO_MODE); std::env::remove_var(crate::env::ENV_NBD_AIO_MODE);
} }
#[test]
fn build_qemu_args_rejects_invalid_ceph_identifiers() {
let vm = VirtualMachine::new("vm1", "org", "proj", VmSpec::default());
let disks = vec![AttachedDisk {
id: "root".into(),
attachment: DiskAttachment::CephRbd {
pool: "pool,inject".into(),
image: "image".into(),
monitors: vec!["10.0.0.10:6789".into()],
user: "admin".into(),
secret: None,
},
bus: DiskBus::Virtio,
cache: DiskCache::None,
boot_index: Some(1),
read_only: false,
}];
let qmp = PathBuf::from("/tmp/qmp.sock");
let console = PathBuf::from("/tmp/console.log");
let error = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap_err();
assert!(error.to_string().contains("unsupported characters"));
}
#[tokio::test] #[tokio::test]
async fn wait_for_qmp_succeeds_after_socket_created() { async fn wait_for_qmp_succeeds_after_socket_created() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();

View file

@ -33,7 +33,9 @@ impl QmpClient {
last_error = Some(format!("Failed to connect QMP: {e}")); last_error = Some(format!("Failed to connect QMP: {e}"));
} }
Err(e) => { Err(e) => {
return Err(Error::HypervisorError(format!("Failed to connect QMP: {e}"))); return Err(Error::HypervisorError(format!(
"Failed to connect QMP: {e}"
)));
} }
} }

View file

@ -33,6 +33,7 @@ serde_json = { workspace = true }
toml = { workspace = true } toml = { workspace = true }
metrics-exporter-prometheus = { workspace = true } metrics-exporter-prometheus = { workspace = true }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
sha2 = "0.10"
chainfire-client = { path = "../../../chainfire/chainfire-client" } chainfire-client = { path = "../../../chainfire/chainfire-client" }
creditservice-client = { path = "../../../creditservice/creditservice-client" } creditservice-client = { path = "../../../creditservice/creditservice-client" }
flaredb-client = { path = "../../../flaredb/crates/flaredb-client" } flaredb-client = { path = "../../../flaredb/crates/flaredb-client" }

View file

@ -1,3 +1,10 @@
use crate::config::ArtifactStoreConfig;
use crate::storage::ImageUploadPart;
use reqwest::header::LOCATION;
use reqwest::{Client as HttpClient, Url};
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::net::IpAddr;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::Arc; use std::sync::Arc;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
@ -13,7 +20,6 @@ use lightningstor_api::proto::{
}; };
use lightningstor_api::{BucketServiceClient, ObjectServiceClient}; use lightningstor_api::{BucketServiceClient, ObjectServiceClient};
use plasmavmc_types::ImageFormat; use plasmavmc_types::ImageFormat;
use reqwest::StatusCode as HttpStatusCode;
use serde::Deserialize; use serde::Deserialize;
use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::process::Command; use tokio::process::Command;
@ -23,25 +29,30 @@ use tonic::metadata::MetadataValue;
use tonic::transport::{Channel, Endpoint}; use tonic::transport::{Channel, Endpoint};
use tonic::{Code, Request, Status}; use tonic::{Code, Request, Status};
const DEFAULT_IMAGE_BUCKET: &str = "plasmavmc-images";
const MAX_OBJECT_GRPC_MESSAGE_SIZE: usize = 1024 * 1024 * 1024; const MAX_OBJECT_GRPC_MESSAGE_SIZE: usize = 1024 * 1024 * 1024;
const OBJECT_GRPC_INITIAL_STREAM_WINDOW: u32 = 64 * 1024 * 1024; const OBJECT_GRPC_INITIAL_STREAM_WINDOW: u32 = 64 * 1024 * 1024;
const OBJECT_GRPC_INITIAL_CONNECTION_WINDOW: u32 = 512 * 1024 * 1024; const OBJECT_GRPC_INITIAL_CONNECTION_WINDOW: u32 = 512 * 1024 * 1024;
const OBJECT_GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30); const OBJECT_GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30);
const OBJECT_GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(10); const OBJECT_GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(10);
const DEFAULT_MULTIPART_UPLOAD_PART_SIZE: usize = 32 * 1024 * 1024;
const MIN_MULTIPART_UPLOAD_PART_SIZE: usize = 8 * 1024 * 1024; const MIN_MULTIPART_UPLOAD_PART_SIZE: usize = 8 * 1024 * 1024;
const MAX_MULTIPART_UPLOAD_PART_SIZE: usize = 128 * 1024 * 1024; const MAX_MULTIPART_UPLOAD_PART_SIZE: usize = 128 * 1024 * 1024;
const DEFAULT_MULTIPART_UPLOAD_CONCURRENCY: usize = 4;
const MAX_MULTIPART_UPLOAD_CONCURRENCY: usize = 32; const MAX_MULTIPART_UPLOAD_CONCURRENCY: usize = 32;
const DEFAULT_RAW_IMAGE_CONVERT_PARALLELISM: usize = 8; const MAX_IMPORT_REDIRECTS: usize = 5;
const DEFAULT_HTTP_SEND_TIMEOUT: Duration = Duration::from_secs(15);
#[derive(Clone)] #[derive(Clone)]
pub struct ArtifactStore { pub struct ArtifactStore {
channel: Channel, channel: Channel,
iam_client: Arc<IamClient>, iam_client: Arc<IamClient>,
http_client: HttpClient,
image_bucket: String, image_bucket: String,
image_cache_dir: PathBuf, image_cache_dir: PathBuf,
multipart_upload_concurrency: usize,
multipart_upload_part_size: usize,
raw_image_convert_parallelism: usize,
max_image_import_size_bytes: u64,
allowed_https_hosts: Arc<HashSet<String>>,
qemu_img_path: PathBuf,
project_tokens: Arc<DashMap<String, CachedToken>>, project_tokens: Arc<DashMap<String, CachedToken>>,
ensured_buckets: Arc<DashSet<String>>, ensured_buckets: Arc<DashSet<String>>,
} }
@ -50,6 +61,8 @@ pub(crate) struct ImportedImage {
pub size_bytes: u64, pub size_bytes: u64,
pub checksum: String, pub checksum: String,
pub format: ImageFormat, pub format: ImageFormat,
pub source_type: String,
pub source_host: Option<String>,
} }
#[derive(Deserialize)] #[derive(Deserialize)]
@ -62,10 +75,24 @@ struct CachedToken {
expires_at: Instant, expires_at: Instant,
} }
struct ValidatedImportUrl {
url: Url,
host: String,
}
struct ImportedImageSource {
source_type: String,
host: String,
}
impl ArtifactStore { impl ArtifactStore {
pub async fn from_env(iam_endpoint: &str) -> Result<Option<Self>, Box<dyn std::error::Error>> { pub async fn from_config(
let Some(raw_endpoint) = std::env::var("PLASMAVMC_LIGHTNINGSTOR_ENDPOINT") config: &ArtifactStoreConfig,
.ok() iam_endpoint: &str,
) -> Result<Option<Self>, Box<dyn std::error::Error>> {
let Some(raw_endpoint) = config
.lightningstor_endpoint
.as_ref()
.map(|value| value.trim().to_string()) .map(|value| value.trim().to_string())
.filter(|value| !value.is_empty()) .filter(|value| !value.is_empty())
else { else {
@ -87,18 +114,41 @@ impl ArtifactStore {
.keep_alive_timeout(OBJECT_GRPC_KEEPALIVE_TIMEOUT) .keep_alive_timeout(OBJECT_GRPC_KEEPALIVE_TIMEOUT)
.connect_lazy(); .connect_lazy();
let image_cache_dir = std::env::var("PLASMAVMC_IMAGE_CACHE_DIR") let image_cache_dir = config.image_cache_dir.clone();
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("/var/lib/plasmavmc/images"));
tokio::fs::create_dir_all(&image_cache_dir).await?; tokio::fs::create_dir_all(&image_cache_dir).await?;
ensure_cache_dir_permissions(&image_cache_dir).await?; ensure_cache_dir_permissions(&image_cache_dir).await?;
let http_client = HttpClient::builder()
.connect_timeout(Duration::from_secs(
config.image_import_connect_timeout_secs.max(1),
))
.timeout(Duration::from_secs(config.image_import_timeout_secs.max(1)))
.redirect(reqwest::redirect::Policy::none())
.build()?;
let allowed_https_hosts = config
.allowed_https_hosts
.iter()
.map(|host| host.trim().to_ascii_lowercase())
.filter(|host| !host.is_empty())
.collect::<HashSet<_>>();
let qemu_img_path = resolve_binary_path(config.qemu_img_path.as_deref(), "qemu-img")?;
Ok(Some(Self { Ok(Some(Self {
channel, channel,
iam_client, iam_client,
image_bucket: std::env::var("PLASMAVMC_IMAGE_BUCKET") http_client,
.unwrap_or_else(|_| DEFAULT_IMAGE_BUCKET.to_string()), image_bucket: config.image_bucket.clone(),
image_cache_dir, image_cache_dir,
multipart_upload_concurrency: config
.multipart_upload_concurrency
.clamp(1, MAX_MULTIPART_UPLOAD_CONCURRENCY),
multipart_upload_part_size: config.multipart_upload_part_size.clamp(
MIN_MULTIPART_UPLOAD_PART_SIZE,
MAX_MULTIPART_UPLOAD_PART_SIZE,
),
raw_image_convert_parallelism: config.raw_image_convert_parallelism.clamp(1, 64),
max_image_import_size_bytes: config.max_image_import_size_bytes.max(1),
allowed_https_hosts: Arc::new(allowed_https_hosts),
qemu_img_path,
project_tokens: Arc::new(DashMap::new()), project_tokens: Arc::new(DashMap::new()),
ensured_buckets: Arc::new(DashSet::new()), ensured_buckets: Arc::new(DashSet::new()),
})) }))
@ -116,50 +166,20 @@ impl ArtifactStore {
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token) self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
.await?; .await?;
let image_path = self.image_path(image_id); let staging_path = self.staging_path(image_id)?;
let staging_path = self.image_cache_dir.join(format!("{image_id}.source")); let ImportedImageSource { source_type, host } =
self.materialize_source(source_url, &staging_path).await?; self.materialize_source(source_url, &staging_path).await?;
if self self.process_staged_image(
.can_reuse_qcow2_source(&staging_path, source_format) org_id,
.await? project_id,
{ image_id,
if tokio::fs::try_exists(&image_path).await.map_err(|e| { &token,
Status::internal(format!("failed to inspect {}: {e}", image_path.display())) &staging_path,
})? { source_format,
let _ = tokio::fs::remove_file(&staging_path).await; source_type,
} else { Some(host),
tokio::fs::rename(&staging_path, &image_path) )
.await .await
.map_err(|e| {
Status::internal(format!(
"failed to move qcow2 image {} into cache {}: {e}",
staging_path.display(),
image_path.display()
))
})?;
ensure_cache_file_permissions(&image_path).await?;
}
} else {
// Normalize non-qcow2 inputs through qemu-img convert so the cached
// artifact has a stable qcow2 representation before upload.
self.convert_to_qcow2(&staging_path, &image_path).await?;
let _ = tokio::fs::remove_file(&staging_path).await;
}
let checksum = self.sha256sum(&image_path).await?;
let metadata = tokio::fs::metadata(&image_path).await.map_err(|e| {
Status::internal(format!("failed to stat {}: {e}", image_path.display()))
})?;
let image_key = image_object_key(org_id, project_id, image_id);
self.upload_file(&self.image_bucket, &image_key, &image_path, &token)
.await?;
Ok(ImportedImage {
size_bytes: metadata.len(),
checksum,
format: ImageFormat::Qcow2,
})
} }
pub async fn materialize_image_cache( pub async fn materialize_image_cache(
@ -171,8 +191,8 @@ impl ArtifactStore {
let token = self.issue_project_token(org_id, project_id).await?; let token = self.issue_project_token(org_id, project_id).await?;
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token) self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
.await?; .await?;
let image_key = image_object_key(org_id, project_id, image_id); let image_key = image_object_key(org_id, project_id, image_id)?;
let image_path = self.image_path(image_id); let image_path = self.image_path(image_id)?;
self.download_object_to_file(&self.image_bucket, &image_key, &image_path, &token) self.download_object_to_file(&self.image_bucket, &image_key, &image_path, &token)
.await?; .await?;
Ok(image_path) Ok(image_path)
@ -187,7 +207,7 @@ impl ArtifactStore {
let image_path = self let image_path = self
.materialize_image_cache(org_id, project_id, image_id) .materialize_image_cache(org_id, project_id, image_id)
.await?; .await?;
let raw_path = self.raw_image_path(image_id); let raw_path = self.raw_image_path(image_id)?;
self.convert_to_raw(&image_path, &raw_path).await?; self.convert_to_raw(&image_path, &raw_path).await?;
Ok(raw_path) Ok(raw_path)
} }
@ -199,7 +219,7 @@ impl ArtifactStore {
image_id: &str, image_id: &str,
) -> Result<(), Status> { ) -> Result<(), Status> {
let token = self.issue_project_token(org_id, project_id).await?; let token = self.issue_project_token(org_id, project_id).await?;
let image_key = image_object_key(org_id, project_id, image_id); let image_key = image_object_key(org_id, project_id, image_id)?;
let mut client = self.object_client().await?; let mut client = self.object_client().await?;
let mut request = Request::new(DeleteObjectRequest { let mut request = Request::new(DeleteObjectRequest {
bucket: self.image_bucket.clone(), bucket: self.image_bucket.clone(),
@ -213,7 +233,7 @@ impl ArtifactStore {
Err(status) => return Err(Status::from_error(Box::new(status))), Err(status) => return Err(Status::from_error(Box::new(status))),
} }
let image_path = self.image_path(image_id); let image_path = self.image_path(image_id)?;
if tokio::fs::try_exists(&image_path).await.map_err(|e| { if tokio::fs::try_exists(&image_path).await.map_err(|e| {
Status::internal(format!("failed to inspect {}: {e}", image_path.display())) Status::internal(format!("failed to inspect {}: {e}", image_path.display()))
})? { })? {
@ -221,7 +241,7 @@ impl ArtifactStore {
Status::internal(format!("failed to remove {}: {e}", image_path.display())) Status::internal(format!("failed to remove {}: {e}", image_path.display()))
})?; })?;
} }
let raw_path = self.raw_image_path(image_id); let raw_path = self.raw_image_path(image_id)?;
if tokio::fs::try_exists(&raw_path).await.map_err(|e| { if tokio::fs::try_exists(&raw_path).await.map_err(|e| {
Status::internal(format!("failed to inspect {}: {e}", raw_path.display())) Status::internal(format!("failed to inspect {}: {e}", raw_path.display()))
})? { })? {
@ -232,6 +252,156 @@ impl ArtifactStore {
Ok(()) Ok(())
} }
pub fn minimum_upload_part_size(&self) -> u32 {
self.multipart_upload_part_size as u32
}
pub(crate) async fn begin_image_upload(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
) -> Result<(String, String), Status> {
let token = self.issue_project_token(org_id, project_id).await?;
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
.await?;
let staging_key = staging_object_key(org_id, project_id, image_id)?;
let mut client = self.object_client().await?;
let mut request = Request::new(CreateMultipartUploadRequest {
bucket: self.image_bucket.clone(),
key: staging_key.clone(),
metadata: None,
});
attach_bearer(&mut request, &token)?;
let upload_id = client
.create_multipart_upload(request)
.await
.map_err(|status| Status::from_error(Box::new(status)))?
.into_inner()
.upload_id;
Ok((upload_id, staging_key))
}
pub(crate) async fn upload_image_part(
&self,
org_id: &str,
project_id: &str,
staging_key: &str,
upload_id: &str,
part_number: u32,
body: Vec<u8>,
) -> Result<ImageUploadPart, Status> {
if part_number == 0 {
return Err(Status::invalid_argument(
"part_number must be greater than zero",
));
}
if body.is_empty() {
return Err(Status::invalid_argument(
"upload part body must not be empty",
));
}
let token = self.issue_project_token(org_id, project_id).await?;
let mut client = self.object_client().await?;
let request_stream = tokio_stream::iter(vec![UploadPartRequest {
bucket: self.image_bucket.clone(),
key: staging_key.to_string(),
upload_id: upload_id.to_string(),
part_number,
body: body.clone().into(),
content_md5: String::new(),
}]);
let mut request = Request::new(request_stream);
attach_bearer(&mut request, &token)?;
let response = client
.upload_part(request)
.await
.map_err(|status| Status::from_error(Box::new(status)))?;
Ok(ImageUploadPart {
part_number,
etag: response.into_inner().etag,
size_bytes: body.len() as u64,
})
}
pub(crate) async fn complete_image_upload(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
staging_key: &str,
upload_id: &str,
parts: &[ImageUploadPart],
source_format: ImageFormat,
) -> Result<ImportedImage, Status> {
if parts.is_empty() {
return Err(Status::failed_precondition(
"upload session does not contain any parts",
));
}
let token = self.issue_project_token(org_id, project_id).await?;
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
.await?;
let mut sorted_parts: Vec<CompletedPart> = parts
.iter()
.map(|part| CompletedPart {
part_number: part.part_number,
etag: part.etag.clone(),
})
.collect();
sorted_parts.sort_by_key(|part| part.part_number);
let mut client = self.object_client().await?;
let mut complete_request = Request::new(CompleteMultipartUploadRequest {
bucket: self.image_bucket.clone(),
key: staging_key.to_string(),
upload_id: upload_id.to_string(),
parts: sorted_parts,
});
attach_bearer(&mut complete_request, &token)?;
client
.complete_multipart_upload(complete_request)
.await
.map_err(|status| Status::from_error(Box::new(status)))?;
let staging_path = self.staging_path(image_id)?;
if tokio::fs::try_exists(&staging_path).await.unwrap_or(false) {
let _ = tokio::fs::remove_file(&staging_path).await;
}
self.download_object_to_file(&self.image_bucket, staging_key, &staging_path, &token)
.await?;
let result = self
.process_staged_image(
org_id,
project_id,
image_id,
&token,
&staging_path,
source_format,
"upload".to_string(),
None,
)
.await;
let _ = self
.delete_object_ignore_not_found(&self.image_bucket, staging_key, &token)
.await;
result
}
pub(crate) async fn abort_image_upload(
&self,
org_id: &str,
project_id: &str,
staging_key: &str,
upload_id: &str,
) -> Result<(), Status> {
let token = self.issue_project_token(org_id, project_id).await?;
self.abort_multipart_upload(&self.image_bucket, staging_key, upload_id, &token)
.await
}
async fn ensure_bucket( async fn ensure_bucket(
&self, &self,
bucket: &str, bucket: &str,
@ -275,7 +445,7 @@ impl ArtifactStore {
let metadata = tokio::fs::metadata(path) let metadata = tokio::fs::metadata(path)
.await .await
.map_err(|e| Status::internal(format!("failed to stat {path:?}: {e}")))?; .map_err(|e| Status::internal(format!("failed to stat {path:?}: {e}")))?;
let multipart_part_size = multipart_upload_part_size(); let multipart_part_size = self.multipart_upload_part_size;
if metadata.len() > multipart_part_size as u64 { if metadata.len() > multipart_part_size as u64 {
return self return self
.upload_file_multipart(bucket, key, path, token, metadata.len()) .upload_file_multipart(bucket, key, path, token, metadata.len())
@ -336,7 +506,7 @@ impl ArtifactStore {
size_bytes: u64, size_bytes: u64,
) -> Result<(), Status> { ) -> Result<(), Status> {
let started = Instant::now(); let started = Instant::now();
let multipart_part_size = multipart_upload_part_size(); let multipart_part_size = self.multipart_upload_part_size;
tracing::info!( tracing::info!(
bucket = bucket, bucket = bucket,
key = key, key = key,
@ -366,7 +536,7 @@ impl ArtifactStore {
let mut part_number = 1u32; let mut part_number = 1u32;
let mut completed_parts = Vec::new(); let mut completed_parts = Vec::new();
let mut uploads = JoinSet::new(); let mut uploads = JoinSet::new();
let upload_concurrency = multipart_upload_concurrency(); let upload_concurrency = self.multipart_upload_concurrency;
let enqueue_part_upload = |uploads: &mut JoinSet<Result<CompletedPart, Status>>, let enqueue_part_upload = |uploads: &mut JoinSet<Result<CompletedPart, Status>>,
client: &ObjectServiceClient<Channel>, client: &ObjectServiceClient<Channel>,
@ -569,76 +739,23 @@ impl ArtifactStore {
Ok(()) Ok(())
} }
async fn materialize_source(&self, source_url: &str, path: &Path) -> Result<(), Status> { async fn materialize_source(
&self,
source_url: &str,
path: &Path,
) -> Result<ImportedImageSource, Status> {
if let Some(parent) = path.parent() { if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent) tokio::fs::create_dir_all(parent)
.await .await
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?; .map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
} }
if let Some(source_path) = source_url.strip_prefix("file://") { let validated = self.validate_import_url(source_url).await?;
tokio::fs::copy(source_path, path).await.map_err(|e| { self.download_https_source(&validated, path).await?;
Status::internal(format!("failed to copy image source {source_path}: {e}")) Ok(ImportedImageSource {
})?; source_type: "https".to_string(),
ensure_cache_file_permissions(path).await?; host: validated.host,
return Ok(()); })
}
if source_url.starts_with('/') {
tokio::fs::copy(source_url, path).await.map_err(|e| {
Status::internal(format!("failed to copy image source {source_url}: {e}"))
})?;
ensure_cache_file_permissions(path).await?;
return Ok(());
}
if source_url.starts_with("http://") || source_url.starts_with("https://") {
let mut response = reqwest::get(source_url).await.map_err(|e| {
Status::unavailable(format!("failed to download image source: {e}"))
})?;
if response.status() != HttpStatusCode::OK {
return Err(Status::failed_precondition(format!(
"image download failed with HTTP {}",
response.status()
)));
}
let temp_path = path.with_extension("download");
let mut file = tokio::fs::File::create(&temp_path).await.map_err(|e| {
Status::internal(format!(
"failed to create downloaded image {}: {e}",
temp_path.display()
))
})?;
while let Some(chunk) = response.chunk().await.map_err(|e| {
Status::unavailable(format!("failed to read image response body: {e}"))
})? {
file.write_all(&chunk).await.map_err(|e| {
Status::internal(format!(
"failed to write downloaded image {}: {e}",
temp_path.display()
))
})?;
}
file.flush().await.map_err(|e| {
Status::internal(format!(
"failed to flush downloaded image {}: {e}",
temp_path.display()
))
})?;
drop(file);
tokio::fs::rename(&temp_path, path).await.map_err(|e| {
Status::internal(format!(
"failed to finalize downloaded image {}: {e}",
path.display()
))
})?;
ensure_cache_file_permissions(path).await?;
return Ok(());
}
Err(Status::invalid_argument(
"source_url must be file://, an absolute path, or http(s)://",
))
} }
async fn convert_to_qcow2(&self, source: &Path, destination: &Path) -> Result<(), Status> { async fn convert_to_qcow2(&self, source: &Path, destination: &Path) -> Result<(), Status> {
@ -654,9 +771,9 @@ impl ArtifactStore {
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?; .map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
} }
let parallelism = raw_image_convert_parallelism().to_string(); let parallelism = self.raw_image_convert_parallelism.to_string();
let args = qemu_img_convert_to_qcow2_args(source, destination, &parallelism); let args = qemu_img_convert_to_qcow2_args(source, destination, &parallelism);
let status = Command::new("qemu-img") let status = Command::new(&self.qemu_img_path)
.args(args.iter().map(String::as_str)) .args(args.iter().map(String::as_str))
.status() .status()
.await .await
@ -685,9 +802,9 @@ impl ArtifactStore {
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?; .map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
} }
let parallelism = raw_image_convert_parallelism().to_string(); let parallelism = self.raw_image_convert_parallelism.to_string();
let args = qemu_img_convert_to_raw_args(source, destination, &parallelism); let args = qemu_img_convert_to_raw_args(source, destination, &parallelism);
let status = Command::new("qemu-img") let status = Command::new(&self.qemu_img_path)
.args(args.iter().map(String::as_str)) .args(args.iter().map(String::as_str))
.status() .status()
.await .await
@ -712,7 +829,7 @@ impl ArtifactStore {
return Ok(false); return Ok(false);
} }
let output = Command::new("qemu-img") let output = Command::new(&self.qemu_img_path)
.args(["info", "--output", "json", path.to_string_lossy().as_ref()]) .args(["info", "--output", "json", path.to_string_lossy().as_ref()])
.output() .output()
.await .await
@ -727,25 +844,262 @@ impl ArtifactStore {
} }
async fn sha256sum(&self, path: &Path) -> Result<String, Status> { async fn sha256sum(&self, path: &Path) -> Result<String, Status> {
let output = Command::new("sha256sum") let mut file = tokio::fs::File::open(path)
.arg(path)
.output()
.await .await
.map_err(|e| Status::internal(format!("failed to spawn sha256sum: {e}")))?; .map_err(|e| Status::internal(format!("failed to open {}: {e}", path.display())))?;
if !output.status.success() { let mut digest = Sha256::new();
return Err(Status::internal(format!( let mut buffer = vec![0u8; 1024 * 1024];
"sha256sum failed for {} with status {}", loop {
path.display(), let read_now = file
output.status .read(&mut buffer)
.await
.map_err(|e| Status::internal(format!("failed to read {}: {e}", path.display())))?;
if read_now == 0 {
break;
}
digest.update(&buffer[..read_now]);
}
Ok(format!("{:x}", digest.finalize()))
}
async fn process_staged_image(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
token: &str,
staging_path: &Path,
source_format: ImageFormat,
source_type: String,
source_host: Option<String>,
) -> Result<ImportedImage, Status> {
let image_path = self.image_path(image_id)?;
if self
.can_reuse_qcow2_source(staging_path, source_format)
.await?
{
if tokio::fs::try_exists(&image_path).await.map_err(|e| {
Status::internal(format!("failed to inspect {}: {e}", image_path.display()))
})? {
let _ = tokio::fs::remove_file(staging_path).await;
} else {
tokio::fs::rename(staging_path, &image_path)
.await
.map_err(|e| {
Status::internal(format!(
"failed to move qcow2 image {} into cache {}: {e}",
staging_path.display(),
image_path.display()
))
})?;
ensure_cache_file_permissions(&image_path).await?;
}
} else {
self.convert_to_qcow2(staging_path, &image_path).await?;
let _ = tokio::fs::remove_file(staging_path).await;
}
let checksum = self.sha256sum(&image_path).await?;
let metadata = tokio::fs::metadata(&image_path).await.map_err(|e| {
Status::internal(format!("failed to stat {}: {e}", image_path.display()))
})?;
let image_key = image_object_key(org_id, project_id, image_id)?;
self.upload_file(&self.image_bucket, &image_key, &image_path, token)
.await?;
Ok(ImportedImage {
size_bytes: metadata.len(),
checksum,
format: ImageFormat::Qcow2,
source_type,
source_host,
})
}
async fn validate_import_url(&self, source_url: &str) -> Result<ValidatedImportUrl, Status> {
if source_url.starts_with("file://") || source_url.starts_with('/') {
return Err(Status::invalid_argument(
"source_url must use https:// and may not reference local files",
));
}
let url = Url::parse(source_url)
.map_err(|e| Status::invalid_argument(format!("invalid source_url: {e}")))?;
if url.scheme() != "https" {
return Err(Status::invalid_argument("source_url must use https://"));
}
if !url.username().is_empty() || url.password().is_some() {
return Err(Status::invalid_argument(
"source_url must not include embedded credentials",
));
}
let host = url
.host_str()
.map(str::to_ascii_lowercase)
.ok_or_else(|| Status::invalid_argument("source_url must include a host"))?;
if host == "localhost" {
return Err(Status::invalid_argument(
"source_url host must not target loopback or local interfaces",
));
}
if !self.allowed_https_hosts.is_empty() && !self.allowed_https_hosts.contains(&host) {
return Err(Status::permission_denied(format!(
"source_url host {host} is not in artifacts.allowed_https_hosts",
))); )));
} }
let stdout = String::from_utf8(output.stdout)
.map_err(|e| Status::internal(format!("invalid sha256sum output: {e}")))?; let port = url.port_or_known_default().unwrap_or(443);
stdout let resolved = tokio::net::lookup_host((host.as_str(), port))
.split_whitespace() .await
.next() .map_err(|e| Status::unavailable(format!("failed to resolve source_url host: {e}")))?;
.map(str::to_string) let mut found_any = false;
.ok_or_else(|| Status::internal("sha256sum output missing digest")) for addr in resolved {
found_any = true;
let ip = addr.ip();
if !is_public_ip(ip) {
return Err(Status::permission_denied(format!(
"source_url resolved to a non-public address: {ip}",
)));
}
}
if !found_any {
return Err(Status::failed_precondition(
"source_url host did not resolve to any public addresses",
));
}
Ok(ValidatedImportUrl { url, host })
}
async fn download_https_source(
&self,
source: &ValidatedImportUrl,
path: &Path,
) -> Result<(), Status> {
let temp_path = path.with_extension("download");
if tokio::fs::try_exists(&temp_path).await.unwrap_or(false) {
let _ = tokio::fs::remove_file(&temp_path).await;
}
let mut current = source.url.clone();
let mut redirects_remaining = MAX_IMPORT_REDIRECTS;
loop {
let response = tokio::time::timeout(
DEFAULT_HTTP_SEND_TIMEOUT,
self.http_client.get(current.clone()).send(),
)
.await
.map_err(|_| Status::deadline_exceeded("timed out waiting for source_url response"))?
.map_err(|e| Status::unavailable(format!("failed to download image source: {e}")))?;
if response.status().is_redirection() {
if redirects_remaining == 0 {
return Err(Status::failed_precondition(
"source_url redirect limit exceeded",
));
}
let location = response
.headers()
.get(LOCATION)
.ok_or_else(|| {
Status::failed_precondition(
"source_url redirect response did not include a Location header",
)
})?
.to_str()
.map_err(|e| {
Status::failed_precondition(format!(
"invalid redirect Location header in source_url response: {e}"
))
})?;
current = current.join(location).map_err(|e| {
Status::failed_precondition(format!(
"failed to resolve redirect target from source_url: {e}"
))
})?;
self.validate_import_url(current.as_ref()).await?;
redirects_remaining -= 1;
continue;
}
if !response.status().is_success() {
return Err(Status::failed_precondition(format!(
"image download failed with HTTP {}",
response.status()
)));
}
if let Some(content_length) = response.content_length() {
if content_length > self.max_image_import_size_bytes {
return Err(Status::resource_exhausted(format!(
"image download exceeds the configured maximum size of {} bytes",
self.max_image_import_size_bytes
)));
}
}
let mut file = tokio::fs::File::create(&temp_path).await.map_err(|e| {
Status::internal(format!(
"failed to create downloaded image {}: {e}",
temp_path.display()
))
})?;
let mut response = response;
let mut downloaded = 0u64;
while let Some(chunk) = response.chunk().await.map_err(|e| {
Status::unavailable(format!("failed to read image response body: {e}"))
})? {
downloaded = downloaded.saturating_add(chunk.len() as u64);
if downloaded > self.max_image_import_size_bytes {
let _ = tokio::fs::remove_file(&temp_path).await;
return Err(Status::resource_exhausted(format!(
"image download exceeds the configured maximum size of {} bytes",
self.max_image_import_size_bytes
)));
}
file.write_all(&chunk).await.map_err(|e| {
Status::internal(format!(
"failed to write downloaded image {}: {e}",
temp_path.display()
))
})?;
}
file.flush().await.map_err(|e| {
Status::internal(format!(
"failed to flush downloaded image {}: {e}",
temp_path.display()
))
})?;
drop(file);
tokio::fs::rename(&temp_path, path).await.map_err(|e| {
Status::internal(format!(
"failed to finalize downloaded image {}: {e}",
path.display()
))
})?;
ensure_cache_file_permissions(path).await?;
return Ok(());
}
}
async fn delete_object_ignore_not_found(
&self,
bucket: &str,
key: &str,
token: &str,
) -> Result<(), Status> {
let mut client = self.object_client().await?;
let mut request = Request::new(DeleteObjectRequest {
bucket: bucket.to_string(),
key: key.to_string(),
version_id: String::new(),
});
attach_bearer(&mut request, token)?;
match client.delete_object(request).await {
Ok(_) => Ok(()),
Err(status) if status.code() == Code::NotFound => Ok(()),
Err(status) => Err(Status::from_error(Box::new(status))),
}
} }
async fn bucket_client(&self) -> Result<BucketServiceClient<Channel>, Status> { async fn bucket_client(&self) -> Result<BucketServiceClient<Channel>, Status> {
@ -832,12 +1186,22 @@ impl ArtifactStore {
Ok(token) Ok(token)
} }
fn image_path(&self, image_id: &str) -> PathBuf { fn image_path(&self, image_id: &str) -> Result<PathBuf, Status> {
self.image_cache_dir.join(format!("{image_id}.qcow2")) Ok(self
.image_cache_dir
.join(format!("{}.qcow2", validated_image_id(image_id)?)))
} }
fn raw_image_path(&self, image_id: &str) -> PathBuf { fn raw_image_path(&self, image_id: &str) -> Result<PathBuf, Status> {
self.image_cache_dir.join(format!("{image_id}.raw")) Ok(self
.image_cache_dir
.join(format!("{}.raw", validated_image_id(image_id)?)))
}
fn staging_path(&self, image_id: &str) -> Result<PathBuf, Status> {
Ok(self
.image_cache_dir
.join(format!("{}.source", validated_image_id(image_id)?)))
} }
async fn abort_multipart_upload( async fn abort_multipart_upload(
@ -915,35 +1279,6 @@ async fn next_uploaded_part(
} }
} }
fn multipart_upload_concurrency() -> usize {
std::env::var("PLASMAVMC_LIGHTNINGSTOR_MULTIPART_CONCURRENCY")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.map(|value| value.clamp(1, MAX_MULTIPART_UPLOAD_CONCURRENCY))
.unwrap_or(DEFAULT_MULTIPART_UPLOAD_CONCURRENCY)
}
fn multipart_upload_part_size() -> usize {
std::env::var("PLASMAVMC_LIGHTNINGSTOR_MULTIPART_PART_SIZE")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.map(|value| {
value.clamp(
MIN_MULTIPART_UPLOAD_PART_SIZE,
MAX_MULTIPART_UPLOAD_PART_SIZE,
)
})
.unwrap_or(DEFAULT_MULTIPART_UPLOAD_PART_SIZE)
}
fn raw_image_convert_parallelism() -> usize {
std::env::var("PLASMAVMC_RAW_IMAGE_CONVERT_PARALLELISM")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.map(|value| value.clamp(1, 64))
.unwrap_or(DEFAULT_RAW_IMAGE_CONVERT_PARALLELISM)
}
fn qemu_img_convert_to_qcow2_args( fn qemu_img_convert_to_qcow2_args(
source: &Path, source: &Path,
destination: &Path, destination: &Path,
@ -1008,8 +1343,74 @@ fn sanitize_identifier(value: &str) -> String {
.collect() .collect()
} }
fn image_object_key(org_id: &str, project_id: &str, image_id: &str) -> String { fn image_object_key(org_id: &str, project_id: &str, image_id: &str) -> Result<String, Status> {
format!("{org_id}/{project_id}/{image_id}.qcow2") Ok(format!(
"{org_id}/{project_id}/{}.qcow2",
validated_image_id(image_id)?
))
}
fn staging_object_key(org_id: &str, project_id: &str, image_id: &str) -> Result<String, Status> {
Ok(format!(
"{org_id}/{project_id}/uploads/{}.source",
validated_image_id(image_id)?
))
}
fn validated_image_id(image_id: &str) -> Result<&str, Status> {
uuid::Uuid::parse_str(image_id)
.map(|_| image_id)
.map_err(|_| Status::invalid_argument("image_id must be a UUID"))
}
fn is_public_ip(ip: IpAddr) -> bool {
match ip {
IpAddr::V4(ip) => {
!(ip.is_private()
|| ip.is_loopback()
|| ip.is_link_local()
|| ip.is_multicast()
|| ip.is_broadcast()
|| ip.is_documentation()
|| ip.is_unspecified())
}
IpAddr::V6(ip) => {
!(ip.is_loopback()
|| ip.is_unspecified()
|| ip.is_multicast()
|| ip.is_unique_local()
|| ip.is_unicast_link_local())
}
}
}
fn resolve_binary_path(
configured_path: Option<&Path>,
binary_name: &str,
) -> Result<PathBuf, Box<dyn std::error::Error>> {
let candidate = match configured_path {
Some(path) => path.to_path_buf(),
None => std::env::var_os("PATH")
.into_iter()
.flat_map(|paths| std::env::split_paths(&paths).collect::<Vec<_>>())
.map(|entry| entry.join(binary_name))
.find(|candidate| candidate.exists())
.ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("failed to locate {binary_name} in PATH"),
)
})?,
};
let metadata = std::fs::metadata(&candidate)?;
if !metadata.is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("{} is not a regular file", candidate.display()),
)
.into());
}
Ok(candidate)
} }
async fn ensure_cache_dir_permissions(path: &Path) -> Result<(), Status> { async fn ensure_cache_dir_permissions(path: &Path) -> Result<(), Status> {
@ -1103,4 +1504,13 @@ mod tests {
] ]
); );
} }
#[test]
fn image_object_key_rejects_non_uuid_identifiers() {
assert!(image_object_key("org", "project", "../passwd").is_err());
assert_eq!(
image_object_key("org", "project", "11111111-1111-1111-1111-111111111111").unwrap(),
"org/project/11111111-1111-1111-1111-111111111111.qcow2"
);
}
} }

View file

@ -1,8 +1,9 @@
//! Server configuration //! Server configuration
use plasmavmc_types::{FireCrackerConfig, KvmConfig};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::net::SocketAddr; use std::net::SocketAddr;
use plasmavmc_types::{FireCrackerConfig, KvmConfig}; use std::path::PathBuf;
/// TLS configuration /// TLS configuration
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
@ -42,12 +43,332 @@ pub struct ServerConfig {
/// Configuration for FireCracker backend /// Configuration for FireCracker backend
#[serde(default)] #[serde(default)]
pub firecracker: FireCrackerConfig, pub firecracker: FireCrackerConfig,
/// Durable/shared state configuration
#[serde(default)]
pub storage: StorageRuntimeConfig,
/// Control-plane and agent heartbeat configuration
#[serde(default)]
pub agent: AgentRuntimeConfig,
/// External service integrations
#[serde(default)]
pub integrations: IntegrationConfig,
/// State watcher and VM watch polling configuration
#[serde(default)]
pub watcher: WatcherRuntimeConfig,
/// Background health monitoring and failover configuration
#[serde(default)]
pub health: HealthRuntimeConfig,
/// Artifact storage and image import/export configuration
#[serde(default)]
pub artifacts: ArtifactStoreConfig,
/// Persistent volume runtime configuration
#[serde(default)]
pub volumes: VolumeRuntimeConfig,
/// Default hypervisor used when requests do not specify one
#[serde(default)]
pub default_hypervisor: DefaultHypervisor,
} }
fn default_http_addr() -> SocketAddr { fn default_http_addr() -> SocketAddr {
"127.0.0.1:8084".parse().unwrap() "127.0.0.1:8084".parse().unwrap()
} }
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "lowercase")]
pub enum StorageBackendKind {
#[default]
Flaredb,
File,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StorageRuntimeConfig {
#[serde(default)]
pub backend: StorageBackendKind,
#[serde(default)]
pub flaredb_endpoint: Option<String>,
#[serde(default)]
pub chainfire_endpoint: Option<String>,
#[serde(default)]
pub state_path: Option<PathBuf>,
}
impl Default for StorageRuntimeConfig {
fn default() -> Self {
Self {
backend: StorageBackendKind::default(),
flaredb_endpoint: None,
chainfire_endpoint: None,
state_path: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AgentRuntimeConfig {
#[serde(default)]
pub control_plane_addr: Option<String>,
#[serde(default)]
pub advertise_endpoint: Option<String>,
#[serde(default)]
pub node_id: Option<String>,
#[serde(default)]
pub node_name: Option<String>,
#[serde(default = "default_agent_heartbeat_interval_secs")]
pub heartbeat_interval_secs: u64,
#[serde(default = "default_shared_live_migration")]
pub shared_live_migration: bool,
}
fn default_agent_heartbeat_interval_secs() -> u64 {
5
}
fn default_shared_live_migration() -> bool {
true
}
impl Default for AgentRuntimeConfig {
fn default() -> Self {
Self {
control_plane_addr: None,
advertise_endpoint: None,
node_id: None,
node_name: None,
heartbeat_interval_secs: default_agent_heartbeat_interval_secs(),
shared_live_migration: default_shared_live_migration(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct IntegrationConfig {
#[serde(default)]
pub prismnet_endpoint: Option<String>,
#[serde(default)]
pub creditservice_endpoint: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WatcherRuntimeConfig {
#[serde(default)]
pub enabled: bool,
#[serde(default = "default_state_watcher_poll_interval_ms")]
pub poll_interval_ms: u64,
#[serde(default = "default_vm_watch_poll_interval_ms")]
pub vm_watch_poll_interval_ms: u64,
}
fn default_state_watcher_poll_interval_ms() -> u64 {
1000
}
fn default_vm_watch_poll_interval_ms() -> u64 {
500
}
impl Default for WatcherRuntimeConfig {
fn default() -> Self {
Self {
enabled: false,
poll_interval_ms: default_state_watcher_poll_interval_ms(),
vm_watch_poll_interval_ms: default_vm_watch_poll_interval_ms(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthRuntimeConfig {
#[serde(default)]
pub vm_monitor_interval_secs: Option<u64>,
#[serde(default)]
pub node_monitor_interval_secs: Option<u64>,
#[serde(default = "default_node_heartbeat_timeout_secs")]
pub node_heartbeat_timeout_secs: u64,
#[serde(default)]
pub auto_restart: bool,
#[serde(default)]
pub failover_enabled: bool,
#[serde(default = "default_failover_min_interval_secs")]
pub failover_min_interval_secs: u64,
}
fn default_node_heartbeat_timeout_secs() -> u64 {
60
}
fn default_failover_min_interval_secs() -> u64 {
60
}
impl Default for HealthRuntimeConfig {
fn default() -> Self {
Self {
vm_monitor_interval_secs: None,
node_monitor_interval_secs: None,
node_heartbeat_timeout_secs: default_node_heartbeat_timeout_secs(),
auto_restart: false,
failover_enabled: false,
failover_min_interval_secs: default_failover_min_interval_secs(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ArtifactStoreConfig {
#[serde(default)]
pub lightningstor_endpoint: Option<String>,
#[serde(default = "default_image_cache_dir")]
pub image_cache_dir: PathBuf,
#[serde(default = "default_image_bucket")]
pub image_bucket: String,
#[serde(default = "default_multipart_upload_concurrency")]
pub multipart_upload_concurrency: usize,
#[serde(default = "default_multipart_upload_part_size")]
pub multipart_upload_part_size: usize,
#[serde(default = "default_raw_image_convert_parallelism")]
pub raw_image_convert_parallelism: usize,
#[serde(default = "default_image_import_connect_timeout_secs")]
pub image_import_connect_timeout_secs: u64,
#[serde(default = "default_image_import_timeout_secs")]
pub image_import_timeout_secs: u64,
#[serde(default = "default_max_image_import_size_bytes")]
pub max_image_import_size_bytes: u64,
#[serde(default)]
pub allowed_https_hosts: Vec<String>,
#[serde(default)]
pub qemu_img_path: Option<PathBuf>,
}
fn default_image_cache_dir() -> PathBuf {
PathBuf::from("/var/lib/plasmavmc/images")
}
fn default_image_bucket() -> String {
"plasmavmc-images".to_string()
}
fn default_multipart_upload_concurrency() -> usize {
4
}
fn default_multipart_upload_part_size() -> usize {
32 * 1024 * 1024
}
fn default_raw_image_convert_parallelism() -> usize {
8
}
fn default_image_import_connect_timeout_secs() -> u64 {
5
}
fn default_image_import_timeout_secs() -> u64 {
60 * 60
}
fn default_max_image_import_size_bytes() -> u64 {
64 * 1024 * 1024 * 1024
}
impl Default for ArtifactStoreConfig {
fn default() -> Self {
Self {
lightningstor_endpoint: None,
image_cache_dir: default_image_cache_dir(),
image_bucket: default_image_bucket(),
multipart_upload_concurrency: default_multipart_upload_concurrency(),
multipart_upload_part_size: default_multipart_upload_part_size(),
raw_image_convert_parallelism: default_raw_image_convert_parallelism(),
image_import_connect_timeout_secs: default_image_import_connect_timeout_secs(),
image_import_timeout_secs: default_image_import_timeout_secs(),
max_image_import_size_bytes: default_max_image_import_size_bytes(),
allowed_https_hosts: Vec::new(),
qemu_img_path: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct CoronaFsConfig {
#[serde(default)]
pub controller_endpoint: Option<String>,
#[serde(default)]
pub node_endpoint: Option<String>,
#[serde(default)]
pub endpoint: Option<String>,
#[serde(default)]
pub node_local_attach: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CephConfig {
#[serde(default)]
pub monitors: Vec<String>,
#[serde(default = "default_ceph_cluster_id")]
pub cluster_id: String,
#[serde(default = "default_ceph_user")]
pub user: String,
#[serde(default)]
pub secret: Option<String>,
}
fn default_ceph_cluster_id() -> String {
"default".to_string()
}
fn default_ceph_user() -> String {
"admin".to_string()
}
impl Default for CephConfig {
fn default() -> Self {
Self {
monitors: Vec::new(),
cluster_id: default_ceph_cluster_id(),
user: default_ceph_user(),
secret: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VolumeRuntimeConfig {
#[serde(default = "default_managed_volume_root")]
pub managed_volume_root: PathBuf,
#[serde(default)]
pub ceph: CephConfig,
#[serde(default)]
pub coronafs: CoronaFsConfig,
#[serde(default)]
pub qemu_img_path: Option<PathBuf>,
}
fn default_managed_volume_root() -> PathBuf {
PathBuf::from("/var/lib/plasmavmc/managed-volumes")
}
impl Default for VolumeRuntimeConfig {
fn default() -> Self {
Self {
managed_volume_root: default_managed_volume_root(),
ceph: CephConfig::default(),
coronafs: CoronaFsConfig::default(),
qemu_img_path: None,
}
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "lowercase")]
pub enum DefaultHypervisor {
#[default]
Kvm,
Firecracker,
Mvisor,
}
/// Authentication configuration /// Authentication configuration
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AuthConfig { pub struct AuthConfig {
@ -78,6 +399,14 @@ impl Default for ServerConfig {
auth: AuthConfig::default(), auth: AuthConfig::default(),
kvm: KvmConfig::default(), kvm: KvmConfig::default(),
firecracker: FireCrackerConfig::default(), firecracker: FireCrackerConfig::default(),
storage: StorageRuntimeConfig::default(),
agent: AgentRuntimeConfig::default(),
integrations: IntegrationConfig::default(),
watcher: WatcherRuntimeConfig::default(),
health: HealthRuntimeConfig::default(),
artifacts: ArtifactStoreConfig::default(),
volumes: VolumeRuntimeConfig::default(),
default_hypervisor: DefaultHypervisor::default(),
} }
} }
} }

View file

@ -15,7 +15,7 @@ use plasmavmc_api::proto::{
use plasmavmc_firecracker::FireCrackerBackend; use plasmavmc_firecracker::FireCrackerBackend;
use plasmavmc_hypervisor::HypervisorRegistry; use plasmavmc_hypervisor::HypervisorRegistry;
use plasmavmc_kvm::KvmBackend; use plasmavmc_kvm::KvmBackend;
use plasmavmc_server::config::ServerConfig; use plasmavmc_server::config::{AgentRuntimeConfig, ServerConfig};
use plasmavmc_server::watcher::{StateSynchronizer, StateWatcher, WatcherConfig}; use plasmavmc_server::watcher::{StateSynchronizer, StateWatcher, WatcherConfig};
use plasmavmc_server::VmServiceImpl; use plasmavmc_server::VmServiceImpl;
use std::net::SocketAddr; use std::net::SocketAddr;
@ -83,16 +83,19 @@ async fn start_agent_heartbeat(
supported_volume_drivers: Vec<i32>, supported_volume_drivers: Vec<i32>,
supported_storage_classes: Vec<String>, supported_storage_classes: Vec<String>,
shared_live_migration: bool, shared_live_migration: bool,
agent_config: &AgentRuntimeConfig,
) { ) {
let Some(control_plane_addr) = std::env::var("PLASMAVMC_CONTROL_PLANE_ADDR") let Some(control_plane_addr) = agent_config
.ok() .control_plane_addr
.as_ref()
.map(|value| value.trim().to_string()) .map(|value| value.trim().to_string())
.filter(|value| !value.is_empty()) .filter(|value| !value.is_empty())
else { else {
return; return;
}; };
let Some(node_id) = std::env::var("PLASMAVMC_NODE_ID") let Some(node_id) = agent_config
.ok() .node_id
.as_ref()
.map(|value| value.trim().to_string()) .map(|value| value.trim().to_string())
.filter(|value| !value.is_empty()) .filter(|value| !value.is_empty())
else { else {
@ -100,19 +103,19 @@ async fn start_agent_heartbeat(
}; };
let endpoint = normalize_endpoint(&control_plane_addr); let endpoint = normalize_endpoint(&control_plane_addr);
let advertise_endpoint = std::env::var("PLASMAVMC_ENDPOINT_ADVERTISE") let advertise_endpoint = agent_config
.ok() .advertise_endpoint
.as_ref()
.map(|value| value.trim().to_string()) .map(|value| value.trim().to_string())
.filter(|value| !value.is_empty()) .filter(|value| !value.is_empty())
.unwrap_or_else(|| local_addr.to_string()); .unwrap_or_else(|| local_addr.to_string());
let node_name = std::env::var("PLASMAVMC_NODE_NAME") let node_name = agent_config
.ok() .node_name
.as_ref()
.cloned()
.filter(|value| !value.trim().is_empty()) .filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| node_id.clone()); .unwrap_or_else(|| node_id.clone());
let heartbeat_secs = std::env::var("PLASMAVMC_NODE_HEARTBEAT_INTERVAL_SECS") let heartbeat_secs = agent_config.heartbeat_interval_secs.max(1);
.ok()
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(5);
tokio::spawn(async move { tokio::spawn(async move {
let mut ticker = tokio::time::interval(Duration::from_secs(heartbeat_secs)); let mut ticker = tokio::time::interval(Duration::from_secs(heartbeat_secs));
@ -222,7 +225,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let registry = Arc::new(HypervisorRegistry::new()); let registry = Arc::new(HypervisorRegistry::new());
// Register KVM backend (always available) // Register KVM backend (always available)
let kvm_backend = Arc::new(KvmBackend::with_defaults()); let kvm_backend = Arc::new(KvmBackend::new(
config
.kvm
.qemu_path
.clone()
.unwrap_or_else(|| PathBuf::from("/usr/bin/qemu-system-x86_64")),
config
.kvm
.runtime_dir
.clone()
.unwrap_or_else(|| PathBuf::from("/run/libvirt/plasmavmc")),
));
registry.register(kvm_backend); registry.register(kvm_backend);
// Register FireCracker backend if kernel/rootfs paths are configured (config or env) // Register FireCracker backend if kernel/rootfs paths are configured (config or env)
@ -285,17 +299,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
registry, registry,
auth_service.clone(), auth_service.clone(),
config.auth.iam_server_addr.clone(), config.auth.iam_server_addr.clone(),
&config,
) )
.await?, .await?,
); );
// Optional: start state watcher for multi-instance HA sync // Optional: start state watcher for multi-instance HA sync
if std::env::var("PLASMAVMC_STATE_WATCHER") if config.watcher.enabled {
.map(|v| matches!(v.as_str(), "1" | "true" | "yes")) let watcher_config = WatcherConfig {
.unwrap_or(false) poll_interval: Duration::from_millis(config.watcher.poll_interval_ms.max(100)),
{ buffer_size: 256,
let config = WatcherConfig::default(); };
let (watcher, rx) = StateWatcher::new(vm_service.store(), config); let (watcher, rx) = StateWatcher::new(vm_service.store(), watcher_config);
let synchronizer = StateSynchronizer::new(vm_service.clone()); let synchronizer = StateSynchronizer::new(vm_service.clone());
tokio::spawn(async move { tokio::spawn(async move {
if let Err(e) = watcher.start().await { if let Err(e) = watcher.start().await {
@ -305,13 +320,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
tokio::spawn(async move { tokio::spawn(async move {
synchronizer.run(rx).await; synchronizer.run(rx).await;
}); });
tracing::info!("State watcher enabled (PLASMAVMC_STATE_WATCHER)"); tracing::info!("State watcher enabled");
} }
// Optional: start health monitor to refresh VM status periodically // Optional: start health monitor to refresh VM status periodically
if let Some(secs) = std::env::var("PLASMAVMC_HEALTH_MONITOR_INTERVAL_SECS") if let Some(secs) = config
.ok() .health
.and_then(|v| v.parse::<u64>().ok()) .vm_monitor_interval_secs
.filter(|secs| *secs > 0)
{ {
if secs > 0 { if secs > 0 {
vm_service vm_service
@ -321,18 +337,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
} }
// Optional: start node health monitor to detect stale heartbeats // Optional: start node health monitor to detect stale heartbeats
if let Some(interval_secs) = std::env::var("PLASMAVMC_NODE_HEALTH_MONITOR_INTERVAL_SECS") if let Some(interval_secs) = config
.ok() .health
.and_then(|v| v.parse::<u64>().ok()) .node_monitor_interval_secs
.filter(|secs| *secs > 0)
{ {
if interval_secs > 0 { if interval_secs > 0 {
let timeout_secs = std::env::var("PLASMAVMC_NODE_HEARTBEAT_TIMEOUT_SECS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(60);
vm_service.clone().start_node_health_monitor( vm_service.clone().start_node_health_monitor(
Duration::from_secs(interval_secs), Duration::from_secs(interval_secs),
Duration::from_secs(timeout_secs), Duration::from_secs(config.health.node_heartbeat_timeout_secs.max(1)),
); );
} }
} }
@ -369,6 +382,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
heartbeat_volume_drivers, heartbeat_volume_drivers,
heartbeat_storage_classes, heartbeat_storage_classes,
shared_live_migration, shared_live_migration,
&config.agent,
) )
.await; .await;

View file

@ -1,7 +1,8 @@
//! Storage abstraction for VM persistence //! Storage abstraction for VM persistence
use crate::config::{StorageBackendKind, StorageRuntimeConfig};
use async_trait::async_trait; use async_trait::async_trait;
use plasmavmc_types::{Image, Node, VirtualMachine, VmHandle, Volume}; use plasmavmc_types::{Image, ImageFormat, Node, VirtualMachine, VmHandle, Volume};
use std::path::PathBuf; use std::path::PathBuf;
use thiserror::Error; use thiserror::Error;
@ -16,14 +17,10 @@ pub enum StorageBackend {
} }
impl StorageBackend { impl StorageBackend {
pub fn from_env() -> Self { pub fn from_config(config: &StorageRuntimeConfig) -> Self {
match std::env::var("PLASMAVMC_STORAGE_BACKEND") match config.backend {
.as_deref() StorageBackendKind::Flaredb => Self::FlareDB,
.unwrap_or("flaredb") StorageBackendKind::File => Self::File,
{
"flaredb" => Self::FlareDB,
"file" => Self::File,
_ => Self::FlareDB,
} }
} }
} }
@ -48,6 +45,29 @@ pub enum StorageError {
/// Result type for storage operations /// Result type for storage operations
pub type StorageResult<T> = Result<T, StorageError>; pub type StorageResult<T> = Result<T, StorageError>;
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ImageUploadPart {
pub part_number: u32,
pub etag: String,
pub size_bytes: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ImageUploadSession {
pub session_id: String,
pub org_id: String,
pub project_id: String,
pub image_id: String,
pub upload_id: String,
pub staging_key: String,
pub source_format: ImageFormat,
#[serde(default)]
pub parts: Vec<ImageUploadPart>,
pub committed_size_bytes: u64,
pub created_at: u64,
pub updated_at: u64,
}
/// Storage trait for VM persistence /// Storage trait for VM persistence
#[async_trait] #[async_trait]
pub trait VmStore: Send + Sync { pub trait VmStore: Send + Sync {
@ -126,6 +146,35 @@ pub trait VmStore: Send + Sync {
/// List images for a tenant /// List images for a tenant
async fn list_images(&self, org_id: &str, project_id: &str) -> StorageResult<Vec<Image>>; async fn list_images(&self, org_id: &str, project_id: &str) -> StorageResult<Vec<Image>>;
/// Save an in-progress image upload session.
async fn save_image_upload_session(&self, session: &ImageUploadSession) -> StorageResult<()>;
/// Load an in-progress image upload session.
async fn load_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<Option<ImageUploadSession>>;
/// Delete an image upload session.
async fn delete_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<()>;
/// List image upload sessions for an image.
async fn list_image_upload_sessions(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
) -> StorageResult<Vec<ImageUploadSession>>;
/// Save a persistent volume /// Save a persistent volume
async fn save_volume(&self, volume: &Volume) -> StorageResult<()>; async fn save_volume(&self, volume: &Volume) -> StorageResult<()>;
@ -191,6 +240,27 @@ fn image_prefix(org_id: &str, project_id: &str) -> String {
format!("/plasmavmc/images/{}/{}/", org_id, project_id) format!("/plasmavmc/images/{}/{}/", org_id, project_id)
} }
/// Build key for image upload session metadata
fn image_upload_session_key(
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> String {
format!(
"/plasmavmc/image-upload-sessions/{}/{}/{}/{}",
org_id, project_id, image_id, session_id
)
}
/// Build prefix for image upload session listing
fn image_upload_session_prefix(org_id: &str, project_id: &str, image_id: &str) -> String {
format!(
"/plasmavmc/image-upload-sessions/{}/{}/{}/",
org_id, project_id, image_id
)
}
/// Build key for volume metadata /// Build key for volume metadata
fn volume_key(org_id: &str, project_id: &str, volume_id: &str) -> String { fn volume_key(org_id: &str, project_id: &str, volume_id: &str) -> String {
format!("/plasmavmc/volumes/{}/{}/{}", org_id, project_id, volume_id) format!("/plasmavmc/volumes/{}/{}/{}", org_id, project_id, volume_id)
@ -207,14 +277,20 @@ pub struct FlareDBStore {
} }
impl FlareDBStore { impl FlareDBStore {
/// Create a new FlareDB store pub async fn new_with_config(config: &StorageRuntimeConfig) -> StorageResult<Self> {
pub async fn new(endpoint: Option<String>) -> StorageResult<Self> { Self::new_with_endpoints(
let endpoint = endpoint.unwrap_or_else(|| { config.flaredb_endpoint.clone(),
std::env::var("PLASMAVMC_FLAREDB_ENDPOINT") config.chainfire_endpoint.clone(),
.unwrap_or_else(|_| "127.0.0.1:2479".to_string()) )
}); .await
let pd_endpoint = std::env::var("PLASMAVMC_CHAINFIRE_ENDPOINT") }
.ok()
pub async fn new_with_endpoints(
endpoint: Option<String>,
pd_endpoint: Option<String>,
) -> StorageResult<Self> {
let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
let pd_endpoint = pd_endpoint
.map(|value| normalize_transport_addr(&value)) .map(|value| normalize_transport_addr(&value))
.unwrap_or_else(|| endpoint.clone()); .unwrap_or_else(|| endpoint.clone());
@ -561,6 +637,58 @@ impl VmStore for FlareDBStore {
Ok(images) Ok(images)
} }
async fn save_image_upload_session(&self, session: &ImageUploadSession) -> StorageResult<()> {
let key = image_upload_session_key(
&session.org_id,
&session.project_id,
&session.image_id,
&session.session_id,
);
let value = serde_json::to_vec(session)?;
self.cas_put(&key, value).await
}
async fn load_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<Option<ImageUploadSession>> {
let key = image_upload_session_key(org_id, project_id, image_id, session_id);
match self.cas_get(&key).await? {
Some(data) => Ok(Some(serde_json::from_slice(&data)?)),
None => Ok(None),
}
}
async fn delete_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<()> {
let key = image_upload_session_key(org_id, project_id, image_id, session_id);
self.cas_delete(&key).await
}
async fn list_image_upload_sessions(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
) -> StorageResult<Vec<ImageUploadSession>> {
let prefix = image_upload_session_prefix(org_id, project_id, image_id);
let mut sessions = Vec::new();
for value in self.cas_scan_values(&prefix).await? {
if let Ok(session) = serde_json::from_slice::<ImageUploadSession>(&value) {
sessions.push(session);
}
}
Ok(sessions)
}
async fn save_volume(&self, volume: &Volume) -> StorageResult<()> { async fn save_volume(&self, volume: &Volume) -> StorageResult<()> {
let key = volume_key(&volume.org_id, &volume.project_id, &volume.id); let key = volume_key(&volume.org_id, &volume.project_id, &volume.id);
let value = serde_json::to_vec(volume)?; let value = serde_json::to_vec(volume)?;
@ -653,6 +781,8 @@ struct PersistedState {
#[serde(default)] #[serde(default)]
images: Vec<Image>, images: Vec<Image>,
#[serde(default)] #[serde(default)]
image_upload_sessions: Vec<ImageUploadSession>,
#[serde(default)]
volumes: Vec<Volume>, volumes: Vec<Volume>,
} }
@ -841,6 +971,71 @@ impl VmStore for FileStore {
.collect()) .collect())
} }
async fn save_image_upload_session(&self, session: &ImageUploadSession) -> StorageResult<()> {
let mut state = self.load_state().unwrap_or_default();
state.image_upload_sessions.retain(|existing| {
!(existing.org_id == session.org_id
&& existing.project_id == session.project_id
&& existing.image_id == session.image_id
&& existing.session_id == session.session_id)
});
state.image_upload_sessions.push(session.clone());
self.save_state(&state)?;
Ok(())
}
async fn load_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<Option<ImageUploadSession>> {
let state = self.load_state().unwrap_or_default();
Ok(state.image_upload_sessions.into_iter().find(|session| {
session.org_id == org_id
&& session.project_id == project_id
&& session.image_id == image_id
&& session.session_id == session_id
}))
}
async fn delete_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<()> {
let mut state = self.load_state().unwrap_or_default();
state.image_upload_sessions.retain(|session| {
!(session.org_id == org_id
&& session.project_id == project_id
&& session.image_id == image_id
&& session.session_id == session_id)
});
self.save_state(&state)?;
Ok(())
}
async fn list_image_upload_sessions(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
) -> StorageResult<Vec<ImageUploadSession>> {
let state = self.load_state().unwrap_or_default();
Ok(state
.image_upload_sessions
.into_iter()
.filter(|session| {
session.org_id == org_id
&& session.project_id == project_id
&& session.image_id == image_id
})
.collect())
}
async fn save_volume(&self, volume: &Volume) -> StorageResult<()> { async fn save_volume(&self, volume: &Volume) -> StorageResult<()> {
let mut state = self.load_state().unwrap_or_default(); let mut state = self.load_state().unwrap_or_default();
state.volumes.retain(|existing| existing.id != volume.id); state.volumes.retain(|existing| existing.id != volume.id);
@ -972,4 +1167,68 @@ mod tests {
.expect("current volume should remain"); .expect("current volume should remain");
assert_eq!(loaded, current); assert_eq!(loaded, current);
} }
#[tokio::test]
async fn filestore_round_trips_image_upload_sessions() {
let tempdir = tempdir().unwrap();
let store = FileStore::new(Some(tempdir.path().join("state.json")));
let session = ImageUploadSession {
session_id: "11111111-1111-1111-1111-111111111111".to_string(),
org_id: "org-1".to_string(),
project_id: "project-1".to_string(),
image_id: "22222222-2222-2222-2222-222222222222".to_string(),
upload_id: "multipart-1".to_string(),
staging_key: "org-1/project-1/uploads/22222222-2222-2222-2222-222222222222.source"
.to_string(),
source_format: ImageFormat::Raw,
parts: vec![ImageUploadPart {
part_number: 1,
etag: "etag-1".to_string(),
size_bytes: 4096,
}],
committed_size_bytes: 4096,
created_at: 1,
updated_at: 2,
};
store.save_image_upload_session(&session).await.unwrap();
let loaded = store
.load_image_upload_session(
&session.org_id,
&session.project_id,
&session.image_id,
&session.session_id,
)
.await
.unwrap()
.expect("session should exist");
assert_eq!(loaded, session);
let listed = store
.list_image_upload_sessions(&session.org_id, &session.project_id, &session.image_id)
.await
.unwrap();
assert_eq!(listed, vec![session.clone()]);
store
.delete_image_upload_session(
&session.org_id,
&session.project_id,
&session.image_id,
&session.session_id,
)
.await
.unwrap();
assert!(store
.load_image_upload_session(
&session.org_id,
&session.project_id,
&session.image_id,
&session.session_id,
)
.await
.unwrap()
.is_none());
}
} }

View file

@ -1,8 +1,9 @@
//! VM Service implementation //! VM Service implementation
use crate::artifact_store::ArtifactStore; use crate::artifact_store::ArtifactStore;
use crate::config::{DefaultHypervisor, ServerConfig};
use crate::prismnet_client::PrismNETClient; use crate::prismnet_client::PrismNETClient;
use crate::storage::{FileStore, FlareDBStore, StorageBackend, VmStore}; use crate::storage::{FileStore, FlareDBStore, ImageUploadSession, StorageBackend, VmStore};
use crate::volume_manager::VolumeManager; use crate::volume_manager::VolumeManager;
use crate::watcher::StateSink; use crate::watcher::StateSink;
use creditservice_client::{Client as CreditServiceClient, ResourceType as CreditResourceType}; use creditservice_client::{Client as CreditServiceClient, ResourceType as CreditResourceType};
@ -11,13 +12,15 @@ use iam_client::client::IamClientConfig;
use iam_client::IamClient; use iam_client::IamClient;
use iam_service_auth::{ use iam_service_auth::{
get_tenant_context, resolve_tenant_ids_from_context, resource_for_tenant, AuthService, get_tenant_context, resolve_tenant_ids_from_context, resource_for_tenant, AuthService,
TenantContext,
}; };
use iam_types::{PolicyBinding, PrincipalRef, Scope}; use iam_types::{PolicyBinding, PrincipalKind, PrincipalRef, Scope};
use plasmavmc_api::proto::{ use plasmavmc_api::proto::{
disk_source::Source as ProtoDiskSourceKind, image_service_server::ImageService, disk_source::Source as ProtoDiskSourceKind, image_service_server::ImageService,
node_service_server::NodeService, vm_service_client::VmServiceClient, node_service_server::NodeService, vm_service_client::VmServiceClient,
vm_service_server::VmService, volume_service_server::VolumeService, vm_service_server::VmService, volume_service_server::VolumeService, AbortImageUploadRequest,
Architecture as ProtoArchitecture, AttachDiskRequest, AttachNicRequest, CephRbdBacking, Architecture as ProtoArchitecture, AttachDiskRequest, AttachNicRequest,
BeginImageUploadRequest, BeginImageUploadResponse, CephRbdBacking, CompleteImageUploadRequest,
CordonNodeRequest, CreateImageRequest, CreateVmRequest, CreateVolumeRequest, CordonNodeRequest, CreateImageRequest, CreateVmRequest, CreateVolumeRequest,
DeleteImageRequest, DeleteVmRequest, DeleteVolumeRequest, DetachDiskRequest, DetachNicRequest, DeleteImageRequest, DeleteVmRequest, DeleteVolumeRequest, DetachDiskRequest, DetachNicRequest,
DiskBus as ProtoDiskBus, DiskCache as ProtoDiskCache, DiskSource as ProtoDiskSource, DiskBus as ProtoDiskBus, DiskCache as ProtoDiskCache, DiskSource as ProtoDiskSource,
@ -30,9 +33,9 @@ use plasmavmc_api::proto::{
NodeState as ProtoNodeState, OsType as ProtoOsType, PrepareVmMigrationRequest, RebootVmRequest, NodeState as ProtoNodeState, OsType as ProtoOsType, PrepareVmMigrationRequest, RebootVmRequest,
RecoverVmRequest, RegisterExternalVolumeRequest, ResetVmRequest, ResizeVolumeRequest, RecoverVmRequest, RegisterExternalVolumeRequest, ResetVmRequest, ResizeVolumeRequest,
StartVmRequest, StopVmRequest, UncordonNodeRequest, UpdateImageRequest, UpdateVmRequest, StartVmRequest, StopVmRequest, UncordonNodeRequest, UpdateImageRequest, UpdateVmRequest,
VirtualMachine, Visibility as ProtoVisibility, VmEvent, VmEventType as ProtoVmEventType, UploadImagePartRequest, UploadImagePartResponse, VirtualMachine, Visibility as ProtoVisibility,
VmSpec as ProtoVmSpec, VmState as ProtoVmState, VmStatus as ProtoVmStatus, VmEvent, VmEventType as ProtoVmEventType, VmSpec as ProtoVmSpec, VmState as ProtoVmState,
Volume as ProtoVolume, VolumeBacking as ProtoVolumeBacking, VmStatus as ProtoVmStatus, Volume as ProtoVolume, VolumeBacking as ProtoVolumeBacking,
VolumeDriverKind as ProtoVolumeDriverKind, VolumeFormat as ProtoVolumeFormat, VolumeDriverKind as ProtoVolumeDriverKind, VolumeFormat as ProtoVolumeFormat,
VolumeStatus as ProtoVolumeStatus, WatchVmRequest, VolumeStatus as ProtoVolumeStatus, WatchVmRequest,
}; };
@ -91,6 +94,12 @@ pub struct VmServiceImpl {
credit_service: Option<Arc<RwLock<CreditServiceClient>>>, credit_service: Option<Arc<RwLock<CreditServiceClient>>>,
/// Local node identifier (optional) /// Local node identifier (optional)
local_node_id: Option<String>, local_node_id: Option<String>,
shared_live_migration: bool,
default_hypervisor: HypervisorType,
watch_poll_interval: Duration,
auto_restart: bool,
failover_enabled: bool,
failover_min_interval_secs: u64,
artifact_store: Option<Arc<ArtifactStore>>, artifact_store: Option<Arc<ArtifactStore>>,
volume_manager: Arc<VolumeManager>, volume_manager: Arc<VolumeManager>,
iam_client: Arc<IamClient>, iam_client: Arc<IamClient>,
@ -149,33 +158,34 @@ impl VmServiceImpl {
hypervisor_registry: Arc<HypervisorRegistry>, hypervisor_registry: Arc<HypervisorRegistry>,
auth: Arc<AuthService>, auth: Arc<AuthService>,
iam_endpoint: impl Into<String>, iam_endpoint: impl Into<String>,
config: &ServerConfig,
) -> Result<Self, Box<dyn std::error::Error>> { ) -> Result<Self, Box<dyn std::error::Error>> {
let backend = StorageBackend::from_env(); let backend = StorageBackend::from_config(&config.storage);
let store: Arc<dyn VmStore> = match backend { let store: Arc<dyn VmStore> = match backend {
StorageBackend::FlareDB => match FlareDBStore::new(None).await { StorageBackend::FlareDB => match FlareDBStore::new_with_config(&config.storage).await {
Ok(flaredb_store) => Arc::new(flaredb_store), Ok(flaredb_store) => Arc::new(flaredb_store),
Err(e) => { Err(e) => {
tracing::warn!( tracing::warn!(
"Failed to connect to FlareDB, falling back to file storage: {}", "Failed to connect to FlareDB, falling back to file storage: {}",
e e
); );
Arc::new(FileStore::new(None)) Arc::new(FileStore::new(config.storage.state_path.clone()))
} }
}, },
StorageBackend::File => { StorageBackend::File => {
let file_store = FileStore::new(None); let file_store = FileStore::new(config.storage.state_path.clone());
Arc::new(file_store) Arc::new(file_store)
} }
}; };
let prismnet_endpoint = std::env::var("PRISMNET_ENDPOINT").ok(); let prismnet_endpoint = config.integrations.prismnet_endpoint.clone();
if let Some(ref endpoint) = prismnet_endpoint { if let Some(ref endpoint) = prismnet_endpoint {
tracing::info!("PrismNET integration enabled: {}", endpoint); tracing::info!("PrismNET integration enabled: {}", endpoint);
} }
// Initialize CreditService client if endpoint is configured // Initialize CreditService client if endpoint is configured
let credit_service = match std::env::var("CREDITSERVICE_ENDPOINT") { let credit_service = match config.integrations.creditservice_endpoint.as_deref() {
Ok(endpoint) => match CreditServiceClient::connect(&endpoint).await { Some(endpoint) => match CreditServiceClient::connect(endpoint).await {
Ok(client) => { Ok(client) => {
tracing::info!("CreditService admission control enabled: {}", endpoint); tracing::info!("CreditService admission control enabled: {}", endpoint);
Some(Arc::new(RwLock::new(client))) Some(Arc::new(RwLock::new(client)))
@ -188,13 +198,17 @@ impl VmServiceImpl {
None None
} }
}, },
Err(_) => { None => {
tracing::info!("CREDITSERVICE_ENDPOINT not set, admission control disabled"); tracing::info!("CreditService endpoint not configured, admission control disabled");
None None
} }
}; };
let local_node_id = std::env::var("PLASMAVMC_NODE_ID").ok(); let local_node_id = config
.agent
.node_id
.clone()
.filter(|value| !value.trim().is_empty());
if let Some(ref node_id) = local_node_id { if let Some(ref node_id) = local_node_id {
tracing::info!("Local node ID: {}", node_id); tracing::info!("Local node ID: {}", node_id);
} }
@ -206,13 +220,25 @@ impl VmServiceImpl {
iam_config = iam_config.without_tls(); iam_config = iam_config.without_tls();
} }
let iam_client = Arc::new(IamClient::connect(iam_config).await?); let iam_client = Arc::new(IamClient::connect(iam_config).await?);
let artifact_store = ArtifactStore::from_env(&normalized_iam_endpoint) let artifact_store =
ArtifactStore::from_config(&config.artifacts, &normalized_iam_endpoint)
.await? .await?
.map(Arc::new); .map(Arc::new);
if artifact_store.is_some() { if artifact_store.is_some() {
tracing::info!("LightningStor artifact backing enabled for VM disks"); tracing::info!("LightningStor artifact backing enabled for VM disks");
} }
let volume_manager = Arc::new(VolumeManager::new(store.clone(), artifact_store.clone())); let volume_manager = Arc::new(VolumeManager::new_with_config(
store.clone(),
artifact_store.clone(),
&config.volumes,
local_node_id.clone(),
)?);
let default_hypervisor = match config.default_hypervisor {
DefaultHypervisor::Kvm => HypervisorType::Kvm,
DefaultHypervisor::Firecracker => HypervisorType::Firecracker,
DefaultHypervisor::Mvisor => HypervisorType::Mvisor,
};
let svc = Self { let svc = Self {
hypervisor_registry, hypervisor_registry,
@ -224,6 +250,14 @@ impl VmServiceImpl {
prismnet_endpoint, prismnet_endpoint,
credit_service, credit_service,
local_node_id, local_node_id,
shared_live_migration: config.agent.shared_live_migration,
default_hypervisor,
watch_poll_interval: Duration::from_millis(
config.watcher.vm_watch_poll_interval_ms.max(100),
),
auto_restart: config.health.auto_restart,
failover_enabled: config.health.failover_enabled,
failover_min_interval_secs: config.health.failover_min_interval_secs.max(1),
artifact_store, artifact_store,
volume_manager, volume_manager,
iam_client, iam_client,
@ -242,10 +276,7 @@ impl VmServiceImpl {
} }
pub fn shared_live_migration(&self) -> bool { pub fn shared_live_migration(&self) -> bool {
std::env::var("PLASMAVMC_SHARED_LIVE_MIGRATION") self.shared_live_migration
.ok()
.map(|value| matches!(value.as_str(), "1" | "true" | "yes"))
.unwrap_or(true)
} }
pub fn store(&self) -> Arc<dyn VmStore> { pub fn store(&self) -> Arc<dyn VmStore> {
@ -256,24 +287,12 @@ impl VmServiceImpl {
Status::internal(err.to_string()) Status::internal(err.to_string())
} }
fn map_hv(typ: ProtoHypervisorType) -> HypervisorType { fn map_hv(&self, typ: ProtoHypervisorType) -> HypervisorType {
match typ { match typ {
ProtoHypervisorType::Kvm => HypervisorType::Kvm, ProtoHypervisorType::Kvm => HypervisorType::Kvm,
ProtoHypervisorType::Firecracker => HypervisorType::Firecracker, ProtoHypervisorType::Firecracker => HypervisorType::Firecracker,
ProtoHypervisorType::Mvisor => HypervisorType::Mvisor, ProtoHypervisorType::Mvisor => HypervisorType::Mvisor,
ProtoHypervisorType::Unspecified => { ProtoHypervisorType::Unspecified => self.default_hypervisor,
// Use environment variable for default, fallback to KVM
match std::env::var("PLASMAVMC_HYPERVISOR")
.as_deref()
.map(|s| s.to_lowercase())
.as_deref()
{
Ok("firecracker") => HypervisorType::Firecracker,
Ok("kvm") => HypervisorType::Kvm,
Ok("mvisor") => HypervisorType::Mvisor,
_ => HypervisorType::Kvm, // Default to KVM for backwards compatibility
}
}
} }
} }
@ -292,13 +311,81 @@ impl VmServiceImpl {
.as_secs() .as_secs()
} }
fn watch_poll_interval() -> Duration { fn require_uuid(value: &str, field_name: &str) -> Result<(), Status> {
let poll_interval_ms = std::env::var("PLASMAVMC_VM_WATCH_POLL_INTERVAL_MS") Uuid::parse_str(value)
.ok() .map(|_| ())
.and_then(|value| value.parse::<u64>().ok()) .map_err(|_| Status::invalid_argument(format!("{field_name} must be a UUID")))
.unwrap_or(500) }
.max(100);
Duration::from_millis(poll_interval_ms) fn validate_disk_reference(disk: &plasmavmc_types::DiskSpec) -> Result<(), Status> {
match &disk.source {
DiskSource::Image { image_id } => Self::require_uuid(image_id, "image_id"),
DiskSource::Volume { volume_id } => Self::require_uuid(volume_id, "volume_id"),
DiskSource::Blank => Ok(()),
}
}
fn validate_vm_disk_references(spec: &plasmavmc_types::VmSpec) -> Result<(), Status> {
for disk in &spec.disks {
Self::validate_disk_reference(disk)?;
}
Ok(())
}
fn ensure_internal_rpc(tenant: &TenantContext) -> Result<(), Status> {
if tenant.principal_kind != PrincipalKind::ServiceAccount
|| !tenant.principal_id.starts_with("plasmavmc-")
{
return Err(Status::permission_denied(
"this RPC is restricted to internal PlasmaVMC service accounts",
));
}
Ok(())
}
fn expected_migration_listener_uri(&self, vm_id: &str) -> Result<String, Status> {
let node_id = self.local_node_id.as_deref().ok_or_else(|| {
Status::failed_precondition("local node id is required for migration preparation")
})?;
let mut hasher = std::collections::hash_map::DefaultHasher::new();
node_id.hash(&mut hasher);
vm_id.hash(&mut hasher);
let port = 4400 + (hasher.finish() % 1000) as u16;
Ok(format!("tcp:0.0.0.0:{port}"))
}
fn apply_imported_image_metadata(
image: &mut Image,
imported: &crate::artifact_store::ImportedImage,
source_format: ImageFormat,
) {
image.status = ImageStatus::Available;
image.format = imported.format;
image.size_bytes = imported.size_bytes;
image.checksum = imported.checksum.clone();
image.updated_at = Self::now_epoch();
image
.metadata
.insert("source_type".to_string(), imported.source_type.clone());
if let Some(host) = &imported.source_host {
image
.metadata
.insert("source_host".to_string(), host.clone());
}
image.metadata.insert(
"artifact_key".to_string(),
format!("{}/{}/{}.qcow2", image.org_id, image.project_id, image.id),
);
if source_format != image.format {
image.metadata.insert(
"source_format".to_string(),
format!("{source_format:?}").to_lowercase(),
);
}
}
fn watch_poll_interval(&self) -> Duration {
self.watch_poll_interval
} }
fn vm_values_differ( fn vm_values_differ(
@ -954,7 +1041,10 @@ impl VmServiceImpl {
} }
} }
fn proto_vm_to_types(vm: &VirtualMachine) -> Result<plasmavmc_types::VirtualMachine, Status> { fn proto_vm_to_types(
&self,
vm: &VirtualMachine,
) -> Result<plasmavmc_types::VirtualMachine, Status> {
let spec = Self::proto_spec_to_types(vm.spec.clone()); let spec = Self::proto_spec_to_types(vm.spec.clone());
let mut typed = plasmavmc_types::VirtualMachine::new( let mut typed = plasmavmc_types::VirtualMachine::new(
vm.name.clone(), vm.name.clone(),
@ -965,7 +1055,7 @@ impl VmServiceImpl {
typed.id = VmId::from_uuid( typed.id = VmId::from_uuid(
Uuid::parse_str(&vm.id).map_err(|e| Status::internal(format!("invalid VM id: {e}")))?, Uuid::parse_str(&vm.id).map_err(|e| Status::internal(format!("invalid VM id: {e}")))?,
); );
typed.hypervisor = Self::map_hv( typed.hypervisor = self.map_hv(
ProtoHypervisorType::try_from(vm.hypervisor).unwrap_or(ProtoHypervisorType::Kvm), ProtoHypervisorType::try_from(vm.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
); );
typed.node_id = if vm.node_id.is_empty() { typed.node_id = if vm.node_id.is_empty() {
@ -1690,9 +1780,6 @@ impl VmServiceImpl {
.map(|entry| (entry.key().clone(), entry.value().clone())) .map(|entry| (entry.key().clone(), entry.value().clone()))
.collect(); .collect();
let auto_restart = std::env::var("PLASMAVMC_AUTO_RESTART")
.map(|v| matches!(v.as_str(), "1" | "true" | "yes"))
.unwrap_or(false);
let local_node = self.local_node_id.as_deref(); let local_node = self.local_node_id.as_deref();
for (key, mut vm) in entries { for (key, mut vm) in entries {
@ -1733,7 +1820,7 @@ impl VmServiceImpl {
match backend.status(&handle).await { match backend.status(&handle).await {
Ok(status) => { Ok(status) => {
if auto_restart if self.auto_restart
&& vm.state == VmState::Running && vm.state == VmState::Running
&& matches!(status.actual_state, VmState::Stopped | VmState::Error) && matches!(status.actual_state, VmState::Stopped | VmState::Error)
{ {
@ -1812,18 +1899,10 @@ impl VmServiceImpl {
return; return;
} }
let failover_enabled = std::env::var("PLASMAVMC_FAILOVER_CONTROLLER")
.map(|v| matches!(v.as_str(), "1" | "true" | "yes"))
.unwrap_or(false);
let min_interval_secs = std::env::var("PLASMAVMC_FAILOVER_MIN_INTERVAL_SECS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(60);
let mut failed_over: HashSet<String> = HashSet::new(); let mut failed_over: HashSet<String> = HashSet::new();
if failover_enabled { if self.failover_enabled {
failed_over = self failed_over = self
.failover_vms_on_unhealthy(&unhealthy, &nodes, min_interval_secs) .failover_vms_on_unhealthy(&unhealthy, &nodes, self.failover_min_interval_secs)
.await; .await;
} }
@ -2047,6 +2126,43 @@ mod tests {
.unwrap(); .unwrap();
assert_eq!(deleted.event_type, ProtoVmEventType::Deleted as i32); assert_eq!(deleted.event_type, ProtoVmEventType::Deleted as i32);
} }
#[test]
fn internal_rpc_guard_requires_plasmavmc_service_account() {
let internal = TenantContext {
org_id: "org".to_string(),
project_id: "project".to_string(),
principal_id: "plasmavmc-org-project".to_string(),
principal_name: "plasmavmc".to_string(),
principal_kind: PrincipalKind::ServiceAccount,
node_id: None,
};
let external = TenantContext {
principal_id: "user-1".to_string(),
principal_kind: PrincipalKind::User,
..internal.clone()
};
assert!(VmServiceImpl::ensure_internal_rpc(&internal).is_ok());
assert!(VmServiceImpl::ensure_internal_rpc(&external).is_err());
}
#[test]
fn vm_disk_reference_validation_requires_uuid_identifiers() {
let mut spec = VmSpec::default();
spec.disks.push(plasmavmc_types::DiskSpec {
id: "root".to_string(),
source: DiskSource::Image {
image_id: "../passwd".to_string(),
},
size_gib: 10,
bus: DiskBus::Virtio,
cache: DiskCache::Writeback,
boot_index: Some(1),
});
assert!(VmServiceImpl::validate_vm_disk_references(&spec).is_err());
}
} }
impl StateSink for VmServiceImpl { impl StateSink for VmServiceImpl {
@ -2121,13 +2237,14 @@ impl VmService for VmServiceImpl {
"CreateVm request" "CreateVm request"
); );
let hv = Self::map_hv( let hv = self.map_hv(
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm), ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
); );
if req.spec.is_none() { if req.spec.is_none() {
return Err(Status::invalid_argument("spec is required")); return Err(Status::invalid_argument("spec is required"));
} }
let spec = Self::proto_spec_to_types(req.spec.clone()); let spec = Self::proto_spec_to_types(req.spec.clone());
Self::validate_vm_disk_references(&spec)?;
if self.is_control_plane_scheduler() { if self.is_control_plane_scheduler() {
if let Some(target) = self if let Some(target) = self
.select_target_node(hv, &req.org_id, &req.project_id, &spec) .select_target_node(hv, &req.org_id, &req.project_id, &spec)
@ -2137,7 +2254,7 @@ impl VmService for VmServiceImpl {
let forwarded = self let forwarded = self
.forward_create_to_node(endpoint, &req.org_id, &req.project_id, &req) .forward_create_to_node(endpoint, &req.org_id, &req.project_id, &req)
.await?; .await?;
let forwarded_vm = Self::proto_vm_to_types(&forwarded)?; let forwarded_vm = self.proto_vm_to_types(&forwarded)?;
let key = TenantKey::new( let key = TenantKey::new(
&forwarded_vm.org_id, &forwarded_vm.org_id,
&forwarded_vm.project_id, &forwarded_vm.project_id,
@ -2395,7 +2512,7 @@ impl VmService for VmServiceImpl {
.await .await
.map_err(|status| Status::from_error(Box::new(status)))? .map_err(|status| Status::from_error(Box::new(status)))?
.into_inner(); .into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?; let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key.clone(), typed_vm.clone()); self.vms.insert(key.clone(), typed_vm.clone());
self.persist_vm(&typed_vm).await; self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm)); return Ok(Response::new(remote_vm));
@ -2700,7 +2817,7 @@ impl VmService for VmServiceImpl {
.await .await
.map_err(|status| Status::from_error(Box::new(status)))? .map_err(|status| Status::from_error(Box::new(status)))?
.into_inner(); .into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?; let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone()); self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await; self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm)); return Ok(Response::new(remote_vm));
@ -2802,7 +2919,7 @@ impl VmService for VmServiceImpl {
.await .await
.map_err(|status| Status::from_error(Box::new(status)))? .map_err(|status| Status::from_error(Box::new(status)))?
.into_inner(); .into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?; let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone()); self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await; self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm)); return Ok(Response::new(remote_vm));
@ -2889,7 +3006,7 @@ impl VmService for VmServiceImpl {
.await .await
.map_err(|status| Status::from_error(Box::new(status)))? .map_err(|status| Status::from_error(Box::new(status)))?
.into_inner(); .into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?; let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone()); self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await; self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm)); return Ok(Response::new(remote_vm));
@ -2970,7 +3087,7 @@ impl VmService for VmServiceImpl {
.await .await
.map_err(|status| Status::from_error(Box::new(status)))? .map_err(|status| Status::from_error(Box::new(status)))?
.into_inner(); .into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?; let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone()); self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await; self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm)); return Ok(Response::new(remote_vm));
@ -3061,7 +3178,7 @@ impl VmService for VmServiceImpl {
.await .await
.map_err(|status| Status::from_error(Box::new(status)))? .map_err(|status| Status::from_error(Box::new(status)))?
.into_inner(); .into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?; let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone()); self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await; self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm)); return Ok(Response::new(remote_vm));
@ -3171,7 +3288,7 @@ impl VmService for VmServiceImpl {
return match client.recover_vm(recover_req).await { return match client.recover_vm(recover_req).await {
Ok(remote_vm) => { Ok(remote_vm) => {
let remote_vm = remote_vm.into_inner(); let remote_vm = remote_vm.into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?; let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone()); self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await; self.persist_vm(&typed_vm).await;
Ok(Response::new(remote_vm)) Ok(Response::new(remote_vm))
@ -3352,6 +3469,7 @@ impl VmService for VmServiceImpl {
request: Request<PrepareVmMigrationRequest>, request: Request<PrepareVmMigrationRequest>,
) -> Result<Response<VirtualMachine>, Status> { ) -> Result<Response<VirtualMachine>, Status> {
let tenant = get_tenant_context(&request)?; let tenant = get_tenant_context(&request)?;
Self::ensure_internal_rpc(&tenant)?;
let (org_id, project_id) = resolve_tenant_ids_from_context( let (org_id, project_id) = resolve_tenant_ids_from_context(
&tenant, &tenant,
&request.get_ref().org_id, &request.get_ref().org_id,
@ -3378,6 +3496,12 @@ impl VmService for VmServiceImpl {
if req.listen_uri.is_empty() { if req.listen_uri.is_empty() {
return Err(Status::invalid_argument("listen_uri is required")); return Err(Status::invalid_argument("listen_uri is required"));
} }
let expected_listen_uri = self.expected_migration_listener_uri(&req.vm_id)?;
if req.listen_uri != expected_listen_uri {
return Err(Status::invalid_argument(format!(
"listen_uri must exactly match {expected_listen_uri}",
)));
}
self.ensure_destination_slot_available(&req.org_id, &req.project_id, &req.vm_id) self.ensure_destination_slot_available(&req.org_id, &req.project_id, &req.vm_id)
.await?; .await?;
@ -3385,7 +3509,7 @@ impl VmService for VmServiceImpl {
let vm_uuid = Uuid::parse_str(&req.vm_id) let vm_uuid = Uuid::parse_str(&req.vm_id)
.map_err(|_| Status::invalid_argument("vm_id must be a UUID"))?; .map_err(|_| Status::invalid_argument("vm_id must be a UUID"))?;
let hv = Self::map_hv( let hv = self.map_hv(
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm), ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
); );
let backend = self let backend = self
@ -3399,6 +3523,7 @@ impl VmService for VmServiceImpl {
} }
let spec = Self::proto_spec_to_types(req.spec); let spec = Self::proto_spec_to_types(req.spec);
Self::validate_vm_disk_references(&spec)?;
let name = if req.name.is_empty() { let name = if req.name.is_empty() {
req.vm_id.clone() req.vm_id.clone()
} else { } else {
@ -3440,6 +3565,7 @@ impl VmService for VmServiceImpl {
request: Request<RecoverVmRequest>, request: Request<RecoverVmRequest>,
) -> Result<Response<VirtualMachine>, Status> { ) -> Result<Response<VirtualMachine>, Status> {
let tenant = get_tenant_context(&request)?; let tenant = get_tenant_context(&request)?;
Self::ensure_internal_rpc(&tenant)?;
let (org_id, project_id) = resolve_tenant_ids_from_context( let (org_id, project_id) = resolve_tenant_ids_from_context(
&tenant, &tenant,
&request.get_ref().org_id, &request.get_ref().org_id,
@ -3469,7 +3595,7 @@ impl VmService for VmServiceImpl {
let vm_uuid = Uuid::parse_str(&req.vm_id) let vm_uuid = Uuid::parse_str(&req.vm_id)
.map_err(|_| Status::invalid_argument("vm_id must be a UUID"))?; .map_err(|_| Status::invalid_argument("vm_id must be a UUID"))?;
let hv = Self::map_hv( let hv = self.map_hv(
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm), ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
); );
let backend = self let backend = self
@ -3481,6 +3607,7 @@ impl VmService for VmServiceImpl {
.await?; .await?;
let spec = Self::proto_spec_to_types(req.spec); let spec = Self::proto_spec_to_types(req.spec);
Self::validate_vm_disk_references(&spec)?;
let name = if req.name.is_empty() { let name = if req.name.is_empty() {
req.vm_id.clone() req.vm_id.clone()
} else { } else {
@ -3584,6 +3711,7 @@ impl VmService for VmServiceImpl {
.disk .disk
.ok_or_else(|| Status::invalid_argument("disk spec required"))?; .ok_or_else(|| Status::invalid_argument("disk spec required"))?;
let mut disk_spec = Self::proto_disk_to_types(proto_disk); let mut disk_spec = Self::proto_disk_to_types(proto_disk);
Self::validate_disk_reference(&disk_spec)?;
if vm.spec.disks.iter().any(|disk| disk.id == disk_spec.id) { if vm.spec.disks.iter().any(|disk| disk.id == disk_spec.id) {
return Err(Status::already_exists("disk already attached")); return Err(Status::already_exists("disk already attached"));
} }
@ -3858,7 +3986,7 @@ impl VmService for VmServiceImpl {
.ensure_vm_loaded(&req.org_id, &req.project_id, &req.vm_id) .ensure_vm_loaded(&req.org_id, &req.project_id, &req.vm_id)
.await .await
.ok_or_else(|| Status::not_found("VM not found"))?; .ok_or_else(|| Status::not_found("VM not found"))?;
let poll_interval = Self::watch_poll_interval(); let poll_interval = self.watch_poll_interval();
let store = Arc::clone(&self.store); let store = Arc::clone(&self.store);
let org_id = req.org_id.clone(); let org_id = req.org_id.clone();
let project_id = req.project_id.clone(); let project_id = req.project_id.clone();
@ -3948,6 +4076,9 @@ impl VolumeService for VmServiceImpl {
"size_gib must be greater than zero", "size_gib must be greater than zero",
)); ));
} }
if !req.image_id.trim().is_empty() {
Self::require_uuid(&req.image_id, "image_id")?;
}
let driver = Self::map_volume_driver( let driver = Self::map_volume_driver(
ProtoVolumeDriverKind::try_from(req.driver).unwrap_or(ProtoVolumeDriverKind::Managed), ProtoVolumeDriverKind::try_from(req.driver).unwrap_or(ProtoVolumeDriverKind::Managed),
@ -4206,6 +4337,9 @@ impl ImageService for VmServiceImpl {
if req.source_url.trim().is_empty() { if req.source_url.trim().is_empty() {
return Err(Status::invalid_argument("source_url is required")); return Err(Status::invalid_argument("source_url is required"));
} }
if !req.source_url.starts_with("https://") {
return Err(Status::invalid_argument("source_url must use https://"));
}
let Some(store) = self.artifact_store.as_ref() else { let Some(store) = self.artifact_store.as_ref() else {
return Err(Status::failed_precondition( return Err(Status::failed_precondition(
"LightningStor artifact backing is required for image imports", "LightningStor artifact backing is required for image imports",
@ -4245,20 +4379,7 @@ impl ImageService for VmServiceImpl {
.await .await
{ {
Ok(imported) => { Ok(imported) => {
image.status = ImageStatus::Available; Self::apply_imported_image_metadata(&mut image, &imported, source_format);
image.format = imported.format;
image.size_bytes = imported.size_bytes;
image.checksum = imported.checksum;
image.updated_at = Self::now_epoch();
image
.metadata
.insert("source_url".to_string(), req.source_url.clone());
if source_format != image.format {
image.metadata.insert(
"source_format".to_string(),
format!("{source_format:?}").to_lowercase(),
);
}
self.images.insert(key, image.clone()); self.images.insert(key, image.clone());
self.persist_image(&image).await; self.persist_image(&image).await;
Ok(Response::new(Self::types_image_to_proto(&image))) Ok(Response::new(Self::types_image_to_proto(&image)))
@ -4276,6 +4397,332 @@ impl ImageService for VmServiceImpl {
} }
} }
async fn begin_image_upload(
&self,
request: Request<BeginImageUploadRequest>,
) -> Result<Response<BeginImageUploadResponse>, Status> {
let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
self.auth
.authorize(
&tenant,
ACTION_IMAGE_CREATE,
&resource_for_tenant("image", "*", &org_id, &project_id),
)
.await?;
let mut req = request.into_inner();
if req.name.trim().is_empty() {
return Err(Status::invalid_argument("name is required"));
}
let Some(store) = self.artifact_store.as_ref() else {
return Err(Status::failed_precondition(
"LightningStor artifact backing is required for image uploads",
));
};
let source_format = Self::map_image_format(
ProtoImageFormat::try_from(req.format).unwrap_or(ProtoImageFormat::Qcow2),
);
let mut image = Image::new(req.name, &org_id, &project_id);
image.visibility = Self::map_visibility(
ProtoVisibility::try_from(req.visibility).unwrap_or(ProtoVisibility::Private),
);
image.os_type = Self::map_os_type(
ProtoOsType::try_from(req.os_type).unwrap_or(ProtoOsType::Unspecified),
);
image.os_version = req.os_version;
image.architecture = Self::map_architecture(req.architecture);
image.min_disk_gib = req.min_disk_gib;
image.min_memory_mib = req.min_memory_mib;
image.metadata = std::mem::take(&mut req.metadata);
image.status = ImageStatus::Uploading;
let key = TenantKey::new(&org_id, &project_id, &image.id);
self.images.insert(key.clone(), image.clone());
self.persist_image(&image).await;
let begin_result = store
.begin_image_upload(&org_id, &project_id, &image.id)
.await;
match begin_result {
Ok((upload_id, staging_key)) => {
let now = Self::now_epoch();
let session = ImageUploadSession {
session_id: Uuid::new_v4().to_string(),
org_id: org_id.clone(),
project_id: project_id.clone(),
image_id: image.id.clone(),
upload_id,
staging_key,
source_format,
parts: Vec::new(),
committed_size_bytes: 0,
created_at: now,
updated_at: now,
};
if let Err(error) = self.store.save_image_upload_session(&session).await {
let _ = store
.abort_image_upload(
&org_id,
&project_id,
&session.staging_key,
&session.upload_id,
)
.await;
image.status = ImageStatus::Error;
image.updated_at = Self::now_epoch();
image.metadata.insert(
"last_error".to_string(),
format!("failed to persist image upload session: {error}"),
);
self.images.insert(key, image.clone());
self.persist_image(&image).await;
return Err(Status::internal(format!(
"failed to persist image upload session: {error}"
)));
}
Ok(Response::new(BeginImageUploadResponse {
image: Some(Self::types_image_to_proto(&image)),
upload_session_id: session.session_id,
minimum_part_size_bytes: store.minimum_upload_part_size(),
}))
}
Err(error) => {
image.status = ImageStatus::Error;
image.updated_at = Self::now_epoch();
image
.metadata
.insert("last_error".to_string(), error.message().to_string());
self.images.insert(key, image.clone());
self.persist_image(&image).await;
Err(error)
}
}
}
async fn upload_image_part(
&self,
request: Request<UploadImagePartRequest>,
) -> Result<Response<UploadImagePartResponse>, Status> {
let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
Self::require_uuid(&req.upload_session_id, "upload_session_id")?;
self.auth
.authorize(
&tenant,
ACTION_IMAGE_CREATE,
&resource_for_tenant("image", req.image_id.clone(), &org_id, &project_id),
)
.await?;
let Some(store) = self.artifact_store.as_ref() else {
return Err(Status::failed_precondition(
"LightningStor artifact backing is required for image uploads",
));
};
let Some(mut session) = self
.store
.load_image_upload_session(&org_id, &project_id, &req.image_id, &req.upload_session_id)
.await
.map_err(|error| {
Status::internal(format!("failed to load image upload session: {error}"))
})?
else {
return Err(Status::not_found("image upload session not found"));
};
let Some(image) = self
.ensure_image_loaded(&org_id, &project_id, &req.image_id)
.await
else {
return Err(Status::not_found("image not found"));
};
if image.status != ImageStatus::Uploading {
return Err(Status::failed_precondition(
"image is not accepting uploaded parts",
));
}
let part = store
.upload_image_part(
&org_id,
&project_id,
&session.staging_key,
&session.upload_id,
req.part_number,
req.body,
)
.await?;
session
.parts
.retain(|existing| existing.part_number != part.part_number);
session.parts.push(part);
session.parts.sort_by_key(|part| part.part_number);
session.committed_size_bytes = session.parts.iter().map(|part| part.size_bytes).sum();
session.updated_at = Self::now_epoch();
self.store
.save_image_upload_session(&session)
.await
.map_err(|error| {
Status::internal(format!("failed to persist image upload session: {error}"))
})?;
Ok(Response::new(UploadImagePartResponse {
part_number: req.part_number,
committed_size_bytes: session.committed_size_bytes,
}))
}
async fn complete_image_upload(
&self,
request: Request<CompleteImageUploadRequest>,
) -> Result<Response<ProtoImage>, Status> {
let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
Self::require_uuid(&req.upload_session_id, "upload_session_id")?;
self.auth
.authorize(
&tenant,
ACTION_IMAGE_CREATE,
&resource_for_tenant("image", req.image_id.clone(), &org_id, &project_id),
)
.await?;
let Some(store) = self.artifact_store.as_ref() else {
return Err(Status::failed_precondition(
"LightningStor artifact backing is required for image uploads",
));
};
let key = TenantKey::new(&org_id, &project_id, &req.image_id);
let Some(mut image) = self
.ensure_image_loaded(&org_id, &project_id, &req.image_id)
.await
else {
return Err(Status::not_found("image not found"));
};
let Some(session) = self
.store
.load_image_upload_session(&org_id, &project_id, &req.image_id, &req.upload_session_id)
.await
.map_err(|error| {
Status::internal(format!("failed to load image upload session: {error}"))
})?
else {
return Err(Status::not_found("image upload session not found"));
};
match store
.complete_image_upload(
&org_id,
&project_id,
&req.image_id,
&session.staging_key,
&session.upload_id,
&session.parts,
session.source_format,
)
.await
{
Ok(imported) => {
Self::apply_imported_image_metadata(&mut image, &imported, session.source_format);
self.images.insert(key, image.clone());
self.persist_image(&image).await;
let _ = self
.store
.delete_image_upload_session(
&org_id,
&project_id,
&req.image_id,
&req.upload_session_id,
)
.await;
Ok(Response::new(Self::types_image_to_proto(&image)))
}
Err(error) => {
image.status = ImageStatus::Error;
image.updated_at = Self::now_epoch();
image
.metadata
.insert("last_error".to_string(), error.message().to_string());
self.images.insert(key, image.clone());
self.persist_image(&image).await;
Err(error)
}
}
}
async fn abort_image_upload(
&self,
request: Request<AbortImageUploadRequest>,
) -> Result<Response<Empty>, Status> {
let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
Self::require_uuid(&req.upload_session_id, "upload_session_id")?;
self.auth
.authorize(
&tenant,
ACTION_IMAGE_CREATE,
&resource_for_tenant("image", req.image_id.clone(), &org_id, &project_id),
)
.await?;
let Some(store) = self.artifact_store.as_ref() else {
return Err(Status::failed_precondition(
"LightningStor artifact backing is required for image uploads",
));
};
let Some(session) = self
.store
.load_image_upload_session(&org_id, &project_id, &req.image_id, &req.upload_session_id)
.await
.map_err(|error| {
Status::internal(format!("failed to load image upload session: {error}"))
})?
else {
return Err(Status::not_found("image upload session not found"));
};
store
.abort_image_upload(
&org_id,
&project_id,
&session.staging_key,
&session.upload_id,
)
.await?;
self.store
.delete_image_upload_session(
&org_id,
&project_id,
&req.image_id,
&req.upload_session_id,
)
.await
.map_err(|error| {
Status::internal(format!("failed to delete image upload session: {error}"))
})?;
if let Some(mut image) = self
.ensure_image_loaded(&org_id, &project_id, &req.image_id)
.await
{
let key = TenantKey::new(&org_id, &project_id, &req.image_id);
image.status = ImageStatus::Error;
image.updated_at = Self::now_epoch();
image
.metadata
.insert("last_error".to_string(), "upload aborted".to_string());
self.images.insert(key, image.clone());
self.persist_image(&image).await;
}
Ok(Response::new(Empty {}))
}
async fn get_image( async fn get_image(
&self, &self,
request: Request<GetImageRequest>, request: Request<GetImageRequest>,
@ -4283,6 +4730,7 @@ impl ImageService for VmServiceImpl {
let tenant = get_tenant_context(&request)?; let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?; let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner(); let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
self.auth self.auth
.authorize( .authorize(
&tenant, &tenant,
@ -4337,6 +4785,7 @@ impl ImageService for VmServiceImpl {
let tenant = get_tenant_context(&request)?; let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?; let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner(); let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
self.auth self.auth
.authorize( .authorize(
&tenant, &tenant,
@ -4378,6 +4827,7 @@ impl ImageService for VmServiceImpl {
let tenant = get_tenant_context(&request)?; let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?; let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner(); let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
self.auth self.auth
.authorize( .authorize(
&tenant, &tenant,
@ -4396,6 +4846,31 @@ impl ImageService for VmServiceImpl {
} }
if let Some(store) = self.artifact_store.as_ref() { if let Some(store) = self.artifact_store.as_ref() {
if let Ok(sessions) = self
.store
.list_image_upload_sessions(&org_id, &project_id, &req.image_id)
.await
{
for session in sessions {
let _ = store
.abort_image_upload(
&org_id,
&project_id,
&session.staging_key,
&session.upload_id,
)
.await;
let _ = self
.store
.delete_image_upload_session(
&org_id,
&project_id,
&req.image_id,
&session.session_id,
)
.await;
}
}
store store
.delete_image(&org_id, &project_id, &req.image_id) .delete_image(&org_id, &project_id, &req.image_id)
.await?; .await?;
@ -4540,7 +5015,7 @@ impl NodeService for VmServiceImpl {
.hypervisors .hypervisors
.iter() .iter()
.filter_map(|h| ProtoHypervisorType::try_from(*h).ok()) .filter_map(|h| ProtoHypervisorType::try_from(*h).ok())
.map(Self::map_hv) .map(|hypervisor| self.map_hv(hypervisor))
.collect(); .collect();
} }
if !req.supported_volume_drivers.is_empty() { if !req.supported_volume_drivers.is_empty() {

View file

@ -1,4 +1,5 @@
use crate::artifact_store::ArtifactStore; use crate::artifact_store::ArtifactStore;
use crate::config::{CephConfig, CoronaFsConfig, VolumeRuntimeConfig};
use crate::storage::VmStore; use crate::storage::VmStore;
use plasmavmc_types::{ use plasmavmc_types::{
AttachedDisk, DiskAttachment, DiskSource, DiskSpec, VirtualMachine, Volume, VolumeBacking, AttachedDisk, DiskAttachment, DiskSource, DiskSpec, VirtualMachine, Volume, VolumeBacking,
@ -18,6 +19,7 @@ const AUTO_DELETE_VOLUME_SOURCE_METADATA_KEY: &str = "plasmavmc.auto_delete_sour
const CORONAFS_IMAGE_SOURCE_ID_METADATA_KEY: &str = "plasmavmc.coronafs_image_source_id"; const CORONAFS_IMAGE_SOURCE_ID_METADATA_KEY: &str = "plasmavmc.coronafs_image_source_id";
const CORONAFS_IMAGE_SEED_PENDING_METADATA_KEY: &str = "plasmavmc.coronafs_image_seed_pending"; const CORONAFS_IMAGE_SEED_PENDING_METADATA_KEY: &str = "plasmavmc.coronafs_image_seed_pending";
const VOLUME_METADATA_CAS_RETRIES: usize = 16; const VOLUME_METADATA_CAS_RETRIES: usize = 16;
const CEPH_IDENTIFIER_MAX_LEN: usize = 128;
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
struct CephClusterConfig { struct CephClusterConfig {
@ -116,6 +118,7 @@ pub struct VolumeManager {
artifact_store: Option<Arc<ArtifactStore>>, artifact_store: Option<Arc<ArtifactStore>>,
managed_root: PathBuf, managed_root: PathBuf,
supported_storage_classes: Vec<String>, supported_storage_classes: Vec<String>,
qemu_img_path: PathBuf,
ceph_cluster: Option<CephClusterConfig>, ceph_cluster: Option<CephClusterConfig>,
coronafs_controller: Option<CoronaFsClient>, coronafs_controller: Option<CoronaFsClient>,
coronafs_node: Option<CoronaFsClient>, coronafs_node: Option<CoronaFsClient>,
@ -124,31 +127,43 @@ pub struct VolumeManager {
} }
impl VolumeManager { impl VolumeManager {
pub fn new(store: Arc<dyn VmStore>, artifact_store: Option<Arc<ArtifactStore>>) -> Self { pub fn new_with_config(
let managed_root = std::env::var("PLASMAVMC_MANAGED_VOLUME_ROOT") store: Arc<dyn VmStore>,
.map(PathBuf::from) artifact_store: Option<Arc<ArtifactStore>>,
.unwrap_or_else(|_| PathBuf::from("/var/lib/plasmavmc/managed-volumes")); config: &VolumeRuntimeConfig,
let ceph_cluster = std::env::var("PLASMAVMC_CEPH_MONITORS") local_node_id: Option<String>,
.ok() ) -> Result<Self, Box<dyn std::error::Error>> {
.filter(|value| !value.trim().is_empty()) let managed_root = config.managed_volume_root.clone();
.map(|monitors| CephClusterConfig { let ceph_cluster = ceph_cluster_from_config(&config.ceph);
cluster_id: std::env::var("PLASMAVMC_CEPH_CLUSTER_ID") let (coronafs_controller, coronafs_node) =
.unwrap_or_else(|_| "default".to_string()), resolve_coronafs_clients_with_config(&config.coronafs);
monitors: monitors let coronafs_node_local_attach = config.coronafs.node_local_attach;
.split(',') let qemu_img_path = resolve_binary_path(config.qemu_img_path.as_deref(), "qemu-img")?;
.map(str::trim)
.filter(|item| !item.is_empty())
.map(ToOwned::to_owned)
.collect(),
user: std::env::var("PLASMAVMC_CEPH_USER").unwrap_or_else(|_| "admin".to_string()),
secret: std::env::var("PLASMAVMC_CEPH_SECRET").ok(),
});
let (coronafs_controller, coronafs_node) = resolve_coronafs_clients();
let coronafs_node_local_attach = coronafs_node_local_attach_enabled();
let local_node_id = std::env::var("PLASMAVMC_NODE_ID")
.ok()
.filter(|value| !value.trim().is_empty());
Ok(Self::build(
store,
artifact_store,
managed_root,
qemu_img_path,
ceph_cluster,
coronafs_controller,
coronafs_node,
coronafs_node_local_attach,
local_node_id,
))
}
fn build(
store: Arc<dyn VmStore>,
artifact_store: Option<Arc<ArtifactStore>>,
managed_root: PathBuf,
qemu_img_path: PathBuf,
ceph_cluster: Option<CephClusterConfig>,
coronafs_controller: Option<CoronaFsClient>,
coronafs_node: Option<CoronaFsClient>,
coronafs_node_local_attach: bool,
local_node_id: Option<String>,
) -> Self {
Self { Self {
store, store,
artifact_store, artifact_store,
@ -161,6 +176,7 @@ impl VolumeManager {
classes.push("ceph-rbd".to_string()); classes.push("ceph-rbd".to_string());
classes classes
}, },
qemu_img_path,
ceph_cluster, ceph_cluster,
coronafs_controller, coronafs_controller,
coronafs_node, coronafs_node,
@ -320,6 +336,7 @@ impl VolumeManager {
Some(export) => export, Some(export) => export,
None => controller.ensure_export_read_only(volume_id).await?, None => controller.ensure_export_read_only(volume_id).await?,
}; };
validate_coronafs_export(&export)?;
node_client node_client
.materialize_from_export( .materialize_from_export(
volume_id, volume_id,
@ -377,6 +394,9 @@ impl VolumeManager {
{ {
return Ok(existing); return Ok(existing);
} }
if let Some(image_id) = image_id {
validate_uuid(image_id, "image_id")?;
}
let path = self.managed_volume_path(volume_id); let path = self.managed_volume_path(volume_id);
let mut metadata = metadata; let mut metadata = metadata;
@ -503,6 +523,9 @@ impl VolumeManager {
"Ceph RBD support is not configured on this node", "Ceph RBD support is not configured on this node",
)); ));
} }
validate_ceph_identifier("ceph_rbd.cluster_id", cluster_id)?;
validate_ceph_identifier("ceph_rbd.pool", pool)?;
validate_ceph_identifier("ceph_rbd.image", image)?;
let volume_id = Uuid::new_v4().to_string(); let volume_id = Uuid::new_v4().to_string();
let mut volume = Volume::new(volume_id, name.to_string(), org_id, project_id, size_gib); let mut volume = Volume::new(volume_id, name.to_string(), org_id, project_id, size_gib);
volume.driver = VolumeDriverKind::CephRbd; volume.driver = VolumeDriverKind::CephRbd;
@ -943,6 +966,7 @@ impl VolumeManager {
let Some(local_volume) = node_client.get_volume_optional(volume_id).await? else { let Some(local_volume) = node_client.get_volume_optional(volume_id).await? else {
return Ok(()); return Ok(());
}; };
validate_coronafs_volume_response(&local_volume)?;
if local_volume.export.is_some() { if local_volume.export.is_some() {
node_client.release_export(volume_id).await?; node_client.release_export(volume_id).await?;
} }
@ -975,6 +999,7 @@ impl VolumeManager {
Some(export) => export, Some(export) => export,
None => controller.ensure_export(volume_id).await?, None => controller.ensure_export(volume_id).await?,
}; };
validate_coronafs_export(&export)?;
tracing::info!( tracing::info!(
volume_id, volume_id,
node_endpoint = %node_client.endpoint, node_endpoint = %node_client.endpoint,
@ -983,7 +1008,7 @@ impl VolumeManager {
export_uri = %export.uri, export_uri = %export.uri,
"Syncing node-local CoronaFS volume back to controller" "Syncing node-local CoronaFS volume back to controller"
); );
sync_local_coronafs_volume_to_export(&local_volume, &export.uri).await sync_local_coronafs_volume_to_export(&self.qemu_img_path, &local_volume, &export.uri).await
} }
async fn materialize_pending_coronafs_image_seed_on_node( async fn materialize_pending_coronafs_image_seed_on_node(
@ -1185,6 +1210,7 @@ impl VolumeManager {
if self.coronafs_attachment_backend().is_some() { if self.coronafs_attachment_backend().is_some() {
let (coronafs_volume, coronafs) = let (coronafs_volume, coronafs) =
self.load_coronafs_volume_for_attachment(volume).await?; self.load_coronafs_volume_for_attachment(volume).await?;
validate_coronafs_volume_response(&coronafs_volume)?;
if coronafs.supports_local_backing_file().await if coronafs.supports_local_backing_file().await
&& !should_prefer_coronafs_export_attachment(&coronafs_volume) && !should_prefer_coronafs_export_attachment(&coronafs_volume)
&& coronafs_local_target_ready(&coronafs_volume.path).await && coronafs_local_target_ready(&coronafs_volume.path).await
@ -1226,6 +1252,9 @@ impl VolumeManager {
cluster_id cluster_id
))); )));
} }
validate_ceph_identifier("ceph cluster_id", cluster_id)?;
validate_ceph_identifier("ceph pool", pool)?;
validate_ceph_identifier("ceph image", image)?;
DiskAttachment::CephRbd { DiskAttachment::CephRbd {
pool: pool.clone(), pool: pool.clone(),
image: image.clone(), image: image.clone(),
@ -1266,7 +1295,7 @@ impl VolumeManager {
.await .await
.map_err(|e| Status::internal(format!("failed to create volume dir: {e}")))?; .map_err(|e| Status::internal(format!("failed to create volume dir: {e}")))?;
} }
let status = Command::new("qemu-img") let status = Command::new(&self.qemu_img_path)
.args([ .args([
"convert", "convert",
"-O", "-O",
@ -1307,7 +1336,7 @@ impl VolumeManager {
.materialize_image_cache(org_id, project_id, image_id) .materialize_image_cache(org_id, project_id, image_id)
.await?; .await?;
let requested_size = gib_to_bytes(size_gib); let requested_size = gib_to_bytes(size_gib);
let image_info = inspect_qemu_image(&image_path).await?; let image_info = inspect_qemu_image(&self.qemu_img_path, &image_path).await?;
if requested_size < image_info.virtual_size { if requested_size < image_info.virtual_size {
return Err(Status::failed_precondition(format!( return Err(Status::failed_precondition(format!(
"requested volume {} GiB is smaller than image virtual size {} bytes", "requested volume {} GiB is smaller than image virtual size {} bytes",
@ -1413,6 +1442,7 @@ impl VolumeManager {
&raw_image_path, &raw_image_path,
Path::new(&volume.path), Path::new(&volume.path),
requested_size, requested_size,
&self.qemu_img_path,
) )
.await?; .await?;
return Ok(CoronaFsProvisionOutcome { return Ok(CoronaFsProvisionOutcome {
@ -1431,7 +1461,7 @@ impl VolumeManager {
export_uri = %export.uri, export_uri = %export.uri,
"Populating CoronaFS-backed VM volume over NBD from image cache" "Populating CoronaFS-backed VM volume over NBD from image cache"
); );
let status = Command::new("qemu-img") let status = Command::new(&self.qemu_img_path)
.args([ .args([
"convert", "convert",
"-t", "-t",
@ -1481,7 +1511,7 @@ impl VolumeManager {
.await .await
.map_err(|e| Status::internal(format!("failed to create volume dir: {e}")))?; .map_err(|e| Status::internal(format!("failed to create volume dir: {e}")))?;
} }
let status = Command::new("qemu-img") let status = Command::new(&self.qemu_img_path)
.args([ .args([
"create", "create",
"-f", "-f",
@ -1508,7 +1538,7 @@ impl VolumeManager {
format: VolumeFormat, format: VolumeFormat,
size_gib: u64, size_gib: u64,
) -> Result<(), Status> { ) -> Result<(), Status> {
let status = Command::new("qemu-img") let status = Command::new(&self.qemu_img_path)
.args([ .args([
"resize", "resize",
"-f", "-f",
@ -1542,8 +1572,8 @@ impl VolumeManager {
} }
} }
async fn inspect_qemu_image(path: &Path) -> Result<QemuImageInfo, Status> { async fn inspect_qemu_image(qemu_img_path: &Path, path: &Path) -> Result<QemuImageInfo, Status> {
let output = Command::new("qemu-img") let output = Command::new(qemu_img_path)
.args(["info", "--output", "json", path.to_string_lossy().as_ref()]) .args(["info", "--output", "json", path.to_string_lossy().as_ref()])
.output() .output()
.await .await
@ -1585,6 +1615,7 @@ async fn clone_local_raw_into_coronafs_volume(
source: &Path, source: &Path,
destination: &Path, destination: &Path,
requested_size: u64, requested_size: u64,
_qemu_img_path: &Path,
) -> Result<(), Status> { ) -> Result<(), Status> {
let temp_path = destination.with_extension("clone.tmp"); let temp_path = destination.with_extension("clone.tmp");
if let Some(parent) = temp_path.parent() { if let Some(parent) = temp_path.parent() {
@ -1595,33 +1626,13 @@ async fn clone_local_raw_into_coronafs_volume(
if tokio::fs::try_exists(&temp_path).await.unwrap_or(false) { if tokio::fs::try_exists(&temp_path).await.unwrap_or(false) {
let _ = tokio::fs::remove_file(&temp_path).await; let _ = tokio::fs::remove_file(&temp_path).await;
} }
tokio::fs::copy(source, &temp_path).await.map_err(|e| {
let copy_output = Command::new("cp") Status::internal(format!(
.args([ "failed to clone raw image {} into {}: {e}",
"--reflink=auto",
"--sparse=always",
source.to_string_lossy().as_ref(),
temp_path.to_string_lossy().as_ref(),
])
.output()
.await
.map_err(|e| Status::internal(format!("failed to spawn raw clone copy: {e}")))?;
if !copy_output.status.success() {
let stderr = String::from_utf8_lossy(&copy_output.stderr)
.trim()
.to_string();
return Err(Status::internal(format!(
"failed to clone raw image {} into {} with status {}{}",
source.display(), source.display(),
temp_path.display(), temp_path.display()
copy_output.status, ))
if stderr.is_empty() { })?;
String::new()
} else {
format!(": {stderr}")
}
)));
}
let file = tokio::fs::OpenOptions::new() let file = tokio::fs::OpenOptions::new()
.write(true) .write(true)
@ -1656,11 +1667,12 @@ async fn clone_local_raw_into_coronafs_volume(
} }
async fn sync_local_coronafs_volume_to_export( async fn sync_local_coronafs_volume_to_export(
qemu_img_path: &Path,
local_volume: &CoronaFsVolumeResponse, local_volume: &CoronaFsVolumeResponse,
export_uri: &str, export_uri: &str,
) -> Result<(), Status> { ) -> Result<(), Status> {
let local_format = local_volume.format.unwrap_or(CoronaFsVolumeFormat::Raw); let local_format = local_volume.format.unwrap_or(CoronaFsVolumeFormat::Raw);
let status = Command::new("qemu-img") let status = Command::new(qemu_img_path)
.args([ .args([
"convert", "convert",
"-t", "-t",
@ -1773,17 +1785,121 @@ fn gib_to_bytes(size_gib: u64) -> u64 {
size_gib.saturating_mul(1024 * 1024 * 1024) size_gib.saturating_mul(1024 * 1024 * 1024)
} }
fn coronafs_node_local_attach_enabled() -> bool { fn validate_uuid(value: &str, field_name: &str) -> Result<(), Status> {
coronafs_node_local_attach_enabled_from_values( Uuid::parse_str(value)
std::env::var("PLASMAVMC_CORONAFS_NODE_LOCAL_ATTACH") .map(|_| ())
.ok() .map_err(|_| Status::invalid_argument(format!("{field_name} must be a UUID")))
.as_deref(),
std::env::var("PLASMAVMC_CORONAFS_ENABLE_EXPERIMENTAL_NODE_LOCAL_ATTACH")
.ok()
.as_deref(),
)
} }
fn validate_ceph_identifier(field_name: &str, value: &str) -> Result<(), Status> {
let trimmed = value.trim();
if trimmed.is_empty() {
return Err(Status::invalid_argument(format!(
"{field_name} is required"
)));
}
if trimmed.len() > CEPH_IDENTIFIER_MAX_LEN {
return Err(Status::invalid_argument(format!(
"{field_name} exceeds {CEPH_IDENTIFIER_MAX_LEN} characters"
)));
}
let mut chars = trimmed.chars();
let Some(first) = chars.next() else {
return Err(Status::invalid_argument(format!(
"{field_name} is required"
)));
};
if !first.is_ascii_alphanumeric() {
return Err(Status::invalid_argument(format!(
"{field_name} must start with an ASCII alphanumeric character"
)));
}
if chars.any(|ch| !(ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-'))) {
return Err(Status::invalid_argument(format!(
"{field_name} contains unsupported characters"
)));
}
Ok(())
}
fn validate_absolute_path(path: &str, field_name: &str) -> Result<(), Status> {
let candidate = Path::new(path);
if !candidate.is_absolute() {
return Err(Status::failed_precondition(format!(
"{field_name} must be an absolute path"
)));
}
Ok(())
}
fn validate_nbd_uri(uri: &str, field_name: &str) -> Result<(), Status> {
let parsed = reqwest::Url::parse(uri)
.map_err(|e| Status::failed_precondition(format!("{field_name} is invalid: {e}")))?;
if parsed.scheme() != "nbd" {
return Err(Status::failed_precondition(format!(
"{field_name} must use nbd://"
)));
}
if !parsed.username().is_empty() || parsed.password().is_some() {
return Err(Status::failed_precondition(format!(
"{field_name} must not contain embedded credentials"
)));
}
if parsed.host_str().is_none() || parsed.port().is_none() {
return Err(Status::failed_precondition(format!(
"{field_name} must include host and port"
)));
}
Ok(())
}
fn validate_coronafs_export(export: &CoronaFsExport) -> Result<(), Status> {
validate_nbd_uri(&export.uri, "coronafs export uri")
}
fn validate_coronafs_volume_response(volume: &CoronaFsVolumeResponse) -> Result<(), Status> {
validate_absolute_path(&volume.path, "coronafs volume path")?;
if let Some(export) = &volume.export {
validate_coronafs_export(export)?;
}
if let Some(source) = &volume.materialized_from {
if source.starts_with("nbd://") {
validate_nbd_uri(source, "coronafs materialized_from")?;
}
}
Ok(())
}
fn resolve_binary_path(
configured_path: Option<&Path>,
binary_name: &str,
) -> Result<PathBuf, Box<dyn std::error::Error>> {
let candidate = match configured_path {
Some(path) => path.to_path_buf(),
None => std::env::var_os("PATH")
.into_iter()
.flat_map(|paths| std::env::split_paths(&paths).collect::<Vec<_>>())
.map(|entry| entry.join(binary_name))
.find(|candidate| candidate.exists())
.ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("failed to locate {binary_name} in PATH"),
)
})?,
};
let metadata = std::fs::metadata(&candidate)?;
if !metadata.is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("{} is not a regular file", candidate.display()),
)
.into());
}
Ok(candidate)
}
#[cfg(test)]
fn coronafs_node_local_attach_enabled_from_values( fn coronafs_node_local_attach_enabled_from_values(
stable_value: Option<&str>, stable_value: Option<&str>,
legacy_value: Option<&str>, legacy_value: Option<&str>,
@ -1794,6 +1910,23 @@ fn coronafs_node_local_attach_enabled_from_values(
.unwrap_or(false) .unwrap_or(false)
} }
fn ceph_cluster_from_config(config: &CephConfig) -> Option<CephClusterConfig> {
if config.monitors.is_empty() {
None
} else {
Some(CephClusterConfig {
cluster_id: config.cluster_id.clone(),
monitors: config.monitors.clone(),
user: config.user.clone(),
secret: config
.secret
.clone()
.or_else(|| std::env::var("PLASMAVMC_CEPH_SECRET").ok()),
})
}
}
#[cfg(test)]
fn parse_truthy(value: &str) -> bool { fn parse_truthy(value: &str) -> bool {
matches!( matches!(
value.trim().to_ascii_lowercase().as_str(), value.trim().to_ascii_lowercase().as_str(),
@ -1801,16 +1934,21 @@ fn parse_truthy(value: &str) -> bool {
) )
} }
fn resolve_coronafs_clients() -> (Option<CoronaFsClient>, Option<CoronaFsClient>) { fn resolve_coronafs_clients_with_config(
config: &CoronaFsConfig,
) -> (Option<CoronaFsClient>, Option<CoronaFsClient>) {
let (controller_endpoint, node_endpoint) = resolve_coronafs_endpoints( let (controller_endpoint, node_endpoint) = resolve_coronafs_endpoints(
std::env::var("PLASMAVMC_CORONAFS_CONTROLLER_ENDPOINT") config
.ok() .controller_endpoint
.clone()
.and_then(normalize_coronafs_endpoint), .and_then(normalize_coronafs_endpoint),
std::env::var("PLASMAVMC_CORONAFS_NODE_ENDPOINT") config
.ok() .node_endpoint
.clone()
.and_then(normalize_coronafs_endpoint), .and_then(normalize_coronafs_endpoint),
std::env::var("PLASMAVMC_CORONAFS_ENDPOINT") config
.ok() .endpoint
.clone()
.and_then(normalize_coronafs_endpoint), .and_then(normalize_coronafs_endpoint),
); );
( (
@ -1911,14 +2049,19 @@ impl CoronaFsClient {
backing_file: None, backing_file: None,
backing_format: None, backing_format: None,
}; };
self.try_request("create CoronaFS volume", |endpoint, http| { let volume = self
.try_request("create CoronaFS volume", |endpoint, http| {
http.put(format!("{endpoint}/v1/volumes/{volume_id}")) http.put(format!("{endpoint}/v1/volumes/{volume_id}"))
.json(&request) .json(&request)
}) })
.await? .await?
.json::<CoronaFsVolumeResponse>() .json::<CoronaFsVolumeResponse>()
.await .await
.map_err(|e| Status::internal(format!("failed to decode CoronaFS create response: {e}"))) .map_err(|e| {
Status::internal(format!("failed to decode CoronaFS create response: {e}"))
})?;
validate_coronafs_volume_response(&volume)?;
Ok(volume)
} }
async fn create_image_backed( async fn create_image_backed(
@ -1934,7 +2077,8 @@ impl CoronaFsClient {
backing_file: Some(backing_file.to_string_lossy().into_owned()), backing_file: Some(backing_file.to_string_lossy().into_owned()),
backing_format: Some(backing_format), backing_format: Some(backing_format),
}; };
self.try_request("create CoronaFS image-backed volume", |endpoint, http| { let volume = self
.try_request("create CoronaFS image-backed volume", |endpoint, http| {
http.put(format!("{endpoint}/v1/volumes/{volume_id}")) http.put(format!("{endpoint}/v1/volumes/{volume_id}"))
.json(&request) .json(&request)
}) })
@ -1945,17 +2089,24 @@ impl CoronaFsClient {
Status::internal(format!( Status::internal(format!(
"failed to decode CoronaFS image-backed create response: {e}" "failed to decode CoronaFS image-backed create response: {e}"
)) ))
}) })?;
validate_coronafs_volume_response(&volume)?;
Ok(volume)
} }
async fn get_volume(&self, volume_id: &str) -> Result<CoronaFsVolumeResponse, Status> { async fn get_volume(&self, volume_id: &str) -> Result<CoronaFsVolumeResponse, Status> {
self.try_request("inspect CoronaFS volume", |endpoint, http| { let volume = self
.try_request("inspect CoronaFS volume", |endpoint, http| {
http.get(format!("{endpoint}/v1/volumes/{volume_id}")) http.get(format!("{endpoint}/v1/volumes/{volume_id}"))
}) })
.await? .await?
.json::<CoronaFsVolumeResponse>() .json::<CoronaFsVolumeResponse>()
.await .await
.map_err(|e| Status::internal(format!("failed to decode CoronaFS inspect response: {e}"))) .map_err(|e| {
Status::internal(format!("failed to decode CoronaFS inspect response: {e}"))
})?;
validate_coronafs_volume_response(&volume)?;
Ok(volume)
} }
async fn get_volume_optional( async fn get_volume_optional(
@ -2001,9 +2152,11 @@ impl CoronaFsClient {
.map_err(|e| { .map_err(|e| {
Status::internal(format!("failed to decode CoronaFS export response: {e}")) Status::internal(format!("failed to decode CoronaFS export response: {e}"))
})?; })?;
response.export.ok_or_else(|| { let export = response.export.ok_or_else(|| {
Status::internal("CoronaFS export response did not include an export URI") Status::internal("CoronaFS export response did not include an export URI")
}) })?;
validate_coronafs_export(&export)?;
Ok(export)
} }
async fn materialize_from_export( async fn materialize_from_export(
@ -2014,13 +2167,15 @@ impl CoronaFsClient {
format: Option<CoronaFsVolumeFormat>, format: Option<CoronaFsVolumeFormat>,
lazy: bool, lazy: bool,
) -> Result<CoronaFsVolumeResponse, Status> { ) -> Result<CoronaFsVolumeResponse, Status> {
validate_nbd_uri(source_uri, "coronafs materialize source_uri")?;
let request = CoronaFsMaterializeRequest { let request = CoronaFsMaterializeRequest {
source_uri: source_uri.to_string(), source_uri: source_uri.to_string(),
size_bytes, size_bytes,
format, format,
lazy, lazy,
}; };
self.try_request("materialize CoronaFS volume", |endpoint, http| { let volume = self
.try_request("materialize CoronaFS volume", |endpoint, http| {
http.post(format!("{endpoint}/v1/volumes/{volume_id}/materialize")) http.post(format!("{endpoint}/v1/volumes/{volume_id}/materialize"))
.json(&request) .json(&request)
}) })
@ -2031,7 +2186,9 @@ impl CoronaFsClient {
Status::internal(format!( Status::internal(format!(
"failed to decode CoronaFS materialize response: {e}" "failed to decode CoronaFS materialize response: {e}"
)) ))
}) })?;
validate_coronafs_volume_response(&volume)?;
Ok(volume)
} }
async fn resize_volume(&self, volume_id: &str, size_bytes: u64) -> Result<(), Status> { async fn resize_volume(&self, volume_id: &str, size_bytes: u64) -> Result<(), Status> {
@ -2210,7 +2367,12 @@ mod tests {
.unwrap(); .unwrap();
tokio::fs::write(&source, b"raw-seed").await.unwrap(); tokio::fs::write(&source, b"raw-seed").await.unwrap();
clone_local_raw_into_coronafs_volume(&source, &destination, 4096) clone_local_raw_into_coronafs_volume(
&source,
&destination,
4096,
Path::new("/usr/bin/qemu-img"),
)
.await .await
.unwrap(); .unwrap();
@ -2398,4 +2560,27 @@ mod tests {
.remove(CORONAFS_IMAGE_SEED_PENDING_METADATA_KEY); .remove(CORONAFS_IMAGE_SEED_PENDING_METADATA_KEY);
assert!(!volume_has_pending_coronafs_image_seed(&volume)); assert!(!volume_has_pending_coronafs_image_seed(&volume));
} }
#[test]
fn ceph_identifier_validation_rejects_option_injection_characters() {
assert!(validate_ceph_identifier("pool", "pool-alpha").is_ok());
assert!(validate_ceph_identifier("pool", "pool,inject").is_err());
assert!(validate_ceph_identifier("image", "image:inject").is_err());
}
#[test]
fn coronafs_export_validation_requires_nbd_uri() {
let valid = CoronaFsExport {
uri: "nbd://10.0.0.1:11000".to_string(),
port: 11000,
pid: None,
};
let invalid = CoronaFsExport {
uri: "http://10.0.0.1:11000".to_string(),
..valid.clone()
};
assert!(validate_coronafs_export(&valid).is_ok());
assert!(validate_coronafs_export(&invalid).is_err());
}
} }

View file

@ -64,13 +64,8 @@ pub struct WatcherConfig {
impl Default for WatcherConfig { impl Default for WatcherConfig {
fn default() -> Self { fn default() -> Self {
let poll_interval_ms = std::env::var("PLASMAVMC_STATE_WATCHER_POLL_INTERVAL_MS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(1000)
.max(100);
Self { Self {
poll_interval: Duration::from_millis(poll_interval_ms), poll_interval: Duration::from_millis(1000),
buffer_size: 256, buffer_size: 256,
} }
} }

View file

@ -4,8 +4,7 @@ use serde::{Deserialize, Serialize};
use std::path::PathBuf; use std::path::PathBuf;
/// FireCracker backend configuration /// FireCracker backend configuration
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[derive(Default)]
pub struct FireCrackerConfig { pub struct FireCrackerConfig {
/// Path to the Firecracker binary /// Path to the Firecracker binary
pub firecracker_path: Option<PathBuf>, pub firecracker_path: Option<PathBuf>,
@ -27,11 +26,11 @@ pub struct FireCrackerConfig {
pub use_jailer: Option<bool>, pub use_jailer: Option<bool>,
} }
/// KVM backend configuration /// KVM backend configuration
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[derive(Default)]
pub struct KvmConfig { pub struct KvmConfig {
// Add KVM specific configuration fields here if any /// Path to the QEMU binary
pub qemu_path: Option<PathBuf>,
/// Runtime directory used for QMP sockets and console logs
pub runtime_dir: Option<PathBuf>,
} }

View file

@ -44,6 +44,10 @@ service VmService {
service ImageService { service ImageService {
rpc CreateImage(CreateImageRequest) returns (Image); rpc CreateImage(CreateImageRequest) returns (Image);
rpc BeginImageUpload(BeginImageUploadRequest) returns (BeginImageUploadResponse);
rpc UploadImagePart(UploadImagePartRequest) returns (UploadImagePartResponse);
rpc CompleteImageUpload(CompleteImageUploadRequest) returns (Image);
rpc AbortImageUpload(AbortImageUploadRequest) returns (Empty);
rpc GetImage(GetImageRequest) returns (Image); rpc GetImage(GetImageRequest) returns (Image);
rpc ListImages(ListImagesRequest) returns (ListImagesResponse); rpc ListImages(ListImagesRequest) returns (ListImagesResponse);
rpc UpdateImage(UpdateImageRequest) returns (Image); rpc UpdateImage(UpdateImageRequest) returns (Image);
@ -475,6 +479,50 @@ message CreateImageRequest {
string source_url = 11; string source_url = 11;
} }
message BeginImageUploadRequest {
string name = 1;
string org_id = 2;
Visibility visibility = 3;
ImageFormat format = 4;
OsType os_type = 5;
string os_version = 6;
Architecture architecture = 7;
uint32 min_disk_gib = 8;
uint32 min_memory_mib = 9;
map<string, string> metadata = 10;
}
message BeginImageUploadResponse {
Image image = 1;
string upload_session_id = 2;
uint32 minimum_part_size_bytes = 3;
}
message UploadImagePartRequest {
string org_id = 1;
string image_id = 2;
string upload_session_id = 3;
uint32 part_number = 4;
bytes body = 5;
}
message UploadImagePartResponse {
uint32 part_number = 1;
uint64 committed_size_bytes = 2;
}
message CompleteImageUploadRequest {
string org_id = 1;
string image_id = 2;
string upload_session_id = 3;
}
message AbortImageUploadRequest {
string org_id = 1;
string image_id = 2;
string upload_session_id = 3;
}
message GetImageRequest { message GetImageRequest {
string org_id = 1; string org_id = 1;
string image_id = 2; string image_id = 2;

View file

@ -20,6 +20,36 @@ pub struct TlsConfig {
pub require_client_cert: bool, pub require_client_cert: bool,
} }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum OvnMode {
Mock,
Real,
}
impl Default for OvnMode {
fn default() -> Self {
Self::Mock
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OvnConfig {
#[serde(default)]
pub mode: OvnMode,
#[serde(default = "default_ovn_nb_addr")]
pub nb_addr: String,
}
impl Default for OvnConfig {
fn default() -> Self {
Self {
mode: OvnMode::default(),
nb_addr: default_ovn_nb_addr(),
}
}
}
/// Metadata storage backend /// Metadata storage backend
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")] #[serde(rename_all = "lowercase")]
@ -74,6 +104,10 @@ pub struct ServerConfig {
/// Authentication configuration /// Authentication configuration
#[serde(default)] #[serde(default)]
pub auth: AuthConfig, pub auth: AuthConfig,
/// OVN integration settings
#[serde(default)]
pub ovn: OvnConfig,
} }
/// Authentication configuration /// Authentication configuration
@ -100,6 +134,10 @@ fn default_http_addr() -> SocketAddr {
"127.0.0.1:8087".parse().unwrap() "127.0.0.1:8087".parse().unwrap()
} }
fn default_ovn_nb_addr() -> String {
"tcp:127.0.0.1:6641".to_string()
}
impl Default for ServerConfig { impl Default for ServerConfig {
fn default() -> Self { fn default() -> Self {
Self { Self {
@ -113,6 +151,7 @@ impl Default for ServerConfig {
log_level: "info".to_string(), log_level: "info".to_string(),
tls: None, tls: None,
auth: AuthConfig::default(), auth: AuthConfig::default(),
ovn: OvnConfig::default(),
} }
} }
} }

View file

@ -11,9 +11,8 @@ use prismnet_api::{
subnet_service_server::SubnetServiceServer, vpc_service_server::VpcServiceServer, subnet_service_server::SubnetServiceServer, vpc_service_server::VpcServiceServer,
}; };
use prismnet_server::{ use prismnet_server::{
config::MetadataBackend, config::MetadataBackend, IpamServiceImpl, NetworkMetadataStore, OvnClient, PortServiceImpl,
IpamServiceImpl, NetworkMetadataStore, OvnClient, PortServiceImpl, SecurityGroupServiceImpl, SecurityGroupServiceImpl, ServerConfig, SubnetServiceImpl, VpcServiceImpl,
ServerConfig, SubnetServiceImpl, VpcServiceImpl,
}; };
use std::net::SocketAddr; use std::net::SocketAddr;
use std::path::PathBuf; use std::path::PathBuf;
@ -36,26 +35,6 @@ struct Args {
#[arg(long)] #[arg(long)]
grpc_addr: Option<String>, grpc_addr: Option<String>,
/// ChainFire endpoint for cluster coordination (optional)
#[arg(long)]
chainfire_endpoint: Option<String>,
/// FlareDB endpoint for metadata and tenant data storage
#[arg(long)]
flaredb_endpoint: Option<String>,
/// Metadata backend (flaredb, postgres, sqlite)
#[arg(long)]
metadata_backend: Option<String>,
/// SQL database URL for metadata (required for postgres/sqlite backend)
#[arg(long)]
metadata_database_url: Option<String>,
/// Run in single-node mode (required when metadata backend is SQLite)
#[arg(long)]
single_node: bool,
/// Log level (overrides config) /// Log level (overrides config)
#[arg(short, long)] #[arg(short, long)]
log_level: Option<String>, log_level: Option<String>,
@ -88,58 +67,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
if let Some(log_level) = args.log_level { if let Some(log_level) = args.log_level {
config.log_level = log_level; config.log_level = log_level;
} }
if let Some(chainfire_endpoint) = args.chainfire_endpoint {
config.chainfire_endpoint = Some(chainfire_endpoint);
}
if let Some(flaredb_endpoint) = args.flaredb_endpoint {
config.flaredb_endpoint = Some(flaredb_endpoint);
}
if let Some(metadata_backend) = args.metadata_backend {
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
}
if let Some(metadata_database_url) = args.metadata_database_url {
config.metadata_database_url = Some(metadata_database_url);
}
if args.single_node {
config.single_node = true;
}
if config.chainfire_endpoint.is_none() {
if let Ok(chainfire_endpoint) = std::env::var("PRISMNET_CHAINFIRE_ENDPOINT") {
let trimmed = chainfire_endpoint.trim();
if !trimmed.is_empty() {
config.chainfire_endpoint = Some(trimmed.to_string());
}
}
}
if config.flaredb_endpoint.is_none() {
if let Ok(flaredb_endpoint) = std::env::var("PRISMNET_FLAREDB_ENDPOINT") {
let trimmed = flaredb_endpoint.trim();
if !trimmed.is_empty() {
config.flaredb_endpoint = Some(trimmed.to_string());
}
}
}
if let Ok(metadata_backend) = std::env::var("PRISMNET_METADATA_BACKEND") {
let trimmed = metadata_backend.trim();
if !trimmed.is_empty() {
config.metadata_backend = parse_metadata_backend(trimmed)?;
}
}
if config.metadata_database_url.is_none() {
if let Ok(metadata_database_url) = std::env::var("PRISMNET_METADATA_DATABASE_URL") {
let trimmed = metadata_database_url.trim();
if !trimmed.is_empty() {
config.metadata_database_url = Some(trimmed.to_string());
}
}
}
if !config.single_node {
if let Ok(single_node) = std::env::var("PRISMNET_SINGLE_NODE") {
let parsed = single_node.trim().to_ascii_lowercase();
config.single_node = matches!(parsed.as_str(), "1" | "true" | "yes" | "on");
}
}
// Initialize tracing // Initialize tracing
tracing_subscriber::fmt() tracing_subscriber::fmt()
.with_env_filter( .with_env_filter(
@ -192,12 +119,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
) )
} }
MetadataBackend::Postgres | MetadataBackend::Sqlite => { MetadataBackend::Postgres | MetadataBackend::Sqlite => {
let database_url = config let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
.metadata_database_url
.as_deref()
.ok_or_else(|| {
anyhow!( anyhow!(
"metadata_database_url is required when metadata_backend={} (env: PRISMNET_METADATA_DATABASE_URL)", "metadata_database_url is required when metadata_backend={}",
metadata_backend_name(config.metadata_backend) metadata_backend_name(config.metadata_backend)
) )
})?; })?;
@ -216,8 +140,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}; };
// Initialize OVN client (default: mock) // Initialize OVN client (default: mock)
let ovn = let ovn = Arc::new(
Arc::new(OvnClient::from_env().map_err(|e| anyhow!("Failed to init OVN client: {}", e))?); OvnClient::from_config(&config.ovn)
.map_err(|e| anyhow!("Failed to init OVN client: {}", e))?,
);
// Initialize IAM authentication service // Initialize IAM authentication service
tracing::info!( tracing::info!(
@ -374,19 +300,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(()) Ok(())
} }
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
match value.trim().to_ascii_lowercase().as_str() {
"flaredb" => Ok(MetadataBackend::FlareDb),
"postgres" => Ok(MetadataBackend::Postgres),
"sqlite" => Ok(MetadataBackend::Sqlite),
other => Err(format!(
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
other
)
.into()),
}
}
fn metadata_backend_name(backend: MetadataBackend) -> &'static str { fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
match backend { match backend {
MetadataBackend::FlareDb => "flaredb", MetadataBackend::FlareDb => "flaredb",

View file

@ -60,12 +60,8 @@ impl NetworkMetadataStore {
endpoint: Option<String>, endpoint: Option<String>,
pd_endpoint: Option<String>, pd_endpoint: Option<String>,
) -> Result<Self> { ) -> Result<Self> {
let endpoint = endpoint.unwrap_or_else(|| { let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
std::env::var("PRISMNET_FLAREDB_ENDPOINT")
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
});
let pd_endpoint = pd_endpoint let pd_endpoint = pd_endpoint
.or_else(|| std::env::var("PRISMNET_CHAINFIRE_ENDPOINT").ok())
.map(|value| normalize_transport_addr(&value)) .map(|value| normalize_transport_addr(&value))
.unwrap_or_else(|| endpoint.clone()); .unwrap_or_else(|| endpoint.clone());
@ -168,7 +164,9 @@ impl NetworkMetadataStore {
) )
.execute(pool) .execute(pool)
.await .await
.map_err(|e| MetadataError::Storage(format!("Failed to initialize Postgres schema: {}", e)))?; .map_err(|e| {
MetadataError::Storage(format!("Failed to initialize Postgres schema: {}", e))
})?;
Ok(()) Ok(())
} }
@ -181,7 +179,9 @@ impl NetworkMetadataStore {
) )
.execute(pool) .execute(pool)
.await .await
.map_err(|e| MetadataError::Storage(format!("Failed to initialize SQLite schema: {}", e)))?; .map_err(|e| {
MetadataError::Storage(format!("Failed to initialize SQLite schema: {}", e))
})?;
Ok(()) Ok(())
} }
@ -208,9 +208,7 @@ impl NetworkMetadataStore {
.bind(value) .bind(value)
.execute(pool.as_ref()) .execute(pool.as_ref())
.await .await
.map_err(|e| { .map_err(|e| MetadataError::Storage(format!("Postgres put failed: {}", e)))?;
MetadataError::Storage(format!("Postgres put failed: {}", e))
})?;
} }
SqlStorageBackend::Sqlite(pool) => { SqlStorageBackend::Sqlite(pool) => {
sqlx::query( sqlx::query(

View file

@ -1,9 +1,12 @@
use std::sync::Arc; use std::sync::Arc;
use prismnet_types::{DhcpOptions, Port, PortId, SecurityGroupId, SecurityGroupRule, SecurityGroupRuleId, VpcId}; use prismnet_types::{
DhcpOptions, Port, PortId, SecurityGroupId, SecurityGroupRule, SecurityGroupRuleId, VpcId,
};
use tokio::process::Command; use tokio::process::Command;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use crate::config::{OvnConfig, OvnMode as ConfigOvnMode};
use crate::ovn::mock::MockOvnState; use crate::ovn::mock::MockOvnState;
/// OVN client mode /// OVN client mode
@ -31,22 +34,19 @@ pub struct OvnClient {
} }
impl OvnClient { impl OvnClient {
/// Build an OVN client from environment variables (default: mock) /// Build an OVN client from configuration (default: mock)
/// - PRISMNET_OVN_MODE: "mock" (default) or "real" pub fn from_config(config: &OvnConfig) -> OvnResult<Self> {
/// - PRISMNET_OVN_NB_ADDR: ovsdb northbound address (real mode only) match config.mode {
pub fn from_env() -> OvnResult<Self> { ConfigOvnMode::Mock => Ok(Self::new_mock()),
let mode = std::env::var("PRISMNET_OVN_MODE").unwrap_or_else(|_| "mock".to_string()); ConfigOvnMode::Real => {
match mode.to_lowercase().as_str() { if config.nb_addr.trim().is_empty() {
"mock" => Ok(Self::new_mock()), Err(OvnError::InvalidArgument(
"real" => { "OVN nb_addr must not be empty in real mode".to_string(),
let nb_addr = std::env::var("PRISMNET_OVN_NB_ADDR") ))
.unwrap_or_else(|_| "tcp:127.0.0.1:6641".to_string()); } else {
Ok(Self::new_real(nb_addr)) Ok(Self::new_real(config.nb_addr.clone()))
}
} }
other => Err(OvnError::InvalidArgument(format!(
"Unknown OVN mode: {}",
other
))),
} }
} }
@ -305,10 +305,7 @@ impl OvnClient {
} }
if !options.dns_servers.is_empty() { if !options.dns_servers.is_empty() {
opts.push(format!( opts.push(format!("dns_server={{{}}}", options.dns_servers.join(",")));
"dns_server={{{}}}",
options.dns_servers.join(",")
));
} }
opts.push(format!("lease_time={}", options.lease_time)); opts.push(format!("lease_time={}", options.lease_time));
@ -389,7 +386,10 @@ impl OvnClient {
if !output.status.success() { if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr).to_string(); let stderr = String::from_utf8_lossy(&output.stderr).to_string();
return Err(OvnError::Command(format!("lr-add query failed: {}", stderr))); return Err(OvnError::Command(format!(
"lr-add query failed: {}",
stderr
)));
} }
let uuid = String::from_utf8_lossy(&output.stdout).trim().to_string(); let uuid = String::from_utf8_lossy(&output.stdout).trim().to_string();
@ -456,11 +456,7 @@ impl OvnClient {
let ls_name = Self::logical_switch_name(switch_id); let ls_name = Self::logical_switch_name(switch_id);
// ovn-nbctl lsp-add <switch> <port-name> // ovn-nbctl lsp-add <switch> <port-name>
self.run_nbctl(vec![ self.run_nbctl(vec!["lsp-add".into(), ls_name, switch_port_name.clone()])
"lsp-add".into(),
ls_name,
switch_port_name.clone(),
])
.await?; .await?;
// ovn-nbctl lsp-set-type <port> router // ovn-nbctl lsp-set-type <port> router
@ -698,10 +694,7 @@ mod tests {
let client = OvnClient::new_mock(); let client = OvnClient::new_mock();
// Create a router // Create a router
let router_id = client let router_id = client.create_logical_router("test-router").await.unwrap();
.create_logical_router("test-router")
.await
.unwrap();
assert!(!router_id.is_empty()); assert!(!router_id.is_empty());
assert!(router_id.starts_with("router-")); assert!(router_id.starts_with("router-"));
@ -732,10 +725,7 @@ mod tests {
.unwrap(); .unwrap();
// Create router // Create router
let router_id = client let router_id = client.create_logical_router("test-router").await.unwrap();
.create_logical_router("test-router")
.await
.unwrap();
// Add router port // Add router port
let mac = "02:00:00:00:00:01"; let mac = "02:00:00:00:00:01";
@ -760,10 +750,7 @@ mod tests {
let client = OvnClient::new_mock(); let client = OvnClient::new_mock();
// Create router // Create router
let router_id = client let router_id = client.create_logical_router("test-router").await.unwrap();
.create_logical_router("test-router")
.await
.unwrap();
// Configure SNAT // Configure SNAT
let external_ip = "203.0.113.10"; let external_ip = "203.0.113.10";
@ -791,10 +778,7 @@ mod tests {
.unwrap(); .unwrap();
// Create router // Create router
let router_id = client let router_id = client.create_logical_router("test-router").await.unwrap();
.create_logical_router("test-router")
.await
.unwrap();
// Add router port // Add router port
let mac = "02:00:00:00:00:01"; let mac = "02:00:00:00:00:01";
@ -840,10 +824,7 @@ mod tests {
.unwrap(); .unwrap();
// Create router // Create router
let router_id = client let router_id = client.create_logical_router("test-router").await.unwrap();
.create_logical_router("test-router")
.await
.unwrap();
// Add router ports to both switches // Add router ports to both switches
let port1_id = client let port1_id = client
@ -876,10 +857,7 @@ mod tests {
.unwrap(); .unwrap();
// Step 2: Create router // Step 2: Create router
let router_id = client let router_id = client.create_logical_router("vpc-router").await.unwrap();
.create_logical_router("vpc-router")
.await
.unwrap();
// Step 3: Attach router to switch // Step 3: Attach router to switch
let mac = "02:00:00:00:00:01"; let mac = "02:00:00:00:00:01";
@ -920,10 +898,7 @@ mod tests {
let client = OvnClient::new_mock(); let client = OvnClient::new_mock();
// Create router // Create router
let router_id = client let router_id = client.create_logical_router("test-router").await.unwrap();
.create_logical_router("test-router")
.await
.unwrap();
// Configure multiple SNAT rules // Configure multiple SNAT rules
client client