centra/photoncloud-monorepo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

harden plasmavmc image ingestion and internal execution paths

Browse source
This commit is contained in:
centra 2026-04-02 07:57:25 +09:00
parent 260fb4c576
commit 0745216107
Signed by: centra
GPG key ID: 0C09689D20B25ACA
64 changed files with 3531 additions and 1434 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
deployer/crates/deployer-ctl/src/power.rs
View file

@ -161,7 +161,9 @@ impl RedfishTarget {
match action {
PowerAction::Cycle => Ok(PowerState::Cycling),
PowerAction::On | PowerAction::Off | PowerAction::Refresh => self.refresh(&client).await,
PowerAction::On | PowerAction::Off | PowerAction::Refresh => {
self.refresh(&client).await
}
}
}
@ -295,7 +297,12 @@ pub async fn request_reinstall(
#[cfg(test)]
mod tests {
use super::*;
use axum::{extract::State, http::StatusCode, routing::{get, post}, Json, Router};
use axum::{
extract::State,
http::StatusCode,
routing::{get, post},
Json, Router,
};
use serde_json::Value;
use std::sync::{Arc, Mutex};
use tokio::net::TcpListener;
@ -303,7 +310,10 @@ mod tests {
#[test]
fn parse_redfish_short_reference_defaults_to_https() {
let parsed = RedfishTarget::parse("redfish://lab-bmc/node01").unwrap();
assert_eq!(parsed.resource_url.as_str(), "https://lab-bmc/redfish/v1/Systems/node01");
assert_eq!(
parsed.resource_url.as_str(),
"https://lab-bmc/redfish/v1/Systems/node01"
);
}
#[test]
@ -361,8 +371,14 @@ mod tests {
addr
))
.unwrap();
assert_eq!(target.perform(PowerAction::Refresh).await.unwrap(), PowerState::On);
assert_eq!(target.perform(PowerAction::Off).await.unwrap(), PowerState::On);
assert_eq!(
target.perform(PowerAction::Refresh).await.unwrap(),
PowerState::On
);
assert_eq!(
target.perform(PowerAction::Off).await.unwrap(),
PowerState::On
);
let payloads = state.seen_payloads.lock().unwrap().clone();
assert_eq!(payloads, vec![r#"{"ResetType":"ForceOff"}"#.to_string()]);

2
deployer/crates/deployer-ctl/src/remote.rs
View file

@ -31,5 +31,3 @@ pub async fn run_deployer_command(endpoint: &str, action: &str) -> Result<()> {
Ok(())
}

5
fiberlb/crates/fiberlb-server/src/config.rs
View file

@ -177,6 +177,10 @@ pub struct VipOwnershipConfig {
/// Interface used for local VIP ownership.
#[serde(default = "default_vip_ownership_interface")]
pub interface: String,
/// Optional explicit `ip` command path used for local VIP ownership.
#[serde(default)]
pub ip_command: Option<String>,
}
fn default_vip_ownership_interface() -> String {
@ -188,6 +192,7 @@ impl Default for VipOwnershipConfig {
Self {
enabled: false,
interface: default_vip_ownership_interface(),
ip_command: None,
}
}
}

66
fiberlb/crates/fiberlb-server/src/main.rs
View file

@ -41,26 +41,6 @@ struct Args {
#[arg(long)]
grpc_addr: Option<String>,
/// ChainFire endpoint for cluster coordination
#[arg(long, env = "FIBERLB_CHAINFIRE_ENDPOINT")]
chainfire_endpoint: Option<String>,
/// FlareDB endpoint for metadata and tenant data storage
#[arg(long, env = "FIBERLB_FLAREDB_ENDPOINT")]
flaredb_endpoint: Option<String>,
/// Metadata backend (flaredb, postgres, sqlite)
#[arg(long, env = "FIBERLB_METADATA_BACKEND")]
metadata_backend: Option<String>,
/// SQL database URL for metadata (required for postgres/sqlite backend)
#[arg(long, env = "FIBERLB_METADATA_DATABASE_URL")]
metadata_database_url: Option<String>,
/// Run in single-node mode (required when metadata backend is SQLite)
#[arg(long, env = "FIBERLB_SINGLE_NODE")]
single_node: bool,
/// Log level (overrides config)
#[arg(short, long)]
log_level: Option<String>,
@ -93,21 +73,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
if let Some(log_level) = args.log_level {
config.log_level = log_level;
}
if let Some(chainfire_endpoint) = args.chainfire_endpoint {
config.chainfire_endpoint = Some(chainfire_endpoint);
}
if let Some(flaredb_endpoint) = args.flaredb_endpoint {
config.flaredb_endpoint = Some(flaredb_endpoint);
}
if let Some(metadata_backend) = args.metadata_backend {
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
}
if let Some(metadata_database_url) = args.metadata_database_url {
config.metadata_database_url = Some(metadata_database_url);
}
if args.single_node {
config.single_node = true;
}
// Initialize tracing
tracing_subscriber::fmt()
@ -194,15 +159,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
)
}
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
let database_url = config
.metadata_database_url
.as_deref()
.ok_or_else(|| {
format!(
"metadata_database_url is required when metadata_backend={} (env: FIBERLB_METADATA_DATABASE_URL)",
metadata_backend_name(config.metadata_backend)
)
})?;
let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
format!(
"metadata_database_url is required when metadata_backend={}",
metadata_backend_name(config.metadata_backend)
)
})?;
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
tracing::info!(
" Metadata backend: {} @ {}",
@ -282,8 +244,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
})?;
let bgp = create_bgp_client(config.bgp.clone()).await?;
let vip_owner: Option<Arc<dyn VipAddressOwner>> = if config.vip_ownership.enabled {
Some(Arc::new(KernelVipAddressOwner::new(
Some(Arc::new(KernelVipAddressOwner::with_ip_command(
config.vip_ownership.interface.clone(),
config.vip_ownership.ip_command.clone(),
)))
} else {
None
@ -439,19 +402,6 @@ async fn wait_for_shutdown_signal() -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
match value.trim().to_ascii_lowercase().as_str() {
"flaredb" => Ok(MetadataBackend::FlareDb),
"postgres" => Ok(MetadataBackend::Postgres),
"sqlite" => Ok(MetadataBackend::Sqlite),
other => Err(format!(
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
other
)
.into()),
}
}
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
match backend {
MetadataBackend::FlareDb => "flaredb",

10
fiberlb/crates/fiberlb-server/src/metadata.rs
View file

@ -61,12 +61,12 @@ impl LbMetadataStore {
endpoint: Option<String>,
pd_endpoint: Option<String>,
) -> Result<Self> {
let endpoint = endpoint.unwrap_or_else(|| {
std::env::var("FIBERLB_FLAREDB_ENDPOINT")
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
});
let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
Self::connect_flaredb(endpoint, pd_endpoint).await
}
async fn connect_flaredb(endpoint: String, pd_endpoint: Option<String>) -> Result<Self> {
let pd_endpoint = pd_endpoint
.or_else(|| std::env::var("FIBERLB_CHAINFIRE_ENDPOINT").ok())
.map(|value| normalize_transport_addr(&value))
.unwrap_or_else(|| endpoint.clone());

53
fiberlb/crates/fiberlb-server/src/vip_owner.rs
View file

@ -41,9 +41,17 @@ pub struct KernelVipAddressOwner {
impl KernelVipAddressOwner {
/// Create a kernel-backed VIP owner for the given interface.
pub fn new(interface: impl Into<String>) -> Self {
Self::with_ip_command(interface, None::<String>)
}
/// Create a kernel-backed VIP owner with an optional explicit `ip` command path.
pub fn with_ip_command(
interface: impl Into<String>,
ip_command: Option<impl Into<String>>,
) -> Self {
Self {
interface: interface.into(),
ip_command: resolve_ip_command(),
ip_command: resolve_ip_command(ip_command.map(Into::into)),
}
}
@ -123,7 +131,14 @@ impl VipAddressOwner for KernelVipAddressOwner {
}
}
fn resolve_ip_command() -> String {
fn resolve_ip_command(configured: Option<String>) -> String {
if let Some(path) = configured {
let trimmed = path.trim();
if !trimmed.is_empty() {
return trimmed.to_string();
}
}
if let Ok(path) = std::env::var("FIBERLB_IP_COMMAND") {
let trimmed = path.trim();
if !trimmed.is_empty() {
@ -159,3 +174,37 @@ fn render_command_output(output: &std::process::Output) -> String {
format!("exit status {}", output.status)
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{Mutex, OnceLock};
fn env_lock() -> &'static Mutex<()> {
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
LOCK.get_or_init(|| Mutex::new(()))
}
#[test]
fn configured_ip_command_wins_over_env() {
let _guard = env_lock().lock().unwrap();
std::env::set_var("FIBERLB_IP_COMMAND", "/env/ip");
assert_eq!(
resolve_ip_command(Some(" /configured/ip ".to_string())),
"/configured/ip"
);
std::env::remove_var("FIBERLB_IP_COMMAND");
}
#[test]
fn env_ip_command_is_used_when_config_is_absent() {
let _guard = env_lock().lock().unwrap();
std::env::set_var("FIBERLB_IP_COMMAND", " /env/ip ");
assert_eq!(resolve_ip_command(None), "/env/ip");
std::env::remove_var("FIBERLB_IP_COMMAND");
}
}

129
flashdns/crates/flashdns-server/src/main.rs
View file

@ -1,27 +1,24 @@
//! FlashDNS authoritative DNS server binary
use anyhow::Result;
use chainfire_client::Client as ChainFireClient;
use clap::Parser;
use flashdns_api::{RecordServiceServer, ZoneServiceServer};
use flashdns_server::{
config::{MetadataBackend, ServerConfig},
dns::DnsHandler,
metadata::DnsMetadataStore,
RecordServiceImpl,
ZoneServiceImpl,
RecordServiceImpl, ZoneServiceImpl,
};
use chainfire_client::Client as ChainFireClient;
use iam_service_auth::AuthService;
use metrics_exporter_prometheus::PrometheusBuilder;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tonic::transport::{Certificate, Identity, Server, ServerTlsConfig};
use tonic::{Request, Status};
use tonic_health::server::health_reporter;
use tracing_subscriber::EnvFilter;
use anyhow::Result;
use clap::Parser;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use config::{Config as Cfg, Environment, File, FileFormat};
/// Command-line arguments for FlashDNS server.
#[derive(Parser, Debug)]
@ -39,26 +36,6 @@ struct CliArgs {
#[arg(long)]
dns_addr: Option<String>,
/// ChainFire endpoint for cluster coordination (overrides config)
#[arg(long, env = "FLASHDNS_CHAINFIRE_ENDPOINT")]
chainfire_endpoint: Option<String>,
/// FlareDB endpoint for metadata and tenant data storage (overrides config)
#[arg(long, env = "FLASHDNS_FLAREDB_ENDPOINT")]
flaredb_endpoint: Option<String>,
/// Metadata backend (flaredb, postgres, sqlite)
#[arg(long, env = "FLASHDNS_METADATA_BACKEND")]
metadata_backend: Option<String>,
/// SQL database URL for metadata (required for postgres/sqlite backend)
#[arg(long, env = "FLASHDNS_METADATA_DATABASE_URL")]
metadata_database_url: Option<String>,
/// Run in single-node mode (required when metadata backend is SQLite)
#[arg(long, env = "FLASHDNS_SINGLE_NODE")]
single_node: bool,
/// Log level (overrides config)
#[arg(short, long)]
log_level: Option<String>,
@ -72,54 +49,22 @@ struct CliArgs {
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let cli_args = CliArgs::parse();
// Load configuration using config-rs
let mut settings = Cfg::builder()
// Layer 1: Application defaults. Serialize ServerConfig::default() into TOML.
.add_source(File::from_str(
toml::to_string(&ServerConfig::default())?.as_str(),
FileFormat::Toml,
))
// Layer 2: Environment variables (e.g., FLASHDNS_GRPC_ADDR, FLASHDNS_LOG_LEVEL)
.add_source(
Environment::with_prefix("FLASHDNS")
.separator("__") // Use double underscore for nested fields
);
// Layer 3: Configuration file (if specified)
if cli_args.config.exists() {
let mut config = if cli_args.config.exists() {
tracing::info!("Loading config from file: {}", cli_args.config.display());
settings = settings.add_source(File::from(cli_args.config.as_path()));
let contents = tokio::fs::read_to_string(&cli_args.config).await?;
toml::from_str(&contents)?
} else {
tracing::info!("Config file not found, using defaults and environment variables.");
}
tracing::info!("Config file not found, using defaults.");
ServerConfig::default()
};
let mut config: ServerConfig = settings
.build()?
.try_deserialize()
.map_err(|e| anyhow::anyhow!("Failed to load configuration: {}", e))?;
// Apply command line overrides (Layer 4: highest precedence)
// Apply command line overrides
if let Some(grpc_addr_str) = cli_args.grpc_addr {
config.grpc_addr = grpc_addr_str.parse()?;
}
if let Some(dns_addr_str) = cli_args.dns_addr {
config.dns_addr = dns_addr_str.parse()?;
}
if let Some(chainfire_endpoint) = cli_args.chainfire_endpoint {
config.chainfire_endpoint = Some(chainfire_endpoint);
}
if let Some(flaredb_endpoint) = cli_args.flaredb_endpoint {
config.flaredb_endpoint = Some(flaredb_endpoint);
}
if let Some(metadata_backend) = cli_args.metadata_backend {
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
}
if let Some(metadata_database_url) = cli_args.metadata_database_url {
config.metadata_database_url = Some(metadata_database_url);
}
if cli_args.single_node {
config.single_node = true;
}
if let Some(log_level) = cli_args.log_level {
config.log_level = log_level;
}
@ -172,20 +117,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
config.flaredb_endpoint.clone(),
config.chainfire_endpoint.clone(),
)
.await
.map_err(|e| anyhow::anyhow!("Failed to initialize FlareDB metadata store: {}", e))?,
.await
.map_err(|e| {
anyhow::anyhow!("Failed to initialize FlareDB metadata store: {}", e)
})?,
)
}
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
let database_url = config
.metadata_database_url
.as_deref()
.ok_or_else(|| {
anyhow::anyhow!(
"metadata_database_url is required when metadata_backend={} (env: FLASHDNS_METADATA_DATABASE_URL)",
metadata_backend_name(config.metadata_backend)
)
})?;
let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
anyhow::anyhow!(
"metadata_database_url is required when metadata_backend={}",
metadata_backend_name(config.metadata_backend)
)
})?;
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
tracing::info!(
" Metadata backend: {} @ {}",
@ -195,13 +139,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Arc::new(
DnsMetadataStore::new_sql(database_url, config.single_node)
.await
.map_err(|e| anyhow::anyhow!("Failed to initialize SQL metadata store: {}", e))?,
.map_err(|e| {
anyhow::anyhow!("Failed to initialize SQL metadata store: {}", e)
})?,
)
}
};
// Initialize IAM authentication service
tracing::info!("Connecting to IAM server at {}", config.auth.iam_server_addr);
tracing::info!(
"Connecting to IAM server at {}",
config.auth.iam_server_addr
);
let auth_service = AuthService::new(&config.auth.iam_server_addr)
.await
.map_err(|e| anyhow::anyhow!("Failed to connect to IAM server: {}", e))?;
@ -300,18 +249,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend> {
match value.trim().to_ascii_lowercase().as_str() {
"flaredb" => Ok(MetadataBackend::FlareDb),
"postgres" => Ok(MetadataBackend::Postgres),
"sqlite" => Ok(MetadataBackend::Sqlite),
other => Err(anyhow::anyhow!(
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
other
)),
}
}
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
match backend {
MetadataBackend::FlareDb => "flaredb",
@ -345,11 +282,7 @@ fn ensure_sql_backend_matches_url(backend: MetadataBackend, database_url: &str)
}
}
async fn register_chainfire_membership(
endpoint: &str,
service: &str,
addr: String,
) -> Result<()> {
async fn register_chainfire_membership(endpoint: &str, service: &str, addr: String) -> Result<()> {
let node_id =
std::env::var("HOSTNAME").unwrap_or_else(|_| format!("{}-{}", service, std::process::id()));
let ts = SystemTime::now()

6
flashdns/crates/flashdns-server/src/metadata.rs
View file

@ -57,12 +57,8 @@ impl DnsMetadataStore {
endpoint: Option<String>,
pd_endpoint: Option<String>,
) -> Result<Self> {
let endpoint = endpoint.unwrap_or_else(|| {
std::env::var("FLASHDNS_FLAREDB_ENDPOINT")
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
});
let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
let pd_endpoint = pd_endpoint
.or_else(|| std::env::var("FLASHDNS_CHAINFIRE_ENDPOINT").ok())
.map(|value| normalize_transport_addr(&value))
.unwrap_or_else(|| endpoint.clone());

53
iam/crates/iam-server/src/config.rs
View file

@ -27,6 +27,14 @@ pub struct ServerConfig {
/// Logging configuration
#[serde(default)]
pub logging: LoggingConfig,
/// Admin API policy configuration
#[serde(default)]
pub admin: AdminConfig,
/// Development-only safety valves
#[serde(default)]
pub dev: DevConfig,
}
impl ServerConfig {
@ -102,6 +110,29 @@ impl ServerConfig {
}
}
if let Ok(value) = std::env::var("IAM_ALLOW_UNAUTHENTICATED_ADMIN")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_UNAUTHENTICATED_ADMIN"))
{
let value = value.trim().to_ascii_lowercase();
config.admin.allow_unauthenticated =
matches!(value.as_str(), "1" | "true" | "yes" | "on");
}
if let Ok(value) = std::env::var("IAM_ALLOW_RANDOM_SIGNING_KEY")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_RANDOM_SIGNING_KEY"))
{
let value = value.trim().to_ascii_lowercase();
config.dev.allow_random_signing_key =
matches!(value.as_str(), "1" | "true" | "yes" | "on");
}
if let Ok(value) = std::env::var("IAM_ALLOW_MEMORY_BACKEND")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_MEMORY_BACKEND"))
{
let value = value.trim().to_ascii_lowercase();
config.dev.allow_memory_backend = matches!(value.as_str(), "1" | "true" | "yes" | "on");
}
Ok(config)
}
@ -132,6 +163,8 @@ impl ServerConfig {
},
},
logging: LoggingConfig::default(),
admin: AdminConfig::default(),
dev: DevConfig::default(),
}
}
}
@ -226,6 +259,26 @@ pub struct ClusterConfig {
pub chainfire_endpoint: Option<String>,
}
/// Admin API policy configuration
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct AdminConfig {
/// Allow admin APIs to run without an explicit admin token (dev only)
#[serde(default)]
pub allow_unauthenticated: bool,
}
/// Development-only safety valves
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct DevConfig {
/// Allow generating a random signing key when none is configured
#[serde(default)]
pub allow_random_signing_key: bool,
/// Allow the in-memory backend
#[serde(default)]
pub allow_memory_backend: bool,
}
/// Backend type
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]

73
iam/crates/iam-server/src/main.rs
View file

@ -64,17 +64,18 @@ fn load_admin_token() -> Option<String> {
.filter(|value| !value.is_empty())
}
fn allow_unauthenticated_admin() -> bool {
std::env::var("IAM_ALLOW_UNAUTHENTICATED_ADMIN")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_UNAUTHENTICATED_ADMIN"))
.ok()
.map(|value| {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "y" | "on"
)
})
.unwrap_or(false)
fn allow_unauthenticated_admin(config: &ServerConfig) -> bool {
config.admin.allow_unauthenticated
|| std::env::var("IAM_ALLOW_UNAUTHENTICATED_ADMIN")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_UNAUTHENTICATED_ADMIN"))
.ok()
.map(|value| {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "y" | "on"
)
})
.unwrap_or(false)
}
fn admin_token_valid(metadata: &MetadataMap, token: &str) -> bool {
@ -334,19 +335,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Create token service
let signing_key = if config.authn.internal_token.signing_key.is_empty() {
let allow_random = std::env::var("IAM_ALLOW_RANDOM_SIGNING_KEY")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_RANDOM_SIGNING_KEY"))
.ok()
.map(|value| {
matches!(
value.trim().to_lowercase().as_str(),
"1" | "true" | "yes" | "y" | "on"
)
})
.unwrap_or(false);
let allow_random = config.dev.allow_random_signing_key
|| std::env::var("IAM_ALLOW_RANDOM_SIGNING_KEY")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_RANDOM_SIGNING_KEY"))
.ok()
.map(|value| {
matches!(
value.trim().to_lowercase().as_str(),
"1" | "true" | "yes" | "y" | "on"
)
})
.unwrap_or(false);
if !allow_random {
return Err("No signing key configured. Set IAM_ALLOW_RANDOM_SIGNING_KEY=true for dev or configure authn.internal_token.signing_key.".into());
return Err("No signing key configured. Set dev.allow_random_signing_key=true (or IAM_ALLOW_RANDOM_SIGNING_KEY=true for legacy dev mode) or configure authn.internal_token.signing_key.".into());
}
warn!("No signing key configured, generating random key (dev-only)");
@ -369,9 +371,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let token_service = Arc::new(InternalTokenService::new(token_config));
let admin_token = load_admin_token();
if admin_token.is_none() && !allow_unauthenticated_admin() {
if admin_token.is_none() && !allow_unauthenticated_admin(&config) {
return Err(
"IAM admin token not configured. Set IAM_ADMIN_TOKEN or explicitly allow dev mode with IAM_ALLOW_UNAUTHENTICATED_ADMIN=true."
"IAM admin token not configured. Set IAM_ADMIN_TOKEN or explicitly allow dev mode with admin.allow_unauthenticated=true."
.into(),
);
}
@ -550,19 +552,20 @@ async fn create_backend(
) -> Result<Backend, Box<dyn std::error::Error>> {
match config.store.backend {
BackendKind::Memory => {
let allow_memory = std::env::var("IAM_ALLOW_MEMORY_BACKEND")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_MEMORY_BACKEND"))
.ok()
.map(|value| {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
})
.unwrap_or(false);
let allow_memory = config.dev.allow_memory_backend
|| std::env::var("IAM_ALLOW_MEMORY_BACKEND")
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_MEMORY_BACKEND"))
.ok()
.map(|value| {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
})
.unwrap_or(false);
if !allow_memory {
return Err(
"In-memory IAM backend is disabled. Use FlareDB backend, or set IAM_ALLOW_MEMORY_BACKEND=true for tests/dev only."
"In-memory IAM backend is disabled. Use FlareDB backend, or set dev.allow_memory_backend=true for tests/dev only."
.into(),
);
}

14
k8shost/crates/k8shost-server/src/config.rs
View file

@ -91,6 +91,18 @@ impl Default for PrismNetConfig {
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CreditServiceConfig {
#[serde(default)]
pub server_addr: Option<String>,
}
impl Default for CreditServiceConfig {
fn default() -> Self {
Self { server_addr: None }
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ChainFireConfig {
pub endpoint: Option<String>,
@ -110,6 +122,8 @@ pub struct Config {
pub flaredb: FlareDbConfig,
pub chainfire: ChainFireConfig,
pub iam: IamConfig,
#[serde(default)]
pub creditservice: CreditServiceConfig,
pub fiberlb: FiberLbConfig,
pub flashdns: FlashDnsConfig,
pub prismnet: PrismNetConfig,

42
k8shost/crates/k8shost-server/src/main.rs
View file

@ -38,8 +38,8 @@ use tracing_subscriber::EnvFilter;
#[command(about = "Kubernetes API server for PlasmaCloud's k8shost component")]
struct Args {
/// Configuration file path
#[arg(short, long)]
config: Option<PathBuf>,
#[arg(short, long, default_value = "k8shost.toml")]
config: PathBuf,
/// Listen address for gRPC server (e.g., "[::]:6443")
#[arg(long)]
@ -91,17 +91,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let args = Args::parse();
// Load configuration
let mut settings = ::config::Config::builder()
.add_source(::config::File::from_str(
toml::to_string(&Config::default())?.as_str(),
::config::FileFormat::Toml,
))
.add_source(::config::Environment::with_prefix("K8SHOST").separator("_"));
let mut settings = ::config::Config::builder().add_source(::config::File::from_str(
toml::to_string(&Config::default())?.as_str(),
::config::FileFormat::Toml,
));
// Add config file if specified
if let Some(config_path) = &args.config {
info!("Loading config from file: {}", config_path.display());
settings = settings.add_source(::config::File::from(config_path.as_path()));
if args.config.exists() {
info!("Loading config from file: {}", args.config.display());
settings = settings.add_source(::config::File::from(args.config.as_path()));
} else {
info!(
"Config file not found: {}, using defaults",
args.config.display()
);
}
let loaded_config: Config = settings
@ -136,6 +138,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.iam_server_addr
.unwrap_or(loaded_config.iam.server_addr),
},
creditservice: config::CreditServiceConfig {
server_addr: loaded_config.creditservice.server_addr,
},
fiberlb: config::FiberLbConfig {
server_addr: args
.fiberlb_server_addr
@ -275,8 +280,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let ipam_client = Arc::new(IpamClient::new(config.prismnet.server_addr.clone()));
// Create service implementations with storage
let creditservice_endpoint = config.creditservice.server_addr.as_deref();
let pod_service = Arc::new(
PodServiceImpl::new_with_credit_service(storage.clone(), auth_service.clone()).await,
PodServiceImpl::new_with_credit_service(
storage.clone(),
auth_service.clone(),
creditservice_endpoint,
)
.await,
);
let service_service = Arc::new(ServiceServiceImpl::new(
storage.clone(),
@ -290,7 +301,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
));
// Start scheduler in background with CreditService integration
let scheduler = Arc::new(scheduler::Scheduler::new_with_credit_service(storage.clone()).await);
let scheduler = Arc::new(
scheduler::Scheduler::new_with_credit_service(storage.clone(), creditservice_endpoint)
.await,
);
tokio::spawn(async move {
scheduler.run().await;
});

36
k8shost/crates/k8shost-server/src/scheduler.rs
View file

@ -33,27 +33,31 @@ impl Scheduler {
}
/// Create a new scheduler with CreditService quota enforcement
pub async fn new_with_credit_service(storage: Arc<Storage>) -> Self {
pub async fn new_with_credit_service(storage: Arc<Storage>, endpoint: Option<&str>) -> Self {
// Initialize CreditService client if endpoint is configured
let credit_service = match std::env::var("CREDITSERVICE_ENDPOINT") {
Ok(endpoint) => match CreditServiceClient::connect(&endpoint).await {
Ok(client) => {
info!(
"Scheduler: CreditService quota enforcement enabled: {}",
endpoint
);
Some(Arc::new(RwLock::new(client)))
}
Err(e) => {
warn!(
let credit_service = match endpoint {
Some(endpoint) if !endpoint.trim().is_empty() => {
match CreditServiceClient::connect(endpoint).await {
Ok(client) => {
info!(
"Scheduler: CreditService quota enforcement enabled: {}",
endpoint
);
Some(Arc::new(RwLock::new(client)))
}
Err(e) => {
warn!(
"Scheduler: Failed to connect to CreditService (quota enforcement disabled): {}",
e
);
None
None
}
}
},
Err(_) => {
info!("Scheduler: CREDITSERVICE_ENDPOINT not set, quota enforcement disabled");
}
_ => {
info!(
"Scheduler: CreditService endpoint not configured, quota enforcement disabled"
);
None
}
};

38
k8shost/crates/k8shost-server/src/services/pod.rs
View file

@ -45,24 +45,30 @@ impl PodServiceImpl {
}
}
pub async fn new_with_credit_service(storage: Arc<Storage>, auth: Arc<AuthService>) -> Self {
pub async fn new_with_credit_service(
storage: Arc<Storage>,
auth: Arc<AuthService>,
endpoint: Option<&str>,
) -> Self {
// Initialize CreditService client if endpoint is configured
let credit_service = match std::env::var("CREDITSERVICE_ENDPOINT") {
Ok(endpoint) => match CreditServiceClient::connect(&endpoint).await {
Ok(client) => {
tracing::info!("CreditService admission control enabled: {}", endpoint);
Some(Arc::new(RwLock::new(client)))
let credit_service = match endpoint {
Some(endpoint) if !endpoint.trim().is_empty() => {
match CreditServiceClient::connect(endpoint).await {
Ok(client) => {
tracing::info!("CreditService admission control enabled: {}", endpoint);
Some(Arc::new(RwLock::new(client)))
}
Err(e) => {
tracing::warn!(
"Failed to connect to CreditService (admission control disabled): {}",
e
);
None
}
}
Err(e) => {
tracing::warn!(
"Failed to connect to CreditService (admission control disabled): {}",
e
);
None
}
},
Err(_) => {
tracing::info!("CREDITSERVICE_ENDPOINT not set, admission control disabled");
}
_ => {
tracing::info!("CreditService endpoint not configured, admission control disabled");
None
}
};

114
lightningstor/crates/lightningstor-server/src/config.rs
View file

@ -50,6 +50,75 @@ pub enum ObjectStorageBackend {
Distributed,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct S3AuthConfig {
#[serde(default = "default_s3_auth_enabled")]
pub enabled: bool,
#[serde(default = "default_s3_aws_region")]
pub aws_region: String,
#[serde(default = "default_s3_iam_cache_ttl_secs")]
pub iam_cache_ttl_secs: u64,
#[serde(default = "default_s3_default_org_id")]
pub default_org_id: Option<String>,
#[serde(default = "default_s3_default_project_id")]
pub default_project_id: Option<String>,
#[serde(default = "default_s3_max_auth_body_bytes")]
pub max_auth_body_bytes: usize,
}
impl Default for S3AuthConfig {
fn default() -> Self {
Self {
enabled: default_s3_auth_enabled(),
aws_region: default_s3_aws_region(),
iam_cache_ttl_secs: default_s3_iam_cache_ttl_secs(),
default_org_id: default_s3_default_org_id(),
default_project_id: default_s3_default_project_id(),
max_auth_body_bytes: default_s3_max_auth_body_bytes(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct S3PerformanceConfig {
#[serde(default = "default_s3_streaming_put_threshold_bytes")]
pub streaming_put_threshold_bytes: usize,
#[serde(default = "default_s3_inline_put_max_bytes")]
pub inline_put_max_bytes: usize,
#[serde(default = "default_s3_multipart_put_concurrency")]
pub multipart_put_concurrency: usize,
#[serde(default = "default_s3_multipart_fetch_concurrency")]
pub multipart_fetch_concurrency: usize,
}
impl Default for S3PerformanceConfig {
fn default() -> Self {
Self {
streaming_put_threshold_bytes: default_s3_streaming_put_threshold_bytes(),
inline_put_max_bytes: default_s3_inline_put_max_bytes(),
multipart_put_concurrency: default_s3_multipart_put_concurrency(),
multipart_fetch_concurrency: default_s3_multipart_fetch_concurrency(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct S3Config {
#[serde(default)]
pub auth: S3AuthConfig,
#[serde(default)]
pub performance: S3PerformanceConfig,
}
/// Server configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ServerConfig {
@ -100,6 +169,10 @@ pub struct ServerConfig {
/// Authentication configuration
#[serde(default)]
pub auth: AuthConfig,
/// S3 API runtime settings
#[serde(default)]
pub s3: S3Config,
}
/// Authentication configuration
@ -114,6 +187,46 @@ fn default_iam_server_addr() -> String {
"127.0.0.1:50051".to_string()
}
fn default_s3_auth_enabled() -> bool {
true
}
fn default_s3_aws_region() -> String {
"us-east-1".to_string()
}
fn default_s3_iam_cache_ttl_secs() -> u64 {
30
}
fn default_s3_default_org_id() -> Option<String> {
Some("default".to_string())
}
fn default_s3_default_project_id() -> Option<String> {
Some("default".to_string())
}
fn default_s3_max_auth_body_bytes() -> usize {
1024 * 1024 * 1024
}
fn default_s3_streaming_put_threshold_bytes() -> usize {
16 * 1024 * 1024
}
fn default_s3_inline_put_max_bytes() -> usize {
128 * 1024 * 1024
}
fn default_s3_multipart_put_concurrency() -> usize {
4
}
fn default_s3_multipart_fetch_concurrency() -> usize {
4
}
impl Default for AuthConfig {
fn default() -> Self {
Self {
@ -139,6 +252,7 @@ impl Default for ServerConfig {
sync_on_write: false,
tls: None,
auth: AuthConfig::default(),
s3: S3Config::default(),
}
}
}

76
lightningstor/crates/lightningstor-server/src/main.rs
View file

@ -5,7 +5,7 @@ use clap::Parser;
use iam_service_auth::AuthService;
use lightningstor_api::{BucketServiceServer, ObjectServiceServer};
use lightningstor_distributed::{
DistributedConfig, ErasureCodedBackend, RedundancyMode, ReplicatedBackend, RepairQueue,
DistributedConfig, ErasureCodedBackend, RedundancyMode, RepairQueue, ReplicatedBackend,
StaticNodeRegistry,
};
use lightningstor_server::{
@ -57,26 +57,6 @@ struct Args {
#[arg(short, long)]
log_level: Option<String>,
/// ChainFire endpoint for cluster coordination (overrides config)
#[arg(long, env = "LIGHTNINGSTOR_CHAINFIRE_ENDPOINT")]
chainfire_endpoint: Option<String>,
/// FlareDB endpoint for metadata and tenant data storage (overrides config)
#[arg(long, env = "LIGHTNINGSTOR_FLAREDB_ENDPOINT")]
flaredb_endpoint: Option<String>,
/// Metadata backend (flaredb, postgres, sqlite)
#[arg(long, env = "LIGHTNINGSTOR_METADATA_BACKEND")]
metadata_backend: Option<String>,
/// SQL database URL for metadata (required for postgres/sqlite backend)
#[arg(long, env = "LIGHTNINGSTOR_METADATA_DATABASE_URL")]
metadata_database_url: Option<String>,
/// Run in single-node mode (required when metadata backend is SQLite)
#[arg(long, env = "LIGHTNINGSTOR_SINGLE_NODE")]
single_node: bool,
/// Data directory for object storage (overrides config)
#[arg(long)]
data_dir: Option<String>,
@ -112,21 +92,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
if let Some(log_level) = args.log_level {
config.log_level = log_level;
}
if let Some(chainfire_endpoint) = args.chainfire_endpoint {
config.chainfire_endpoint = Some(chainfire_endpoint);
}
if let Some(flaredb_endpoint) = args.flaredb_endpoint {
config.flaredb_endpoint = Some(flaredb_endpoint);
}
if let Some(metadata_backend) = args.metadata_backend {
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
}
if let Some(metadata_database_url) = args.metadata_database_url {
config.metadata_database_url = Some(metadata_database_url);
}
if args.single_node {
config.single_node = true;
}
if let Some(data_dir) = args.data_dir {
config.data_dir = data_dir;
}
@ -182,20 +147,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
config.flaredb_endpoint.clone(),
config.chainfire_endpoint.clone(),
)
.await
.map_err(|e| format!("Failed to initialize FlareDB metadata store: {}", e))?,
.await
.map_err(|e| format!("Failed to initialize FlareDB metadata store: {}", e))?,
)
}
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
let database_url = config
.metadata_database_url
.as_deref()
.ok_or_else(|| {
format!(
"metadata_database_url is required when metadata_backend={} (env: LIGHTNINGSTOR_METADATA_DATABASE_URL)",
metadata_backend_name(config.metadata_backend)
)
})?;
let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
format!(
"metadata_database_url is required when metadata_backend={}",
metadata_backend_name(config.metadata_backend)
)
})?;
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
tracing::info!(
"Metadata backend: {} @ {}",
@ -263,10 +225,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let s3_addr: SocketAddr = config.s3_addr;
// Start S3 HTTP server with shared state
let s3_router = s3::create_router_with_auth(
let s3_router = s3::create_router_with_auth_config(
storage.clone(),
metadata.clone(),
Some(config.auth.iam_server_addr.clone()),
config.s3.clone(),
);
let s3_server = tokio::spawn(async move {
tracing::info!("S3 HTTP server listening on {}", s3_addr);
@ -341,19 +304,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
match value.trim().to_ascii_lowercase().as_str() {
"flaredb" => Ok(MetadataBackend::FlareDb),
"postgres" => Ok(MetadataBackend::Postgres),
"sqlite" => Ok(MetadataBackend::Sqlite),
other => Err(format!(
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
other
)
.into()),
}
}
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
match backend {
MetadataBackend::FlareDb => "flaredb",
@ -442,7 +392,9 @@ async fn create_storage_backend(
ObjectStorageBackend::LocalFs => {
tracing::info!("Object storage backend: local_fs");
Ok(StorageRuntime {
backend: Arc::new(LocalFsBackend::new(&config.data_dir, config.sync_on_write).await?),
backend: Arc::new(
LocalFsBackend::new(&config.data_dir, config.sync_on_write).await?,
),
repair_worker: None,
})
}

104
lightningstor/crates/lightningstor-server/src/metadata.rs
View file

@ -55,26 +55,25 @@ impl MetadataStore {
endpoint: Option<String>,
pd_endpoint: Option<String>,
) -> Result<Self> {
let endpoint = endpoint.unwrap_or_else(|| {
std::env::var("LIGHTNINGSTOR_FLAREDB_ENDPOINT")
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
});
let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
let pd_endpoint = pd_endpoint
.or_else(|| std::env::var("LIGHTNINGSTOR_CHAINFIRE_ENDPOINT").ok())
.map(|value| normalize_transport_addr(&value))
.unwrap_or_else(|| endpoint.clone());
let mut clients = Vec::with_capacity(FLAREDB_CLIENT_POOL_SIZE);
for _ in 0..FLAREDB_CLIENT_POOL_SIZE {
let client =
RdbClient::connect_with_pd_namespace(endpoint.clone(), pd_endpoint.clone(), "lightningstor")
.await
.map_err(|e| {
lightningstor_types::Error::StorageError(format!(
"Failed to connect to FlareDB: {}",
e
))
})?;
let client = RdbClient::connect_with_pd_namespace(
endpoint.clone(),
pd_endpoint.clone(),
"lightningstor",
)
.await
.map_err(|e| {
lightningstor_types::Error::StorageError(format!(
"Failed to connect to FlareDB: {}",
e
))
})?;
clients.push(Arc::new(Mutex::new(client)));
}
@ -321,7 +320,11 @@ impl MetadataStore {
Ok((results, next))
}
async fn flaredb_put(clients: &[Arc<Mutex<RdbClient>>], key: &[u8], value: &[u8]) -> Result<()> {
async fn flaredb_put(
clients: &[Arc<Mutex<RdbClient>>],
key: &[u8],
value: &[u8],
) -> Result<()> {
let client = Self::flaredb_client_for_key(clients, key);
let raw_result = {
let mut c = client.lock().await;
@ -443,7 +446,8 @@ impl MetadataStore {
let client = Self::flaredb_scan_client(clients);
let (mut items, next) = match {
let mut c = client.lock().await;
c.raw_scan(start_key.clone(), end_key.clone(), fetch_limit).await
c.raw_scan(start_key.clone(), end_key.clone(), fetch_limit)
.await
} {
Ok((keys, values, next)) => {
let items = keys
@ -697,12 +701,13 @@ impl MetadataStore {
.await
}
StorageBackend::Sql(sql) => {
let prefix_end = String::from_utf8(Self::prefix_end(prefix.as_bytes())).map_err(|e| {
lightningstor_types::Error::StorageError(format!(
"Failed to encode prefix end: {}",
e
))
})?;
let prefix_end =
String::from_utf8(Self::prefix_end(prefix.as_bytes())).map_err(|e| {
lightningstor_types::Error::StorageError(format!(
"Failed to encode prefix end: {}",
e
))
})?;
let fetch_limit = (limit.saturating_add(1)) as i64;
match sql {
SqlStorageBackend::Postgres(pool) => {
@ -908,7 +913,10 @@ impl MetadataStore {
}
fn multipart_bucket_prefix(bucket_id: &BucketId, prefix: &str) -> String {
format!("/lightningstor/multipart/by-bucket/{}/{}", bucket_id, prefix)
format!(
"/lightningstor/multipart/by-bucket/{}/{}",
bucket_id, prefix
)
}
fn multipart_object_key(object_id: &ObjectId) -> String {
@ -955,7 +963,8 @@ impl MetadataStore {
}
pub async fn delete_replicated_repair_task(&self, task_id: &str) -> Result<()> {
self.delete_key(&Self::replicated_repair_task_key(task_id)).await
self.delete_key(&Self::replicated_repair_task_key(task_id))
.await
}
/// Save bucket metadata
@ -1246,7 +1255,8 @@ impl MetadataStore {
))
.await?;
}
self.delete_key(&Self::multipart_upload_key(upload_id)).await
self.delete_key(&Self::multipart_upload_key(upload_id))
.await
}
pub async fn list_multipart_uploads(
@ -1331,7 +1341,8 @@ impl MetadataStore {
}
pub async fn delete_object_multipart_upload(&self, object_id: &ObjectId) -> Result<()> {
self.delete_key(&Self::multipart_object_key(object_id)).await
self.delete_key(&Self::multipart_object_key(object_id))
.await
}
}
@ -1380,13 +1391,11 @@ mod tests {
store.delete_bucket(&bucket).await.unwrap();
assert!(!store.bucket_cache.contains_key(&cache_key));
assert!(!store.bucket_cache.contains_key(&cache_id_key));
assert!(
store
.load_bucket("org-a", "project-a", "bench-bucket")
.await
.unwrap()
.is_none()
);
assert!(store
.load_bucket("org-a", "project-a", "bench-bucket")
.await
.unwrap()
.is_none());
}
#[tokio::test]
@ -1426,13 +1435,11 @@ mod tests {
.await
.unwrap();
assert!(!store.object_cache.contains_key(&cache_key));
assert!(
store
.load_object(&bucket.id, object.key.as_str(), None)
.await
.unwrap()
.is_none()
);
assert!(store
.load_object(&bucket.id, object.key.as_str(), None)
.await
.unwrap()
.is_none());
}
#[tokio::test]
@ -1496,8 +1503,10 @@ mod tests {
);
store.save_bucket(&bucket).await.unwrap();
let upload_a = MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/one.bin").unwrap());
let upload_b = MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/two.bin").unwrap());
let upload_a =
MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/one.bin").unwrap());
let upload_b =
MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/two.bin").unwrap());
let other_bucket = Bucket::new(
BucketName::new("other-bucket").unwrap(),
"org-a",
@ -1505,8 +1514,10 @@ mod tests {
"default",
);
store.save_bucket(&other_bucket).await.unwrap();
let upload_other =
MultipartUpload::new(other_bucket.id.to_string(), ObjectKey::new("a/three.bin").unwrap());
let upload_other = MultipartUpload::new(
other_bucket.id.to_string(),
ObjectKey::new("a/three.bin").unwrap(),
);
store.save_multipart_upload(&upload_a).await.unwrap();
store.save_multipart_upload(&upload_b).await.unwrap();
@ -1543,10 +1554,7 @@ mod tests {
assert_eq!(tasks[0].attempt_count, 1);
assert_eq!(tasks[0].last_error.as_deref(), Some("transient failure"));
store
.delete_replicated_repair_task(&task.id)
.await
.unwrap();
store.delete_replicated_repair_task(&task.id).await.unwrap();
assert!(store
.list_replicated_repair_tasks(10)
.await

69
lightningstor/crates/lightningstor-server/src/s3/auth.rs
View file

@ -3,6 +3,7 @@
//! Implements simplified SigV4 authentication compatible with AWS S3 SDKs and aws-cli.
//! Integrates with IAM for access key validation.
use crate::config::S3AuthConfig;
use crate::tenant::TenantContext;
use axum::{
body::{Body, Bytes},
@ -23,8 +24,6 @@ use tracing::{debug, warn};
use url::form_urlencoded;
type HmacSha256 = Hmac<Sha256>;
const DEFAULT_MAX_AUTH_BODY_BYTES: usize = 1024 * 1024 * 1024;
#[derive(Clone, Debug)]
pub(crate) struct VerifiedBodyBytes(pub Bytes);
@ -49,6 +48,8 @@ pub struct AuthState {
aws_region: String,
/// AWS service name for SigV4 (e.g., s3)
aws_service: String,
/// Maximum request body size to buffer during auth verification
max_auth_body_bytes: usize,
}
pub struct IamClient {
@ -60,6 +61,8 @@ pub struct IamClient {
enum IamClientMode {
Env {
credentials: std::collections::HashMap<String, String>,
default_org_id: Option<String>,
default_project_id: Option<String>,
},
Grpc {
endpoint: String,
@ -83,11 +86,11 @@ struct CachedCredential {
impl IamClient {
/// Create a new IAM client. If an endpoint is supplied, use the IAM gRPC API.
pub fn new(iam_endpoint: Option<String>) -> Self {
let cache_ttl = std::env::var("LIGHTNINGSTOR_S3_IAM_CACHE_TTL_SECS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.map(StdDuration::from_secs)
.unwrap_or_else(|| StdDuration::from_secs(30));
Self::new_with_config(iam_endpoint, &S3AuthConfig::default())
}
pub fn new_with_config(iam_endpoint: Option<String>, config: &S3AuthConfig) -> Self {
let cache_ttl = StdDuration::from_secs(config.iam_cache_ttl_secs);
if let Some(endpoint) = iam_endpoint
.map(|value| normalize_iam_endpoint(&value))
@ -106,6 +109,8 @@ impl IamClient {
Self {
mode: IamClientMode::Env {
credentials: Self::load_env_credentials(),
default_org_id: config.default_org_id.clone(),
default_project_id: config.default_project_id.clone(),
},
credential_cache: Arc::new(RwLock::new(HashMap::new())),
cache_ttl,
@ -160,32 +165,32 @@ impl IamClient {
#[cfg(test)]
fn env_credentials(&self) -> Option<&std::collections::HashMap<String, String>> {
match &self.mode {
IamClientMode::Env { credentials } => Some(credentials),
IamClientMode::Env { credentials, .. } => Some(credentials),
IamClientMode::Grpc { .. } => None,
}
}
fn env_default_tenant() -> (Option<String>, Option<String>) {
let org_id = std::env::var("S3_TENANT_ORG_ID")
.ok()
.or_else(|| std::env::var("S3_ORG_ID").ok())
.or_else(|| Some("default".to_string()));
let project_id = std::env::var("S3_TENANT_PROJECT_ID")
.ok()
.or_else(|| std::env::var("S3_PROJECT_ID").ok())
.or_else(|| Some("default".to_string()));
(org_id, project_id)
fn env_default_tenant(
default_org_id: Option<String>,
default_project_id: Option<String>,
) -> (Option<String>, Option<String>) {
(default_org_id, default_project_id)
}
/// Validate access key and resolve the credential context.
pub async fn get_credential(&self, access_key_id: &str) -> Result<ResolvedCredential, String> {
match &self.mode {
IamClientMode::Env { credentials } => {
IamClientMode::Env {
credentials,
default_org_id,
default_project_id,
} => {
let secret_key = credentials
.get(access_key_id)
.cloned()
.ok_or_else(|| "Access key ID not found".to_string())?;
let (org_id, project_id) = Self::env_default_tenant();
let (org_id, project_id) =
Self::env_default_tenant(default_org_id.clone(), default_project_id.clone());
Ok(ResolvedCredential {
secret_key,
principal_id: access_key_id.to_string(),
@ -318,16 +323,21 @@ fn iam_admin_token() -> Option<String> {
impl AuthState {
/// Create new auth state with IAM integration
pub fn new(iam_endpoint: Option<String>) -> Self {
let iam_client = Some(Arc::new(RwLock::new(IamClient::new(iam_endpoint))));
Self::new_with_config(iam_endpoint, &S3AuthConfig::default())
}
pub fn new_with_config(iam_endpoint: Option<String>, config: &S3AuthConfig) -> Self {
let iam_client = Some(Arc::new(RwLock::new(IamClient::new_with_config(
iam_endpoint,
config,
))));
Self {
iam_client,
enabled: std::env::var("S3_AUTH_ENABLED")
.unwrap_or_else(|_| "true".to_string())
.parse()
.unwrap_or(true),
aws_region: std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string()), // Default S3 region
enabled: config.enabled,
aws_region: config.aws_region.clone(),
aws_service: "s3".to_string(),
max_auth_body_bytes: config.max_auth_body_bytes,
}
}
@ -338,6 +348,7 @@ impl AuthState {
enabled: false,
aws_region: "us-east-1".to_string(),
aws_service: "s3".to_string(),
max_auth_body_bytes: S3AuthConfig::default().max_auth_body_bytes,
}
}
}
@ -438,12 +449,8 @@ pub async fn sigv4_auth_middleware(
let should_buffer_body = should_buffer_auth_body(payload_hash_header.as_deref());
let body_bytes = if should_buffer_body {
let max_body_bytes = std::env::var("S3_MAX_AUTH_BODY_BYTES")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(DEFAULT_MAX_AUTH_BODY_BYTES);
let (parts, body) = request.into_parts();
let body_bytes = match axum::body::to_bytes(body, max_body_bytes).await {
let body_bytes = match axum::body::to_bytes(body, auth_state.max_auth_body_bytes).await {
Ok(b) => b,
Err(e) => {
return error_response(

7
lightningstor/crates/lightningstor-server/src/s3/mod.rs
View file

@ -6,5 +6,8 @@ mod auth;
mod router;
mod xml;
pub use auth::{AuthState, sigv4_auth_middleware};
pub use router::{create_router, create_router_with_auth, create_router_with_state};
pub use auth::{sigv4_auth_middleware, AuthState};
pub use router::{
create_router, create_router_with_auth, create_router_with_auth_config,
create_router_with_state,
};

112
lightningstor/crates/lightningstor-server/src/s3/router.rs
View file

@ -1,5 +1,6 @@
//! S3 API router using Axum
use crate::config::{S3Config, S3PerformanceConfig};
use axum::{
body::{Body, Bytes},
extract::{Request, State},
@ -14,8 +15,8 @@ use futures::{stream, stream::FuturesUnordered, StreamExt};
use http_body_util::BodyExt;
use md5::{Digest, Md5};
use serde::Deserialize;
use std::io;
use sha2::Sha256;
use std::io;
use std::sync::Arc;
use tokio::task::JoinHandle;
@ -38,17 +39,24 @@ use super::xml::{
pub struct S3State {
pub storage: Arc<dyn StorageBackend>,
pub metadata: Arc<MetadataStore>,
pub performance: S3PerformanceConfig,
}
// Keep streamed single-PUT parts aligned with the distributed backend's
// large-object chunking so GET does not degrade into many small serial reads.
const DEFAULT_STREAMING_PUT_THRESHOLD_BYTES: usize = 16 * 1024 * 1024;
const DEFAULT_INLINE_PUT_MAX_BYTES: usize = 128 * 1024 * 1024;
const DEFAULT_MULTIPART_PUT_CONCURRENCY: usize = 4;
impl S3State {
pub fn new(storage: Arc<dyn StorageBackend>, metadata: Arc<MetadataStore>) -> Self {
Self { storage, metadata }
Self::new_with_config(storage, metadata, S3PerformanceConfig::default())
}
pub fn new_with_config(
storage: Arc<dyn StorageBackend>,
metadata: Arc<MetadataStore>,
performance: S3PerformanceConfig,
) -> Self {
Self {
storage,
metadata,
performance,
}
}
}
@ -57,7 +65,12 @@ pub fn create_router_with_state(
storage: Arc<dyn StorageBackend>,
metadata: Arc<MetadataStore>,
) -> Router {
create_router_with_auth_state(storage, metadata, Arc::new(AuthState::new(None)))
create_router_with_auth_state(
storage,
metadata,
Arc::new(AuthState::new(None)),
S3PerformanceConfig::default(),
)
}
/// Create the S3-compatible HTTP router with auth and storage backends
@ -66,15 +79,30 @@ pub fn create_router_with_auth(
metadata: Arc<MetadataStore>,
iam_endpoint: Option<String>,
) -> Router {
create_router_with_auth_state(storage, metadata, Arc::new(AuthState::new(iam_endpoint)))
create_router_with_auth_config(storage, metadata, iam_endpoint, S3Config::default())
}
pub fn create_router_with_auth_config(
storage: Arc<dyn StorageBackend>,
metadata: Arc<MetadataStore>,
iam_endpoint: Option<String>,
config: S3Config,
) -> Router {
create_router_with_auth_state(
storage,
metadata,
Arc::new(AuthState::new_with_config(iam_endpoint, &config.auth)),
config.performance,
)
}
fn create_router_with_auth_state(
storage: Arc<dyn StorageBackend>,
metadata: Arc<MetadataStore>,
auth_state: Arc<AuthState>,
performance: S3PerformanceConfig,
) -> Router {
let state = Arc::new(S3State::new(storage, metadata));
let state = Arc::new(S3State::new_with_config(storage, metadata, performance));
Router::new()
// Catch-all route for ALL operations (including root /)
@ -126,28 +154,16 @@ fn request_tenant(extensions: &axum::http::Extensions) -> TenantContext {
})
}
fn streaming_put_threshold_bytes() -> usize {
std::env::var("LIGHTNINGSTOR_S3_STREAMING_PUT_THRESHOLD_BYTES")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_STREAMING_PUT_THRESHOLD_BYTES)
fn streaming_put_threshold_bytes(state: &Arc<S3State>) -> usize {
state.performance.streaming_put_threshold_bytes
}
fn inline_put_max_bytes() -> usize {
std::env::var("LIGHTNINGSTOR_S3_INLINE_PUT_MAX_BYTES")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_INLINE_PUT_MAX_BYTES)
fn inline_put_max_bytes(state: &Arc<S3State>) -> usize {
state.performance.inline_put_max_bytes
}
fn multipart_put_concurrency() -> usize {
std::env::var("LIGHTNINGSTOR_S3_MULTIPART_PUT_CONCURRENCY")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_MULTIPART_PUT_CONCURRENCY)
fn multipart_put_concurrency(state: &Arc<S3State>) -> usize {
state.performance.multipart_put_concurrency
}
fn request_content_length(headers: &HeaderMap) -> Option<usize> {
@ -751,13 +767,13 @@ async fn put_object(
(body_len, etag, None, Some(body_bytes))
}
None => {
let prepared = if let Some(content_length) =
content_length.filter(|content_length| *content_length <= inline_put_max_bytes())
let prepared = if let Some(content_length) = content_length
.filter(|content_length| *content_length <= inline_put_max_bytes(&state))
{
read_inline_put_body(
body,
verified_payload_hash.as_deref(),
inline_put_max_bytes(),
inline_put_max_bytes(&state),
Some(content_length),
)
.await
@ -948,7 +964,7 @@ async fn stream_put_body(
) -> Result<PreparedPutBody, Response<Body>> {
let verify_payload_hash = expected_payload_hash
.filter(|expected_payload_hash| *expected_payload_hash != "UNSIGNED-PAYLOAD");
let threshold = streaming_put_threshold_bytes();
let threshold = streaming_put_threshold_bytes(state);
let mut buffered = BytesMut::with_capacity(threshold);
let mut full_md5 = Some(Md5::new());
let mut full_sha256 = verify_payload_hash.map(|_| Sha256::new());
@ -958,7 +974,7 @@ async fn stream_put_body(
let mut scheduled_part_numbers = Vec::new();
let mut completed_parts = Vec::new();
let mut in_flight_uploads = FuturesUnordered::new();
let max_in_flight_uploads = multipart_put_concurrency();
let max_in_flight_uploads = multipart_put_concurrency(state);
while let Some(frame) = body.frame().await {
let frame = match frame {
@ -1282,7 +1298,10 @@ fn multipart_object_body(state: Arc<S3State>, object: &Object, upload: Multipart
)));
}
return Ok(Some((bytes.slice(0..body_end), (storage, upload, idx, offset))));
return Ok(Some((
bytes.slice(0..body_end),
(storage, upload, idx, offset),
)));
}
Ok(None)
@ -1374,7 +1393,11 @@ async fn get_object(
);
}
let multipart_upload = match state.metadata.load_object_multipart_upload(&object.id).await {
let multipart_upload = match state
.metadata
.load_object_multipart_upload(&object.id)
.await
{
Ok(upload) => upload,
Err(e) => {
return error_response(
@ -1385,7 +1408,10 @@ async fn get_object(
}
};
let (body, content_length) = if let Some(upload) = multipart_upload {
(multipart_object_body(Arc::clone(&state), &object, upload), object.size as usize)
(
multipart_object_body(Arc::clone(&state), &object, upload),
object.size as usize,
)
} else {
let data = match state.storage.get_object(&object.id).await {
Ok(data) => data,
@ -1653,6 +1679,7 @@ mod tests {
storage,
metadata.clone(),
Arc::new(AuthState::disabled()),
S3PerformanceConfig::default(),
);
std::mem::forget(tempdir);
(router, metadata)
@ -1982,7 +2009,8 @@ mod tests {
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = vec![b'x'; DEFAULT_STREAMING_PUT_THRESHOLD_BYTES + 1024];
let threshold = S3PerformanceConfig::default().streaming_put_threshold_bytes;
let body = vec![b'x'; threshold + 1024];
let response = router
.clone()
.oneshot(
@ -2197,10 +2225,12 @@ mod tests {
async fn large_put_streams_multipart_parts_with_parallel_uploads() {
let storage = Arc::new(DelayedMultipartStorage::new(Duration::from_millis(25)));
let metadata = Arc::new(MetadataStore::new_in_memory());
let performance = S3PerformanceConfig::default();
let router = create_router_with_auth_state(
storage.clone(),
metadata,
Arc::new(AuthState::disabled()),
performance.clone(),
);
let response = router
@ -2216,7 +2246,7 @@ mod tests {
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = vec![b'x'; (DEFAULT_STREAMING_PUT_THRESHOLD_BYTES * 2) + 4096];
let body = vec![b'x'; (performance.streaming_put_threshold_bytes * 2) + 4096];
let response = router
.clone()
.oneshot(
@ -2250,10 +2280,12 @@ mod tests {
async fn moderate_put_with_content_length_stays_inline() {
let storage = Arc::new(DelayedMultipartStorage::new(Duration::from_millis(25)));
let metadata = Arc::new(MetadataStore::new_in_memory());
let performance = S3PerformanceConfig::default();
let router = create_router_with_auth_state(
storage.clone(),
metadata.clone(),
Arc::new(AuthState::disabled()),
performance.clone(),
);
let response = router
@ -2269,7 +2301,7 @@ mod tests {
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = vec![b'y'; DEFAULT_STREAMING_PUT_THRESHOLD_BYTES + 4096];
let body = vec![b'y'; performance.streaming_put_threshold_bytes + 4096];
let response = router
.clone()
.oneshot(

5
lightningstor/crates/lightningstor-server/src/s3/xml.rs
View file

@ -170,5 +170,8 @@ pub struct CompleteMultipartUploadResult {
/// Convert to XML with declaration
pub fn to_xml<T: Serialize>(value: &T) -> Result<String, quick_xml::DeError> {
let xml = xml_to_string(value)?;
Ok(format!("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n{}", xml))
Ok(format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n{}",
xml
))
}

24
nightlight/crates/nightlight-server/src/config.rs
View file

@ -6,6 +6,7 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::Path;
/// Main server configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
@ -123,7 +124,7 @@ pub struct TlsConfig {
impl Config {
/// Load configuration from a YAML file
pub fn from_file(path: &str) -> Result<Self> {
pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
let content = fs::read_to_string(path)?;
let config = serde_yaml::from_str(&content)?;
Ok(config)
@ -136,27 +137,6 @@ impl Config {
fs::write(path, content)?;
Ok(())
}
/// Apply environment variable overrides
///
/// This allows NixOS service module to override configuration via environment variables.
/// Environment variables take precedence over configuration file values.
pub fn apply_env_overrides(&mut self) {
if let Ok(val) = std::env::var("NIGHTLIGHT_HTTP_ADDR") {
self.server.http_addr = val;
}
if let Ok(val) = std::env::var("NIGHTLIGHT_GRPC_ADDR") {
self.server.grpc_addr = val;
}
if let Ok(val) = std::env::var("NIGHTLIGHT_DATA_DIR") {
self.storage.data_dir = val;
}
if let Ok(val) = std::env::var("NIGHTLIGHT_RETENTION_DAYS") {
if let Ok(days) = val.parse() {
self.storage.retention_days = days;
}
}
}
}
impl Default for Config {

49
nightlight/crates/nightlight-server/src/grpc.rs
View file

@ -183,7 +183,11 @@ impl Admin for AdminServiceImpl {
_request: Request<HealthRequest>,
) -> Result<Response<HealthResponse>, Status> {
let storage_result = self.storage.stats().await;
let status = if storage_result.is_ok() { "ok" } else { "degraded" };
let status = if storage_result.is_ok() {
"ok"
} else {
"degraded"
};
let storage_message = match &storage_result {
Ok(_) => "storage ready".to_string(),
Err(error) => error.to_string(),
@ -253,7 +257,9 @@ impl Admin for AdminServiceImpl {
version: env!("CARGO_PKG_VERSION").to_string(),
commit: option_env!("GIT_COMMIT").unwrap_or("unknown").to_string(),
build_time: option_env!("BUILD_TIME").unwrap_or("unknown").to_string(),
rust_version: option_env!("RUSTC_VERSION").unwrap_or("unknown").to_string(),
rust_version: option_env!("RUSTC_VERSION")
.unwrap_or("unknown")
.to_string(),
target: format!("{}-{}", std::env::consts::ARCH, std::env::consts::OS),
}))
}
@ -330,9 +336,7 @@ mod tests {
use super::*;
use crate::ingestion::IngestionService;
use crate::storage::Storage;
use nightlight_api::nightlight::{
InstantQueryRequest, LabelValuesRequest, SeriesQueryRequest,
};
use nightlight_api::nightlight::{InstantQueryRequest, LabelValuesRequest, SeriesQueryRequest};
use nightlight_api::prometheus::{Label, Sample, TimeSeries, WriteRequest};
#[tokio::test]
@ -380,7 +384,10 @@ mod tests {
data.result[0].metric.get("__name__").map(String::as_str),
Some("grpc_metric")
);
assert_eq!(data.result[0].value.as_ref().map(|value| value.value), Some(12.5));
assert_eq!(
data.result[0].value.as_ref().map(|value| value.value),
Some(12.5)
);
}
#[tokio::test]
@ -452,7 +459,10 @@ mod tests {
.unwrap()
.into_inner();
assert_eq!(label_values.status, "success");
assert_eq!(label_values.data, vec!["api".to_string(), "worker".to_string()]);
assert_eq!(
label_values.data,
vec!["api".to_string(), "worker".to_string()]
);
}
#[tokio::test]
@ -477,26 +487,33 @@ mod tests {
.unwrap();
let query = QueryService::from_storage(storage.queryable());
query.execute_instant_query("admin_metric", 2_000).await.unwrap();
query
.execute_instant_query("admin_metric", 2_000)
.await
.unwrap();
let admin = AdminServiceImpl::new(
Arc::clone(&storage),
ingestion.metrics(),
query.metrics(),
);
let admin =
AdminServiceImpl::new(Arc::clone(&storage), ingestion.metrics(), query.metrics());
let stats = admin
.stats(Request::new(StatsRequest {}))
.await
.unwrap()
.into_inner();
assert_eq!(stats.storage.as_ref().map(|value| value.total_samples), Some(1));
assert_eq!(
stats.ingestion
stats.storage.as_ref().map(|value| value.total_samples),
Some(1)
);
assert_eq!(
stats
.ingestion
.as_ref()
.map(|value| value.samples_ingested_total),
Some(1)
);
assert_eq!(stats.query.as_ref().map(|value| value.queries_total), Some(1));
assert_eq!(
stats.query.as_ref().map(|value| value.queries_total),
Some(1)
);
}
}

53
nightlight/crates/nightlight-server/src/ingestion.rs
View file

@ -15,8 +15,8 @@ use nightlight_api::prometheus::{Label, WriteRequest};
use nightlight_types::Error;
use prost::Message;
use snap::raw::Decoder as SnappyDecoder;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Instant;
use tracing::{debug, error, info, warn};
@ -113,9 +113,8 @@ impl IngestionService {
}
// Store series with samples in shared storage
let series_id = nightlight_types::SeriesId(
compute_series_fingerprint(&internal_labels)
);
let series_id =
nightlight_types::SeriesId(compute_series_fingerprint(&internal_labels));
let time_series = nightlight_types::TimeSeries {
id: series_id,
@ -178,11 +177,11 @@ impl IngestionMetrics {
}
/// Axum handler for /api/v1/write endpoint
async fn handle_remote_write(
State(service): State<IngestionService>,
body: Bytes,
) -> Response {
service.metrics.requests_total.fetch_add(1, Ordering::Relaxed);
async fn handle_remote_write(State(service): State<IngestionService>, body: Bytes) -> Response {
service
.metrics
.requests_total
.fetch_add(1, Ordering::Relaxed);
debug!("Received remote_write request, size: {} bytes", body.len());
@ -225,17 +224,26 @@ async fn handle_remote_write(
}
Err(Error::Storage(msg)) if msg.contains("buffer full") => {
warn!("Write buffer full, returning 429");
service.metrics.requests_failed.fetch_add(1, Ordering::Relaxed);
service
.metrics
.requests_failed
.fetch_add(1, Ordering::Relaxed);
IngestionError::Backpressure.into_response()
}
Err(Error::InvalidLabel(msg)) => {
warn!("Invalid labels: {}", msg);
service.metrics.requests_failed.fetch_add(1, Ordering::Relaxed);
service
.metrics
.requests_failed
.fetch_add(1, Ordering::Relaxed);
IngestionError::InvalidLabels.into_response()
}
Err(e) => {
error!("Failed to process write request: {}", e);
service.metrics.requests_failed.fetch_add(1, Ordering::Relaxed);
service
.metrics
.requests_failed
.fetch_add(1, Ordering::Relaxed);
IngestionError::StorageError.into_response()
}
}
@ -285,7 +293,11 @@ fn validate_labels(labels: Vec<Label>) -> Result<Vec<Label>, Error> {
}
// Label names must contain only [a-zA-Z0-9_]
if !label.name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
if !label
.name
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_')
{
return Err(Error::InvalidLabel(format!(
"Invalid label name '{}': must contain only [a-zA-Z0-9_]",
label.name
@ -337,15 +349,12 @@ impl IntoResponse for IngestionError {
IngestionError::InvalidProtobuf => {
(StatusCode::BAD_REQUEST, "Invalid protobuf encoding")
}
IngestionError::InvalidLabels => {
(StatusCode::BAD_REQUEST, "Invalid metric labels")
}
IngestionError::StorageError => {
(StatusCode::INTERNAL_SERVER_ERROR, "Storage error")
}
IngestionError::Backpressure => {
(StatusCode::TOO_MANY_REQUESTS, "Write buffer full, retry later")
}
IngestionError::InvalidLabels => (StatusCode::BAD_REQUEST, "Invalid metric labels"),
IngestionError::StorageError => (StatusCode::INTERNAL_SERVER_ERROR, "Storage error"),
IngestionError::Backpressure => (
StatusCode::TOO_MANY_REQUESTS,
"Write buffer full, retry later",
),
};
(status, message).into_response()

31
nightlight/crates/nightlight-server/src/main.rs
View file

@ -12,6 +12,7 @@ use std::time::Duration;
use anyhow::Result;
use axum::{routing::get, Router};
use clap::Parser;
use nightlight_api::nightlight::admin_server::AdminServer;
use nightlight_api::nightlight::metric_query_server::MetricQueryServer;
use tokio::time::MissedTickBehavior;
@ -33,8 +34,18 @@ use storage::Storage;
const DEFAULT_SNAPSHOT_INTERVAL_SECS: u64 = 30;
#[derive(Parser, Debug)]
#[command(name = "nightlight-server")]
struct Args {
/// Configuration file path
#[arg(short, long, default_value = "nightlight.yaml")]
config: std::path::PathBuf,
}
#[tokio::main]
async fn main() -> Result<()> {
let args = Args::parse();
tracing_subscriber::fmt()
.with_max_level(Level::INFO)
.with_target(false)
@ -44,17 +55,20 @@ async fn main() -> Result<()> {
info!("Nightlight server starting");
info!("Version: {}", env!("CARGO_PKG_VERSION"));
let mut config = match Config::from_file("config.yaml") {
let config = match Config::from_file(&args.config) {
Ok(config) => {
info!("Configuration loaded from config.yaml");
info!("Configuration loaded from {}", args.config.display());
config
}
Err(error) => {
info!("Failed to load config.yaml: {}, using defaults", error);
info!(
"Failed to load {}: {}, using defaults",
args.config.display(),
error
);
Config::default()
}
};
config.apply_env_overrides();
if config.tls.is_some() {
warn!("Nightlight TLS configuration is currently ignored; starting plaintext listeners");
@ -133,7 +147,9 @@ async fn main() -> Result<()> {
info!(" - Admin.Health / Stats / BuildInfo");
let shutdown = async {
tokio::signal::ctrl_c().await.expect("failed to install Ctrl+C handler");
tokio::signal::ctrl_c()
.await
.expect("failed to install Ctrl+C handler");
};
tokio::pin!(shutdown);
@ -163,8 +179,9 @@ async fn maintenance_loop(
config: StorageConfig,
mut shutdown: tokio::sync::broadcast::Receiver<()>,
) {
let snapshot_interval_secs =
config.compaction_interval_seconds.clamp(5, DEFAULT_SNAPSHOT_INTERVAL_SECS);
let snapshot_interval_secs = config
.compaction_interval_seconds
.clamp(5, DEFAULT_SNAPSHOT_INTERVAL_SECS);
let mut snapshot_interval = tokio::time::interval(Duration::from_secs(snapshot_interval_secs));
snapshot_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);

3
nix/modules/chainfire.nix
View file

@ -49,7 +49,7 @@ let
raft_addr = raftAddr;
})
cfg.initialPeers;
chainfireConfigFile = tomlFormat.generate "chainfire.toml" {
generatedConfig = {
node = {
id = numericId cfg.nodeId;
name = cfg.nodeId;
@ -73,6 +73,7 @@ let
role = cfg.raftRole;
};
};
chainfireConfigFile = tomlFormat.generate "chainfire.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
in
{
options.services.chainfire = {

37
nix/modules/fiberlb.nix
View file

@ -75,6 +75,8 @@ let
fiberlbBaseConfig = {
grpc_addr = "0.0.0.0:${toString cfg.port}";
log_level = "info";
metadata_backend = cfg.metadataBackend;
single_node = cfg.singleNode;
auth = {
iam_server_addr =
if cfg.iamAddr != null
@ -93,6 +95,8 @@ let
vip_ownership = {
enabled = cfg.vipOwnership.enable;
interface = cfg.vipOwnership.interface;
} // lib.optionalAttrs (cfg.vipOwnership.ipCommand != null) {
ip_command = cfg.vipOwnership.ipCommand;
};
} // lib.optionalAttrs cfg.bgp.enable {
bgp =
@ -127,6 +131,15 @@ let
// lib.optionalAttrs (cfg.bgp.nextHop != null) {
next_hop = cfg.bgp.nextHop;
};
}
// lib.optionalAttrs (cfg.flaredbAddr != null) {
flaredb_endpoint = cfg.flaredbAddr;
}
// lib.optionalAttrs (normalizedDatabaseUrl != null) {
metadata_database_url = normalizedDatabaseUrl;
}
// lib.optionalAttrs (cfg.chainfireAddr != null) {
chainfire_endpoint = "http://${cfg.chainfireAddr}";
};
fiberlbConfigFile = tomlFormat.generate "fiberlb.toml" (lib.recursiveUpdate fiberlbBaseConfig cfg.settings);
flaredbDependencies = lib.optional (cfg.metadataBackend == "flaredb") "flaredb.service";
@ -222,6 +235,13 @@ in
default = "lo";
description = "Interface where FiberLB should claim VIP /32 addresses.";
};
ipCommand = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Optional explicit path to the `ip` command used for VIP ownership.";
example = "/run/current-system/sw/bin/ip";
};
};
vipDrain = {
@ -367,22 +387,7 @@ in
AmbientCapabilities = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
CapabilityBoundingSet = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
# Environment variables for service endpoints
Environment = [
"RUST_LOG=info"
"FIBERLB_FLAREDB_ENDPOINT=${if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479"}"
"FIBERLB_METADATA_BACKEND=${cfg.metadataBackend}"
] ++ lib.optional (normalizedDatabaseUrl != null) "FIBERLB_METADATA_DATABASE_URL=${normalizedDatabaseUrl}"
++ lib.optional cfg.singleNode "FIBERLB_SINGLE_NODE=true"
++ lib.optional (cfg.chainfireAddr != null) "FIBERLB_CHAINFIRE_ENDPOINT=http://${cfg.chainfireAddr}";
# Start command
ExecStart = lib.concatStringsSep " " ([
"${cfg.package}/bin/fiberlb"
"--config ${fiberlbConfigFile}"
"--grpc-addr 0.0.0.0:${toString cfg.port}"
"--flaredb-endpoint ${if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479"}"
]);
ExecStart = "${cfg.package}/bin/fiberlb --config ${fiberlbConfigFile}";
};
};
};

2
nix/modules/flashdns.nix
View file

@ -25,7 +25,7 @@ let
// lib.optionalAttrs (cfg.databaseUrl != null) {
metadata_database_url = cfg.databaseUrl;
};
configFile = tomlFormat.generate "flashdns.toml" generatedConfig;
configFile = tomlFormat.generate "flashdns.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
in
{
options.services.flashdns = {

36
nix/modules/iam.nix
View file

@ -3,12 +3,19 @@
let
cfg = config.services.iam;
tomlFormat = pkgs.formats.toml { };
iamConfigFile = tomlFormat.generate "iam.toml" {
generatedConfig = {
server = {
addr = "0.0.0.0:${toString cfg.port}";
http_addr = "0.0.0.0:${toString cfg.httpPort}";
};
logging.level = "info";
admin = {
allow_unauthenticated = cfg.allowUnauthenticatedAdmin;
};
dev = {
allow_random_signing_key = cfg.allowRandomSigningKey;
allow_memory_backend = cfg.storeBackend == "memory";
};
store = {
backend = cfg.storeBackend;
flaredb_endpoint =
@ -25,6 +32,7 @@ let
chainfire_endpoint = cfg.chainfireAddr;
};
};
iamConfigFile = tomlFormat.generate "iam.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
in
{
options.services.iam = {
@ -87,6 +95,18 @@ in
description = "Admin token injected as IAM_ADMIN_TOKEN for privileged IAM APIs.";
};
allowRandomSigningKey = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Allow IAM to generate a random signing key when authn.internal_token.signing_key is unset (dev only).";
};
allowUnauthenticatedAdmin = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Allow privileged admin APIs without an admin token (dev only).";
};
settings = lib.mkOption {
type = lib.types.attrs;
default = {};
@ -119,20 +139,6 @@ in
wants = [ "chainfire.service" "flaredb.service" ];
environment = lib.mkMerge [
{
CHAINFIRE_ENDPOINT = if cfg.chainfireAddr != null then cfg.chainfireAddr else "127.0.0.1:2379";
FLAREDB_ENDPOINT = if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479";
IAM_STORE_BACKEND = cfg.storeBackend;
}
(lib.mkIf (cfg.databaseUrl != null) {
IAM_DATABASE_URL = cfg.databaseUrl;
})
(lib.mkIf cfg.singleNode {
IAM_SINGLE_NODE = "true";
})
(lib.mkIf (cfg.storeBackend == "memory") {
IAM_ALLOW_MEMORY_BACKEND = "1";
})
(lib.mkIf (cfg.adminToken != null) {
IAM_ADMIN_TOKEN = cfg.adminToken;
})

80
nix/modules/k8shost.nix
View file

@ -2,6 +2,59 @@
let
cfg = config.services.k8shost;
tomlFormat = pkgs.formats.toml { };
generatedConfig = {
server = {
addr = "0.0.0.0:${toString cfg.port}";
http_addr = "127.0.0.1:${toString cfg.httpPort}";
log_level = "info";
};
flaredb = {
pd_addr = cfg.flaredbPdAddr;
direct_addr = cfg.flaredbDirectAddr;
};
chainfire = {
endpoint = cfg.chainfireAddr;
};
iam = {
server_addr =
if cfg.iamAddr != null
then cfg.iamAddr
else "http://127.0.0.1:50080";
};
creditservice =
lib.optionalAttrs
(
cfg.creditserviceAddr != null
|| ((config.services ? creditservice) && config.services.creditservice.enable)
)
{
server_addr =
if cfg.creditserviceAddr != null
then cfg.creditserviceAddr
else "http://127.0.0.1:${toString config.services.creditservice.grpcPort}";
};
fiberlb = {
server_addr =
if cfg.fiberlbAddr != null
then cfg.fiberlbAddr
else "http://127.0.0.1:50085";
};
flashdns = {
server_addr =
if cfg.flashdnsAddr != null
then cfg.flashdnsAddr
else "http://127.0.0.1:50084";
};
prismnet = {
server_addr =
if cfg.prismnetAddr != null
then cfg.prismnetAddr
else "http://127.0.0.1:50081";
};
};
configFile = tomlFormat.generate "k8shost.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
configPath = "/etc/k8shost/k8shost.toml";
in
{
options.services.k8shost = {
@ -26,6 +79,13 @@ in
example = "http://10.0.0.1:50080";
};
creditserviceAddr = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "CreditService endpoint address (http://host:port) for pod admission and scheduler quota enforcement.";
example = "http://10.0.0.1:3010";
};
chainfireAddr = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -123,24 +183,10 @@ in
ProtectHome = true;
ReadWritePaths = [ cfg.dataDir ];
# Environment variables for service endpoints
Environment = [
"RUST_LOG=info"
];
# Start command
ExecStart = lib.concatStringsSep " " ([
"${cfg.package}/bin/k8shost-server"
"--addr 0.0.0.0:${toString cfg.port}"
"--http-addr 127.0.0.1:${toString cfg.httpPort}"
] ++ lib.optional (cfg.iamAddr != null) "--iam-server-addr ${cfg.iamAddr}"
++ lib.optional (cfg.chainfireAddr != null) "--chainfire-endpoint ${cfg.chainfireAddr}"
++ lib.optional (cfg.prismnetAddr != null) "--prismnet-server-addr ${cfg.prismnetAddr}"
++ lib.optional (cfg.flaredbPdAddr != null) "--flaredb-pd-addr ${cfg.flaredbPdAddr}"
++ lib.optional (cfg.flaredbDirectAddr != null) "--flaredb-direct-addr ${cfg.flaredbDirectAddr}"
++ lib.optional (cfg.fiberlbAddr != null) "--fiberlb-server-addr ${cfg.fiberlbAddr}"
++ lib.optional (cfg.flashdnsAddr != null) "--flashdns-server-addr ${cfg.flashdnsAddr}");
ExecStart = "${cfg.package}/bin/k8shost-server --config ${configPath}";
};
};
environment.etc."k8shost/k8shost.toml".source = configFile;
};
}

88
nix/modules/lightningstor.nix
View file

@ -73,6 +73,22 @@ let
// lib.optionalAttrs (cfg.distributedRegistryEndpoint != null) {
registry_endpoint = cfg.distributedRegistryEndpoint;
};
s3 = {
auth = {
enabled = cfg.s3AuthEnabled;
aws_region = cfg.s3AwsRegion;
iam_cache_ttl_secs = cfg.s3IamCacheTtlSecs;
default_org_id = cfg.s3DefaultOrgId;
default_project_id = cfg.s3DefaultProjectId;
max_auth_body_bytes = cfg.s3MaxAuthBodyBytes;
};
performance = {
streaming_put_threshold_bytes = cfg.s3StreamingPutThresholdBytes;
inline_put_max_bytes = cfg.s3InlinePutMaxBytes;
multipart_put_concurrency = cfg.s3MultipartPutConcurrency;
multipart_fetch_concurrency = cfg.s3MultipartFetchConcurrency;
};
};
}
// lib.optionalAttrs (cfg.flaredbAddr != null) {
flaredb_endpoint = cfg.flaredbAddr;
@ -321,6 +337,54 @@ in
description = "Maximum concurrent multipart GET part fetches.";
};
s3AuthEnabled = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Enable S3 Signature V4 authentication.";
};
s3AwsRegion = lib.mkOption {
type = lib.types.str;
default = "us-east-1";
description = "AWS region name exposed by the S3 compatibility layer.";
};
s3IamCacheTtlSecs = lib.mkOption {
type = lib.types.int;
default = 30;
description = "Seconds to cache IAM-backed S3 credential lookups.";
};
s3DefaultOrgId = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "default";
description = "Default org ID used for static S3 credentials.";
};
s3DefaultProjectId = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "default";
description = "Default project ID used for static S3 credentials.";
};
s3MaxAuthBodyBytes = lib.mkOption {
type = lib.types.int;
default = 1024 * 1024 * 1024;
description = "Maximum request body size buffered during S3 auth verification.";
};
s3AccessKeyId = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Optional static S3 access key ID injected via environment for dev/test.";
};
s3SecretKey = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Optional static S3 secret key injected via environment for dev/test.";
};
databaseUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -360,6 +424,13 @@ in
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = (cfg.s3AccessKeyId == null) == (cfg.s3SecretKey == null);
message = "services.lightningstor.s3AccessKeyId and services.lightningstor.s3SecretKey must be set together";
}
];
users.users.lightningstor = {
isSystemUser = true;
group = "lightningstor";
@ -391,17 +462,12 @@ in
ExecStart = execStart;
};
environment = {
RUST_LOG = "info";
LIGHTNINGSTOR_S3_STREAMING_PUT_THRESHOLD_BYTES =
toString cfg.s3StreamingPutThresholdBytes;
LIGHTNINGSTOR_S3_INLINE_PUT_MAX_BYTES =
toString cfg.s3InlinePutMaxBytes;
LIGHTNINGSTOR_S3_MULTIPART_PUT_CONCURRENCY =
toString cfg.s3MultipartPutConcurrency;
LIGHTNINGSTOR_S3_MULTIPART_FETCH_CONCURRENCY =
toString cfg.s3MultipartFetchConcurrency;
};
environment = lib.mkMerge [
(lib.mkIf (cfg.s3AccessKeyId != null) {
S3_ACCESS_KEY_ID = cfg.s3AccessKeyId;
S3_SECRET_KEY = cfg.s3SecretKey;
})
];
};
};
}

28
nix/modules/nightlight.nix
View file

@ -2,6 +2,19 @@
let
cfg = config.services.nightlight;
yamlFormat = pkgs.formats.yaml { };
generatedConfig = {
server = {
grpc_addr = "0.0.0.0:${toString cfg.grpcPort}";
http_addr = "0.0.0.0:${toString cfg.httpPort}";
};
storage = {
data_dir = toString cfg.dataDir;
retention_days = cfg.retentionDays;
};
};
configFile = yamlFormat.generate "nightlight.yaml" (lib.recursiveUpdate generatedConfig cfg.settings);
configPath = "/etc/nightlight/nightlight.yaml";
in
{
options.services.nightlight = {
@ -79,19 +92,10 @@ in
ProtectHome = true;
ReadWritePaths = [ cfg.dataDir ];
# Start command
# Note: nightlight-server uses a config file, so we'll pass data-dir directly
# The config will be auto-generated or use defaults from Config::default()
ExecStart = "${cfg.package}/bin/nightlight-server";
# Environment variables for configuration
Environment = [
"NIGHTLIGHT_HTTP_ADDR=0.0.0.0:${toString cfg.httpPort}"
"NIGHTLIGHT_GRPC_ADDR=0.0.0.0:${toString cfg.grpcPort}"
"NIGHTLIGHT_DATA_DIR=${cfg.dataDir}"
"NIGHTLIGHT_RETENTION_DAYS=${toString cfg.retentionDays}"
];
ExecStart = "${cfg.package}/bin/nightlight-server --config ${configPath}";
};
};
environment.etc."nightlight/nightlight.yaml".source = configFile;
};
}

134
nix/modules/plasmavmc.nix
View file

@ -27,7 +27,7 @@ let
else if cfg.coronafsEndpoint != null then cfg.coronafsEndpoint
else null;
tomlFormat = pkgs.formats.toml { };
plasmavmcConfigFile = tomlFormat.generate "plasmavmc.toml" {
generatedConfig = {
addr = "0.0.0.0:${toString cfg.port}";
http_addr = "0.0.0.0:${toString cfg.httpPort}";
log_level = "info";
@ -37,7 +37,78 @@ let
then cfg.iamAddr
else "127.0.0.1:50080";
};
kvm = {
qemu_path = "${pkgs.qemu}/bin/qemu-system-x86_64";
runtime_dir = "/run/libvirt/plasmavmc";
};
storage = {
backend = "flaredb";
flaredb_endpoint =
if cfg.flaredbAddr != null
then cfg.flaredbAddr
else "127.0.0.1:2479";
} // lib.optionalAttrs (cfg.chainfireAddr != null) {
chainfire_endpoint = "http://${cfg.chainfireAddr}";
};
agent =
{
shared_live_migration = cfg.sharedLiveMigration;
}
// lib.optionalAttrs (cfg.mode != "server") {
node_id = cfg.nodeId;
node_name = cfg.nodeName;
heartbeat_interval_secs = cfg.heartbeatIntervalSeconds;
}
// lib.optionalAttrs (cfg.controlPlaneAddr != null) {
control_plane_addr = cfg.controlPlaneAddr;
}
// lib.optionalAttrs (cfg.advertiseAddr != null) {
advertise_endpoint = cfg.advertiseAddr;
};
integrations =
lib.optionalAttrs (cfg.prismnetAddr != null) {
prismnet_endpoint = "http://${cfg.prismnetAddr}";
};
watcher =
lib.optionalAttrs (cfg.chainfireAddr != null) {
enabled = true;
};
health =
lib.optionalAttrs (cfg.mode == "server") {
node_monitor_interval_secs = 5;
node_heartbeat_timeout_secs = 30;
};
artifacts =
{
image_cache_dir = "${toString cfg.dataDir}/images";
}
// lib.optionalAttrs (cfg.lightningstorAddr != null) {
lightningstor_endpoint = cfg.lightningstorAddr;
};
volumes = {
managed_volume_root = toString cfg.managedVolumeRoot;
coronafs =
(lib.optionalAttrs (effectiveCoronafsControllerEndpoint != null) {
controller_endpoint = effectiveCoronafsControllerEndpoint;
})
// (lib.optionalAttrs (effectiveCoronafsNodeEndpoint != null) {
node_endpoint = effectiveCoronafsNodeEndpoint;
})
// (lib.optionalAttrs (cfg.coronafsEndpoint != null) {
endpoint = cfg.coronafsEndpoint;
})
// {
node_local_attach = cfg.coronafsNodeLocalAttach || cfg.experimentalCoronafsNodeLocalAttach;
};
ceph =
(lib.optionalAttrs (cfg.cephMonitors != [ ]) {
monitors = cfg.cephMonitors;
cluster_id = cfg.cephClusterId;
user = cfg.cephUser;
});
};
};
plasmavmcConfigFile = tomlFormat.generate "plasmavmc.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
in
{
options.services.plasmavmc = {
@ -286,64 +357,9 @@ in
exit 1
'';
environment = lib.mkMerge [
{
PLASMAVMC_MODE = cfg.mode;
PLASMAVMC_STORAGE_BACKEND = "flaredb";
PLASMAVMC_FLAREDB_ENDPOINT = if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479";
PLASMAVMC_QEMU_PATH = "${pkgs.qemu}/bin/qemu-system-x86_64";
PLASMAVMC_RUNTIME_DIR = "/run/libvirt/plasmavmc";
PLASMAVMC_IMAGE_CACHE_DIR = "${toString cfg.dataDir}/images";
PLASMAVMC_MANAGED_VOLUME_ROOT = toString cfg.managedVolumeRoot;
PLASMAVMC_SHARED_LIVE_MIGRATION = lib.boolToString cfg.sharedLiveMigration;
}
(lib.mkIf (cfg.prismnetAddr != null) {
PRISMNET_ENDPOINT = "http://${cfg.prismnetAddr}";
})
(lib.mkIf (cfg.chainfireAddr != null) {
PLASMAVMC_CHAINFIRE_ENDPOINT = "http://${cfg.chainfireAddr}";
PLASMAVMC_STATE_WATCHER = "1";
})
(lib.mkIf (cfg.lightningstorAddr != null) {
PLASMAVMC_LIGHTNINGSTOR_ENDPOINT = cfg.lightningstorAddr;
})
(lib.mkIf (effectiveCoronafsControllerEndpoint != null) {
PLASMAVMC_CORONAFS_CONTROLLER_ENDPOINT = effectiveCoronafsControllerEndpoint;
})
(lib.mkIf (effectiveCoronafsNodeEndpoint != null) {
PLASMAVMC_CORONAFS_NODE_ENDPOINT = effectiveCoronafsNodeEndpoint;
})
(lib.mkIf (cfg.coronafsNodeLocalAttach || cfg.experimentalCoronafsNodeLocalAttach) {
PLASMAVMC_CORONAFS_NODE_LOCAL_ATTACH = "1";
PLASMAVMC_CORONAFS_ENABLE_EXPERIMENTAL_NODE_LOCAL_ATTACH = "1";
})
(lib.mkIf (cfg.coronafsEndpoint != null) {
PLASMAVMC_CORONAFS_ENDPOINT = cfg.coronafsEndpoint;
})
(lib.mkIf (cfg.cephMonitors != [ ]) {
PLASMAVMC_CEPH_MONITORS = lib.concatStringsSep "," cfg.cephMonitors;
PLASMAVMC_CEPH_CLUSTER_ID = cfg.cephClusterId;
PLASMAVMC_CEPH_USER = cfg.cephUser;
})
(lib.mkIf (cfg.cephSecret != null) {
PLASMAVMC_CEPH_SECRET = cfg.cephSecret;
})
(lib.mkIf (cfg.mode != "server") {
PLASMAVMC_NODE_ID = cfg.nodeId;
PLASMAVMC_NODE_NAME = cfg.nodeName;
PLASMAVMC_NODE_HEARTBEAT_INTERVAL_SECS = toString cfg.heartbeatIntervalSeconds;
})
(lib.mkIf (cfg.controlPlaneAddr != null) {
PLASMAVMC_CONTROL_PLANE_ADDR = cfg.controlPlaneAddr;
})
(lib.mkIf (cfg.advertiseAddr != null) {
PLASMAVMC_ENDPOINT_ADVERTISE = cfg.advertiseAddr;
})
(lib.mkIf (cfg.mode == "server") {
PLASMAVMC_NODE_HEALTH_MONITOR_INTERVAL_SECS = "5";
PLASMAVMC_NODE_HEARTBEAT_TIMEOUT_SECS = "30";
})
];
environment = lib.optionalAttrs (cfg.cephSecret != null) {
PLASMAVMC_CEPH_SECRET = cfg.cephSecret;
};
serviceConfig = {
Type = "simple";

37
nix/modules/prismnet.nix
View file

@ -3,17 +3,29 @@
let
cfg = config.services.prismnet;
tomlFormat = pkgs.formats.toml { };
prismnetConfigFile = tomlFormat.generate "prismnet.toml" {
generatedConfig = {
grpc_addr = "0.0.0.0:${toString cfg.port}";
http_addr = "0.0.0.0:${toString cfg.httpPort}";
log_level = "info";
metadata_backend = cfg.metadataBackend;
single_node = cfg.singleNode;
auth = {
iam_server_addr =
if cfg.iamAddr != null
then cfg.iamAddr
else "127.0.0.1:50080";
};
}
// lib.optionalAttrs (cfg.chainfireAddr != null) {
chainfire_endpoint = "http://${cfg.chainfireAddr}";
}
// lib.optionalAttrs (cfg.flaredbAddr != null) {
flaredb_endpoint = cfg.flaredbAddr;
}
// lib.optionalAttrs (cfg.databaseUrl != null) {
metadata_database_url = cfg.databaseUrl;
};
prismnetConfigFile = tomlFormat.generate "prismnet.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
in
{
options.services.prismnet = {
@ -108,22 +120,6 @@ in
after = [ "network.target" "iam.service" "flaredb.service" "chainfire.service" ];
wants = [ "iam.service" "flaredb.service" "chainfire.service" ];
environment = lib.mkMerge [
{
PRISMNET_FLAREDB_ENDPOINT = if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479";
PRISMNET_METADATA_BACKEND = cfg.metadataBackend;
}
(lib.mkIf (cfg.databaseUrl != null) {
PRISMNET_METADATA_DATABASE_URL = cfg.databaseUrl;
})
(lib.mkIf cfg.singleNode {
PRISMNET_SINGLE_NODE = "1";
})
(lib.mkIf (cfg.chainfireAddr != null) {
PRISMNET_CHAINFIRE_ENDPOINT = "http://${cfg.chainfireAddr}";
})
];
serviceConfig = {
Type = "simple";
User = "prismnet";
@ -143,12 +139,7 @@ in
ReadWritePaths = [ cfg.dataDir ];
# Start command
ExecStart = lib.concatStringsSep " " [
"${cfg.package}/bin/prismnet-server"
"--config ${prismnetConfigFile}"
"--grpc-addr 0.0.0.0:${toString cfg.port}"
"--flaredb-endpoint ${if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479"}"
];
ExecStart = "${cfg.package}/bin/prismnet-server --config ${prismnetConfigFile}";
};
};
};

8
nix/templates/plasmacloud-single-node.nix
View file

@ -15,17 +15,13 @@
services.flaredb = {
enable = true;
dataDir = "/var/lib/flaredb";
settings = {
chainfire_endpoint = "http://127.0.0.1:${toString config.services.chainfire.port}";
};
pdAddr = "127.0.0.1:${toString config.services.chainfire.port}";
};
services.iam = {
enable = true;
dataDir = "/var/lib/iam";
settings = {
flaredb_endpoint = "http://127.0.0.1:${toString config.services.flaredb.port}";
};
flaredbAddr = "127.0.0.1:${toString config.services.flaredb.port}";
};
services.plasmavmc.enable = true;

13
nix/test-cluster/node01.nix
View file

@ -71,6 +71,8 @@
port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
};
services.prismnet = {
@ -162,13 +164,6 @@
flashdnsAddr = "http://10.100.0.11:50084";
};
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
};
systemd.services.lightningstor.environment = {
S3_ACCESS_KEY_ID = "photoncloud-test";
S3_SECRET_KEY = "photoncloud-test-secret";
};
services.lightningstor.s3AccessKeyId = "photoncloud-test";
services.lightningstor.s3SecretKey = "photoncloud-test-secret";
}

7
nix/test-cluster/node02.nix
View file

@ -64,10 +64,7 @@
port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
};
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
};
}

7
nix/test-cluster/node03.nix
View file

@ -64,10 +64,7 @@
port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
};
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
};
}

13
nix/test-cluster/storage-node01.nix
View file

@ -69,6 +69,8 @@
port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
};
services.plasmavmc = {
@ -124,13 +126,6 @@
region = "test";
};
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
};
systemd.services.lightningstor.environment = {
S3_ACCESS_KEY_ID = "photoncloud-test";
S3_SECRET_KEY = "photoncloud-test-secret";
};
services.lightningstor.s3AccessKeyId = "photoncloud-test";
services.lightningstor.s3SecretKey = "photoncloud-test-secret";
}

7
nix/test-cluster/storage-node02.nix
View file

@ -66,10 +66,7 @@
port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
};
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
};
}

7
nix/test-cluster/storage-node03.nix
View file

@ -66,10 +66,7 @@
port = 50080;
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
};
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
allowRandomSigningKey = true;
allowUnauthenticatedAdmin = true;
};
}

10
nix/tests/fiberlb-native-bgp-ecmp-drain-vm-smoke.nix
View file

@ -173,10 +173,7 @@ in
port = 50080;
httpPort = 8083;
storeBackend = "memory";
};
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
allowRandomSigningKey = true;
};
services.fiberlb = {
@ -260,10 +257,7 @@ in
port = 50080;
httpPort = 8083;
storeBackend = "memory";
};
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
allowRandomSigningKey = true;
};
services.fiberlb = {

5
nix/tests/fiberlb-native-bgp-interop-vm-smoke.nix
View file

@ -260,10 +260,7 @@ in
port = 50080;
httpPort = 8083;
storeBackend = "memory";
};
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
allowRandomSigningKey = true;
};
services.fiberlb = {

5
nix/tests/fiberlb-native-bgp-multipath-vm-smoke.nix
View file

@ -165,10 +165,7 @@ in
port = 50080;
httpPort = 8083;
storeBackend = "memory";
};
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
allowRandomSigningKey = true;
};
services.fiberlb = {

5
nix/tests/fiberlb-native-bgp-vm-smoke.nix
View file

@ -120,10 +120,7 @@ in
port = 50080;
httpPort = 8083;
storeBackend = "memory";
};
systemd.services.iam.environment = {
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
allowRandomSigningKey = true;
};
services.fiberlb = {

14
plasmavmc/Cargo.lock generated
View file

@ -2458,6 +2458,18 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
[[package]]
name = "nix"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
dependencies = [
"bitflags 2.11.0",
"cfg-if",
"cfg_aliases",
"libc",
]
[[package]]
name = "nom"
version = "7.1.3"
@ -2807,6 +2819,7 @@ name = "plasmavmc-kvm"
version = "0.1.0"
dependencies = [
"async-trait",
"nix",
"plasmavmc-hypervisor",
"plasmavmc-types",
"serde",
@ -2853,6 +2866,7 @@ dependencies = [
"reqwest 0.12.28",
"serde",
"serde_json",
"sha2",
"tempfile",
"thiserror 1.0.69",
"tokio",

1
plasmavmc/crates/plasmavmc-kvm/Cargo.toml
View file

@ -15,6 +15,7 @@ tracing = { workspace = true }
thiserror = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
nix = { version = "0.29", features = ["signal"] }
[lints]
workspace = true

10
plasmavmc/crates/plasmavmc-kvm/src/env.rs
View file

@ -115,14 +115,20 @@ mod tests {
fn resolve_runtime_dir_defaults() {
let _guard = env_test_lock().lock().unwrap();
std::env::remove_var(ENV_RUNTIME_DIR);
assert_eq!(resolve_runtime_dir(), PathBuf::from("/run/libvirt/plasmavmc"));
assert_eq!(
resolve_runtime_dir(),
PathBuf::from("/run/libvirt/plasmavmc")
);
}
#[test]
fn resolve_runtime_dir_from_env() {
let _guard = env_test_lock().lock().unwrap();
std::env::set_var(ENV_RUNTIME_DIR, "/tmp/plasmavmc-runtime");
assert_eq!(resolve_runtime_dir(), PathBuf::from("/tmp/plasmavmc-runtime"));
assert_eq!(
resolve_runtime_dir(),
PathBuf::from("/tmp/plasmavmc-runtime")
);
std::env::remove_var(ENV_RUNTIME_DIR);
}

321
plasmavmc/crates/plasmavmc-kvm/src/lib.rs
View file

@ -8,18 +8,19 @@ mod qmp;
use async_trait::async_trait;
use env::{
resolve_kernel_initrd, resolve_nbd_aio_mode, resolve_nbd_max_queues, resolve_qcow2_path, resolve_qemu_path,
resolve_qmp_timeout_secs, resolve_runtime_dir, ENV_QCOW2_PATH,
resolve_kernel_initrd, resolve_nbd_aio_mode, resolve_nbd_max_queues, resolve_qcow2_path,
resolve_qemu_path, resolve_qmp_timeout_secs, resolve_runtime_dir, ENV_QCOW2_PATH,
};
use nix::sys::signal::{kill as nix_kill, Signal};
use nix::unistd::Pid;
use plasmavmc_hypervisor::{BackendCapabilities, HypervisorBackend, UnsupportedReason};
use plasmavmc_types::{
AttachedDisk, DiskAttachment, DiskBus, DiskCache, Error, HypervisorType, NetworkSpec,
NicModel, Result, VirtualMachine, VmHandle, VmSpec, VmState, VmStatus, VolumeFormat,
AttachedDisk, DiskAttachment, DiskBus, DiskCache, Error, HypervisorType, NetworkSpec, NicModel,
Result, VirtualMachine, VmHandle, VmSpec, VmState, VmStatus, VolumeFormat,
};
use qmp::QmpClient;
use serde_json::{json, Value};
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::time::Duration;
use tokio::process::Command;
use tokio::{net::UnixStream, time::Instant};
@ -62,31 +63,6 @@ fn volume_format_name(format: VolumeFormat) -> &'static str {
}
}
fn build_rbd_uri(pool: &str, image: &str, monitors: &[String], user: &str) -> String {
let mut uri = format!("rbd:{pool}/{image}");
if !user.is_empty() {
uri.push_str(&format!(":id={user}"));
}
if !monitors.is_empty() {
uri.push_str(&format!(":mon_host={}", monitors.join(";")));
}
uri
}
fn disk_source_arg(disk: &AttachedDisk) -> Result<(String, &'static str)> {
match &disk.attachment {
DiskAttachment::File { path, format } => Ok((path.clone(), volume_format_name(*format))),
DiskAttachment::Nbd { uri, format } => Ok((uri.clone(), volume_format_name(*format))),
DiskAttachment::CephRbd {
pool,
image,
monitors,
user,
..
} => Ok((build_rbd_uri(pool, image, monitors, user), "raw")),
}
}
fn effective_disk_cache(disk: &AttachedDisk) -> DiskCache {
match (&disk.attachment, disk.cache) {
// Shared NBD-backed volumes perform better and behave more predictably
@ -96,14 +72,6 @@ fn effective_disk_cache(disk: &AttachedDisk) -> DiskCache {
}
}
fn disk_cache_mode(cache: DiskCache) -> &'static str {
match cache {
DiskCache::None => "none",
DiskCache::Writeback => "writeback",
DiskCache::Writethrough => "writethrough",
}
}
fn disk_aio_mode(disk: &AttachedDisk) -> Option<&'static str> {
match (&disk.attachment, disk.cache) {
(DiskAttachment::File { .. }, DiskCache::None) => Some("native"),
@ -125,7 +93,8 @@ fn disk_queue_count(vm: &VirtualMachine, disk: &AttachedDisk) -> u16 {
return 1;
}
vm.spec.cpu
vm.spec
.cpu
.vcpus
.clamp(1, resolve_nbd_max_queues().max(1) as u32) as u16
}
@ -153,6 +122,169 @@ fn qmp_timeout() -> Duration {
Duration::from_secs(resolve_qmp_timeout_secs())
}
fn disk_cache_json(cache: DiskCache) -> Value {
json!({
"direct": matches!(cache, DiskCache::None),
"no-flush": false
})
}
fn validate_ceph_component(field_name: &str, value: &str) -> Result<()> {
if value.is_empty() {
return Err(Error::HypervisorError(format!("{field_name} is required")));
}
let mut chars = value.chars();
let Some(first) = chars.next() else {
return Err(Error::HypervisorError(format!("{field_name} is required")));
};
if !first.is_ascii_alphanumeric() {
return Err(Error::HypervisorError(format!(
"{field_name} must start with an ASCII alphanumeric character"
)));
}
if chars.any(|ch| !(ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-'))) {
return Err(Error::HypervisorError(format!(
"{field_name} contains unsupported characters"
)));
}
Ok(())
}
fn parse_host_port(authority: &str, default_port: u16) -> Result<(String, u16)> {
if let Some(rest) = authority.strip_prefix('[') {
let (host, tail) = rest
.split_once(']')
.ok_or_else(|| Error::HypervisorError("invalid IPv6 authority".into()))?;
let port = tail
.strip_prefix(':')
.and_then(|value| value.parse::<u16>().ok())
.unwrap_or(default_port);
return Ok((host.to_string(), port));
}
if let Some((host, port)) = authority.rsplit_once(':') {
if !host.is_empty() {
if let Ok(port) = port.parse::<u16>() {
return Ok((host.to_string(), port));
}
}
}
Ok((authority.to_string(), default_port))
}
fn parse_nbd_uri(uri: &str) -> Result<(String, u16, Option<String>)> {
let remainder = uri
.strip_prefix("nbd://")
.ok_or_else(|| Error::HypervisorError(format!("unsupported NBD URI: {uri}")))?;
if remainder.contains('@') || remainder.contains('?') || remainder.contains('#') {
return Err(Error::HypervisorError(format!(
"unsupported NBD URI components: {uri}"
)));
}
let (authority, path) = remainder.split_once('/').unwrap_or((remainder, ""));
let (host, port) = parse_host_port(authority, 10809)?;
if host.is_empty() {
return Err(Error::HypervisorError(format!(
"missing NBD host in URI: {uri}"
)));
}
let export = (!path.is_empty()).then(|| path.to_string());
Ok((host, port, export))
}
fn parse_ceph_monitor(monitor: &str) -> Result<Value> {
let (host, port) = parse_host_port(monitor, 6789)?;
if host.is_empty() {
return Err(Error::HypervisorError(format!(
"invalid Ceph monitor address: {monitor}"
)));
}
Ok(json!({
"type": "inet",
"host": host,
"port": port.to_string()
}))
}
fn disk_blockdev_arg(disk: &AttachedDisk, disk_id: &str) -> Result<String> {
let effective_cache = effective_disk_cache(disk);
let aio_mode = disk_aio_mode(disk);
let file = match &disk.attachment {
DiskAttachment::File { path, .. } => {
let mut file = json!({
"driver": "file",
"filename": path
});
if let Some(aio_mode) = aio_mode {
file["aio"] = json!(aio_mode);
}
file
}
DiskAttachment::Nbd { uri, .. } => {
let (host, port, export) = parse_nbd_uri(uri)?;
let mut nbd = json!({
"driver": "nbd",
"server": {
"type": "inet",
"host": host,
"port": port.to_string()
}
});
if let Some(export) = export {
nbd["export"] = json!(export);
}
if let Some(aio_mode) = aio_mode {
nbd["aio"] = json!(aio_mode);
}
nbd
}
DiskAttachment::CephRbd {
pool,
image,
monitors,
user,
..
} => {
validate_ceph_component("ceph pool", pool)?;
validate_ceph_component("ceph image", image)?;
if !user.is_empty() {
validate_ceph_component("ceph user", user)?;
}
let servers: Vec<Value> = monitors
.iter()
.map(|monitor| parse_ceph_monitor(monitor))
.collect::<Result<Vec<_>>>()?;
let mut rbd = json!({
"driver": "rbd",
"pool": pool,
"image": image,
"server": servers
});
if !user.is_empty() {
rbd["user"] = json!(user);
}
rbd
}
};
let format_driver = match &disk.attachment {
DiskAttachment::File { format, .. } | DiskAttachment::Nbd { format, .. } => {
volume_format_name(*format)
}
DiskAttachment::CephRbd { .. } => "raw",
};
Ok(json!({
"node-name": format!("drive-{disk_id}"),
"driver": format_driver,
"read-only": disk.read_only,
"cache": disk_cache_json(effective_cache),
"file": file
})
.to_string())
}
fn build_disk_args(vm: &VirtualMachine, disks: &[AttachedDisk]) -> Result<Vec<String>> {
if disks.is_empty() && vm.spec.disks.is_empty() {
let qcow_path = resolve_qcow2_path().ok_or_else(|| {
@ -167,8 +299,20 @@ fn build_disk_args(vm: &VirtualMachine, disks: &[AttachedDisk]) -> Result<Vec<St
)));
}
return Ok(vec![
"-drive".into(),
format!("file={},if=virtio,format=qcow2", qcow_path.display()),
"-blockdev".into(),
json!({
"node-name": "drive-root",
"driver": "qcow2",
"read-only": false,
"cache": disk_cache_json(DiskCache::Writeback),
"file": {
"driver": "file",
"filename": qcow_path.display().to_string()
}
})
.to_string(),
"-device".into(),
"virtio-blk-pci,drive=drive-root,id=disk-root".into(),
]);
}
@ -205,21 +349,12 @@ fn build_disk_args(vm: &VirtualMachine, disks: &[AttachedDisk]) -> Result<Vec<St
for (index, disk) in disks.into_iter().enumerate() {
let disk_id = sanitize_device_component(&disk.id, index);
let (source, format_name) = disk_source_arg(disk)?;
if disk_uses_dedicated_iothread(disk) {
args.push("-object".into());
args.push(format!("iothread,id=iothread-{disk_id}"));
}
let effective_cache = effective_disk_cache(disk);
let mut drive_arg = format!(
"file={source},if=none,format={format_name},id=drive-{disk_id},cache={}",
disk_cache_mode(effective_cache)
);
if let Some(aio_mode) = disk_aio_mode(disk) {
drive_arg.push_str(&format!(",aio={aio_mode}"));
}
args.push("-drive".into());
args.push(drive_arg);
args.push("-blockdev".into());
args.push(disk_blockdev_arg(disk, &disk_id)?);
let bootindex = bootindex_suffix(disk.boot_index);
let device_arg = match disk.bus {
@ -359,33 +494,24 @@ async fn wait_for_qmp(qmp_socket: &Path, timeout: Duration) -> Result<()> {
}
fn kill_pid(pid: u32) -> Result<()> {
let status = std::process::Command::new("kill")
.arg("-9")
.arg(pid.to_string())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map_err(|e| Error::HypervisorError(format!("Failed to invoke kill -9: {e}")))?;
if status.success() {
Ok(())
} else if !pid_running(pid) {
Ok(())
} else {
Err(Error::HypervisorError(format!(
"kill -9 exited with status: {status}"
)))
let pid = Pid::from_raw(pid as i32);
match nix_kill(pid, Signal::SIGKILL) {
Ok(()) => Ok(()),
Err(nix::errno::Errno::ESRCH) => Ok(()),
Err(error) => Err(Error::HypervisorError(format!(
"failed to send SIGKILL to pid {}: {error}",
pid.as_raw()
))),
}
}
fn pid_running(pid: u32) -> bool {
std::process::Command::new("kill")
.arg("-0")
.arg(pid.to_string())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map(|status| status.success())
.unwrap_or(false)
match nix_kill(Pid::from_raw(pid as i32), None::<Signal>) {
Ok(()) => true,
Err(nix::errno::Errno::EPERM) => true,
Err(nix::errno::Errno::ESRCH) => false,
Err(_) => false,
}
}
fn vm_stopped_out_of_band(handle: &VmHandle, qmp_socket: &Path) -> bool {
@ -569,7 +695,9 @@ impl HypervisorBackend for KvmBackend {
match QmpClient::connect(&qmp_socket).await {
Ok(mut client) => match client.query_status().await {
Ok(status) if matches!(status.actual_state, VmState::Stopped | VmState::Failed) => {
Ok(status)
if matches!(status.actual_state, VmState::Stopped | VmState::Failed) =>
{
break;
}
Ok(_) => {}
@ -1109,10 +1237,12 @@ mod tests {
let args_joined = args.join(" ");
assert!(args_joined.contains("vm-root.qcow2"));
assert!(args_joined.contains("vm-data.qcow2"));
assert!(args_joined.contains("-blockdev"));
assert!(args_joined.contains("bootindex=1"));
assert!(args_joined.contains("cache=writeback"));
assert!(args_joined.contains("cache=none,aio=native"));
assert!(args_joined.contains("cache=writeback,aio=threads"));
assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
assert!(args_joined.contains("\"cache\":{\"direct\":false,\"no-flush\":false}"));
assert!(args_joined.contains("\"aio\":\"native\""));
assert!(args_joined.contains("\"aio\":\"threads\""));
}
#[test]
@ -1135,6 +1265,7 @@ mod tests {
let console = PathBuf::from("/tmp/console.log");
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
let args_joined = args.join(" ");
assert!(args_joined.contains("\"driver\":\"nbd\""));
assert!(args_joined.contains("-object iothread,id=iothread-root"));
assert!(args_joined.contains("virtio-blk-pci,drive=drive-root,id=disk-root,iothread=iothread-root,num-queues=4,queue-size=1024,bootindex=1"));
}
@ -1159,7 +1290,8 @@ mod tests {
let console = PathBuf::from("/tmp/console.log");
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
let args_joined = args.join(" ");
assert!(args_joined.contains("cache=none,aio=io_uring"));
assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
assert!(args_joined.contains("\"aio\":\"io_uring\""));
}
#[test]
@ -1182,7 +1314,8 @@ mod tests {
let console = PathBuf::from("/tmp/console.log");
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
let args_joined = args.join(" ");
assert!(args_joined.contains("cache=none,aio=io_uring"));
assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
assert!(args_joined.contains("\"aio\":\"io_uring\""));
}
#[test]
@ -1205,10 +1338,34 @@ mod tests {
let console = PathBuf::from("/tmp/console.log");
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
let args_joined = args.join(" ");
assert!(args_joined.contains("cache=none,aio=threads"));
assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
assert!(args_joined.contains("\"aio\":\"threads\""));
std::env::remove_var(crate::env::ENV_NBD_AIO_MODE);
}
#[test]
fn build_qemu_args_rejects_invalid_ceph_identifiers() {
let vm = VirtualMachine::new("vm1", "org", "proj", VmSpec::default());
let disks = vec![AttachedDisk {
id: "root".into(),
attachment: DiskAttachment::CephRbd {
pool: "pool,inject".into(),
image: "image".into(),
monitors: vec!["10.0.0.10:6789".into()],
user: "admin".into(),
secret: None,
},
bus: DiskBus::Virtio,
cache: DiskCache::None,
boot_index: Some(1),
read_only: false,
}];
let qmp = PathBuf::from("/tmp/qmp.sock");
let console = PathBuf::from("/tmp/console.log");
let error = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap_err();
assert!(error.to_string().contains("unsupported characters"));
}
#[tokio::test]
async fn wait_for_qmp_succeeds_after_socket_created() {
let dir = tempfile::tempdir().unwrap();

4
plasmavmc/crates/plasmavmc-kvm/src/qmp.rs
View file

@ -33,7 +33,9 @@ impl QmpClient {
last_error = Some(format!("Failed to connect QMP: {e}"));
}
Err(e) => {
return Err(Error::HypervisorError(format!("Failed to connect QMP: {e}")));
return Err(Error::HypervisorError(format!(
"Failed to connect QMP: {e}"
)));
}
}

1
plasmavmc/crates/plasmavmc-server/Cargo.toml
View file

@ -33,6 +33,7 @@ serde_json = { workspace = true }
toml = { workspace = true }
metrics-exporter-prometheus = { workspace = true }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
sha2 = "0.10"
chainfire-client = { path = "../../../chainfire/chainfire-client" }
creditservice-client = { path = "../../../creditservice/creditservice-client" }
flaredb-client = { path = "../../../flaredb/crates/flaredb-client" }

782
plasmavmc/crates/plasmavmc-server/src/artifact_store.rs
View file

@ -1,3 +1,10 @@
use crate::config::ArtifactStoreConfig;
use crate::storage::ImageUploadPart;
use reqwest::header::LOCATION;
use reqwest::{Client as HttpClient, Url};
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::net::IpAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
@ -13,7 +20,6 @@ use lightningstor_api::proto::{
};
use lightningstor_api::{BucketServiceClient, ObjectServiceClient};
use plasmavmc_types::ImageFormat;
use reqwest::StatusCode as HttpStatusCode;
use serde::Deserialize;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::process::Command;
@ -23,25 +29,30 @@ use tonic::metadata::MetadataValue;
use tonic::transport::{Channel, Endpoint};
use tonic::{Code, Request, Status};
const DEFAULT_IMAGE_BUCKET: &str = "plasmavmc-images";
const MAX_OBJECT_GRPC_MESSAGE_SIZE: usize = 1024 * 1024 * 1024;
const OBJECT_GRPC_INITIAL_STREAM_WINDOW: u32 = 64 * 1024 * 1024;
const OBJECT_GRPC_INITIAL_CONNECTION_WINDOW: u32 = 512 * 1024 * 1024;
const OBJECT_GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30);
const OBJECT_GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(10);
const DEFAULT_MULTIPART_UPLOAD_PART_SIZE: usize = 32 * 1024 * 1024;
const MIN_MULTIPART_UPLOAD_PART_SIZE: usize = 8 * 1024 * 1024;
const MAX_MULTIPART_UPLOAD_PART_SIZE: usize = 128 * 1024 * 1024;
const DEFAULT_MULTIPART_UPLOAD_CONCURRENCY: usize = 4;
const MAX_MULTIPART_UPLOAD_CONCURRENCY: usize = 32;
const DEFAULT_RAW_IMAGE_CONVERT_PARALLELISM: usize = 8;
const MAX_IMPORT_REDIRECTS: usize = 5;
const DEFAULT_HTTP_SEND_TIMEOUT: Duration = Duration::from_secs(15);
#[derive(Clone)]
pub struct ArtifactStore {
channel: Channel,
iam_client: Arc<IamClient>,
http_client: HttpClient,
image_bucket: String,
image_cache_dir: PathBuf,
multipart_upload_concurrency: usize,
multipart_upload_part_size: usize,
raw_image_convert_parallelism: usize,
max_image_import_size_bytes: u64,
allowed_https_hosts: Arc<HashSet<String>>,
qemu_img_path: PathBuf,
project_tokens: Arc<DashMap<String, CachedToken>>,
ensured_buckets: Arc<DashSet<String>>,
}
@ -50,6 +61,8 @@ pub(crate) struct ImportedImage {
pub size_bytes: u64,
pub checksum: String,
pub format: ImageFormat,
pub source_type: String,
pub source_host: Option<String>,
}
#[derive(Deserialize)]
@ -62,10 +75,24 @@ struct CachedToken {
expires_at: Instant,
}
struct ValidatedImportUrl {
url: Url,
host: String,
}
struct ImportedImageSource {
source_type: String,
host: String,
}
impl ArtifactStore {
pub async fn from_env(iam_endpoint: &str) -> Result<Option<Self>, Box<dyn std::error::Error>> {
let Some(raw_endpoint) = std::env::var("PLASMAVMC_LIGHTNINGSTOR_ENDPOINT")
.ok()
pub async fn from_config(
config: &ArtifactStoreConfig,
iam_endpoint: &str,
) -> Result<Option<Self>, Box<dyn std::error::Error>> {
let Some(raw_endpoint) = config
.lightningstor_endpoint
.as_ref()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
else {
@ -87,18 +114,41 @@ impl ArtifactStore {
.keep_alive_timeout(OBJECT_GRPC_KEEPALIVE_TIMEOUT)
.connect_lazy();
let image_cache_dir = std::env::var("PLASMAVMC_IMAGE_CACHE_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("/var/lib/plasmavmc/images"));
let image_cache_dir = config.image_cache_dir.clone();
tokio::fs::create_dir_all(&image_cache_dir).await?;
ensure_cache_dir_permissions(&image_cache_dir).await?;
let http_client = HttpClient::builder()
.connect_timeout(Duration::from_secs(
config.image_import_connect_timeout_secs.max(1),
))
.timeout(Duration::from_secs(config.image_import_timeout_secs.max(1)))
.redirect(reqwest::redirect::Policy::none())
.build()?;
let allowed_https_hosts = config
.allowed_https_hosts
.iter()
.map(|host| host.trim().to_ascii_lowercase())
.filter(|host| !host.is_empty())
.collect::<HashSet<_>>();
let qemu_img_path = resolve_binary_path(config.qemu_img_path.as_deref(), "qemu-img")?;
Ok(Some(Self {
channel,
iam_client,
image_bucket: std::env::var("PLASMAVMC_IMAGE_BUCKET")
.unwrap_or_else(|_| DEFAULT_IMAGE_BUCKET.to_string()),
http_client,
image_bucket: config.image_bucket.clone(),
image_cache_dir,
multipart_upload_concurrency: config
.multipart_upload_concurrency
.clamp(1, MAX_MULTIPART_UPLOAD_CONCURRENCY),
multipart_upload_part_size: config.multipart_upload_part_size.clamp(
MIN_MULTIPART_UPLOAD_PART_SIZE,
MAX_MULTIPART_UPLOAD_PART_SIZE,
),
raw_image_convert_parallelism: config.raw_image_convert_parallelism.clamp(1, 64),
max_image_import_size_bytes: config.max_image_import_size_bytes.max(1),
allowed_https_hosts: Arc::new(allowed_https_hosts),
qemu_img_path,
project_tokens: Arc::new(DashMap::new()),
ensured_buckets: Arc::new(DashSet::new()),
}))
@ -116,50 +166,20 @@ impl ArtifactStore {
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
.await?;
let image_path = self.image_path(image_id);
let staging_path = self.image_cache_dir.join(format!("{image_id}.source"));
self.materialize_source(source_url, &staging_path).await?;
if self
.can_reuse_qcow2_source(&staging_path, source_format)
.await?
{
if tokio::fs::try_exists(&image_path).await.map_err(|e| {
Status::internal(format!("failed to inspect {}: {e}", image_path.display()))
})? {
let _ = tokio::fs::remove_file(&staging_path).await;
} else {
tokio::fs::rename(&staging_path, &image_path)
.await
.map_err(|e| {
Status::internal(format!(
"failed to move qcow2 image {} into cache {}: {e}",
staging_path.display(),
image_path.display()
))
})?;
ensure_cache_file_permissions(&image_path).await?;
}
} else {
// Normalize non-qcow2 inputs through qemu-img convert so the cached
// artifact has a stable qcow2 representation before upload.
self.convert_to_qcow2(&staging_path, &image_path).await?;
let _ = tokio::fs::remove_file(&staging_path).await;
}
let checksum = self.sha256sum(&image_path).await?;
let metadata = tokio::fs::metadata(&image_path).await.map_err(|e| {
Status::internal(format!("failed to stat {}: {e}", image_path.display()))
})?;
let image_key = image_object_key(org_id, project_id, image_id);
self.upload_file(&self.image_bucket, &image_key, &image_path, &token)
.await?;
Ok(ImportedImage {
size_bytes: metadata.len(),
checksum,
format: ImageFormat::Qcow2,
})
let staging_path = self.staging_path(image_id)?;
let ImportedImageSource { source_type, host } =
self.materialize_source(source_url, &staging_path).await?;
self.process_staged_image(
org_id,
project_id,
image_id,
&token,
&staging_path,
source_format,
source_type,
Some(host),
)
.await
}
pub async fn materialize_image_cache(
@ -171,8 +191,8 @@ impl ArtifactStore {
let token = self.issue_project_token(org_id, project_id).await?;
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
.await?;
let image_key = image_object_key(org_id, project_id, image_id);
let image_path = self.image_path(image_id);
let image_key = image_object_key(org_id, project_id, image_id)?;
let image_path = self.image_path(image_id)?;
self.download_object_to_file(&self.image_bucket, &image_key, &image_path, &token)
.await?;
Ok(image_path)
@ -187,7 +207,7 @@ impl ArtifactStore {
let image_path = self
.materialize_image_cache(org_id, project_id, image_id)
.await?;
let raw_path = self.raw_image_path(image_id);
let raw_path = self.raw_image_path(image_id)?;
self.convert_to_raw(&image_path, &raw_path).await?;
Ok(raw_path)
}
@ -199,7 +219,7 @@ impl ArtifactStore {
image_id: &str,
) -> Result<(), Status> {
let token = self.issue_project_token(org_id, project_id).await?;
let image_key = image_object_key(org_id, project_id, image_id);
let image_key = image_object_key(org_id, project_id, image_id)?;
let mut client = self.object_client().await?;
let mut request = Request::new(DeleteObjectRequest {
bucket: self.image_bucket.clone(),
@ -213,7 +233,7 @@ impl ArtifactStore {
Err(status) => return Err(Status::from_error(Box::new(status))),
}
let image_path = self.image_path(image_id);
let image_path = self.image_path(image_id)?;
if tokio::fs::try_exists(&image_path).await.map_err(|e| {
Status::internal(format!("failed to inspect {}: {e}", image_path.display()))
})? {
@ -221,7 +241,7 @@ impl ArtifactStore {
Status::internal(format!("failed to remove {}: {e}", image_path.display()))
})?;
}
let raw_path = self.raw_image_path(image_id);
let raw_path = self.raw_image_path(image_id)?;
if tokio::fs::try_exists(&raw_path).await.map_err(|e| {
Status::internal(format!("failed to inspect {}: {e}", raw_path.display()))
})? {
@ -232,6 +252,156 @@ impl ArtifactStore {
Ok(())
}
pub fn minimum_upload_part_size(&self) -> u32 {
self.multipart_upload_part_size as u32
}
pub(crate) async fn begin_image_upload(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
) -> Result<(String, String), Status> {
let token = self.issue_project_token(org_id, project_id).await?;
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
.await?;
let staging_key = staging_object_key(org_id, project_id, image_id)?;
let mut client = self.object_client().await?;
let mut request = Request::new(CreateMultipartUploadRequest {
bucket: self.image_bucket.clone(),
key: staging_key.clone(),
metadata: None,
});
attach_bearer(&mut request, &token)?;
let upload_id = client
.create_multipart_upload(request)
.await
.map_err(|status| Status::from_error(Box::new(status)))?
.into_inner()
.upload_id;
Ok((upload_id, staging_key))
}
pub(crate) async fn upload_image_part(
&self,
org_id: &str,
project_id: &str,
staging_key: &str,
upload_id: &str,
part_number: u32,
body: Vec<u8>,
) -> Result<ImageUploadPart, Status> {
if part_number == 0 {
return Err(Status::invalid_argument(
"part_number must be greater than zero",
));
}
if body.is_empty() {
return Err(Status::invalid_argument(
"upload part body must not be empty",
));
}
let token = self.issue_project_token(org_id, project_id).await?;
let mut client = self.object_client().await?;
let request_stream = tokio_stream::iter(vec![UploadPartRequest {
bucket: self.image_bucket.clone(),
key: staging_key.to_string(),
upload_id: upload_id.to_string(),
part_number,
body: body.clone().into(),
content_md5: String::new(),
}]);
let mut request = Request::new(request_stream);
attach_bearer(&mut request, &token)?;
let response = client
.upload_part(request)
.await
.map_err(|status| Status::from_error(Box::new(status)))?;
Ok(ImageUploadPart {
part_number,
etag: response.into_inner().etag,
size_bytes: body.len() as u64,
})
}
pub(crate) async fn complete_image_upload(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
staging_key: &str,
upload_id: &str,
parts: &[ImageUploadPart],
source_format: ImageFormat,
) -> Result<ImportedImage, Status> {
if parts.is_empty() {
return Err(Status::failed_precondition(
"upload session does not contain any parts",
));
}
let token = self.issue_project_token(org_id, project_id).await?;
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
.await?;
let mut sorted_parts: Vec<CompletedPart> = parts
.iter()
.map(|part| CompletedPart {
part_number: part.part_number,
etag: part.etag.clone(),
})
.collect();
sorted_parts.sort_by_key(|part| part.part_number);
let mut client = self.object_client().await?;
let mut complete_request = Request::new(CompleteMultipartUploadRequest {
bucket: self.image_bucket.clone(),
key: staging_key.to_string(),
upload_id: upload_id.to_string(),
parts: sorted_parts,
});
attach_bearer(&mut complete_request, &token)?;
client
.complete_multipart_upload(complete_request)
.await
.map_err(|status| Status::from_error(Box::new(status)))?;
let staging_path = self.staging_path(image_id)?;
if tokio::fs::try_exists(&staging_path).await.unwrap_or(false) {
let _ = tokio::fs::remove_file(&staging_path).await;
}
self.download_object_to_file(&self.image_bucket, staging_key, &staging_path, &token)
.await?;
let result = self
.process_staged_image(
org_id,
project_id,
image_id,
&token,
&staging_path,
source_format,
"upload".to_string(),
None,
)
.await;
let _ = self
.delete_object_ignore_not_found(&self.image_bucket, staging_key, &token)
.await;
result
}
pub(crate) async fn abort_image_upload(
&self,
org_id: &str,
project_id: &str,
staging_key: &str,
upload_id: &str,
) -> Result<(), Status> {
let token = self.issue_project_token(org_id, project_id).await?;
self.abort_multipart_upload(&self.image_bucket, staging_key, upload_id, &token)
.await
}
async fn ensure_bucket(
&self,
bucket: &str,
@ -275,7 +445,7 @@ impl ArtifactStore {
let metadata = tokio::fs::metadata(path)
.await
.map_err(|e| Status::internal(format!("failed to stat {path:?}: {e}")))?;
let multipart_part_size = multipart_upload_part_size();
let multipart_part_size = self.multipart_upload_part_size;
if metadata.len() > multipart_part_size as u64 {
return self
.upload_file_multipart(bucket, key, path, token, metadata.len())
@ -336,7 +506,7 @@ impl ArtifactStore {
size_bytes: u64,
) -> Result<(), Status> {
let started = Instant::now();
let multipart_part_size = multipart_upload_part_size();
let multipart_part_size = self.multipart_upload_part_size;
tracing::info!(
bucket = bucket,
key = key,
@ -366,7 +536,7 @@ impl ArtifactStore {
let mut part_number = 1u32;
let mut completed_parts = Vec::new();
let mut uploads = JoinSet::new();
let upload_concurrency = multipart_upload_concurrency();
let upload_concurrency = self.multipart_upload_concurrency;
let enqueue_part_upload = |uploads: &mut JoinSet<Result<CompletedPart, Status>>,
client: &ObjectServiceClient<Channel>,
@ -569,76 +739,23 @@ impl ArtifactStore {
Ok(())
}
async fn materialize_source(&self, source_url: &str, path: &Path) -> Result<(), Status> {
async fn materialize_source(
&self,
source_url: &str,
path: &Path,
) -> Result<ImportedImageSource, Status> {
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent)
.await
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
}
if let Some(source_path) = source_url.strip_prefix("file://") {
tokio::fs::copy(source_path, path).await.map_err(|e| {
Status::internal(format!("failed to copy image source {source_path}: {e}"))
})?;
ensure_cache_file_permissions(path).await?;
return Ok(());
}
if source_url.starts_with('/') {
tokio::fs::copy(source_url, path).await.map_err(|e| {
Status::internal(format!("failed to copy image source {source_url}: {e}"))
})?;
ensure_cache_file_permissions(path).await?;
return Ok(());
}
if source_url.starts_with("http://") || source_url.starts_with("https://") {
let mut response = reqwest::get(source_url).await.map_err(|e| {
Status::unavailable(format!("failed to download image source: {e}"))
})?;
if response.status() != HttpStatusCode::OK {
return Err(Status::failed_precondition(format!(
"image download failed with HTTP {}",
response.status()
)));
}
let temp_path = path.with_extension("download");
let mut file = tokio::fs::File::create(&temp_path).await.map_err(|e| {
Status::internal(format!(
"failed to create downloaded image {}: {e}",
temp_path.display()
))
})?;
while let Some(chunk) = response.chunk().await.map_err(|e| {
Status::unavailable(format!("failed to read image response body: {e}"))
})? {
file.write_all(&chunk).await.map_err(|e| {
Status::internal(format!(
"failed to write downloaded image {}: {e}",
temp_path.display()
))
})?;
}
file.flush().await.map_err(|e| {
Status::internal(format!(
"failed to flush downloaded image {}: {e}",
temp_path.display()
))
})?;
drop(file);
tokio::fs::rename(&temp_path, path).await.map_err(|e| {
Status::internal(format!(
"failed to finalize downloaded image {}: {e}",
path.display()
))
})?;
ensure_cache_file_permissions(path).await?;
return Ok(());
}
Err(Status::invalid_argument(
"source_url must be file://, an absolute path, or http(s)://",
))
let validated = self.validate_import_url(source_url).await?;
self.download_https_source(&validated, path).await?;
Ok(ImportedImageSource {
source_type: "https".to_string(),
host: validated.host,
})
}
async fn convert_to_qcow2(&self, source: &Path, destination: &Path) -> Result<(), Status> {
@ -654,9 +771,9 @@ impl ArtifactStore {
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
}
let parallelism = raw_image_convert_parallelism().to_string();
let parallelism = self.raw_image_convert_parallelism.to_string();
let args = qemu_img_convert_to_qcow2_args(source, destination, &parallelism);
let status = Command::new("qemu-img")
let status = Command::new(&self.qemu_img_path)
.args(args.iter().map(String::as_str))
.status()
.await
@ -685,9 +802,9 @@ impl ArtifactStore {
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
}
let parallelism = raw_image_convert_parallelism().to_string();
let parallelism = self.raw_image_convert_parallelism.to_string();
let args = qemu_img_convert_to_raw_args(source, destination, &parallelism);
let status = Command::new("qemu-img")
let status = Command::new(&self.qemu_img_path)
.args(args.iter().map(String::as_str))
.status()
.await
@ -712,7 +829,7 @@ impl ArtifactStore {
return Ok(false);
}
let output = Command::new("qemu-img")
let output = Command::new(&self.qemu_img_path)
.args(["info", "--output", "json", path.to_string_lossy().as_ref()])
.output()
.await
@ -727,25 +844,262 @@ impl ArtifactStore {
}
async fn sha256sum(&self, path: &Path) -> Result<String, Status> {
let output = Command::new("sha256sum")
.arg(path)
.output()
let mut file = tokio::fs::File::open(path)
.await
.map_err(|e| Status::internal(format!("failed to spawn sha256sum: {e}")))?;
if !output.status.success() {
return Err(Status::internal(format!(
"sha256sum failed for {} with status {}",
path.display(),
output.status
.map_err(|e| Status::internal(format!("failed to open {}: {e}", path.display())))?;
let mut digest = Sha256::new();
let mut buffer = vec![0u8; 1024 * 1024];
loop {
let read_now = file
.read(&mut buffer)
.await
.map_err(|e| Status::internal(format!("failed to read {}: {e}", path.display())))?;
if read_now == 0 {
break;
}
digest.update(&buffer[..read_now]);
}
Ok(format!("{:x}", digest.finalize()))
}
async fn process_staged_image(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
token: &str,
staging_path: &Path,
source_format: ImageFormat,
source_type: String,
source_host: Option<String>,
) -> Result<ImportedImage, Status> {
let image_path = self.image_path(image_id)?;
if self
.can_reuse_qcow2_source(staging_path, source_format)
.await?
{
if tokio::fs::try_exists(&image_path).await.map_err(|e| {
Status::internal(format!("failed to inspect {}: {e}", image_path.display()))
})? {
let _ = tokio::fs::remove_file(staging_path).await;
} else {
tokio::fs::rename(staging_path, &image_path)
.await
.map_err(|e| {
Status::internal(format!(
"failed to move qcow2 image {} into cache {}: {e}",
staging_path.display(),
image_path.display()
))
})?;
ensure_cache_file_permissions(&image_path).await?;
}
} else {
self.convert_to_qcow2(staging_path, &image_path).await?;
let _ = tokio::fs::remove_file(staging_path).await;
}
let checksum = self.sha256sum(&image_path).await?;
let metadata = tokio::fs::metadata(&image_path).await.map_err(|e| {
Status::internal(format!("failed to stat {}: {e}", image_path.display()))
})?;
let image_key = image_object_key(org_id, project_id, image_id)?;
self.upload_file(&self.image_bucket, &image_key, &image_path, token)
.await?;
Ok(ImportedImage {
size_bytes: metadata.len(),
checksum,
format: ImageFormat::Qcow2,
source_type,
source_host,
})
}
async fn validate_import_url(&self, source_url: &str) -> Result<ValidatedImportUrl, Status> {
if source_url.starts_with("file://") || source_url.starts_with('/') {
return Err(Status::invalid_argument(
"source_url must use https:// and may not reference local files",
));
}
let url = Url::parse(source_url)
.map_err(|e| Status::invalid_argument(format!("invalid source_url: {e}")))?;
if url.scheme() != "https" {
return Err(Status::invalid_argument("source_url must use https://"));
}
if !url.username().is_empty() || url.password().is_some() {
return Err(Status::invalid_argument(
"source_url must not include embedded credentials",
));
}
let host = url
.host_str()
.map(str::to_ascii_lowercase)
.ok_or_else(|| Status::invalid_argument("source_url must include a host"))?;
if host == "localhost" {
return Err(Status::invalid_argument(
"source_url host must not target loopback or local interfaces",
));
}
if !self.allowed_https_hosts.is_empty() && !self.allowed_https_hosts.contains(&host) {
return Err(Status::permission_denied(format!(
"source_url host {host} is not in artifacts.allowed_https_hosts",
)));
}
let stdout = String::from_utf8(output.stdout)
.map_err(|e| Status::internal(format!("invalid sha256sum output: {e}")))?;
stdout
.split_whitespace()
.next()
.map(str::to_string)
.ok_or_else(|| Status::internal("sha256sum output missing digest"))
let port = url.port_or_known_default().unwrap_or(443);
let resolved = tokio::net::lookup_host((host.as_str(), port))
.await
.map_err(|e| Status::unavailable(format!("failed to resolve source_url host: {e}")))?;
let mut found_any = false;
for addr in resolved {
found_any = true;
let ip = addr.ip();
if !is_public_ip(ip) {
return Err(Status::permission_denied(format!(
"source_url resolved to a non-public address: {ip}",
)));
}
}
if !found_any {
return Err(Status::failed_precondition(
"source_url host did not resolve to any public addresses",
));
}
Ok(ValidatedImportUrl { url, host })
}
async fn download_https_source(
&self,
source: &ValidatedImportUrl,
path: &Path,
) -> Result<(), Status> {
let temp_path = path.with_extension("download");
if tokio::fs::try_exists(&temp_path).await.unwrap_or(false) {
let _ = tokio::fs::remove_file(&temp_path).await;
}
let mut current = source.url.clone();
let mut redirects_remaining = MAX_IMPORT_REDIRECTS;
loop {
let response = tokio::time::timeout(
DEFAULT_HTTP_SEND_TIMEOUT,
self.http_client.get(current.clone()).send(),
)
.await
.map_err(|_| Status::deadline_exceeded("timed out waiting for source_url response"))?
.map_err(|e| Status::unavailable(format!("failed to download image source: {e}")))?;
if response.status().is_redirection() {
if redirects_remaining == 0 {
return Err(Status::failed_precondition(
"source_url redirect limit exceeded",
));
}
let location = response
.headers()
.get(LOCATION)
.ok_or_else(|| {
Status::failed_precondition(
"source_url redirect response did not include a Location header",
)
})?
.to_str()
.map_err(|e| {
Status::failed_precondition(format!(
"invalid redirect Location header in source_url response: {e}"
))
})?;
current = current.join(location).map_err(|e| {
Status::failed_precondition(format!(
"failed to resolve redirect target from source_url: {e}"
))
})?;
self.validate_import_url(current.as_ref()).await?;
redirects_remaining -= 1;
continue;
}
if !response.status().is_success() {
return Err(Status::failed_precondition(format!(
"image download failed with HTTP {}",
response.status()
)));
}
if let Some(content_length) = response.content_length() {
if content_length > self.max_image_import_size_bytes {
return Err(Status::resource_exhausted(format!(
"image download exceeds the configured maximum size of {} bytes",
self.max_image_import_size_bytes
)));
}
}
let mut file = tokio::fs::File::create(&temp_path).await.map_err(|e| {
Status::internal(format!(
"failed to create downloaded image {}: {e}",
temp_path.display()
))
})?;
let mut response = response;
let mut downloaded = 0u64;
while let Some(chunk) = response.chunk().await.map_err(|e| {
Status::unavailable(format!("failed to read image response body: {e}"))
})? {
downloaded = downloaded.saturating_add(chunk.len() as u64);
if downloaded > self.max_image_import_size_bytes {
let _ = tokio::fs::remove_file(&temp_path).await;
return Err(Status::resource_exhausted(format!(
"image download exceeds the configured maximum size of {} bytes",
self.max_image_import_size_bytes
)));
}
file.write_all(&chunk).await.map_err(|e| {
Status::internal(format!(
"failed to write downloaded image {}: {e}",
temp_path.display()
))
})?;
}
file.flush().await.map_err(|e| {
Status::internal(format!(
"failed to flush downloaded image {}: {e}",
temp_path.display()
))
})?;
drop(file);
tokio::fs::rename(&temp_path, path).await.map_err(|e| {
Status::internal(format!(
"failed to finalize downloaded image {}: {e}",
path.display()
))
})?;
ensure_cache_file_permissions(path).await?;
return Ok(());
}
}
async fn delete_object_ignore_not_found(
&self,
bucket: &str,
key: &str,
token: &str,
) -> Result<(), Status> {
let mut client = self.object_client().await?;
let mut request = Request::new(DeleteObjectRequest {
bucket: bucket.to_string(),
key: key.to_string(),
version_id: String::new(),
});
attach_bearer(&mut request, token)?;
match client.delete_object(request).await {
Ok(_) => Ok(()),
Err(status) if status.code() == Code::NotFound => Ok(()),
Err(status) => Err(Status::from_error(Box::new(status))),
}
}
async fn bucket_client(&self) -> Result<BucketServiceClient<Channel>, Status> {
@ -832,12 +1186,22 @@ impl ArtifactStore {
Ok(token)
}
fn image_path(&self, image_id: &str) -> PathBuf {
self.image_cache_dir.join(format!("{image_id}.qcow2"))
fn image_path(&self, image_id: &str) -> Result<PathBuf, Status> {
Ok(self
.image_cache_dir
.join(format!("{}.qcow2", validated_image_id(image_id)?)))
}
fn raw_image_path(&self, image_id: &str) -> PathBuf {
self.image_cache_dir.join(format!("{image_id}.raw"))
fn raw_image_path(&self, image_id: &str) -> Result<PathBuf, Status> {
Ok(self
.image_cache_dir
.join(format!("{}.raw", validated_image_id(image_id)?)))
}
fn staging_path(&self, image_id: &str) -> Result<PathBuf, Status> {
Ok(self
.image_cache_dir
.join(format!("{}.source", validated_image_id(image_id)?)))
}
async fn abort_multipart_upload(
@ -915,35 +1279,6 @@ async fn next_uploaded_part(
}
}
fn multipart_upload_concurrency() -> usize {
std::env::var("PLASMAVMC_LIGHTNINGSTOR_MULTIPART_CONCURRENCY")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.map(|value| value.clamp(1, MAX_MULTIPART_UPLOAD_CONCURRENCY))
.unwrap_or(DEFAULT_MULTIPART_UPLOAD_CONCURRENCY)
}
fn multipart_upload_part_size() -> usize {
std::env::var("PLASMAVMC_LIGHTNINGSTOR_MULTIPART_PART_SIZE")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.map(|value| {
value.clamp(
MIN_MULTIPART_UPLOAD_PART_SIZE,
MAX_MULTIPART_UPLOAD_PART_SIZE,
)
})
.unwrap_or(DEFAULT_MULTIPART_UPLOAD_PART_SIZE)
}
fn raw_image_convert_parallelism() -> usize {
std::env::var("PLASMAVMC_RAW_IMAGE_CONVERT_PARALLELISM")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.map(|value| value.clamp(1, 64))
.unwrap_or(DEFAULT_RAW_IMAGE_CONVERT_PARALLELISM)
}
fn qemu_img_convert_to_qcow2_args(
source: &Path,
destination: &Path,
@ -1008,8 +1343,74 @@ fn sanitize_identifier(value: &str) -> String {
.collect()
}
fn image_object_key(org_id: &str, project_id: &str, image_id: &str) -> String {
format!("{org_id}/{project_id}/{image_id}.qcow2")
fn image_object_key(org_id: &str, project_id: &str, image_id: &str) -> Result<String, Status> {
Ok(format!(
"{org_id}/{project_id}/{}.qcow2",
validated_image_id(image_id)?
))
}
fn staging_object_key(org_id: &str, project_id: &str, image_id: &str) -> Result<String, Status> {
Ok(format!(
"{org_id}/{project_id}/uploads/{}.source",
validated_image_id(image_id)?
))
}
fn validated_image_id(image_id: &str) -> Result<&str, Status> {
uuid::Uuid::parse_str(image_id)
.map(|_| image_id)
.map_err(|_| Status::invalid_argument("image_id must be a UUID"))
}
fn is_public_ip(ip: IpAddr) -> bool {
match ip {
IpAddr::V4(ip) => {
!(ip.is_private()
|| ip.is_loopback()
|| ip.is_link_local()
|| ip.is_multicast()
|| ip.is_broadcast()
|| ip.is_documentation()
|| ip.is_unspecified())
}
IpAddr::V6(ip) => {
!(ip.is_loopback()
|| ip.is_unspecified()
|| ip.is_multicast()
|| ip.is_unique_local()
|| ip.is_unicast_link_local())
}
}
}
fn resolve_binary_path(
configured_path: Option<&Path>,
binary_name: &str,
) -> Result<PathBuf, Box<dyn std::error::Error>> {
let candidate = match configured_path {
Some(path) => path.to_path_buf(),
None => std::env::var_os("PATH")
.into_iter()
.flat_map(|paths| std::env::split_paths(&paths).collect::<Vec<_>>())
.map(|entry| entry.join(binary_name))
.find(|candidate| candidate.exists())
.ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("failed to locate {binary_name} in PATH"),
)
})?,
};
let metadata = std::fs::metadata(&candidate)?;
if !metadata.is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("{} is not a regular file", candidate.display()),
)
.into());
}
Ok(candidate)
}
async fn ensure_cache_dir_permissions(path: &Path) -> Result<(), Status> {
@ -1103,4 +1504,13 @@ mod tests {
]
);
}
#[test]
fn image_object_key_rejects_non_uuid_identifiers() {
assert!(image_object_key("org", "project", "../passwd").is_err());
assert_eq!(
image_object_key("org", "project", "11111111-1111-1111-1111-111111111111").unwrap(),
"org/project/11111111-1111-1111-1111-111111111111.qcow2"
);
}
}

331
plasmavmc/crates/plasmavmc-server/src/config.rs
View file

@ -1,8 +1,9 @@
//! Server configuration
use plasmavmc_types::{FireCrackerConfig, KvmConfig};
use serde::{Deserialize, Serialize};
use std::net::SocketAddr;
use plasmavmc_types::{FireCrackerConfig, KvmConfig};
use std::path::PathBuf;
/// TLS configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
@ -42,12 +43,332 @@ pub struct ServerConfig {
/// Configuration for FireCracker backend
#[serde(default)]
pub firecracker: FireCrackerConfig,
/// Durable/shared state configuration
#[serde(default)]
pub storage: StorageRuntimeConfig,
/// Control-plane and agent heartbeat configuration
#[serde(default)]
pub agent: AgentRuntimeConfig,
/// External service integrations
#[serde(default)]
pub integrations: IntegrationConfig,
/// State watcher and VM watch polling configuration
#[serde(default)]
pub watcher: WatcherRuntimeConfig,
/// Background health monitoring and failover configuration
#[serde(default)]
pub health: HealthRuntimeConfig,
/// Artifact storage and image import/export configuration
#[serde(default)]
pub artifacts: ArtifactStoreConfig,
/// Persistent volume runtime configuration
#[serde(default)]
pub volumes: VolumeRuntimeConfig,
/// Default hypervisor used when requests do not specify one
#[serde(default)]
pub default_hypervisor: DefaultHypervisor,
}
fn default_http_addr() -> SocketAddr {
"127.0.0.1:8084".parse().unwrap()
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "lowercase")]
pub enum StorageBackendKind {
#[default]
Flaredb,
File,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StorageRuntimeConfig {
#[serde(default)]
pub backend: StorageBackendKind,
#[serde(default)]
pub flaredb_endpoint: Option<String>,
#[serde(default)]
pub chainfire_endpoint: Option<String>,
#[serde(default)]
pub state_path: Option<PathBuf>,
}
impl Default for StorageRuntimeConfig {
fn default() -> Self {
Self {
backend: StorageBackendKind::default(),
flaredb_endpoint: None,
chainfire_endpoint: None,
state_path: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AgentRuntimeConfig {
#[serde(default)]
pub control_plane_addr: Option<String>,
#[serde(default)]
pub advertise_endpoint: Option<String>,
#[serde(default)]
pub node_id: Option<String>,
#[serde(default)]
pub node_name: Option<String>,
#[serde(default = "default_agent_heartbeat_interval_secs")]
pub heartbeat_interval_secs: u64,
#[serde(default = "default_shared_live_migration")]
pub shared_live_migration: bool,
}
fn default_agent_heartbeat_interval_secs() -> u64 {
5
}
fn default_shared_live_migration() -> bool {
true
}
impl Default for AgentRuntimeConfig {
fn default() -> Self {
Self {
control_plane_addr: None,
advertise_endpoint: None,
node_id: None,
node_name: None,
heartbeat_interval_secs: default_agent_heartbeat_interval_secs(),
shared_live_migration: default_shared_live_migration(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct IntegrationConfig {
#[serde(default)]
pub prismnet_endpoint: Option<String>,
#[serde(default)]
pub creditservice_endpoint: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WatcherRuntimeConfig {
#[serde(default)]
pub enabled: bool,
#[serde(default = "default_state_watcher_poll_interval_ms")]
pub poll_interval_ms: u64,
#[serde(default = "default_vm_watch_poll_interval_ms")]
pub vm_watch_poll_interval_ms: u64,
}
fn default_state_watcher_poll_interval_ms() -> u64 {
1000
}
fn default_vm_watch_poll_interval_ms() -> u64 {
500
}
impl Default for WatcherRuntimeConfig {
fn default() -> Self {
Self {
enabled: false,
poll_interval_ms: default_state_watcher_poll_interval_ms(),
vm_watch_poll_interval_ms: default_vm_watch_poll_interval_ms(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthRuntimeConfig {
#[serde(default)]
pub vm_monitor_interval_secs: Option<u64>,
#[serde(default)]
pub node_monitor_interval_secs: Option<u64>,
#[serde(default = "default_node_heartbeat_timeout_secs")]
pub node_heartbeat_timeout_secs: u64,
#[serde(default)]
pub auto_restart: bool,
#[serde(default)]
pub failover_enabled: bool,
#[serde(default = "default_failover_min_interval_secs")]
pub failover_min_interval_secs: u64,
}
fn default_node_heartbeat_timeout_secs() -> u64 {
60
}
fn default_failover_min_interval_secs() -> u64 {
60
}
impl Default for HealthRuntimeConfig {
fn default() -> Self {
Self {
vm_monitor_interval_secs: None,
node_monitor_interval_secs: None,
node_heartbeat_timeout_secs: default_node_heartbeat_timeout_secs(),
auto_restart: false,
failover_enabled: false,
failover_min_interval_secs: default_failover_min_interval_secs(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ArtifactStoreConfig {
#[serde(default)]
pub lightningstor_endpoint: Option<String>,
#[serde(default = "default_image_cache_dir")]
pub image_cache_dir: PathBuf,
#[serde(default = "default_image_bucket")]
pub image_bucket: String,
#[serde(default = "default_multipart_upload_concurrency")]
pub multipart_upload_concurrency: usize,
#[serde(default = "default_multipart_upload_part_size")]
pub multipart_upload_part_size: usize,
#[serde(default = "default_raw_image_convert_parallelism")]
pub raw_image_convert_parallelism: usize,
#[serde(default = "default_image_import_connect_timeout_secs")]
pub image_import_connect_timeout_secs: u64,
#[serde(default = "default_image_import_timeout_secs")]
pub image_import_timeout_secs: u64,
#[serde(default = "default_max_image_import_size_bytes")]
pub max_image_import_size_bytes: u64,
#[serde(default)]
pub allowed_https_hosts: Vec<String>,
#[serde(default)]
pub qemu_img_path: Option<PathBuf>,
}
fn default_image_cache_dir() -> PathBuf {
PathBuf::from("/var/lib/plasmavmc/images")
}
fn default_image_bucket() -> String {
"plasmavmc-images".to_string()
}
fn default_multipart_upload_concurrency() -> usize {
4
}
fn default_multipart_upload_part_size() -> usize {
32 * 1024 * 1024
}
fn default_raw_image_convert_parallelism() -> usize {
8
}
fn default_image_import_connect_timeout_secs() -> u64 {
5
}
fn default_image_import_timeout_secs() -> u64 {
60 * 60
}
fn default_max_image_import_size_bytes() -> u64 {
64 * 1024 * 1024 * 1024
}
impl Default for ArtifactStoreConfig {
fn default() -> Self {
Self {
lightningstor_endpoint: None,
image_cache_dir: default_image_cache_dir(),
image_bucket: default_image_bucket(),
multipart_upload_concurrency: default_multipart_upload_concurrency(),
multipart_upload_part_size: default_multipart_upload_part_size(),
raw_image_convert_parallelism: default_raw_image_convert_parallelism(),
image_import_connect_timeout_secs: default_image_import_connect_timeout_secs(),
image_import_timeout_secs: default_image_import_timeout_secs(),
max_image_import_size_bytes: default_max_image_import_size_bytes(),
allowed_https_hosts: Vec::new(),
qemu_img_path: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct CoronaFsConfig {
#[serde(default)]
pub controller_endpoint: Option<String>,
#[serde(default)]
pub node_endpoint: Option<String>,
#[serde(default)]
pub endpoint: Option<String>,
#[serde(default)]
pub node_local_attach: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CephConfig {
#[serde(default)]
pub monitors: Vec<String>,
#[serde(default = "default_ceph_cluster_id")]
pub cluster_id: String,
#[serde(default = "default_ceph_user")]
pub user: String,
#[serde(default)]
pub secret: Option<String>,
}
fn default_ceph_cluster_id() -> String {
"default".to_string()
}
fn default_ceph_user() -> String {
"admin".to_string()
}
impl Default for CephConfig {
fn default() -> Self {
Self {
monitors: Vec::new(),
cluster_id: default_ceph_cluster_id(),
user: default_ceph_user(),
secret: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VolumeRuntimeConfig {
#[serde(default = "default_managed_volume_root")]
pub managed_volume_root: PathBuf,
#[serde(default)]
pub ceph: CephConfig,
#[serde(default)]
pub coronafs: CoronaFsConfig,
#[serde(default)]
pub qemu_img_path: Option<PathBuf>,
}
fn default_managed_volume_root() -> PathBuf {
PathBuf::from("/var/lib/plasmavmc/managed-volumes")
}
impl Default for VolumeRuntimeConfig {
fn default() -> Self {
Self {
managed_volume_root: default_managed_volume_root(),
ceph: CephConfig::default(),
coronafs: CoronaFsConfig::default(),
qemu_img_path: None,
}
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "lowercase")]
pub enum DefaultHypervisor {
#[default]
Kvm,
Firecracker,
Mvisor,
}
/// Authentication configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AuthConfig {
@ -78,6 +399,14 @@ impl Default for ServerConfig {
auth: AuthConfig::default(),
kvm: KvmConfig::default(),
firecracker: FireCrackerConfig::default(),
storage: StorageRuntimeConfig::default(),
agent: AgentRuntimeConfig::default(),
integrations: IntegrationConfig::default(),
watcher: WatcherRuntimeConfig::default(),
health: HealthRuntimeConfig::default(),
artifacts: ArtifactStoreConfig::default(),
volumes: VolumeRuntimeConfig::default(),
default_hypervisor: DefaultHypervisor::default(),
}
}
}

78
plasmavmc/crates/plasmavmc-server/src/main.rs
View file

@ -15,7 +15,7 @@ use plasmavmc_api::proto::{
use plasmavmc_firecracker::FireCrackerBackend;
use plasmavmc_hypervisor::HypervisorRegistry;
use plasmavmc_kvm::KvmBackend;
use plasmavmc_server::config::ServerConfig;
use plasmavmc_server::config::{AgentRuntimeConfig, ServerConfig};
use plasmavmc_server::watcher::{StateSynchronizer, StateWatcher, WatcherConfig};
use plasmavmc_server::VmServiceImpl;
use std::net::SocketAddr;
@ -83,16 +83,19 @@ async fn start_agent_heartbeat(
supported_volume_drivers: Vec<i32>,
supported_storage_classes: Vec<String>,
shared_live_migration: bool,
agent_config: &AgentRuntimeConfig,
) {
let Some(control_plane_addr) = std::env::var("PLASMAVMC_CONTROL_PLANE_ADDR")
.ok()
let Some(control_plane_addr) = agent_config
.control_plane_addr
.as_ref()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
else {
return;
};
let Some(node_id) = std::env::var("PLASMAVMC_NODE_ID")
.ok()
let Some(node_id) = agent_config
.node_id
.as_ref()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
else {
@ -100,19 +103,19 @@ async fn start_agent_heartbeat(
};
let endpoint = normalize_endpoint(&control_plane_addr);
let advertise_endpoint = std::env::var("PLASMAVMC_ENDPOINT_ADVERTISE")
.ok()
let advertise_endpoint = agent_config
.advertise_endpoint
.as_ref()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
.unwrap_or_else(|| local_addr.to_string());
let node_name = std::env::var("PLASMAVMC_NODE_NAME")
.ok()
let node_name = agent_config
.node_name
.as_ref()
.cloned()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| node_id.clone());
let heartbeat_secs = std::env::var("PLASMAVMC_NODE_HEARTBEAT_INTERVAL_SECS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(5);
let heartbeat_secs = agent_config.heartbeat_interval_secs.max(1);
tokio::spawn(async move {
let mut ticker = tokio::time::interval(Duration::from_secs(heartbeat_secs));
@ -222,7 +225,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let registry = Arc::new(HypervisorRegistry::new());
// Register KVM backend (always available)
let kvm_backend = Arc::new(KvmBackend::with_defaults());
let kvm_backend = Arc::new(KvmBackend::new(
config
.kvm
.qemu_path
.clone()
.unwrap_or_else(|| PathBuf::from("/usr/bin/qemu-system-x86_64")),
config
.kvm
.runtime_dir
.clone()
.unwrap_or_else(|| PathBuf::from("/run/libvirt/plasmavmc")),
));
registry.register(kvm_backend);
// Register FireCracker backend if kernel/rootfs paths are configured (config or env)
@ -285,17 +299,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
registry,
auth_service.clone(),
config.auth.iam_server_addr.clone(),
&config,
)
.await?,
);
// Optional: start state watcher for multi-instance HA sync
if std::env::var("PLASMAVMC_STATE_WATCHER")
.map(|v| matches!(v.as_str(), "1" | "true" | "yes"))
.unwrap_or(false)
{
let config = WatcherConfig::default();
let (watcher, rx) = StateWatcher::new(vm_service.store(), config);
if config.watcher.enabled {
let watcher_config = WatcherConfig {
poll_interval: Duration::from_millis(config.watcher.poll_interval_ms.max(100)),
buffer_size: 256,
};
let (watcher, rx) = StateWatcher::new(vm_service.store(), watcher_config);
let synchronizer = StateSynchronizer::new(vm_service.clone());
tokio::spawn(async move {
if let Err(e) = watcher.start().await {
@ -305,13 +320,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
tokio::spawn(async move {
synchronizer.run(rx).await;
});
tracing::info!("State watcher enabled (PLASMAVMC_STATE_WATCHER)");
tracing::info!("State watcher enabled");
}
// Optional: start health monitor to refresh VM status periodically
if let Some(secs) = std::env::var("PLASMAVMC_HEALTH_MONITOR_INTERVAL_SECS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
if let Some(secs) = config
.health
.vm_monitor_interval_secs
.filter(|secs| *secs > 0)
{
if secs > 0 {
vm_service
@ -321,18 +337,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}
// Optional: start node health monitor to detect stale heartbeats
if let Some(interval_secs) = std::env::var("PLASMAVMC_NODE_HEALTH_MONITOR_INTERVAL_SECS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
if let Some(interval_secs) = config
.health
.node_monitor_interval_secs
.filter(|secs| *secs > 0)
{
if interval_secs > 0 {
let timeout_secs = std::env::var("PLASMAVMC_NODE_HEARTBEAT_TIMEOUT_SECS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(60);
vm_service.clone().start_node_health_monitor(
Duration::from_secs(interval_secs),
Duration::from_secs(timeout_secs),
Duration::from_secs(config.health.node_heartbeat_timeout_secs.max(1)),
);
}
}
@ -369,6 +382,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
heartbeat_volume_drivers,
heartbeat_storage_classes,
shared_live_migration,
&config.agent,
)
.await;

293
plasmavmc/crates/plasmavmc-server/src/storage.rs
View file

@ -1,7 +1,8 @@
//! Storage abstraction for VM persistence
use crate::config::{StorageBackendKind, StorageRuntimeConfig};
use async_trait::async_trait;
use plasmavmc_types::{Image, Node, VirtualMachine, VmHandle, Volume};
use plasmavmc_types::{Image, ImageFormat, Node, VirtualMachine, VmHandle, Volume};
use std::path::PathBuf;
use thiserror::Error;
@ -16,14 +17,10 @@ pub enum StorageBackend {
}
impl StorageBackend {
pub fn from_env() -> Self {
match std::env::var("PLASMAVMC_STORAGE_BACKEND")
.as_deref()
.unwrap_or("flaredb")
{
"flaredb" => Self::FlareDB,
"file" => Self::File,
_ => Self::FlareDB,
pub fn from_config(config: &StorageRuntimeConfig) -> Self {
match config.backend {
StorageBackendKind::Flaredb => Self::FlareDB,
StorageBackendKind::File => Self::File,
}
}
}
@ -48,6 +45,29 @@ pub enum StorageError {
/// Result type for storage operations
pub type StorageResult<T> = Result<T, StorageError>;
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ImageUploadPart {
pub part_number: u32,
pub etag: String,
pub size_bytes: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ImageUploadSession {
pub session_id: String,
pub org_id: String,
pub project_id: String,
pub image_id: String,
pub upload_id: String,
pub staging_key: String,
pub source_format: ImageFormat,
#[serde(default)]
pub parts: Vec<ImageUploadPart>,
pub committed_size_bytes: u64,
pub created_at: u64,
pub updated_at: u64,
}
/// Storage trait for VM persistence
#[async_trait]
pub trait VmStore: Send + Sync {
@ -126,6 +146,35 @@ pub trait VmStore: Send + Sync {
/// List images for a tenant
async fn list_images(&self, org_id: &str, project_id: &str) -> StorageResult<Vec<Image>>;
/// Save an in-progress image upload session.
async fn save_image_upload_session(&self, session: &ImageUploadSession) -> StorageResult<()>;
/// Load an in-progress image upload session.
async fn load_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<Option<ImageUploadSession>>;
/// Delete an image upload session.
async fn delete_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<()>;
/// List image upload sessions for an image.
async fn list_image_upload_sessions(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
) -> StorageResult<Vec<ImageUploadSession>>;
/// Save a persistent volume
async fn save_volume(&self, volume: &Volume) -> StorageResult<()>;
@ -191,6 +240,27 @@ fn image_prefix(org_id: &str, project_id: &str) -> String {
format!("/plasmavmc/images/{}/{}/", org_id, project_id)
}
/// Build key for image upload session metadata
fn image_upload_session_key(
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> String {
format!(
"/plasmavmc/image-upload-sessions/{}/{}/{}/{}",
org_id, project_id, image_id, session_id
)
}
/// Build prefix for image upload session listing
fn image_upload_session_prefix(org_id: &str, project_id: &str, image_id: &str) -> String {
format!(
"/plasmavmc/image-upload-sessions/{}/{}/{}/",
org_id, project_id, image_id
)
}
/// Build key for volume metadata
fn volume_key(org_id: &str, project_id: &str, volume_id: &str) -> String {
format!("/plasmavmc/volumes/{}/{}/{}", org_id, project_id, volume_id)
@ -207,14 +277,20 @@ pub struct FlareDBStore {
}
impl FlareDBStore {
/// Create a new FlareDB store
pub async fn new(endpoint: Option<String>) -> StorageResult<Self> {
let endpoint = endpoint.unwrap_or_else(|| {
std::env::var("PLASMAVMC_FLAREDB_ENDPOINT")
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
});
let pd_endpoint = std::env::var("PLASMAVMC_CHAINFIRE_ENDPOINT")
.ok()
pub async fn new_with_config(config: &StorageRuntimeConfig) -> StorageResult<Self> {
Self::new_with_endpoints(
config.flaredb_endpoint.clone(),
config.chainfire_endpoint.clone(),
)
.await
}
pub async fn new_with_endpoints(
endpoint: Option<String>,
pd_endpoint: Option<String>,
) -> StorageResult<Self> {
let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
let pd_endpoint = pd_endpoint
.map(|value| normalize_transport_addr(&value))
.unwrap_or_else(|| endpoint.clone());
@ -561,6 +637,58 @@ impl VmStore for FlareDBStore {
Ok(images)
}
async fn save_image_upload_session(&self, session: &ImageUploadSession) -> StorageResult<()> {
let key = image_upload_session_key(
&session.org_id,
&session.project_id,
&session.image_id,
&session.session_id,
);
let value = serde_json::to_vec(session)?;
self.cas_put(&key, value).await
}
async fn load_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<Option<ImageUploadSession>> {
let key = image_upload_session_key(org_id, project_id, image_id, session_id);
match self.cas_get(&key).await? {
Some(data) => Ok(Some(serde_json::from_slice(&data)?)),
None => Ok(None),
}
}
async fn delete_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<()> {
let key = image_upload_session_key(org_id, project_id, image_id, session_id);
self.cas_delete(&key).await
}
async fn list_image_upload_sessions(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
) -> StorageResult<Vec<ImageUploadSession>> {
let prefix = image_upload_session_prefix(org_id, project_id, image_id);
let mut sessions = Vec::new();
for value in self.cas_scan_values(&prefix).await? {
if let Ok(session) = serde_json::from_slice::<ImageUploadSession>(&value) {
sessions.push(session);
}
}
Ok(sessions)
}
async fn save_volume(&self, volume: &Volume) -> StorageResult<()> {
let key = volume_key(&volume.org_id, &volume.project_id, &volume.id);
let value = serde_json::to_vec(volume)?;
@ -653,6 +781,8 @@ struct PersistedState {
#[serde(default)]
images: Vec<Image>,
#[serde(default)]
image_upload_sessions: Vec<ImageUploadSession>,
#[serde(default)]
volumes: Vec<Volume>,
}
@ -841,6 +971,71 @@ impl VmStore for FileStore {
.collect())
}
async fn save_image_upload_session(&self, session: &ImageUploadSession) -> StorageResult<()> {
let mut state = self.load_state().unwrap_or_default();
state.image_upload_sessions.retain(|existing| {
!(existing.org_id == session.org_id
&& existing.project_id == session.project_id
&& existing.image_id == session.image_id
&& existing.session_id == session.session_id)
});
state.image_upload_sessions.push(session.clone());
self.save_state(&state)?;
Ok(())
}
async fn load_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<Option<ImageUploadSession>> {
let state = self.load_state().unwrap_or_default();
Ok(state.image_upload_sessions.into_iter().find(|session| {
session.org_id == org_id
&& session.project_id == project_id
&& session.image_id == image_id
&& session.session_id == session_id
}))
}
async fn delete_image_upload_session(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
session_id: &str,
) -> StorageResult<()> {
let mut state = self.load_state().unwrap_or_default();
state.image_upload_sessions.retain(|session| {
!(session.org_id == org_id
&& session.project_id == project_id
&& session.image_id == image_id
&& session.session_id == session_id)
});
self.save_state(&state)?;
Ok(())
}
async fn list_image_upload_sessions(
&self,
org_id: &str,
project_id: &str,
image_id: &str,
) -> StorageResult<Vec<ImageUploadSession>> {
let state = self.load_state().unwrap_or_default();
Ok(state
.image_upload_sessions
.into_iter()
.filter(|session| {
session.org_id == org_id
&& session.project_id == project_id
&& session.image_id == image_id
})
.collect())
}
async fn save_volume(&self, volume: &Volume) -> StorageResult<()> {
let mut state = self.load_state().unwrap_or_default();
state.volumes.retain(|existing| existing.id != volume.id);
@ -972,4 +1167,68 @@ mod tests {
.expect("current volume should remain");
assert_eq!(loaded, current);
}
#[tokio::test]
async fn filestore_round_trips_image_upload_sessions() {
let tempdir = tempdir().unwrap();
let store = FileStore::new(Some(tempdir.path().join("state.json")));
let session = ImageUploadSession {
session_id: "11111111-1111-1111-1111-111111111111".to_string(),
org_id: "org-1".to_string(),
project_id: "project-1".to_string(),
image_id: "22222222-2222-2222-2222-222222222222".to_string(),
upload_id: "multipart-1".to_string(),
staging_key: "org-1/project-1/uploads/22222222-2222-2222-2222-222222222222.source"
.to_string(),
source_format: ImageFormat::Raw,
parts: vec![ImageUploadPart {
part_number: 1,
etag: "etag-1".to_string(),
size_bytes: 4096,
}],
committed_size_bytes: 4096,
created_at: 1,
updated_at: 2,
};
store.save_image_upload_session(&session).await.unwrap();
let loaded = store
.load_image_upload_session(
&session.org_id,
&session.project_id,
&session.image_id,
&session.session_id,
)
.await
.unwrap()
.expect("session should exist");
assert_eq!(loaded, session);
let listed = store
.list_image_upload_sessions(&session.org_id, &session.project_id, &session.image_id)
.await
.unwrap();
assert_eq!(listed, vec![session.clone()]);
store
.delete_image_upload_session(
&session.org_id,
&session.project_id,
&session.image_id,
&session.session_id,
)
.await
.unwrap();
assert!(store
.load_image_upload_session(
&session.org_id,
&session.project_id,
&session.image_id,
&session.session_id,
)
.await
.unwrap()
.is_none());
}
}

653
plasmavmc/crates/plasmavmc-server/src/vm_service.rs
View file

@ -1,8 +1,9 @@
//! VM Service implementation
use crate::artifact_store::ArtifactStore;
use crate::config::{DefaultHypervisor, ServerConfig};
use crate::prismnet_client::PrismNETClient;
use crate::storage::{FileStore, FlareDBStore, StorageBackend, VmStore};
use crate::storage::{FileStore, FlareDBStore, ImageUploadSession, StorageBackend, VmStore};
use crate::volume_manager::VolumeManager;
use crate::watcher::StateSink;
use creditservice_client::{Client as CreditServiceClient, ResourceType as CreditResourceType};
@ -11,13 +12,15 @@ use iam_client::client::IamClientConfig;
use iam_client::IamClient;
use iam_service_auth::{
get_tenant_context, resolve_tenant_ids_from_context, resource_for_tenant, AuthService,
TenantContext,
};
use iam_types::{PolicyBinding, PrincipalRef, Scope};
use iam_types::{PolicyBinding, PrincipalKind, PrincipalRef, Scope};
use plasmavmc_api::proto::{
disk_source::Source as ProtoDiskSourceKind, image_service_server::ImageService,
node_service_server::NodeService, vm_service_client::VmServiceClient,
vm_service_server::VmService, volume_service_server::VolumeService,
Architecture as ProtoArchitecture, AttachDiskRequest, AttachNicRequest, CephRbdBacking,
vm_service_server::VmService, volume_service_server::VolumeService, AbortImageUploadRequest,
Architecture as ProtoArchitecture, AttachDiskRequest, AttachNicRequest,
BeginImageUploadRequest, BeginImageUploadResponse, CephRbdBacking, CompleteImageUploadRequest,
CordonNodeRequest, CreateImageRequest, CreateVmRequest, CreateVolumeRequest,
DeleteImageRequest, DeleteVmRequest, DeleteVolumeRequest, DetachDiskRequest, DetachNicRequest,
DiskBus as ProtoDiskBus, DiskCache as ProtoDiskCache, DiskSource as ProtoDiskSource,
@ -30,9 +33,9 @@ use plasmavmc_api::proto::{
NodeState as ProtoNodeState, OsType as ProtoOsType, PrepareVmMigrationRequest, RebootVmRequest,
RecoverVmRequest, RegisterExternalVolumeRequest, ResetVmRequest, ResizeVolumeRequest,
StartVmRequest, StopVmRequest, UncordonNodeRequest, UpdateImageRequest, UpdateVmRequest,
VirtualMachine, Visibility as ProtoVisibility, VmEvent, VmEventType as ProtoVmEventType,
VmSpec as ProtoVmSpec, VmState as ProtoVmState, VmStatus as ProtoVmStatus,
Volume as ProtoVolume, VolumeBacking as ProtoVolumeBacking,
UploadImagePartRequest, UploadImagePartResponse, VirtualMachine, Visibility as ProtoVisibility,
VmEvent, VmEventType as ProtoVmEventType, VmSpec as ProtoVmSpec, VmState as ProtoVmState,
VmStatus as ProtoVmStatus, Volume as ProtoVolume, VolumeBacking as ProtoVolumeBacking,
VolumeDriverKind as ProtoVolumeDriverKind, VolumeFormat as ProtoVolumeFormat,
VolumeStatus as ProtoVolumeStatus, WatchVmRequest,
};
@ -91,6 +94,12 @@ pub struct VmServiceImpl {
credit_service: Option<Arc<RwLock<CreditServiceClient>>>,
/// Local node identifier (optional)
local_node_id: Option<String>,
shared_live_migration: bool,
default_hypervisor: HypervisorType,
watch_poll_interval: Duration,
auto_restart: bool,
failover_enabled: bool,
failover_min_interval_secs: u64,
artifact_store: Option<Arc<ArtifactStore>>,
volume_manager: Arc<VolumeManager>,
iam_client: Arc<IamClient>,
@ -149,33 +158,34 @@ impl VmServiceImpl {
hypervisor_registry: Arc<HypervisorRegistry>,
auth: Arc<AuthService>,
iam_endpoint: impl Into<String>,
config: &ServerConfig,
) -> Result<Self, Box<dyn std::error::Error>> {
let backend = StorageBackend::from_env();
let backend = StorageBackend::from_config(&config.storage);
let store: Arc<dyn VmStore> = match backend {
StorageBackend::FlareDB => match FlareDBStore::new(None).await {
StorageBackend::FlareDB => match FlareDBStore::new_with_config(&config.storage).await {
Ok(flaredb_store) => Arc::new(flaredb_store),
Err(e) => {
tracing::warn!(
"Failed to connect to FlareDB, falling back to file storage: {}",
e
);
Arc::new(FileStore::new(None))
Arc::new(FileStore::new(config.storage.state_path.clone()))
}
},
StorageBackend::File => {
let file_store = FileStore::new(None);
let file_store = FileStore::new(config.storage.state_path.clone());
Arc::new(file_store)
}
};
let prismnet_endpoint = std::env::var("PRISMNET_ENDPOINT").ok();
let prismnet_endpoint = config.integrations.prismnet_endpoint.clone();
if let Some(ref endpoint) = prismnet_endpoint {
tracing::info!("PrismNET integration enabled: {}", endpoint);
}
// Initialize CreditService client if endpoint is configured
let credit_service = match std::env::var("CREDITSERVICE_ENDPOINT") {
Ok(endpoint) => match CreditServiceClient::connect(&endpoint).await {
let credit_service = match config.integrations.creditservice_endpoint.as_deref() {
Some(endpoint) => match CreditServiceClient::connect(endpoint).await {
Ok(client) => {
tracing::info!("CreditService admission control enabled: {}", endpoint);
Some(Arc::new(RwLock::new(client)))
@ -188,13 +198,17 @@ impl VmServiceImpl {
None
}
},
Err(_) => {
tracing::info!("CREDITSERVICE_ENDPOINT not set, admission control disabled");
None => {
tracing::info!("CreditService endpoint not configured, admission control disabled");
None
}
};
let local_node_id = std::env::var("PLASMAVMC_NODE_ID").ok();
let local_node_id = config
.agent
.node_id
.clone()
.filter(|value| !value.trim().is_empty());
if let Some(ref node_id) = local_node_id {
tracing::info!("Local node ID: {}", node_id);
}
@ -206,13 +220,25 @@ impl VmServiceImpl {
iam_config = iam_config.without_tls();
}
let iam_client = Arc::new(IamClient::connect(iam_config).await?);
let artifact_store = ArtifactStore::from_env(&normalized_iam_endpoint)
.await?
.map(Arc::new);
let artifact_store =
ArtifactStore::from_config(&config.artifacts, &normalized_iam_endpoint)
.await?
.map(Arc::new);
if artifact_store.is_some() {
tracing::info!("LightningStor artifact backing enabled for VM disks");
}
let volume_manager = Arc::new(VolumeManager::new(store.clone(), artifact_store.clone()));
let volume_manager = Arc::new(VolumeManager::new_with_config(
store.clone(),
artifact_store.clone(),
&config.volumes,
local_node_id.clone(),
)?);
let default_hypervisor = match config.default_hypervisor {
DefaultHypervisor::Kvm => HypervisorType::Kvm,
DefaultHypervisor::Firecracker => HypervisorType::Firecracker,
DefaultHypervisor::Mvisor => HypervisorType::Mvisor,
};
let svc = Self {
hypervisor_registry,
@ -224,6 +250,14 @@ impl VmServiceImpl {
prismnet_endpoint,
credit_service,
local_node_id,
shared_live_migration: config.agent.shared_live_migration,
default_hypervisor,
watch_poll_interval: Duration::from_millis(
config.watcher.vm_watch_poll_interval_ms.max(100),
),
auto_restart: config.health.auto_restart,
failover_enabled: config.health.failover_enabled,
failover_min_interval_secs: config.health.failover_min_interval_secs.max(1),
artifact_store,
volume_manager,
iam_client,
@ -242,10 +276,7 @@ impl VmServiceImpl {
}
pub fn shared_live_migration(&self) -> bool {
std::env::var("PLASMAVMC_SHARED_LIVE_MIGRATION")
.ok()
.map(|value| matches!(value.as_str(), "1" | "true" | "yes"))
.unwrap_or(true)
self.shared_live_migration
}
pub fn store(&self) -> Arc<dyn VmStore> {
@ -256,24 +287,12 @@ impl VmServiceImpl {
Status::internal(err.to_string())
}
fn map_hv(typ: ProtoHypervisorType) -> HypervisorType {
fn map_hv(&self, typ: ProtoHypervisorType) -> HypervisorType {
match typ {
ProtoHypervisorType::Kvm => HypervisorType::Kvm,
ProtoHypervisorType::Firecracker => HypervisorType::Firecracker,
ProtoHypervisorType::Mvisor => HypervisorType::Mvisor,
ProtoHypervisorType::Unspecified => {
// Use environment variable for default, fallback to KVM
match std::env::var("PLASMAVMC_HYPERVISOR")
.as_deref()
.map(|s| s.to_lowercase())
.as_deref()
{
Ok("firecracker") => HypervisorType::Firecracker,
Ok("kvm") => HypervisorType::Kvm,
Ok("mvisor") => HypervisorType::Mvisor,
_ => HypervisorType::Kvm, // Default to KVM for backwards compatibility
}
}
ProtoHypervisorType::Unspecified => self.default_hypervisor,
}
}
@ -292,13 +311,81 @@ impl VmServiceImpl {
.as_secs()
}
fn watch_poll_interval() -> Duration {
let poll_interval_ms = std::env::var("PLASMAVMC_VM_WATCH_POLL_INTERVAL_MS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(500)
.max(100);
Duration::from_millis(poll_interval_ms)
fn require_uuid(value: &str, field_name: &str) -> Result<(), Status> {
Uuid::parse_str(value)
.map(|_| ())
.map_err(|_| Status::invalid_argument(format!("{field_name} must be a UUID")))
}
fn validate_disk_reference(disk: &plasmavmc_types::DiskSpec) -> Result<(), Status> {
match &disk.source {
DiskSource::Image { image_id } => Self::require_uuid(image_id, "image_id"),
DiskSource::Volume { volume_id } => Self::require_uuid(volume_id, "volume_id"),
DiskSource::Blank => Ok(()),
}
}
fn validate_vm_disk_references(spec: &plasmavmc_types::VmSpec) -> Result<(), Status> {
for disk in &spec.disks {
Self::validate_disk_reference(disk)?;
}
Ok(())
}
fn ensure_internal_rpc(tenant: &TenantContext) -> Result<(), Status> {
if tenant.principal_kind != PrincipalKind::ServiceAccount
|| !tenant.principal_id.starts_with("plasmavmc-")
{
return Err(Status::permission_denied(
"this RPC is restricted to internal PlasmaVMC service accounts",
));
}
Ok(())
}
fn expected_migration_listener_uri(&self, vm_id: &str) -> Result<String, Status> {
let node_id = self.local_node_id.as_deref().ok_or_else(|| {
Status::failed_precondition("local node id is required for migration preparation")
})?;
let mut hasher = std::collections::hash_map::DefaultHasher::new();
node_id.hash(&mut hasher);
vm_id.hash(&mut hasher);
let port = 4400 + (hasher.finish() % 1000) as u16;
Ok(format!("tcp:0.0.0.0:{port}"))
}
fn apply_imported_image_metadata(
image: &mut Image,
imported: &crate::artifact_store::ImportedImage,
source_format: ImageFormat,
) {
image.status = ImageStatus::Available;
image.format = imported.format;
image.size_bytes = imported.size_bytes;
image.checksum = imported.checksum.clone();
image.updated_at = Self::now_epoch();
image
.metadata
.insert("source_type".to_string(), imported.source_type.clone());
if let Some(host) = &imported.source_host {
image
.metadata
.insert("source_host".to_string(), host.clone());
}
image.metadata.insert(
"artifact_key".to_string(),
format!("{}/{}/{}.qcow2", image.org_id, image.project_id, image.id),
);
if source_format != image.format {
image.metadata.insert(
"source_format".to_string(),
format!("{source_format:?}").to_lowercase(),
);
}
}
fn watch_poll_interval(&self) -> Duration {
self.watch_poll_interval
}
fn vm_values_differ(
@ -954,7 +1041,10 @@ impl VmServiceImpl {
}
}
fn proto_vm_to_types(vm: &VirtualMachine) -> Result<plasmavmc_types::VirtualMachine, Status> {
fn proto_vm_to_types(
&self,
vm: &VirtualMachine,
) -> Result<plasmavmc_types::VirtualMachine, Status> {
let spec = Self::proto_spec_to_types(vm.spec.clone());
let mut typed = plasmavmc_types::VirtualMachine::new(
vm.name.clone(),
@ -965,7 +1055,7 @@ impl VmServiceImpl {
typed.id = VmId::from_uuid(
Uuid::parse_str(&vm.id).map_err(|e| Status::internal(format!("invalid VM id: {e}")))?,
);
typed.hypervisor = Self::map_hv(
typed.hypervisor = self.map_hv(
ProtoHypervisorType::try_from(vm.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
);
typed.node_id = if vm.node_id.is_empty() {
@ -1690,9 +1780,6 @@ impl VmServiceImpl {
.map(|entry| (entry.key().clone(), entry.value().clone()))
.collect();
let auto_restart = std::env::var("PLASMAVMC_AUTO_RESTART")
.map(|v| matches!(v.as_str(), "1" | "true" | "yes"))
.unwrap_or(false);
let local_node = self.local_node_id.as_deref();
for (key, mut vm) in entries {
@ -1733,7 +1820,7 @@ impl VmServiceImpl {
match backend.status(&handle).await {
Ok(status) => {
if auto_restart
if self.auto_restart
&& vm.state == VmState::Running
&& matches!(status.actual_state, VmState::Stopped | VmState::Error)
{
@ -1812,18 +1899,10 @@ impl VmServiceImpl {
return;
}
let failover_enabled = std::env::var("PLASMAVMC_FAILOVER_CONTROLLER")
.map(|v| matches!(v.as_str(), "1" | "true" | "yes"))
.unwrap_or(false);
let min_interval_secs = std::env::var("PLASMAVMC_FAILOVER_MIN_INTERVAL_SECS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(60);
let mut failed_over: HashSet<String> = HashSet::new();
if failover_enabled {
if self.failover_enabled {
failed_over = self
.failover_vms_on_unhealthy(&unhealthy, &nodes, min_interval_secs)
.failover_vms_on_unhealthy(&unhealthy, &nodes, self.failover_min_interval_secs)
.await;
}
@ -2047,6 +2126,43 @@ mod tests {
.unwrap();
assert_eq!(deleted.event_type, ProtoVmEventType::Deleted as i32);
}
#[test]
fn internal_rpc_guard_requires_plasmavmc_service_account() {
let internal = TenantContext {
org_id: "org".to_string(),
project_id: "project".to_string(),
principal_id: "plasmavmc-org-project".to_string(),
principal_name: "plasmavmc".to_string(),
principal_kind: PrincipalKind::ServiceAccount,
node_id: None,
};
let external = TenantContext {
principal_id: "user-1".to_string(),
principal_kind: PrincipalKind::User,
..internal.clone()
};
assert!(VmServiceImpl::ensure_internal_rpc(&internal).is_ok());
assert!(VmServiceImpl::ensure_internal_rpc(&external).is_err());
}
#[test]
fn vm_disk_reference_validation_requires_uuid_identifiers() {
let mut spec = VmSpec::default();
spec.disks.push(plasmavmc_types::DiskSpec {
id: "root".to_string(),
source: DiskSource::Image {
image_id: "../passwd".to_string(),
},
size_gib: 10,
bus: DiskBus::Virtio,
cache: DiskCache::Writeback,
boot_index: Some(1),
});
assert!(VmServiceImpl::validate_vm_disk_references(&spec).is_err());
}
}
impl StateSink for VmServiceImpl {
@ -2121,13 +2237,14 @@ impl VmService for VmServiceImpl {
"CreateVm request"
);
let hv = Self::map_hv(
let hv = self.map_hv(
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
);
if req.spec.is_none() {
return Err(Status::invalid_argument("spec is required"));
}
let spec = Self::proto_spec_to_types(req.spec.clone());
Self::validate_vm_disk_references(&spec)?;
if self.is_control_plane_scheduler() {
if let Some(target) = self
.select_target_node(hv, &req.org_id, &req.project_id, &spec)
@ -2137,7 +2254,7 @@ impl VmService for VmServiceImpl {
let forwarded = self
.forward_create_to_node(endpoint, &req.org_id, &req.project_id, &req)
.await?;
let forwarded_vm = Self::proto_vm_to_types(&forwarded)?;
let forwarded_vm = self.proto_vm_to_types(&forwarded)?;
let key = TenantKey::new(
&forwarded_vm.org_id,
&forwarded_vm.project_id,
@ -2395,7 +2512,7 @@ impl VmService for VmServiceImpl {
.await
.map_err(|status| Status::from_error(Box::new(status)))?
.into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key.clone(), typed_vm.clone());
self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm));
@ -2700,7 +2817,7 @@ impl VmService for VmServiceImpl {
.await
.map_err(|status| Status::from_error(Box::new(status)))?
.into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm));
@ -2802,7 +2919,7 @@ impl VmService for VmServiceImpl {
.await
.map_err(|status| Status::from_error(Box::new(status)))?
.into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm));
@ -2889,7 +3006,7 @@ impl VmService for VmServiceImpl {
.await
.map_err(|status| Status::from_error(Box::new(status)))?
.into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm));
@ -2970,7 +3087,7 @@ impl VmService for VmServiceImpl {
.await
.map_err(|status| Status::from_error(Box::new(status)))?
.into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm));
@ -3061,7 +3178,7 @@ impl VmService for VmServiceImpl {
.await
.map_err(|status| Status::from_error(Box::new(status)))?
.into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await;
return Ok(Response::new(remote_vm));
@ -3171,7 +3288,7 @@ impl VmService for VmServiceImpl {
return match client.recover_vm(recover_req).await {
Ok(remote_vm) => {
let remote_vm = remote_vm.into_inner();
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
self.vms.insert(key, typed_vm.clone());
self.persist_vm(&typed_vm).await;
Ok(Response::new(remote_vm))
@ -3352,6 +3469,7 @@ impl VmService for VmServiceImpl {
request: Request<PrepareVmMigrationRequest>,
) -> Result<Response<VirtualMachine>, Status> {
let tenant = get_tenant_context(&request)?;
Self::ensure_internal_rpc(&tenant)?;
let (org_id, project_id) = resolve_tenant_ids_from_context(
&tenant,
&request.get_ref().org_id,
@ -3378,6 +3496,12 @@ impl VmService for VmServiceImpl {
if req.listen_uri.is_empty() {
return Err(Status::invalid_argument("listen_uri is required"));
}
let expected_listen_uri = self.expected_migration_listener_uri(&req.vm_id)?;
if req.listen_uri != expected_listen_uri {
return Err(Status::invalid_argument(format!(
"listen_uri must exactly match {expected_listen_uri}",
)));
}
self.ensure_destination_slot_available(&req.org_id, &req.project_id, &req.vm_id)
.await?;
@ -3385,7 +3509,7 @@ impl VmService for VmServiceImpl {
let vm_uuid = Uuid::parse_str(&req.vm_id)
.map_err(|_| Status::invalid_argument("vm_id must be a UUID"))?;
let hv = Self::map_hv(
let hv = self.map_hv(
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
);
let backend = self
@ -3399,6 +3523,7 @@ impl VmService for VmServiceImpl {
}
let spec = Self::proto_spec_to_types(req.spec);
Self::validate_vm_disk_references(&spec)?;
let name = if req.name.is_empty() {
req.vm_id.clone()
} else {
@ -3440,6 +3565,7 @@ impl VmService for VmServiceImpl {
request: Request<RecoverVmRequest>,
) -> Result<Response<VirtualMachine>, Status> {
let tenant = get_tenant_context(&request)?;
Self::ensure_internal_rpc(&tenant)?;
let (org_id, project_id) = resolve_tenant_ids_from_context(
&tenant,
&request.get_ref().org_id,
@ -3469,7 +3595,7 @@ impl VmService for VmServiceImpl {
let vm_uuid = Uuid::parse_str(&req.vm_id)
.map_err(|_| Status::invalid_argument("vm_id must be a UUID"))?;
let hv = Self::map_hv(
let hv = self.map_hv(
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
);
let backend = self
@ -3481,6 +3607,7 @@ impl VmService for VmServiceImpl {
.await?;
let spec = Self::proto_spec_to_types(req.spec);
Self::validate_vm_disk_references(&spec)?;
let name = if req.name.is_empty() {
req.vm_id.clone()
} else {
@ -3584,6 +3711,7 @@ impl VmService for VmServiceImpl {
.disk
.ok_or_else(|| Status::invalid_argument("disk spec required"))?;
let mut disk_spec = Self::proto_disk_to_types(proto_disk);
Self::validate_disk_reference(&disk_spec)?;
if vm.spec.disks.iter().any(|disk| disk.id == disk_spec.id) {
return Err(Status::already_exists("disk already attached"));
}
@ -3858,7 +3986,7 @@ impl VmService for VmServiceImpl {
.ensure_vm_loaded(&req.org_id, &req.project_id, &req.vm_id)
.await
.ok_or_else(|| Status::not_found("VM not found"))?;
let poll_interval = Self::watch_poll_interval();
let poll_interval = self.watch_poll_interval();
let store = Arc::clone(&self.store);
let org_id = req.org_id.clone();
let project_id = req.project_id.clone();
@ -3948,6 +4076,9 @@ impl VolumeService for VmServiceImpl {
"size_gib must be greater than zero",
));
}
if !req.image_id.trim().is_empty() {
Self::require_uuid(&req.image_id, "image_id")?;
}
let driver = Self::map_volume_driver(
ProtoVolumeDriverKind::try_from(req.driver).unwrap_or(ProtoVolumeDriverKind::Managed),
@ -4206,6 +4337,9 @@ impl ImageService for VmServiceImpl {
if req.source_url.trim().is_empty() {
return Err(Status::invalid_argument("source_url is required"));
}
if !req.source_url.starts_with("https://") {
return Err(Status::invalid_argument("source_url must use https://"));
}
let Some(store) = self.artifact_store.as_ref() else {
return Err(Status::failed_precondition(
"LightningStor artifact backing is required for image imports",
@ -4245,20 +4379,7 @@ impl ImageService for VmServiceImpl {
.await
{
Ok(imported) => {
image.status = ImageStatus::Available;
image.format = imported.format;
image.size_bytes = imported.size_bytes;
image.checksum = imported.checksum;
image.updated_at = Self::now_epoch();
image
.metadata
.insert("source_url".to_string(), req.source_url.clone());
if source_format != image.format {
image.metadata.insert(
"source_format".to_string(),
format!("{source_format:?}").to_lowercase(),
);
}
Self::apply_imported_image_metadata(&mut image, &imported, source_format);
self.images.insert(key, image.clone());
self.persist_image(&image).await;
Ok(Response::new(Self::types_image_to_proto(&image)))
@ -4276,6 +4397,332 @@ impl ImageService for VmServiceImpl {
}
}
async fn begin_image_upload(
&self,
request: Request<BeginImageUploadRequest>,
) -> Result<Response<BeginImageUploadResponse>, Status> {
let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
self.auth
.authorize(
&tenant,
ACTION_IMAGE_CREATE,
&resource_for_tenant("image", "*", &org_id, &project_id),
)
.await?;
let mut req = request.into_inner();
if req.name.trim().is_empty() {
return Err(Status::invalid_argument("name is required"));
}
let Some(store) = self.artifact_store.as_ref() else {
return Err(Status::failed_precondition(
"LightningStor artifact backing is required for image uploads",
));
};
let source_format = Self::map_image_format(
ProtoImageFormat::try_from(req.format).unwrap_or(ProtoImageFormat::Qcow2),
);
let mut image = Image::new(req.name, &org_id, &project_id);
image.visibility = Self::map_visibility(
ProtoVisibility::try_from(req.visibility).unwrap_or(ProtoVisibility::Private),
);
image.os_type = Self::map_os_type(
ProtoOsType::try_from(req.os_type).unwrap_or(ProtoOsType::Unspecified),
);
image.os_version = req.os_version;
image.architecture = Self::map_architecture(req.architecture);
image.min_disk_gib = req.min_disk_gib;
image.min_memory_mib = req.min_memory_mib;
image.metadata = std::mem::take(&mut req.metadata);
image.status = ImageStatus::Uploading;
let key = TenantKey::new(&org_id, &project_id, &image.id);
self.images.insert(key.clone(), image.clone());
self.persist_image(&image).await;
let begin_result = store
.begin_image_upload(&org_id, &project_id, &image.id)
.await;
match begin_result {
Ok((upload_id, staging_key)) => {
let now = Self::now_epoch();
let session = ImageUploadSession {
session_id: Uuid::new_v4().to_string(),
org_id: org_id.clone(),
project_id: project_id.clone(),
image_id: image.id.clone(),
upload_id,
staging_key,
source_format,
parts: Vec::new(),
committed_size_bytes: 0,
created_at: now,
updated_at: now,
};
if let Err(error) = self.store.save_image_upload_session(&session).await {
let _ = store
.abort_image_upload(
&org_id,
&project_id,
&session.staging_key,
&session.upload_id,
)
.await;
image.status = ImageStatus::Error;
image.updated_at = Self::now_epoch();
image.metadata.insert(
"last_error".to_string(),
format!("failed to persist image upload session: {error}"),
);
self.images.insert(key, image.clone());
self.persist_image(&image).await;
return Err(Status::internal(format!(
"failed to persist image upload session: {error}"
)));
}
Ok(Response::new(BeginImageUploadResponse {
image: Some(Self::types_image_to_proto(&image)),
upload_session_id: session.session_id,
minimum_part_size_bytes: store.minimum_upload_part_size(),
}))
}
Err(error) => {
image.status = ImageStatus::Error;
image.updated_at = Self::now_epoch();
image
.metadata
.insert("last_error".to_string(), error.message().to_string());
self.images.insert(key, image.clone());
self.persist_image(&image).await;
Err(error)
}
}
}
async fn upload_image_part(
&self,
request: Request<UploadImagePartRequest>,
) -> Result<Response<UploadImagePartResponse>, Status> {
let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
Self::require_uuid(&req.upload_session_id, "upload_session_id")?;
self.auth
.authorize(
&tenant,
ACTION_IMAGE_CREATE,
&resource_for_tenant("image", req.image_id.clone(), &org_id, &project_id),
)
.await?;
let Some(store) = self.artifact_store.as_ref() else {
return Err(Status::failed_precondition(
"LightningStor artifact backing is required for image uploads",
));
};
let Some(mut session) = self
.store
.load_image_upload_session(&org_id, &project_id, &req.image_id, &req.upload_session_id)
.await
.map_err(|error| {
Status::internal(format!("failed to load image upload session: {error}"))
})?
else {
return Err(Status::not_found("image upload session not found"));
};
let Some(image) = self
.ensure_image_loaded(&org_id, &project_id, &req.image_id)
.await
else {
return Err(Status::not_found("image not found"));
};
if image.status != ImageStatus::Uploading {
return Err(Status::failed_precondition(
"image is not accepting uploaded parts",
));
}
let part = store
.upload_image_part(
&org_id,
&project_id,
&session.staging_key,
&session.upload_id,
req.part_number,
req.body,
)
.await?;
session
.parts
.retain(|existing| existing.part_number != part.part_number);
session.parts.push(part);
session.parts.sort_by_key(|part| part.part_number);
session.committed_size_bytes = session.parts.iter().map(|part| part.size_bytes).sum();
session.updated_at = Self::now_epoch();
self.store
.save_image_upload_session(&session)
.await
.map_err(|error| {
Status::internal(format!("failed to persist image upload session: {error}"))
})?;
Ok(Response::new(UploadImagePartResponse {
part_number: req.part_number,
committed_size_bytes: session.committed_size_bytes,
}))
}
async fn complete_image_upload(
&self,
request: Request<CompleteImageUploadRequest>,
) -> Result<Response<ProtoImage>, Status> {
let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
Self::require_uuid(&req.upload_session_id, "upload_session_id")?;
self.auth
.authorize(
&tenant,
ACTION_IMAGE_CREATE,
&resource_for_tenant("image", req.image_id.clone(), &org_id, &project_id),
)
.await?;
let Some(store) = self.artifact_store.as_ref() else {
return Err(Status::failed_precondition(
"LightningStor artifact backing is required for image uploads",
));
};
let key = TenantKey::new(&org_id, &project_id, &req.image_id);
let Some(mut image) = self
.ensure_image_loaded(&org_id, &project_id, &req.image_id)
.await
else {
return Err(Status::not_found("image not found"));
};
let Some(session) = self
.store
.load_image_upload_session(&org_id, &project_id, &req.image_id, &req.upload_session_id)
.await
.map_err(|error| {
Status::internal(format!("failed to load image upload session: {error}"))
})?
else {
return Err(Status::not_found("image upload session not found"));
};
match store
.complete_image_upload(
&org_id,
&project_id,
&req.image_id,
&session.staging_key,
&session.upload_id,
&session.parts,
session.source_format,
)
.await
{
Ok(imported) => {
Self::apply_imported_image_metadata(&mut image, &imported, session.source_format);
self.images.insert(key, image.clone());
self.persist_image(&image).await;
let _ = self
.store
.delete_image_upload_session(
&org_id,
&project_id,
&req.image_id,
&req.upload_session_id,
)
.await;
Ok(Response::new(Self::types_image_to_proto(&image)))
}
Err(error) => {
image.status = ImageStatus::Error;
image.updated_at = Self::now_epoch();
image
.metadata
.insert("last_error".to_string(), error.message().to_string());
self.images.insert(key, image.clone());
self.persist_image(&image).await;
Err(error)
}
}
}
async fn abort_image_upload(
&self,
request: Request<AbortImageUploadRequest>,
) -> Result<Response<Empty>, Status> {
let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
Self::require_uuid(&req.upload_session_id, "upload_session_id")?;
self.auth
.authorize(
&tenant,
ACTION_IMAGE_CREATE,
&resource_for_tenant("image", req.image_id.clone(), &org_id, &project_id),
)
.await?;
let Some(store) = self.artifact_store.as_ref() else {
return Err(Status::failed_precondition(
"LightningStor artifact backing is required for image uploads",
));
};
let Some(session) = self
.store
.load_image_upload_session(&org_id, &project_id, &req.image_id, &req.upload_session_id)
.await
.map_err(|error| {
Status::internal(format!("failed to load image upload session: {error}"))
})?
else {
return Err(Status::not_found("image upload session not found"));
};
store
.abort_image_upload(
&org_id,
&project_id,
&session.staging_key,
&session.upload_id,
)
.await?;
self.store
.delete_image_upload_session(
&org_id,
&project_id,
&req.image_id,
&req.upload_session_id,
)
.await
.map_err(|error| {
Status::internal(format!("failed to delete image upload session: {error}"))
})?;
if let Some(mut image) = self
.ensure_image_loaded(&org_id, &project_id, &req.image_id)
.await
{
let key = TenantKey::new(&org_id, &project_id, &req.image_id);
image.status = ImageStatus::Error;
image.updated_at = Self::now_epoch();
image
.metadata
.insert("last_error".to_string(), "upload aborted".to_string());
self.images.insert(key, image.clone());
self.persist_image(&image).await;
}
Ok(Response::new(Empty {}))
}
async fn get_image(
&self,
request: Request<GetImageRequest>,
@ -4283,6 +4730,7 @@ impl ImageService for VmServiceImpl {
let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
self.auth
.authorize(
&tenant,
@ -4337,6 +4785,7 @@ impl ImageService for VmServiceImpl {
let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
self.auth
.authorize(
&tenant,
@ -4378,6 +4827,7 @@ impl ImageService for VmServiceImpl {
let tenant = get_tenant_context(&request)?;
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
let req = request.into_inner();
Self::require_uuid(&req.image_id, "image_id")?;
self.auth
.authorize(
&tenant,
@ -4396,6 +4846,31 @@ impl ImageService for VmServiceImpl {
}
if let Some(store) = self.artifact_store.as_ref() {
if let Ok(sessions) = self
.store
.list_image_upload_sessions(&org_id, &project_id, &req.image_id)
.await
{
for session in sessions {
let _ = store
.abort_image_upload(
&org_id,
&project_id,
&session.staging_key,
&session.upload_id,
)
.await;
let _ = self
.store
.delete_image_upload_session(
&org_id,
&project_id,
&req.image_id,
&session.session_id,
)
.await;
}
}
store
.delete_image(&org_id, &project_id, &req.image_id)
.await?;
@ -4540,7 +5015,7 @@ impl NodeService for VmServiceImpl {
.hypervisors
.iter()
.filter_map(|h| ProtoHypervisorType::try_from(*h).ok())
.map(Self::map_hv)
.map(|hypervisor| self.map_hv(hypervisor))
.collect();
}
if !req.supported_volume_drivers.is_empty() {

423
plasmavmc/crates/plasmavmc-server/src/volume_manager.rs
View file

@ -1,4 +1,5 @@
use crate::artifact_store::ArtifactStore;
use crate::config::{CephConfig, CoronaFsConfig, VolumeRuntimeConfig};
use crate::storage::VmStore;
use plasmavmc_types::{
AttachedDisk, DiskAttachment, DiskSource, DiskSpec, VirtualMachine, Volume, VolumeBacking,
@ -18,6 +19,7 @@ const AUTO_DELETE_VOLUME_SOURCE_METADATA_KEY: &str = "plasmavmc.auto_delete_sour
const CORONAFS_IMAGE_SOURCE_ID_METADATA_KEY: &str = "plasmavmc.coronafs_image_source_id";
const CORONAFS_IMAGE_SEED_PENDING_METADATA_KEY: &str = "plasmavmc.coronafs_image_seed_pending";
const VOLUME_METADATA_CAS_RETRIES: usize = 16;
const CEPH_IDENTIFIER_MAX_LEN: usize = 128;
#[derive(Clone, Debug)]
struct CephClusterConfig {
@ -116,6 +118,7 @@ pub struct VolumeManager {
artifact_store: Option<Arc<ArtifactStore>>,
managed_root: PathBuf,
supported_storage_classes: Vec<String>,
qemu_img_path: PathBuf,
ceph_cluster: Option<CephClusterConfig>,
coronafs_controller: Option<CoronaFsClient>,
coronafs_node: Option<CoronaFsClient>,
@ -124,31 +127,43 @@ pub struct VolumeManager {
}
impl VolumeManager {
pub fn new(store: Arc<dyn VmStore>, artifact_store: Option<Arc<ArtifactStore>>) -> Self {
let managed_root = std::env::var("PLASMAVMC_MANAGED_VOLUME_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("/var/lib/plasmavmc/managed-volumes"));
let ceph_cluster = std::env::var("PLASMAVMC_CEPH_MONITORS")
.ok()
.filter(|value| !value.trim().is_empty())
.map(|monitors| CephClusterConfig {
cluster_id: std::env::var("PLASMAVMC_CEPH_CLUSTER_ID")
.unwrap_or_else(|_| "default".to_string()),
monitors: monitors
.split(',')
.map(str::trim)
.filter(|item| !item.is_empty())
.map(ToOwned::to_owned)
.collect(),
user: std::env::var("PLASMAVMC_CEPH_USER").unwrap_or_else(|_| "admin".to_string()),
secret: std::env::var("PLASMAVMC_CEPH_SECRET").ok(),
});
let (coronafs_controller, coronafs_node) = resolve_coronafs_clients();
let coronafs_node_local_attach = coronafs_node_local_attach_enabled();
let local_node_id = std::env::var("PLASMAVMC_NODE_ID")
.ok()
.filter(|value| !value.trim().is_empty());
pub fn new_with_config(
store: Arc<dyn VmStore>,
artifact_store: Option<Arc<ArtifactStore>>,
config: &VolumeRuntimeConfig,
local_node_id: Option<String>,
) -> Result<Self, Box<dyn std::error::Error>> {
let managed_root = config.managed_volume_root.clone();
let ceph_cluster = ceph_cluster_from_config(&config.ceph);
let (coronafs_controller, coronafs_node) =
resolve_coronafs_clients_with_config(&config.coronafs);
let coronafs_node_local_attach = config.coronafs.node_local_attach;
let qemu_img_path = resolve_binary_path(config.qemu_img_path.as_deref(), "qemu-img")?;
Ok(Self::build(
store,
artifact_store,
managed_root,
qemu_img_path,
ceph_cluster,
coronafs_controller,
coronafs_node,
coronafs_node_local_attach,
local_node_id,
))
}
fn build(
store: Arc<dyn VmStore>,
artifact_store: Option<Arc<ArtifactStore>>,
managed_root: PathBuf,
qemu_img_path: PathBuf,
ceph_cluster: Option<CephClusterConfig>,
coronafs_controller: Option<CoronaFsClient>,
coronafs_node: Option<CoronaFsClient>,
coronafs_node_local_attach: bool,
local_node_id: Option<String>,
) -> Self {
Self {
store,
artifact_store,
@ -161,6 +176,7 @@ impl VolumeManager {
classes.push("ceph-rbd".to_string());
classes
},
qemu_img_path,
ceph_cluster,
coronafs_controller,
coronafs_node,
@ -320,6 +336,7 @@ impl VolumeManager {
Some(export) => export,
None => controller.ensure_export_read_only(volume_id).await?,
};
validate_coronafs_export(&export)?;
node_client
.materialize_from_export(
volume_id,
@ -377,6 +394,9 @@ impl VolumeManager {
{
return Ok(existing);
}
if let Some(image_id) = image_id {
validate_uuid(image_id, "image_id")?;
}
let path = self.managed_volume_path(volume_id);
let mut metadata = metadata;
@ -503,6 +523,9 @@ impl VolumeManager {
"Ceph RBD support is not configured on this node",
));
}
validate_ceph_identifier("ceph_rbd.cluster_id", cluster_id)?;
validate_ceph_identifier("ceph_rbd.pool", pool)?;
validate_ceph_identifier("ceph_rbd.image", image)?;
let volume_id = Uuid::new_v4().to_string();
let mut volume = Volume::new(volume_id, name.to_string(), org_id, project_id, size_gib);
volume.driver = VolumeDriverKind::CephRbd;
@ -943,6 +966,7 @@ impl VolumeManager {
let Some(local_volume) = node_client.get_volume_optional(volume_id).await? else {
return Ok(());
};
validate_coronafs_volume_response(&local_volume)?;
if local_volume.export.is_some() {
node_client.release_export(volume_id).await?;
}
@ -975,6 +999,7 @@ impl VolumeManager {
Some(export) => export,
None => controller.ensure_export(volume_id).await?,
};
validate_coronafs_export(&export)?;
tracing::info!(
volume_id,
node_endpoint = %node_client.endpoint,
@ -983,7 +1008,7 @@ impl VolumeManager {
export_uri = %export.uri,
"Syncing node-local CoronaFS volume back to controller"
);
sync_local_coronafs_volume_to_export(&local_volume, &export.uri).await
sync_local_coronafs_volume_to_export(&self.qemu_img_path, &local_volume, &export.uri).await
}
async fn materialize_pending_coronafs_image_seed_on_node(
@ -1185,6 +1210,7 @@ impl VolumeManager {
if self.coronafs_attachment_backend().is_some() {
let (coronafs_volume, coronafs) =
self.load_coronafs_volume_for_attachment(volume).await?;
validate_coronafs_volume_response(&coronafs_volume)?;
if coronafs.supports_local_backing_file().await
&& !should_prefer_coronafs_export_attachment(&coronafs_volume)
&& coronafs_local_target_ready(&coronafs_volume.path).await
@ -1226,6 +1252,9 @@ impl VolumeManager {
cluster_id
)));
}
validate_ceph_identifier("ceph cluster_id", cluster_id)?;
validate_ceph_identifier("ceph pool", pool)?;
validate_ceph_identifier("ceph image", image)?;
DiskAttachment::CephRbd {
pool: pool.clone(),
image: image.clone(),
@ -1266,7 +1295,7 @@ impl VolumeManager {
.await
.map_err(|e| Status::internal(format!("failed to create volume dir: {e}")))?;
}
let status = Command::new("qemu-img")
let status = Command::new(&self.qemu_img_path)
.args([
"convert",
"-O",
@ -1307,7 +1336,7 @@ impl VolumeManager {
.materialize_image_cache(org_id, project_id, image_id)
.await?;
let requested_size = gib_to_bytes(size_gib);
let image_info = inspect_qemu_image(&image_path).await?;
let image_info = inspect_qemu_image(&self.qemu_img_path, &image_path).await?;
if requested_size < image_info.virtual_size {
return Err(Status::failed_precondition(format!(
"requested volume {} GiB is smaller than image virtual size {} bytes",
@ -1413,6 +1442,7 @@ impl VolumeManager {
&raw_image_path,
Path::new(&volume.path),
requested_size,
&self.qemu_img_path,
)
.await?;
return Ok(CoronaFsProvisionOutcome {
@ -1431,7 +1461,7 @@ impl VolumeManager {
export_uri = %export.uri,
"Populating CoronaFS-backed VM volume over NBD from image cache"
);
let status = Command::new("qemu-img")
let status = Command::new(&self.qemu_img_path)
.args([
"convert",
"-t",
@ -1481,7 +1511,7 @@ impl VolumeManager {
.await
.map_err(|e| Status::internal(format!("failed to create volume dir: {e}")))?;
}
let status = Command::new("qemu-img")
let status = Command::new(&self.qemu_img_path)
.args([
"create",
"-f",
@ -1508,7 +1538,7 @@ impl VolumeManager {
format: VolumeFormat,
size_gib: u64,
) -> Result<(), Status> {
let status = Command::new("qemu-img")
let status = Command::new(&self.qemu_img_path)
.args([
"resize",
"-f",
@ -1542,8 +1572,8 @@ impl VolumeManager {
}
}
async fn inspect_qemu_image(path: &Path) -> Result<QemuImageInfo, Status> {
let output = Command::new("qemu-img")
async fn inspect_qemu_image(qemu_img_path: &Path, path: &Path) -> Result<QemuImageInfo, Status> {
let output = Command::new(qemu_img_path)
.args(["info", "--output", "json", path.to_string_lossy().as_ref()])
.output()
.await
@ -1585,6 +1615,7 @@ async fn clone_local_raw_into_coronafs_volume(
source: &Path,
destination: &Path,
requested_size: u64,
_qemu_img_path: &Path,
) -> Result<(), Status> {
let temp_path = destination.with_extension("clone.tmp");
if let Some(parent) = temp_path.parent() {
@ -1595,33 +1626,13 @@ async fn clone_local_raw_into_coronafs_volume(
if tokio::fs::try_exists(&temp_path).await.unwrap_or(false) {
let _ = tokio::fs::remove_file(&temp_path).await;
}
let copy_output = Command::new("cp")
.args([
"--reflink=auto",
"--sparse=always",
source.to_string_lossy().as_ref(),
temp_path.to_string_lossy().as_ref(),
])
.output()
.await
.map_err(|e| Status::internal(format!("failed to spawn raw clone copy: {e}")))?;
if !copy_output.status.success() {
let stderr = String::from_utf8_lossy(&copy_output.stderr)
.trim()
.to_string();
return Err(Status::internal(format!(
"failed to clone raw image {} into {} with status {}{}",
tokio::fs::copy(source, &temp_path).await.map_err(|e| {
Status::internal(format!(
"failed to clone raw image {} into {}: {e}",
source.display(),
temp_path.display(),
copy_output.status,
if stderr.is_empty() {
String::new()
} else {
format!(": {stderr}")
}
)));
}
temp_path.display()
))
})?;
let file = tokio::fs::OpenOptions::new()
.write(true)
@ -1656,11 +1667,12 @@ async fn clone_local_raw_into_coronafs_volume(
}
async fn sync_local_coronafs_volume_to_export(
qemu_img_path: &Path,
local_volume: &CoronaFsVolumeResponse,
export_uri: &str,
) -> Result<(), Status> {
let local_format = local_volume.format.unwrap_or(CoronaFsVolumeFormat::Raw);
let status = Command::new("qemu-img")
let status = Command::new(qemu_img_path)
.args([
"convert",
"-t",
@ -1773,17 +1785,121 @@ fn gib_to_bytes(size_gib: u64) -> u64 {
size_gib.saturating_mul(1024 * 1024 * 1024)
}
fn coronafs_node_local_attach_enabled() -> bool {
coronafs_node_local_attach_enabled_from_values(
std::env::var("PLASMAVMC_CORONAFS_NODE_LOCAL_ATTACH")
.ok()
.as_deref(),
std::env::var("PLASMAVMC_CORONAFS_ENABLE_EXPERIMENTAL_NODE_LOCAL_ATTACH")
.ok()
.as_deref(),
)
fn validate_uuid(value: &str, field_name: &str) -> Result<(), Status> {
Uuid::parse_str(value)
.map(|_| ())
.map_err(|_| Status::invalid_argument(format!("{field_name} must be a UUID")))
}
fn validate_ceph_identifier(field_name: &str, value: &str) -> Result<(), Status> {
let trimmed = value.trim();
if trimmed.is_empty() {
return Err(Status::invalid_argument(format!(
"{field_name} is required"
)));
}
if trimmed.len() > CEPH_IDENTIFIER_MAX_LEN {
return Err(Status::invalid_argument(format!(
"{field_name} exceeds {CEPH_IDENTIFIER_MAX_LEN} characters"
)));
}
let mut chars = trimmed.chars();
let Some(first) = chars.next() else {
return Err(Status::invalid_argument(format!(
"{field_name} is required"
)));
};
if !first.is_ascii_alphanumeric() {
return Err(Status::invalid_argument(format!(
"{field_name} must start with an ASCII alphanumeric character"
)));
}
if chars.any(|ch| !(ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-'))) {
return Err(Status::invalid_argument(format!(
"{field_name} contains unsupported characters"
)));
}
Ok(())
}
fn validate_absolute_path(path: &str, field_name: &str) -> Result<(), Status> {
let candidate = Path::new(path);
if !candidate.is_absolute() {
return Err(Status::failed_precondition(format!(
"{field_name} must be an absolute path"
)));
}
Ok(())
}
fn validate_nbd_uri(uri: &str, field_name: &str) -> Result<(), Status> {
let parsed = reqwest::Url::parse(uri)
.map_err(|e| Status::failed_precondition(format!("{field_name} is invalid: {e}")))?;
if parsed.scheme() != "nbd" {
return Err(Status::failed_precondition(format!(
"{field_name} must use nbd://"
)));
}
if !parsed.username().is_empty() || parsed.password().is_some() {
return Err(Status::failed_precondition(format!(
"{field_name} must not contain embedded credentials"
)));
}
if parsed.host_str().is_none() || parsed.port().is_none() {
return Err(Status::failed_precondition(format!(
"{field_name} must include host and port"
)));
}
Ok(())
}
fn validate_coronafs_export(export: &CoronaFsExport) -> Result<(), Status> {
validate_nbd_uri(&export.uri, "coronafs export uri")
}
fn validate_coronafs_volume_response(volume: &CoronaFsVolumeResponse) -> Result<(), Status> {
validate_absolute_path(&volume.path, "coronafs volume path")?;
if let Some(export) = &volume.export {
validate_coronafs_export(export)?;
}
if let Some(source) = &volume.materialized_from {
if source.starts_with("nbd://") {
validate_nbd_uri(source, "coronafs materialized_from")?;
}
}
Ok(())
}
fn resolve_binary_path(
configured_path: Option<&Path>,
binary_name: &str,
) -> Result<PathBuf, Box<dyn std::error::Error>> {
let candidate = match configured_path {
Some(path) => path.to_path_buf(),
None => std::env::var_os("PATH")
.into_iter()
.flat_map(|paths| std::env::split_paths(&paths).collect::<Vec<_>>())
.map(|entry| entry.join(binary_name))
.find(|candidate| candidate.exists())
.ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("failed to locate {binary_name} in PATH"),
)
})?,
};
let metadata = std::fs::metadata(&candidate)?;
if !metadata.is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("{} is not a regular file", candidate.display()),
)
.into());
}
Ok(candidate)
}
#[cfg(test)]
fn coronafs_node_local_attach_enabled_from_values(
stable_value: Option<&str>,
legacy_value: Option<&str>,
@ -1794,6 +1910,23 @@ fn coronafs_node_local_attach_enabled_from_values(
.unwrap_or(false)
}
fn ceph_cluster_from_config(config: &CephConfig) -> Option<CephClusterConfig> {
if config.monitors.is_empty() {
None
} else {
Some(CephClusterConfig {
cluster_id: config.cluster_id.clone(),
monitors: config.monitors.clone(),
user: config.user.clone(),
secret: config
.secret
.clone()
.or_else(|| std::env::var("PLASMAVMC_CEPH_SECRET").ok()),
})
}
}
#[cfg(test)]
fn parse_truthy(value: &str) -> bool {
matches!(
value.trim().to_ascii_lowercase().as_str(),
@ -1801,16 +1934,21 @@ fn parse_truthy(value: &str) -> bool {
)
}
fn resolve_coronafs_clients() -> (Option<CoronaFsClient>, Option<CoronaFsClient>) {
fn resolve_coronafs_clients_with_config(
config: &CoronaFsConfig,
) -> (Option<CoronaFsClient>, Option<CoronaFsClient>) {
let (controller_endpoint, node_endpoint) = resolve_coronafs_endpoints(
std::env::var("PLASMAVMC_CORONAFS_CONTROLLER_ENDPOINT")
.ok()
config
.controller_endpoint
.clone()
.and_then(normalize_coronafs_endpoint),
std::env::var("PLASMAVMC_CORONAFS_NODE_ENDPOINT")
.ok()
config
.node_endpoint
.clone()
.and_then(normalize_coronafs_endpoint),
std::env::var("PLASMAVMC_CORONAFS_ENDPOINT")
.ok()
config
.endpoint
.clone()
.and_then(normalize_coronafs_endpoint),
);
(
@ -1911,14 +2049,19 @@ impl CoronaFsClient {
backing_file: None,
backing_format: None,
};
self.try_request("create CoronaFS volume", |endpoint, http| {
http.put(format!("{endpoint}/v1/volumes/{volume_id}"))
.json(&request)
})
.await?
.json::<CoronaFsVolumeResponse>()
.await
.map_err(|e| Status::internal(format!("failed to decode CoronaFS create response: {e}")))
let volume = self
.try_request("create CoronaFS volume", |endpoint, http| {
http.put(format!("{endpoint}/v1/volumes/{volume_id}"))
.json(&request)
})
.await?
.json::<CoronaFsVolumeResponse>()
.await
.map_err(|e| {
Status::internal(format!("failed to decode CoronaFS create response: {e}"))
})?;
validate_coronafs_volume_response(&volume)?;
Ok(volume)
}
async fn create_image_backed(
@ -1934,28 +2077,36 @@ impl CoronaFsClient {
backing_file: Some(backing_file.to_string_lossy().into_owned()),
backing_format: Some(backing_format),
};
self.try_request("create CoronaFS image-backed volume", |endpoint, http| {
http.put(format!("{endpoint}/v1/volumes/{volume_id}"))
.json(&request)
})
.await?
.json::<CoronaFsVolumeResponse>()
.await
.map_err(|e| {
Status::internal(format!(
"failed to decode CoronaFS image-backed create response: {e}"
))
})
let volume = self
.try_request("create CoronaFS image-backed volume", |endpoint, http| {
http.put(format!("{endpoint}/v1/volumes/{volume_id}"))
.json(&request)
})
.await?
.json::<CoronaFsVolumeResponse>()
.await
.map_err(|e| {
Status::internal(format!(
"failed to decode CoronaFS image-backed create response: {e}"
))
})?;
validate_coronafs_volume_response(&volume)?;
Ok(volume)
}
async fn get_volume(&self, volume_id: &str) -> Result<CoronaFsVolumeResponse, Status> {
self.try_request("inspect CoronaFS volume", |endpoint, http| {
http.get(format!("{endpoint}/v1/volumes/{volume_id}"))
})
.await?
.json::<CoronaFsVolumeResponse>()
.await
.map_err(|e| Status::internal(format!("failed to decode CoronaFS inspect response: {e}")))
let volume = self
.try_request("inspect CoronaFS volume", |endpoint, http| {
http.get(format!("{endpoint}/v1/volumes/{volume_id}"))
})
.await?
.json::<CoronaFsVolumeResponse>()
.await
.map_err(|e| {
Status::internal(format!("failed to decode CoronaFS inspect response: {e}"))
})?;
validate_coronafs_volume_response(&volume)?;
Ok(volume)
}
async fn get_volume_optional(
@ -2001,9 +2152,11 @@ impl CoronaFsClient {
.map_err(|e| {
Status::internal(format!("failed to decode CoronaFS export response: {e}"))
})?;
response.export.ok_or_else(|| {
let export = response.export.ok_or_else(|| {
Status::internal("CoronaFS export response did not include an export URI")
})
})?;
validate_coronafs_export(&export)?;
Ok(export)
}
async fn materialize_from_export(
@ -2014,24 +2167,28 @@ impl CoronaFsClient {
format: Option<CoronaFsVolumeFormat>,
lazy: bool,
) -> Result<CoronaFsVolumeResponse, Status> {
validate_nbd_uri(source_uri, "coronafs materialize source_uri")?;
let request = CoronaFsMaterializeRequest {
source_uri: source_uri.to_string(),
size_bytes,
format,
lazy,
};
self.try_request("materialize CoronaFS volume", |endpoint, http| {
http.post(format!("{endpoint}/v1/volumes/{volume_id}/materialize"))
.json(&request)
})
.await?
.json::<CoronaFsVolumeResponse>()
.await
.map_err(|e| {
Status::internal(format!(
"failed to decode CoronaFS materialize response: {e}"
))
})
let volume = self
.try_request("materialize CoronaFS volume", |endpoint, http| {
http.post(format!("{endpoint}/v1/volumes/{volume_id}/materialize"))
.json(&request)
})
.await?
.json::<CoronaFsVolumeResponse>()
.await
.map_err(|e| {
Status::internal(format!(
"failed to decode CoronaFS materialize response: {e}"
))
})?;
validate_coronafs_volume_response(&volume)?;
Ok(volume)
}
async fn resize_volume(&self, volume_id: &str, size_bytes: u64) -> Result<(), Status> {
@ -2210,9 +2367,14 @@ mod tests {
.unwrap();
tokio::fs::write(&source, b"raw-seed").await.unwrap();
clone_local_raw_into_coronafs_volume(&source, &destination, 4096)
.await
.unwrap();
clone_local_raw_into_coronafs_volume(
&source,
&destination,
4096,
Path::new("/usr/bin/qemu-img"),
)
.await
.unwrap();
let cloned = tokio::fs::read(&destination).await.unwrap();
assert_eq!(&cloned[..8], b"raw-seed");
@ -2398,4 +2560,27 @@ mod tests {
.remove(CORONAFS_IMAGE_SEED_PENDING_METADATA_KEY);
assert!(!volume_has_pending_coronafs_image_seed(&volume));
}
#[test]
fn ceph_identifier_validation_rejects_option_injection_characters() {
assert!(validate_ceph_identifier("pool", "pool-alpha").is_ok());
assert!(validate_ceph_identifier("pool", "pool,inject").is_err());
assert!(validate_ceph_identifier("image", "image:inject").is_err());
}
#[test]
fn coronafs_export_validation_requires_nbd_uri() {
let valid = CoronaFsExport {
uri: "nbd://10.0.0.1:11000".to_string(),
port: 11000,
pid: None,
};
let invalid = CoronaFsExport {
uri: "http://10.0.0.1:11000".to_string(),
..valid.clone()
};
assert!(validate_coronafs_export(&valid).is_ok());
assert!(validate_coronafs_export(&invalid).is_err());
}
}

7
plasmavmc/crates/plasmavmc-server/src/watcher.rs
View file

@ -64,13 +64,8 @@ pub struct WatcherConfig {
impl Default for WatcherConfig {
fn default() -> Self {
let poll_interval_ms = std::env::var("PLASMAVMC_STATE_WATCHER_POLL_INTERVAL_MS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(1000)
.max(100);
Self {
poll_interval: Duration::from_millis(poll_interval_ms),
poll_interval: Duration::from_millis(1000),
buffer_size: 256,
}
}

13
plasmavmc/crates/plasmavmc-types/src/config.rs
View file

@ -4,8 +4,7 @@ use serde::{Deserialize, Serialize};
use std::path::PathBuf;
/// FireCracker backend configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Default)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct FireCrackerConfig {
/// Path to the Firecracker binary
pub firecracker_path: Option<PathBuf>,
@ -27,11 +26,11 @@ pub struct FireCrackerConfig {
pub use_jailer: Option<bool>,
}
/// KVM backend configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Default)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct KvmConfig {
// Add KVM specific configuration fields here if any
/// Path to the QEMU binary
pub qemu_path: Option<PathBuf>,
/// Runtime directory used for QMP sockets and console logs
pub runtime_dir: Option<PathBuf>,
}

48
plasmavmc/proto/plasmavmc.proto
View file

@ -44,6 +44,10 @@ service VmService {
service ImageService {
rpc CreateImage(CreateImageRequest) returns (Image);
rpc BeginImageUpload(BeginImageUploadRequest) returns (BeginImageUploadResponse);
rpc UploadImagePart(UploadImagePartRequest) returns (UploadImagePartResponse);
rpc CompleteImageUpload(CompleteImageUploadRequest) returns (Image);
rpc AbortImageUpload(AbortImageUploadRequest) returns (Empty);
rpc GetImage(GetImageRequest) returns (Image);
rpc ListImages(ListImagesRequest) returns (ListImagesResponse);
rpc UpdateImage(UpdateImageRequest) returns (Image);
@ -475,6 +479,50 @@ message CreateImageRequest {
string source_url = 11;
}
message BeginImageUploadRequest {
string name = 1;
string org_id = 2;
Visibility visibility = 3;
ImageFormat format = 4;
OsType os_type = 5;
string os_version = 6;
Architecture architecture = 7;
uint32 min_disk_gib = 8;
uint32 min_memory_mib = 9;
map<string, string> metadata = 10;
}
message BeginImageUploadResponse {
Image image = 1;
string upload_session_id = 2;
uint32 minimum_part_size_bytes = 3;
}
message UploadImagePartRequest {
string org_id = 1;
string image_id = 2;
string upload_session_id = 3;
uint32 part_number = 4;
bytes body = 5;
}
message UploadImagePartResponse {
uint32 part_number = 1;
uint64 committed_size_bytes = 2;
}
message CompleteImageUploadRequest {
string org_id = 1;
string image_id = 2;
string upload_session_id = 3;
}
message AbortImageUploadRequest {
string org_id = 1;
string image_id = 2;
string upload_session_id = 3;
}
message GetImageRequest {
string org_id = 1;
string image_id = 2;

39
prismnet/crates/prismnet-server/src/config.rs
View file

@ -20,6 +20,36 @@ pub struct TlsConfig {
pub require_client_cert: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum OvnMode {
Mock,
Real,
}
impl Default for OvnMode {
fn default() -> Self {
Self::Mock
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OvnConfig {
#[serde(default)]
pub mode: OvnMode,
#[serde(default = "default_ovn_nb_addr")]
pub nb_addr: String,
}
impl Default for OvnConfig {
fn default() -> Self {
Self {
mode: OvnMode::default(),
nb_addr: default_ovn_nb_addr(),
}
}
}
/// Metadata storage backend
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
@ -74,6 +104,10 @@ pub struct ServerConfig {
/// Authentication configuration
#[serde(default)]
pub auth: AuthConfig,
/// OVN integration settings
#[serde(default)]
pub ovn: OvnConfig,
}
/// Authentication configuration
@ -100,6 +134,10 @@ fn default_http_addr() -> SocketAddr {
"127.0.0.1:8087".parse().unwrap()
}
fn default_ovn_nb_addr() -> String {
"tcp:127.0.0.1:6641".to_string()
}
impl Default for ServerConfig {
fn default() -> Self {
Self {
@ -113,6 +151,7 @@ impl Default for ServerConfig {
log_level: "info".to_string(),
tls: None,
auth: AuthConfig::default(),
ovn: OvnConfig::default(),
}
}
}

115
prismnet/crates/prismnet-server/src/main.rs
View file

@ -11,9 +11,8 @@ use prismnet_api::{
subnet_service_server::SubnetServiceServer, vpc_service_server::VpcServiceServer,
};
use prismnet_server::{
config::MetadataBackend,
IpamServiceImpl, NetworkMetadataStore, OvnClient, PortServiceImpl, SecurityGroupServiceImpl,
ServerConfig, SubnetServiceImpl, VpcServiceImpl,
config::MetadataBackend, IpamServiceImpl, NetworkMetadataStore, OvnClient, PortServiceImpl,
SecurityGroupServiceImpl, ServerConfig, SubnetServiceImpl, VpcServiceImpl,
};
use std::net::SocketAddr;
use std::path::PathBuf;
@ -36,26 +35,6 @@ struct Args {
#[arg(long)]
grpc_addr: Option<String>,
/// ChainFire endpoint for cluster coordination (optional)
#[arg(long)]
chainfire_endpoint: Option<String>,
/// FlareDB endpoint for metadata and tenant data storage
#[arg(long)]
flaredb_endpoint: Option<String>,
/// Metadata backend (flaredb, postgres, sqlite)
#[arg(long)]
metadata_backend: Option<String>,
/// SQL database URL for metadata (required for postgres/sqlite backend)
#[arg(long)]
metadata_database_url: Option<String>,
/// Run in single-node mode (required when metadata backend is SQLite)
#[arg(long)]
single_node: bool,
/// Log level (overrides config)
#[arg(short, long)]
log_level: Option<String>,
@ -88,58 +67,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
if let Some(log_level) = args.log_level {
config.log_level = log_level;
}
if let Some(chainfire_endpoint) = args.chainfire_endpoint {
config.chainfire_endpoint = Some(chainfire_endpoint);
}
if let Some(flaredb_endpoint) = args.flaredb_endpoint {
config.flaredb_endpoint = Some(flaredb_endpoint);
}
if let Some(metadata_backend) = args.metadata_backend {
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
}
if let Some(metadata_database_url) = args.metadata_database_url {
config.metadata_database_url = Some(metadata_database_url);
}
if args.single_node {
config.single_node = true;
}
if config.chainfire_endpoint.is_none() {
if let Ok(chainfire_endpoint) = std::env::var("PRISMNET_CHAINFIRE_ENDPOINT") {
let trimmed = chainfire_endpoint.trim();
if !trimmed.is_empty() {
config.chainfire_endpoint = Some(trimmed.to_string());
}
}
}
if config.flaredb_endpoint.is_none() {
if let Ok(flaredb_endpoint) = std::env::var("PRISMNET_FLAREDB_ENDPOINT") {
let trimmed = flaredb_endpoint.trim();
if !trimmed.is_empty() {
config.flaredb_endpoint = Some(trimmed.to_string());
}
}
}
if let Ok(metadata_backend) = std::env::var("PRISMNET_METADATA_BACKEND") {
let trimmed = metadata_backend.trim();
if !trimmed.is_empty() {
config.metadata_backend = parse_metadata_backend(trimmed)?;
}
}
if config.metadata_database_url.is_none() {
if let Ok(metadata_database_url) = std::env::var("PRISMNET_METADATA_DATABASE_URL") {
let trimmed = metadata_database_url.trim();
if !trimmed.is_empty() {
config.metadata_database_url = Some(trimmed.to_string());
}
}
}
if !config.single_node {
if let Ok(single_node) = std::env::var("PRISMNET_SINGLE_NODE") {
let parsed = single_node.trim().to_ascii_lowercase();
config.single_node = matches!(parsed.as_str(), "1" | "true" | "yes" | "on");
}
}
// Initialize tracing
tracing_subscriber::fmt()
.with_env_filter(
@ -187,20 +114,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
config.flaredb_endpoint.clone(),
config.chainfire_endpoint.clone(),
)
.await
.map_err(|e| anyhow!("Failed to init FlareDB metadata store: {}", e))?,
.await
.map_err(|e| anyhow!("Failed to init FlareDB metadata store: {}", e))?,
)
}
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
let database_url = config
.metadata_database_url
.as_deref()
.ok_or_else(|| {
anyhow!(
"metadata_database_url is required when metadata_backend={} (env: PRISMNET_METADATA_DATABASE_URL)",
metadata_backend_name(config.metadata_backend)
)
})?;
let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
anyhow!(
"metadata_database_url is required when metadata_backend={}",
metadata_backend_name(config.metadata_backend)
)
})?;
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
tracing::info!(
" Metadata backend: {} @ {}",
@ -216,8 +140,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
};
// Initialize OVN client (default: mock)
let ovn =
Arc::new(OvnClient::from_env().map_err(|e| anyhow!("Failed to init OVN client: {}", e))?);
let ovn = Arc::new(
OvnClient::from_config(&config.ovn)
.map_err(|e| anyhow!("Failed to init OVN client: {}", e))?,
);
// Initialize IAM authentication service
tracing::info!(
@ -374,19 +300,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
match value.trim().to_ascii_lowercase().as_str() {
"flaredb" => Ok(MetadataBackend::FlareDb),
"postgres" => Ok(MetadataBackend::Postgres),
"sqlite" => Ok(MetadataBackend::Sqlite),
other => Err(format!(
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
other
)
.into()),
}
}
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
match backend {
MetadataBackend::FlareDb => "flaredb",

18
prismnet/crates/prismnet-server/src/metadata.rs
View file

@ -60,12 +60,8 @@ impl NetworkMetadataStore {
endpoint: Option<String>,
pd_endpoint: Option<String>,
) -> Result<Self> {
let endpoint = endpoint.unwrap_or_else(|| {
std::env::var("PRISMNET_FLAREDB_ENDPOINT")
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
});
let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
let pd_endpoint = pd_endpoint
.or_else(|| std::env::var("PRISMNET_CHAINFIRE_ENDPOINT").ok())
.map(|value| normalize_transport_addr(&value))
.unwrap_or_else(|| endpoint.clone());
@ -168,7 +164,9 @@ impl NetworkMetadataStore {
)
.execute(pool)
.await
.map_err(|e| MetadataError::Storage(format!("Failed to initialize Postgres schema: {}", e)))?;
.map_err(|e| {
MetadataError::Storage(format!("Failed to initialize Postgres schema: {}", e))
})?;
Ok(())
}
@ -181,7 +179,9 @@ impl NetworkMetadataStore {
)
.execute(pool)
.await
.map_err(|e| MetadataError::Storage(format!("Failed to initialize SQLite schema: {}", e)))?;
.map_err(|e| {
MetadataError::Storage(format!("Failed to initialize SQLite schema: {}", e))
})?;
Ok(())
}
@ -208,9 +208,7 @@ impl NetworkMetadataStore {
.bind(value)
.execute(pool.as_ref())
.await
.map_err(|e| {
MetadataError::Storage(format!("Postgres put failed: {}", e))
})?;
.map_err(|e| MetadataError::Storage(format!("Postgres put failed: {}", e)))?;
}
SqlStorageBackend::Sqlite(pool) => {
sqlx::query(

85
prismnet/crates/prismnet-server/src/ovn/client.rs
View file

@ -1,9 +1,12 @@
use std::sync::Arc;
use prismnet_types::{DhcpOptions, Port, PortId, SecurityGroupId, SecurityGroupRule, SecurityGroupRuleId, VpcId};
use prismnet_types::{
DhcpOptions, Port, PortId, SecurityGroupId, SecurityGroupRule, SecurityGroupRuleId, VpcId,
};
use tokio::process::Command;
use tokio::sync::Mutex;
use crate::config::{OvnConfig, OvnMode as ConfigOvnMode};
use crate::ovn::mock::MockOvnState;
/// OVN client mode
@ -31,22 +34,19 @@ pub struct OvnClient {
}
impl OvnClient {
/// Build an OVN client from environment variables (default: mock)
/// - PRISMNET_OVN_MODE: "mock" (default) or "real"
/// - PRISMNET_OVN_NB_ADDR: ovsdb northbound address (real mode only)
pub fn from_env() -> OvnResult<Self> {
let mode = std::env::var("PRISMNET_OVN_MODE").unwrap_or_else(|_| "mock".to_string());
match mode.to_lowercase().as_str() {
"mock" => Ok(Self::new_mock()),
"real" => {
let nb_addr = std::env::var("PRISMNET_OVN_NB_ADDR")
.unwrap_or_else(|_| "tcp:127.0.0.1:6641".to_string());
Ok(Self::new_real(nb_addr))
/// Build an OVN client from configuration (default: mock)
pub fn from_config(config: &OvnConfig) -> OvnResult<Self> {
match config.mode {
ConfigOvnMode::Mock => Ok(Self::new_mock()),
ConfigOvnMode::Real => {
if config.nb_addr.trim().is_empty() {
Err(OvnError::InvalidArgument(
"OVN nb_addr must not be empty in real mode".to_string(),
))
} else {
Ok(Self::new_real(config.nb_addr.clone()))
}
}
other => Err(OvnError::InvalidArgument(format!(
"Unknown OVN mode: {}",
other
))),
}
}
@ -305,10 +305,7 @@ impl OvnClient {
}
if !options.dns_servers.is_empty() {
opts.push(format!(
"dns_server={{{}}}",
options.dns_servers.join(",")
));
opts.push(format!("dns_server={{{}}}", options.dns_servers.join(",")));
}
opts.push(format!("lease_time={}", options.lease_time));
@ -389,7 +386,10 @@ impl OvnClient {
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
return Err(OvnError::Command(format!("lr-add query failed: {}", stderr)));
return Err(OvnError::Command(format!(
"lr-add query failed: {}",
stderr
)));
}
let uuid = String::from_utf8_lossy(&output.stdout).trim().to_string();
@ -456,12 +456,8 @@ impl OvnClient {
let ls_name = Self::logical_switch_name(switch_id);
// ovn-nbctl lsp-add <switch> <port-name>
self.run_nbctl(vec![
"lsp-add".into(),
ls_name,
switch_port_name.clone(),
])
.await?;
self.run_nbctl(vec!["lsp-add".into(), ls_name, switch_port_name.clone()])
.await?;
// ovn-nbctl lsp-set-type <port> router
self.run_nbctl(vec![
@ -698,10 +694,7 @@ mod tests {
let client = OvnClient::new_mock();
// Create a router
let router_id = client
.create_logical_router("test-router")
.await
.unwrap();
let router_id = client.create_logical_router("test-router").await.unwrap();
assert!(!router_id.is_empty());
assert!(router_id.starts_with("router-"));
@ -732,10 +725,7 @@ mod tests {
.unwrap();
// Create router
let router_id = client
.create_logical_router("test-router")
.await
.unwrap();
let router_id = client.create_logical_router("test-router").await.unwrap();
// Add router port
let mac = "02:00:00:00:00:01";
@ -760,10 +750,7 @@ mod tests {
let client = OvnClient::new_mock();
// Create router
let router_id = client
.create_logical_router("test-router")
.await
.unwrap();
let router_id = client.create_logical_router("test-router").await.unwrap();
// Configure SNAT
let external_ip = "203.0.113.10";
@ -791,10 +778,7 @@ mod tests {
.unwrap();
// Create router
let router_id = client
.create_logical_router("test-router")
.await
.unwrap();
let router_id = client.create_logical_router("test-router").await.unwrap();
// Add router port
let mac = "02:00:00:00:00:01";
@ -840,10 +824,7 @@ mod tests {
.unwrap();
// Create router
let router_id = client
.create_logical_router("test-router")
.await
.unwrap();
let router_id = client.create_logical_router("test-router").await.unwrap();
// Add router ports to both switches
let port1_id = client
@ -876,10 +857,7 @@ mod tests {
.unwrap();
// Step 2: Create router
let router_id = client
.create_logical_router("vpc-router")
.await
.unwrap();
let router_id = client.create_logical_router("vpc-router").await.unwrap();
// Step 3: Attach router to switch
let mac = "02:00:00:00:00:01";
@ -920,10 +898,7 @@ mod tests {
let client = OvnClient::new_mock();
// Create router
let router_id = client
.create_logical_router("test-router")
.await
.unwrap();
let router_id = client.create_logical_router("test-router").await.unwrap();
// Configure multiple SNAT rules
client