harden plasmavmc image ingestion and internal execution paths
This commit is contained in:
parent
260fb4c576
commit
0745216107
64 changed files with 3531 additions and 1434 deletions
|
|
@ -161,7 +161,9 @@ impl RedfishTarget {
|
||||||
|
|
||||||
match action {
|
match action {
|
||||||
PowerAction::Cycle => Ok(PowerState::Cycling),
|
PowerAction::Cycle => Ok(PowerState::Cycling),
|
||||||
PowerAction::On | PowerAction::Off | PowerAction::Refresh => self.refresh(&client).await,
|
PowerAction::On | PowerAction::Off | PowerAction::Refresh => {
|
||||||
|
self.refresh(&client).await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -295,7 +297,12 @@ pub async fn request_reinstall(
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use axum::{extract::State, http::StatusCode, routing::{get, post}, Json, Router};
|
use axum::{
|
||||||
|
extract::State,
|
||||||
|
http::StatusCode,
|
||||||
|
routing::{get, post},
|
||||||
|
Json, Router,
|
||||||
|
};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
|
|
@ -303,7 +310,10 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn parse_redfish_short_reference_defaults_to_https() {
|
fn parse_redfish_short_reference_defaults_to_https() {
|
||||||
let parsed = RedfishTarget::parse("redfish://lab-bmc/node01").unwrap();
|
let parsed = RedfishTarget::parse("redfish://lab-bmc/node01").unwrap();
|
||||||
assert_eq!(parsed.resource_url.as_str(), "https://lab-bmc/redfish/v1/Systems/node01");
|
assert_eq!(
|
||||||
|
parsed.resource_url.as_str(),
|
||||||
|
"https://lab-bmc/redfish/v1/Systems/node01"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -361,8 +371,14 @@ mod tests {
|
||||||
addr
|
addr
|
||||||
))
|
))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(target.perform(PowerAction::Refresh).await.unwrap(), PowerState::On);
|
assert_eq!(
|
||||||
assert_eq!(target.perform(PowerAction::Off).await.unwrap(), PowerState::On);
|
target.perform(PowerAction::Refresh).await.unwrap(),
|
||||||
|
PowerState::On
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
target.perform(PowerAction::Off).await.unwrap(),
|
||||||
|
PowerState::On
|
||||||
|
);
|
||||||
|
|
||||||
let payloads = state.seen_payloads.lock().unwrap().clone();
|
let payloads = state.seen_payloads.lock().unwrap().clone();
|
||||||
assert_eq!(payloads, vec![r#"{"ResetType":"ForceOff"}"#.to_string()]);
|
assert_eq!(payloads, vec![r#"{"ResetType":"ForceOff"}"#.to_string()]);
|
||||||
|
|
|
||||||
|
|
@ -31,5 +31,3 @@ pub async fn run_deployer_command(endpoint: &str, action: &str) -> Result<()> {
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -177,6 +177,10 @@ pub struct VipOwnershipConfig {
|
||||||
/// Interface used for local VIP ownership.
|
/// Interface used for local VIP ownership.
|
||||||
#[serde(default = "default_vip_ownership_interface")]
|
#[serde(default = "default_vip_ownership_interface")]
|
||||||
pub interface: String,
|
pub interface: String,
|
||||||
|
|
||||||
|
/// Optional explicit `ip` command path used for local VIP ownership.
|
||||||
|
#[serde(default)]
|
||||||
|
pub ip_command: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_vip_ownership_interface() -> String {
|
fn default_vip_ownership_interface() -> String {
|
||||||
|
|
@ -188,6 +192,7 @@ impl Default for VipOwnershipConfig {
|
||||||
Self {
|
Self {
|
||||||
enabled: false,
|
enabled: false,
|
||||||
interface: default_vip_ownership_interface(),
|
interface: default_vip_ownership_interface(),
|
||||||
|
ip_command: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -41,26 +41,6 @@ struct Args {
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
grpc_addr: Option<String>,
|
grpc_addr: Option<String>,
|
||||||
|
|
||||||
/// ChainFire endpoint for cluster coordination
|
|
||||||
#[arg(long, env = "FIBERLB_CHAINFIRE_ENDPOINT")]
|
|
||||||
chainfire_endpoint: Option<String>,
|
|
||||||
|
|
||||||
/// FlareDB endpoint for metadata and tenant data storage
|
|
||||||
#[arg(long, env = "FIBERLB_FLAREDB_ENDPOINT")]
|
|
||||||
flaredb_endpoint: Option<String>,
|
|
||||||
|
|
||||||
/// Metadata backend (flaredb, postgres, sqlite)
|
|
||||||
#[arg(long, env = "FIBERLB_METADATA_BACKEND")]
|
|
||||||
metadata_backend: Option<String>,
|
|
||||||
|
|
||||||
/// SQL database URL for metadata (required for postgres/sqlite backend)
|
|
||||||
#[arg(long, env = "FIBERLB_METADATA_DATABASE_URL")]
|
|
||||||
metadata_database_url: Option<String>,
|
|
||||||
|
|
||||||
/// Run in single-node mode (required when metadata backend is SQLite)
|
|
||||||
#[arg(long, env = "FIBERLB_SINGLE_NODE")]
|
|
||||||
single_node: bool,
|
|
||||||
|
|
||||||
/// Log level (overrides config)
|
/// Log level (overrides config)
|
||||||
#[arg(short, long)]
|
#[arg(short, long)]
|
||||||
log_level: Option<String>,
|
log_level: Option<String>,
|
||||||
|
|
@ -93,21 +73,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
if let Some(log_level) = args.log_level {
|
if let Some(log_level) = args.log_level {
|
||||||
config.log_level = log_level;
|
config.log_level = log_level;
|
||||||
}
|
}
|
||||||
if let Some(chainfire_endpoint) = args.chainfire_endpoint {
|
|
||||||
config.chainfire_endpoint = Some(chainfire_endpoint);
|
|
||||||
}
|
|
||||||
if let Some(flaredb_endpoint) = args.flaredb_endpoint {
|
|
||||||
config.flaredb_endpoint = Some(flaredb_endpoint);
|
|
||||||
}
|
|
||||||
if let Some(metadata_backend) = args.metadata_backend {
|
|
||||||
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
|
|
||||||
}
|
|
||||||
if let Some(metadata_database_url) = args.metadata_database_url {
|
|
||||||
config.metadata_database_url = Some(metadata_database_url);
|
|
||||||
}
|
|
||||||
if args.single_node {
|
|
||||||
config.single_node = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize tracing
|
// Initialize tracing
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
|
|
@ -194,15 +159,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
|
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
|
||||||
let database_url = config
|
let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
|
||||||
.metadata_database_url
|
format!(
|
||||||
.as_deref()
|
"metadata_database_url is required when metadata_backend={}",
|
||||||
.ok_or_else(|| {
|
metadata_backend_name(config.metadata_backend)
|
||||||
format!(
|
)
|
||||||
"metadata_database_url is required when metadata_backend={} (env: FIBERLB_METADATA_DATABASE_URL)",
|
})?;
|
||||||
metadata_backend_name(config.metadata_backend)
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
|
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
" Metadata backend: {} @ {}",
|
" Metadata backend: {} @ {}",
|
||||||
|
|
@ -282,8 +244,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
})?;
|
})?;
|
||||||
let bgp = create_bgp_client(config.bgp.clone()).await?;
|
let bgp = create_bgp_client(config.bgp.clone()).await?;
|
||||||
let vip_owner: Option<Arc<dyn VipAddressOwner>> = if config.vip_ownership.enabled {
|
let vip_owner: Option<Arc<dyn VipAddressOwner>> = if config.vip_ownership.enabled {
|
||||||
Some(Arc::new(KernelVipAddressOwner::new(
|
Some(Arc::new(KernelVipAddressOwner::with_ip_command(
|
||||||
config.vip_ownership.interface.clone(),
|
config.vip_ownership.interface.clone(),
|
||||||
|
config.vip_ownership.ip_command.clone(),
|
||||||
)))
|
)))
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
|
|
@ -439,19 +402,6 @@ async fn wait_for_shutdown_signal() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
|
|
||||||
match value.trim().to_ascii_lowercase().as_str() {
|
|
||||||
"flaredb" => Ok(MetadataBackend::FlareDb),
|
|
||||||
"postgres" => Ok(MetadataBackend::Postgres),
|
|
||||||
"sqlite" => Ok(MetadataBackend::Sqlite),
|
|
||||||
other => Err(format!(
|
|
||||||
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
|
|
||||||
other
|
|
||||||
)
|
|
||||||
.into()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
|
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
|
||||||
match backend {
|
match backend {
|
||||||
MetadataBackend::FlareDb => "flaredb",
|
MetadataBackend::FlareDb => "flaredb",
|
||||||
|
|
|
||||||
|
|
@ -61,12 +61,12 @@ impl LbMetadataStore {
|
||||||
endpoint: Option<String>,
|
endpoint: Option<String>,
|
||||||
pd_endpoint: Option<String>,
|
pd_endpoint: Option<String>,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let endpoint = endpoint.unwrap_or_else(|| {
|
let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
|
||||||
std::env::var("FIBERLB_FLAREDB_ENDPOINT")
|
Self::connect_flaredb(endpoint, pd_endpoint).await
|
||||||
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
|
}
|
||||||
});
|
|
||||||
|
async fn connect_flaredb(endpoint: String, pd_endpoint: Option<String>) -> Result<Self> {
|
||||||
let pd_endpoint = pd_endpoint
|
let pd_endpoint = pd_endpoint
|
||||||
.or_else(|| std::env::var("FIBERLB_CHAINFIRE_ENDPOINT").ok())
|
|
||||||
.map(|value| normalize_transport_addr(&value))
|
.map(|value| normalize_transport_addr(&value))
|
||||||
.unwrap_or_else(|| endpoint.clone());
|
.unwrap_or_else(|| endpoint.clone());
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -41,9 +41,17 @@ pub struct KernelVipAddressOwner {
|
||||||
impl KernelVipAddressOwner {
|
impl KernelVipAddressOwner {
|
||||||
/// Create a kernel-backed VIP owner for the given interface.
|
/// Create a kernel-backed VIP owner for the given interface.
|
||||||
pub fn new(interface: impl Into<String>) -> Self {
|
pub fn new(interface: impl Into<String>) -> Self {
|
||||||
|
Self::with_ip_command(interface, None::<String>)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a kernel-backed VIP owner with an optional explicit `ip` command path.
|
||||||
|
pub fn with_ip_command(
|
||||||
|
interface: impl Into<String>,
|
||||||
|
ip_command: Option<impl Into<String>>,
|
||||||
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
interface: interface.into(),
|
interface: interface.into(),
|
||||||
ip_command: resolve_ip_command(),
|
ip_command: resolve_ip_command(ip_command.map(Into::into)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -123,7 +131,14 @@ impl VipAddressOwner for KernelVipAddressOwner {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn resolve_ip_command() -> String {
|
fn resolve_ip_command(configured: Option<String>) -> String {
|
||||||
|
if let Some(path) = configured {
|
||||||
|
let trimmed = path.trim();
|
||||||
|
if !trimmed.is_empty() {
|
||||||
|
return trimmed.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if let Ok(path) = std::env::var("FIBERLB_IP_COMMAND") {
|
if let Ok(path) = std::env::var("FIBERLB_IP_COMMAND") {
|
||||||
let trimmed = path.trim();
|
let trimmed = path.trim();
|
||||||
if !trimmed.is_empty() {
|
if !trimmed.is_empty() {
|
||||||
|
|
@ -159,3 +174,37 @@ fn render_command_output(output: &std::process::Output) -> String {
|
||||||
|
|
||||||
format!("exit status {}", output.status)
|
format!("exit status {}", output.status)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::sync::{Mutex, OnceLock};
|
||||||
|
|
||||||
|
fn env_lock() -> &'static Mutex<()> {
|
||||||
|
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||||
|
LOCK.get_or_init(|| Mutex::new(()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn configured_ip_command_wins_over_env() {
|
||||||
|
let _guard = env_lock().lock().unwrap();
|
||||||
|
std::env::set_var("FIBERLB_IP_COMMAND", "/env/ip");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
resolve_ip_command(Some(" /configured/ip ".to_string())),
|
||||||
|
"/configured/ip"
|
||||||
|
);
|
||||||
|
|
||||||
|
std::env::remove_var("FIBERLB_IP_COMMAND");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn env_ip_command_is_used_when_config_is_absent() {
|
||||||
|
let _guard = env_lock().lock().unwrap();
|
||||||
|
std::env::set_var("FIBERLB_IP_COMMAND", " /env/ip ");
|
||||||
|
|
||||||
|
assert_eq!(resolve_ip_command(None), "/env/ip");
|
||||||
|
|
||||||
|
std::env::remove_var("FIBERLB_IP_COMMAND");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,27 +1,24 @@
|
||||||
//! FlashDNS authoritative DNS server binary
|
//! FlashDNS authoritative DNS server binary
|
||||||
|
|
||||||
|
use anyhow::Result;
|
||||||
|
use chainfire_client::Client as ChainFireClient;
|
||||||
|
use clap::Parser;
|
||||||
use flashdns_api::{RecordServiceServer, ZoneServiceServer};
|
use flashdns_api::{RecordServiceServer, ZoneServiceServer};
|
||||||
use flashdns_server::{
|
use flashdns_server::{
|
||||||
config::{MetadataBackend, ServerConfig},
|
config::{MetadataBackend, ServerConfig},
|
||||||
dns::DnsHandler,
|
dns::DnsHandler,
|
||||||
metadata::DnsMetadataStore,
|
metadata::DnsMetadataStore,
|
||||||
RecordServiceImpl,
|
RecordServiceImpl, ZoneServiceImpl,
|
||||||
ZoneServiceImpl,
|
|
||||||
};
|
};
|
||||||
use chainfire_client::Client as ChainFireClient;
|
|
||||||
use iam_service_auth::AuthService;
|
use iam_service_auth::AuthService;
|
||||||
use metrics_exporter_prometheus::PrometheusBuilder;
|
use metrics_exporter_prometheus::PrometheusBuilder;
|
||||||
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
use tonic::transport::{Certificate, Identity, Server, ServerTlsConfig};
|
use tonic::transport::{Certificate, Identity, Server, ServerTlsConfig};
|
||||||
use tonic::{Request, Status};
|
use tonic::{Request, Status};
|
||||||
use tonic_health::server::health_reporter;
|
use tonic_health::server::health_reporter;
|
||||||
use tracing_subscriber::EnvFilter;
|
use tracing_subscriber::EnvFilter;
|
||||||
use anyhow::Result;
|
|
||||||
use clap::Parser;
|
|
||||||
use std::path::PathBuf;
|
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
|
||||||
|
|
||||||
use config::{Config as Cfg, Environment, File, FileFormat};
|
|
||||||
|
|
||||||
/// Command-line arguments for FlashDNS server.
|
/// Command-line arguments for FlashDNS server.
|
||||||
#[derive(Parser, Debug)]
|
#[derive(Parser, Debug)]
|
||||||
|
|
@ -39,26 +36,6 @@ struct CliArgs {
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
dns_addr: Option<String>,
|
dns_addr: Option<String>,
|
||||||
|
|
||||||
/// ChainFire endpoint for cluster coordination (overrides config)
|
|
||||||
#[arg(long, env = "FLASHDNS_CHAINFIRE_ENDPOINT")]
|
|
||||||
chainfire_endpoint: Option<String>,
|
|
||||||
|
|
||||||
/// FlareDB endpoint for metadata and tenant data storage (overrides config)
|
|
||||||
#[arg(long, env = "FLASHDNS_FLAREDB_ENDPOINT")]
|
|
||||||
flaredb_endpoint: Option<String>,
|
|
||||||
|
|
||||||
/// Metadata backend (flaredb, postgres, sqlite)
|
|
||||||
#[arg(long, env = "FLASHDNS_METADATA_BACKEND")]
|
|
||||||
metadata_backend: Option<String>,
|
|
||||||
|
|
||||||
/// SQL database URL for metadata (required for postgres/sqlite backend)
|
|
||||||
#[arg(long, env = "FLASHDNS_METADATA_DATABASE_URL")]
|
|
||||||
metadata_database_url: Option<String>,
|
|
||||||
|
|
||||||
/// Run in single-node mode (required when metadata backend is SQLite)
|
|
||||||
#[arg(long, env = "FLASHDNS_SINGLE_NODE")]
|
|
||||||
single_node: bool,
|
|
||||||
|
|
||||||
/// Log level (overrides config)
|
/// Log level (overrides config)
|
||||||
#[arg(short, long)]
|
#[arg(short, long)]
|
||||||
log_level: Option<String>,
|
log_level: Option<String>,
|
||||||
|
|
@ -72,54 +49,22 @@ struct CliArgs {
|
||||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let cli_args = CliArgs::parse();
|
let cli_args = CliArgs::parse();
|
||||||
|
|
||||||
// Load configuration using config-rs
|
let mut config = if cli_args.config.exists() {
|
||||||
let mut settings = Cfg::builder()
|
|
||||||
// Layer 1: Application defaults. Serialize ServerConfig::default() into TOML.
|
|
||||||
.add_source(File::from_str(
|
|
||||||
toml::to_string(&ServerConfig::default())?.as_str(),
|
|
||||||
FileFormat::Toml,
|
|
||||||
))
|
|
||||||
// Layer 2: Environment variables (e.g., FLASHDNS_GRPC_ADDR, FLASHDNS_LOG_LEVEL)
|
|
||||||
.add_source(
|
|
||||||
Environment::with_prefix("FLASHDNS")
|
|
||||||
.separator("__") // Use double underscore for nested fields
|
|
||||||
);
|
|
||||||
|
|
||||||
// Layer 3: Configuration file (if specified)
|
|
||||||
if cli_args.config.exists() {
|
|
||||||
tracing::info!("Loading config from file: {}", cli_args.config.display());
|
tracing::info!("Loading config from file: {}", cli_args.config.display());
|
||||||
settings = settings.add_source(File::from(cli_args.config.as_path()));
|
let contents = tokio::fs::read_to_string(&cli_args.config).await?;
|
||||||
|
toml::from_str(&contents)?
|
||||||
} else {
|
} else {
|
||||||
tracing::info!("Config file not found, using defaults and environment variables.");
|
tracing::info!("Config file not found, using defaults.");
|
||||||
}
|
ServerConfig::default()
|
||||||
|
};
|
||||||
|
|
||||||
let mut config: ServerConfig = settings
|
// Apply command line overrides
|
||||||
.build()?
|
|
||||||
.try_deserialize()
|
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to load configuration: {}", e))?;
|
|
||||||
|
|
||||||
// Apply command line overrides (Layer 4: highest precedence)
|
|
||||||
if let Some(grpc_addr_str) = cli_args.grpc_addr {
|
if let Some(grpc_addr_str) = cli_args.grpc_addr {
|
||||||
config.grpc_addr = grpc_addr_str.parse()?;
|
config.grpc_addr = grpc_addr_str.parse()?;
|
||||||
}
|
}
|
||||||
if let Some(dns_addr_str) = cli_args.dns_addr {
|
if let Some(dns_addr_str) = cli_args.dns_addr {
|
||||||
config.dns_addr = dns_addr_str.parse()?;
|
config.dns_addr = dns_addr_str.parse()?;
|
||||||
}
|
}
|
||||||
if let Some(chainfire_endpoint) = cli_args.chainfire_endpoint {
|
|
||||||
config.chainfire_endpoint = Some(chainfire_endpoint);
|
|
||||||
}
|
|
||||||
if let Some(flaredb_endpoint) = cli_args.flaredb_endpoint {
|
|
||||||
config.flaredb_endpoint = Some(flaredb_endpoint);
|
|
||||||
}
|
|
||||||
if let Some(metadata_backend) = cli_args.metadata_backend {
|
|
||||||
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
|
|
||||||
}
|
|
||||||
if let Some(metadata_database_url) = cli_args.metadata_database_url {
|
|
||||||
config.metadata_database_url = Some(metadata_database_url);
|
|
||||||
}
|
|
||||||
if cli_args.single_node {
|
|
||||||
config.single_node = true;
|
|
||||||
}
|
|
||||||
if let Some(log_level) = cli_args.log_level {
|
if let Some(log_level) = cli_args.log_level {
|
||||||
config.log_level = log_level;
|
config.log_level = log_level;
|
||||||
}
|
}
|
||||||
|
|
@ -172,20 +117,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
config.flaredb_endpoint.clone(),
|
config.flaredb_endpoint.clone(),
|
||||||
config.chainfire_endpoint.clone(),
|
config.chainfire_endpoint.clone(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to initialize FlareDB metadata store: {}", e))?,
|
.map_err(|e| {
|
||||||
|
anyhow::anyhow!("Failed to initialize FlareDB metadata store: {}", e)
|
||||||
|
})?,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
|
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
|
||||||
let database_url = config
|
let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
|
||||||
.metadata_database_url
|
anyhow::anyhow!(
|
||||||
.as_deref()
|
"metadata_database_url is required when metadata_backend={}",
|
||||||
.ok_or_else(|| {
|
metadata_backend_name(config.metadata_backend)
|
||||||
anyhow::anyhow!(
|
)
|
||||||
"metadata_database_url is required when metadata_backend={} (env: FLASHDNS_METADATA_DATABASE_URL)",
|
})?;
|
||||||
metadata_backend_name(config.metadata_backend)
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
|
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
" Metadata backend: {} @ {}",
|
" Metadata backend: {} @ {}",
|
||||||
|
|
@ -195,13 +139,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
Arc::new(
|
Arc::new(
|
||||||
DnsMetadataStore::new_sql(database_url, config.single_node)
|
DnsMetadataStore::new_sql(database_url, config.single_node)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to initialize SQL metadata store: {}", e))?,
|
.map_err(|e| {
|
||||||
|
anyhow::anyhow!("Failed to initialize SQL metadata store: {}", e)
|
||||||
|
})?,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Initialize IAM authentication service
|
// Initialize IAM authentication service
|
||||||
tracing::info!("Connecting to IAM server at {}", config.auth.iam_server_addr);
|
tracing::info!(
|
||||||
|
"Connecting to IAM server at {}",
|
||||||
|
config.auth.iam_server_addr
|
||||||
|
);
|
||||||
let auth_service = AuthService::new(&config.auth.iam_server_addr)
|
let auth_service = AuthService::new(&config.auth.iam_server_addr)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to connect to IAM server: {}", e))?;
|
.map_err(|e| anyhow::anyhow!("Failed to connect to IAM server: {}", e))?;
|
||||||
|
|
@ -300,18 +249,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend> {
|
|
||||||
match value.trim().to_ascii_lowercase().as_str() {
|
|
||||||
"flaredb" => Ok(MetadataBackend::FlareDb),
|
|
||||||
"postgres" => Ok(MetadataBackend::Postgres),
|
|
||||||
"sqlite" => Ok(MetadataBackend::Sqlite),
|
|
||||||
other => Err(anyhow::anyhow!(
|
|
||||||
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
|
|
||||||
other
|
|
||||||
)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
|
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
|
||||||
match backend {
|
match backend {
|
||||||
MetadataBackend::FlareDb => "flaredb",
|
MetadataBackend::FlareDb => "flaredb",
|
||||||
|
|
@ -345,11 +282,7 @@ fn ensure_sql_backend_matches_url(backend: MetadataBackend, database_url: &str)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn register_chainfire_membership(
|
async fn register_chainfire_membership(endpoint: &str, service: &str, addr: String) -> Result<()> {
|
||||||
endpoint: &str,
|
|
||||||
service: &str,
|
|
||||||
addr: String,
|
|
||||||
) -> Result<()> {
|
|
||||||
let node_id =
|
let node_id =
|
||||||
std::env::var("HOSTNAME").unwrap_or_else(|_| format!("{}-{}", service, std::process::id()));
|
std::env::var("HOSTNAME").unwrap_or_else(|_| format!("{}-{}", service, std::process::id()));
|
||||||
let ts = SystemTime::now()
|
let ts = SystemTime::now()
|
||||||
|
|
|
||||||
|
|
@ -57,12 +57,8 @@ impl DnsMetadataStore {
|
||||||
endpoint: Option<String>,
|
endpoint: Option<String>,
|
||||||
pd_endpoint: Option<String>,
|
pd_endpoint: Option<String>,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let endpoint = endpoint.unwrap_or_else(|| {
|
let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
|
||||||
std::env::var("FLASHDNS_FLAREDB_ENDPOINT")
|
|
||||||
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
|
|
||||||
});
|
|
||||||
let pd_endpoint = pd_endpoint
|
let pd_endpoint = pd_endpoint
|
||||||
.or_else(|| std::env::var("FLASHDNS_CHAINFIRE_ENDPOINT").ok())
|
|
||||||
.map(|value| normalize_transport_addr(&value))
|
.map(|value| normalize_transport_addr(&value))
|
||||||
.unwrap_or_else(|| endpoint.clone());
|
.unwrap_or_else(|| endpoint.clone());
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -27,6 +27,14 @@ pub struct ServerConfig {
|
||||||
/// Logging configuration
|
/// Logging configuration
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub logging: LoggingConfig,
|
pub logging: LoggingConfig,
|
||||||
|
|
||||||
|
/// Admin API policy configuration
|
||||||
|
#[serde(default)]
|
||||||
|
pub admin: AdminConfig,
|
||||||
|
|
||||||
|
/// Development-only safety valves
|
||||||
|
#[serde(default)]
|
||||||
|
pub dev: DevConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ServerConfig {
|
impl ServerConfig {
|
||||||
|
|
@ -102,6 +110,29 @@ impl ServerConfig {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let Ok(value) = std::env::var("IAM_ALLOW_UNAUTHENTICATED_ADMIN")
|
||||||
|
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_UNAUTHENTICATED_ADMIN"))
|
||||||
|
{
|
||||||
|
let value = value.trim().to_ascii_lowercase();
|
||||||
|
config.admin.allow_unauthenticated =
|
||||||
|
matches!(value.as_str(), "1" | "true" | "yes" | "on");
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Ok(value) = std::env::var("IAM_ALLOW_RANDOM_SIGNING_KEY")
|
||||||
|
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_RANDOM_SIGNING_KEY"))
|
||||||
|
{
|
||||||
|
let value = value.trim().to_ascii_lowercase();
|
||||||
|
config.dev.allow_random_signing_key =
|
||||||
|
matches!(value.as_str(), "1" | "true" | "yes" | "on");
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Ok(value) = std::env::var("IAM_ALLOW_MEMORY_BACKEND")
|
||||||
|
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_MEMORY_BACKEND"))
|
||||||
|
{
|
||||||
|
let value = value.trim().to_ascii_lowercase();
|
||||||
|
config.dev.allow_memory_backend = matches!(value.as_str(), "1" | "true" | "yes" | "on");
|
||||||
|
}
|
||||||
|
|
||||||
Ok(config)
|
Ok(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -132,6 +163,8 @@ impl ServerConfig {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
logging: LoggingConfig::default(),
|
logging: LoggingConfig::default(),
|
||||||
|
admin: AdminConfig::default(),
|
||||||
|
dev: DevConfig::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -226,6 +259,26 @@ pub struct ClusterConfig {
|
||||||
pub chainfire_endpoint: Option<String>,
|
pub chainfire_endpoint: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Admin API policy configuration
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
|
pub struct AdminConfig {
|
||||||
|
/// Allow admin APIs to run without an explicit admin token (dev only)
|
||||||
|
#[serde(default)]
|
||||||
|
pub allow_unauthenticated: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Development-only safety valves
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
|
pub struct DevConfig {
|
||||||
|
/// Allow generating a random signing key when none is configured
|
||||||
|
#[serde(default)]
|
||||||
|
pub allow_random_signing_key: bool,
|
||||||
|
|
||||||
|
/// Allow the in-memory backend
|
||||||
|
#[serde(default)]
|
||||||
|
pub allow_memory_backend: bool,
|
||||||
|
}
|
||||||
|
|
||||||
/// Backend type
|
/// Backend type
|
||||||
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "lowercase")]
|
#[serde(rename_all = "lowercase")]
|
||||||
|
|
|
||||||
|
|
@ -64,17 +64,18 @@ fn load_admin_token() -> Option<String> {
|
||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn allow_unauthenticated_admin() -> bool {
|
fn allow_unauthenticated_admin(config: &ServerConfig) -> bool {
|
||||||
std::env::var("IAM_ALLOW_UNAUTHENTICATED_ADMIN")
|
config.admin.allow_unauthenticated
|
||||||
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_UNAUTHENTICATED_ADMIN"))
|
|| std::env::var("IAM_ALLOW_UNAUTHENTICATED_ADMIN")
|
||||||
.ok()
|
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_UNAUTHENTICATED_ADMIN"))
|
||||||
.map(|value| {
|
.ok()
|
||||||
matches!(
|
.map(|value| {
|
||||||
value.trim().to_ascii_lowercase().as_str(),
|
matches!(
|
||||||
"1" | "true" | "yes" | "y" | "on"
|
value.trim().to_ascii_lowercase().as_str(),
|
||||||
)
|
"1" | "true" | "yes" | "y" | "on"
|
||||||
})
|
)
|
||||||
.unwrap_or(false)
|
})
|
||||||
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn admin_token_valid(metadata: &MetadataMap, token: &str) -> bool {
|
fn admin_token_valid(metadata: &MetadataMap, token: &str) -> bool {
|
||||||
|
|
@ -334,19 +335,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
|
||||||
// Create token service
|
// Create token service
|
||||||
let signing_key = if config.authn.internal_token.signing_key.is_empty() {
|
let signing_key = if config.authn.internal_token.signing_key.is_empty() {
|
||||||
let allow_random = std::env::var("IAM_ALLOW_RANDOM_SIGNING_KEY")
|
let allow_random = config.dev.allow_random_signing_key
|
||||||
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_RANDOM_SIGNING_KEY"))
|
|| std::env::var("IAM_ALLOW_RANDOM_SIGNING_KEY")
|
||||||
.ok()
|
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_RANDOM_SIGNING_KEY"))
|
||||||
.map(|value| {
|
.ok()
|
||||||
matches!(
|
.map(|value| {
|
||||||
value.trim().to_lowercase().as_str(),
|
matches!(
|
||||||
"1" | "true" | "yes" | "y" | "on"
|
value.trim().to_lowercase().as_str(),
|
||||||
)
|
"1" | "true" | "yes" | "y" | "on"
|
||||||
})
|
)
|
||||||
.unwrap_or(false);
|
})
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
if !allow_random {
|
if !allow_random {
|
||||||
return Err("No signing key configured. Set IAM_ALLOW_RANDOM_SIGNING_KEY=true for dev or configure authn.internal_token.signing_key.".into());
|
return Err("No signing key configured. Set dev.allow_random_signing_key=true (or IAM_ALLOW_RANDOM_SIGNING_KEY=true for legacy dev mode) or configure authn.internal_token.signing_key.".into());
|
||||||
}
|
}
|
||||||
|
|
||||||
warn!("No signing key configured, generating random key (dev-only)");
|
warn!("No signing key configured, generating random key (dev-only)");
|
||||||
|
|
@ -369,9 +371,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
|
||||||
let token_service = Arc::new(InternalTokenService::new(token_config));
|
let token_service = Arc::new(InternalTokenService::new(token_config));
|
||||||
let admin_token = load_admin_token();
|
let admin_token = load_admin_token();
|
||||||
if admin_token.is_none() && !allow_unauthenticated_admin() {
|
if admin_token.is_none() && !allow_unauthenticated_admin(&config) {
|
||||||
return Err(
|
return Err(
|
||||||
"IAM admin token not configured. Set IAM_ADMIN_TOKEN or explicitly allow dev mode with IAM_ALLOW_UNAUTHENTICATED_ADMIN=true."
|
"IAM admin token not configured. Set IAM_ADMIN_TOKEN or explicitly allow dev mode with admin.allow_unauthenticated=true."
|
||||||
.into(),
|
.into(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -550,19 +552,20 @@ async fn create_backend(
|
||||||
) -> Result<Backend, Box<dyn std::error::Error>> {
|
) -> Result<Backend, Box<dyn std::error::Error>> {
|
||||||
match config.store.backend {
|
match config.store.backend {
|
||||||
BackendKind::Memory => {
|
BackendKind::Memory => {
|
||||||
let allow_memory = std::env::var("IAM_ALLOW_MEMORY_BACKEND")
|
let allow_memory = config.dev.allow_memory_backend
|
||||||
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_MEMORY_BACKEND"))
|
|| std::env::var("IAM_ALLOW_MEMORY_BACKEND")
|
||||||
.ok()
|
.or_else(|_| std::env::var("PHOTON_IAM_ALLOW_MEMORY_BACKEND"))
|
||||||
.map(|value| {
|
.ok()
|
||||||
matches!(
|
.map(|value| {
|
||||||
value.trim().to_ascii_lowercase().as_str(),
|
matches!(
|
||||||
"1" | "true" | "yes" | "on"
|
value.trim().to_ascii_lowercase().as_str(),
|
||||||
)
|
"1" | "true" | "yes" | "on"
|
||||||
})
|
)
|
||||||
.unwrap_or(false);
|
})
|
||||||
|
.unwrap_or(false);
|
||||||
if !allow_memory {
|
if !allow_memory {
|
||||||
return Err(
|
return Err(
|
||||||
"In-memory IAM backend is disabled. Use FlareDB backend, or set IAM_ALLOW_MEMORY_BACKEND=true for tests/dev only."
|
"In-memory IAM backend is disabled. Use FlareDB backend, or set dev.allow_memory_backend=true for tests/dev only."
|
||||||
.into(),
|
.into(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -91,6 +91,18 @@ impl Default for PrismNetConfig {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub struct CreditServiceConfig {
|
||||||
|
#[serde(default)]
|
||||||
|
pub server_addr: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for CreditServiceConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self { server_addr: None }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct ChainFireConfig {
|
pub struct ChainFireConfig {
|
||||||
pub endpoint: Option<String>,
|
pub endpoint: Option<String>,
|
||||||
|
|
@ -110,6 +122,8 @@ pub struct Config {
|
||||||
pub flaredb: FlareDbConfig,
|
pub flaredb: FlareDbConfig,
|
||||||
pub chainfire: ChainFireConfig,
|
pub chainfire: ChainFireConfig,
|
||||||
pub iam: IamConfig,
|
pub iam: IamConfig,
|
||||||
|
#[serde(default)]
|
||||||
|
pub creditservice: CreditServiceConfig,
|
||||||
pub fiberlb: FiberLbConfig,
|
pub fiberlb: FiberLbConfig,
|
||||||
pub flashdns: FlashDnsConfig,
|
pub flashdns: FlashDnsConfig,
|
||||||
pub prismnet: PrismNetConfig,
|
pub prismnet: PrismNetConfig,
|
||||||
|
|
|
||||||
|
|
@ -38,8 +38,8 @@ use tracing_subscriber::EnvFilter;
|
||||||
#[command(about = "Kubernetes API server for PlasmaCloud's k8shost component")]
|
#[command(about = "Kubernetes API server for PlasmaCloud's k8shost component")]
|
||||||
struct Args {
|
struct Args {
|
||||||
/// Configuration file path
|
/// Configuration file path
|
||||||
#[arg(short, long)]
|
#[arg(short, long, default_value = "k8shost.toml")]
|
||||||
config: Option<PathBuf>,
|
config: PathBuf,
|
||||||
|
|
||||||
/// Listen address for gRPC server (e.g., "[::]:6443")
|
/// Listen address for gRPC server (e.g., "[::]:6443")
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
|
|
@ -91,17 +91,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let args = Args::parse();
|
let args = Args::parse();
|
||||||
|
|
||||||
// Load configuration
|
// Load configuration
|
||||||
let mut settings = ::config::Config::builder()
|
let mut settings = ::config::Config::builder().add_source(::config::File::from_str(
|
||||||
.add_source(::config::File::from_str(
|
toml::to_string(&Config::default())?.as_str(),
|
||||||
toml::to_string(&Config::default())?.as_str(),
|
::config::FileFormat::Toml,
|
||||||
::config::FileFormat::Toml,
|
));
|
||||||
))
|
|
||||||
.add_source(::config::Environment::with_prefix("K8SHOST").separator("_"));
|
|
||||||
|
|
||||||
// Add config file if specified
|
if args.config.exists() {
|
||||||
if let Some(config_path) = &args.config {
|
info!("Loading config from file: {}", args.config.display());
|
||||||
info!("Loading config from file: {}", config_path.display());
|
settings = settings.add_source(::config::File::from(args.config.as_path()));
|
||||||
settings = settings.add_source(::config::File::from(config_path.as_path()));
|
} else {
|
||||||
|
info!(
|
||||||
|
"Config file not found: {}, using defaults",
|
||||||
|
args.config.display()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let loaded_config: Config = settings
|
let loaded_config: Config = settings
|
||||||
|
|
@ -136,6 +138,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
.iam_server_addr
|
.iam_server_addr
|
||||||
.unwrap_or(loaded_config.iam.server_addr),
|
.unwrap_or(loaded_config.iam.server_addr),
|
||||||
},
|
},
|
||||||
|
creditservice: config::CreditServiceConfig {
|
||||||
|
server_addr: loaded_config.creditservice.server_addr,
|
||||||
|
},
|
||||||
fiberlb: config::FiberLbConfig {
|
fiberlb: config::FiberLbConfig {
|
||||||
server_addr: args
|
server_addr: args
|
||||||
.fiberlb_server_addr
|
.fiberlb_server_addr
|
||||||
|
|
@ -275,8 +280,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let ipam_client = Arc::new(IpamClient::new(config.prismnet.server_addr.clone()));
|
let ipam_client = Arc::new(IpamClient::new(config.prismnet.server_addr.clone()));
|
||||||
|
|
||||||
// Create service implementations with storage
|
// Create service implementations with storage
|
||||||
|
let creditservice_endpoint = config.creditservice.server_addr.as_deref();
|
||||||
let pod_service = Arc::new(
|
let pod_service = Arc::new(
|
||||||
PodServiceImpl::new_with_credit_service(storage.clone(), auth_service.clone()).await,
|
PodServiceImpl::new_with_credit_service(
|
||||||
|
storage.clone(),
|
||||||
|
auth_service.clone(),
|
||||||
|
creditservice_endpoint,
|
||||||
|
)
|
||||||
|
.await,
|
||||||
);
|
);
|
||||||
let service_service = Arc::new(ServiceServiceImpl::new(
|
let service_service = Arc::new(ServiceServiceImpl::new(
|
||||||
storage.clone(),
|
storage.clone(),
|
||||||
|
|
@ -290,7 +301,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
));
|
));
|
||||||
|
|
||||||
// Start scheduler in background with CreditService integration
|
// Start scheduler in background with CreditService integration
|
||||||
let scheduler = Arc::new(scheduler::Scheduler::new_with_credit_service(storage.clone()).await);
|
let scheduler = Arc::new(
|
||||||
|
scheduler::Scheduler::new_with_credit_service(storage.clone(), creditservice_endpoint)
|
||||||
|
.await,
|
||||||
|
);
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
scheduler.run().await;
|
scheduler.run().await;
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -33,27 +33,31 @@ impl Scheduler {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create a new scheduler with CreditService quota enforcement
|
/// Create a new scheduler with CreditService quota enforcement
|
||||||
pub async fn new_with_credit_service(storage: Arc<Storage>) -> Self {
|
pub async fn new_with_credit_service(storage: Arc<Storage>, endpoint: Option<&str>) -> Self {
|
||||||
// Initialize CreditService client if endpoint is configured
|
// Initialize CreditService client if endpoint is configured
|
||||||
let credit_service = match std::env::var("CREDITSERVICE_ENDPOINT") {
|
let credit_service = match endpoint {
|
||||||
Ok(endpoint) => match CreditServiceClient::connect(&endpoint).await {
|
Some(endpoint) if !endpoint.trim().is_empty() => {
|
||||||
Ok(client) => {
|
match CreditServiceClient::connect(endpoint).await {
|
||||||
info!(
|
Ok(client) => {
|
||||||
"Scheduler: CreditService quota enforcement enabled: {}",
|
info!(
|
||||||
endpoint
|
"Scheduler: CreditService quota enforcement enabled: {}",
|
||||||
);
|
endpoint
|
||||||
Some(Arc::new(RwLock::new(client)))
|
);
|
||||||
}
|
Some(Arc::new(RwLock::new(client)))
|
||||||
Err(e) => {
|
}
|
||||||
warn!(
|
Err(e) => {
|
||||||
|
warn!(
|
||||||
"Scheduler: Failed to connect to CreditService (quota enforcement disabled): {}",
|
"Scheduler: Failed to connect to CreditService (quota enforcement disabled): {}",
|
||||||
e
|
e
|
||||||
);
|
);
|
||||||
None
|
None
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
Err(_) => {
|
_ => {
|
||||||
info!("Scheduler: CREDITSERVICE_ENDPOINT not set, quota enforcement disabled");
|
info!(
|
||||||
|
"Scheduler: CreditService endpoint not configured, quota enforcement disabled"
|
||||||
|
);
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -45,24 +45,30 @@ impl PodServiceImpl {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn new_with_credit_service(storage: Arc<Storage>, auth: Arc<AuthService>) -> Self {
|
pub async fn new_with_credit_service(
|
||||||
|
storage: Arc<Storage>,
|
||||||
|
auth: Arc<AuthService>,
|
||||||
|
endpoint: Option<&str>,
|
||||||
|
) -> Self {
|
||||||
// Initialize CreditService client if endpoint is configured
|
// Initialize CreditService client if endpoint is configured
|
||||||
let credit_service = match std::env::var("CREDITSERVICE_ENDPOINT") {
|
let credit_service = match endpoint {
|
||||||
Ok(endpoint) => match CreditServiceClient::connect(&endpoint).await {
|
Some(endpoint) if !endpoint.trim().is_empty() => {
|
||||||
Ok(client) => {
|
match CreditServiceClient::connect(endpoint).await {
|
||||||
tracing::info!("CreditService admission control enabled: {}", endpoint);
|
Ok(client) => {
|
||||||
Some(Arc::new(RwLock::new(client)))
|
tracing::info!("CreditService admission control enabled: {}", endpoint);
|
||||||
|
Some(Arc::new(RwLock::new(client)))
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(
|
||||||
|
"Failed to connect to CreditService (admission control disabled): {}",
|
||||||
|
e
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
}
|
||||||
tracing::warn!(
|
_ => {
|
||||||
"Failed to connect to CreditService (admission control disabled): {}",
|
tracing::info!("CreditService endpoint not configured, admission control disabled");
|
||||||
e
|
|
||||||
);
|
|
||||||
None
|
|
||||||
}
|
|
||||||
},
|
|
||||||
Err(_) => {
|
|
||||||
tracing::info!("CREDITSERVICE_ENDPOINT not set, admission control disabled");
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -50,6 +50,75 @@ pub enum ObjectStorageBackend {
|
||||||
Distributed,
|
Distributed,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct S3AuthConfig {
|
||||||
|
#[serde(default = "default_s3_auth_enabled")]
|
||||||
|
pub enabled: bool,
|
||||||
|
|
||||||
|
#[serde(default = "default_s3_aws_region")]
|
||||||
|
pub aws_region: String,
|
||||||
|
|
||||||
|
#[serde(default = "default_s3_iam_cache_ttl_secs")]
|
||||||
|
pub iam_cache_ttl_secs: u64,
|
||||||
|
|
||||||
|
#[serde(default = "default_s3_default_org_id")]
|
||||||
|
pub default_org_id: Option<String>,
|
||||||
|
|
||||||
|
#[serde(default = "default_s3_default_project_id")]
|
||||||
|
pub default_project_id: Option<String>,
|
||||||
|
|
||||||
|
#[serde(default = "default_s3_max_auth_body_bytes")]
|
||||||
|
pub max_auth_body_bytes: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for S3AuthConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
enabled: default_s3_auth_enabled(),
|
||||||
|
aws_region: default_s3_aws_region(),
|
||||||
|
iam_cache_ttl_secs: default_s3_iam_cache_ttl_secs(),
|
||||||
|
default_org_id: default_s3_default_org_id(),
|
||||||
|
default_project_id: default_s3_default_project_id(),
|
||||||
|
max_auth_body_bytes: default_s3_max_auth_body_bytes(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct S3PerformanceConfig {
|
||||||
|
#[serde(default = "default_s3_streaming_put_threshold_bytes")]
|
||||||
|
pub streaming_put_threshold_bytes: usize,
|
||||||
|
|
||||||
|
#[serde(default = "default_s3_inline_put_max_bytes")]
|
||||||
|
pub inline_put_max_bytes: usize,
|
||||||
|
|
||||||
|
#[serde(default = "default_s3_multipart_put_concurrency")]
|
||||||
|
pub multipart_put_concurrency: usize,
|
||||||
|
|
||||||
|
#[serde(default = "default_s3_multipart_fetch_concurrency")]
|
||||||
|
pub multipart_fetch_concurrency: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for S3PerformanceConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
streaming_put_threshold_bytes: default_s3_streaming_put_threshold_bytes(),
|
||||||
|
inline_put_max_bytes: default_s3_inline_put_max_bytes(),
|
||||||
|
multipart_put_concurrency: default_s3_multipart_put_concurrency(),
|
||||||
|
multipart_fetch_concurrency: default_s3_multipart_fetch_concurrency(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
|
pub struct S3Config {
|
||||||
|
#[serde(default)]
|
||||||
|
pub auth: S3AuthConfig,
|
||||||
|
|
||||||
|
#[serde(default)]
|
||||||
|
pub performance: S3PerformanceConfig,
|
||||||
|
}
|
||||||
|
|
||||||
/// Server configuration
|
/// Server configuration
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct ServerConfig {
|
pub struct ServerConfig {
|
||||||
|
|
@ -100,6 +169,10 @@ pub struct ServerConfig {
|
||||||
/// Authentication configuration
|
/// Authentication configuration
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub auth: AuthConfig,
|
pub auth: AuthConfig,
|
||||||
|
|
||||||
|
/// S3 API runtime settings
|
||||||
|
#[serde(default)]
|
||||||
|
pub s3: S3Config,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Authentication configuration
|
/// Authentication configuration
|
||||||
|
|
@ -114,6 +187,46 @@ fn default_iam_server_addr() -> String {
|
||||||
"127.0.0.1:50051".to_string()
|
"127.0.0.1:50051".to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_s3_auth_enabled() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_s3_aws_region() -> String {
|
||||||
|
"us-east-1".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_s3_iam_cache_ttl_secs() -> u64 {
|
||||||
|
30
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_s3_default_org_id() -> Option<String> {
|
||||||
|
Some("default".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_s3_default_project_id() -> Option<String> {
|
||||||
|
Some("default".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_s3_max_auth_body_bytes() -> usize {
|
||||||
|
1024 * 1024 * 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_s3_streaming_put_threshold_bytes() -> usize {
|
||||||
|
16 * 1024 * 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_s3_inline_put_max_bytes() -> usize {
|
||||||
|
128 * 1024 * 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_s3_multipart_put_concurrency() -> usize {
|
||||||
|
4
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_s3_multipart_fetch_concurrency() -> usize {
|
||||||
|
4
|
||||||
|
}
|
||||||
|
|
||||||
impl Default for AuthConfig {
|
impl Default for AuthConfig {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
|
@ -139,6 +252,7 @@ impl Default for ServerConfig {
|
||||||
sync_on_write: false,
|
sync_on_write: false,
|
||||||
tls: None,
|
tls: None,
|
||||||
auth: AuthConfig::default(),
|
auth: AuthConfig::default(),
|
||||||
|
s3: S3Config::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ use clap::Parser;
|
||||||
use iam_service_auth::AuthService;
|
use iam_service_auth::AuthService;
|
||||||
use lightningstor_api::{BucketServiceServer, ObjectServiceServer};
|
use lightningstor_api::{BucketServiceServer, ObjectServiceServer};
|
||||||
use lightningstor_distributed::{
|
use lightningstor_distributed::{
|
||||||
DistributedConfig, ErasureCodedBackend, RedundancyMode, ReplicatedBackend, RepairQueue,
|
DistributedConfig, ErasureCodedBackend, RedundancyMode, RepairQueue, ReplicatedBackend,
|
||||||
StaticNodeRegistry,
|
StaticNodeRegistry,
|
||||||
};
|
};
|
||||||
use lightningstor_server::{
|
use lightningstor_server::{
|
||||||
|
|
@ -57,26 +57,6 @@ struct Args {
|
||||||
#[arg(short, long)]
|
#[arg(short, long)]
|
||||||
log_level: Option<String>,
|
log_level: Option<String>,
|
||||||
|
|
||||||
/// ChainFire endpoint for cluster coordination (overrides config)
|
|
||||||
#[arg(long, env = "LIGHTNINGSTOR_CHAINFIRE_ENDPOINT")]
|
|
||||||
chainfire_endpoint: Option<String>,
|
|
||||||
|
|
||||||
/// FlareDB endpoint for metadata and tenant data storage (overrides config)
|
|
||||||
#[arg(long, env = "LIGHTNINGSTOR_FLAREDB_ENDPOINT")]
|
|
||||||
flaredb_endpoint: Option<String>,
|
|
||||||
|
|
||||||
/// Metadata backend (flaredb, postgres, sqlite)
|
|
||||||
#[arg(long, env = "LIGHTNINGSTOR_METADATA_BACKEND")]
|
|
||||||
metadata_backend: Option<String>,
|
|
||||||
|
|
||||||
/// SQL database URL for metadata (required for postgres/sqlite backend)
|
|
||||||
#[arg(long, env = "LIGHTNINGSTOR_METADATA_DATABASE_URL")]
|
|
||||||
metadata_database_url: Option<String>,
|
|
||||||
|
|
||||||
/// Run in single-node mode (required when metadata backend is SQLite)
|
|
||||||
#[arg(long, env = "LIGHTNINGSTOR_SINGLE_NODE")]
|
|
||||||
single_node: bool,
|
|
||||||
|
|
||||||
/// Data directory for object storage (overrides config)
|
/// Data directory for object storage (overrides config)
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
data_dir: Option<String>,
|
data_dir: Option<String>,
|
||||||
|
|
@ -112,21 +92,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
if let Some(log_level) = args.log_level {
|
if let Some(log_level) = args.log_level {
|
||||||
config.log_level = log_level;
|
config.log_level = log_level;
|
||||||
}
|
}
|
||||||
if let Some(chainfire_endpoint) = args.chainfire_endpoint {
|
|
||||||
config.chainfire_endpoint = Some(chainfire_endpoint);
|
|
||||||
}
|
|
||||||
if let Some(flaredb_endpoint) = args.flaredb_endpoint {
|
|
||||||
config.flaredb_endpoint = Some(flaredb_endpoint);
|
|
||||||
}
|
|
||||||
if let Some(metadata_backend) = args.metadata_backend {
|
|
||||||
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
|
|
||||||
}
|
|
||||||
if let Some(metadata_database_url) = args.metadata_database_url {
|
|
||||||
config.metadata_database_url = Some(metadata_database_url);
|
|
||||||
}
|
|
||||||
if args.single_node {
|
|
||||||
config.single_node = true;
|
|
||||||
}
|
|
||||||
if let Some(data_dir) = args.data_dir {
|
if let Some(data_dir) = args.data_dir {
|
||||||
config.data_dir = data_dir;
|
config.data_dir = data_dir;
|
||||||
}
|
}
|
||||||
|
|
@ -182,20 +147,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
config.flaredb_endpoint.clone(),
|
config.flaredb_endpoint.clone(),
|
||||||
config.chainfire_endpoint.clone(),
|
config.chainfire_endpoint.clone(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("Failed to initialize FlareDB metadata store: {}", e))?,
|
.map_err(|e| format!("Failed to initialize FlareDB metadata store: {}", e))?,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
|
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
|
||||||
let database_url = config
|
let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
|
||||||
.metadata_database_url
|
format!(
|
||||||
.as_deref()
|
"metadata_database_url is required when metadata_backend={}",
|
||||||
.ok_or_else(|| {
|
metadata_backend_name(config.metadata_backend)
|
||||||
format!(
|
)
|
||||||
"metadata_database_url is required when metadata_backend={} (env: LIGHTNINGSTOR_METADATA_DATABASE_URL)",
|
})?;
|
||||||
metadata_backend_name(config.metadata_backend)
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
|
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
"Metadata backend: {} @ {}",
|
"Metadata backend: {} @ {}",
|
||||||
|
|
@ -263,10 +225,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let s3_addr: SocketAddr = config.s3_addr;
|
let s3_addr: SocketAddr = config.s3_addr;
|
||||||
|
|
||||||
// Start S3 HTTP server with shared state
|
// Start S3 HTTP server with shared state
|
||||||
let s3_router = s3::create_router_with_auth(
|
let s3_router = s3::create_router_with_auth_config(
|
||||||
storage.clone(),
|
storage.clone(),
|
||||||
metadata.clone(),
|
metadata.clone(),
|
||||||
Some(config.auth.iam_server_addr.clone()),
|
Some(config.auth.iam_server_addr.clone()),
|
||||||
|
config.s3.clone(),
|
||||||
);
|
);
|
||||||
let s3_server = tokio::spawn(async move {
|
let s3_server = tokio::spawn(async move {
|
||||||
tracing::info!("S3 HTTP server listening on {}", s3_addr);
|
tracing::info!("S3 HTTP server listening on {}", s3_addr);
|
||||||
|
|
@ -341,19 +304,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
|
|
||||||
match value.trim().to_ascii_lowercase().as_str() {
|
|
||||||
"flaredb" => Ok(MetadataBackend::FlareDb),
|
|
||||||
"postgres" => Ok(MetadataBackend::Postgres),
|
|
||||||
"sqlite" => Ok(MetadataBackend::Sqlite),
|
|
||||||
other => Err(format!(
|
|
||||||
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
|
|
||||||
other
|
|
||||||
)
|
|
||||||
.into()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
|
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
|
||||||
match backend {
|
match backend {
|
||||||
MetadataBackend::FlareDb => "flaredb",
|
MetadataBackend::FlareDb => "flaredb",
|
||||||
|
|
@ -442,7 +392,9 @@ async fn create_storage_backend(
|
||||||
ObjectStorageBackend::LocalFs => {
|
ObjectStorageBackend::LocalFs => {
|
||||||
tracing::info!("Object storage backend: local_fs");
|
tracing::info!("Object storage backend: local_fs");
|
||||||
Ok(StorageRuntime {
|
Ok(StorageRuntime {
|
||||||
backend: Arc::new(LocalFsBackend::new(&config.data_dir, config.sync_on_write).await?),
|
backend: Arc::new(
|
||||||
|
LocalFsBackend::new(&config.data_dir, config.sync_on_write).await?,
|
||||||
|
),
|
||||||
repair_worker: None,
|
repair_worker: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -55,26 +55,25 @@ impl MetadataStore {
|
||||||
endpoint: Option<String>,
|
endpoint: Option<String>,
|
||||||
pd_endpoint: Option<String>,
|
pd_endpoint: Option<String>,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let endpoint = endpoint.unwrap_or_else(|| {
|
let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
|
||||||
std::env::var("LIGHTNINGSTOR_FLAREDB_ENDPOINT")
|
|
||||||
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
|
|
||||||
});
|
|
||||||
let pd_endpoint = pd_endpoint
|
let pd_endpoint = pd_endpoint
|
||||||
.or_else(|| std::env::var("LIGHTNINGSTOR_CHAINFIRE_ENDPOINT").ok())
|
|
||||||
.map(|value| normalize_transport_addr(&value))
|
.map(|value| normalize_transport_addr(&value))
|
||||||
.unwrap_or_else(|| endpoint.clone());
|
.unwrap_or_else(|| endpoint.clone());
|
||||||
|
|
||||||
let mut clients = Vec::with_capacity(FLAREDB_CLIENT_POOL_SIZE);
|
let mut clients = Vec::with_capacity(FLAREDB_CLIENT_POOL_SIZE);
|
||||||
for _ in 0..FLAREDB_CLIENT_POOL_SIZE {
|
for _ in 0..FLAREDB_CLIENT_POOL_SIZE {
|
||||||
let client =
|
let client = RdbClient::connect_with_pd_namespace(
|
||||||
RdbClient::connect_with_pd_namespace(endpoint.clone(), pd_endpoint.clone(), "lightningstor")
|
endpoint.clone(),
|
||||||
.await
|
pd_endpoint.clone(),
|
||||||
.map_err(|e| {
|
"lightningstor",
|
||||||
lightningstor_types::Error::StorageError(format!(
|
)
|
||||||
"Failed to connect to FlareDB: {}",
|
.await
|
||||||
e
|
.map_err(|e| {
|
||||||
))
|
lightningstor_types::Error::StorageError(format!(
|
||||||
})?;
|
"Failed to connect to FlareDB: {}",
|
||||||
|
e
|
||||||
|
))
|
||||||
|
})?;
|
||||||
clients.push(Arc::new(Mutex::new(client)));
|
clients.push(Arc::new(Mutex::new(client)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -321,7 +320,11 @@ impl MetadataStore {
|
||||||
Ok((results, next))
|
Ok((results, next))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn flaredb_put(clients: &[Arc<Mutex<RdbClient>>], key: &[u8], value: &[u8]) -> Result<()> {
|
async fn flaredb_put(
|
||||||
|
clients: &[Arc<Mutex<RdbClient>>],
|
||||||
|
key: &[u8],
|
||||||
|
value: &[u8],
|
||||||
|
) -> Result<()> {
|
||||||
let client = Self::flaredb_client_for_key(clients, key);
|
let client = Self::flaredb_client_for_key(clients, key);
|
||||||
let raw_result = {
|
let raw_result = {
|
||||||
let mut c = client.lock().await;
|
let mut c = client.lock().await;
|
||||||
|
|
@ -443,7 +446,8 @@ impl MetadataStore {
|
||||||
let client = Self::flaredb_scan_client(clients);
|
let client = Self::flaredb_scan_client(clients);
|
||||||
let (mut items, next) = match {
|
let (mut items, next) = match {
|
||||||
let mut c = client.lock().await;
|
let mut c = client.lock().await;
|
||||||
c.raw_scan(start_key.clone(), end_key.clone(), fetch_limit).await
|
c.raw_scan(start_key.clone(), end_key.clone(), fetch_limit)
|
||||||
|
.await
|
||||||
} {
|
} {
|
||||||
Ok((keys, values, next)) => {
|
Ok((keys, values, next)) => {
|
||||||
let items = keys
|
let items = keys
|
||||||
|
|
@ -697,12 +701,13 @@ impl MetadataStore {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
StorageBackend::Sql(sql) => {
|
StorageBackend::Sql(sql) => {
|
||||||
let prefix_end = String::from_utf8(Self::prefix_end(prefix.as_bytes())).map_err(|e| {
|
let prefix_end =
|
||||||
lightningstor_types::Error::StorageError(format!(
|
String::from_utf8(Self::prefix_end(prefix.as_bytes())).map_err(|e| {
|
||||||
"Failed to encode prefix end: {}",
|
lightningstor_types::Error::StorageError(format!(
|
||||||
e
|
"Failed to encode prefix end: {}",
|
||||||
))
|
e
|
||||||
})?;
|
))
|
||||||
|
})?;
|
||||||
let fetch_limit = (limit.saturating_add(1)) as i64;
|
let fetch_limit = (limit.saturating_add(1)) as i64;
|
||||||
match sql {
|
match sql {
|
||||||
SqlStorageBackend::Postgres(pool) => {
|
SqlStorageBackend::Postgres(pool) => {
|
||||||
|
|
@ -908,7 +913,10 @@ impl MetadataStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn multipart_bucket_prefix(bucket_id: &BucketId, prefix: &str) -> String {
|
fn multipart_bucket_prefix(bucket_id: &BucketId, prefix: &str) -> String {
|
||||||
format!("/lightningstor/multipart/by-bucket/{}/{}", bucket_id, prefix)
|
format!(
|
||||||
|
"/lightningstor/multipart/by-bucket/{}/{}",
|
||||||
|
bucket_id, prefix
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn multipart_object_key(object_id: &ObjectId) -> String {
|
fn multipart_object_key(object_id: &ObjectId) -> String {
|
||||||
|
|
@ -955,7 +963,8 @@ impl MetadataStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn delete_replicated_repair_task(&self, task_id: &str) -> Result<()> {
|
pub async fn delete_replicated_repair_task(&self, task_id: &str) -> Result<()> {
|
||||||
self.delete_key(&Self::replicated_repair_task_key(task_id)).await
|
self.delete_key(&Self::replicated_repair_task_key(task_id))
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Save bucket metadata
|
/// Save bucket metadata
|
||||||
|
|
@ -1246,7 +1255,8 @@ impl MetadataStore {
|
||||||
))
|
))
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
self.delete_key(&Self::multipart_upload_key(upload_id)).await
|
self.delete_key(&Self::multipart_upload_key(upload_id))
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn list_multipart_uploads(
|
pub async fn list_multipart_uploads(
|
||||||
|
|
@ -1331,7 +1341,8 @@ impl MetadataStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn delete_object_multipart_upload(&self, object_id: &ObjectId) -> Result<()> {
|
pub async fn delete_object_multipart_upload(&self, object_id: &ObjectId) -> Result<()> {
|
||||||
self.delete_key(&Self::multipart_object_key(object_id)).await
|
self.delete_key(&Self::multipart_object_key(object_id))
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1380,13 +1391,11 @@ mod tests {
|
||||||
store.delete_bucket(&bucket).await.unwrap();
|
store.delete_bucket(&bucket).await.unwrap();
|
||||||
assert!(!store.bucket_cache.contains_key(&cache_key));
|
assert!(!store.bucket_cache.contains_key(&cache_key));
|
||||||
assert!(!store.bucket_cache.contains_key(&cache_id_key));
|
assert!(!store.bucket_cache.contains_key(&cache_id_key));
|
||||||
assert!(
|
assert!(store
|
||||||
store
|
.load_bucket("org-a", "project-a", "bench-bucket")
|
||||||
.load_bucket("org-a", "project-a", "bench-bucket")
|
.await
|
||||||
.await
|
.unwrap()
|
||||||
.unwrap()
|
.is_none());
|
||||||
.is_none()
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|
@ -1426,13 +1435,11 @@ mod tests {
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(!store.object_cache.contains_key(&cache_key));
|
assert!(!store.object_cache.contains_key(&cache_key));
|
||||||
assert!(
|
assert!(store
|
||||||
store
|
.load_object(&bucket.id, object.key.as_str(), None)
|
||||||
.load_object(&bucket.id, object.key.as_str(), None)
|
.await
|
||||||
.await
|
.unwrap()
|
||||||
.unwrap()
|
.is_none());
|
||||||
.is_none()
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|
@ -1496,8 +1503,10 @@ mod tests {
|
||||||
);
|
);
|
||||||
store.save_bucket(&bucket).await.unwrap();
|
store.save_bucket(&bucket).await.unwrap();
|
||||||
|
|
||||||
let upload_a = MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/one.bin").unwrap());
|
let upload_a =
|
||||||
let upload_b = MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/two.bin").unwrap());
|
MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/one.bin").unwrap());
|
||||||
|
let upload_b =
|
||||||
|
MultipartUpload::new(bucket.id.to_string(), ObjectKey::new("a/two.bin").unwrap());
|
||||||
let other_bucket = Bucket::new(
|
let other_bucket = Bucket::new(
|
||||||
BucketName::new("other-bucket").unwrap(),
|
BucketName::new("other-bucket").unwrap(),
|
||||||
"org-a",
|
"org-a",
|
||||||
|
|
@ -1505,8 +1514,10 @@ mod tests {
|
||||||
"default",
|
"default",
|
||||||
);
|
);
|
||||||
store.save_bucket(&other_bucket).await.unwrap();
|
store.save_bucket(&other_bucket).await.unwrap();
|
||||||
let upload_other =
|
let upload_other = MultipartUpload::new(
|
||||||
MultipartUpload::new(other_bucket.id.to_string(), ObjectKey::new("a/three.bin").unwrap());
|
other_bucket.id.to_string(),
|
||||||
|
ObjectKey::new("a/three.bin").unwrap(),
|
||||||
|
);
|
||||||
|
|
||||||
store.save_multipart_upload(&upload_a).await.unwrap();
|
store.save_multipart_upload(&upload_a).await.unwrap();
|
||||||
store.save_multipart_upload(&upload_b).await.unwrap();
|
store.save_multipart_upload(&upload_b).await.unwrap();
|
||||||
|
|
@ -1543,10 +1554,7 @@ mod tests {
|
||||||
assert_eq!(tasks[0].attempt_count, 1);
|
assert_eq!(tasks[0].attempt_count, 1);
|
||||||
assert_eq!(tasks[0].last_error.as_deref(), Some("transient failure"));
|
assert_eq!(tasks[0].last_error.as_deref(), Some("transient failure"));
|
||||||
|
|
||||||
store
|
store.delete_replicated_repair_task(&task.id).await.unwrap();
|
||||||
.delete_replicated_repair_task(&task.id)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert!(store
|
assert!(store
|
||||||
.list_replicated_repair_tasks(10)
|
.list_replicated_repair_tasks(10)
|
||||||
.await
|
.await
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
//! Implements simplified SigV4 authentication compatible with AWS S3 SDKs and aws-cli.
|
//! Implements simplified SigV4 authentication compatible with AWS S3 SDKs and aws-cli.
|
||||||
//! Integrates with IAM for access key validation.
|
//! Integrates with IAM for access key validation.
|
||||||
|
|
||||||
|
use crate::config::S3AuthConfig;
|
||||||
use crate::tenant::TenantContext;
|
use crate::tenant::TenantContext;
|
||||||
use axum::{
|
use axum::{
|
||||||
body::{Body, Bytes},
|
body::{Body, Bytes},
|
||||||
|
|
@ -23,8 +24,6 @@ use tracing::{debug, warn};
|
||||||
use url::form_urlencoded;
|
use url::form_urlencoded;
|
||||||
|
|
||||||
type HmacSha256 = Hmac<Sha256>;
|
type HmacSha256 = Hmac<Sha256>;
|
||||||
const DEFAULT_MAX_AUTH_BODY_BYTES: usize = 1024 * 1024 * 1024;
|
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub(crate) struct VerifiedBodyBytes(pub Bytes);
|
pub(crate) struct VerifiedBodyBytes(pub Bytes);
|
||||||
|
|
||||||
|
|
@ -49,6 +48,8 @@ pub struct AuthState {
|
||||||
aws_region: String,
|
aws_region: String,
|
||||||
/// AWS service name for SigV4 (e.g., s3)
|
/// AWS service name for SigV4 (e.g., s3)
|
||||||
aws_service: String,
|
aws_service: String,
|
||||||
|
/// Maximum request body size to buffer during auth verification
|
||||||
|
max_auth_body_bytes: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct IamClient {
|
pub struct IamClient {
|
||||||
|
|
@ -60,6 +61,8 @@ pub struct IamClient {
|
||||||
enum IamClientMode {
|
enum IamClientMode {
|
||||||
Env {
|
Env {
|
||||||
credentials: std::collections::HashMap<String, String>,
|
credentials: std::collections::HashMap<String, String>,
|
||||||
|
default_org_id: Option<String>,
|
||||||
|
default_project_id: Option<String>,
|
||||||
},
|
},
|
||||||
Grpc {
|
Grpc {
|
||||||
endpoint: String,
|
endpoint: String,
|
||||||
|
|
@ -83,11 +86,11 @@ struct CachedCredential {
|
||||||
impl IamClient {
|
impl IamClient {
|
||||||
/// Create a new IAM client. If an endpoint is supplied, use the IAM gRPC API.
|
/// Create a new IAM client. If an endpoint is supplied, use the IAM gRPC API.
|
||||||
pub fn new(iam_endpoint: Option<String>) -> Self {
|
pub fn new(iam_endpoint: Option<String>) -> Self {
|
||||||
let cache_ttl = std::env::var("LIGHTNINGSTOR_S3_IAM_CACHE_TTL_SECS")
|
Self::new_with_config(iam_endpoint, &S3AuthConfig::default())
|
||||||
.ok()
|
}
|
||||||
.and_then(|value| value.parse::<u64>().ok())
|
|
||||||
.map(StdDuration::from_secs)
|
pub fn new_with_config(iam_endpoint: Option<String>, config: &S3AuthConfig) -> Self {
|
||||||
.unwrap_or_else(|| StdDuration::from_secs(30));
|
let cache_ttl = StdDuration::from_secs(config.iam_cache_ttl_secs);
|
||||||
|
|
||||||
if let Some(endpoint) = iam_endpoint
|
if let Some(endpoint) = iam_endpoint
|
||||||
.map(|value| normalize_iam_endpoint(&value))
|
.map(|value| normalize_iam_endpoint(&value))
|
||||||
|
|
@ -106,6 +109,8 @@ impl IamClient {
|
||||||
Self {
|
Self {
|
||||||
mode: IamClientMode::Env {
|
mode: IamClientMode::Env {
|
||||||
credentials: Self::load_env_credentials(),
|
credentials: Self::load_env_credentials(),
|
||||||
|
default_org_id: config.default_org_id.clone(),
|
||||||
|
default_project_id: config.default_project_id.clone(),
|
||||||
},
|
},
|
||||||
credential_cache: Arc::new(RwLock::new(HashMap::new())),
|
credential_cache: Arc::new(RwLock::new(HashMap::new())),
|
||||||
cache_ttl,
|
cache_ttl,
|
||||||
|
|
@ -160,32 +165,32 @@ impl IamClient {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
fn env_credentials(&self) -> Option<&std::collections::HashMap<String, String>> {
|
fn env_credentials(&self) -> Option<&std::collections::HashMap<String, String>> {
|
||||||
match &self.mode {
|
match &self.mode {
|
||||||
IamClientMode::Env { credentials } => Some(credentials),
|
IamClientMode::Env { credentials, .. } => Some(credentials),
|
||||||
IamClientMode::Grpc { .. } => None,
|
IamClientMode::Grpc { .. } => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn env_default_tenant() -> (Option<String>, Option<String>) {
|
fn env_default_tenant(
|
||||||
let org_id = std::env::var("S3_TENANT_ORG_ID")
|
default_org_id: Option<String>,
|
||||||
.ok()
|
default_project_id: Option<String>,
|
||||||
.or_else(|| std::env::var("S3_ORG_ID").ok())
|
) -> (Option<String>, Option<String>) {
|
||||||
.or_else(|| Some("default".to_string()));
|
(default_org_id, default_project_id)
|
||||||
let project_id = std::env::var("S3_TENANT_PROJECT_ID")
|
|
||||||
.ok()
|
|
||||||
.or_else(|| std::env::var("S3_PROJECT_ID").ok())
|
|
||||||
.or_else(|| Some("default".to_string()));
|
|
||||||
(org_id, project_id)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Validate access key and resolve the credential context.
|
/// Validate access key and resolve the credential context.
|
||||||
pub async fn get_credential(&self, access_key_id: &str) -> Result<ResolvedCredential, String> {
|
pub async fn get_credential(&self, access_key_id: &str) -> Result<ResolvedCredential, String> {
|
||||||
match &self.mode {
|
match &self.mode {
|
||||||
IamClientMode::Env { credentials } => {
|
IamClientMode::Env {
|
||||||
|
credentials,
|
||||||
|
default_org_id,
|
||||||
|
default_project_id,
|
||||||
|
} => {
|
||||||
let secret_key = credentials
|
let secret_key = credentials
|
||||||
.get(access_key_id)
|
.get(access_key_id)
|
||||||
.cloned()
|
.cloned()
|
||||||
.ok_or_else(|| "Access key ID not found".to_string())?;
|
.ok_or_else(|| "Access key ID not found".to_string())?;
|
||||||
let (org_id, project_id) = Self::env_default_tenant();
|
let (org_id, project_id) =
|
||||||
|
Self::env_default_tenant(default_org_id.clone(), default_project_id.clone());
|
||||||
Ok(ResolvedCredential {
|
Ok(ResolvedCredential {
|
||||||
secret_key,
|
secret_key,
|
||||||
principal_id: access_key_id.to_string(),
|
principal_id: access_key_id.to_string(),
|
||||||
|
|
@ -318,16 +323,21 @@ fn iam_admin_token() -> Option<String> {
|
||||||
impl AuthState {
|
impl AuthState {
|
||||||
/// Create new auth state with IAM integration
|
/// Create new auth state with IAM integration
|
||||||
pub fn new(iam_endpoint: Option<String>) -> Self {
|
pub fn new(iam_endpoint: Option<String>) -> Self {
|
||||||
let iam_client = Some(Arc::new(RwLock::new(IamClient::new(iam_endpoint))));
|
Self::new_with_config(iam_endpoint, &S3AuthConfig::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn new_with_config(iam_endpoint: Option<String>, config: &S3AuthConfig) -> Self {
|
||||||
|
let iam_client = Some(Arc::new(RwLock::new(IamClient::new_with_config(
|
||||||
|
iam_endpoint,
|
||||||
|
config,
|
||||||
|
))));
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
iam_client,
|
iam_client,
|
||||||
enabled: std::env::var("S3_AUTH_ENABLED")
|
enabled: config.enabled,
|
||||||
.unwrap_or_else(|_| "true".to_string())
|
aws_region: config.aws_region.clone(),
|
||||||
.parse()
|
|
||||||
.unwrap_or(true),
|
|
||||||
aws_region: std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string()), // Default S3 region
|
|
||||||
aws_service: "s3".to_string(),
|
aws_service: "s3".to_string(),
|
||||||
|
max_auth_body_bytes: config.max_auth_body_bytes,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -338,6 +348,7 @@ impl AuthState {
|
||||||
enabled: false,
|
enabled: false,
|
||||||
aws_region: "us-east-1".to_string(),
|
aws_region: "us-east-1".to_string(),
|
||||||
aws_service: "s3".to_string(),
|
aws_service: "s3".to_string(),
|
||||||
|
max_auth_body_bytes: S3AuthConfig::default().max_auth_body_bytes,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -438,12 +449,8 @@ pub async fn sigv4_auth_middleware(
|
||||||
let should_buffer_body = should_buffer_auth_body(payload_hash_header.as_deref());
|
let should_buffer_body = should_buffer_auth_body(payload_hash_header.as_deref());
|
||||||
|
|
||||||
let body_bytes = if should_buffer_body {
|
let body_bytes = if should_buffer_body {
|
||||||
let max_body_bytes = std::env::var("S3_MAX_AUTH_BODY_BYTES")
|
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.parse::<usize>().ok())
|
|
||||||
.unwrap_or(DEFAULT_MAX_AUTH_BODY_BYTES);
|
|
||||||
let (parts, body) = request.into_parts();
|
let (parts, body) = request.into_parts();
|
||||||
let body_bytes = match axum::body::to_bytes(body, max_body_bytes).await {
|
let body_bytes = match axum::body::to_bytes(body, auth_state.max_auth_body_bytes).await {
|
||||||
Ok(b) => b,
|
Ok(b) => b,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
return error_response(
|
return error_response(
|
||||||
|
|
|
||||||
|
|
@ -6,5 +6,8 @@ mod auth;
|
||||||
mod router;
|
mod router;
|
||||||
mod xml;
|
mod xml;
|
||||||
|
|
||||||
pub use auth::{AuthState, sigv4_auth_middleware};
|
pub use auth::{sigv4_auth_middleware, AuthState};
|
||||||
pub use router::{create_router, create_router_with_auth, create_router_with_state};
|
pub use router::{
|
||||||
|
create_router, create_router_with_auth, create_router_with_auth_config,
|
||||||
|
create_router_with_state,
|
||||||
|
};
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
//! S3 API router using Axum
|
//! S3 API router using Axum
|
||||||
|
|
||||||
|
use crate::config::{S3Config, S3PerformanceConfig};
|
||||||
use axum::{
|
use axum::{
|
||||||
body::{Body, Bytes},
|
body::{Body, Bytes},
|
||||||
extract::{Request, State},
|
extract::{Request, State},
|
||||||
|
|
@ -14,8 +15,8 @@ use futures::{stream, stream::FuturesUnordered, StreamExt};
|
||||||
use http_body_util::BodyExt;
|
use http_body_util::BodyExt;
|
||||||
use md5::{Digest, Md5};
|
use md5::{Digest, Md5};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use std::io;
|
|
||||||
use sha2::Sha256;
|
use sha2::Sha256;
|
||||||
|
use std::io;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
|
|
||||||
|
|
@ -38,17 +39,24 @@ use super::xml::{
|
||||||
pub struct S3State {
|
pub struct S3State {
|
||||||
pub storage: Arc<dyn StorageBackend>,
|
pub storage: Arc<dyn StorageBackend>,
|
||||||
pub metadata: Arc<MetadataStore>,
|
pub metadata: Arc<MetadataStore>,
|
||||||
|
pub performance: S3PerformanceConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Keep streamed single-PUT parts aligned with the distributed backend's
|
|
||||||
// large-object chunking so GET does not degrade into many small serial reads.
|
|
||||||
const DEFAULT_STREAMING_PUT_THRESHOLD_BYTES: usize = 16 * 1024 * 1024;
|
|
||||||
const DEFAULT_INLINE_PUT_MAX_BYTES: usize = 128 * 1024 * 1024;
|
|
||||||
const DEFAULT_MULTIPART_PUT_CONCURRENCY: usize = 4;
|
|
||||||
|
|
||||||
impl S3State {
|
impl S3State {
|
||||||
pub fn new(storage: Arc<dyn StorageBackend>, metadata: Arc<MetadataStore>) -> Self {
|
pub fn new(storage: Arc<dyn StorageBackend>, metadata: Arc<MetadataStore>) -> Self {
|
||||||
Self { storage, metadata }
|
Self::new_with_config(storage, metadata, S3PerformanceConfig::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn new_with_config(
|
||||||
|
storage: Arc<dyn StorageBackend>,
|
||||||
|
metadata: Arc<MetadataStore>,
|
||||||
|
performance: S3PerformanceConfig,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
storage,
|
||||||
|
metadata,
|
||||||
|
performance,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -57,7 +65,12 @@ pub fn create_router_with_state(
|
||||||
storage: Arc<dyn StorageBackend>,
|
storage: Arc<dyn StorageBackend>,
|
||||||
metadata: Arc<MetadataStore>,
|
metadata: Arc<MetadataStore>,
|
||||||
) -> Router {
|
) -> Router {
|
||||||
create_router_with_auth_state(storage, metadata, Arc::new(AuthState::new(None)))
|
create_router_with_auth_state(
|
||||||
|
storage,
|
||||||
|
metadata,
|
||||||
|
Arc::new(AuthState::new(None)),
|
||||||
|
S3PerformanceConfig::default(),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create the S3-compatible HTTP router with auth and storage backends
|
/// Create the S3-compatible HTTP router with auth and storage backends
|
||||||
|
|
@ -66,15 +79,30 @@ pub fn create_router_with_auth(
|
||||||
metadata: Arc<MetadataStore>,
|
metadata: Arc<MetadataStore>,
|
||||||
iam_endpoint: Option<String>,
|
iam_endpoint: Option<String>,
|
||||||
) -> Router {
|
) -> Router {
|
||||||
create_router_with_auth_state(storage, metadata, Arc::new(AuthState::new(iam_endpoint)))
|
create_router_with_auth_config(storage, metadata, iam_endpoint, S3Config::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn create_router_with_auth_config(
|
||||||
|
storage: Arc<dyn StorageBackend>,
|
||||||
|
metadata: Arc<MetadataStore>,
|
||||||
|
iam_endpoint: Option<String>,
|
||||||
|
config: S3Config,
|
||||||
|
) -> Router {
|
||||||
|
create_router_with_auth_state(
|
||||||
|
storage,
|
||||||
|
metadata,
|
||||||
|
Arc::new(AuthState::new_with_config(iam_endpoint, &config.auth)),
|
||||||
|
config.performance,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn create_router_with_auth_state(
|
fn create_router_with_auth_state(
|
||||||
storage: Arc<dyn StorageBackend>,
|
storage: Arc<dyn StorageBackend>,
|
||||||
metadata: Arc<MetadataStore>,
|
metadata: Arc<MetadataStore>,
|
||||||
auth_state: Arc<AuthState>,
|
auth_state: Arc<AuthState>,
|
||||||
|
performance: S3PerformanceConfig,
|
||||||
) -> Router {
|
) -> Router {
|
||||||
let state = Arc::new(S3State::new(storage, metadata));
|
let state = Arc::new(S3State::new_with_config(storage, metadata, performance));
|
||||||
|
|
||||||
Router::new()
|
Router::new()
|
||||||
// Catch-all route for ALL operations (including root /)
|
// Catch-all route for ALL operations (including root /)
|
||||||
|
|
@ -126,28 +154,16 @@ fn request_tenant(extensions: &axum::http::Extensions) -> TenantContext {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn streaming_put_threshold_bytes() -> usize {
|
fn streaming_put_threshold_bytes(state: &Arc<S3State>) -> usize {
|
||||||
std::env::var("LIGHTNINGSTOR_S3_STREAMING_PUT_THRESHOLD_BYTES")
|
state.performance.streaming_put_threshold_bytes
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.parse::<usize>().ok())
|
|
||||||
.filter(|value| *value > 0)
|
|
||||||
.unwrap_or(DEFAULT_STREAMING_PUT_THRESHOLD_BYTES)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn inline_put_max_bytes() -> usize {
|
fn inline_put_max_bytes(state: &Arc<S3State>) -> usize {
|
||||||
std::env::var("LIGHTNINGSTOR_S3_INLINE_PUT_MAX_BYTES")
|
state.performance.inline_put_max_bytes
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.parse::<usize>().ok())
|
|
||||||
.filter(|value| *value > 0)
|
|
||||||
.unwrap_or(DEFAULT_INLINE_PUT_MAX_BYTES)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn multipart_put_concurrency() -> usize {
|
fn multipart_put_concurrency(state: &Arc<S3State>) -> usize {
|
||||||
std::env::var("LIGHTNINGSTOR_S3_MULTIPART_PUT_CONCURRENCY")
|
state.performance.multipart_put_concurrency
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.parse::<usize>().ok())
|
|
||||||
.filter(|value| *value > 0)
|
|
||||||
.unwrap_or(DEFAULT_MULTIPART_PUT_CONCURRENCY)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn request_content_length(headers: &HeaderMap) -> Option<usize> {
|
fn request_content_length(headers: &HeaderMap) -> Option<usize> {
|
||||||
|
|
@ -751,13 +767,13 @@ async fn put_object(
|
||||||
(body_len, etag, None, Some(body_bytes))
|
(body_len, etag, None, Some(body_bytes))
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
let prepared = if let Some(content_length) =
|
let prepared = if let Some(content_length) = content_length
|
||||||
content_length.filter(|content_length| *content_length <= inline_put_max_bytes())
|
.filter(|content_length| *content_length <= inline_put_max_bytes(&state))
|
||||||
{
|
{
|
||||||
read_inline_put_body(
|
read_inline_put_body(
|
||||||
body,
|
body,
|
||||||
verified_payload_hash.as_deref(),
|
verified_payload_hash.as_deref(),
|
||||||
inline_put_max_bytes(),
|
inline_put_max_bytes(&state),
|
||||||
Some(content_length),
|
Some(content_length),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|
@ -948,7 +964,7 @@ async fn stream_put_body(
|
||||||
) -> Result<PreparedPutBody, Response<Body>> {
|
) -> Result<PreparedPutBody, Response<Body>> {
|
||||||
let verify_payload_hash = expected_payload_hash
|
let verify_payload_hash = expected_payload_hash
|
||||||
.filter(|expected_payload_hash| *expected_payload_hash != "UNSIGNED-PAYLOAD");
|
.filter(|expected_payload_hash| *expected_payload_hash != "UNSIGNED-PAYLOAD");
|
||||||
let threshold = streaming_put_threshold_bytes();
|
let threshold = streaming_put_threshold_bytes(state);
|
||||||
let mut buffered = BytesMut::with_capacity(threshold);
|
let mut buffered = BytesMut::with_capacity(threshold);
|
||||||
let mut full_md5 = Some(Md5::new());
|
let mut full_md5 = Some(Md5::new());
|
||||||
let mut full_sha256 = verify_payload_hash.map(|_| Sha256::new());
|
let mut full_sha256 = verify_payload_hash.map(|_| Sha256::new());
|
||||||
|
|
@ -958,7 +974,7 @@ async fn stream_put_body(
|
||||||
let mut scheduled_part_numbers = Vec::new();
|
let mut scheduled_part_numbers = Vec::new();
|
||||||
let mut completed_parts = Vec::new();
|
let mut completed_parts = Vec::new();
|
||||||
let mut in_flight_uploads = FuturesUnordered::new();
|
let mut in_flight_uploads = FuturesUnordered::new();
|
||||||
let max_in_flight_uploads = multipart_put_concurrency();
|
let max_in_flight_uploads = multipart_put_concurrency(state);
|
||||||
|
|
||||||
while let Some(frame) = body.frame().await {
|
while let Some(frame) = body.frame().await {
|
||||||
let frame = match frame {
|
let frame = match frame {
|
||||||
|
|
@ -1282,7 +1298,10 @@ fn multipart_object_body(state: Arc<S3State>, object: &Object, upload: Multipart
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
return Ok(Some((bytes.slice(0..body_end), (storage, upload, idx, offset))));
|
return Ok(Some((
|
||||||
|
bytes.slice(0..body_end),
|
||||||
|
(storage, upload, idx, offset),
|
||||||
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(None)
|
Ok(None)
|
||||||
|
|
@ -1374,7 +1393,11 @@ async fn get_object(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let multipart_upload = match state.metadata.load_object_multipart_upload(&object.id).await {
|
let multipart_upload = match state
|
||||||
|
.metadata
|
||||||
|
.load_object_multipart_upload(&object.id)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(upload) => upload,
|
Ok(upload) => upload,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
return error_response(
|
return error_response(
|
||||||
|
|
@ -1385,7 +1408,10 @@ async fn get_object(
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let (body, content_length) = if let Some(upload) = multipart_upload {
|
let (body, content_length) = if let Some(upload) = multipart_upload {
|
||||||
(multipart_object_body(Arc::clone(&state), &object, upload), object.size as usize)
|
(
|
||||||
|
multipart_object_body(Arc::clone(&state), &object, upload),
|
||||||
|
object.size as usize,
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
let data = match state.storage.get_object(&object.id).await {
|
let data = match state.storage.get_object(&object.id).await {
|
||||||
Ok(data) => data,
|
Ok(data) => data,
|
||||||
|
|
@ -1653,6 +1679,7 @@ mod tests {
|
||||||
storage,
|
storage,
|
||||||
metadata.clone(),
|
metadata.clone(),
|
||||||
Arc::new(AuthState::disabled()),
|
Arc::new(AuthState::disabled()),
|
||||||
|
S3PerformanceConfig::default(),
|
||||||
);
|
);
|
||||||
std::mem::forget(tempdir);
|
std::mem::forget(tempdir);
|
||||||
(router, metadata)
|
(router, metadata)
|
||||||
|
|
@ -1982,7 +2009,8 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(response.status(), StatusCode::OK);
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
|
||||||
let body = vec![b'x'; DEFAULT_STREAMING_PUT_THRESHOLD_BYTES + 1024];
|
let threshold = S3PerformanceConfig::default().streaming_put_threshold_bytes;
|
||||||
|
let body = vec![b'x'; threshold + 1024];
|
||||||
let response = router
|
let response = router
|
||||||
.clone()
|
.clone()
|
||||||
.oneshot(
|
.oneshot(
|
||||||
|
|
@ -2197,10 +2225,12 @@ mod tests {
|
||||||
async fn large_put_streams_multipart_parts_with_parallel_uploads() {
|
async fn large_put_streams_multipart_parts_with_parallel_uploads() {
|
||||||
let storage = Arc::new(DelayedMultipartStorage::new(Duration::from_millis(25)));
|
let storage = Arc::new(DelayedMultipartStorage::new(Duration::from_millis(25)));
|
||||||
let metadata = Arc::new(MetadataStore::new_in_memory());
|
let metadata = Arc::new(MetadataStore::new_in_memory());
|
||||||
|
let performance = S3PerformanceConfig::default();
|
||||||
let router = create_router_with_auth_state(
|
let router = create_router_with_auth_state(
|
||||||
storage.clone(),
|
storage.clone(),
|
||||||
metadata,
|
metadata,
|
||||||
Arc::new(AuthState::disabled()),
|
Arc::new(AuthState::disabled()),
|
||||||
|
performance.clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
let response = router
|
let response = router
|
||||||
|
|
@ -2216,7 +2246,7 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(response.status(), StatusCode::OK);
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
|
||||||
let body = vec![b'x'; (DEFAULT_STREAMING_PUT_THRESHOLD_BYTES * 2) + 4096];
|
let body = vec![b'x'; (performance.streaming_put_threshold_bytes * 2) + 4096];
|
||||||
let response = router
|
let response = router
|
||||||
.clone()
|
.clone()
|
||||||
.oneshot(
|
.oneshot(
|
||||||
|
|
@ -2250,10 +2280,12 @@ mod tests {
|
||||||
async fn moderate_put_with_content_length_stays_inline() {
|
async fn moderate_put_with_content_length_stays_inline() {
|
||||||
let storage = Arc::new(DelayedMultipartStorage::new(Duration::from_millis(25)));
|
let storage = Arc::new(DelayedMultipartStorage::new(Duration::from_millis(25)));
|
||||||
let metadata = Arc::new(MetadataStore::new_in_memory());
|
let metadata = Arc::new(MetadataStore::new_in_memory());
|
||||||
|
let performance = S3PerformanceConfig::default();
|
||||||
let router = create_router_with_auth_state(
|
let router = create_router_with_auth_state(
|
||||||
storage.clone(),
|
storage.clone(),
|
||||||
metadata.clone(),
|
metadata.clone(),
|
||||||
Arc::new(AuthState::disabled()),
|
Arc::new(AuthState::disabled()),
|
||||||
|
performance.clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
let response = router
|
let response = router
|
||||||
|
|
@ -2269,7 +2301,7 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(response.status(), StatusCode::OK);
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
|
||||||
let body = vec![b'y'; DEFAULT_STREAMING_PUT_THRESHOLD_BYTES + 4096];
|
let body = vec![b'y'; performance.streaming_put_threshold_bytes + 4096];
|
||||||
let response = router
|
let response = router
|
||||||
.clone()
|
.clone()
|
||||||
.oneshot(
|
.oneshot(
|
||||||
|
|
|
||||||
|
|
@ -170,5 +170,8 @@ pub struct CompleteMultipartUploadResult {
|
||||||
/// Convert to XML with declaration
|
/// Convert to XML with declaration
|
||||||
pub fn to_xml<T: Serialize>(value: &T) -> Result<String, quick_xml::DeError> {
|
pub fn to_xml<T: Serialize>(value: &T) -> Result<String, quick_xml::DeError> {
|
||||||
let xml = xml_to_string(value)?;
|
let xml = xml_to_string(value)?;
|
||||||
Ok(format!("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n{}", xml))
|
Ok(format!(
|
||||||
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n{}",
|
||||||
|
xml
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::fs;
|
use std::fs;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
/// Main server configuration
|
/// Main server configuration
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
|
@ -123,7 +124,7 @@ pub struct TlsConfig {
|
||||||
|
|
||||||
impl Config {
|
impl Config {
|
||||||
/// Load configuration from a YAML file
|
/// Load configuration from a YAML file
|
||||||
pub fn from_file(path: &str) -> Result<Self> {
|
pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
|
||||||
let content = fs::read_to_string(path)?;
|
let content = fs::read_to_string(path)?;
|
||||||
let config = serde_yaml::from_str(&content)?;
|
let config = serde_yaml::from_str(&content)?;
|
||||||
Ok(config)
|
Ok(config)
|
||||||
|
|
@ -136,27 +137,6 @@ impl Config {
|
||||||
fs::write(path, content)?;
|
fs::write(path, content)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Apply environment variable overrides
|
|
||||||
///
|
|
||||||
/// This allows NixOS service module to override configuration via environment variables.
|
|
||||||
/// Environment variables take precedence over configuration file values.
|
|
||||||
pub fn apply_env_overrides(&mut self) {
|
|
||||||
if let Ok(val) = std::env::var("NIGHTLIGHT_HTTP_ADDR") {
|
|
||||||
self.server.http_addr = val;
|
|
||||||
}
|
|
||||||
if let Ok(val) = std::env::var("NIGHTLIGHT_GRPC_ADDR") {
|
|
||||||
self.server.grpc_addr = val;
|
|
||||||
}
|
|
||||||
if let Ok(val) = std::env::var("NIGHTLIGHT_DATA_DIR") {
|
|
||||||
self.storage.data_dir = val;
|
|
||||||
}
|
|
||||||
if let Ok(val) = std::env::var("NIGHTLIGHT_RETENTION_DAYS") {
|
|
||||||
if let Ok(days) = val.parse() {
|
|
||||||
self.storage.retention_days = days;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for Config {
|
impl Default for Config {
|
||||||
|
|
|
||||||
|
|
@ -183,7 +183,11 @@ impl Admin for AdminServiceImpl {
|
||||||
_request: Request<HealthRequest>,
|
_request: Request<HealthRequest>,
|
||||||
) -> Result<Response<HealthResponse>, Status> {
|
) -> Result<Response<HealthResponse>, Status> {
|
||||||
let storage_result = self.storage.stats().await;
|
let storage_result = self.storage.stats().await;
|
||||||
let status = if storage_result.is_ok() { "ok" } else { "degraded" };
|
let status = if storage_result.is_ok() {
|
||||||
|
"ok"
|
||||||
|
} else {
|
||||||
|
"degraded"
|
||||||
|
};
|
||||||
let storage_message = match &storage_result {
|
let storage_message = match &storage_result {
|
||||||
Ok(_) => "storage ready".to_string(),
|
Ok(_) => "storage ready".to_string(),
|
||||||
Err(error) => error.to_string(),
|
Err(error) => error.to_string(),
|
||||||
|
|
@ -253,7 +257,9 @@ impl Admin for AdminServiceImpl {
|
||||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||||
commit: option_env!("GIT_COMMIT").unwrap_or("unknown").to_string(),
|
commit: option_env!("GIT_COMMIT").unwrap_or("unknown").to_string(),
|
||||||
build_time: option_env!("BUILD_TIME").unwrap_or("unknown").to_string(),
|
build_time: option_env!("BUILD_TIME").unwrap_or("unknown").to_string(),
|
||||||
rust_version: option_env!("RUSTC_VERSION").unwrap_or("unknown").to_string(),
|
rust_version: option_env!("RUSTC_VERSION")
|
||||||
|
.unwrap_or("unknown")
|
||||||
|
.to_string(),
|
||||||
target: format!("{}-{}", std::env::consts::ARCH, std::env::consts::OS),
|
target: format!("{}-{}", std::env::consts::ARCH, std::env::consts::OS),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
@ -330,9 +336,7 @@ mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::ingestion::IngestionService;
|
use crate::ingestion::IngestionService;
|
||||||
use crate::storage::Storage;
|
use crate::storage::Storage;
|
||||||
use nightlight_api::nightlight::{
|
use nightlight_api::nightlight::{InstantQueryRequest, LabelValuesRequest, SeriesQueryRequest};
|
||||||
InstantQueryRequest, LabelValuesRequest, SeriesQueryRequest,
|
|
||||||
};
|
|
||||||
use nightlight_api::prometheus::{Label, Sample, TimeSeries, WriteRequest};
|
use nightlight_api::prometheus::{Label, Sample, TimeSeries, WriteRequest};
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|
@ -380,7 +384,10 @@ mod tests {
|
||||||
data.result[0].metric.get("__name__").map(String::as_str),
|
data.result[0].metric.get("__name__").map(String::as_str),
|
||||||
Some("grpc_metric")
|
Some("grpc_metric")
|
||||||
);
|
);
|
||||||
assert_eq!(data.result[0].value.as_ref().map(|value| value.value), Some(12.5));
|
assert_eq!(
|
||||||
|
data.result[0].value.as_ref().map(|value| value.value),
|
||||||
|
Some(12.5)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|
@ -452,7 +459,10 @@ mod tests {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.into_inner();
|
.into_inner();
|
||||||
assert_eq!(label_values.status, "success");
|
assert_eq!(label_values.status, "success");
|
||||||
assert_eq!(label_values.data, vec!["api".to_string(), "worker".to_string()]);
|
assert_eq!(
|
||||||
|
label_values.data,
|
||||||
|
vec!["api".to_string(), "worker".to_string()]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|
@ -477,26 +487,33 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let query = QueryService::from_storage(storage.queryable());
|
let query = QueryService::from_storage(storage.queryable());
|
||||||
query.execute_instant_query("admin_metric", 2_000).await.unwrap();
|
query
|
||||||
|
.execute_instant_query("admin_metric", 2_000)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let admin = AdminServiceImpl::new(
|
let admin =
|
||||||
Arc::clone(&storage),
|
AdminServiceImpl::new(Arc::clone(&storage), ingestion.metrics(), query.metrics());
|
||||||
ingestion.metrics(),
|
|
||||||
query.metrics(),
|
|
||||||
);
|
|
||||||
let stats = admin
|
let stats = admin
|
||||||
.stats(Request::new(StatsRequest {}))
|
.stats(Request::new(StatsRequest {}))
|
||||||
.await
|
.await
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.into_inner();
|
.into_inner();
|
||||||
|
|
||||||
assert_eq!(stats.storage.as_ref().map(|value| value.total_samples), Some(1));
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
stats.ingestion
|
stats.storage.as_ref().map(|value| value.total_samples),
|
||||||
|
Some(1)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
stats
|
||||||
|
.ingestion
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|value| value.samples_ingested_total),
|
.map(|value| value.samples_ingested_total),
|
||||||
Some(1)
|
Some(1)
|
||||||
);
|
);
|
||||||
assert_eq!(stats.query.as_ref().map(|value| value.queries_total), Some(1));
|
assert_eq!(
|
||||||
|
stats.query.as_ref().map(|value| value.queries_total),
|
||||||
|
Some(1)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -15,8 +15,8 @@ use nightlight_api::prometheus::{Label, WriteRequest};
|
||||||
use nightlight_types::Error;
|
use nightlight_types::Error;
|
||||||
use prost::Message;
|
use prost::Message;
|
||||||
use snap::raw::Decoder as SnappyDecoder;
|
use snap::raw::Decoder as SnappyDecoder;
|
||||||
use std::sync::Arc;
|
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
use std::sync::Arc;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
|
|
||||||
|
|
@ -113,9 +113,8 @@ impl IngestionService {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store series with samples in shared storage
|
// Store series with samples in shared storage
|
||||||
let series_id = nightlight_types::SeriesId(
|
let series_id =
|
||||||
compute_series_fingerprint(&internal_labels)
|
nightlight_types::SeriesId(compute_series_fingerprint(&internal_labels));
|
||||||
);
|
|
||||||
|
|
||||||
let time_series = nightlight_types::TimeSeries {
|
let time_series = nightlight_types::TimeSeries {
|
||||||
id: series_id,
|
id: series_id,
|
||||||
|
|
@ -178,11 +177,11 @@ impl IngestionMetrics {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Axum handler for /api/v1/write endpoint
|
/// Axum handler for /api/v1/write endpoint
|
||||||
async fn handle_remote_write(
|
async fn handle_remote_write(State(service): State<IngestionService>, body: Bytes) -> Response {
|
||||||
State(service): State<IngestionService>,
|
service
|
||||||
body: Bytes,
|
.metrics
|
||||||
) -> Response {
|
.requests_total
|
||||||
service.metrics.requests_total.fetch_add(1, Ordering::Relaxed);
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
|
||||||
debug!("Received remote_write request, size: {} bytes", body.len());
|
debug!("Received remote_write request, size: {} bytes", body.len());
|
||||||
|
|
||||||
|
|
@ -225,17 +224,26 @@ async fn handle_remote_write(
|
||||||
}
|
}
|
||||||
Err(Error::Storage(msg)) if msg.contains("buffer full") => {
|
Err(Error::Storage(msg)) if msg.contains("buffer full") => {
|
||||||
warn!("Write buffer full, returning 429");
|
warn!("Write buffer full, returning 429");
|
||||||
service.metrics.requests_failed.fetch_add(1, Ordering::Relaxed);
|
service
|
||||||
|
.metrics
|
||||||
|
.requests_failed
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
IngestionError::Backpressure.into_response()
|
IngestionError::Backpressure.into_response()
|
||||||
}
|
}
|
||||||
Err(Error::InvalidLabel(msg)) => {
|
Err(Error::InvalidLabel(msg)) => {
|
||||||
warn!("Invalid labels: {}", msg);
|
warn!("Invalid labels: {}", msg);
|
||||||
service.metrics.requests_failed.fetch_add(1, Ordering::Relaxed);
|
service
|
||||||
|
.metrics
|
||||||
|
.requests_failed
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
IngestionError::InvalidLabels.into_response()
|
IngestionError::InvalidLabels.into_response()
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Failed to process write request: {}", e);
|
error!("Failed to process write request: {}", e);
|
||||||
service.metrics.requests_failed.fetch_add(1, Ordering::Relaxed);
|
service
|
||||||
|
.metrics
|
||||||
|
.requests_failed
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
IngestionError::StorageError.into_response()
|
IngestionError::StorageError.into_response()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -285,7 +293,11 @@ fn validate_labels(labels: Vec<Label>) -> Result<Vec<Label>, Error> {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Label names must contain only [a-zA-Z0-9_]
|
// Label names must contain only [a-zA-Z0-9_]
|
||||||
if !label.name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
|
if !label
|
||||||
|
.name
|
||||||
|
.chars()
|
||||||
|
.all(|c| c.is_ascii_alphanumeric() || c == '_')
|
||||||
|
{
|
||||||
return Err(Error::InvalidLabel(format!(
|
return Err(Error::InvalidLabel(format!(
|
||||||
"Invalid label name '{}': must contain only [a-zA-Z0-9_]",
|
"Invalid label name '{}': must contain only [a-zA-Z0-9_]",
|
||||||
label.name
|
label.name
|
||||||
|
|
@ -337,15 +349,12 @@ impl IntoResponse for IngestionError {
|
||||||
IngestionError::InvalidProtobuf => {
|
IngestionError::InvalidProtobuf => {
|
||||||
(StatusCode::BAD_REQUEST, "Invalid protobuf encoding")
|
(StatusCode::BAD_REQUEST, "Invalid protobuf encoding")
|
||||||
}
|
}
|
||||||
IngestionError::InvalidLabels => {
|
IngestionError::InvalidLabels => (StatusCode::BAD_REQUEST, "Invalid metric labels"),
|
||||||
(StatusCode::BAD_REQUEST, "Invalid metric labels")
|
IngestionError::StorageError => (StatusCode::INTERNAL_SERVER_ERROR, "Storage error"),
|
||||||
}
|
IngestionError::Backpressure => (
|
||||||
IngestionError::StorageError => {
|
StatusCode::TOO_MANY_REQUESTS,
|
||||||
(StatusCode::INTERNAL_SERVER_ERROR, "Storage error")
|
"Write buffer full, retry later",
|
||||||
}
|
),
|
||||||
IngestionError::Backpressure => {
|
|
||||||
(StatusCode::TOO_MANY_REQUESTS, "Write buffer full, retry later")
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
(status, message).into_response()
|
(status, message).into_response()
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ use std::time::Duration;
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use axum::{routing::get, Router};
|
use axum::{routing::get, Router};
|
||||||
|
use clap::Parser;
|
||||||
use nightlight_api::nightlight::admin_server::AdminServer;
|
use nightlight_api::nightlight::admin_server::AdminServer;
|
||||||
use nightlight_api::nightlight::metric_query_server::MetricQueryServer;
|
use nightlight_api::nightlight::metric_query_server::MetricQueryServer;
|
||||||
use tokio::time::MissedTickBehavior;
|
use tokio::time::MissedTickBehavior;
|
||||||
|
|
@ -33,8 +34,18 @@ use storage::Storage;
|
||||||
|
|
||||||
const DEFAULT_SNAPSHOT_INTERVAL_SECS: u64 = 30;
|
const DEFAULT_SNAPSHOT_INTERVAL_SECS: u64 = 30;
|
||||||
|
|
||||||
|
#[derive(Parser, Debug)]
|
||||||
|
#[command(name = "nightlight-server")]
|
||||||
|
struct Args {
|
||||||
|
/// Configuration file path
|
||||||
|
#[arg(short, long, default_value = "nightlight.yaml")]
|
||||||
|
config: std::path::PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> Result<()> {
|
||||||
|
let args = Args::parse();
|
||||||
|
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
.with_max_level(Level::INFO)
|
.with_max_level(Level::INFO)
|
||||||
.with_target(false)
|
.with_target(false)
|
||||||
|
|
@ -44,17 +55,20 @@ async fn main() -> Result<()> {
|
||||||
info!("Nightlight server starting");
|
info!("Nightlight server starting");
|
||||||
info!("Version: {}", env!("CARGO_PKG_VERSION"));
|
info!("Version: {}", env!("CARGO_PKG_VERSION"));
|
||||||
|
|
||||||
let mut config = match Config::from_file("config.yaml") {
|
let config = match Config::from_file(&args.config) {
|
||||||
Ok(config) => {
|
Ok(config) => {
|
||||||
info!("Configuration loaded from config.yaml");
|
info!("Configuration loaded from {}", args.config.display());
|
||||||
config
|
config
|
||||||
}
|
}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
info!("Failed to load config.yaml: {}, using defaults", error);
|
info!(
|
||||||
|
"Failed to load {}: {}, using defaults",
|
||||||
|
args.config.display(),
|
||||||
|
error
|
||||||
|
);
|
||||||
Config::default()
|
Config::default()
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
config.apply_env_overrides();
|
|
||||||
|
|
||||||
if config.tls.is_some() {
|
if config.tls.is_some() {
|
||||||
warn!("Nightlight TLS configuration is currently ignored; starting plaintext listeners");
|
warn!("Nightlight TLS configuration is currently ignored; starting plaintext listeners");
|
||||||
|
|
@ -133,7 +147,9 @@ async fn main() -> Result<()> {
|
||||||
info!(" - Admin.Health / Stats / BuildInfo");
|
info!(" - Admin.Health / Stats / BuildInfo");
|
||||||
|
|
||||||
let shutdown = async {
|
let shutdown = async {
|
||||||
tokio::signal::ctrl_c().await.expect("failed to install Ctrl+C handler");
|
tokio::signal::ctrl_c()
|
||||||
|
.await
|
||||||
|
.expect("failed to install Ctrl+C handler");
|
||||||
};
|
};
|
||||||
tokio::pin!(shutdown);
|
tokio::pin!(shutdown);
|
||||||
|
|
||||||
|
|
@ -163,8 +179,9 @@ async fn maintenance_loop(
|
||||||
config: StorageConfig,
|
config: StorageConfig,
|
||||||
mut shutdown: tokio::sync::broadcast::Receiver<()>,
|
mut shutdown: tokio::sync::broadcast::Receiver<()>,
|
||||||
) {
|
) {
|
||||||
let snapshot_interval_secs =
|
let snapshot_interval_secs = config
|
||||||
config.compaction_interval_seconds.clamp(5, DEFAULT_SNAPSHOT_INTERVAL_SECS);
|
.compaction_interval_seconds
|
||||||
|
.clamp(5, DEFAULT_SNAPSHOT_INTERVAL_SECS);
|
||||||
let mut snapshot_interval = tokio::time::interval(Duration::from_secs(snapshot_interval_secs));
|
let mut snapshot_interval = tokio::time::interval(Duration::from_secs(snapshot_interval_secs));
|
||||||
snapshot_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
snapshot_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -49,7 +49,7 @@ let
|
||||||
raft_addr = raftAddr;
|
raft_addr = raftAddr;
|
||||||
})
|
})
|
||||||
cfg.initialPeers;
|
cfg.initialPeers;
|
||||||
chainfireConfigFile = tomlFormat.generate "chainfire.toml" {
|
generatedConfig = {
|
||||||
node = {
|
node = {
|
||||||
id = numericId cfg.nodeId;
|
id = numericId cfg.nodeId;
|
||||||
name = cfg.nodeId;
|
name = cfg.nodeId;
|
||||||
|
|
@ -73,6 +73,7 @@ let
|
||||||
role = cfg.raftRole;
|
role = cfg.raftRole;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
chainfireConfigFile = tomlFormat.generate "chainfire.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options.services.chainfire = {
|
options.services.chainfire = {
|
||||||
|
|
|
||||||
|
|
@ -75,6 +75,8 @@ let
|
||||||
fiberlbBaseConfig = {
|
fiberlbBaseConfig = {
|
||||||
grpc_addr = "0.0.0.0:${toString cfg.port}";
|
grpc_addr = "0.0.0.0:${toString cfg.port}";
|
||||||
log_level = "info";
|
log_level = "info";
|
||||||
|
metadata_backend = cfg.metadataBackend;
|
||||||
|
single_node = cfg.singleNode;
|
||||||
auth = {
|
auth = {
|
||||||
iam_server_addr =
|
iam_server_addr =
|
||||||
if cfg.iamAddr != null
|
if cfg.iamAddr != null
|
||||||
|
|
@ -93,6 +95,8 @@ let
|
||||||
vip_ownership = {
|
vip_ownership = {
|
||||||
enabled = cfg.vipOwnership.enable;
|
enabled = cfg.vipOwnership.enable;
|
||||||
interface = cfg.vipOwnership.interface;
|
interface = cfg.vipOwnership.interface;
|
||||||
|
} // lib.optionalAttrs (cfg.vipOwnership.ipCommand != null) {
|
||||||
|
ip_command = cfg.vipOwnership.ipCommand;
|
||||||
};
|
};
|
||||||
} // lib.optionalAttrs cfg.bgp.enable {
|
} // lib.optionalAttrs cfg.bgp.enable {
|
||||||
bgp =
|
bgp =
|
||||||
|
|
@ -127,6 +131,15 @@ let
|
||||||
// lib.optionalAttrs (cfg.bgp.nextHop != null) {
|
// lib.optionalAttrs (cfg.bgp.nextHop != null) {
|
||||||
next_hop = cfg.bgp.nextHop;
|
next_hop = cfg.bgp.nextHop;
|
||||||
};
|
};
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.flaredbAddr != null) {
|
||||||
|
flaredb_endpoint = cfg.flaredbAddr;
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (normalizedDatabaseUrl != null) {
|
||||||
|
metadata_database_url = normalizedDatabaseUrl;
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.chainfireAddr != null) {
|
||||||
|
chainfire_endpoint = "http://${cfg.chainfireAddr}";
|
||||||
};
|
};
|
||||||
fiberlbConfigFile = tomlFormat.generate "fiberlb.toml" (lib.recursiveUpdate fiberlbBaseConfig cfg.settings);
|
fiberlbConfigFile = tomlFormat.generate "fiberlb.toml" (lib.recursiveUpdate fiberlbBaseConfig cfg.settings);
|
||||||
flaredbDependencies = lib.optional (cfg.metadataBackend == "flaredb") "flaredb.service";
|
flaredbDependencies = lib.optional (cfg.metadataBackend == "flaredb") "flaredb.service";
|
||||||
|
|
@ -222,6 +235,13 @@ in
|
||||||
default = "lo";
|
default = "lo";
|
||||||
description = "Interface where FiberLB should claim VIP /32 addresses.";
|
description = "Interface where FiberLB should claim VIP /32 addresses.";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
ipCommand = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.str;
|
||||||
|
default = null;
|
||||||
|
description = "Optional explicit path to the `ip` command used for VIP ownership.";
|
||||||
|
example = "/run/current-system/sw/bin/ip";
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
vipDrain = {
|
vipDrain = {
|
||||||
|
|
@ -367,22 +387,7 @@ in
|
||||||
AmbientCapabilities = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
|
AmbientCapabilities = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
|
||||||
CapabilityBoundingSet = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
|
CapabilityBoundingSet = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
|
||||||
|
|
||||||
# Environment variables for service endpoints
|
ExecStart = "${cfg.package}/bin/fiberlb --config ${fiberlbConfigFile}";
|
||||||
Environment = [
|
|
||||||
"RUST_LOG=info"
|
|
||||||
"FIBERLB_FLAREDB_ENDPOINT=${if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479"}"
|
|
||||||
"FIBERLB_METADATA_BACKEND=${cfg.metadataBackend}"
|
|
||||||
] ++ lib.optional (normalizedDatabaseUrl != null) "FIBERLB_METADATA_DATABASE_URL=${normalizedDatabaseUrl}"
|
|
||||||
++ lib.optional cfg.singleNode "FIBERLB_SINGLE_NODE=true"
|
|
||||||
++ lib.optional (cfg.chainfireAddr != null) "FIBERLB_CHAINFIRE_ENDPOINT=http://${cfg.chainfireAddr}";
|
|
||||||
|
|
||||||
# Start command
|
|
||||||
ExecStart = lib.concatStringsSep " " ([
|
|
||||||
"${cfg.package}/bin/fiberlb"
|
|
||||||
"--config ${fiberlbConfigFile}"
|
|
||||||
"--grpc-addr 0.0.0.0:${toString cfg.port}"
|
|
||||||
"--flaredb-endpoint ${if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479"}"
|
|
||||||
]);
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,7 @@ let
|
||||||
// lib.optionalAttrs (cfg.databaseUrl != null) {
|
// lib.optionalAttrs (cfg.databaseUrl != null) {
|
||||||
metadata_database_url = cfg.databaseUrl;
|
metadata_database_url = cfg.databaseUrl;
|
||||||
};
|
};
|
||||||
configFile = tomlFormat.generate "flashdns.toml" generatedConfig;
|
configFile = tomlFormat.generate "flashdns.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options.services.flashdns = {
|
options.services.flashdns = {
|
||||||
|
|
|
||||||
|
|
@ -3,12 +3,19 @@
|
||||||
let
|
let
|
||||||
cfg = config.services.iam;
|
cfg = config.services.iam;
|
||||||
tomlFormat = pkgs.formats.toml { };
|
tomlFormat = pkgs.formats.toml { };
|
||||||
iamConfigFile = tomlFormat.generate "iam.toml" {
|
generatedConfig = {
|
||||||
server = {
|
server = {
|
||||||
addr = "0.0.0.0:${toString cfg.port}";
|
addr = "0.0.0.0:${toString cfg.port}";
|
||||||
http_addr = "0.0.0.0:${toString cfg.httpPort}";
|
http_addr = "0.0.0.0:${toString cfg.httpPort}";
|
||||||
};
|
};
|
||||||
logging.level = "info";
|
logging.level = "info";
|
||||||
|
admin = {
|
||||||
|
allow_unauthenticated = cfg.allowUnauthenticatedAdmin;
|
||||||
|
};
|
||||||
|
dev = {
|
||||||
|
allow_random_signing_key = cfg.allowRandomSigningKey;
|
||||||
|
allow_memory_backend = cfg.storeBackend == "memory";
|
||||||
|
};
|
||||||
store = {
|
store = {
|
||||||
backend = cfg.storeBackend;
|
backend = cfg.storeBackend;
|
||||||
flaredb_endpoint =
|
flaredb_endpoint =
|
||||||
|
|
@ -25,6 +32,7 @@ let
|
||||||
chainfire_endpoint = cfg.chainfireAddr;
|
chainfire_endpoint = cfg.chainfireAddr;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
iamConfigFile = tomlFormat.generate "iam.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options.services.iam = {
|
options.services.iam = {
|
||||||
|
|
@ -87,6 +95,18 @@ in
|
||||||
description = "Admin token injected as IAM_ADMIN_TOKEN for privileged IAM APIs.";
|
description = "Admin token injected as IAM_ADMIN_TOKEN for privileged IAM APIs.";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
allowRandomSigningKey = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = false;
|
||||||
|
description = "Allow IAM to generate a random signing key when authn.internal_token.signing_key is unset (dev only).";
|
||||||
|
};
|
||||||
|
|
||||||
|
allowUnauthenticatedAdmin = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = false;
|
||||||
|
description = "Allow privileged admin APIs without an admin token (dev only).";
|
||||||
|
};
|
||||||
|
|
||||||
settings = lib.mkOption {
|
settings = lib.mkOption {
|
||||||
type = lib.types.attrs;
|
type = lib.types.attrs;
|
||||||
default = {};
|
default = {};
|
||||||
|
|
@ -119,20 +139,6 @@ in
|
||||||
wants = [ "chainfire.service" "flaredb.service" ];
|
wants = [ "chainfire.service" "flaredb.service" ];
|
||||||
|
|
||||||
environment = lib.mkMerge [
|
environment = lib.mkMerge [
|
||||||
{
|
|
||||||
CHAINFIRE_ENDPOINT = if cfg.chainfireAddr != null then cfg.chainfireAddr else "127.0.0.1:2379";
|
|
||||||
FLAREDB_ENDPOINT = if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479";
|
|
||||||
IAM_STORE_BACKEND = cfg.storeBackend;
|
|
||||||
}
|
|
||||||
(lib.mkIf (cfg.databaseUrl != null) {
|
|
||||||
IAM_DATABASE_URL = cfg.databaseUrl;
|
|
||||||
})
|
|
||||||
(lib.mkIf cfg.singleNode {
|
|
||||||
IAM_SINGLE_NODE = "true";
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.storeBackend == "memory") {
|
|
||||||
IAM_ALLOW_MEMORY_BACKEND = "1";
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.adminToken != null) {
|
(lib.mkIf (cfg.adminToken != null) {
|
||||||
IAM_ADMIN_TOKEN = cfg.adminToken;
|
IAM_ADMIN_TOKEN = cfg.adminToken;
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,59 @@
|
||||||
|
|
||||||
let
|
let
|
||||||
cfg = config.services.k8shost;
|
cfg = config.services.k8shost;
|
||||||
|
tomlFormat = pkgs.formats.toml { };
|
||||||
|
generatedConfig = {
|
||||||
|
server = {
|
||||||
|
addr = "0.0.0.0:${toString cfg.port}";
|
||||||
|
http_addr = "127.0.0.1:${toString cfg.httpPort}";
|
||||||
|
log_level = "info";
|
||||||
|
};
|
||||||
|
flaredb = {
|
||||||
|
pd_addr = cfg.flaredbPdAddr;
|
||||||
|
direct_addr = cfg.flaredbDirectAddr;
|
||||||
|
};
|
||||||
|
chainfire = {
|
||||||
|
endpoint = cfg.chainfireAddr;
|
||||||
|
};
|
||||||
|
iam = {
|
||||||
|
server_addr =
|
||||||
|
if cfg.iamAddr != null
|
||||||
|
then cfg.iamAddr
|
||||||
|
else "http://127.0.0.1:50080";
|
||||||
|
};
|
||||||
|
creditservice =
|
||||||
|
lib.optionalAttrs
|
||||||
|
(
|
||||||
|
cfg.creditserviceAddr != null
|
||||||
|
|| ((config.services ? creditservice) && config.services.creditservice.enable)
|
||||||
|
)
|
||||||
|
{
|
||||||
|
server_addr =
|
||||||
|
if cfg.creditserviceAddr != null
|
||||||
|
then cfg.creditserviceAddr
|
||||||
|
else "http://127.0.0.1:${toString config.services.creditservice.grpcPort}";
|
||||||
|
};
|
||||||
|
fiberlb = {
|
||||||
|
server_addr =
|
||||||
|
if cfg.fiberlbAddr != null
|
||||||
|
then cfg.fiberlbAddr
|
||||||
|
else "http://127.0.0.1:50085";
|
||||||
|
};
|
||||||
|
flashdns = {
|
||||||
|
server_addr =
|
||||||
|
if cfg.flashdnsAddr != null
|
||||||
|
then cfg.flashdnsAddr
|
||||||
|
else "http://127.0.0.1:50084";
|
||||||
|
};
|
||||||
|
prismnet = {
|
||||||
|
server_addr =
|
||||||
|
if cfg.prismnetAddr != null
|
||||||
|
then cfg.prismnetAddr
|
||||||
|
else "http://127.0.0.1:50081";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
configFile = tomlFormat.generate "k8shost.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
|
||||||
|
configPath = "/etc/k8shost/k8shost.toml";
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options.services.k8shost = {
|
options.services.k8shost = {
|
||||||
|
|
@ -26,6 +79,13 @@ in
|
||||||
example = "http://10.0.0.1:50080";
|
example = "http://10.0.0.1:50080";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
creditserviceAddr = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.str;
|
||||||
|
default = null;
|
||||||
|
description = "CreditService endpoint address (http://host:port) for pod admission and scheduler quota enforcement.";
|
||||||
|
example = "http://10.0.0.1:3010";
|
||||||
|
};
|
||||||
|
|
||||||
chainfireAddr = lib.mkOption {
|
chainfireAddr = lib.mkOption {
|
||||||
type = lib.types.nullOr lib.types.str;
|
type = lib.types.nullOr lib.types.str;
|
||||||
default = null;
|
default = null;
|
||||||
|
|
@ -123,24 +183,10 @@ in
|
||||||
ProtectHome = true;
|
ProtectHome = true;
|
||||||
ReadWritePaths = [ cfg.dataDir ];
|
ReadWritePaths = [ cfg.dataDir ];
|
||||||
|
|
||||||
# Environment variables for service endpoints
|
ExecStart = "${cfg.package}/bin/k8shost-server --config ${configPath}";
|
||||||
Environment = [
|
|
||||||
"RUST_LOG=info"
|
|
||||||
];
|
|
||||||
|
|
||||||
# Start command
|
|
||||||
ExecStart = lib.concatStringsSep " " ([
|
|
||||||
"${cfg.package}/bin/k8shost-server"
|
|
||||||
"--addr 0.0.0.0:${toString cfg.port}"
|
|
||||||
"--http-addr 127.0.0.1:${toString cfg.httpPort}"
|
|
||||||
] ++ lib.optional (cfg.iamAddr != null) "--iam-server-addr ${cfg.iamAddr}"
|
|
||||||
++ lib.optional (cfg.chainfireAddr != null) "--chainfire-endpoint ${cfg.chainfireAddr}"
|
|
||||||
++ lib.optional (cfg.prismnetAddr != null) "--prismnet-server-addr ${cfg.prismnetAddr}"
|
|
||||||
++ lib.optional (cfg.flaredbPdAddr != null) "--flaredb-pd-addr ${cfg.flaredbPdAddr}"
|
|
||||||
++ lib.optional (cfg.flaredbDirectAddr != null) "--flaredb-direct-addr ${cfg.flaredbDirectAddr}"
|
|
||||||
++ lib.optional (cfg.fiberlbAddr != null) "--fiberlb-server-addr ${cfg.fiberlbAddr}"
|
|
||||||
++ lib.optional (cfg.flashdnsAddr != null) "--flashdns-server-addr ${cfg.flashdnsAddr}");
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
environment.etc."k8shost/k8shost.toml".source = configFile;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -73,6 +73,22 @@ let
|
||||||
// lib.optionalAttrs (cfg.distributedRegistryEndpoint != null) {
|
// lib.optionalAttrs (cfg.distributedRegistryEndpoint != null) {
|
||||||
registry_endpoint = cfg.distributedRegistryEndpoint;
|
registry_endpoint = cfg.distributedRegistryEndpoint;
|
||||||
};
|
};
|
||||||
|
s3 = {
|
||||||
|
auth = {
|
||||||
|
enabled = cfg.s3AuthEnabled;
|
||||||
|
aws_region = cfg.s3AwsRegion;
|
||||||
|
iam_cache_ttl_secs = cfg.s3IamCacheTtlSecs;
|
||||||
|
default_org_id = cfg.s3DefaultOrgId;
|
||||||
|
default_project_id = cfg.s3DefaultProjectId;
|
||||||
|
max_auth_body_bytes = cfg.s3MaxAuthBodyBytes;
|
||||||
|
};
|
||||||
|
performance = {
|
||||||
|
streaming_put_threshold_bytes = cfg.s3StreamingPutThresholdBytes;
|
||||||
|
inline_put_max_bytes = cfg.s3InlinePutMaxBytes;
|
||||||
|
multipart_put_concurrency = cfg.s3MultipartPutConcurrency;
|
||||||
|
multipart_fetch_concurrency = cfg.s3MultipartFetchConcurrency;
|
||||||
|
};
|
||||||
|
};
|
||||||
}
|
}
|
||||||
// lib.optionalAttrs (cfg.flaredbAddr != null) {
|
// lib.optionalAttrs (cfg.flaredbAddr != null) {
|
||||||
flaredb_endpoint = cfg.flaredbAddr;
|
flaredb_endpoint = cfg.flaredbAddr;
|
||||||
|
|
@ -321,6 +337,54 @@ in
|
||||||
description = "Maximum concurrent multipart GET part fetches.";
|
description = "Maximum concurrent multipart GET part fetches.";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
s3AuthEnabled = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = true;
|
||||||
|
description = "Enable S3 Signature V4 authentication.";
|
||||||
|
};
|
||||||
|
|
||||||
|
s3AwsRegion = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "us-east-1";
|
||||||
|
description = "AWS region name exposed by the S3 compatibility layer.";
|
||||||
|
};
|
||||||
|
|
||||||
|
s3IamCacheTtlSecs = lib.mkOption {
|
||||||
|
type = lib.types.int;
|
||||||
|
default = 30;
|
||||||
|
description = "Seconds to cache IAM-backed S3 credential lookups.";
|
||||||
|
};
|
||||||
|
|
||||||
|
s3DefaultOrgId = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.str;
|
||||||
|
default = "default";
|
||||||
|
description = "Default org ID used for static S3 credentials.";
|
||||||
|
};
|
||||||
|
|
||||||
|
s3DefaultProjectId = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.str;
|
||||||
|
default = "default";
|
||||||
|
description = "Default project ID used for static S3 credentials.";
|
||||||
|
};
|
||||||
|
|
||||||
|
s3MaxAuthBodyBytes = lib.mkOption {
|
||||||
|
type = lib.types.int;
|
||||||
|
default = 1024 * 1024 * 1024;
|
||||||
|
description = "Maximum request body size buffered during S3 auth verification.";
|
||||||
|
};
|
||||||
|
|
||||||
|
s3AccessKeyId = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.str;
|
||||||
|
default = null;
|
||||||
|
description = "Optional static S3 access key ID injected via environment for dev/test.";
|
||||||
|
};
|
||||||
|
|
||||||
|
s3SecretKey = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.str;
|
||||||
|
default = null;
|
||||||
|
description = "Optional static S3 secret key injected via environment for dev/test.";
|
||||||
|
};
|
||||||
|
|
||||||
databaseUrl = lib.mkOption {
|
databaseUrl = lib.mkOption {
|
||||||
type = lib.types.nullOr lib.types.str;
|
type = lib.types.nullOr lib.types.str;
|
||||||
default = null;
|
default = null;
|
||||||
|
|
@ -360,6 +424,13 @@ in
|
||||||
};
|
};
|
||||||
|
|
||||||
config = lib.mkIf cfg.enable {
|
config = lib.mkIf cfg.enable {
|
||||||
|
assertions = [
|
||||||
|
{
|
||||||
|
assertion = (cfg.s3AccessKeyId == null) == (cfg.s3SecretKey == null);
|
||||||
|
message = "services.lightningstor.s3AccessKeyId and services.lightningstor.s3SecretKey must be set together";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
users.users.lightningstor = {
|
users.users.lightningstor = {
|
||||||
isSystemUser = true;
|
isSystemUser = true;
|
||||||
group = "lightningstor";
|
group = "lightningstor";
|
||||||
|
|
@ -391,17 +462,12 @@ in
|
||||||
ExecStart = execStart;
|
ExecStart = execStart;
|
||||||
};
|
};
|
||||||
|
|
||||||
environment = {
|
environment = lib.mkMerge [
|
||||||
RUST_LOG = "info";
|
(lib.mkIf (cfg.s3AccessKeyId != null) {
|
||||||
LIGHTNINGSTOR_S3_STREAMING_PUT_THRESHOLD_BYTES =
|
S3_ACCESS_KEY_ID = cfg.s3AccessKeyId;
|
||||||
toString cfg.s3StreamingPutThresholdBytes;
|
S3_SECRET_KEY = cfg.s3SecretKey;
|
||||||
LIGHTNINGSTOR_S3_INLINE_PUT_MAX_BYTES =
|
})
|
||||||
toString cfg.s3InlinePutMaxBytes;
|
];
|
||||||
LIGHTNINGSTOR_S3_MULTIPART_PUT_CONCURRENCY =
|
|
||||||
toString cfg.s3MultipartPutConcurrency;
|
|
||||||
LIGHTNINGSTOR_S3_MULTIPART_FETCH_CONCURRENCY =
|
|
||||||
toString cfg.s3MultipartFetchConcurrency;
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,19 @@
|
||||||
|
|
||||||
let
|
let
|
||||||
cfg = config.services.nightlight;
|
cfg = config.services.nightlight;
|
||||||
|
yamlFormat = pkgs.formats.yaml { };
|
||||||
|
generatedConfig = {
|
||||||
|
server = {
|
||||||
|
grpc_addr = "0.0.0.0:${toString cfg.grpcPort}";
|
||||||
|
http_addr = "0.0.0.0:${toString cfg.httpPort}";
|
||||||
|
};
|
||||||
|
storage = {
|
||||||
|
data_dir = toString cfg.dataDir;
|
||||||
|
retention_days = cfg.retentionDays;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
configFile = yamlFormat.generate "nightlight.yaml" (lib.recursiveUpdate generatedConfig cfg.settings);
|
||||||
|
configPath = "/etc/nightlight/nightlight.yaml";
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options.services.nightlight = {
|
options.services.nightlight = {
|
||||||
|
|
@ -79,19 +92,10 @@ in
|
||||||
ProtectHome = true;
|
ProtectHome = true;
|
||||||
ReadWritePaths = [ cfg.dataDir ];
|
ReadWritePaths = [ cfg.dataDir ];
|
||||||
|
|
||||||
# Start command
|
ExecStart = "${cfg.package}/bin/nightlight-server --config ${configPath}";
|
||||||
# Note: nightlight-server uses a config file, so we'll pass data-dir directly
|
|
||||||
# The config will be auto-generated or use defaults from Config::default()
|
|
||||||
ExecStart = "${cfg.package}/bin/nightlight-server";
|
|
||||||
|
|
||||||
# Environment variables for configuration
|
|
||||||
Environment = [
|
|
||||||
"NIGHTLIGHT_HTTP_ADDR=0.0.0.0:${toString cfg.httpPort}"
|
|
||||||
"NIGHTLIGHT_GRPC_ADDR=0.0.0.0:${toString cfg.grpcPort}"
|
|
||||||
"NIGHTLIGHT_DATA_DIR=${cfg.dataDir}"
|
|
||||||
"NIGHTLIGHT_RETENTION_DAYS=${toString cfg.retentionDays}"
|
|
||||||
];
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
environment.etc."nightlight/nightlight.yaml".source = configFile;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ let
|
||||||
else if cfg.coronafsEndpoint != null then cfg.coronafsEndpoint
|
else if cfg.coronafsEndpoint != null then cfg.coronafsEndpoint
|
||||||
else null;
|
else null;
|
||||||
tomlFormat = pkgs.formats.toml { };
|
tomlFormat = pkgs.formats.toml { };
|
||||||
plasmavmcConfigFile = tomlFormat.generate "plasmavmc.toml" {
|
generatedConfig = {
|
||||||
addr = "0.0.0.0:${toString cfg.port}";
|
addr = "0.0.0.0:${toString cfg.port}";
|
||||||
http_addr = "0.0.0.0:${toString cfg.httpPort}";
|
http_addr = "0.0.0.0:${toString cfg.httpPort}";
|
||||||
log_level = "info";
|
log_level = "info";
|
||||||
|
|
@ -37,7 +37,78 @@ let
|
||||||
then cfg.iamAddr
|
then cfg.iamAddr
|
||||||
else "127.0.0.1:50080";
|
else "127.0.0.1:50080";
|
||||||
};
|
};
|
||||||
|
kvm = {
|
||||||
|
qemu_path = "${pkgs.qemu}/bin/qemu-system-x86_64";
|
||||||
|
runtime_dir = "/run/libvirt/plasmavmc";
|
||||||
|
};
|
||||||
|
storage = {
|
||||||
|
backend = "flaredb";
|
||||||
|
flaredb_endpoint =
|
||||||
|
if cfg.flaredbAddr != null
|
||||||
|
then cfg.flaredbAddr
|
||||||
|
else "127.0.0.1:2479";
|
||||||
|
} // lib.optionalAttrs (cfg.chainfireAddr != null) {
|
||||||
|
chainfire_endpoint = "http://${cfg.chainfireAddr}";
|
||||||
|
};
|
||||||
|
agent =
|
||||||
|
{
|
||||||
|
shared_live_migration = cfg.sharedLiveMigration;
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.mode != "server") {
|
||||||
|
node_id = cfg.nodeId;
|
||||||
|
node_name = cfg.nodeName;
|
||||||
|
heartbeat_interval_secs = cfg.heartbeatIntervalSeconds;
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.controlPlaneAddr != null) {
|
||||||
|
control_plane_addr = cfg.controlPlaneAddr;
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.advertiseAddr != null) {
|
||||||
|
advertise_endpoint = cfg.advertiseAddr;
|
||||||
|
};
|
||||||
|
integrations =
|
||||||
|
lib.optionalAttrs (cfg.prismnetAddr != null) {
|
||||||
|
prismnet_endpoint = "http://${cfg.prismnetAddr}";
|
||||||
|
};
|
||||||
|
watcher =
|
||||||
|
lib.optionalAttrs (cfg.chainfireAddr != null) {
|
||||||
|
enabled = true;
|
||||||
|
};
|
||||||
|
health =
|
||||||
|
lib.optionalAttrs (cfg.mode == "server") {
|
||||||
|
node_monitor_interval_secs = 5;
|
||||||
|
node_heartbeat_timeout_secs = 30;
|
||||||
|
};
|
||||||
|
artifacts =
|
||||||
|
{
|
||||||
|
image_cache_dir = "${toString cfg.dataDir}/images";
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.lightningstorAddr != null) {
|
||||||
|
lightningstor_endpoint = cfg.lightningstorAddr;
|
||||||
|
};
|
||||||
|
volumes = {
|
||||||
|
managed_volume_root = toString cfg.managedVolumeRoot;
|
||||||
|
coronafs =
|
||||||
|
(lib.optionalAttrs (effectiveCoronafsControllerEndpoint != null) {
|
||||||
|
controller_endpoint = effectiveCoronafsControllerEndpoint;
|
||||||
|
})
|
||||||
|
// (lib.optionalAttrs (effectiveCoronafsNodeEndpoint != null) {
|
||||||
|
node_endpoint = effectiveCoronafsNodeEndpoint;
|
||||||
|
})
|
||||||
|
// (lib.optionalAttrs (cfg.coronafsEndpoint != null) {
|
||||||
|
endpoint = cfg.coronafsEndpoint;
|
||||||
|
})
|
||||||
|
// {
|
||||||
|
node_local_attach = cfg.coronafsNodeLocalAttach || cfg.experimentalCoronafsNodeLocalAttach;
|
||||||
|
};
|
||||||
|
ceph =
|
||||||
|
(lib.optionalAttrs (cfg.cephMonitors != [ ]) {
|
||||||
|
monitors = cfg.cephMonitors;
|
||||||
|
cluster_id = cfg.cephClusterId;
|
||||||
|
user = cfg.cephUser;
|
||||||
|
});
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
plasmavmcConfigFile = tomlFormat.generate "plasmavmc.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options.services.plasmavmc = {
|
options.services.plasmavmc = {
|
||||||
|
|
@ -286,64 +357,9 @@ in
|
||||||
exit 1
|
exit 1
|
||||||
'';
|
'';
|
||||||
|
|
||||||
environment = lib.mkMerge [
|
environment = lib.optionalAttrs (cfg.cephSecret != null) {
|
||||||
{
|
PLASMAVMC_CEPH_SECRET = cfg.cephSecret;
|
||||||
PLASMAVMC_MODE = cfg.mode;
|
};
|
||||||
PLASMAVMC_STORAGE_BACKEND = "flaredb";
|
|
||||||
PLASMAVMC_FLAREDB_ENDPOINT = if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479";
|
|
||||||
PLASMAVMC_QEMU_PATH = "${pkgs.qemu}/bin/qemu-system-x86_64";
|
|
||||||
PLASMAVMC_RUNTIME_DIR = "/run/libvirt/plasmavmc";
|
|
||||||
PLASMAVMC_IMAGE_CACHE_DIR = "${toString cfg.dataDir}/images";
|
|
||||||
PLASMAVMC_MANAGED_VOLUME_ROOT = toString cfg.managedVolumeRoot;
|
|
||||||
PLASMAVMC_SHARED_LIVE_MIGRATION = lib.boolToString cfg.sharedLiveMigration;
|
|
||||||
}
|
|
||||||
(lib.mkIf (cfg.prismnetAddr != null) {
|
|
||||||
PRISMNET_ENDPOINT = "http://${cfg.prismnetAddr}";
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.chainfireAddr != null) {
|
|
||||||
PLASMAVMC_CHAINFIRE_ENDPOINT = "http://${cfg.chainfireAddr}";
|
|
||||||
PLASMAVMC_STATE_WATCHER = "1";
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.lightningstorAddr != null) {
|
|
||||||
PLASMAVMC_LIGHTNINGSTOR_ENDPOINT = cfg.lightningstorAddr;
|
|
||||||
})
|
|
||||||
(lib.mkIf (effectiveCoronafsControllerEndpoint != null) {
|
|
||||||
PLASMAVMC_CORONAFS_CONTROLLER_ENDPOINT = effectiveCoronafsControllerEndpoint;
|
|
||||||
})
|
|
||||||
(lib.mkIf (effectiveCoronafsNodeEndpoint != null) {
|
|
||||||
PLASMAVMC_CORONAFS_NODE_ENDPOINT = effectiveCoronafsNodeEndpoint;
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.coronafsNodeLocalAttach || cfg.experimentalCoronafsNodeLocalAttach) {
|
|
||||||
PLASMAVMC_CORONAFS_NODE_LOCAL_ATTACH = "1";
|
|
||||||
PLASMAVMC_CORONAFS_ENABLE_EXPERIMENTAL_NODE_LOCAL_ATTACH = "1";
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.coronafsEndpoint != null) {
|
|
||||||
PLASMAVMC_CORONAFS_ENDPOINT = cfg.coronafsEndpoint;
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.cephMonitors != [ ]) {
|
|
||||||
PLASMAVMC_CEPH_MONITORS = lib.concatStringsSep "," cfg.cephMonitors;
|
|
||||||
PLASMAVMC_CEPH_CLUSTER_ID = cfg.cephClusterId;
|
|
||||||
PLASMAVMC_CEPH_USER = cfg.cephUser;
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.cephSecret != null) {
|
|
||||||
PLASMAVMC_CEPH_SECRET = cfg.cephSecret;
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.mode != "server") {
|
|
||||||
PLASMAVMC_NODE_ID = cfg.nodeId;
|
|
||||||
PLASMAVMC_NODE_NAME = cfg.nodeName;
|
|
||||||
PLASMAVMC_NODE_HEARTBEAT_INTERVAL_SECS = toString cfg.heartbeatIntervalSeconds;
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.controlPlaneAddr != null) {
|
|
||||||
PLASMAVMC_CONTROL_PLANE_ADDR = cfg.controlPlaneAddr;
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.advertiseAddr != null) {
|
|
||||||
PLASMAVMC_ENDPOINT_ADVERTISE = cfg.advertiseAddr;
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.mode == "server") {
|
|
||||||
PLASMAVMC_NODE_HEALTH_MONITOR_INTERVAL_SECS = "5";
|
|
||||||
PLASMAVMC_NODE_HEARTBEAT_TIMEOUT_SECS = "30";
|
|
||||||
})
|
|
||||||
];
|
|
||||||
|
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
Type = "simple";
|
Type = "simple";
|
||||||
|
|
|
||||||
|
|
@ -3,17 +3,29 @@
|
||||||
let
|
let
|
||||||
cfg = config.services.prismnet;
|
cfg = config.services.prismnet;
|
||||||
tomlFormat = pkgs.formats.toml { };
|
tomlFormat = pkgs.formats.toml { };
|
||||||
prismnetConfigFile = tomlFormat.generate "prismnet.toml" {
|
generatedConfig = {
|
||||||
grpc_addr = "0.0.0.0:${toString cfg.port}";
|
grpc_addr = "0.0.0.0:${toString cfg.port}";
|
||||||
http_addr = "0.0.0.0:${toString cfg.httpPort}";
|
http_addr = "0.0.0.0:${toString cfg.httpPort}";
|
||||||
log_level = "info";
|
log_level = "info";
|
||||||
|
metadata_backend = cfg.metadataBackend;
|
||||||
|
single_node = cfg.singleNode;
|
||||||
auth = {
|
auth = {
|
||||||
iam_server_addr =
|
iam_server_addr =
|
||||||
if cfg.iamAddr != null
|
if cfg.iamAddr != null
|
||||||
then cfg.iamAddr
|
then cfg.iamAddr
|
||||||
else "127.0.0.1:50080";
|
else "127.0.0.1:50080";
|
||||||
};
|
};
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.chainfireAddr != null) {
|
||||||
|
chainfire_endpoint = "http://${cfg.chainfireAddr}";
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.flaredbAddr != null) {
|
||||||
|
flaredb_endpoint = cfg.flaredbAddr;
|
||||||
|
}
|
||||||
|
// lib.optionalAttrs (cfg.databaseUrl != null) {
|
||||||
|
metadata_database_url = cfg.databaseUrl;
|
||||||
};
|
};
|
||||||
|
prismnetConfigFile = tomlFormat.generate "prismnet.toml" (lib.recursiveUpdate generatedConfig cfg.settings);
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options.services.prismnet = {
|
options.services.prismnet = {
|
||||||
|
|
@ -108,22 +120,6 @@ in
|
||||||
after = [ "network.target" "iam.service" "flaredb.service" "chainfire.service" ];
|
after = [ "network.target" "iam.service" "flaredb.service" "chainfire.service" ];
|
||||||
wants = [ "iam.service" "flaredb.service" "chainfire.service" ];
|
wants = [ "iam.service" "flaredb.service" "chainfire.service" ];
|
||||||
|
|
||||||
environment = lib.mkMerge [
|
|
||||||
{
|
|
||||||
PRISMNET_FLAREDB_ENDPOINT = if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479";
|
|
||||||
PRISMNET_METADATA_BACKEND = cfg.metadataBackend;
|
|
||||||
}
|
|
||||||
(lib.mkIf (cfg.databaseUrl != null) {
|
|
||||||
PRISMNET_METADATA_DATABASE_URL = cfg.databaseUrl;
|
|
||||||
})
|
|
||||||
(lib.mkIf cfg.singleNode {
|
|
||||||
PRISMNET_SINGLE_NODE = "1";
|
|
||||||
})
|
|
||||||
(lib.mkIf (cfg.chainfireAddr != null) {
|
|
||||||
PRISMNET_CHAINFIRE_ENDPOINT = "http://${cfg.chainfireAddr}";
|
|
||||||
})
|
|
||||||
];
|
|
||||||
|
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
Type = "simple";
|
Type = "simple";
|
||||||
User = "prismnet";
|
User = "prismnet";
|
||||||
|
|
@ -143,12 +139,7 @@ in
|
||||||
ReadWritePaths = [ cfg.dataDir ];
|
ReadWritePaths = [ cfg.dataDir ];
|
||||||
|
|
||||||
# Start command
|
# Start command
|
||||||
ExecStart = lib.concatStringsSep " " [
|
ExecStart = "${cfg.package}/bin/prismnet-server --config ${prismnetConfigFile}";
|
||||||
"${cfg.package}/bin/prismnet-server"
|
|
||||||
"--config ${prismnetConfigFile}"
|
|
||||||
"--grpc-addr 0.0.0.0:${toString cfg.port}"
|
|
||||||
"--flaredb-endpoint ${if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479"}"
|
|
||||||
];
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -15,17 +15,13 @@
|
||||||
services.flaredb = {
|
services.flaredb = {
|
||||||
enable = true;
|
enable = true;
|
||||||
dataDir = "/var/lib/flaredb";
|
dataDir = "/var/lib/flaredb";
|
||||||
settings = {
|
pdAddr = "127.0.0.1:${toString config.services.chainfire.port}";
|
||||||
chainfire_endpoint = "http://127.0.0.1:${toString config.services.chainfire.port}";
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
services.iam = {
|
services.iam = {
|
||||||
enable = true;
|
enable = true;
|
||||||
dataDir = "/var/lib/iam";
|
dataDir = "/var/lib/iam";
|
||||||
settings = {
|
flaredbAddr = "127.0.0.1:${toString config.services.flaredb.port}";
|
||||||
flaredb_endpoint = "http://127.0.0.1:${toString config.services.flaredb.port}";
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
services.plasmavmc.enable = true;
|
services.plasmavmc.enable = true;
|
||||||
|
|
|
||||||
|
|
@ -71,6 +71,8 @@
|
||||||
port = 50080;
|
port = 50080;
|
||||||
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
||||||
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
||||||
|
allowRandomSigningKey = true;
|
||||||
|
allowUnauthenticatedAdmin = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
services.prismnet = {
|
services.prismnet = {
|
||||||
|
|
@ -162,13 +164,6 @@
|
||||||
flashdnsAddr = "http://10.100.0.11:50084";
|
flashdnsAddr = "http://10.100.0.11:50084";
|
||||||
};
|
};
|
||||||
|
|
||||||
systemd.services.iam.environment = {
|
services.lightningstor.s3AccessKeyId = "photoncloud-test";
|
||||||
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
services.lightningstor.s3SecretKey = "photoncloud-test-secret";
|
||||||
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
|
|
||||||
};
|
|
||||||
|
|
||||||
systemd.services.lightningstor.environment = {
|
|
||||||
S3_ACCESS_KEY_ID = "photoncloud-test";
|
|
||||||
S3_SECRET_KEY = "photoncloud-test-secret";
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -64,10 +64,7 @@
|
||||||
port = 50080;
|
port = 50080;
|
||||||
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
||||||
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
||||||
};
|
allowRandomSigningKey = true;
|
||||||
|
allowUnauthenticatedAdmin = true;
|
||||||
systemd.services.iam.environment = {
|
|
||||||
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
|
||||||
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -64,10 +64,7 @@
|
||||||
port = 50080;
|
port = 50080;
|
||||||
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
||||||
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
||||||
};
|
allowRandomSigningKey = true;
|
||||||
|
allowUnauthenticatedAdmin = true;
|
||||||
systemd.services.iam.environment = {
|
|
||||||
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
|
||||||
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -69,6 +69,8 @@
|
||||||
port = 50080;
|
port = 50080;
|
||||||
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
||||||
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
||||||
|
allowRandomSigningKey = true;
|
||||||
|
allowUnauthenticatedAdmin = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
services.plasmavmc = {
|
services.plasmavmc = {
|
||||||
|
|
@ -124,13 +126,6 @@
|
||||||
region = "test";
|
region = "test";
|
||||||
};
|
};
|
||||||
|
|
||||||
systemd.services.iam.environment = {
|
services.lightningstor.s3AccessKeyId = "photoncloud-test";
|
||||||
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
services.lightningstor.s3SecretKey = "photoncloud-test-secret";
|
||||||
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
|
|
||||||
};
|
|
||||||
|
|
||||||
systemd.services.lightningstor.environment = {
|
|
||||||
S3_ACCESS_KEY_ID = "photoncloud-test";
|
|
||||||
S3_SECRET_KEY = "photoncloud-test-secret";
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -66,10 +66,7 @@
|
||||||
port = 50080;
|
port = 50080;
|
||||||
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
||||||
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
||||||
};
|
allowRandomSigningKey = true;
|
||||||
|
allowUnauthenticatedAdmin = true;
|
||||||
systemd.services.iam.environment = {
|
|
||||||
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
|
||||||
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -66,10 +66,7 @@
|
||||||
port = 50080;
|
port = 50080;
|
||||||
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
chainfireAddr = config.photonTestCluster.chainfireControlPlaneAddrs;
|
||||||
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
flaredbAddr = config.photonTestCluster.flaredbControlPlaneAddrs;
|
||||||
};
|
allowRandomSigningKey = true;
|
||||||
|
allowUnauthenticatedAdmin = true;
|
||||||
systemd.services.iam.environment = {
|
|
||||||
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
|
||||||
IAM_ALLOW_UNAUTHENTICATED_ADMIN = "true";
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -173,10 +173,7 @@ in
|
||||||
port = 50080;
|
port = 50080;
|
||||||
httpPort = 8083;
|
httpPort = 8083;
|
||||||
storeBackend = "memory";
|
storeBackend = "memory";
|
||||||
};
|
allowRandomSigningKey = true;
|
||||||
|
|
||||||
systemd.services.iam.environment = {
|
|
||||||
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
|
||||||
};
|
};
|
||||||
|
|
||||||
services.fiberlb = {
|
services.fiberlb = {
|
||||||
|
|
@ -260,10 +257,7 @@ in
|
||||||
port = 50080;
|
port = 50080;
|
||||||
httpPort = 8083;
|
httpPort = 8083;
|
||||||
storeBackend = "memory";
|
storeBackend = "memory";
|
||||||
};
|
allowRandomSigningKey = true;
|
||||||
|
|
||||||
systemd.services.iam.environment = {
|
|
||||||
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
|
||||||
};
|
};
|
||||||
|
|
||||||
services.fiberlb = {
|
services.fiberlb = {
|
||||||
|
|
|
||||||
|
|
@ -260,10 +260,7 @@ in
|
||||||
port = 50080;
|
port = 50080;
|
||||||
httpPort = 8083;
|
httpPort = 8083;
|
||||||
storeBackend = "memory";
|
storeBackend = "memory";
|
||||||
};
|
allowRandomSigningKey = true;
|
||||||
|
|
||||||
systemd.services.iam.environment = {
|
|
||||||
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
|
||||||
};
|
};
|
||||||
|
|
||||||
services.fiberlb = {
|
services.fiberlb = {
|
||||||
|
|
|
||||||
|
|
@ -165,10 +165,7 @@ in
|
||||||
port = 50080;
|
port = 50080;
|
||||||
httpPort = 8083;
|
httpPort = 8083;
|
||||||
storeBackend = "memory";
|
storeBackend = "memory";
|
||||||
};
|
allowRandomSigningKey = true;
|
||||||
|
|
||||||
systemd.services.iam.environment = {
|
|
||||||
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
|
||||||
};
|
};
|
||||||
|
|
||||||
services.fiberlb = {
|
services.fiberlb = {
|
||||||
|
|
|
||||||
|
|
@ -120,10 +120,7 @@ in
|
||||||
port = 50080;
|
port = 50080;
|
||||||
httpPort = 8083;
|
httpPort = 8083;
|
||||||
storeBackend = "memory";
|
storeBackend = "memory";
|
||||||
};
|
allowRandomSigningKey = true;
|
||||||
|
|
||||||
systemd.services.iam.environment = {
|
|
||||||
IAM_ALLOW_RANDOM_SIGNING_KEY = "1";
|
|
||||||
};
|
};
|
||||||
|
|
||||||
services.fiberlb = {
|
services.fiberlb = {
|
||||||
|
|
|
||||||
14
plasmavmc/Cargo.lock
generated
14
plasmavmc/Cargo.lock
generated
|
|
@ -2458,6 +2458,18 @@ version = "0.10.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
|
checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nix"
|
||||||
|
version = "0.29.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.11.0",
|
||||||
|
"cfg-if",
|
||||||
|
"cfg_aliases",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "nom"
|
name = "nom"
|
||||||
version = "7.1.3"
|
version = "7.1.3"
|
||||||
|
|
@ -2807,6 +2819,7 @@ name = "plasmavmc-kvm"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
"nix",
|
||||||
"plasmavmc-hypervisor",
|
"plasmavmc-hypervisor",
|
||||||
"plasmavmc-types",
|
"plasmavmc-types",
|
||||||
"serde",
|
"serde",
|
||||||
|
|
@ -2853,6 +2866,7 @@ dependencies = [
|
||||||
"reqwest 0.12.28",
|
"reqwest 0.12.28",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"sha2",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror 1.0.69",
|
"thiserror 1.0.69",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@ tracing = { workspace = true }
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
serde = { workspace = true }
|
serde = { workspace = true }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
|
nix = { version = "0.29", features = ["signal"] }
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
workspace = true
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -115,14 +115,20 @@ mod tests {
|
||||||
fn resolve_runtime_dir_defaults() {
|
fn resolve_runtime_dir_defaults() {
|
||||||
let _guard = env_test_lock().lock().unwrap();
|
let _guard = env_test_lock().lock().unwrap();
|
||||||
std::env::remove_var(ENV_RUNTIME_DIR);
|
std::env::remove_var(ENV_RUNTIME_DIR);
|
||||||
assert_eq!(resolve_runtime_dir(), PathBuf::from("/run/libvirt/plasmavmc"));
|
assert_eq!(
|
||||||
|
resolve_runtime_dir(),
|
||||||
|
PathBuf::from("/run/libvirt/plasmavmc")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn resolve_runtime_dir_from_env() {
|
fn resolve_runtime_dir_from_env() {
|
||||||
let _guard = env_test_lock().lock().unwrap();
|
let _guard = env_test_lock().lock().unwrap();
|
||||||
std::env::set_var(ENV_RUNTIME_DIR, "/tmp/plasmavmc-runtime");
|
std::env::set_var(ENV_RUNTIME_DIR, "/tmp/plasmavmc-runtime");
|
||||||
assert_eq!(resolve_runtime_dir(), PathBuf::from("/tmp/plasmavmc-runtime"));
|
assert_eq!(
|
||||||
|
resolve_runtime_dir(),
|
||||||
|
PathBuf::from("/tmp/plasmavmc-runtime")
|
||||||
|
);
|
||||||
std::env::remove_var(ENV_RUNTIME_DIR);
|
std::env::remove_var(ENV_RUNTIME_DIR);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -8,18 +8,19 @@ mod qmp;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use env::{
|
use env::{
|
||||||
resolve_kernel_initrd, resolve_nbd_aio_mode, resolve_nbd_max_queues, resolve_qcow2_path, resolve_qemu_path,
|
resolve_kernel_initrd, resolve_nbd_aio_mode, resolve_nbd_max_queues, resolve_qcow2_path,
|
||||||
resolve_qmp_timeout_secs, resolve_runtime_dir, ENV_QCOW2_PATH,
|
resolve_qemu_path, resolve_qmp_timeout_secs, resolve_runtime_dir, ENV_QCOW2_PATH,
|
||||||
};
|
};
|
||||||
|
use nix::sys::signal::{kill as nix_kill, Signal};
|
||||||
|
use nix::unistd::Pid;
|
||||||
use plasmavmc_hypervisor::{BackendCapabilities, HypervisorBackend, UnsupportedReason};
|
use plasmavmc_hypervisor::{BackendCapabilities, HypervisorBackend, UnsupportedReason};
|
||||||
use plasmavmc_types::{
|
use plasmavmc_types::{
|
||||||
AttachedDisk, DiskAttachment, DiskBus, DiskCache, Error, HypervisorType, NetworkSpec,
|
AttachedDisk, DiskAttachment, DiskBus, DiskCache, Error, HypervisorType, NetworkSpec, NicModel,
|
||||||
NicModel, Result, VirtualMachine, VmHandle, VmSpec, VmState, VmStatus, VolumeFormat,
|
Result, VirtualMachine, VmHandle, VmSpec, VmState, VmStatus, VolumeFormat,
|
||||||
};
|
};
|
||||||
use qmp::QmpClient;
|
use qmp::QmpClient;
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::Stdio;
|
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
use tokio::{net::UnixStream, time::Instant};
|
use tokio::{net::UnixStream, time::Instant};
|
||||||
|
|
@ -62,31 +63,6 @@ fn volume_format_name(format: VolumeFormat) -> &'static str {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_rbd_uri(pool: &str, image: &str, monitors: &[String], user: &str) -> String {
|
|
||||||
let mut uri = format!("rbd:{pool}/{image}");
|
|
||||||
if !user.is_empty() {
|
|
||||||
uri.push_str(&format!(":id={user}"));
|
|
||||||
}
|
|
||||||
if !monitors.is_empty() {
|
|
||||||
uri.push_str(&format!(":mon_host={}", monitors.join(";")));
|
|
||||||
}
|
|
||||||
uri
|
|
||||||
}
|
|
||||||
|
|
||||||
fn disk_source_arg(disk: &AttachedDisk) -> Result<(String, &'static str)> {
|
|
||||||
match &disk.attachment {
|
|
||||||
DiskAttachment::File { path, format } => Ok((path.clone(), volume_format_name(*format))),
|
|
||||||
DiskAttachment::Nbd { uri, format } => Ok((uri.clone(), volume_format_name(*format))),
|
|
||||||
DiskAttachment::CephRbd {
|
|
||||||
pool,
|
|
||||||
image,
|
|
||||||
monitors,
|
|
||||||
user,
|
|
||||||
..
|
|
||||||
} => Ok((build_rbd_uri(pool, image, monitors, user), "raw")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn effective_disk_cache(disk: &AttachedDisk) -> DiskCache {
|
fn effective_disk_cache(disk: &AttachedDisk) -> DiskCache {
|
||||||
match (&disk.attachment, disk.cache) {
|
match (&disk.attachment, disk.cache) {
|
||||||
// Shared NBD-backed volumes perform better and behave more predictably
|
// Shared NBD-backed volumes perform better and behave more predictably
|
||||||
|
|
@ -96,14 +72,6 @@ fn effective_disk_cache(disk: &AttachedDisk) -> DiskCache {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn disk_cache_mode(cache: DiskCache) -> &'static str {
|
|
||||||
match cache {
|
|
||||||
DiskCache::None => "none",
|
|
||||||
DiskCache::Writeback => "writeback",
|
|
||||||
DiskCache::Writethrough => "writethrough",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn disk_aio_mode(disk: &AttachedDisk) -> Option<&'static str> {
|
fn disk_aio_mode(disk: &AttachedDisk) -> Option<&'static str> {
|
||||||
match (&disk.attachment, disk.cache) {
|
match (&disk.attachment, disk.cache) {
|
||||||
(DiskAttachment::File { .. }, DiskCache::None) => Some("native"),
|
(DiskAttachment::File { .. }, DiskCache::None) => Some("native"),
|
||||||
|
|
@ -125,7 +93,8 @@ fn disk_queue_count(vm: &VirtualMachine, disk: &AttachedDisk) -> u16 {
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
vm.spec.cpu
|
vm.spec
|
||||||
|
.cpu
|
||||||
.vcpus
|
.vcpus
|
||||||
.clamp(1, resolve_nbd_max_queues().max(1) as u32) as u16
|
.clamp(1, resolve_nbd_max_queues().max(1) as u32) as u16
|
||||||
}
|
}
|
||||||
|
|
@ -153,6 +122,169 @@ fn qmp_timeout() -> Duration {
|
||||||
Duration::from_secs(resolve_qmp_timeout_secs())
|
Duration::from_secs(resolve_qmp_timeout_secs())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn disk_cache_json(cache: DiskCache) -> Value {
|
||||||
|
json!({
|
||||||
|
"direct": matches!(cache, DiskCache::None),
|
||||||
|
"no-flush": false
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_ceph_component(field_name: &str, value: &str) -> Result<()> {
|
||||||
|
if value.is_empty() {
|
||||||
|
return Err(Error::HypervisorError(format!("{field_name} is required")));
|
||||||
|
}
|
||||||
|
let mut chars = value.chars();
|
||||||
|
let Some(first) = chars.next() else {
|
||||||
|
return Err(Error::HypervisorError(format!("{field_name} is required")));
|
||||||
|
};
|
||||||
|
if !first.is_ascii_alphanumeric() {
|
||||||
|
return Err(Error::HypervisorError(format!(
|
||||||
|
"{field_name} must start with an ASCII alphanumeric character"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if chars.any(|ch| !(ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-'))) {
|
||||||
|
return Err(Error::HypervisorError(format!(
|
||||||
|
"{field_name} contains unsupported characters"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_host_port(authority: &str, default_port: u16) -> Result<(String, u16)> {
|
||||||
|
if let Some(rest) = authority.strip_prefix('[') {
|
||||||
|
let (host, tail) = rest
|
||||||
|
.split_once(']')
|
||||||
|
.ok_or_else(|| Error::HypervisorError("invalid IPv6 authority".into()))?;
|
||||||
|
let port = tail
|
||||||
|
.strip_prefix(':')
|
||||||
|
.and_then(|value| value.parse::<u16>().ok())
|
||||||
|
.unwrap_or(default_port);
|
||||||
|
return Ok((host.to_string(), port));
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some((host, port)) = authority.rsplit_once(':') {
|
||||||
|
if !host.is_empty() {
|
||||||
|
if let Ok(port) = port.parse::<u16>() {
|
||||||
|
return Ok((host.to_string(), port));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok((authority.to_string(), default_port))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_nbd_uri(uri: &str) -> Result<(String, u16, Option<String>)> {
|
||||||
|
let remainder = uri
|
||||||
|
.strip_prefix("nbd://")
|
||||||
|
.ok_or_else(|| Error::HypervisorError(format!("unsupported NBD URI: {uri}")))?;
|
||||||
|
if remainder.contains('@') || remainder.contains('?') || remainder.contains('#') {
|
||||||
|
return Err(Error::HypervisorError(format!(
|
||||||
|
"unsupported NBD URI components: {uri}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let (authority, path) = remainder.split_once('/').unwrap_or((remainder, ""));
|
||||||
|
let (host, port) = parse_host_port(authority, 10809)?;
|
||||||
|
if host.is_empty() {
|
||||||
|
return Err(Error::HypervisorError(format!(
|
||||||
|
"missing NBD host in URI: {uri}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let export = (!path.is_empty()).then(|| path.to_string());
|
||||||
|
Ok((host, port, export))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_ceph_monitor(monitor: &str) -> Result<Value> {
|
||||||
|
let (host, port) = parse_host_port(monitor, 6789)?;
|
||||||
|
if host.is_empty() {
|
||||||
|
return Err(Error::HypervisorError(format!(
|
||||||
|
"invalid Ceph monitor address: {monitor}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(json!({
|
||||||
|
"type": "inet",
|
||||||
|
"host": host,
|
||||||
|
"port": port.to_string()
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn disk_blockdev_arg(disk: &AttachedDisk, disk_id: &str) -> Result<String> {
|
||||||
|
let effective_cache = effective_disk_cache(disk);
|
||||||
|
let aio_mode = disk_aio_mode(disk);
|
||||||
|
let file = match &disk.attachment {
|
||||||
|
DiskAttachment::File { path, .. } => {
|
||||||
|
let mut file = json!({
|
||||||
|
"driver": "file",
|
||||||
|
"filename": path
|
||||||
|
});
|
||||||
|
if let Some(aio_mode) = aio_mode {
|
||||||
|
file["aio"] = json!(aio_mode);
|
||||||
|
}
|
||||||
|
file
|
||||||
|
}
|
||||||
|
DiskAttachment::Nbd { uri, .. } => {
|
||||||
|
let (host, port, export) = parse_nbd_uri(uri)?;
|
||||||
|
let mut nbd = json!({
|
||||||
|
"driver": "nbd",
|
||||||
|
"server": {
|
||||||
|
"type": "inet",
|
||||||
|
"host": host,
|
||||||
|
"port": port.to_string()
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if let Some(export) = export {
|
||||||
|
nbd["export"] = json!(export);
|
||||||
|
}
|
||||||
|
if let Some(aio_mode) = aio_mode {
|
||||||
|
nbd["aio"] = json!(aio_mode);
|
||||||
|
}
|
||||||
|
nbd
|
||||||
|
}
|
||||||
|
DiskAttachment::CephRbd {
|
||||||
|
pool,
|
||||||
|
image,
|
||||||
|
monitors,
|
||||||
|
user,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
validate_ceph_component("ceph pool", pool)?;
|
||||||
|
validate_ceph_component("ceph image", image)?;
|
||||||
|
if !user.is_empty() {
|
||||||
|
validate_ceph_component("ceph user", user)?;
|
||||||
|
}
|
||||||
|
let servers: Vec<Value> = monitors
|
||||||
|
.iter()
|
||||||
|
.map(|monitor| parse_ceph_monitor(monitor))
|
||||||
|
.collect::<Result<Vec<_>>>()?;
|
||||||
|
let mut rbd = json!({
|
||||||
|
"driver": "rbd",
|
||||||
|
"pool": pool,
|
||||||
|
"image": image,
|
||||||
|
"server": servers
|
||||||
|
});
|
||||||
|
if !user.is_empty() {
|
||||||
|
rbd["user"] = json!(user);
|
||||||
|
}
|
||||||
|
rbd
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let format_driver = match &disk.attachment {
|
||||||
|
DiskAttachment::File { format, .. } | DiskAttachment::Nbd { format, .. } => {
|
||||||
|
volume_format_name(*format)
|
||||||
|
}
|
||||||
|
DiskAttachment::CephRbd { .. } => "raw",
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(json!({
|
||||||
|
"node-name": format!("drive-{disk_id}"),
|
||||||
|
"driver": format_driver,
|
||||||
|
"read-only": disk.read_only,
|
||||||
|
"cache": disk_cache_json(effective_cache),
|
||||||
|
"file": file
|
||||||
|
})
|
||||||
|
.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
fn build_disk_args(vm: &VirtualMachine, disks: &[AttachedDisk]) -> Result<Vec<String>> {
|
fn build_disk_args(vm: &VirtualMachine, disks: &[AttachedDisk]) -> Result<Vec<String>> {
|
||||||
if disks.is_empty() && vm.spec.disks.is_empty() {
|
if disks.is_empty() && vm.spec.disks.is_empty() {
|
||||||
let qcow_path = resolve_qcow2_path().ok_or_else(|| {
|
let qcow_path = resolve_qcow2_path().ok_or_else(|| {
|
||||||
|
|
@ -167,8 +299,20 @@ fn build_disk_args(vm: &VirtualMachine, disks: &[AttachedDisk]) -> Result<Vec<St
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
return Ok(vec![
|
return Ok(vec![
|
||||||
"-drive".into(),
|
"-blockdev".into(),
|
||||||
format!("file={},if=virtio,format=qcow2", qcow_path.display()),
|
json!({
|
||||||
|
"node-name": "drive-root",
|
||||||
|
"driver": "qcow2",
|
||||||
|
"read-only": false,
|
||||||
|
"cache": disk_cache_json(DiskCache::Writeback),
|
||||||
|
"file": {
|
||||||
|
"driver": "file",
|
||||||
|
"filename": qcow_path.display().to_string()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
"-device".into(),
|
||||||
|
"virtio-blk-pci,drive=drive-root,id=disk-root".into(),
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -205,21 +349,12 @@ fn build_disk_args(vm: &VirtualMachine, disks: &[AttachedDisk]) -> Result<Vec<St
|
||||||
|
|
||||||
for (index, disk) in disks.into_iter().enumerate() {
|
for (index, disk) in disks.into_iter().enumerate() {
|
||||||
let disk_id = sanitize_device_component(&disk.id, index);
|
let disk_id = sanitize_device_component(&disk.id, index);
|
||||||
let (source, format_name) = disk_source_arg(disk)?;
|
|
||||||
if disk_uses_dedicated_iothread(disk) {
|
if disk_uses_dedicated_iothread(disk) {
|
||||||
args.push("-object".into());
|
args.push("-object".into());
|
||||||
args.push(format!("iothread,id=iothread-{disk_id}"));
|
args.push(format!("iothread,id=iothread-{disk_id}"));
|
||||||
}
|
}
|
||||||
let effective_cache = effective_disk_cache(disk);
|
args.push("-blockdev".into());
|
||||||
let mut drive_arg = format!(
|
args.push(disk_blockdev_arg(disk, &disk_id)?);
|
||||||
"file={source},if=none,format={format_name},id=drive-{disk_id},cache={}",
|
|
||||||
disk_cache_mode(effective_cache)
|
|
||||||
);
|
|
||||||
if let Some(aio_mode) = disk_aio_mode(disk) {
|
|
||||||
drive_arg.push_str(&format!(",aio={aio_mode}"));
|
|
||||||
}
|
|
||||||
args.push("-drive".into());
|
|
||||||
args.push(drive_arg);
|
|
||||||
|
|
||||||
let bootindex = bootindex_suffix(disk.boot_index);
|
let bootindex = bootindex_suffix(disk.boot_index);
|
||||||
let device_arg = match disk.bus {
|
let device_arg = match disk.bus {
|
||||||
|
|
@ -359,33 +494,24 @@ async fn wait_for_qmp(qmp_socket: &Path, timeout: Duration) -> Result<()> {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn kill_pid(pid: u32) -> Result<()> {
|
fn kill_pid(pid: u32) -> Result<()> {
|
||||||
let status = std::process::Command::new("kill")
|
let pid = Pid::from_raw(pid as i32);
|
||||||
.arg("-9")
|
match nix_kill(pid, Signal::SIGKILL) {
|
||||||
.arg(pid.to_string())
|
Ok(()) => Ok(()),
|
||||||
.stdout(Stdio::null())
|
Err(nix::errno::Errno::ESRCH) => Ok(()),
|
||||||
.stderr(Stdio::null())
|
Err(error) => Err(Error::HypervisorError(format!(
|
||||||
.status()
|
"failed to send SIGKILL to pid {}: {error}",
|
||||||
.map_err(|e| Error::HypervisorError(format!("Failed to invoke kill -9: {e}")))?;
|
pid.as_raw()
|
||||||
if status.success() {
|
))),
|
||||||
Ok(())
|
|
||||||
} else if !pid_running(pid) {
|
|
||||||
Ok(())
|
|
||||||
} else {
|
|
||||||
Err(Error::HypervisorError(format!(
|
|
||||||
"kill -9 exited with status: {status}"
|
|
||||||
)))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pid_running(pid: u32) -> bool {
|
fn pid_running(pid: u32) -> bool {
|
||||||
std::process::Command::new("kill")
|
match nix_kill(Pid::from_raw(pid as i32), None::<Signal>) {
|
||||||
.arg("-0")
|
Ok(()) => true,
|
||||||
.arg(pid.to_string())
|
Err(nix::errno::Errno::EPERM) => true,
|
||||||
.stdout(Stdio::null())
|
Err(nix::errno::Errno::ESRCH) => false,
|
||||||
.stderr(Stdio::null())
|
Err(_) => false,
|
||||||
.status()
|
}
|
||||||
.map(|status| status.success())
|
|
||||||
.unwrap_or(false)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn vm_stopped_out_of_band(handle: &VmHandle, qmp_socket: &Path) -> bool {
|
fn vm_stopped_out_of_band(handle: &VmHandle, qmp_socket: &Path) -> bool {
|
||||||
|
|
@ -569,7 +695,9 @@ impl HypervisorBackend for KvmBackend {
|
||||||
|
|
||||||
match QmpClient::connect(&qmp_socket).await {
|
match QmpClient::connect(&qmp_socket).await {
|
||||||
Ok(mut client) => match client.query_status().await {
|
Ok(mut client) => match client.query_status().await {
|
||||||
Ok(status) if matches!(status.actual_state, VmState::Stopped | VmState::Failed) => {
|
Ok(status)
|
||||||
|
if matches!(status.actual_state, VmState::Stopped | VmState::Failed) =>
|
||||||
|
{
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Ok(_) => {}
|
Ok(_) => {}
|
||||||
|
|
@ -1109,10 +1237,12 @@ mod tests {
|
||||||
let args_joined = args.join(" ");
|
let args_joined = args.join(" ");
|
||||||
assert!(args_joined.contains("vm-root.qcow2"));
|
assert!(args_joined.contains("vm-root.qcow2"));
|
||||||
assert!(args_joined.contains("vm-data.qcow2"));
|
assert!(args_joined.contains("vm-data.qcow2"));
|
||||||
|
assert!(args_joined.contains("-blockdev"));
|
||||||
assert!(args_joined.contains("bootindex=1"));
|
assert!(args_joined.contains("bootindex=1"));
|
||||||
assert!(args_joined.contains("cache=writeback"));
|
assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
|
||||||
assert!(args_joined.contains("cache=none,aio=native"));
|
assert!(args_joined.contains("\"cache\":{\"direct\":false,\"no-flush\":false}"));
|
||||||
assert!(args_joined.contains("cache=writeback,aio=threads"));
|
assert!(args_joined.contains("\"aio\":\"native\""));
|
||||||
|
assert!(args_joined.contains("\"aio\":\"threads\""));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -1135,6 +1265,7 @@ mod tests {
|
||||||
let console = PathBuf::from("/tmp/console.log");
|
let console = PathBuf::from("/tmp/console.log");
|
||||||
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
|
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
|
||||||
let args_joined = args.join(" ");
|
let args_joined = args.join(" ");
|
||||||
|
assert!(args_joined.contains("\"driver\":\"nbd\""));
|
||||||
assert!(args_joined.contains("-object iothread,id=iothread-root"));
|
assert!(args_joined.contains("-object iothread,id=iothread-root"));
|
||||||
assert!(args_joined.contains("virtio-blk-pci,drive=drive-root,id=disk-root,iothread=iothread-root,num-queues=4,queue-size=1024,bootindex=1"));
|
assert!(args_joined.contains("virtio-blk-pci,drive=drive-root,id=disk-root,iothread=iothread-root,num-queues=4,queue-size=1024,bootindex=1"));
|
||||||
}
|
}
|
||||||
|
|
@ -1159,7 +1290,8 @@ mod tests {
|
||||||
let console = PathBuf::from("/tmp/console.log");
|
let console = PathBuf::from("/tmp/console.log");
|
||||||
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
|
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
|
||||||
let args_joined = args.join(" ");
|
let args_joined = args.join(" ");
|
||||||
assert!(args_joined.contains("cache=none,aio=io_uring"));
|
assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
|
||||||
|
assert!(args_joined.contains("\"aio\":\"io_uring\""));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -1182,7 +1314,8 @@ mod tests {
|
||||||
let console = PathBuf::from("/tmp/console.log");
|
let console = PathBuf::from("/tmp/console.log");
|
||||||
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
|
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
|
||||||
let args_joined = args.join(" ");
|
let args_joined = args.join(" ");
|
||||||
assert!(args_joined.contains("cache=none,aio=io_uring"));
|
assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
|
||||||
|
assert!(args_joined.contains("\"aio\":\"io_uring\""));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -1205,10 +1338,34 @@ mod tests {
|
||||||
let console = PathBuf::from("/tmp/console.log");
|
let console = PathBuf::from("/tmp/console.log");
|
||||||
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
|
let args = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap();
|
||||||
let args_joined = args.join(" ");
|
let args_joined = args.join(" ");
|
||||||
assert!(args_joined.contains("cache=none,aio=threads"));
|
assert!(args_joined.contains("\"cache\":{\"direct\":true,\"no-flush\":false}"));
|
||||||
|
assert!(args_joined.contains("\"aio\":\"threads\""));
|
||||||
std::env::remove_var(crate::env::ENV_NBD_AIO_MODE);
|
std::env::remove_var(crate::env::ENV_NBD_AIO_MODE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_qemu_args_rejects_invalid_ceph_identifiers() {
|
||||||
|
let vm = VirtualMachine::new("vm1", "org", "proj", VmSpec::default());
|
||||||
|
let disks = vec![AttachedDisk {
|
||||||
|
id: "root".into(),
|
||||||
|
attachment: DiskAttachment::CephRbd {
|
||||||
|
pool: "pool,inject".into(),
|
||||||
|
image: "image".into(),
|
||||||
|
monitors: vec!["10.0.0.10:6789".into()],
|
||||||
|
user: "admin".into(),
|
||||||
|
secret: None,
|
||||||
|
},
|
||||||
|
bus: DiskBus::Virtio,
|
||||||
|
cache: DiskCache::None,
|
||||||
|
boot_index: Some(1),
|
||||||
|
read_only: false,
|
||||||
|
}];
|
||||||
|
let qmp = PathBuf::from("/tmp/qmp.sock");
|
||||||
|
let console = PathBuf::from("/tmp/console.log");
|
||||||
|
let error = build_qemu_args(&vm, &disks, &qmp, &console, None, None).unwrap_err();
|
||||||
|
assert!(error.to_string().contains("unsupported characters"));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn wait_for_qmp_succeeds_after_socket_created() {
|
async fn wait_for_qmp_succeeds_after_socket_created() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|
|
||||||
|
|
@ -33,7 +33,9 @@ impl QmpClient {
|
||||||
last_error = Some(format!("Failed to connect QMP: {e}"));
|
last_error = Some(format!("Failed to connect QMP: {e}"));
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
return Err(Error::HypervisorError(format!("Failed to connect QMP: {e}")));
|
return Err(Error::HypervisorError(format!(
|
||||||
|
"Failed to connect QMP: {e}"
|
||||||
|
)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,7 @@ serde_json = { workspace = true }
|
||||||
toml = { workspace = true }
|
toml = { workspace = true }
|
||||||
metrics-exporter-prometheus = { workspace = true }
|
metrics-exporter-prometheus = { workspace = true }
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
|
||||||
|
sha2 = "0.10"
|
||||||
chainfire-client = { path = "../../../chainfire/chainfire-client" }
|
chainfire-client = { path = "../../../chainfire/chainfire-client" }
|
||||||
creditservice-client = { path = "../../../creditservice/creditservice-client" }
|
creditservice-client = { path = "../../../creditservice/creditservice-client" }
|
||||||
flaredb-client = { path = "../../../flaredb/crates/flaredb-client" }
|
flaredb-client = { path = "../../../flaredb/crates/flaredb-client" }
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,10 @@
|
||||||
|
use crate::config::ArtifactStoreConfig;
|
||||||
|
use crate::storage::ImageUploadPart;
|
||||||
|
use reqwest::header::LOCATION;
|
||||||
|
use reqwest::{Client as HttpClient, Url};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::collections::HashSet;
|
||||||
|
use std::net::IpAddr;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
@ -13,7 +20,6 @@ use lightningstor_api::proto::{
|
||||||
};
|
};
|
||||||
use lightningstor_api::{BucketServiceClient, ObjectServiceClient};
|
use lightningstor_api::{BucketServiceClient, ObjectServiceClient};
|
||||||
use plasmavmc_types::ImageFormat;
|
use plasmavmc_types::ImageFormat;
|
||||||
use reqwest::StatusCode as HttpStatusCode;
|
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
|
|
@ -23,25 +29,30 @@ use tonic::metadata::MetadataValue;
|
||||||
use tonic::transport::{Channel, Endpoint};
|
use tonic::transport::{Channel, Endpoint};
|
||||||
use tonic::{Code, Request, Status};
|
use tonic::{Code, Request, Status};
|
||||||
|
|
||||||
const DEFAULT_IMAGE_BUCKET: &str = "plasmavmc-images";
|
|
||||||
const MAX_OBJECT_GRPC_MESSAGE_SIZE: usize = 1024 * 1024 * 1024;
|
const MAX_OBJECT_GRPC_MESSAGE_SIZE: usize = 1024 * 1024 * 1024;
|
||||||
const OBJECT_GRPC_INITIAL_STREAM_WINDOW: u32 = 64 * 1024 * 1024;
|
const OBJECT_GRPC_INITIAL_STREAM_WINDOW: u32 = 64 * 1024 * 1024;
|
||||||
const OBJECT_GRPC_INITIAL_CONNECTION_WINDOW: u32 = 512 * 1024 * 1024;
|
const OBJECT_GRPC_INITIAL_CONNECTION_WINDOW: u32 = 512 * 1024 * 1024;
|
||||||
const OBJECT_GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30);
|
const OBJECT_GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30);
|
||||||
const OBJECT_GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(10);
|
const OBJECT_GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
const DEFAULT_MULTIPART_UPLOAD_PART_SIZE: usize = 32 * 1024 * 1024;
|
|
||||||
const MIN_MULTIPART_UPLOAD_PART_SIZE: usize = 8 * 1024 * 1024;
|
const MIN_MULTIPART_UPLOAD_PART_SIZE: usize = 8 * 1024 * 1024;
|
||||||
const MAX_MULTIPART_UPLOAD_PART_SIZE: usize = 128 * 1024 * 1024;
|
const MAX_MULTIPART_UPLOAD_PART_SIZE: usize = 128 * 1024 * 1024;
|
||||||
const DEFAULT_MULTIPART_UPLOAD_CONCURRENCY: usize = 4;
|
|
||||||
const MAX_MULTIPART_UPLOAD_CONCURRENCY: usize = 32;
|
const MAX_MULTIPART_UPLOAD_CONCURRENCY: usize = 32;
|
||||||
const DEFAULT_RAW_IMAGE_CONVERT_PARALLELISM: usize = 8;
|
const MAX_IMPORT_REDIRECTS: usize = 5;
|
||||||
|
const DEFAULT_HTTP_SEND_TIMEOUT: Duration = Duration::from_secs(15);
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct ArtifactStore {
|
pub struct ArtifactStore {
|
||||||
channel: Channel,
|
channel: Channel,
|
||||||
iam_client: Arc<IamClient>,
|
iam_client: Arc<IamClient>,
|
||||||
|
http_client: HttpClient,
|
||||||
image_bucket: String,
|
image_bucket: String,
|
||||||
image_cache_dir: PathBuf,
|
image_cache_dir: PathBuf,
|
||||||
|
multipart_upload_concurrency: usize,
|
||||||
|
multipart_upload_part_size: usize,
|
||||||
|
raw_image_convert_parallelism: usize,
|
||||||
|
max_image_import_size_bytes: u64,
|
||||||
|
allowed_https_hosts: Arc<HashSet<String>>,
|
||||||
|
qemu_img_path: PathBuf,
|
||||||
project_tokens: Arc<DashMap<String, CachedToken>>,
|
project_tokens: Arc<DashMap<String, CachedToken>>,
|
||||||
ensured_buckets: Arc<DashSet<String>>,
|
ensured_buckets: Arc<DashSet<String>>,
|
||||||
}
|
}
|
||||||
|
|
@ -50,6 +61,8 @@ pub(crate) struct ImportedImage {
|
||||||
pub size_bytes: u64,
|
pub size_bytes: u64,
|
||||||
pub checksum: String,
|
pub checksum: String,
|
||||||
pub format: ImageFormat,
|
pub format: ImageFormat,
|
||||||
|
pub source_type: String,
|
||||||
|
pub source_host: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
|
|
@ -62,10 +75,24 @@ struct CachedToken {
|
||||||
expires_at: Instant,
|
expires_at: Instant,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct ValidatedImportUrl {
|
||||||
|
url: Url,
|
||||||
|
host: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ImportedImageSource {
|
||||||
|
source_type: String,
|
||||||
|
host: String,
|
||||||
|
}
|
||||||
|
|
||||||
impl ArtifactStore {
|
impl ArtifactStore {
|
||||||
pub async fn from_env(iam_endpoint: &str) -> Result<Option<Self>, Box<dyn std::error::Error>> {
|
pub async fn from_config(
|
||||||
let Some(raw_endpoint) = std::env::var("PLASMAVMC_LIGHTNINGSTOR_ENDPOINT")
|
config: &ArtifactStoreConfig,
|
||||||
.ok()
|
iam_endpoint: &str,
|
||||||
|
) -> Result<Option<Self>, Box<dyn std::error::Error>> {
|
||||||
|
let Some(raw_endpoint) = config
|
||||||
|
.lightningstor_endpoint
|
||||||
|
.as_ref()
|
||||||
.map(|value| value.trim().to_string())
|
.map(|value| value.trim().to_string())
|
||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
else {
|
else {
|
||||||
|
|
@ -87,18 +114,41 @@ impl ArtifactStore {
|
||||||
.keep_alive_timeout(OBJECT_GRPC_KEEPALIVE_TIMEOUT)
|
.keep_alive_timeout(OBJECT_GRPC_KEEPALIVE_TIMEOUT)
|
||||||
.connect_lazy();
|
.connect_lazy();
|
||||||
|
|
||||||
let image_cache_dir = std::env::var("PLASMAVMC_IMAGE_CACHE_DIR")
|
let image_cache_dir = config.image_cache_dir.clone();
|
||||||
.map(PathBuf::from)
|
|
||||||
.unwrap_or_else(|_| PathBuf::from("/var/lib/plasmavmc/images"));
|
|
||||||
tokio::fs::create_dir_all(&image_cache_dir).await?;
|
tokio::fs::create_dir_all(&image_cache_dir).await?;
|
||||||
ensure_cache_dir_permissions(&image_cache_dir).await?;
|
ensure_cache_dir_permissions(&image_cache_dir).await?;
|
||||||
|
let http_client = HttpClient::builder()
|
||||||
|
.connect_timeout(Duration::from_secs(
|
||||||
|
config.image_import_connect_timeout_secs.max(1),
|
||||||
|
))
|
||||||
|
.timeout(Duration::from_secs(config.image_import_timeout_secs.max(1)))
|
||||||
|
.redirect(reqwest::redirect::Policy::none())
|
||||||
|
.build()?;
|
||||||
|
let allowed_https_hosts = config
|
||||||
|
.allowed_https_hosts
|
||||||
|
.iter()
|
||||||
|
.map(|host| host.trim().to_ascii_lowercase())
|
||||||
|
.filter(|host| !host.is_empty())
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
let qemu_img_path = resolve_binary_path(config.qemu_img_path.as_deref(), "qemu-img")?;
|
||||||
|
|
||||||
Ok(Some(Self {
|
Ok(Some(Self {
|
||||||
channel,
|
channel,
|
||||||
iam_client,
|
iam_client,
|
||||||
image_bucket: std::env::var("PLASMAVMC_IMAGE_BUCKET")
|
http_client,
|
||||||
.unwrap_or_else(|_| DEFAULT_IMAGE_BUCKET.to_string()),
|
image_bucket: config.image_bucket.clone(),
|
||||||
image_cache_dir,
|
image_cache_dir,
|
||||||
|
multipart_upload_concurrency: config
|
||||||
|
.multipart_upload_concurrency
|
||||||
|
.clamp(1, MAX_MULTIPART_UPLOAD_CONCURRENCY),
|
||||||
|
multipart_upload_part_size: config.multipart_upload_part_size.clamp(
|
||||||
|
MIN_MULTIPART_UPLOAD_PART_SIZE,
|
||||||
|
MAX_MULTIPART_UPLOAD_PART_SIZE,
|
||||||
|
),
|
||||||
|
raw_image_convert_parallelism: config.raw_image_convert_parallelism.clamp(1, 64),
|
||||||
|
max_image_import_size_bytes: config.max_image_import_size_bytes.max(1),
|
||||||
|
allowed_https_hosts: Arc::new(allowed_https_hosts),
|
||||||
|
qemu_img_path,
|
||||||
project_tokens: Arc::new(DashMap::new()),
|
project_tokens: Arc::new(DashMap::new()),
|
||||||
ensured_buckets: Arc::new(DashSet::new()),
|
ensured_buckets: Arc::new(DashSet::new()),
|
||||||
}))
|
}))
|
||||||
|
|
@ -116,50 +166,20 @@ impl ArtifactStore {
|
||||||
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
|
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let image_path = self.image_path(image_id);
|
let staging_path = self.staging_path(image_id)?;
|
||||||
let staging_path = self.image_cache_dir.join(format!("{image_id}.source"));
|
let ImportedImageSource { source_type, host } =
|
||||||
|
self.materialize_source(source_url, &staging_path).await?;
|
||||||
self.materialize_source(source_url, &staging_path).await?;
|
self.process_staged_image(
|
||||||
if self
|
org_id,
|
||||||
.can_reuse_qcow2_source(&staging_path, source_format)
|
project_id,
|
||||||
.await?
|
image_id,
|
||||||
{
|
&token,
|
||||||
if tokio::fs::try_exists(&image_path).await.map_err(|e| {
|
&staging_path,
|
||||||
Status::internal(format!("failed to inspect {}: {e}", image_path.display()))
|
source_format,
|
||||||
})? {
|
source_type,
|
||||||
let _ = tokio::fs::remove_file(&staging_path).await;
|
Some(host),
|
||||||
} else {
|
)
|
||||||
tokio::fs::rename(&staging_path, &image_path)
|
.await
|
||||||
.await
|
|
||||||
.map_err(|e| {
|
|
||||||
Status::internal(format!(
|
|
||||||
"failed to move qcow2 image {} into cache {}: {e}",
|
|
||||||
staging_path.display(),
|
|
||||||
image_path.display()
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
ensure_cache_file_permissions(&image_path).await?;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Normalize non-qcow2 inputs through qemu-img convert so the cached
|
|
||||||
// artifact has a stable qcow2 representation before upload.
|
|
||||||
self.convert_to_qcow2(&staging_path, &image_path).await?;
|
|
||||||
let _ = tokio::fs::remove_file(&staging_path).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
let checksum = self.sha256sum(&image_path).await?;
|
|
||||||
let metadata = tokio::fs::metadata(&image_path).await.map_err(|e| {
|
|
||||||
Status::internal(format!("failed to stat {}: {e}", image_path.display()))
|
|
||||||
})?;
|
|
||||||
let image_key = image_object_key(org_id, project_id, image_id);
|
|
||||||
self.upload_file(&self.image_bucket, &image_key, &image_path, &token)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(ImportedImage {
|
|
||||||
size_bytes: metadata.len(),
|
|
||||||
checksum,
|
|
||||||
format: ImageFormat::Qcow2,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn materialize_image_cache(
|
pub async fn materialize_image_cache(
|
||||||
|
|
@ -171,8 +191,8 @@ impl ArtifactStore {
|
||||||
let token = self.issue_project_token(org_id, project_id).await?;
|
let token = self.issue_project_token(org_id, project_id).await?;
|
||||||
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
|
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
|
||||||
.await?;
|
.await?;
|
||||||
let image_key = image_object_key(org_id, project_id, image_id);
|
let image_key = image_object_key(org_id, project_id, image_id)?;
|
||||||
let image_path = self.image_path(image_id);
|
let image_path = self.image_path(image_id)?;
|
||||||
self.download_object_to_file(&self.image_bucket, &image_key, &image_path, &token)
|
self.download_object_to_file(&self.image_bucket, &image_key, &image_path, &token)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(image_path)
|
Ok(image_path)
|
||||||
|
|
@ -187,7 +207,7 @@ impl ArtifactStore {
|
||||||
let image_path = self
|
let image_path = self
|
||||||
.materialize_image_cache(org_id, project_id, image_id)
|
.materialize_image_cache(org_id, project_id, image_id)
|
||||||
.await?;
|
.await?;
|
||||||
let raw_path = self.raw_image_path(image_id);
|
let raw_path = self.raw_image_path(image_id)?;
|
||||||
self.convert_to_raw(&image_path, &raw_path).await?;
|
self.convert_to_raw(&image_path, &raw_path).await?;
|
||||||
Ok(raw_path)
|
Ok(raw_path)
|
||||||
}
|
}
|
||||||
|
|
@ -199,7 +219,7 @@ impl ArtifactStore {
|
||||||
image_id: &str,
|
image_id: &str,
|
||||||
) -> Result<(), Status> {
|
) -> Result<(), Status> {
|
||||||
let token = self.issue_project_token(org_id, project_id).await?;
|
let token = self.issue_project_token(org_id, project_id).await?;
|
||||||
let image_key = image_object_key(org_id, project_id, image_id);
|
let image_key = image_object_key(org_id, project_id, image_id)?;
|
||||||
let mut client = self.object_client().await?;
|
let mut client = self.object_client().await?;
|
||||||
let mut request = Request::new(DeleteObjectRequest {
|
let mut request = Request::new(DeleteObjectRequest {
|
||||||
bucket: self.image_bucket.clone(),
|
bucket: self.image_bucket.clone(),
|
||||||
|
|
@ -213,7 +233,7 @@ impl ArtifactStore {
|
||||||
Err(status) => return Err(Status::from_error(Box::new(status))),
|
Err(status) => return Err(Status::from_error(Box::new(status))),
|
||||||
}
|
}
|
||||||
|
|
||||||
let image_path = self.image_path(image_id);
|
let image_path = self.image_path(image_id)?;
|
||||||
if tokio::fs::try_exists(&image_path).await.map_err(|e| {
|
if tokio::fs::try_exists(&image_path).await.map_err(|e| {
|
||||||
Status::internal(format!("failed to inspect {}: {e}", image_path.display()))
|
Status::internal(format!("failed to inspect {}: {e}", image_path.display()))
|
||||||
})? {
|
})? {
|
||||||
|
|
@ -221,7 +241,7 @@ impl ArtifactStore {
|
||||||
Status::internal(format!("failed to remove {}: {e}", image_path.display()))
|
Status::internal(format!("failed to remove {}: {e}", image_path.display()))
|
||||||
})?;
|
})?;
|
||||||
}
|
}
|
||||||
let raw_path = self.raw_image_path(image_id);
|
let raw_path = self.raw_image_path(image_id)?;
|
||||||
if tokio::fs::try_exists(&raw_path).await.map_err(|e| {
|
if tokio::fs::try_exists(&raw_path).await.map_err(|e| {
|
||||||
Status::internal(format!("failed to inspect {}: {e}", raw_path.display()))
|
Status::internal(format!("failed to inspect {}: {e}", raw_path.display()))
|
||||||
})? {
|
})? {
|
||||||
|
|
@ -232,6 +252,156 @@ impl ArtifactStore {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn minimum_upload_part_size(&self) -> u32 {
|
||||||
|
self.multipart_upload_part_size as u32
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn begin_image_upload(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
) -> Result<(String, String), Status> {
|
||||||
|
let token = self.issue_project_token(org_id, project_id).await?;
|
||||||
|
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
|
||||||
|
.await?;
|
||||||
|
let staging_key = staging_object_key(org_id, project_id, image_id)?;
|
||||||
|
let mut client = self.object_client().await?;
|
||||||
|
let mut request = Request::new(CreateMultipartUploadRequest {
|
||||||
|
bucket: self.image_bucket.clone(),
|
||||||
|
key: staging_key.clone(),
|
||||||
|
metadata: None,
|
||||||
|
});
|
||||||
|
attach_bearer(&mut request, &token)?;
|
||||||
|
let upload_id = client
|
||||||
|
.create_multipart_upload(request)
|
||||||
|
.await
|
||||||
|
.map_err(|status| Status::from_error(Box::new(status)))?
|
||||||
|
.into_inner()
|
||||||
|
.upload_id;
|
||||||
|
Ok((upload_id, staging_key))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn upload_image_part(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
staging_key: &str,
|
||||||
|
upload_id: &str,
|
||||||
|
part_number: u32,
|
||||||
|
body: Vec<u8>,
|
||||||
|
) -> Result<ImageUploadPart, Status> {
|
||||||
|
if part_number == 0 {
|
||||||
|
return Err(Status::invalid_argument(
|
||||||
|
"part_number must be greater than zero",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if body.is_empty() {
|
||||||
|
return Err(Status::invalid_argument(
|
||||||
|
"upload part body must not be empty",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let token = self.issue_project_token(org_id, project_id).await?;
|
||||||
|
let mut client = self.object_client().await?;
|
||||||
|
let request_stream = tokio_stream::iter(vec![UploadPartRequest {
|
||||||
|
bucket: self.image_bucket.clone(),
|
||||||
|
key: staging_key.to_string(),
|
||||||
|
upload_id: upload_id.to_string(),
|
||||||
|
part_number,
|
||||||
|
body: body.clone().into(),
|
||||||
|
content_md5: String::new(),
|
||||||
|
}]);
|
||||||
|
let mut request = Request::new(request_stream);
|
||||||
|
attach_bearer(&mut request, &token)?;
|
||||||
|
let response = client
|
||||||
|
.upload_part(request)
|
||||||
|
.await
|
||||||
|
.map_err(|status| Status::from_error(Box::new(status)))?;
|
||||||
|
Ok(ImageUploadPart {
|
||||||
|
part_number,
|
||||||
|
etag: response.into_inner().etag,
|
||||||
|
size_bytes: body.len() as u64,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn complete_image_upload(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
staging_key: &str,
|
||||||
|
upload_id: &str,
|
||||||
|
parts: &[ImageUploadPart],
|
||||||
|
source_format: ImageFormat,
|
||||||
|
) -> Result<ImportedImage, Status> {
|
||||||
|
if parts.is_empty() {
|
||||||
|
return Err(Status::failed_precondition(
|
||||||
|
"upload session does not contain any parts",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let token = self.issue_project_token(org_id, project_id).await?;
|
||||||
|
self.ensure_bucket(&self.image_bucket, org_id, project_id, &token)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let mut sorted_parts: Vec<CompletedPart> = parts
|
||||||
|
.iter()
|
||||||
|
.map(|part| CompletedPart {
|
||||||
|
part_number: part.part_number,
|
||||||
|
etag: part.etag.clone(),
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
sorted_parts.sort_by_key(|part| part.part_number);
|
||||||
|
|
||||||
|
let mut client = self.object_client().await?;
|
||||||
|
let mut complete_request = Request::new(CompleteMultipartUploadRequest {
|
||||||
|
bucket: self.image_bucket.clone(),
|
||||||
|
key: staging_key.to_string(),
|
||||||
|
upload_id: upload_id.to_string(),
|
||||||
|
parts: sorted_parts,
|
||||||
|
});
|
||||||
|
attach_bearer(&mut complete_request, &token)?;
|
||||||
|
client
|
||||||
|
.complete_multipart_upload(complete_request)
|
||||||
|
.await
|
||||||
|
.map_err(|status| Status::from_error(Box::new(status)))?;
|
||||||
|
|
||||||
|
let staging_path = self.staging_path(image_id)?;
|
||||||
|
if tokio::fs::try_exists(&staging_path).await.unwrap_or(false) {
|
||||||
|
let _ = tokio::fs::remove_file(&staging_path).await;
|
||||||
|
}
|
||||||
|
self.download_object_to_file(&self.image_bucket, staging_key, &staging_path, &token)
|
||||||
|
.await?;
|
||||||
|
let result = self
|
||||||
|
.process_staged_image(
|
||||||
|
org_id,
|
||||||
|
project_id,
|
||||||
|
image_id,
|
||||||
|
&token,
|
||||||
|
&staging_path,
|
||||||
|
source_format,
|
||||||
|
"upload".to_string(),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let _ = self
|
||||||
|
.delete_object_ignore_not_found(&self.image_bucket, staging_key, &token)
|
||||||
|
.await;
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn abort_image_upload(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
staging_key: &str,
|
||||||
|
upload_id: &str,
|
||||||
|
) -> Result<(), Status> {
|
||||||
|
let token = self.issue_project_token(org_id, project_id).await?;
|
||||||
|
self.abort_multipart_upload(&self.image_bucket, staging_key, upload_id, &token)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
async fn ensure_bucket(
|
async fn ensure_bucket(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|
@ -275,7 +445,7 @@ impl ArtifactStore {
|
||||||
let metadata = tokio::fs::metadata(path)
|
let metadata = tokio::fs::metadata(path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| Status::internal(format!("failed to stat {path:?}: {e}")))?;
|
.map_err(|e| Status::internal(format!("failed to stat {path:?}: {e}")))?;
|
||||||
let multipart_part_size = multipart_upload_part_size();
|
let multipart_part_size = self.multipart_upload_part_size;
|
||||||
if metadata.len() > multipart_part_size as u64 {
|
if metadata.len() > multipart_part_size as u64 {
|
||||||
return self
|
return self
|
||||||
.upload_file_multipart(bucket, key, path, token, metadata.len())
|
.upload_file_multipart(bucket, key, path, token, metadata.len())
|
||||||
|
|
@ -336,7 +506,7 @@ impl ArtifactStore {
|
||||||
size_bytes: u64,
|
size_bytes: u64,
|
||||||
) -> Result<(), Status> {
|
) -> Result<(), Status> {
|
||||||
let started = Instant::now();
|
let started = Instant::now();
|
||||||
let multipart_part_size = multipart_upload_part_size();
|
let multipart_part_size = self.multipart_upload_part_size;
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
bucket = bucket,
|
bucket = bucket,
|
||||||
key = key,
|
key = key,
|
||||||
|
|
@ -366,7 +536,7 @@ impl ArtifactStore {
|
||||||
let mut part_number = 1u32;
|
let mut part_number = 1u32;
|
||||||
let mut completed_parts = Vec::new();
|
let mut completed_parts = Vec::new();
|
||||||
let mut uploads = JoinSet::new();
|
let mut uploads = JoinSet::new();
|
||||||
let upload_concurrency = multipart_upload_concurrency();
|
let upload_concurrency = self.multipart_upload_concurrency;
|
||||||
|
|
||||||
let enqueue_part_upload = |uploads: &mut JoinSet<Result<CompletedPart, Status>>,
|
let enqueue_part_upload = |uploads: &mut JoinSet<Result<CompletedPart, Status>>,
|
||||||
client: &ObjectServiceClient<Channel>,
|
client: &ObjectServiceClient<Channel>,
|
||||||
|
|
@ -569,76 +739,23 @@ impl ArtifactStore {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn materialize_source(&self, source_url: &str, path: &Path) -> Result<(), Status> {
|
async fn materialize_source(
|
||||||
|
&self,
|
||||||
|
source_url: &str,
|
||||||
|
path: &Path,
|
||||||
|
) -> Result<ImportedImageSource, Status> {
|
||||||
if let Some(parent) = path.parent() {
|
if let Some(parent) = path.parent() {
|
||||||
tokio::fs::create_dir_all(parent)
|
tokio::fs::create_dir_all(parent)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
|
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(source_path) = source_url.strip_prefix("file://") {
|
let validated = self.validate_import_url(source_url).await?;
|
||||||
tokio::fs::copy(source_path, path).await.map_err(|e| {
|
self.download_https_source(&validated, path).await?;
|
||||||
Status::internal(format!("failed to copy image source {source_path}: {e}"))
|
Ok(ImportedImageSource {
|
||||||
})?;
|
source_type: "https".to_string(),
|
||||||
ensure_cache_file_permissions(path).await?;
|
host: validated.host,
|
||||||
return Ok(());
|
})
|
||||||
}
|
|
||||||
|
|
||||||
if source_url.starts_with('/') {
|
|
||||||
tokio::fs::copy(source_url, path).await.map_err(|e| {
|
|
||||||
Status::internal(format!("failed to copy image source {source_url}: {e}"))
|
|
||||||
})?;
|
|
||||||
ensure_cache_file_permissions(path).await?;
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
|
|
||||||
if source_url.starts_with("http://") || source_url.starts_with("https://") {
|
|
||||||
let mut response = reqwest::get(source_url).await.map_err(|e| {
|
|
||||||
Status::unavailable(format!("failed to download image source: {e}"))
|
|
||||||
})?;
|
|
||||||
if response.status() != HttpStatusCode::OK {
|
|
||||||
return Err(Status::failed_precondition(format!(
|
|
||||||
"image download failed with HTTP {}",
|
|
||||||
response.status()
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let temp_path = path.with_extension("download");
|
|
||||||
let mut file = tokio::fs::File::create(&temp_path).await.map_err(|e| {
|
|
||||||
Status::internal(format!(
|
|
||||||
"failed to create downloaded image {}: {e}",
|
|
||||||
temp_path.display()
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
while let Some(chunk) = response.chunk().await.map_err(|e| {
|
|
||||||
Status::unavailable(format!("failed to read image response body: {e}"))
|
|
||||||
})? {
|
|
||||||
file.write_all(&chunk).await.map_err(|e| {
|
|
||||||
Status::internal(format!(
|
|
||||||
"failed to write downloaded image {}: {e}",
|
|
||||||
temp_path.display()
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
}
|
|
||||||
file.flush().await.map_err(|e| {
|
|
||||||
Status::internal(format!(
|
|
||||||
"failed to flush downloaded image {}: {e}",
|
|
||||||
temp_path.display()
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
drop(file);
|
|
||||||
tokio::fs::rename(&temp_path, path).await.map_err(|e| {
|
|
||||||
Status::internal(format!(
|
|
||||||
"failed to finalize downloaded image {}: {e}",
|
|
||||||
path.display()
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
ensure_cache_file_permissions(path).await?;
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
|
|
||||||
Err(Status::invalid_argument(
|
|
||||||
"source_url must be file://, an absolute path, or http(s)://",
|
|
||||||
))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn convert_to_qcow2(&self, source: &Path, destination: &Path) -> Result<(), Status> {
|
async fn convert_to_qcow2(&self, source: &Path, destination: &Path) -> Result<(), Status> {
|
||||||
|
|
@ -654,9 +771,9 @@ impl ArtifactStore {
|
||||||
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
|
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let parallelism = raw_image_convert_parallelism().to_string();
|
let parallelism = self.raw_image_convert_parallelism.to_string();
|
||||||
let args = qemu_img_convert_to_qcow2_args(source, destination, ¶llelism);
|
let args = qemu_img_convert_to_qcow2_args(source, destination, ¶llelism);
|
||||||
let status = Command::new("qemu-img")
|
let status = Command::new(&self.qemu_img_path)
|
||||||
.args(args.iter().map(String::as_str))
|
.args(args.iter().map(String::as_str))
|
||||||
.status()
|
.status()
|
||||||
.await
|
.await
|
||||||
|
|
@ -685,9 +802,9 @@ impl ArtifactStore {
|
||||||
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
|
.map_err(|e| Status::internal(format!("failed to create {parent:?}: {e}")))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let parallelism = raw_image_convert_parallelism().to_string();
|
let parallelism = self.raw_image_convert_parallelism.to_string();
|
||||||
let args = qemu_img_convert_to_raw_args(source, destination, ¶llelism);
|
let args = qemu_img_convert_to_raw_args(source, destination, ¶llelism);
|
||||||
let status = Command::new("qemu-img")
|
let status = Command::new(&self.qemu_img_path)
|
||||||
.args(args.iter().map(String::as_str))
|
.args(args.iter().map(String::as_str))
|
||||||
.status()
|
.status()
|
||||||
.await
|
.await
|
||||||
|
|
@ -712,7 +829,7 @@ impl ArtifactStore {
|
||||||
return Ok(false);
|
return Ok(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
let output = Command::new("qemu-img")
|
let output = Command::new(&self.qemu_img_path)
|
||||||
.args(["info", "--output", "json", path.to_string_lossy().as_ref()])
|
.args(["info", "--output", "json", path.to_string_lossy().as_ref()])
|
||||||
.output()
|
.output()
|
||||||
.await
|
.await
|
||||||
|
|
@ -727,25 +844,262 @@ impl ArtifactStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn sha256sum(&self, path: &Path) -> Result<String, Status> {
|
async fn sha256sum(&self, path: &Path) -> Result<String, Status> {
|
||||||
let output = Command::new("sha256sum")
|
let mut file = tokio::fs::File::open(path)
|
||||||
.arg(path)
|
|
||||||
.output()
|
|
||||||
.await
|
.await
|
||||||
.map_err(|e| Status::internal(format!("failed to spawn sha256sum: {e}")))?;
|
.map_err(|e| Status::internal(format!("failed to open {}: {e}", path.display())))?;
|
||||||
if !output.status.success() {
|
let mut digest = Sha256::new();
|
||||||
return Err(Status::internal(format!(
|
let mut buffer = vec![0u8; 1024 * 1024];
|
||||||
"sha256sum failed for {} with status {}",
|
loop {
|
||||||
path.display(),
|
let read_now = file
|
||||||
output.status
|
.read(&mut buffer)
|
||||||
|
.await
|
||||||
|
.map_err(|e| Status::internal(format!("failed to read {}: {e}", path.display())))?;
|
||||||
|
if read_now == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
digest.update(&buffer[..read_now]);
|
||||||
|
}
|
||||||
|
Ok(format!("{:x}", digest.finalize()))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn process_staged_image(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
token: &str,
|
||||||
|
staging_path: &Path,
|
||||||
|
source_format: ImageFormat,
|
||||||
|
source_type: String,
|
||||||
|
source_host: Option<String>,
|
||||||
|
) -> Result<ImportedImage, Status> {
|
||||||
|
let image_path = self.image_path(image_id)?;
|
||||||
|
if self
|
||||||
|
.can_reuse_qcow2_source(staging_path, source_format)
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
if tokio::fs::try_exists(&image_path).await.map_err(|e| {
|
||||||
|
Status::internal(format!("failed to inspect {}: {e}", image_path.display()))
|
||||||
|
})? {
|
||||||
|
let _ = tokio::fs::remove_file(staging_path).await;
|
||||||
|
} else {
|
||||||
|
tokio::fs::rename(staging_path, &image_path)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
Status::internal(format!(
|
||||||
|
"failed to move qcow2 image {} into cache {}: {e}",
|
||||||
|
staging_path.display(),
|
||||||
|
image_path.display()
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
ensure_cache_file_permissions(&image_path).await?;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
self.convert_to_qcow2(staging_path, &image_path).await?;
|
||||||
|
let _ = tokio::fs::remove_file(staging_path).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
let checksum = self.sha256sum(&image_path).await?;
|
||||||
|
let metadata = tokio::fs::metadata(&image_path).await.map_err(|e| {
|
||||||
|
Status::internal(format!("failed to stat {}: {e}", image_path.display()))
|
||||||
|
})?;
|
||||||
|
let image_key = image_object_key(org_id, project_id, image_id)?;
|
||||||
|
self.upload_file(&self.image_bucket, &image_key, &image_path, token)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(ImportedImage {
|
||||||
|
size_bytes: metadata.len(),
|
||||||
|
checksum,
|
||||||
|
format: ImageFormat::Qcow2,
|
||||||
|
source_type,
|
||||||
|
source_host,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn validate_import_url(&self, source_url: &str) -> Result<ValidatedImportUrl, Status> {
|
||||||
|
if source_url.starts_with("file://") || source_url.starts_with('/') {
|
||||||
|
return Err(Status::invalid_argument(
|
||||||
|
"source_url must use https:// and may not reference local files",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let url = Url::parse(source_url)
|
||||||
|
.map_err(|e| Status::invalid_argument(format!("invalid source_url: {e}")))?;
|
||||||
|
if url.scheme() != "https" {
|
||||||
|
return Err(Status::invalid_argument("source_url must use https://"));
|
||||||
|
}
|
||||||
|
if !url.username().is_empty() || url.password().is_some() {
|
||||||
|
return Err(Status::invalid_argument(
|
||||||
|
"source_url must not include embedded credentials",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let host = url
|
||||||
|
.host_str()
|
||||||
|
.map(str::to_ascii_lowercase)
|
||||||
|
.ok_or_else(|| Status::invalid_argument("source_url must include a host"))?;
|
||||||
|
if host == "localhost" {
|
||||||
|
return Err(Status::invalid_argument(
|
||||||
|
"source_url host must not target loopback or local interfaces",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !self.allowed_https_hosts.is_empty() && !self.allowed_https_hosts.contains(&host) {
|
||||||
|
return Err(Status::permission_denied(format!(
|
||||||
|
"source_url host {host} is not in artifacts.allowed_https_hosts",
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
let stdout = String::from_utf8(output.stdout)
|
|
||||||
.map_err(|e| Status::internal(format!("invalid sha256sum output: {e}")))?;
|
let port = url.port_or_known_default().unwrap_or(443);
|
||||||
stdout
|
let resolved = tokio::net::lookup_host((host.as_str(), port))
|
||||||
.split_whitespace()
|
.await
|
||||||
.next()
|
.map_err(|e| Status::unavailable(format!("failed to resolve source_url host: {e}")))?;
|
||||||
.map(str::to_string)
|
let mut found_any = false;
|
||||||
.ok_or_else(|| Status::internal("sha256sum output missing digest"))
|
for addr in resolved {
|
||||||
|
found_any = true;
|
||||||
|
let ip = addr.ip();
|
||||||
|
if !is_public_ip(ip) {
|
||||||
|
return Err(Status::permission_denied(format!(
|
||||||
|
"source_url resolved to a non-public address: {ip}",
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found_any {
|
||||||
|
return Err(Status::failed_precondition(
|
||||||
|
"source_url host did not resolve to any public addresses",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(ValidatedImportUrl { url, host })
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn download_https_source(
|
||||||
|
&self,
|
||||||
|
source: &ValidatedImportUrl,
|
||||||
|
path: &Path,
|
||||||
|
) -> Result<(), Status> {
|
||||||
|
let temp_path = path.with_extension("download");
|
||||||
|
if tokio::fs::try_exists(&temp_path).await.unwrap_or(false) {
|
||||||
|
let _ = tokio::fs::remove_file(&temp_path).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut current = source.url.clone();
|
||||||
|
let mut redirects_remaining = MAX_IMPORT_REDIRECTS;
|
||||||
|
loop {
|
||||||
|
let response = tokio::time::timeout(
|
||||||
|
DEFAULT_HTTP_SEND_TIMEOUT,
|
||||||
|
self.http_client.get(current.clone()).send(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| Status::deadline_exceeded("timed out waiting for source_url response"))?
|
||||||
|
.map_err(|e| Status::unavailable(format!("failed to download image source: {e}")))?;
|
||||||
|
|
||||||
|
if response.status().is_redirection() {
|
||||||
|
if redirects_remaining == 0 {
|
||||||
|
return Err(Status::failed_precondition(
|
||||||
|
"source_url redirect limit exceeded",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let location = response
|
||||||
|
.headers()
|
||||||
|
.get(LOCATION)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
Status::failed_precondition(
|
||||||
|
"source_url redirect response did not include a Location header",
|
||||||
|
)
|
||||||
|
})?
|
||||||
|
.to_str()
|
||||||
|
.map_err(|e| {
|
||||||
|
Status::failed_precondition(format!(
|
||||||
|
"invalid redirect Location header in source_url response: {e}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
current = current.join(location).map_err(|e| {
|
||||||
|
Status::failed_precondition(format!(
|
||||||
|
"failed to resolve redirect target from source_url: {e}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
self.validate_import_url(current.as_ref()).await?;
|
||||||
|
redirects_remaining -= 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if !response.status().is_success() {
|
||||||
|
return Err(Status::failed_precondition(format!(
|
||||||
|
"image download failed with HTTP {}",
|
||||||
|
response.status()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(content_length) = response.content_length() {
|
||||||
|
if content_length > self.max_image_import_size_bytes {
|
||||||
|
return Err(Status::resource_exhausted(format!(
|
||||||
|
"image download exceeds the configured maximum size of {} bytes",
|
||||||
|
self.max_image_import_size_bytes
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut file = tokio::fs::File::create(&temp_path).await.map_err(|e| {
|
||||||
|
Status::internal(format!(
|
||||||
|
"failed to create downloaded image {}: {e}",
|
||||||
|
temp_path.display()
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let mut response = response;
|
||||||
|
let mut downloaded = 0u64;
|
||||||
|
while let Some(chunk) = response.chunk().await.map_err(|e| {
|
||||||
|
Status::unavailable(format!("failed to read image response body: {e}"))
|
||||||
|
})? {
|
||||||
|
downloaded = downloaded.saturating_add(chunk.len() as u64);
|
||||||
|
if downloaded > self.max_image_import_size_bytes {
|
||||||
|
let _ = tokio::fs::remove_file(&temp_path).await;
|
||||||
|
return Err(Status::resource_exhausted(format!(
|
||||||
|
"image download exceeds the configured maximum size of {} bytes",
|
||||||
|
self.max_image_import_size_bytes
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
file.write_all(&chunk).await.map_err(|e| {
|
||||||
|
Status::internal(format!(
|
||||||
|
"failed to write downloaded image {}: {e}",
|
||||||
|
temp_path.display()
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
file.flush().await.map_err(|e| {
|
||||||
|
Status::internal(format!(
|
||||||
|
"failed to flush downloaded image {}: {e}",
|
||||||
|
temp_path.display()
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
drop(file);
|
||||||
|
tokio::fs::rename(&temp_path, path).await.map_err(|e| {
|
||||||
|
Status::internal(format!(
|
||||||
|
"failed to finalize downloaded image {}: {e}",
|
||||||
|
path.display()
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
ensure_cache_file_permissions(path).await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_object_ignore_not_found(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
token: &str,
|
||||||
|
) -> Result<(), Status> {
|
||||||
|
let mut client = self.object_client().await?;
|
||||||
|
let mut request = Request::new(DeleteObjectRequest {
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
key: key.to_string(),
|
||||||
|
version_id: String::new(),
|
||||||
|
});
|
||||||
|
attach_bearer(&mut request, token)?;
|
||||||
|
match client.delete_object(request).await {
|
||||||
|
Ok(_) => Ok(()),
|
||||||
|
Err(status) if status.code() == Code::NotFound => Ok(()),
|
||||||
|
Err(status) => Err(Status::from_error(Box::new(status))),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn bucket_client(&self) -> Result<BucketServiceClient<Channel>, Status> {
|
async fn bucket_client(&self) -> Result<BucketServiceClient<Channel>, Status> {
|
||||||
|
|
@ -832,12 +1186,22 @@ impl ArtifactStore {
|
||||||
Ok(token)
|
Ok(token)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn image_path(&self, image_id: &str) -> PathBuf {
|
fn image_path(&self, image_id: &str) -> Result<PathBuf, Status> {
|
||||||
self.image_cache_dir.join(format!("{image_id}.qcow2"))
|
Ok(self
|
||||||
|
.image_cache_dir
|
||||||
|
.join(format!("{}.qcow2", validated_image_id(image_id)?)))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn raw_image_path(&self, image_id: &str) -> PathBuf {
|
fn raw_image_path(&self, image_id: &str) -> Result<PathBuf, Status> {
|
||||||
self.image_cache_dir.join(format!("{image_id}.raw"))
|
Ok(self
|
||||||
|
.image_cache_dir
|
||||||
|
.join(format!("{}.raw", validated_image_id(image_id)?)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn staging_path(&self, image_id: &str) -> Result<PathBuf, Status> {
|
||||||
|
Ok(self
|
||||||
|
.image_cache_dir
|
||||||
|
.join(format!("{}.source", validated_image_id(image_id)?)))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn abort_multipart_upload(
|
async fn abort_multipart_upload(
|
||||||
|
|
@ -915,35 +1279,6 @@ async fn next_uploaded_part(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn multipart_upload_concurrency() -> usize {
|
|
||||||
std::env::var("PLASMAVMC_LIGHTNINGSTOR_MULTIPART_CONCURRENCY")
|
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.parse::<usize>().ok())
|
|
||||||
.map(|value| value.clamp(1, MAX_MULTIPART_UPLOAD_CONCURRENCY))
|
|
||||||
.unwrap_or(DEFAULT_MULTIPART_UPLOAD_CONCURRENCY)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn multipart_upload_part_size() -> usize {
|
|
||||||
std::env::var("PLASMAVMC_LIGHTNINGSTOR_MULTIPART_PART_SIZE")
|
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.parse::<usize>().ok())
|
|
||||||
.map(|value| {
|
|
||||||
value.clamp(
|
|
||||||
MIN_MULTIPART_UPLOAD_PART_SIZE,
|
|
||||||
MAX_MULTIPART_UPLOAD_PART_SIZE,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.unwrap_or(DEFAULT_MULTIPART_UPLOAD_PART_SIZE)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn raw_image_convert_parallelism() -> usize {
|
|
||||||
std::env::var("PLASMAVMC_RAW_IMAGE_CONVERT_PARALLELISM")
|
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.parse::<usize>().ok())
|
|
||||||
.map(|value| value.clamp(1, 64))
|
|
||||||
.unwrap_or(DEFAULT_RAW_IMAGE_CONVERT_PARALLELISM)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn qemu_img_convert_to_qcow2_args(
|
fn qemu_img_convert_to_qcow2_args(
|
||||||
source: &Path,
|
source: &Path,
|
||||||
destination: &Path,
|
destination: &Path,
|
||||||
|
|
@ -1008,8 +1343,74 @@ fn sanitize_identifier(value: &str) -> String {
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn image_object_key(org_id: &str, project_id: &str, image_id: &str) -> String {
|
fn image_object_key(org_id: &str, project_id: &str, image_id: &str) -> Result<String, Status> {
|
||||||
format!("{org_id}/{project_id}/{image_id}.qcow2")
|
Ok(format!(
|
||||||
|
"{org_id}/{project_id}/{}.qcow2",
|
||||||
|
validated_image_id(image_id)?
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn staging_object_key(org_id: &str, project_id: &str, image_id: &str) -> Result<String, Status> {
|
||||||
|
Ok(format!(
|
||||||
|
"{org_id}/{project_id}/uploads/{}.source",
|
||||||
|
validated_image_id(image_id)?
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validated_image_id(image_id: &str) -> Result<&str, Status> {
|
||||||
|
uuid::Uuid::parse_str(image_id)
|
||||||
|
.map(|_| image_id)
|
||||||
|
.map_err(|_| Status::invalid_argument("image_id must be a UUID"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_public_ip(ip: IpAddr) -> bool {
|
||||||
|
match ip {
|
||||||
|
IpAddr::V4(ip) => {
|
||||||
|
!(ip.is_private()
|
||||||
|
|| ip.is_loopback()
|
||||||
|
|| ip.is_link_local()
|
||||||
|
|| ip.is_multicast()
|
||||||
|
|| ip.is_broadcast()
|
||||||
|
|| ip.is_documentation()
|
||||||
|
|| ip.is_unspecified())
|
||||||
|
}
|
||||||
|
IpAddr::V6(ip) => {
|
||||||
|
!(ip.is_loopback()
|
||||||
|
|| ip.is_unspecified()
|
||||||
|
|| ip.is_multicast()
|
||||||
|
|| ip.is_unique_local()
|
||||||
|
|| ip.is_unicast_link_local())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_binary_path(
|
||||||
|
configured_path: Option<&Path>,
|
||||||
|
binary_name: &str,
|
||||||
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
||||||
|
let candidate = match configured_path {
|
||||||
|
Some(path) => path.to_path_buf(),
|
||||||
|
None => std::env::var_os("PATH")
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|paths| std::env::split_paths(&paths).collect::<Vec<_>>())
|
||||||
|
.map(|entry| entry.join(binary_name))
|
||||||
|
.find(|candidate| candidate.exists())
|
||||||
|
.ok_or_else(|| {
|
||||||
|
std::io::Error::new(
|
||||||
|
std::io::ErrorKind::NotFound,
|
||||||
|
format!("failed to locate {binary_name} in PATH"),
|
||||||
|
)
|
||||||
|
})?,
|
||||||
|
};
|
||||||
|
let metadata = std::fs::metadata(&candidate)?;
|
||||||
|
if !metadata.is_file() {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidInput,
|
||||||
|
format!("{} is not a regular file", candidate.display()),
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
Ok(candidate)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn ensure_cache_dir_permissions(path: &Path) -> Result<(), Status> {
|
async fn ensure_cache_dir_permissions(path: &Path) -> Result<(), Status> {
|
||||||
|
|
@ -1103,4 +1504,13 @@ mod tests {
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn image_object_key_rejects_non_uuid_identifiers() {
|
||||||
|
assert!(image_object_key("org", "project", "../passwd").is_err());
|
||||||
|
assert_eq!(
|
||||||
|
image_object_key("org", "project", "11111111-1111-1111-1111-111111111111").unwrap(),
|
||||||
|
"org/project/11111111-1111-1111-1111-111111111111.qcow2"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,9 @@
|
||||||
//! Server configuration
|
//! Server configuration
|
||||||
|
|
||||||
|
use plasmavmc_types::{FireCrackerConfig, KvmConfig};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use plasmavmc_types::{FireCrackerConfig, KvmConfig};
|
use std::path::PathBuf;
|
||||||
|
|
||||||
/// TLS configuration
|
/// TLS configuration
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
|
@ -42,12 +43,332 @@ pub struct ServerConfig {
|
||||||
/// Configuration for FireCracker backend
|
/// Configuration for FireCracker backend
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub firecracker: FireCrackerConfig,
|
pub firecracker: FireCrackerConfig,
|
||||||
|
/// Durable/shared state configuration
|
||||||
|
#[serde(default)]
|
||||||
|
pub storage: StorageRuntimeConfig,
|
||||||
|
/// Control-plane and agent heartbeat configuration
|
||||||
|
#[serde(default)]
|
||||||
|
pub agent: AgentRuntimeConfig,
|
||||||
|
/// External service integrations
|
||||||
|
#[serde(default)]
|
||||||
|
pub integrations: IntegrationConfig,
|
||||||
|
/// State watcher and VM watch polling configuration
|
||||||
|
#[serde(default)]
|
||||||
|
pub watcher: WatcherRuntimeConfig,
|
||||||
|
/// Background health monitoring and failover configuration
|
||||||
|
#[serde(default)]
|
||||||
|
pub health: HealthRuntimeConfig,
|
||||||
|
/// Artifact storage and image import/export configuration
|
||||||
|
#[serde(default)]
|
||||||
|
pub artifacts: ArtifactStoreConfig,
|
||||||
|
/// Persistent volume runtime configuration
|
||||||
|
#[serde(default)]
|
||||||
|
pub volumes: VolumeRuntimeConfig,
|
||||||
|
/// Default hypervisor used when requests do not specify one
|
||||||
|
#[serde(default)]
|
||||||
|
pub default_hypervisor: DefaultHypervisor,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_http_addr() -> SocketAddr {
|
fn default_http_addr() -> SocketAddr {
|
||||||
"127.0.0.1:8084".parse().unwrap()
|
"127.0.0.1:8084".parse().unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum StorageBackendKind {
|
||||||
|
#[default]
|
||||||
|
Flaredb,
|
||||||
|
File,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct StorageRuntimeConfig {
|
||||||
|
#[serde(default)]
|
||||||
|
pub backend: StorageBackendKind,
|
||||||
|
#[serde(default)]
|
||||||
|
pub flaredb_endpoint: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub chainfire_endpoint: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub state_path: Option<PathBuf>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for StorageRuntimeConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
backend: StorageBackendKind::default(),
|
||||||
|
flaredb_endpoint: None,
|
||||||
|
chainfire_endpoint: None,
|
||||||
|
state_path: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct AgentRuntimeConfig {
|
||||||
|
#[serde(default)]
|
||||||
|
pub control_plane_addr: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub advertise_endpoint: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub node_id: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub node_name: Option<String>,
|
||||||
|
#[serde(default = "default_agent_heartbeat_interval_secs")]
|
||||||
|
pub heartbeat_interval_secs: u64,
|
||||||
|
#[serde(default = "default_shared_live_migration")]
|
||||||
|
pub shared_live_migration: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_agent_heartbeat_interval_secs() -> u64 {
|
||||||
|
5
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_shared_live_migration() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for AgentRuntimeConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
control_plane_addr: None,
|
||||||
|
advertise_endpoint: None,
|
||||||
|
node_id: None,
|
||||||
|
node_name: None,
|
||||||
|
heartbeat_interval_secs: default_agent_heartbeat_interval_secs(),
|
||||||
|
shared_live_migration: default_shared_live_migration(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
|
pub struct IntegrationConfig {
|
||||||
|
#[serde(default)]
|
||||||
|
pub prismnet_endpoint: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub creditservice_endpoint: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct WatcherRuntimeConfig {
|
||||||
|
#[serde(default)]
|
||||||
|
pub enabled: bool,
|
||||||
|
#[serde(default = "default_state_watcher_poll_interval_ms")]
|
||||||
|
pub poll_interval_ms: u64,
|
||||||
|
#[serde(default = "default_vm_watch_poll_interval_ms")]
|
||||||
|
pub vm_watch_poll_interval_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_state_watcher_poll_interval_ms() -> u64 {
|
||||||
|
1000
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_vm_watch_poll_interval_ms() -> u64 {
|
||||||
|
500
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for WatcherRuntimeConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
enabled: false,
|
||||||
|
poll_interval_ms: default_state_watcher_poll_interval_ms(),
|
||||||
|
vm_watch_poll_interval_ms: default_vm_watch_poll_interval_ms(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct HealthRuntimeConfig {
|
||||||
|
#[serde(default)]
|
||||||
|
pub vm_monitor_interval_secs: Option<u64>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub node_monitor_interval_secs: Option<u64>,
|
||||||
|
#[serde(default = "default_node_heartbeat_timeout_secs")]
|
||||||
|
pub node_heartbeat_timeout_secs: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub auto_restart: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
pub failover_enabled: bool,
|
||||||
|
#[serde(default = "default_failover_min_interval_secs")]
|
||||||
|
pub failover_min_interval_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_node_heartbeat_timeout_secs() -> u64 {
|
||||||
|
60
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_failover_min_interval_secs() -> u64 {
|
||||||
|
60
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for HealthRuntimeConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
vm_monitor_interval_secs: None,
|
||||||
|
node_monitor_interval_secs: None,
|
||||||
|
node_heartbeat_timeout_secs: default_node_heartbeat_timeout_secs(),
|
||||||
|
auto_restart: false,
|
||||||
|
failover_enabled: false,
|
||||||
|
failover_min_interval_secs: default_failover_min_interval_secs(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct ArtifactStoreConfig {
|
||||||
|
#[serde(default)]
|
||||||
|
pub lightningstor_endpoint: Option<String>,
|
||||||
|
#[serde(default = "default_image_cache_dir")]
|
||||||
|
pub image_cache_dir: PathBuf,
|
||||||
|
#[serde(default = "default_image_bucket")]
|
||||||
|
pub image_bucket: String,
|
||||||
|
#[serde(default = "default_multipart_upload_concurrency")]
|
||||||
|
pub multipart_upload_concurrency: usize,
|
||||||
|
#[serde(default = "default_multipart_upload_part_size")]
|
||||||
|
pub multipart_upload_part_size: usize,
|
||||||
|
#[serde(default = "default_raw_image_convert_parallelism")]
|
||||||
|
pub raw_image_convert_parallelism: usize,
|
||||||
|
#[serde(default = "default_image_import_connect_timeout_secs")]
|
||||||
|
pub image_import_connect_timeout_secs: u64,
|
||||||
|
#[serde(default = "default_image_import_timeout_secs")]
|
||||||
|
pub image_import_timeout_secs: u64,
|
||||||
|
#[serde(default = "default_max_image_import_size_bytes")]
|
||||||
|
pub max_image_import_size_bytes: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub allowed_https_hosts: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub qemu_img_path: Option<PathBuf>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_image_cache_dir() -> PathBuf {
|
||||||
|
PathBuf::from("/var/lib/plasmavmc/images")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_image_bucket() -> String {
|
||||||
|
"plasmavmc-images".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_multipart_upload_concurrency() -> usize {
|
||||||
|
4
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_multipart_upload_part_size() -> usize {
|
||||||
|
32 * 1024 * 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_raw_image_convert_parallelism() -> usize {
|
||||||
|
8
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_image_import_connect_timeout_secs() -> u64 {
|
||||||
|
5
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_image_import_timeout_secs() -> u64 {
|
||||||
|
60 * 60
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_max_image_import_size_bytes() -> u64 {
|
||||||
|
64 * 1024 * 1024 * 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ArtifactStoreConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
lightningstor_endpoint: None,
|
||||||
|
image_cache_dir: default_image_cache_dir(),
|
||||||
|
image_bucket: default_image_bucket(),
|
||||||
|
multipart_upload_concurrency: default_multipart_upload_concurrency(),
|
||||||
|
multipart_upload_part_size: default_multipart_upload_part_size(),
|
||||||
|
raw_image_convert_parallelism: default_raw_image_convert_parallelism(),
|
||||||
|
image_import_connect_timeout_secs: default_image_import_connect_timeout_secs(),
|
||||||
|
image_import_timeout_secs: default_image_import_timeout_secs(),
|
||||||
|
max_image_import_size_bytes: default_max_image_import_size_bytes(),
|
||||||
|
allowed_https_hosts: Vec::new(),
|
||||||
|
qemu_img_path: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
|
pub struct CoronaFsConfig {
|
||||||
|
#[serde(default)]
|
||||||
|
pub controller_endpoint: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub node_endpoint: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub endpoint: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub node_local_attach: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct CephConfig {
|
||||||
|
#[serde(default)]
|
||||||
|
pub monitors: Vec<String>,
|
||||||
|
#[serde(default = "default_ceph_cluster_id")]
|
||||||
|
pub cluster_id: String,
|
||||||
|
#[serde(default = "default_ceph_user")]
|
||||||
|
pub user: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub secret: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_ceph_cluster_id() -> String {
|
||||||
|
"default".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_ceph_user() -> String {
|
||||||
|
"admin".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for CephConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
monitors: Vec::new(),
|
||||||
|
cluster_id: default_ceph_cluster_id(),
|
||||||
|
user: default_ceph_user(),
|
||||||
|
secret: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct VolumeRuntimeConfig {
|
||||||
|
#[serde(default = "default_managed_volume_root")]
|
||||||
|
pub managed_volume_root: PathBuf,
|
||||||
|
#[serde(default)]
|
||||||
|
pub ceph: CephConfig,
|
||||||
|
#[serde(default)]
|
||||||
|
pub coronafs: CoronaFsConfig,
|
||||||
|
#[serde(default)]
|
||||||
|
pub qemu_img_path: Option<PathBuf>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_managed_volume_root() -> PathBuf {
|
||||||
|
PathBuf::from("/var/lib/plasmavmc/managed-volumes")
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for VolumeRuntimeConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
managed_volume_root: default_managed_volume_root(),
|
||||||
|
ceph: CephConfig::default(),
|
||||||
|
coronafs: CoronaFsConfig::default(),
|
||||||
|
qemu_img_path: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum DefaultHypervisor {
|
||||||
|
#[default]
|
||||||
|
Kvm,
|
||||||
|
Firecracker,
|
||||||
|
Mvisor,
|
||||||
|
}
|
||||||
|
|
||||||
/// Authentication configuration
|
/// Authentication configuration
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct AuthConfig {
|
pub struct AuthConfig {
|
||||||
|
|
@ -78,6 +399,14 @@ impl Default for ServerConfig {
|
||||||
auth: AuthConfig::default(),
|
auth: AuthConfig::default(),
|
||||||
kvm: KvmConfig::default(),
|
kvm: KvmConfig::default(),
|
||||||
firecracker: FireCrackerConfig::default(),
|
firecracker: FireCrackerConfig::default(),
|
||||||
|
storage: StorageRuntimeConfig::default(),
|
||||||
|
agent: AgentRuntimeConfig::default(),
|
||||||
|
integrations: IntegrationConfig::default(),
|
||||||
|
watcher: WatcherRuntimeConfig::default(),
|
||||||
|
health: HealthRuntimeConfig::default(),
|
||||||
|
artifacts: ArtifactStoreConfig::default(),
|
||||||
|
volumes: VolumeRuntimeConfig::default(),
|
||||||
|
default_hypervisor: DefaultHypervisor::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ use plasmavmc_api::proto::{
|
||||||
use plasmavmc_firecracker::FireCrackerBackend;
|
use plasmavmc_firecracker::FireCrackerBackend;
|
||||||
use plasmavmc_hypervisor::HypervisorRegistry;
|
use plasmavmc_hypervisor::HypervisorRegistry;
|
||||||
use plasmavmc_kvm::KvmBackend;
|
use plasmavmc_kvm::KvmBackend;
|
||||||
use plasmavmc_server::config::ServerConfig;
|
use plasmavmc_server::config::{AgentRuntimeConfig, ServerConfig};
|
||||||
use plasmavmc_server::watcher::{StateSynchronizer, StateWatcher, WatcherConfig};
|
use plasmavmc_server::watcher::{StateSynchronizer, StateWatcher, WatcherConfig};
|
||||||
use plasmavmc_server::VmServiceImpl;
|
use plasmavmc_server::VmServiceImpl;
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
|
|
@ -83,16 +83,19 @@ async fn start_agent_heartbeat(
|
||||||
supported_volume_drivers: Vec<i32>,
|
supported_volume_drivers: Vec<i32>,
|
||||||
supported_storage_classes: Vec<String>,
|
supported_storage_classes: Vec<String>,
|
||||||
shared_live_migration: bool,
|
shared_live_migration: bool,
|
||||||
|
agent_config: &AgentRuntimeConfig,
|
||||||
) {
|
) {
|
||||||
let Some(control_plane_addr) = std::env::var("PLASMAVMC_CONTROL_PLANE_ADDR")
|
let Some(control_plane_addr) = agent_config
|
||||||
.ok()
|
.control_plane_addr
|
||||||
|
.as_ref()
|
||||||
.map(|value| value.trim().to_string())
|
.map(|value| value.trim().to_string())
|
||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
else {
|
else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
let Some(node_id) = std::env::var("PLASMAVMC_NODE_ID")
|
let Some(node_id) = agent_config
|
||||||
.ok()
|
.node_id
|
||||||
|
.as_ref()
|
||||||
.map(|value| value.trim().to_string())
|
.map(|value| value.trim().to_string())
|
||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
else {
|
else {
|
||||||
|
|
@ -100,19 +103,19 @@ async fn start_agent_heartbeat(
|
||||||
};
|
};
|
||||||
|
|
||||||
let endpoint = normalize_endpoint(&control_plane_addr);
|
let endpoint = normalize_endpoint(&control_plane_addr);
|
||||||
let advertise_endpoint = std::env::var("PLASMAVMC_ENDPOINT_ADVERTISE")
|
let advertise_endpoint = agent_config
|
||||||
.ok()
|
.advertise_endpoint
|
||||||
|
.as_ref()
|
||||||
.map(|value| value.trim().to_string())
|
.map(|value| value.trim().to_string())
|
||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
.unwrap_or_else(|| local_addr.to_string());
|
.unwrap_or_else(|| local_addr.to_string());
|
||||||
let node_name = std::env::var("PLASMAVMC_NODE_NAME")
|
let node_name = agent_config
|
||||||
.ok()
|
.node_name
|
||||||
|
.as_ref()
|
||||||
|
.cloned()
|
||||||
.filter(|value| !value.trim().is_empty())
|
.filter(|value| !value.trim().is_empty())
|
||||||
.unwrap_or_else(|| node_id.clone());
|
.unwrap_or_else(|| node_id.clone());
|
||||||
let heartbeat_secs = std::env::var("PLASMAVMC_NODE_HEARTBEAT_INTERVAL_SECS")
|
let heartbeat_secs = agent_config.heartbeat_interval_secs.max(1);
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.parse::<u64>().ok())
|
|
||||||
.unwrap_or(5);
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let mut ticker = tokio::time::interval(Duration::from_secs(heartbeat_secs));
|
let mut ticker = tokio::time::interval(Duration::from_secs(heartbeat_secs));
|
||||||
|
|
@ -222,7 +225,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let registry = Arc::new(HypervisorRegistry::new());
|
let registry = Arc::new(HypervisorRegistry::new());
|
||||||
|
|
||||||
// Register KVM backend (always available)
|
// Register KVM backend (always available)
|
||||||
let kvm_backend = Arc::new(KvmBackend::with_defaults());
|
let kvm_backend = Arc::new(KvmBackend::new(
|
||||||
|
config
|
||||||
|
.kvm
|
||||||
|
.qemu_path
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| PathBuf::from("/usr/bin/qemu-system-x86_64")),
|
||||||
|
config
|
||||||
|
.kvm
|
||||||
|
.runtime_dir
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| PathBuf::from("/run/libvirt/plasmavmc")),
|
||||||
|
));
|
||||||
registry.register(kvm_backend);
|
registry.register(kvm_backend);
|
||||||
|
|
||||||
// Register FireCracker backend if kernel/rootfs paths are configured (config or env)
|
// Register FireCracker backend if kernel/rootfs paths are configured (config or env)
|
||||||
|
|
@ -285,17 +299,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
registry,
|
registry,
|
||||||
auth_service.clone(),
|
auth_service.clone(),
|
||||||
config.auth.iam_server_addr.clone(),
|
config.auth.iam_server_addr.clone(),
|
||||||
|
&config,
|
||||||
)
|
)
|
||||||
.await?,
|
.await?,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Optional: start state watcher for multi-instance HA sync
|
// Optional: start state watcher for multi-instance HA sync
|
||||||
if std::env::var("PLASMAVMC_STATE_WATCHER")
|
if config.watcher.enabled {
|
||||||
.map(|v| matches!(v.as_str(), "1" | "true" | "yes"))
|
let watcher_config = WatcherConfig {
|
||||||
.unwrap_or(false)
|
poll_interval: Duration::from_millis(config.watcher.poll_interval_ms.max(100)),
|
||||||
{
|
buffer_size: 256,
|
||||||
let config = WatcherConfig::default();
|
};
|
||||||
let (watcher, rx) = StateWatcher::new(vm_service.store(), config);
|
let (watcher, rx) = StateWatcher::new(vm_service.store(), watcher_config);
|
||||||
let synchronizer = StateSynchronizer::new(vm_service.clone());
|
let synchronizer = StateSynchronizer::new(vm_service.clone());
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if let Err(e) = watcher.start().await {
|
if let Err(e) = watcher.start().await {
|
||||||
|
|
@ -305,13 +320,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
synchronizer.run(rx).await;
|
synchronizer.run(rx).await;
|
||||||
});
|
});
|
||||||
tracing::info!("State watcher enabled (PLASMAVMC_STATE_WATCHER)");
|
tracing::info!("State watcher enabled");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Optional: start health monitor to refresh VM status periodically
|
// Optional: start health monitor to refresh VM status periodically
|
||||||
if let Some(secs) = std::env::var("PLASMAVMC_HEALTH_MONITOR_INTERVAL_SECS")
|
if let Some(secs) = config
|
||||||
.ok()
|
.health
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
.vm_monitor_interval_secs
|
||||||
|
.filter(|secs| *secs > 0)
|
||||||
{
|
{
|
||||||
if secs > 0 {
|
if secs > 0 {
|
||||||
vm_service
|
vm_service
|
||||||
|
|
@ -321,18 +337,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Optional: start node health monitor to detect stale heartbeats
|
// Optional: start node health monitor to detect stale heartbeats
|
||||||
if let Some(interval_secs) = std::env::var("PLASMAVMC_NODE_HEALTH_MONITOR_INTERVAL_SECS")
|
if let Some(interval_secs) = config
|
||||||
.ok()
|
.health
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
.node_monitor_interval_secs
|
||||||
|
.filter(|secs| *secs > 0)
|
||||||
{
|
{
|
||||||
if interval_secs > 0 {
|
if interval_secs > 0 {
|
||||||
let timeout_secs = std::env::var("PLASMAVMC_NODE_HEARTBEAT_TIMEOUT_SECS")
|
|
||||||
.ok()
|
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
|
||||||
.unwrap_or(60);
|
|
||||||
vm_service.clone().start_node_health_monitor(
|
vm_service.clone().start_node_health_monitor(
|
||||||
Duration::from_secs(interval_secs),
|
Duration::from_secs(interval_secs),
|
||||||
Duration::from_secs(timeout_secs),
|
Duration::from_secs(config.health.node_heartbeat_timeout_secs.max(1)),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -369,6 +382,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
heartbeat_volume_drivers,
|
heartbeat_volume_drivers,
|
||||||
heartbeat_storage_classes,
|
heartbeat_storage_classes,
|
||||||
shared_live_migration,
|
shared_live_migration,
|
||||||
|
&config.agent,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
//! Storage abstraction for VM persistence
|
//! Storage abstraction for VM persistence
|
||||||
|
|
||||||
|
use crate::config::{StorageBackendKind, StorageRuntimeConfig};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use plasmavmc_types::{Image, Node, VirtualMachine, VmHandle, Volume};
|
use plasmavmc_types::{Image, ImageFormat, Node, VirtualMachine, VmHandle, Volume};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
|
@ -16,14 +17,10 @@ pub enum StorageBackend {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl StorageBackend {
|
impl StorageBackend {
|
||||||
pub fn from_env() -> Self {
|
pub fn from_config(config: &StorageRuntimeConfig) -> Self {
|
||||||
match std::env::var("PLASMAVMC_STORAGE_BACKEND")
|
match config.backend {
|
||||||
.as_deref()
|
StorageBackendKind::Flaredb => Self::FlareDB,
|
||||||
.unwrap_or("flaredb")
|
StorageBackendKind::File => Self::File,
|
||||||
{
|
|
||||||
"flaredb" => Self::FlareDB,
|
|
||||||
"file" => Self::File,
|
|
||||||
_ => Self::FlareDB,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -48,6 +45,29 @@ pub enum StorageError {
|
||||||
/// Result type for storage operations
|
/// Result type for storage operations
|
||||||
pub type StorageResult<T> = Result<T, StorageError>;
|
pub type StorageResult<T> = Result<T, StorageError>;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct ImageUploadPart {
|
||||||
|
pub part_number: u32,
|
||||||
|
pub etag: String,
|
||||||
|
pub size_bytes: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct ImageUploadSession {
|
||||||
|
pub session_id: String,
|
||||||
|
pub org_id: String,
|
||||||
|
pub project_id: String,
|
||||||
|
pub image_id: String,
|
||||||
|
pub upload_id: String,
|
||||||
|
pub staging_key: String,
|
||||||
|
pub source_format: ImageFormat,
|
||||||
|
#[serde(default)]
|
||||||
|
pub parts: Vec<ImageUploadPart>,
|
||||||
|
pub committed_size_bytes: u64,
|
||||||
|
pub created_at: u64,
|
||||||
|
pub updated_at: u64,
|
||||||
|
}
|
||||||
|
|
||||||
/// Storage trait for VM persistence
|
/// Storage trait for VM persistence
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
pub trait VmStore: Send + Sync {
|
pub trait VmStore: Send + Sync {
|
||||||
|
|
@ -126,6 +146,35 @@ pub trait VmStore: Send + Sync {
|
||||||
/// List images for a tenant
|
/// List images for a tenant
|
||||||
async fn list_images(&self, org_id: &str, project_id: &str) -> StorageResult<Vec<Image>>;
|
async fn list_images(&self, org_id: &str, project_id: &str) -> StorageResult<Vec<Image>>;
|
||||||
|
|
||||||
|
/// Save an in-progress image upload session.
|
||||||
|
async fn save_image_upload_session(&self, session: &ImageUploadSession) -> StorageResult<()>;
|
||||||
|
|
||||||
|
/// Load an in-progress image upload session.
|
||||||
|
async fn load_image_upload_session(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
session_id: &str,
|
||||||
|
) -> StorageResult<Option<ImageUploadSession>>;
|
||||||
|
|
||||||
|
/// Delete an image upload session.
|
||||||
|
async fn delete_image_upload_session(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
session_id: &str,
|
||||||
|
) -> StorageResult<()>;
|
||||||
|
|
||||||
|
/// List image upload sessions for an image.
|
||||||
|
async fn list_image_upload_sessions(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
) -> StorageResult<Vec<ImageUploadSession>>;
|
||||||
|
|
||||||
/// Save a persistent volume
|
/// Save a persistent volume
|
||||||
async fn save_volume(&self, volume: &Volume) -> StorageResult<()>;
|
async fn save_volume(&self, volume: &Volume) -> StorageResult<()>;
|
||||||
|
|
||||||
|
|
@ -191,6 +240,27 @@ fn image_prefix(org_id: &str, project_id: &str) -> String {
|
||||||
format!("/plasmavmc/images/{}/{}/", org_id, project_id)
|
format!("/plasmavmc/images/{}/{}/", org_id, project_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Build key for image upload session metadata
|
||||||
|
fn image_upload_session_key(
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
session_id: &str,
|
||||||
|
) -> String {
|
||||||
|
format!(
|
||||||
|
"/plasmavmc/image-upload-sessions/{}/{}/{}/{}",
|
||||||
|
org_id, project_id, image_id, session_id
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build prefix for image upload session listing
|
||||||
|
fn image_upload_session_prefix(org_id: &str, project_id: &str, image_id: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"/plasmavmc/image-upload-sessions/{}/{}/{}/",
|
||||||
|
org_id, project_id, image_id
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Build key for volume metadata
|
/// Build key for volume metadata
|
||||||
fn volume_key(org_id: &str, project_id: &str, volume_id: &str) -> String {
|
fn volume_key(org_id: &str, project_id: &str, volume_id: &str) -> String {
|
||||||
format!("/plasmavmc/volumes/{}/{}/{}", org_id, project_id, volume_id)
|
format!("/plasmavmc/volumes/{}/{}/{}", org_id, project_id, volume_id)
|
||||||
|
|
@ -207,14 +277,20 @@ pub struct FlareDBStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FlareDBStore {
|
impl FlareDBStore {
|
||||||
/// Create a new FlareDB store
|
pub async fn new_with_config(config: &StorageRuntimeConfig) -> StorageResult<Self> {
|
||||||
pub async fn new(endpoint: Option<String>) -> StorageResult<Self> {
|
Self::new_with_endpoints(
|
||||||
let endpoint = endpoint.unwrap_or_else(|| {
|
config.flaredb_endpoint.clone(),
|
||||||
std::env::var("PLASMAVMC_FLAREDB_ENDPOINT")
|
config.chainfire_endpoint.clone(),
|
||||||
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
|
)
|
||||||
});
|
.await
|
||||||
let pd_endpoint = std::env::var("PLASMAVMC_CHAINFIRE_ENDPOINT")
|
}
|
||||||
.ok()
|
|
||||||
|
pub async fn new_with_endpoints(
|
||||||
|
endpoint: Option<String>,
|
||||||
|
pd_endpoint: Option<String>,
|
||||||
|
) -> StorageResult<Self> {
|
||||||
|
let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
|
||||||
|
let pd_endpoint = pd_endpoint
|
||||||
.map(|value| normalize_transport_addr(&value))
|
.map(|value| normalize_transport_addr(&value))
|
||||||
.unwrap_or_else(|| endpoint.clone());
|
.unwrap_or_else(|| endpoint.clone());
|
||||||
|
|
||||||
|
|
@ -561,6 +637,58 @@ impl VmStore for FlareDBStore {
|
||||||
Ok(images)
|
Ok(images)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn save_image_upload_session(&self, session: &ImageUploadSession) -> StorageResult<()> {
|
||||||
|
let key = image_upload_session_key(
|
||||||
|
&session.org_id,
|
||||||
|
&session.project_id,
|
||||||
|
&session.image_id,
|
||||||
|
&session.session_id,
|
||||||
|
);
|
||||||
|
let value = serde_json::to_vec(session)?;
|
||||||
|
self.cas_put(&key, value).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_image_upload_session(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
session_id: &str,
|
||||||
|
) -> StorageResult<Option<ImageUploadSession>> {
|
||||||
|
let key = image_upload_session_key(org_id, project_id, image_id, session_id);
|
||||||
|
match self.cas_get(&key).await? {
|
||||||
|
Some(data) => Ok(Some(serde_json::from_slice(&data)?)),
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_image_upload_session(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
session_id: &str,
|
||||||
|
) -> StorageResult<()> {
|
||||||
|
let key = image_upload_session_key(org_id, project_id, image_id, session_id);
|
||||||
|
self.cas_delete(&key).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_image_upload_sessions(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
) -> StorageResult<Vec<ImageUploadSession>> {
|
||||||
|
let prefix = image_upload_session_prefix(org_id, project_id, image_id);
|
||||||
|
let mut sessions = Vec::new();
|
||||||
|
for value in self.cas_scan_values(&prefix).await? {
|
||||||
|
if let Ok(session) = serde_json::from_slice::<ImageUploadSession>(&value) {
|
||||||
|
sessions.push(session);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(sessions)
|
||||||
|
}
|
||||||
|
|
||||||
async fn save_volume(&self, volume: &Volume) -> StorageResult<()> {
|
async fn save_volume(&self, volume: &Volume) -> StorageResult<()> {
|
||||||
let key = volume_key(&volume.org_id, &volume.project_id, &volume.id);
|
let key = volume_key(&volume.org_id, &volume.project_id, &volume.id);
|
||||||
let value = serde_json::to_vec(volume)?;
|
let value = serde_json::to_vec(volume)?;
|
||||||
|
|
@ -653,6 +781,8 @@ struct PersistedState {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
images: Vec<Image>,
|
images: Vec<Image>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
image_upload_sessions: Vec<ImageUploadSession>,
|
||||||
|
#[serde(default)]
|
||||||
volumes: Vec<Volume>,
|
volumes: Vec<Volume>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -841,6 +971,71 @@ impl VmStore for FileStore {
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn save_image_upload_session(&self, session: &ImageUploadSession) -> StorageResult<()> {
|
||||||
|
let mut state = self.load_state().unwrap_or_default();
|
||||||
|
state.image_upload_sessions.retain(|existing| {
|
||||||
|
!(existing.org_id == session.org_id
|
||||||
|
&& existing.project_id == session.project_id
|
||||||
|
&& existing.image_id == session.image_id
|
||||||
|
&& existing.session_id == session.session_id)
|
||||||
|
});
|
||||||
|
state.image_upload_sessions.push(session.clone());
|
||||||
|
self.save_state(&state)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_image_upload_session(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
session_id: &str,
|
||||||
|
) -> StorageResult<Option<ImageUploadSession>> {
|
||||||
|
let state = self.load_state().unwrap_or_default();
|
||||||
|
Ok(state.image_upload_sessions.into_iter().find(|session| {
|
||||||
|
session.org_id == org_id
|
||||||
|
&& session.project_id == project_id
|
||||||
|
&& session.image_id == image_id
|
||||||
|
&& session.session_id == session_id
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_image_upload_session(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
session_id: &str,
|
||||||
|
) -> StorageResult<()> {
|
||||||
|
let mut state = self.load_state().unwrap_or_default();
|
||||||
|
state.image_upload_sessions.retain(|session| {
|
||||||
|
!(session.org_id == org_id
|
||||||
|
&& session.project_id == project_id
|
||||||
|
&& session.image_id == image_id
|
||||||
|
&& session.session_id == session_id)
|
||||||
|
});
|
||||||
|
self.save_state(&state)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_image_upload_sessions(
|
||||||
|
&self,
|
||||||
|
org_id: &str,
|
||||||
|
project_id: &str,
|
||||||
|
image_id: &str,
|
||||||
|
) -> StorageResult<Vec<ImageUploadSession>> {
|
||||||
|
let state = self.load_state().unwrap_or_default();
|
||||||
|
Ok(state
|
||||||
|
.image_upload_sessions
|
||||||
|
.into_iter()
|
||||||
|
.filter(|session| {
|
||||||
|
session.org_id == org_id
|
||||||
|
&& session.project_id == project_id
|
||||||
|
&& session.image_id == image_id
|
||||||
|
})
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
async fn save_volume(&self, volume: &Volume) -> StorageResult<()> {
|
async fn save_volume(&self, volume: &Volume) -> StorageResult<()> {
|
||||||
let mut state = self.load_state().unwrap_or_default();
|
let mut state = self.load_state().unwrap_or_default();
|
||||||
state.volumes.retain(|existing| existing.id != volume.id);
|
state.volumes.retain(|existing| existing.id != volume.id);
|
||||||
|
|
@ -972,4 +1167,68 @@ mod tests {
|
||||||
.expect("current volume should remain");
|
.expect("current volume should remain");
|
||||||
assert_eq!(loaded, current);
|
assert_eq!(loaded, current);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn filestore_round_trips_image_upload_sessions() {
|
||||||
|
let tempdir = tempdir().unwrap();
|
||||||
|
let store = FileStore::new(Some(tempdir.path().join("state.json")));
|
||||||
|
let session = ImageUploadSession {
|
||||||
|
session_id: "11111111-1111-1111-1111-111111111111".to_string(),
|
||||||
|
org_id: "org-1".to_string(),
|
||||||
|
project_id: "project-1".to_string(),
|
||||||
|
image_id: "22222222-2222-2222-2222-222222222222".to_string(),
|
||||||
|
upload_id: "multipart-1".to_string(),
|
||||||
|
staging_key: "org-1/project-1/uploads/22222222-2222-2222-2222-222222222222.source"
|
||||||
|
.to_string(),
|
||||||
|
source_format: ImageFormat::Raw,
|
||||||
|
parts: vec![ImageUploadPart {
|
||||||
|
part_number: 1,
|
||||||
|
etag: "etag-1".to_string(),
|
||||||
|
size_bytes: 4096,
|
||||||
|
}],
|
||||||
|
committed_size_bytes: 4096,
|
||||||
|
created_at: 1,
|
||||||
|
updated_at: 2,
|
||||||
|
};
|
||||||
|
|
||||||
|
store.save_image_upload_session(&session).await.unwrap();
|
||||||
|
|
||||||
|
let loaded = store
|
||||||
|
.load_image_upload_session(
|
||||||
|
&session.org_id,
|
||||||
|
&session.project_id,
|
||||||
|
&session.image_id,
|
||||||
|
&session.session_id,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.expect("session should exist");
|
||||||
|
assert_eq!(loaded, session);
|
||||||
|
|
||||||
|
let listed = store
|
||||||
|
.list_image_upload_sessions(&session.org_id, &session.project_id, &session.image_id)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(listed, vec![session.clone()]);
|
||||||
|
|
||||||
|
store
|
||||||
|
.delete_image_upload_session(
|
||||||
|
&session.org_id,
|
||||||
|
&session.project_id,
|
||||||
|
&session.image_id,
|
||||||
|
&session.session_id,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(store
|
||||||
|
.load_image_upload_session(
|
||||||
|
&session.org_id,
|
||||||
|
&session.project_id,
|
||||||
|
&session.image_id,
|
||||||
|
&session.session_id,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_none());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,9 @@
|
||||||
//! VM Service implementation
|
//! VM Service implementation
|
||||||
|
|
||||||
use crate::artifact_store::ArtifactStore;
|
use crate::artifact_store::ArtifactStore;
|
||||||
|
use crate::config::{DefaultHypervisor, ServerConfig};
|
||||||
use crate::prismnet_client::PrismNETClient;
|
use crate::prismnet_client::PrismNETClient;
|
||||||
use crate::storage::{FileStore, FlareDBStore, StorageBackend, VmStore};
|
use crate::storage::{FileStore, FlareDBStore, ImageUploadSession, StorageBackend, VmStore};
|
||||||
use crate::volume_manager::VolumeManager;
|
use crate::volume_manager::VolumeManager;
|
||||||
use crate::watcher::StateSink;
|
use crate::watcher::StateSink;
|
||||||
use creditservice_client::{Client as CreditServiceClient, ResourceType as CreditResourceType};
|
use creditservice_client::{Client as CreditServiceClient, ResourceType as CreditResourceType};
|
||||||
|
|
@ -11,13 +12,15 @@ use iam_client::client::IamClientConfig;
|
||||||
use iam_client::IamClient;
|
use iam_client::IamClient;
|
||||||
use iam_service_auth::{
|
use iam_service_auth::{
|
||||||
get_tenant_context, resolve_tenant_ids_from_context, resource_for_tenant, AuthService,
|
get_tenant_context, resolve_tenant_ids_from_context, resource_for_tenant, AuthService,
|
||||||
|
TenantContext,
|
||||||
};
|
};
|
||||||
use iam_types::{PolicyBinding, PrincipalRef, Scope};
|
use iam_types::{PolicyBinding, PrincipalKind, PrincipalRef, Scope};
|
||||||
use plasmavmc_api::proto::{
|
use plasmavmc_api::proto::{
|
||||||
disk_source::Source as ProtoDiskSourceKind, image_service_server::ImageService,
|
disk_source::Source as ProtoDiskSourceKind, image_service_server::ImageService,
|
||||||
node_service_server::NodeService, vm_service_client::VmServiceClient,
|
node_service_server::NodeService, vm_service_client::VmServiceClient,
|
||||||
vm_service_server::VmService, volume_service_server::VolumeService,
|
vm_service_server::VmService, volume_service_server::VolumeService, AbortImageUploadRequest,
|
||||||
Architecture as ProtoArchitecture, AttachDiskRequest, AttachNicRequest, CephRbdBacking,
|
Architecture as ProtoArchitecture, AttachDiskRequest, AttachNicRequest,
|
||||||
|
BeginImageUploadRequest, BeginImageUploadResponse, CephRbdBacking, CompleteImageUploadRequest,
|
||||||
CordonNodeRequest, CreateImageRequest, CreateVmRequest, CreateVolumeRequest,
|
CordonNodeRequest, CreateImageRequest, CreateVmRequest, CreateVolumeRequest,
|
||||||
DeleteImageRequest, DeleteVmRequest, DeleteVolumeRequest, DetachDiskRequest, DetachNicRequest,
|
DeleteImageRequest, DeleteVmRequest, DeleteVolumeRequest, DetachDiskRequest, DetachNicRequest,
|
||||||
DiskBus as ProtoDiskBus, DiskCache as ProtoDiskCache, DiskSource as ProtoDiskSource,
|
DiskBus as ProtoDiskBus, DiskCache as ProtoDiskCache, DiskSource as ProtoDiskSource,
|
||||||
|
|
@ -30,9 +33,9 @@ use plasmavmc_api::proto::{
|
||||||
NodeState as ProtoNodeState, OsType as ProtoOsType, PrepareVmMigrationRequest, RebootVmRequest,
|
NodeState as ProtoNodeState, OsType as ProtoOsType, PrepareVmMigrationRequest, RebootVmRequest,
|
||||||
RecoverVmRequest, RegisterExternalVolumeRequest, ResetVmRequest, ResizeVolumeRequest,
|
RecoverVmRequest, RegisterExternalVolumeRequest, ResetVmRequest, ResizeVolumeRequest,
|
||||||
StartVmRequest, StopVmRequest, UncordonNodeRequest, UpdateImageRequest, UpdateVmRequest,
|
StartVmRequest, StopVmRequest, UncordonNodeRequest, UpdateImageRequest, UpdateVmRequest,
|
||||||
VirtualMachine, Visibility as ProtoVisibility, VmEvent, VmEventType as ProtoVmEventType,
|
UploadImagePartRequest, UploadImagePartResponse, VirtualMachine, Visibility as ProtoVisibility,
|
||||||
VmSpec as ProtoVmSpec, VmState as ProtoVmState, VmStatus as ProtoVmStatus,
|
VmEvent, VmEventType as ProtoVmEventType, VmSpec as ProtoVmSpec, VmState as ProtoVmState,
|
||||||
Volume as ProtoVolume, VolumeBacking as ProtoVolumeBacking,
|
VmStatus as ProtoVmStatus, Volume as ProtoVolume, VolumeBacking as ProtoVolumeBacking,
|
||||||
VolumeDriverKind as ProtoVolumeDriverKind, VolumeFormat as ProtoVolumeFormat,
|
VolumeDriverKind as ProtoVolumeDriverKind, VolumeFormat as ProtoVolumeFormat,
|
||||||
VolumeStatus as ProtoVolumeStatus, WatchVmRequest,
|
VolumeStatus as ProtoVolumeStatus, WatchVmRequest,
|
||||||
};
|
};
|
||||||
|
|
@ -91,6 +94,12 @@ pub struct VmServiceImpl {
|
||||||
credit_service: Option<Arc<RwLock<CreditServiceClient>>>,
|
credit_service: Option<Arc<RwLock<CreditServiceClient>>>,
|
||||||
/// Local node identifier (optional)
|
/// Local node identifier (optional)
|
||||||
local_node_id: Option<String>,
|
local_node_id: Option<String>,
|
||||||
|
shared_live_migration: bool,
|
||||||
|
default_hypervisor: HypervisorType,
|
||||||
|
watch_poll_interval: Duration,
|
||||||
|
auto_restart: bool,
|
||||||
|
failover_enabled: bool,
|
||||||
|
failover_min_interval_secs: u64,
|
||||||
artifact_store: Option<Arc<ArtifactStore>>,
|
artifact_store: Option<Arc<ArtifactStore>>,
|
||||||
volume_manager: Arc<VolumeManager>,
|
volume_manager: Arc<VolumeManager>,
|
||||||
iam_client: Arc<IamClient>,
|
iam_client: Arc<IamClient>,
|
||||||
|
|
@ -149,33 +158,34 @@ impl VmServiceImpl {
|
||||||
hypervisor_registry: Arc<HypervisorRegistry>,
|
hypervisor_registry: Arc<HypervisorRegistry>,
|
||||||
auth: Arc<AuthService>,
|
auth: Arc<AuthService>,
|
||||||
iam_endpoint: impl Into<String>,
|
iam_endpoint: impl Into<String>,
|
||||||
|
config: &ServerConfig,
|
||||||
) -> Result<Self, Box<dyn std::error::Error>> {
|
) -> Result<Self, Box<dyn std::error::Error>> {
|
||||||
let backend = StorageBackend::from_env();
|
let backend = StorageBackend::from_config(&config.storage);
|
||||||
let store: Arc<dyn VmStore> = match backend {
|
let store: Arc<dyn VmStore> = match backend {
|
||||||
StorageBackend::FlareDB => match FlareDBStore::new(None).await {
|
StorageBackend::FlareDB => match FlareDBStore::new_with_config(&config.storage).await {
|
||||||
Ok(flaredb_store) => Arc::new(flaredb_store),
|
Ok(flaredb_store) => Arc::new(flaredb_store),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
"Failed to connect to FlareDB, falling back to file storage: {}",
|
"Failed to connect to FlareDB, falling back to file storage: {}",
|
||||||
e
|
e
|
||||||
);
|
);
|
||||||
Arc::new(FileStore::new(None))
|
Arc::new(FileStore::new(config.storage.state_path.clone()))
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
StorageBackend::File => {
|
StorageBackend::File => {
|
||||||
let file_store = FileStore::new(None);
|
let file_store = FileStore::new(config.storage.state_path.clone());
|
||||||
Arc::new(file_store)
|
Arc::new(file_store)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let prismnet_endpoint = std::env::var("PRISMNET_ENDPOINT").ok();
|
let prismnet_endpoint = config.integrations.prismnet_endpoint.clone();
|
||||||
if let Some(ref endpoint) = prismnet_endpoint {
|
if let Some(ref endpoint) = prismnet_endpoint {
|
||||||
tracing::info!("PrismNET integration enabled: {}", endpoint);
|
tracing::info!("PrismNET integration enabled: {}", endpoint);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize CreditService client if endpoint is configured
|
// Initialize CreditService client if endpoint is configured
|
||||||
let credit_service = match std::env::var("CREDITSERVICE_ENDPOINT") {
|
let credit_service = match config.integrations.creditservice_endpoint.as_deref() {
|
||||||
Ok(endpoint) => match CreditServiceClient::connect(&endpoint).await {
|
Some(endpoint) => match CreditServiceClient::connect(endpoint).await {
|
||||||
Ok(client) => {
|
Ok(client) => {
|
||||||
tracing::info!("CreditService admission control enabled: {}", endpoint);
|
tracing::info!("CreditService admission control enabled: {}", endpoint);
|
||||||
Some(Arc::new(RwLock::new(client)))
|
Some(Arc::new(RwLock::new(client)))
|
||||||
|
|
@ -188,13 +198,17 @@ impl VmServiceImpl {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
Err(_) => {
|
None => {
|
||||||
tracing::info!("CREDITSERVICE_ENDPOINT not set, admission control disabled");
|
tracing::info!("CreditService endpoint not configured, admission control disabled");
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let local_node_id = std::env::var("PLASMAVMC_NODE_ID").ok();
|
let local_node_id = config
|
||||||
|
.agent
|
||||||
|
.node_id
|
||||||
|
.clone()
|
||||||
|
.filter(|value| !value.trim().is_empty());
|
||||||
if let Some(ref node_id) = local_node_id {
|
if let Some(ref node_id) = local_node_id {
|
||||||
tracing::info!("Local node ID: {}", node_id);
|
tracing::info!("Local node ID: {}", node_id);
|
||||||
}
|
}
|
||||||
|
|
@ -206,13 +220,25 @@ impl VmServiceImpl {
|
||||||
iam_config = iam_config.without_tls();
|
iam_config = iam_config.without_tls();
|
||||||
}
|
}
|
||||||
let iam_client = Arc::new(IamClient::connect(iam_config).await?);
|
let iam_client = Arc::new(IamClient::connect(iam_config).await?);
|
||||||
let artifact_store = ArtifactStore::from_env(&normalized_iam_endpoint)
|
let artifact_store =
|
||||||
.await?
|
ArtifactStore::from_config(&config.artifacts, &normalized_iam_endpoint)
|
||||||
.map(Arc::new);
|
.await?
|
||||||
|
.map(Arc::new);
|
||||||
if artifact_store.is_some() {
|
if artifact_store.is_some() {
|
||||||
tracing::info!("LightningStor artifact backing enabled for VM disks");
|
tracing::info!("LightningStor artifact backing enabled for VM disks");
|
||||||
}
|
}
|
||||||
let volume_manager = Arc::new(VolumeManager::new(store.clone(), artifact_store.clone()));
|
let volume_manager = Arc::new(VolumeManager::new_with_config(
|
||||||
|
store.clone(),
|
||||||
|
artifact_store.clone(),
|
||||||
|
&config.volumes,
|
||||||
|
local_node_id.clone(),
|
||||||
|
)?);
|
||||||
|
|
||||||
|
let default_hypervisor = match config.default_hypervisor {
|
||||||
|
DefaultHypervisor::Kvm => HypervisorType::Kvm,
|
||||||
|
DefaultHypervisor::Firecracker => HypervisorType::Firecracker,
|
||||||
|
DefaultHypervisor::Mvisor => HypervisorType::Mvisor,
|
||||||
|
};
|
||||||
|
|
||||||
let svc = Self {
|
let svc = Self {
|
||||||
hypervisor_registry,
|
hypervisor_registry,
|
||||||
|
|
@ -224,6 +250,14 @@ impl VmServiceImpl {
|
||||||
prismnet_endpoint,
|
prismnet_endpoint,
|
||||||
credit_service,
|
credit_service,
|
||||||
local_node_id,
|
local_node_id,
|
||||||
|
shared_live_migration: config.agent.shared_live_migration,
|
||||||
|
default_hypervisor,
|
||||||
|
watch_poll_interval: Duration::from_millis(
|
||||||
|
config.watcher.vm_watch_poll_interval_ms.max(100),
|
||||||
|
),
|
||||||
|
auto_restart: config.health.auto_restart,
|
||||||
|
failover_enabled: config.health.failover_enabled,
|
||||||
|
failover_min_interval_secs: config.health.failover_min_interval_secs.max(1),
|
||||||
artifact_store,
|
artifact_store,
|
||||||
volume_manager,
|
volume_manager,
|
||||||
iam_client,
|
iam_client,
|
||||||
|
|
@ -242,10 +276,7 @@ impl VmServiceImpl {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn shared_live_migration(&self) -> bool {
|
pub fn shared_live_migration(&self) -> bool {
|
||||||
std::env::var("PLASMAVMC_SHARED_LIVE_MIGRATION")
|
self.shared_live_migration
|
||||||
.ok()
|
|
||||||
.map(|value| matches!(value.as_str(), "1" | "true" | "yes"))
|
|
||||||
.unwrap_or(true)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn store(&self) -> Arc<dyn VmStore> {
|
pub fn store(&self) -> Arc<dyn VmStore> {
|
||||||
|
|
@ -256,24 +287,12 @@ impl VmServiceImpl {
|
||||||
Status::internal(err.to_string())
|
Status::internal(err.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn map_hv(typ: ProtoHypervisorType) -> HypervisorType {
|
fn map_hv(&self, typ: ProtoHypervisorType) -> HypervisorType {
|
||||||
match typ {
|
match typ {
|
||||||
ProtoHypervisorType::Kvm => HypervisorType::Kvm,
|
ProtoHypervisorType::Kvm => HypervisorType::Kvm,
|
||||||
ProtoHypervisorType::Firecracker => HypervisorType::Firecracker,
|
ProtoHypervisorType::Firecracker => HypervisorType::Firecracker,
|
||||||
ProtoHypervisorType::Mvisor => HypervisorType::Mvisor,
|
ProtoHypervisorType::Mvisor => HypervisorType::Mvisor,
|
||||||
ProtoHypervisorType::Unspecified => {
|
ProtoHypervisorType::Unspecified => self.default_hypervisor,
|
||||||
// Use environment variable for default, fallback to KVM
|
|
||||||
match std::env::var("PLASMAVMC_HYPERVISOR")
|
|
||||||
.as_deref()
|
|
||||||
.map(|s| s.to_lowercase())
|
|
||||||
.as_deref()
|
|
||||||
{
|
|
||||||
Ok("firecracker") => HypervisorType::Firecracker,
|
|
||||||
Ok("kvm") => HypervisorType::Kvm,
|
|
||||||
Ok("mvisor") => HypervisorType::Mvisor,
|
|
||||||
_ => HypervisorType::Kvm, // Default to KVM for backwards compatibility
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -292,13 +311,81 @@ impl VmServiceImpl {
|
||||||
.as_secs()
|
.as_secs()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn watch_poll_interval() -> Duration {
|
fn require_uuid(value: &str, field_name: &str) -> Result<(), Status> {
|
||||||
let poll_interval_ms = std::env::var("PLASMAVMC_VM_WATCH_POLL_INTERVAL_MS")
|
Uuid::parse_str(value)
|
||||||
.ok()
|
.map(|_| ())
|
||||||
.and_then(|value| value.parse::<u64>().ok())
|
.map_err(|_| Status::invalid_argument(format!("{field_name} must be a UUID")))
|
||||||
.unwrap_or(500)
|
}
|
||||||
.max(100);
|
|
||||||
Duration::from_millis(poll_interval_ms)
|
fn validate_disk_reference(disk: &plasmavmc_types::DiskSpec) -> Result<(), Status> {
|
||||||
|
match &disk.source {
|
||||||
|
DiskSource::Image { image_id } => Self::require_uuid(image_id, "image_id"),
|
||||||
|
DiskSource::Volume { volume_id } => Self::require_uuid(volume_id, "volume_id"),
|
||||||
|
DiskSource::Blank => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_vm_disk_references(spec: &plasmavmc_types::VmSpec) -> Result<(), Status> {
|
||||||
|
for disk in &spec.disks {
|
||||||
|
Self::validate_disk_reference(disk)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ensure_internal_rpc(tenant: &TenantContext) -> Result<(), Status> {
|
||||||
|
if tenant.principal_kind != PrincipalKind::ServiceAccount
|
||||||
|
|| !tenant.principal_id.starts_with("plasmavmc-")
|
||||||
|
{
|
||||||
|
return Err(Status::permission_denied(
|
||||||
|
"this RPC is restricted to internal PlasmaVMC service accounts",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn expected_migration_listener_uri(&self, vm_id: &str) -> Result<String, Status> {
|
||||||
|
let node_id = self.local_node_id.as_deref().ok_or_else(|| {
|
||||||
|
Status::failed_precondition("local node id is required for migration preparation")
|
||||||
|
})?;
|
||||||
|
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||||
|
node_id.hash(&mut hasher);
|
||||||
|
vm_id.hash(&mut hasher);
|
||||||
|
let port = 4400 + (hasher.finish() % 1000) as u16;
|
||||||
|
Ok(format!("tcp:0.0.0.0:{port}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_imported_image_metadata(
|
||||||
|
image: &mut Image,
|
||||||
|
imported: &crate::artifact_store::ImportedImage,
|
||||||
|
source_format: ImageFormat,
|
||||||
|
) {
|
||||||
|
image.status = ImageStatus::Available;
|
||||||
|
image.format = imported.format;
|
||||||
|
image.size_bytes = imported.size_bytes;
|
||||||
|
image.checksum = imported.checksum.clone();
|
||||||
|
image.updated_at = Self::now_epoch();
|
||||||
|
image
|
||||||
|
.metadata
|
||||||
|
.insert("source_type".to_string(), imported.source_type.clone());
|
||||||
|
if let Some(host) = &imported.source_host {
|
||||||
|
image
|
||||||
|
.metadata
|
||||||
|
.insert("source_host".to_string(), host.clone());
|
||||||
|
}
|
||||||
|
image.metadata.insert(
|
||||||
|
"artifact_key".to_string(),
|
||||||
|
format!("{}/{}/{}.qcow2", image.org_id, image.project_id, image.id),
|
||||||
|
);
|
||||||
|
if source_format != image.format {
|
||||||
|
image.metadata.insert(
|
||||||
|
"source_format".to_string(),
|
||||||
|
format!("{source_format:?}").to_lowercase(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn watch_poll_interval(&self) -> Duration {
|
||||||
|
self.watch_poll_interval
|
||||||
}
|
}
|
||||||
|
|
||||||
fn vm_values_differ(
|
fn vm_values_differ(
|
||||||
|
|
@ -954,7 +1041,10 @@ impl VmServiceImpl {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn proto_vm_to_types(vm: &VirtualMachine) -> Result<plasmavmc_types::VirtualMachine, Status> {
|
fn proto_vm_to_types(
|
||||||
|
&self,
|
||||||
|
vm: &VirtualMachine,
|
||||||
|
) -> Result<plasmavmc_types::VirtualMachine, Status> {
|
||||||
let spec = Self::proto_spec_to_types(vm.spec.clone());
|
let spec = Self::proto_spec_to_types(vm.spec.clone());
|
||||||
let mut typed = plasmavmc_types::VirtualMachine::new(
|
let mut typed = plasmavmc_types::VirtualMachine::new(
|
||||||
vm.name.clone(),
|
vm.name.clone(),
|
||||||
|
|
@ -965,7 +1055,7 @@ impl VmServiceImpl {
|
||||||
typed.id = VmId::from_uuid(
|
typed.id = VmId::from_uuid(
|
||||||
Uuid::parse_str(&vm.id).map_err(|e| Status::internal(format!("invalid VM id: {e}")))?,
|
Uuid::parse_str(&vm.id).map_err(|e| Status::internal(format!("invalid VM id: {e}")))?,
|
||||||
);
|
);
|
||||||
typed.hypervisor = Self::map_hv(
|
typed.hypervisor = self.map_hv(
|
||||||
ProtoHypervisorType::try_from(vm.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
|
ProtoHypervisorType::try_from(vm.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
|
||||||
);
|
);
|
||||||
typed.node_id = if vm.node_id.is_empty() {
|
typed.node_id = if vm.node_id.is_empty() {
|
||||||
|
|
@ -1690,9 +1780,6 @@ impl VmServiceImpl {
|
||||||
.map(|entry| (entry.key().clone(), entry.value().clone()))
|
.map(|entry| (entry.key().clone(), entry.value().clone()))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let auto_restart = std::env::var("PLASMAVMC_AUTO_RESTART")
|
|
||||||
.map(|v| matches!(v.as_str(), "1" | "true" | "yes"))
|
|
||||||
.unwrap_or(false);
|
|
||||||
let local_node = self.local_node_id.as_deref();
|
let local_node = self.local_node_id.as_deref();
|
||||||
|
|
||||||
for (key, mut vm) in entries {
|
for (key, mut vm) in entries {
|
||||||
|
|
@ -1733,7 +1820,7 @@ impl VmServiceImpl {
|
||||||
|
|
||||||
match backend.status(&handle).await {
|
match backend.status(&handle).await {
|
||||||
Ok(status) => {
|
Ok(status) => {
|
||||||
if auto_restart
|
if self.auto_restart
|
||||||
&& vm.state == VmState::Running
|
&& vm.state == VmState::Running
|
||||||
&& matches!(status.actual_state, VmState::Stopped | VmState::Error)
|
&& matches!(status.actual_state, VmState::Stopped | VmState::Error)
|
||||||
{
|
{
|
||||||
|
|
@ -1812,18 +1899,10 @@ impl VmServiceImpl {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let failover_enabled = std::env::var("PLASMAVMC_FAILOVER_CONTROLLER")
|
|
||||||
.map(|v| matches!(v.as_str(), "1" | "true" | "yes"))
|
|
||||||
.unwrap_or(false);
|
|
||||||
let min_interval_secs = std::env::var("PLASMAVMC_FAILOVER_MIN_INTERVAL_SECS")
|
|
||||||
.ok()
|
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
|
||||||
.unwrap_or(60);
|
|
||||||
|
|
||||||
let mut failed_over: HashSet<String> = HashSet::new();
|
let mut failed_over: HashSet<String> = HashSet::new();
|
||||||
if failover_enabled {
|
if self.failover_enabled {
|
||||||
failed_over = self
|
failed_over = self
|
||||||
.failover_vms_on_unhealthy(&unhealthy, &nodes, min_interval_secs)
|
.failover_vms_on_unhealthy(&unhealthy, &nodes, self.failover_min_interval_secs)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2047,6 +2126,43 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(deleted.event_type, ProtoVmEventType::Deleted as i32);
|
assert_eq!(deleted.event_type, ProtoVmEventType::Deleted as i32);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn internal_rpc_guard_requires_plasmavmc_service_account() {
|
||||||
|
let internal = TenantContext {
|
||||||
|
org_id: "org".to_string(),
|
||||||
|
project_id: "project".to_string(),
|
||||||
|
principal_id: "plasmavmc-org-project".to_string(),
|
||||||
|
principal_name: "plasmavmc".to_string(),
|
||||||
|
principal_kind: PrincipalKind::ServiceAccount,
|
||||||
|
node_id: None,
|
||||||
|
};
|
||||||
|
let external = TenantContext {
|
||||||
|
principal_id: "user-1".to_string(),
|
||||||
|
principal_kind: PrincipalKind::User,
|
||||||
|
..internal.clone()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(VmServiceImpl::ensure_internal_rpc(&internal).is_ok());
|
||||||
|
assert!(VmServiceImpl::ensure_internal_rpc(&external).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn vm_disk_reference_validation_requires_uuid_identifiers() {
|
||||||
|
let mut spec = VmSpec::default();
|
||||||
|
spec.disks.push(plasmavmc_types::DiskSpec {
|
||||||
|
id: "root".to_string(),
|
||||||
|
source: DiskSource::Image {
|
||||||
|
image_id: "../passwd".to_string(),
|
||||||
|
},
|
||||||
|
size_gib: 10,
|
||||||
|
bus: DiskBus::Virtio,
|
||||||
|
cache: DiskCache::Writeback,
|
||||||
|
boot_index: Some(1),
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(VmServiceImpl::validate_vm_disk_references(&spec).is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl StateSink for VmServiceImpl {
|
impl StateSink for VmServiceImpl {
|
||||||
|
|
@ -2121,13 +2237,14 @@ impl VmService for VmServiceImpl {
|
||||||
"CreateVm request"
|
"CreateVm request"
|
||||||
);
|
);
|
||||||
|
|
||||||
let hv = Self::map_hv(
|
let hv = self.map_hv(
|
||||||
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
|
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
|
||||||
);
|
);
|
||||||
if req.spec.is_none() {
|
if req.spec.is_none() {
|
||||||
return Err(Status::invalid_argument("spec is required"));
|
return Err(Status::invalid_argument("spec is required"));
|
||||||
}
|
}
|
||||||
let spec = Self::proto_spec_to_types(req.spec.clone());
|
let spec = Self::proto_spec_to_types(req.spec.clone());
|
||||||
|
Self::validate_vm_disk_references(&spec)?;
|
||||||
if self.is_control_plane_scheduler() {
|
if self.is_control_plane_scheduler() {
|
||||||
if let Some(target) = self
|
if let Some(target) = self
|
||||||
.select_target_node(hv, &req.org_id, &req.project_id, &spec)
|
.select_target_node(hv, &req.org_id, &req.project_id, &spec)
|
||||||
|
|
@ -2137,7 +2254,7 @@ impl VmService for VmServiceImpl {
|
||||||
let forwarded = self
|
let forwarded = self
|
||||||
.forward_create_to_node(endpoint, &req.org_id, &req.project_id, &req)
|
.forward_create_to_node(endpoint, &req.org_id, &req.project_id, &req)
|
||||||
.await?;
|
.await?;
|
||||||
let forwarded_vm = Self::proto_vm_to_types(&forwarded)?;
|
let forwarded_vm = self.proto_vm_to_types(&forwarded)?;
|
||||||
let key = TenantKey::new(
|
let key = TenantKey::new(
|
||||||
&forwarded_vm.org_id,
|
&forwarded_vm.org_id,
|
||||||
&forwarded_vm.project_id,
|
&forwarded_vm.project_id,
|
||||||
|
|
@ -2395,7 +2512,7 @@ impl VmService for VmServiceImpl {
|
||||||
.await
|
.await
|
||||||
.map_err(|status| Status::from_error(Box::new(status)))?
|
.map_err(|status| Status::from_error(Box::new(status)))?
|
||||||
.into_inner();
|
.into_inner();
|
||||||
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
|
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
|
||||||
self.vms.insert(key.clone(), typed_vm.clone());
|
self.vms.insert(key.clone(), typed_vm.clone());
|
||||||
self.persist_vm(&typed_vm).await;
|
self.persist_vm(&typed_vm).await;
|
||||||
return Ok(Response::new(remote_vm));
|
return Ok(Response::new(remote_vm));
|
||||||
|
|
@ -2700,7 +2817,7 @@ impl VmService for VmServiceImpl {
|
||||||
.await
|
.await
|
||||||
.map_err(|status| Status::from_error(Box::new(status)))?
|
.map_err(|status| Status::from_error(Box::new(status)))?
|
||||||
.into_inner();
|
.into_inner();
|
||||||
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
|
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
|
||||||
self.vms.insert(key, typed_vm.clone());
|
self.vms.insert(key, typed_vm.clone());
|
||||||
self.persist_vm(&typed_vm).await;
|
self.persist_vm(&typed_vm).await;
|
||||||
return Ok(Response::new(remote_vm));
|
return Ok(Response::new(remote_vm));
|
||||||
|
|
@ -2802,7 +2919,7 @@ impl VmService for VmServiceImpl {
|
||||||
.await
|
.await
|
||||||
.map_err(|status| Status::from_error(Box::new(status)))?
|
.map_err(|status| Status::from_error(Box::new(status)))?
|
||||||
.into_inner();
|
.into_inner();
|
||||||
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
|
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
|
||||||
self.vms.insert(key, typed_vm.clone());
|
self.vms.insert(key, typed_vm.clone());
|
||||||
self.persist_vm(&typed_vm).await;
|
self.persist_vm(&typed_vm).await;
|
||||||
return Ok(Response::new(remote_vm));
|
return Ok(Response::new(remote_vm));
|
||||||
|
|
@ -2889,7 +3006,7 @@ impl VmService for VmServiceImpl {
|
||||||
.await
|
.await
|
||||||
.map_err(|status| Status::from_error(Box::new(status)))?
|
.map_err(|status| Status::from_error(Box::new(status)))?
|
||||||
.into_inner();
|
.into_inner();
|
||||||
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
|
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
|
||||||
self.vms.insert(key, typed_vm.clone());
|
self.vms.insert(key, typed_vm.clone());
|
||||||
self.persist_vm(&typed_vm).await;
|
self.persist_vm(&typed_vm).await;
|
||||||
return Ok(Response::new(remote_vm));
|
return Ok(Response::new(remote_vm));
|
||||||
|
|
@ -2970,7 +3087,7 @@ impl VmService for VmServiceImpl {
|
||||||
.await
|
.await
|
||||||
.map_err(|status| Status::from_error(Box::new(status)))?
|
.map_err(|status| Status::from_error(Box::new(status)))?
|
||||||
.into_inner();
|
.into_inner();
|
||||||
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
|
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
|
||||||
self.vms.insert(key, typed_vm.clone());
|
self.vms.insert(key, typed_vm.clone());
|
||||||
self.persist_vm(&typed_vm).await;
|
self.persist_vm(&typed_vm).await;
|
||||||
return Ok(Response::new(remote_vm));
|
return Ok(Response::new(remote_vm));
|
||||||
|
|
@ -3061,7 +3178,7 @@ impl VmService for VmServiceImpl {
|
||||||
.await
|
.await
|
||||||
.map_err(|status| Status::from_error(Box::new(status)))?
|
.map_err(|status| Status::from_error(Box::new(status)))?
|
||||||
.into_inner();
|
.into_inner();
|
||||||
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
|
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
|
||||||
self.vms.insert(key, typed_vm.clone());
|
self.vms.insert(key, typed_vm.clone());
|
||||||
self.persist_vm(&typed_vm).await;
|
self.persist_vm(&typed_vm).await;
|
||||||
return Ok(Response::new(remote_vm));
|
return Ok(Response::new(remote_vm));
|
||||||
|
|
@ -3171,7 +3288,7 @@ impl VmService for VmServiceImpl {
|
||||||
return match client.recover_vm(recover_req).await {
|
return match client.recover_vm(recover_req).await {
|
||||||
Ok(remote_vm) => {
|
Ok(remote_vm) => {
|
||||||
let remote_vm = remote_vm.into_inner();
|
let remote_vm = remote_vm.into_inner();
|
||||||
let typed_vm = Self::proto_vm_to_types(&remote_vm)?;
|
let typed_vm = self.proto_vm_to_types(&remote_vm)?;
|
||||||
self.vms.insert(key, typed_vm.clone());
|
self.vms.insert(key, typed_vm.clone());
|
||||||
self.persist_vm(&typed_vm).await;
|
self.persist_vm(&typed_vm).await;
|
||||||
Ok(Response::new(remote_vm))
|
Ok(Response::new(remote_vm))
|
||||||
|
|
@ -3352,6 +3469,7 @@ impl VmService for VmServiceImpl {
|
||||||
request: Request<PrepareVmMigrationRequest>,
|
request: Request<PrepareVmMigrationRequest>,
|
||||||
) -> Result<Response<VirtualMachine>, Status> {
|
) -> Result<Response<VirtualMachine>, Status> {
|
||||||
let tenant = get_tenant_context(&request)?;
|
let tenant = get_tenant_context(&request)?;
|
||||||
|
Self::ensure_internal_rpc(&tenant)?;
|
||||||
let (org_id, project_id) = resolve_tenant_ids_from_context(
|
let (org_id, project_id) = resolve_tenant_ids_from_context(
|
||||||
&tenant,
|
&tenant,
|
||||||
&request.get_ref().org_id,
|
&request.get_ref().org_id,
|
||||||
|
|
@ -3378,6 +3496,12 @@ impl VmService for VmServiceImpl {
|
||||||
if req.listen_uri.is_empty() {
|
if req.listen_uri.is_empty() {
|
||||||
return Err(Status::invalid_argument("listen_uri is required"));
|
return Err(Status::invalid_argument("listen_uri is required"));
|
||||||
}
|
}
|
||||||
|
let expected_listen_uri = self.expected_migration_listener_uri(&req.vm_id)?;
|
||||||
|
if req.listen_uri != expected_listen_uri {
|
||||||
|
return Err(Status::invalid_argument(format!(
|
||||||
|
"listen_uri must exactly match {expected_listen_uri}",
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
self.ensure_destination_slot_available(&req.org_id, &req.project_id, &req.vm_id)
|
self.ensure_destination_slot_available(&req.org_id, &req.project_id, &req.vm_id)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
@ -3385,7 +3509,7 @@ impl VmService for VmServiceImpl {
|
||||||
let vm_uuid = Uuid::parse_str(&req.vm_id)
|
let vm_uuid = Uuid::parse_str(&req.vm_id)
|
||||||
.map_err(|_| Status::invalid_argument("vm_id must be a UUID"))?;
|
.map_err(|_| Status::invalid_argument("vm_id must be a UUID"))?;
|
||||||
|
|
||||||
let hv = Self::map_hv(
|
let hv = self.map_hv(
|
||||||
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
|
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
|
||||||
);
|
);
|
||||||
let backend = self
|
let backend = self
|
||||||
|
|
@ -3399,6 +3523,7 @@ impl VmService for VmServiceImpl {
|
||||||
}
|
}
|
||||||
|
|
||||||
let spec = Self::proto_spec_to_types(req.spec);
|
let spec = Self::proto_spec_to_types(req.spec);
|
||||||
|
Self::validate_vm_disk_references(&spec)?;
|
||||||
let name = if req.name.is_empty() {
|
let name = if req.name.is_empty() {
|
||||||
req.vm_id.clone()
|
req.vm_id.clone()
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -3440,6 +3565,7 @@ impl VmService for VmServiceImpl {
|
||||||
request: Request<RecoverVmRequest>,
|
request: Request<RecoverVmRequest>,
|
||||||
) -> Result<Response<VirtualMachine>, Status> {
|
) -> Result<Response<VirtualMachine>, Status> {
|
||||||
let tenant = get_tenant_context(&request)?;
|
let tenant = get_tenant_context(&request)?;
|
||||||
|
Self::ensure_internal_rpc(&tenant)?;
|
||||||
let (org_id, project_id) = resolve_tenant_ids_from_context(
|
let (org_id, project_id) = resolve_tenant_ids_from_context(
|
||||||
&tenant,
|
&tenant,
|
||||||
&request.get_ref().org_id,
|
&request.get_ref().org_id,
|
||||||
|
|
@ -3469,7 +3595,7 @@ impl VmService for VmServiceImpl {
|
||||||
let vm_uuid = Uuid::parse_str(&req.vm_id)
|
let vm_uuid = Uuid::parse_str(&req.vm_id)
|
||||||
.map_err(|_| Status::invalid_argument("vm_id must be a UUID"))?;
|
.map_err(|_| Status::invalid_argument("vm_id must be a UUID"))?;
|
||||||
|
|
||||||
let hv = Self::map_hv(
|
let hv = self.map_hv(
|
||||||
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
|
ProtoHypervisorType::try_from(req.hypervisor).unwrap_or(ProtoHypervisorType::Kvm),
|
||||||
);
|
);
|
||||||
let backend = self
|
let backend = self
|
||||||
|
|
@ -3481,6 +3607,7 @@ impl VmService for VmServiceImpl {
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let spec = Self::proto_spec_to_types(req.spec);
|
let spec = Self::proto_spec_to_types(req.spec);
|
||||||
|
Self::validate_vm_disk_references(&spec)?;
|
||||||
let name = if req.name.is_empty() {
|
let name = if req.name.is_empty() {
|
||||||
req.vm_id.clone()
|
req.vm_id.clone()
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -3584,6 +3711,7 @@ impl VmService for VmServiceImpl {
|
||||||
.disk
|
.disk
|
||||||
.ok_or_else(|| Status::invalid_argument("disk spec required"))?;
|
.ok_or_else(|| Status::invalid_argument("disk spec required"))?;
|
||||||
let mut disk_spec = Self::proto_disk_to_types(proto_disk);
|
let mut disk_spec = Self::proto_disk_to_types(proto_disk);
|
||||||
|
Self::validate_disk_reference(&disk_spec)?;
|
||||||
if vm.spec.disks.iter().any(|disk| disk.id == disk_spec.id) {
|
if vm.spec.disks.iter().any(|disk| disk.id == disk_spec.id) {
|
||||||
return Err(Status::already_exists("disk already attached"));
|
return Err(Status::already_exists("disk already attached"));
|
||||||
}
|
}
|
||||||
|
|
@ -3858,7 +3986,7 @@ impl VmService for VmServiceImpl {
|
||||||
.ensure_vm_loaded(&req.org_id, &req.project_id, &req.vm_id)
|
.ensure_vm_loaded(&req.org_id, &req.project_id, &req.vm_id)
|
||||||
.await
|
.await
|
||||||
.ok_or_else(|| Status::not_found("VM not found"))?;
|
.ok_or_else(|| Status::not_found("VM not found"))?;
|
||||||
let poll_interval = Self::watch_poll_interval();
|
let poll_interval = self.watch_poll_interval();
|
||||||
let store = Arc::clone(&self.store);
|
let store = Arc::clone(&self.store);
|
||||||
let org_id = req.org_id.clone();
|
let org_id = req.org_id.clone();
|
||||||
let project_id = req.project_id.clone();
|
let project_id = req.project_id.clone();
|
||||||
|
|
@ -3948,6 +4076,9 @@ impl VolumeService for VmServiceImpl {
|
||||||
"size_gib must be greater than zero",
|
"size_gib must be greater than zero",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
if !req.image_id.trim().is_empty() {
|
||||||
|
Self::require_uuid(&req.image_id, "image_id")?;
|
||||||
|
}
|
||||||
|
|
||||||
let driver = Self::map_volume_driver(
|
let driver = Self::map_volume_driver(
|
||||||
ProtoVolumeDriverKind::try_from(req.driver).unwrap_or(ProtoVolumeDriverKind::Managed),
|
ProtoVolumeDriverKind::try_from(req.driver).unwrap_or(ProtoVolumeDriverKind::Managed),
|
||||||
|
|
@ -4206,6 +4337,9 @@ impl ImageService for VmServiceImpl {
|
||||||
if req.source_url.trim().is_empty() {
|
if req.source_url.trim().is_empty() {
|
||||||
return Err(Status::invalid_argument("source_url is required"));
|
return Err(Status::invalid_argument("source_url is required"));
|
||||||
}
|
}
|
||||||
|
if !req.source_url.starts_with("https://") {
|
||||||
|
return Err(Status::invalid_argument("source_url must use https://"));
|
||||||
|
}
|
||||||
let Some(store) = self.artifact_store.as_ref() else {
|
let Some(store) = self.artifact_store.as_ref() else {
|
||||||
return Err(Status::failed_precondition(
|
return Err(Status::failed_precondition(
|
||||||
"LightningStor artifact backing is required for image imports",
|
"LightningStor artifact backing is required for image imports",
|
||||||
|
|
@ -4245,20 +4379,7 @@ impl ImageService for VmServiceImpl {
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(imported) => {
|
Ok(imported) => {
|
||||||
image.status = ImageStatus::Available;
|
Self::apply_imported_image_metadata(&mut image, &imported, source_format);
|
||||||
image.format = imported.format;
|
|
||||||
image.size_bytes = imported.size_bytes;
|
|
||||||
image.checksum = imported.checksum;
|
|
||||||
image.updated_at = Self::now_epoch();
|
|
||||||
image
|
|
||||||
.metadata
|
|
||||||
.insert("source_url".to_string(), req.source_url.clone());
|
|
||||||
if source_format != image.format {
|
|
||||||
image.metadata.insert(
|
|
||||||
"source_format".to_string(),
|
|
||||||
format!("{source_format:?}").to_lowercase(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
self.images.insert(key, image.clone());
|
self.images.insert(key, image.clone());
|
||||||
self.persist_image(&image).await;
|
self.persist_image(&image).await;
|
||||||
Ok(Response::new(Self::types_image_to_proto(&image)))
|
Ok(Response::new(Self::types_image_to_proto(&image)))
|
||||||
|
|
@ -4276,6 +4397,332 @@ impl ImageService for VmServiceImpl {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn begin_image_upload(
|
||||||
|
&self,
|
||||||
|
request: Request<BeginImageUploadRequest>,
|
||||||
|
) -> Result<Response<BeginImageUploadResponse>, Status> {
|
||||||
|
let tenant = get_tenant_context(&request)?;
|
||||||
|
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
|
||||||
|
self.auth
|
||||||
|
.authorize(
|
||||||
|
&tenant,
|
||||||
|
ACTION_IMAGE_CREATE,
|
||||||
|
&resource_for_tenant("image", "*", &org_id, &project_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let mut req = request.into_inner();
|
||||||
|
if req.name.trim().is_empty() {
|
||||||
|
return Err(Status::invalid_argument("name is required"));
|
||||||
|
}
|
||||||
|
let Some(store) = self.artifact_store.as_ref() else {
|
||||||
|
return Err(Status::failed_precondition(
|
||||||
|
"LightningStor artifact backing is required for image uploads",
|
||||||
|
));
|
||||||
|
};
|
||||||
|
|
||||||
|
let source_format = Self::map_image_format(
|
||||||
|
ProtoImageFormat::try_from(req.format).unwrap_or(ProtoImageFormat::Qcow2),
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut image = Image::new(req.name, &org_id, &project_id);
|
||||||
|
image.visibility = Self::map_visibility(
|
||||||
|
ProtoVisibility::try_from(req.visibility).unwrap_or(ProtoVisibility::Private),
|
||||||
|
);
|
||||||
|
image.os_type = Self::map_os_type(
|
||||||
|
ProtoOsType::try_from(req.os_type).unwrap_or(ProtoOsType::Unspecified),
|
||||||
|
);
|
||||||
|
image.os_version = req.os_version;
|
||||||
|
image.architecture = Self::map_architecture(req.architecture);
|
||||||
|
image.min_disk_gib = req.min_disk_gib;
|
||||||
|
image.min_memory_mib = req.min_memory_mib;
|
||||||
|
image.metadata = std::mem::take(&mut req.metadata);
|
||||||
|
image.status = ImageStatus::Uploading;
|
||||||
|
|
||||||
|
let key = TenantKey::new(&org_id, &project_id, &image.id);
|
||||||
|
self.images.insert(key.clone(), image.clone());
|
||||||
|
self.persist_image(&image).await;
|
||||||
|
|
||||||
|
let begin_result = store
|
||||||
|
.begin_image_upload(&org_id, &project_id, &image.id)
|
||||||
|
.await;
|
||||||
|
match begin_result {
|
||||||
|
Ok((upload_id, staging_key)) => {
|
||||||
|
let now = Self::now_epoch();
|
||||||
|
let session = ImageUploadSession {
|
||||||
|
session_id: Uuid::new_v4().to_string(),
|
||||||
|
org_id: org_id.clone(),
|
||||||
|
project_id: project_id.clone(),
|
||||||
|
image_id: image.id.clone(),
|
||||||
|
upload_id,
|
||||||
|
staging_key,
|
||||||
|
source_format,
|
||||||
|
parts: Vec::new(),
|
||||||
|
committed_size_bytes: 0,
|
||||||
|
created_at: now,
|
||||||
|
updated_at: now,
|
||||||
|
};
|
||||||
|
if let Err(error) = self.store.save_image_upload_session(&session).await {
|
||||||
|
let _ = store
|
||||||
|
.abort_image_upload(
|
||||||
|
&org_id,
|
||||||
|
&project_id,
|
||||||
|
&session.staging_key,
|
||||||
|
&session.upload_id,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
image.status = ImageStatus::Error;
|
||||||
|
image.updated_at = Self::now_epoch();
|
||||||
|
image.metadata.insert(
|
||||||
|
"last_error".to_string(),
|
||||||
|
format!("failed to persist image upload session: {error}"),
|
||||||
|
);
|
||||||
|
self.images.insert(key, image.clone());
|
||||||
|
self.persist_image(&image).await;
|
||||||
|
return Err(Status::internal(format!(
|
||||||
|
"failed to persist image upload session: {error}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(Response::new(BeginImageUploadResponse {
|
||||||
|
image: Some(Self::types_image_to_proto(&image)),
|
||||||
|
upload_session_id: session.session_id,
|
||||||
|
minimum_part_size_bytes: store.minimum_upload_part_size(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
image.status = ImageStatus::Error;
|
||||||
|
image.updated_at = Self::now_epoch();
|
||||||
|
image
|
||||||
|
.metadata
|
||||||
|
.insert("last_error".to_string(), error.message().to_string());
|
||||||
|
self.images.insert(key, image.clone());
|
||||||
|
self.persist_image(&image).await;
|
||||||
|
Err(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn upload_image_part(
|
||||||
|
&self,
|
||||||
|
request: Request<UploadImagePartRequest>,
|
||||||
|
) -> Result<Response<UploadImagePartResponse>, Status> {
|
||||||
|
let tenant = get_tenant_context(&request)?;
|
||||||
|
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
|
||||||
|
let req = request.into_inner();
|
||||||
|
Self::require_uuid(&req.image_id, "image_id")?;
|
||||||
|
Self::require_uuid(&req.upload_session_id, "upload_session_id")?;
|
||||||
|
self.auth
|
||||||
|
.authorize(
|
||||||
|
&tenant,
|
||||||
|
ACTION_IMAGE_CREATE,
|
||||||
|
&resource_for_tenant("image", req.image_id.clone(), &org_id, &project_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let Some(store) = self.artifact_store.as_ref() else {
|
||||||
|
return Err(Status::failed_precondition(
|
||||||
|
"LightningStor artifact backing is required for image uploads",
|
||||||
|
));
|
||||||
|
};
|
||||||
|
let Some(mut session) = self
|
||||||
|
.store
|
||||||
|
.load_image_upload_session(&org_id, &project_id, &req.image_id, &req.upload_session_id)
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
Status::internal(format!("failed to load image upload session: {error}"))
|
||||||
|
})?
|
||||||
|
else {
|
||||||
|
return Err(Status::not_found("image upload session not found"));
|
||||||
|
};
|
||||||
|
let Some(image) = self
|
||||||
|
.ensure_image_loaded(&org_id, &project_id, &req.image_id)
|
||||||
|
.await
|
||||||
|
else {
|
||||||
|
return Err(Status::not_found("image not found"));
|
||||||
|
};
|
||||||
|
if image.status != ImageStatus::Uploading {
|
||||||
|
return Err(Status::failed_precondition(
|
||||||
|
"image is not accepting uploaded parts",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let part = store
|
||||||
|
.upload_image_part(
|
||||||
|
&org_id,
|
||||||
|
&project_id,
|
||||||
|
&session.staging_key,
|
||||||
|
&session.upload_id,
|
||||||
|
req.part_number,
|
||||||
|
req.body,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
session
|
||||||
|
.parts
|
||||||
|
.retain(|existing| existing.part_number != part.part_number);
|
||||||
|
session.parts.push(part);
|
||||||
|
session.parts.sort_by_key(|part| part.part_number);
|
||||||
|
session.committed_size_bytes = session.parts.iter().map(|part| part.size_bytes).sum();
|
||||||
|
session.updated_at = Self::now_epoch();
|
||||||
|
self.store
|
||||||
|
.save_image_upload_session(&session)
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
Status::internal(format!("failed to persist image upload session: {error}"))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(Response::new(UploadImagePartResponse {
|
||||||
|
part_number: req.part_number,
|
||||||
|
committed_size_bytes: session.committed_size_bytes,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn complete_image_upload(
|
||||||
|
&self,
|
||||||
|
request: Request<CompleteImageUploadRequest>,
|
||||||
|
) -> Result<Response<ProtoImage>, Status> {
|
||||||
|
let tenant = get_tenant_context(&request)?;
|
||||||
|
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
|
||||||
|
let req = request.into_inner();
|
||||||
|
Self::require_uuid(&req.image_id, "image_id")?;
|
||||||
|
Self::require_uuid(&req.upload_session_id, "upload_session_id")?;
|
||||||
|
self.auth
|
||||||
|
.authorize(
|
||||||
|
&tenant,
|
||||||
|
ACTION_IMAGE_CREATE,
|
||||||
|
&resource_for_tenant("image", req.image_id.clone(), &org_id, &project_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let Some(store) = self.artifact_store.as_ref() else {
|
||||||
|
return Err(Status::failed_precondition(
|
||||||
|
"LightningStor artifact backing is required for image uploads",
|
||||||
|
));
|
||||||
|
};
|
||||||
|
let key = TenantKey::new(&org_id, &project_id, &req.image_id);
|
||||||
|
let Some(mut image) = self
|
||||||
|
.ensure_image_loaded(&org_id, &project_id, &req.image_id)
|
||||||
|
.await
|
||||||
|
else {
|
||||||
|
return Err(Status::not_found("image not found"));
|
||||||
|
};
|
||||||
|
let Some(session) = self
|
||||||
|
.store
|
||||||
|
.load_image_upload_session(&org_id, &project_id, &req.image_id, &req.upload_session_id)
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
Status::internal(format!("failed to load image upload session: {error}"))
|
||||||
|
})?
|
||||||
|
else {
|
||||||
|
return Err(Status::not_found("image upload session not found"));
|
||||||
|
};
|
||||||
|
|
||||||
|
match store
|
||||||
|
.complete_image_upload(
|
||||||
|
&org_id,
|
||||||
|
&project_id,
|
||||||
|
&req.image_id,
|
||||||
|
&session.staging_key,
|
||||||
|
&session.upload_id,
|
||||||
|
&session.parts,
|
||||||
|
session.source_format,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(imported) => {
|
||||||
|
Self::apply_imported_image_metadata(&mut image, &imported, session.source_format);
|
||||||
|
self.images.insert(key, image.clone());
|
||||||
|
self.persist_image(&image).await;
|
||||||
|
let _ = self
|
||||||
|
.store
|
||||||
|
.delete_image_upload_session(
|
||||||
|
&org_id,
|
||||||
|
&project_id,
|
||||||
|
&req.image_id,
|
||||||
|
&req.upload_session_id,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
Ok(Response::new(Self::types_image_to_proto(&image)))
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
image.status = ImageStatus::Error;
|
||||||
|
image.updated_at = Self::now_epoch();
|
||||||
|
image
|
||||||
|
.metadata
|
||||||
|
.insert("last_error".to_string(), error.message().to_string());
|
||||||
|
self.images.insert(key, image.clone());
|
||||||
|
self.persist_image(&image).await;
|
||||||
|
Err(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn abort_image_upload(
|
||||||
|
&self,
|
||||||
|
request: Request<AbortImageUploadRequest>,
|
||||||
|
) -> Result<Response<Empty>, Status> {
|
||||||
|
let tenant = get_tenant_context(&request)?;
|
||||||
|
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
|
||||||
|
let req = request.into_inner();
|
||||||
|
Self::require_uuid(&req.image_id, "image_id")?;
|
||||||
|
Self::require_uuid(&req.upload_session_id, "upload_session_id")?;
|
||||||
|
self.auth
|
||||||
|
.authorize(
|
||||||
|
&tenant,
|
||||||
|
ACTION_IMAGE_CREATE,
|
||||||
|
&resource_for_tenant("image", req.image_id.clone(), &org_id, &project_id),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let Some(store) = self.artifact_store.as_ref() else {
|
||||||
|
return Err(Status::failed_precondition(
|
||||||
|
"LightningStor artifact backing is required for image uploads",
|
||||||
|
));
|
||||||
|
};
|
||||||
|
let Some(session) = self
|
||||||
|
.store
|
||||||
|
.load_image_upload_session(&org_id, &project_id, &req.image_id, &req.upload_session_id)
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
Status::internal(format!("failed to load image upload session: {error}"))
|
||||||
|
})?
|
||||||
|
else {
|
||||||
|
return Err(Status::not_found("image upload session not found"));
|
||||||
|
};
|
||||||
|
store
|
||||||
|
.abort_image_upload(
|
||||||
|
&org_id,
|
||||||
|
&project_id,
|
||||||
|
&session.staging_key,
|
||||||
|
&session.upload_id,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
self.store
|
||||||
|
.delete_image_upload_session(
|
||||||
|
&org_id,
|
||||||
|
&project_id,
|
||||||
|
&req.image_id,
|
||||||
|
&req.upload_session_id,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
Status::internal(format!("failed to delete image upload session: {error}"))
|
||||||
|
})?;
|
||||||
|
if let Some(mut image) = self
|
||||||
|
.ensure_image_loaded(&org_id, &project_id, &req.image_id)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
let key = TenantKey::new(&org_id, &project_id, &req.image_id);
|
||||||
|
image.status = ImageStatus::Error;
|
||||||
|
image.updated_at = Self::now_epoch();
|
||||||
|
image
|
||||||
|
.metadata
|
||||||
|
.insert("last_error".to_string(), "upload aborted".to_string());
|
||||||
|
self.images.insert(key, image.clone());
|
||||||
|
self.persist_image(&image).await;
|
||||||
|
}
|
||||||
|
Ok(Response::new(Empty {}))
|
||||||
|
}
|
||||||
|
|
||||||
async fn get_image(
|
async fn get_image(
|
||||||
&self,
|
&self,
|
||||||
request: Request<GetImageRequest>,
|
request: Request<GetImageRequest>,
|
||||||
|
|
@ -4283,6 +4730,7 @@ impl ImageService for VmServiceImpl {
|
||||||
let tenant = get_tenant_context(&request)?;
|
let tenant = get_tenant_context(&request)?;
|
||||||
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
|
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
|
||||||
let req = request.into_inner();
|
let req = request.into_inner();
|
||||||
|
Self::require_uuid(&req.image_id, "image_id")?;
|
||||||
self.auth
|
self.auth
|
||||||
.authorize(
|
.authorize(
|
||||||
&tenant,
|
&tenant,
|
||||||
|
|
@ -4337,6 +4785,7 @@ impl ImageService for VmServiceImpl {
|
||||||
let tenant = get_tenant_context(&request)?;
|
let tenant = get_tenant_context(&request)?;
|
||||||
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
|
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
|
||||||
let req = request.into_inner();
|
let req = request.into_inner();
|
||||||
|
Self::require_uuid(&req.image_id, "image_id")?;
|
||||||
self.auth
|
self.auth
|
||||||
.authorize(
|
.authorize(
|
||||||
&tenant,
|
&tenant,
|
||||||
|
|
@ -4378,6 +4827,7 @@ impl ImageService for VmServiceImpl {
|
||||||
let tenant = get_tenant_context(&request)?;
|
let tenant = get_tenant_context(&request)?;
|
||||||
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
|
let (org_id, project_id) = Self::resolve_image_tenant(&tenant, &request.get_ref().org_id)?;
|
||||||
let req = request.into_inner();
|
let req = request.into_inner();
|
||||||
|
Self::require_uuid(&req.image_id, "image_id")?;
|
||||||
self.auth
|
self.auth
|
||||||
.authorize(
|
.authorize(
|
||||||
&tenant,
|
&tenant,
|
||||||
|
|
@ -4396,6 +4846,31 @@ impl ImageService for VmServiceImpl {
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(store) = self.artifact_store.as_ref() {
|
if let Some(store) = self.artifact_store.as_ref() {
|
||||||
|
if let Ok(sessions) = self
|
||||||
|
.store
|
||||||
|
.list_image_upload_sessions(&org_id, &project_id, &req.image_id)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
for session in sessions {
|
||||||
|
let _ = store
|
||||||
|
.abort_image_upload(
|
||||||
|
&org_id,
|
||||||
|
&project_id,
|
||||||
|
&session.staging_key,
|
||||||
|
&session.upload_id,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let _ = self
|
||||||
|
.store
|
||||||
|
.delete_image_upload_session(
|
||||||
|
&org_id,
|
||||||
|
&project_id,
|
||||||
|
&req.image_id,
|
||||||
|
&session.session_id,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
store
|
store
|
||||||
.delete_image(&org_id, &project_id, &req.image_id)
|
.delete_image(&org_id, &project_id, &req.image_id)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
@ -4540,7 +5015,7 @@ impl NodeService for VmServiceImpl {
|
||||||
.hypervisors
|
.hypervisors
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|h| ProtoHypervisorType::try_from(*h).ok())
|
.filter_map(|h| ProtoHypervisorType::try_from(*h).ok())
|
||||||
.map(Self::map_hv)
|
.map(|hypervisor| self.map_hv(hypervisor))
|
||||||
.collect();
|
.collect();
|
||||||
}
|
}
|
||||||
if !req.supported_volume_drivers.is_empty() {
|
if !req.supported_volume_drivers.is_empty() {
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
use crate::artifact_store::ArtifactStore;
|
use crate::artifact_store::ArtifactStore;
|
||||||
|
use crate::config::{CephConfig, CoronaFsConfig, VolumeRuntimeConfig};
|
||||||
use crate::storage::VmStore;
|
use crate::storage::VmStore;
|
||||||
use plasmavmc_types::{
|
use plasmavmc_types::{
|
||||||
AttachedDisk, DiskAttachment, DiskSource, DiskSpec, VirtualMachine, Volume, VolumeBacking,
|
AttachedDisk, DiskAttachment, DiskSource, DiskSpec, VirtualMachine, Volume, VolumeBacking,
|
||||||
|
|
@ -18,6 +19,7 @@ const AUTO_DELETE_VOLUME_SOURCE_METADATA_KEY: &str = "plasmavmc.auto_delete_sour
|
||||||
const CORONAFS_IMAGE_SOURCE_ID_METADATA_KEY: &str = "plasmavmc.coronafs_image_source_id";
|
const CORONAFS_IMAGE_SOURCE_ID_METADATA_KEY: &str = "plasmavmc.coronafs_image_source_id";
|
||||||
const CORONAFS_IMAGE_SEED_PENDING_METADATA_KEY: &str = "plasmavmc.coronafs_image_seed_pending";
|
const CORONAFS_IMAGE_SEED_PENDING_METADATA_KEY: &str = "plasmavmc.coronafs_image_seed_pending";
|
||||||
const VOLUME_METADATA_CAS_RETRIES: usize = 16;
|
const VOLUME_METADATA_CAS_RETRIES: usize = 16;
|
||||||
|
const CEPH_IDENTIFIER_MAX_LEN: usize = 128;
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
struct CephClusterConfig {
|
struct CephClusterConfig {
|
||||||
|
|
@ -116,6 +118,7 @@ pub struct VolumeManager {
|
||||||
artifact_store: Option<Arc<ArtifactStore>>,
|
artifact_store: Option<Arc<ArtifactStore>>,
|
||||||
managed_root: PathBuf,
|
managed_root: PathBuf,
|
||||||
supported_storage_classes: Vec<String>,
|
supported_storage_classes: Vec<String>,
|
||||||
|
qemu_img_path: PathBuf,
|
||||||
ceph_cluster: Option<CephClusterConfig>,
|
ceph_cluster: Option<CephClusterConfig>,
|
||||||
coronafs_controller: Option<CoronaFsClient>,
|
coronafs_controller: Option<CoronaFsClient>,
|
||||||
coronafs_node: Option<CoronaFsClient>,
|
coronafs_node: Option<CoronaFsClient>,
|
||||||
|
|
@ -124,31 +127,43 @@ pub struct VolumeManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl VolumeManager {
|
impl VolumeManager {
|
||||||
pub fn new(store: Arc<dyn VmStore>, artifact_store: Option<Arc<ArtifactStore>>) -> Self {
|
pub fn new_with_config(
|
||||||
let managed_root = std::env::var("PLASMAVMC_MANAGED_VOLUME_ROOT")
|
store: Arc<dyn VmStore>,
|
||||||
.map(PathBuf::from)
|
artifact_store: Option<Arc<ArtifactStore>>,
|
||||||
.unwrap_or_else(|_| PathBuf::from("/var/lib/plasmavmc/managed-volumes"));
|
config: &VolumeRuntimeConfig,
|
||||||
let ceph_cluster = std::env::var("PLASMAVMC_CEPH_MONITORS")
|
local_node_id: Option<String>,
|
||||||
.ok()
|
) -> Result<Self, Box<dyn std::error::Error>> {
|
||||||
.filter(|value| !value.trim().is_empty())
|
let managed_root = config.managed_volume_root.clone();
|
||||||
.map(|monitors| CephClusterConfig {
|
let ceph_cluster = ceph_cluster_from_config(&config.ceph);
|
||||||
cluster_id: std::env::var("PLASMAVMC_CEPH_CLUSTER_ID")
|
let (coronafs_controller, coronafs_node) =
|
||||||
.unwrap_or_else(|_| "default".to_string()),
|
resolve_coronafs_clients_with_config(&config.coronafs);
|
||||||
monitors: monitors
|
let coronafs_node_local_attach = config.coronafs.node_local_attach;
|
||||||
.split(',')
|
let qemu_img_path = resolve_binary_path(config.qemu_img_path.as_deref(), "qemu-img")?;
|
||||||
.map(str::trim)
|
|
||||||
.filter(|item| !item.is_empty())
|
|
||||||
.map(ToOwned::to_owned)
|
|
||||||
.collect(),
|
|
||||||
user: std::env::var("PLASMAVMC_CEPH_USER").unwrap_or_else(|_| "admin".to_string()),
|
|
||||||
secret: std::env::var("PLASMAVMC_CEPH_SECRET").ok(),
|
|
||||||
});
|
|
||||||
let (coronafs_controller, coronafs_node) = resolve_coronafs_clients();
|
|
||||||
let coronafs_node_local_attach = coronafs_node_local_attach_enabled();
|
|
||||||
let local_node_id = std::env::var("PLASMAVMC_NODE_ID")
|
|
||||||
.ok()
|
|
||||||
.filter(|value| !value.trim().is_empty());
|
|
||||||
|
|
||||||
|
Ok(Self::build(
|
||||||
|
store,
|
||||||
|
artifact_store,
|
||||||
|
managed_root,
|
||||||
|
qemu_img_path,
|
||||||
|
ceph_cluster,
|
||||||
|
coronafs_controller,
|
||||||
|
coronafs_node,
|
||||||
|
coronafs_node_local_attach,
|
||||||
|
local_node_id,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build(
|
||||||
|
store: Arc<dyn VmStore>,
|
||||||
|
artifact_store: Option<Arc<ArtifactStore>>,
|
||||||
|
managed_root: PathBuf,
|
||||||
|
qemu_img_path: PathBuf,
|
||||||
|
ceph_cluster: Option<CephClusterConfig>,
|
||||||
|
coronafs_controller: Option<CoronaFsClient>,
|
||||||
|
coronafs_node: Option<CoronaFsClient>,
|
||||||
|
coronafs_node_local_attach: bool,
|
||||||
|
local_node_id: Option<String>,
|
||||||
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
store,
|
store,
|
||||||
artifact_store,
|
artifact_store,
|
||||||
|
|
@ -161,6 +176,7 @@ impl VolumeManager {
|
||||||
classes.push("ceph-rbd".to_string());
|
classes.push("ceph-rbd".to_string());
|
||||||
classes
|
classes
|
||||||
},
|
},
|
||||||
|
qemu_img_path,
|
||||||
ceph_cluster,
|
ceph_cluster,
|
||||||
coronafs_controller,
|
coronafs_controller,
|
||||||
coronafs_node,
|
coronafs_node,
|
||||||
|
|
@ -320,6 +336,7 @@ impl VolumeManager {
|
||||||
Some(export) => export,
|
Some(export) => export,
|
||||||
None => controller.ensure_export_read_only(volume_id).await?,
|
None => controller.ensure_export_read_only(volume_id).await?,
|
||||||
};
|
};
|
||||||
|
validate_coronafs_export(&export)?;
|
||||||
node_client
|
node_client
|
||||||
.materialize_from_export(
|
.materialize_from_export(
|
||||||
volume_id,
|
volume_id,
|
||||||
|
|
@ -377,6 +394,9 @@ impl VolumeManager {
|
||||||
{
|
{
|
||||||
return Ok(existing);
|
return Ok(existing);
|
||||||
}
|
}
|
||||||
|
if let Some(image_id) = image_id {
|
||||||
|
validate_uuid(image_id, "image_id")?;
|
||||||
|
}
|
||||||
|
|
||||||
let path = self.managed_volume_path(volume_id);
|
let path = self.managed_volume_path(volume_id);
|
||||||
let mut metadata = metadata;
|
let mut metadata = metadata;
|
||||||
|
|
@ -503,6 +523,9 @@ impl VolumeManager {
|
||||||
"Ceph RBD support is not configured on this node",
|
"Ceph RBD support is not configured on this node",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
validate_ceph_identifier("ceph_rbd.cluster_id", cluster_id)?;
|
||||||
|
validate_ceph_identifier("ceph_rbd.pool", pool)?;
|
||||||
|
validate_ceph_identifier("ceph_rbd.image", image)?;
|
||||||
let volume_id = Uuid::new_v4().to_string();
|
let volume_id = Uuid::new_v4().to_string();
|
||||||
let mut volume = Volume::new(volume_id, name.to_string(), org_id, project_id, size_gib);
|
let mut volume = Volume::new(volume_id, name.to_string(), org_id, project_id, size_gib);
|
||||||
volume.driver = VolumeDriverKind::CephRbd;
|
volume.driver = VolumeDriverKind::CephRbd;
|
||||||
|
|
@ -943,6 +966,7 @@ impl VolumeManager {
|
||||||
let Some(local_volume) = node_client.get_volume_optional(volume_id).await? else {
|
let Some(local_volume) = node_client.get_volume_optional(volume_id).await? else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
|
validate_coronafs_volume_response(&local_volume)?;
|
||||||
if local_volume.export.is_some() {
|
if local_volume.export.is_some() {
|
||||||
node_client.release_export(volume_id).await?;
|
node_client.release_export(volume_id).await?;
|
||||||
}
|
}
|
||||||
|
|
@ -975,6 +999,7 @@ impl VolumeManager {
|
||||||
Some(export) => export,
|
Some(export) => export,
|
||||||
None => controller.ensure_export(volume_id).await?,
|
None => controller.ensure_export(volume_id).await?,
|
||||||
};
|
};
|
||||||
|
validate_coronafs_export(&export)?;
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
volume_id,
|
volume_id,
|
||||||
node_endpoint = %node_client.endpoint,
|
node_endpoint = %node_client.endpoint,
|
||||||
|
|
@ -983,7 +1008,7 @@ impl VolumeManager {
|
||||||
export_uri = %export.uri,
|
export_uri = %export.uri,
|
||||||
"Syncing node-local CoronaFS volume back to controller"
|
"Syncing node-local CoronaFS volume back to controller"
|
||||||
);
|
);
|
||||||
sync_local_coronafs_volume_to_export(&local_volume, &export.uri).await
|
sync_local_coronafs_volume_to_export(&self.qemu_img_path, &local_volume, &export.uri).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn materialize_pending_coronafs_image_seed_on_node(
|
async fn materialize_pending_coronafs_image_seed_on_node(
|
||||||
|
|
@ -1185,6 +1210,7 @@ impl VolumeManager {
|
||||||
if self.coronafs_attachment_backend().is_some() {
|
if self.coronafs_attachment_backend().is_some() {
|
||||||
let (coronafs_volume, coronafs) =
|
let (coronafs_volume, coronafs) =
|
||||||
self.load_coronafs_volume_for_attachment(volume).await?;
|
self.load_coronafs_volume_for_attachment(volume).await?;
|
||||||
|
validate_coronafs_volume_response(&coronafs_volume)?;
|
||||||
if coronafs.supports_local_backing_file().await
|
if coronafs.supports_local_backing_file().await
|
||||||
&& !should_prefer_coronafs_export_attachment(&coronafs_volume)
|
&& !should_prefer_coronafs_export_attachment(&coronafs_volume)
|
||||||
&& coronafs_local_target_ready(&coronafs_volume.path).await
|
&& coronafs_local_target_ready(&coronafs_volume.path).await
|
||||||
|
|
@ -1226,6 +1252,9 @@ impl VolumeManager {
|
||||||
cluster_id
|
cluster_id
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
validate_ceph_identifier("ceph cluster_id", cluster_id)?;
|
||||||
|
validate_ceph_identifier("ceph pool", pool)?;
|
||||||
|
validate_ceph_identifier("ceph image", image)?;
|
||||||
DiskAttachment::CephRbd {
|
DiskAttachment::CephRbd {
|
||||||
pool: pool.clone(),
|
pool: pool.clone(),
|
||||||
image: image.clone(),
|
image: image.clone(),
|
||||||
|
|
@ -1266,7 +1295,7 @@ impl VolumeManager {
|
||||||
.await
|
.await
|
||||||
.map_err(|e| Status::internal(format!("failed to create volume dir: {e}")))?;
|
.map_err(|e| Status::internal(format!("failed to create volume dir: {e}")))?;
|
||||||
}
|
}
|
||||||
let status = Command::new("qemu-img")
|
let status = Command::new(&self.qemu_img_path)
|
||||||
.args([
|
.args([
|
||||||
"convert",
|
"convert",
|
||||||
"-O",
|
"-O",
|
||||||
|
|
@ -1307,7 +1336,7 @@ impl VolumeManager {
|
||||||
.materialize_image_cache(org_id, project_id, image_id)
|
.materialize_image_cache(org_id, project_id, image_id)
|
||||||
.await?;
|
.await?;
|
||||||
let requested_size = gib_to_bytes(size_gib);
|
let requested_size = gib_to_bytes(size_gib);
|
||||||
let image_info = inspect_qemu_image(&image_path).await?;
|
let image_info = inspect_qemu_image(&self.qemu_img_path, &image_path).await?;
|
||||||
if requested_size < image_info.virtual_size {
|
if requested_size < image_info.virtual_size {
|
||||||
return Err(Status::failed_precondition(format!(
|
return Err(Status::failed_precondition(format!(
|
||||||
"requested volume {} GiB is smaller than image virtual size {} bytes",
|
"requested volume {} GiB is smaller than image virtual size {} bytes",
|
||||||
|
|
@ -1413,6 +1442,7 @@ impl VolumeManager {
|
||||||
&raw_image_path,
|
&raw_image_path,
|
||||||
Path::new(&volume.path),
|
Path::new(&volume.path),
|
||||||
requested_size,
|
requested_size,
|
||||||
|
&self.qemu_img_path,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
return Ok(CoronaFsProvisionOutcome {
|
return Ok(CoronaFsProvisionOutcome {
|
||||||
|
|
@ -1431,7 +1461,7 @@ impl VolumeManager {
|
||||||
export_uri = %export.uri,
|
export_uri = %export.uri,
|
||||||
"Populating CoronaFS-backed VM volume over NBD from image cache"
|
"Populating CoronaFS-backed VM volume over NBD from image cache"
|
||||||
);
|
);
|
||||||
let status = Command::new("qemu-img")
|
let status = Command::new(&self.qemu_img_path)
|
||||||
.args([
|
.args([
|
||||||
"convert",
|
"convert",
|
||||||
"-t",
|
"-t",
|
||||||
|
|
@ -1481,7 +1511,7 @@ impl VolumeManager {
|
||||||
.await
|
.await
|
||||||
.map_err(|e| Status::internal(format!("failed to create volume dir: {e}")))?;
|
.map_err(|e| Status::internal(format!("failed to create volume dir: {e}")))?;
|
||||||
}
|
}
|
||||||
let status = Command::new("qemu-img")
|
let status = Command::new(&self.qemu_img_path)
|
||||||
.args([
|
.args([
|
||||||
"create",
|
"create",
|
||||||
"-f",
|
"-f",
|
||||||
|
|
@ -1508,7 +1538,7 @@ impl VolumeManager {
|
||||||
format: VolumeFormat,
|
format: VolumeFormat,
|
||||||
size_gib: u64,
|
size_gib: u64,
|
||||||
) -> Result<(), Status> {
|
) -> Result<(), Status> {
|
||||||
let status = Command::new("qemu-img")
|
let status = Command::new(&self.qemu_img_path)
|
||||||
.args([
|
.args([
|
||||||
"resize",
|
"resize",
|
||||||
"-f",
|
"-f",
|
||||||
|
|
@ -1542,8 +1572,8 @@ impl VolumeManager {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn inspect_qemu_image(path: &Path) -> Result<QemuImageInfo, Status> {
|
async fn inspect_qemu_image(qemu_img_path: &Path, path: &Path) -> Result<QemuImageInfo, Status> {
|
||||||
let output = Command::new("qemu-img")
|
let output = Command::new(qemu_img_path)
|
||||||
.args(["info", "--output", "json", path.to_string_lossy().as_ref()])
|
.args(["info", "--output", "json", path.to_string_lossy().as_ref()])
|
||||||
.output()
|
.output()
|
||||||
.await
|
.await
|
||||||
|
|
@ -1585,6 +1615,7 @@ async fn clone_local_raw_into_coronafs_volume(
|
||||||
source: &Path,
|
source: &Path,
|
||||||
destination: &Path,
|
destination: &Path,
|
||||||
requested_size: u64,
|
requested_size: u64,
|
||||||
|
_qemu_img_path: &Path,
|
||||||
) -> Result<(), Status> {
|
) -> Result<(), Status> {
|
||||||
let temp_path = destination.with_extension("clone.tmp");
|
let temp_path = destination.with_extension("clone.tmp");
|
||||||
if let Some(parent) = temp_path.parent() {
|
if let Some(parent) = temp_path.parent() {
|
||||||
|
|
@ -1595,33 +1626,13 @@ async fn clone_local_raw_into_coronafs_volume(
|
||||||
if tokio::fs::try_exists(&temp_path).await.unwrap_or(false) {
|
if tokio::fs::try_exists(&temp_path).await.unwrap_or(false) {
|
||||||
let _ = tokio::fs::remove_file(&temp_path).await;
|
let _ = tokio::fs::remove_file(&temp_path).await;
|
||||||
}
|
}
|
||||||
|
tokio::fs::copy(source, &temp_path).await.map_err(|e| {
|
||||||
let copy_output = Command::new("cp")
|
Status::internal(format!(
|
||||||
.args([
|
"failed to clone raw image {} into {}: {e}",
|
||||||
"--reflink=auto",
|
|
||||||
"--sparse=always",
|
|
||||||
source.to_string_lossy().as_ref(),
|
|
||||||
temp_path.to_string_lossy().as_ref(),
|
|
||||||
])
|
|
||||||
.output()
|
|
||||||
.await
|
|
||||||
.map_err(|e| Status::internal(format!("failed to spawn raw clone copy: {e}")))?;
|
|
||||||
if !copy_output.status.success() {
|
|
||||||
let stderr = String::from_utf8_lossy(©_output.stderr)
|
|
||||||
.trim()
|
|
||||||
.to_string();
|
|
||||||
return Err(Status::internal(format!(
|
|
||||||
"failed to clone raw image {} into {} with status {}{}",
|
|
||||||
source.display(),
|
source.display(),
|
||||||
temp_path.display(),
|
temp_path.display()
|
||||||
copy_output.status,
|
))
|
||||||
if stderr.is_empty() {
|
})?;
|
||||||
String::new()
|
|
||||||
} else {
|
|
||||||
format!(": {stderr}")
|
|
||||||
}
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
let file = tokio::fs::OpenOptions::new()
|
let file = tokio::fs::OpenOptions::new()
|
||||||
.write(true)
|
.write(true)
|
||||||
|
|
@ -1656,11 +1667,12 @@ async fn clone_local_raw_into_coronafs_volume(
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn sync_local_coronafs_volume_to_export(
|
async fn sync_local_coronafs_volume_to_export(
|
||||||
|
qemu_img_path: &Path,
|
||||||
local_volume: &CoronaFsVolumeResponse,
|
local_volume: &CoronaFsVolumeResponse,
|
||||||
export_uri: &str,
|
export_uri: &str,
|
||||||
) -> Result<(), Status> {
|
) -> Result<(), Status> {
|
||||||
let local_format = local_volume.format.unwrap_or(CoronaFsVolumeFormat::Raw);
|
let local_format = local_volume.format.unwrap_or(CoronaFsVolumeFormat::Raw);
|
||||||
let status = Command::new("qemu-img")
|
let status = Command::new(qemu_img_path)
|
||||||
.args([
|
.args([
|
||||||
"convert",
|
"convert",
|
||||||
"-t",
|
"-t",
|
||||||
|
|
@ -1773,17 +1785,121 @@ fn gib_to_bytes(size_gib: u64) -> u64 {
|
||||||
size_gib.saturating_mul(1024 * 1024 * 1024)
|
size_gib.saturating_mul(1024 * 1024 * 1024)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn coronafs_node_local_attach_enabled() -> bool {
|
fn validate_uuid(value: &str, field_name: &str) -> Result<(), Status> {
|
||||||
coronafs_node_local_attach_enabled_from_values(
|
Uuid::parse_str(value)
|
||||||
std::env::var("PLASMAVMC_CORONAFS_NODE_LOCAL_ATTACH")
|
.map(|_| ())
|
||||||
.ok()
|
.map_err(|_| Status::invalid_argument(format!("{field_name} must be a UUID")))
|
||||||
.as_deref(),
|
|
||||||
std::env::var("PLASMAVMC_CORONAFS_ENABLE_EXPERIMENTAL_NODE_LOCAL_ATTACH")
|
|
||||||
.ok()
|
|
||||||
.as_deref(),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn validate_ceph_identifier(field_name: &str, value: &str) -> Result<(), Status> {
|
||||||
|
let trimmed = value.trim();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
return Err(Status::invalid_argument(format!(
|
||||||
|
"{field_name} is required"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if trimmed.len() > CEPH_IDENTIFIER_MAX_LEN {
|
||||||
|
return Err(Status::invalid_argument(format!(
|
||||||
|
"{field_name} exceeds {CEPH_IDENTIFIER_MAX_LEN} characters"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let mut chars = trimmed.chars();
|
||||||
|
let Some(first) = chars.next() else {
|
||||||
|
return Err(Status::invalid_argument(format!(
|
||||||
|
"{field_name} is required"
|
||||||
|
)));
|
||||||
|
};
|
||||||
|
if !first.is_ascii_alphanumeric() {
|
||||||
|
return Err(Status::invalid_argument(format!(
|
||||||
|
"{field_name} must start with an ASCII alphanumeric character"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if chars.any(|ch| !(ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-'))) {
|
||||||
|
return Err(Status::invalid_argument(format!(
|
||||||
|
"{field_name} contains unsupported characters"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_absolute_path(path: &str, field_name: &str) -> Result<(), Status> {
|
||||||
|
let candidate = Path::new(path);
|
||||||
|
if !candidate.is_absolute() {
|
||||||
|
return Err(Status::failed_precondition(format!(
|
||||||
|
"{field_name} must be an absolute path"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_nbd_uri(uri: &str, field_name: &str) -> Result<(), Status> {
|
||||||
|
let parsed = reqwest::Url::parse(uri)
|
||||||
|
.map_err(|e| Status::failed_precondition(format!("{field_name} is invalid: {e}")))?;
|
||||||
|
if parsed.scheme() != "nbd" {
|
||||||
|
return Err(Status::failed_precondition(format!(
|
||||||
|
"{field_name} must use nbd://"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if !parsed.username().is_empty() || parsed.password().is_some() {
|
||||||
|
return Err(Status::failed_precondition(format!(
|
||||||
|
"{field_name} must not contain embedded credentials"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if parsed.host_str().is_none() || parsed.port().is_none() {
|
||||||
|
return Err(Status::failed_precondition(format!(
|
||||||
|
"{field_name} must include host and port"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_coronafs_export(export: &CoronaFsExport) -> Result<(), Status> {
|
||||||
|
validate_nbd_uri(&export.uri, "coronafs export uri")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_coronafs_volume_response(volume: &CoronaFsVolumeResponse) -> Result<(), Status> {
|
||||||
|
validate_absolute_path(&volume.path, "coronafs volume path")?;
|
||||||
|
if let Some(export) = &volume.export {
|
||||||
|
validate_coronafs_export(export)?;
|
||||||
|
}
|
||||||
|
if let Some(source) = &volume.materialized_from {
|
||||||
|
if source.starts_with("nbd://") {
|
||||||
|
validate_nbd_uri(source, "coronafs materialized_from")?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_binary_path(
|
||||||
|
configured_path: Option<&Path>,
|
||||||
|
binary_name: &str,
|
||||||
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
||||||
|
let candidate = match configured_path {
|
||||||
|
Some(path) => path.to_path_buf(),
|
||||||
|
None => std::env::var_os("PATH")
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|paths| std::env::split_paths(&paths).collect::<Vec<_>>())
|
||||||
|
.map(|entry| entry.join(binary_name))
|
||||||
|
.find(|candidate| candidate.exists())
|
||||||
|
.ok_or_else(|| {
|
||||||
|
std::io::Error::new(
|
||||||
|
std::io::ErrorKind::NotFound,
|
||||||
|
format!("failed to locate {binary_name} in PATH"),
|
||||||
|
)
|
||||||
|
})?,
|
||||||
|
};
|
||||||
|
let metadata = std::fs::metadata(&candidate)?;
|
||||||
|
if !metadata.is_file() {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidInput,
|
||||||
|
format!("{} is not a regular file", candidate.display()),
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
Ok(candidate)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
fn coronafs_node_local_attach_enabled_from_values(
|
fn coronafs_node_local_attach_enabled_from_values(
|
||||||
stable_value: Option<&str>,
|
stable_value: Option<&str>,
|
||||||
legacy_value: Option<&str>,
|
legacy_value: Option<&str>,
|
||||||
|
|
@ -1794,6 +1910,23 @@ fn coronafs_node_local_attach_enabled_from_values(
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn ceph_cluster_from_config(config: &CephConfig) -> Option<CephClusterConfig> {
|
||||||
|
if config.monitors.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(CephClusterConfig {
|
||||||
|
cluster_id: config.cluster_id.clone(),
|
||||||
|
monitors: config.monitors.clone(),
|
||||||
|
user: config.user.clone(),
|
||||||
|
secret: config
|
||||||
|
.secret
|
||||||
|
.clone()
|
||||||
|
.or_else(|| std::env::var("PLASMAVMC_CEPH_SECRET").ok()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
fn parse_truthy(value: &str) -> bool {
|
fn parse_truthy(value: &str) -> bool {
|
||||||
matches!(
|
matches!(
|
||||||
value.trim().to_ascii_lowercase().as_str(),
|
value.trim().to_ascii_lowercase().as_str(),
|
||||||
|
|
@ -1801,16 +1934,21 @@ fn parse_truthy(value: &str) -> bool {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn resolve_coronafs_clients() -> (Option<CoronaFsClient>, Option<CoronaFsClient>) {
|
fn resolve_coronafs_clients_with_config(
|
||||||
|
config: &CoronaFsConfig,
|
||||||
|
) -> (Option<CoronaFsClient>, Option<CoronaFsClient>) {
|
||||||
let (controller_endpoint, node_endpoint) = resolve_coronafs_endpoints(
|
let (controller_endpoint, node_endpoint) = resolve_coronafs_endpoints(
|
||||||
std::env::var("PLASMAVMC_CORONAFS_CONTROLLER_ENDPOINT")
|
config
|
||||||
.ok()
|
.controller_endpoint
|
||||||
|
.clone()
|
||||||
.and_then(normalize_coronafs_endpoint),
|
.and_then(normalize_coronafs_endpoint),
|
||||||
std::env::var("PLASMAVMC_CORONAFS_NODE_ENDPOINT")
|
config
|
||||||
.ok()
|
.node_endpoint
|
||||||
|
.clone()
|
||||||
.and_then(normalize_coronafs_endpoint),
|
.and_then(normalize_coronafs_endpoint),
|
||||||
std::env::var("PLASMAVMC_CORONAFS_ENDPOINT")
|
config
|
||||||
.ok()
|
.endpoint
|
||||||
|
.clone()
|
||||||
.and_then(normalize_coronafs_endpoint),
|
.and_then(normalize_coronafs_endpoint),
|
||||||
);
|
);
|
||||||
(
|
(
|
||||||
|
|
@ -1911,14 +2049,19 @@ impl CoronaFsClient {
|
||||||
backing_file: None,
|
backing_file: None,
|
||||||
backing_format: None,
|
backing_format: None,
|
||||||
};
|
};
|
||||||
self.try_request("create CoronaFS volume", |endpoint, http| {
|
let volume = self
|
||||||
http.put(format!("{endpoint}/v1/volumes/{volume_id}"))
|
.try_request("create CoronaFS volume", |endpoint, http| {
|
||||||
.json(&request)
|
http.put(format!("{endpoint}/v1/volumes/{volume_id}"))
|
||||||
})
|
.json(&request)
|
||||||
.await?
|
})
|
||||||
.json::<CoronaFsVolumeResponse>()
|
.await?
|
||||||
.await
|
.json::<CoronaFsVolumeResponse>()
|
||||||
.map_err(|e| Status::internal(format!("failed to decode CoronaFS create response: {e}")))
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
Status::internal(format!("failed to decode CoronaFS create response: {e}"))
|
||||||
|
})?;
|
||||||
|
validate_coronafs_volume_response(&volume)?;
|
||||||
|
Ok(volume)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn create_image_backed(
|
async fn create_image_backed(
|
||||||
|
|
@ -1934,28 +2077,36 @@ impl CoronaFsClient {
|
||||||
backing_file: Some(backing_file.to_string_lossy().into_owned()),
|
backing_file: Some(backing_file.to_string_lossy().into_owned()),
|
||||||
backing_format: Some(backing_format),
|
backing_format: Some(backing_format),
|
||||||
};
|
};
|
||||||
self.try_request("create CoronaFS image-backed volume", |endpoint, http| {
|
let volume = self
|
||||||
http.put(format!("{endpoint}/v1/volumes/{volume_id}"))
|
.try_request("create CoronaFS image-backed volume", |endpoint, http| {
|
||||||
.json(&request)
|
http.put(format!("{endpoint}/v1/volumes/{volume_id}"))
|
||||||
})
|
.json(&request)
|
||||||
.await?
|
})
|
||||||
.json::<CoronaFsVolumeResponse>()
|
.await?
|
||||||
.await
|
.json::<CoronaFsVolumeResponse>()
|
||||||
.map_err(|e| {
|
.await
|
||||||
Status::internal(format!(
|
.map_err(|e| {
|
||||||
"failed to decode CoronaFS image-backed create response: {e}"
|
Status::internal(format!(
|
||||||
))
|
"failed to decode CoronaFS image-backed create response: {e}"
|
||||||
})
|
))
|
||||||
|
})?;
|
||||||
|
validate_coronafs_volume_response(&volume)?;
|
||||||
|
Ok(volume)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_volume(&self, volume_id: &str) -> Result<CoronaFsVolumeResponse, Status> {
|
async fn get_volume(&self, volume_id: &str) -> Result<CoronaFsVolumeResponse, Status> {
|
||||||
self.try_request("inspect CoronaFS volume", |endpoint, http| {
|
let volume = self
|
||||||
http.get(format!("{endpoint}/v1/volumes/{volume_id}"))
|
.try_request("inspect CoronaFS volume", |endpoint, http| {
|
||||||
})
|
http.get(format!("{endpoint}/v1/volumes/{volume_id}"))
|
||||||
.await?
|
})
|
||||||
.json::<CoronaFsVolumeResponse>()
|
.await?
|
||||||
.await
|
.json::<CoronaFsVolumeResponse>()
|
||||||
.map_err(|e| Status::internal(format!("failed to decode CoronaFS inspect response: {e}")))
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
Status::internal(format!("failed to decode CoronaFS inspect response: {e}"))
|
||||||
|
})?;
|
||||||
|
validate_coronafs_volume_response(&volume)?;
|
||||||
|
Ok(volume)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_volume_optional(
|
async fn get_volume_optional(
|
||||||
|
|
@ -2001,9 +2152,11 @@ impl CoronaFsClient {
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
Status::internal(format!("failed to decode CoronaFS export response: {e}"))
|
Status::internal(format!("failed to decode CoronaFS export response: {e}"))
|
||||||
})?;
|
})?;
|
||||||
response.export.ok_or_else(|| {
|
let export = response.export.ok_or_else(|| {
|
||||||
Status::internal("CoronaFS export response did not include an export URI")
|
Status::internal("CoronaFS export response did not include an export URI")
|
||||||
})
|
})?;
|
||||||
|
validate_coronafs_export(&export)?;
|
||||||
|
Ok(export)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn materialize_from_export(
|
async fn materialize_from_export(
|
||||||
|
|
@ -2014,24 +2167,28 @@ impl CoronaFsClient {
|
||||||
format: Option<CoronaFsVolumeFormat>,
|
format: Option<CoronaFsVolumeFormat>,
|
||||||
lazy: bool,
|
lazy: bool,
|
||||||
) -> Result<CoronaFsVolumeResponse, Status> {
|
) -> Result<CoronaFsVolumeResponse, Status> {
|
||||||
|
validate_nbd_uri(source_uri, "coronafs materialize source_uri")?;
|
||||||
let request = CoronaFsMaterializeRequest {
|
let request = CoronaFsMaterializeRequest {
|
||||||
source_uri: source_uri.to_string(),
|
source_uri: source_uri.to_string(),
|
||||||
size_bytes,
|
size_bytes,
|
||||||
format,
|
format,
|
||||||
lazy,
|
lazy,
|
||||||
};
|
};
|
||||||
self.try_request("materialize CoronaFS volume", |endpoint, http| {
|
let volume = self
|
||||||
http.post(format!("{endpoint}/v1/volumes/{volume_id}/materialize"))
|
.try_request("materialize CoronaFS volume", |endpoint, http| {
|
||||||
.json(&request)
|
http.post(format!("{endpoint}/v1/volumes/{volume_id}/materialize"))
|
||||||
})
|
.json(&request)
|
||||||
.await?
|
})
|
||||||
.json::<CoronaFsVolumeResponse>()
|
.await?
|
||||||
.await
|
.json::<CoronaFsVolumeResponse>()
|
||||||
.map_err(|e| {
|
.await
|
||||||
Status::internal(format!(
|
.map_err(|e| {
|
||||||
"failed to decode CoronaFS materialize response: {e}"
|
Status::internal(format!(
|
||||||
))
|
"failed to decode CoronaFS materialize response: {e}"
|
||||||
})
|
))
|
||||||
|
})?;
|
||||||
|
validate_coronafs_volume_response(&volume)?;
|
||||||
|
Ok(volume)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn resize_volume(&self, volume_id: &str, size_bytes: u64) -> Result<(), Status> {
|
async fn resize_volume(&self, volume_id: &str, size_bytes: u64) -> Result<(), Status> {
|
||||||
|
|
@ -2210,9 +2367,14 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
tokio::fs::write(&source, b"raw-seed").await.unwrap();
|
tokio::fs::write(&source, b"raw-seed").await.unwrap();
|
||||||
|
|
||||||
clone_local_raw_into_coronafs_volume(&source, &destination, 4096)
|
clone_local_raw_into_coronafs_volume(
|
||||||
.await
|
&source,
|
||||||
.unwrap();
|
&destination,
|
||||||
|
4096,
|
||||||
|
Path::new("/usr/bin/qemu-img"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let cloned = tokio::fs::read(&destination).await.unwrap();
|
let cloned = tokio::fs::read(&destination).await.unwrap();
|
||||||
assert_eq!(&cloned[..8], b"raw-seed");
|
assert_eq!(&cloned[..8], b"raw-seed");
|
||||||
|
|
@ -2398,4 +2560,27 @@ mod tests {
|
||||||
.remove(CORONAFS_IMAGE_SEED_PENDING_METADATA_KEY);
|
.remove(CORONAFS_IMAGE_SEED_PENDING_METADATA_KEY);
|
||||||
assert!(!volume_has_pending_coronafs_image_seed(&volume));
|
assert!(!volume_has_pending_coronafs_image_seed(&volume));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ceph_identifier_validation_rejects_option_injection_characters() {
|
||||||
|
assert!(validate_ceph_identifier("pool", "pool-alpha").is_ok());
|
||||||
|
assert!(validate_ceph_identifier("pool", "pool,inject").is_err());
|
||||||
|
assert!(validate_ceph_identifier("image", "image:inject").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn coronafs_export_validation_requires_nbd_uri() {
|
||||||
|
let valid = CoronaFsExport {
|
||||||
|
uri: "nbd://10.0.0.1:11000".to_string(),
|
||||||
|
port: 11000,
|
||||||
|
pid: None,
|
||||||
|
};
|
||||||
|
let invalid = CoronaFsExport {
|
||||||
|
uri: "http://10.0.0.1:11000".to_string(),
|
||||||
|
..valid.clone()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(validate_coronafs_export(&valid).is_ok());
|
||||||
|
assert!(validate_coronafs_export(&invalid).is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -64,13 +64,8 @@ pub struct WatcherConfig {
|
||||||
|
|
||||||
impl Default for WatcherConfig {
|
impl Default for WatcherConfig {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
let poll_interval_ms = std::env::var("PLASMAVMC_STATE_WATCHER_POLL_INTERVAL_MS")
|
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.parse::<u64>().ok())
|
|
||||||
.unwrap_or(1000)
|
|
||||||
.max(100);
|
|
||||||
Self {
|
Self {
|
||||||
poll_interval: Duration::from_millis(poll_interval_ms),
|
poll_interval: Duration::from_millis(1000),
|
||||||
buffer_size: 256,
|
buffer_size: 256,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,8 +4,7 @@ use serde::{Deserialize, Serialize};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
/// FireCracker backend configuration
|
/// FireCracker backend configuration
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
#[derive(Default)]
|
|
||||||
pub struct FireCrackerConfig {
|
pub struct FireCrackerConfig {
|
||||||
/// Path to the Firecracker binary
|
/// Path to the Firecracker binary
|
||||||
pub firecracker_path: Option<PathBuf>,
|
pub firecracker_path: Option<PathBuf>,
|
||||||
|
|
@ -27,11 +26,11 @@ pub struct FireCrackerConfig {
|
||||||
pub use_jailer: Option<bool>,
|
pub use_jailer: Option<bool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// KVM backend configuration
|
/// KVM backend configuration
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
#[derive(Default)]
|
|
||||||
pub struct KvmConfig {
|
pub struct KvmConfig {
|
||||||
// Add KVM specific configuration fields here if any
|
/// Path to the QEMU binary
|
||||||
|
pub qemu_path: Option<PathBuf>,
|
||||||
|
/// Runtime directory used for QMP sockets and console logs
|
||||||
|
pub runtime_dir: Option<PathBuf>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,10 @@ service VmService {
|
||||||
|
|
||||||
service ImageService {
|
service ImageService {
|
||||||
rpc CreateImage(CreateImageRequest) returns (Image);
|
rpc CreateImage(CreateImageRequest) returns (Image);
|
||||||
|
rpc BeginImageUpload(BeginImageUploadRequest) returns (BeginImageUploadResponse);
|
||||||
|
rpc UploadImagePart(UploadImagePartRequest) returns (UploadImagePartResponse);
|
||||||
|
rpc CompleteImageUpload(CompleteImageUploadRequest) returns (Image);
|
||||||
|
rpc AbortImageUpload(AbortImageUploadRequest) returns (Empty);
|
||||||
rpc GetImage(GetImageRequest) returns (Image);
|
rpc GetImage(GetImageRequest) returns (Image);
|
||||||
rpc ListImages(ListImagesRequest) returns (ListImagesResponse);
|
rpc ListImages(ListImagesRequest) returns (ListImagesResponse);
|
||||||
rpc UpdateImage(UpdateImageRequest) returns (Image);
|
rpc UpdateImage(UpdateImageRequest) returns (Image);
|
||||||
|
|
@ -475,6 +479,50 @@ message CreateImageRequest {
|
||||||
string source_url = 11;
|
string source_url = 11;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
message BeginImageUploadRequest {
|
||||||
|
string name = 1;
|
||||||
|
string org_id = 2;
|
||||||
|
Visibility visibility = 3;
|
||||||
|
ImageFormat format = 4;
|
||||||
|
OsType os_type = 5;
|
||||||
|
string os_version = 6;
|
||||||
|
Architecture architecture = 7;
|
||||||
|
uint32 min_disk_gib = 8;
|
||||||
|
uint32 min_memory_mib = 9;
|
||||||
|
map<string, string> metadata = 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
message BeginImageUploadResponse {
|
||||||
|
Image image = 1;
|
||||||
|
string upload_session_id = 2;
|
||||||
|
uint32 minimum_part_size_bytes = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
message UploadImagePartRequest {
|
||||||
|
string org_id = 1;
|
||||||
|
string image_id = 2;
|
||||||
|
string upload_session_id = 3;
|
||||||
|
uint32 part_number = 4;
|
||||||
|
bytes body = 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
message UploadImagePartResponse {
|
||||||
|
uint32 part_number = 1;
|
||||||
|
uint64 committed_size_bytes = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message CompleteImageUploadRequest {
|
||||||
|
string org_id = 1;
|
||||||
|
string image_id = 2;
|
||||||
|
string upload_session_id = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
message AbortImageUploadRequest {
|
||||||
|
string org_id = 1;
|
||||||
|
string image_id = 2;
|
||||||
|
string upload_session_id = 3;
|
||||||
|
}
|
||||||
|
|
||||||
message GetImageRequest {
|
message GetImageRequest {
|
||||||
string org_id = 1;
|
string org_id = 1;
|
||||||
string image_id = 2;
|
string image_id = 2;
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,36 @@ pub struct TlsConfig {
|
||||||
pub require_client_cert: bool,
|
pub require_client_cert: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum OvnMode {
|
||||||
|
Mock,
|
||||||
|
Real,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for OvnMode {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::Mock
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct OvnConfig {
|
||||||
|
#[serde(default)]
|
||||||
|
pub mode: OvnMode,
|
||||||
|
#[serde(default = "default_ovn_nb_addr")]
|
||||||
|
pub nb_addr: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for OvnConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
mode: OvnMode::default(),
|
||||||
|
nb_addr: default_ovn_nb_addr(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Metadata storage backend
|
/// Metadata storage backend
|
||||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
#[serde(rename_all = "lowercase")]
|
#[serde(rename_all = "lowercase")]
|
||||||
|
|
@ -74,6 +104,10 @@ pub struct ServerConfig {
|
||||||
/// Authentication configuration
|
/// Authentication configuration
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub auth: AuthConfig,
|
pub auth: AuthConfig,
|
||||||
|
|
||||||
|
/// OVN integration settings
|
||||||
|
#[serde(default)]
|
||||||
|
pub ovn: OvnConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Authentication configuration
|
/// Authentication configuration
|
||||||
|
|
@ -100,6 +134,10 @@ fn default_http_addr() -> SocketAddr {
|
||||||
"127.0.0.1:8087".parse().unwrap()
|
"127.0.0.1:8087".parse().unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_ovn_nb_addr() -> String {
|
||||||
|
"tcp:127.0.0.1:6641".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
impl Default for ServerConfig {
|
impl Default for ServerConfig {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
|
@ -113,6 +151,7 @@ impl Default for ServerConfig {
|
||||||
log_level: "info".to_string(),
|
log_level: "info".to_string(),
|
||||||
tls: None,
|
tls: None,
|
||||||
auth: AuthConfig::default(),
|
auth: AuthConfig::default(),
|
||||||
|
ovn: OvnConfig::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,9 +11,8 @@ use prismnet_api::{
|
||||||
subnet_service_server::SubnetServiceServer, vpc_service_server::VpcServiceServer,
|
subnet_service_server::SubnetServiceServer, vpc_service_server::VpcServiceServer,
|
||||||
};
|
};
|
||||||
use prismnet_server::{
|
use prismnet_server::{
|
||||||
config::MetadataBackend,
|
config::MetadataBackend, IpamServiceImpl, NetworkMetadataStore, OvnClient, PortServiceImpl,
|
||||||
IpamServiceImpl, NetworkMetadataStore, OvnClient, PortServiceImpl, SecurityGroupServiceImpl,
|
SecurityGroupServiceImpl, ServerConfig, SubnetServiceImpl, VpcServiceImpl,
|
||||||
ServerConfig, SubnetServiceImpl, VpcServiceImpl,
|
|
||||||
};
|
};
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
@ -36,26 +35,6 @@ struct Args {
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
grpc_addr: Option<String>,
|
grpc_addr: Option<String>,
|
||||||
|
|
||||||
/// ChainFire endpoint for cluster coordination (optional)
|
|
||||||
#[arg(long)]
|
|
||||||
chainfire_endpoint: Option<String>,
|
|
||||||
|
|
||||||
/// FlareDB endpoint for metadata and tenant data storage
|
|
||||||
#[arg(long)]
|
|
||||||
flaredb_endpoint: Option<String>,
|
|
||||||
|
|
||||||
/// Metadata backend (flaredb, postgres, sqlite)
|
|
||||||
#[arg(long)]
|
|
||||||
metadata_backend: Option<String>,
|
|
||||||
|
|
||||||
/// SQL database URL for metadata (required for postgres/sqlite backend)
|
|
||||||
#[arg(long)]
|
|
||||||
metadata_database_url: Option<String>,
|
|
||||||
|
|
||||||
/// Run in single-node mode (required when metadata backend is SQLite)
|
|
||||||
#[arg(long)]
|
|
||||||
single_node: bool,
|
|
||||||
|
|
||||||
/// Log level (overrides config)
|
/// Log level (overrides config)
|
||||||
#[arg(short, long)]
|
#[arg(short, long)]
|
||||||
log_level: Option<String>,
|
log_level: Option<String>,
|
||||||
|
|
@ -88,58 +67,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
if let Some(log_level) = args.log_level {
|
if let Some(log_level) = args.log_level {
|
||||||
config.log_level = log_level;
|
config.log_level = log_level;
|
||||||
}
|
}
|
||||||
if let Some(chainfire_endpoint) = args.chainfire_endpoint {
|
|
||||||
config.chainfire_endpoint = Some(chainfire_endpoint);
|
|
||||||
}
|
|
||||||
if let Some(flaredb_endpoint) = args.flaredb_endpoint {
|
|
||||||
config.flaredb_endpoint = Some(flaredb_endpoint);
|
|
||||||
}
|
|
||||||
if let Some(metadata_backend) = args.metadata_backend {
|
|
||||||
config.metadata_backend = parse_metadata_backend(&metadata_backend)?;
|
|
||||||
}
|
|
||||||
if let Some(metadata_database_url) = args.metadata_database_url {
|
|
||||||
config.metadata_database_url = Some(metadata_database_url);
|
|
||||||
}
|
|
||||||
if args.single_node {
|
|
||||||
config.single_node = true;
|
|
||||||
}
|
|
||||||
if config.chainfire_endpoint.is_none() {
|
|
||||||
if let Ok(chainfire_endpoint) = std::env::var("PRISMNET_CHAINFIRE_ENDPOINT") {
|
|
||||||
let trimmed = chainfire_endpoint.trim();
|
|
||||||
if !trimmed.is_empty() {
|
|
||||||
config.chainfire_endpoint = Some(trimmed.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if config.flaredb_endpoint.is_none() {
|
|
||||||
if let Ok(flaredb_endpoint) = std::env::var("PRISMNET_FLAREDB_ENDPOINT") {
|
|
||||||
let trimmed = flaredb_endpoint.trim();
|
|
||||||
if !trimmed.is_empty() {
|
|
||||||
config.flaredb_endpoint = Some(trimmed.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Ok(metadata_backend) = std::env::var("PRISMNET_METADATA_BACKEND") {
|
|
||||||
let trimmed = metadata_backend.trim();
|
|
||||||
if !trimmed.is_empty() {
|
|
||||||
config.metadata_backend = parse_metadata_backend(trimmed)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if config.metadata_database_url.is_none() {
|
|
||||||
if let Ok(metadata_database_url) = std::env::var("PRISMNET_METADATA_DATABASE_URL") {
|
|
||||||
let trimmed = metadata_database_url.trim();
|
|
||||||
if !trimmed.is_empty() {
|
|
||||||
config.metadata_database_url = Some(trimmed.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !config.single_node {
|
|
||||||
if let Ok(single_node) = std::env::var("PRISMNET_SINGLE_NODE") {
|
|
||||||
let parsed = single_node.trim().to_ascii_lowercase();
|
|
||||||
config.single_node = matches!(parsed.as_str(), "1" | "true" | "yes" | "on");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize tracing
|
// Initialize tracing
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
.with_env_filter(
|
.with_env_filter(
|
||||||
|
|
@ -187,20 +114,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
config.flaredb_endpoint.clone(),
|
config.flaredb_endpoint.clone(),
|
||||||
config.chainfire_endpoint.clone(),
|
config.chainfire_endpoint.clone(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow!("Failed to init FlareDB metadata store: {}", e))?,
|
.map_err(|e| anyhow!("Failed to init FlareDB metadata store: {}", e))?,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
|
MetadataBackend::Postgres | MetadataBackend::Sqlite => {
|
||||||
let database_url = config
|
let database_url = config.metadata_database_url.as_deref().ok_or_else(|| {
|
||||||
.metadata_database_url
|
anyhow!(
|
||||||
.as_deref()
|
"metadata_database_url is required when metadata_backend={}",
|
||||||
.ok_or_else(|| {
|
metadata_backend_name(config.metadata_backend)
|
||||||
anyhow!(
|
)
|
||||||
"metadata_database_url is required when metadata_backend={} (env: PRISMNET_METADATA_DATABASE_URL)",
|
})?;
|
||||||
metadata_backend_name(config.metadata_backend)
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
|
ensure_sql_backend_matches_url(config.metadata_backend, database_url)?;
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
" Metadata backend: {} @ {}",
|
" Metadata backend: {} @ {}",
|
||||||
|
|
@ -216,8 +140,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
};
|
};
|
||||||
|
|
||||||
// Initialize OVN client (default: mock)
|
// Initialize OVN client (default: mock)
|
||||||
let ovn =
|
let ovn = Arc::new(
|
||||||
Arc::new(OvnClient::from_env().map_err(|e| anyhow!("Failed to init OVN client: {}", e))?);
|
OvnClient::from_config(&config.ovn)
|
||||||
|
.map_err(|e| anyhow!("Failed to init OVN client: {}", e))?,
|
||||||
|
);
|
||||||
|
|
||||||
// Initialize IAM authentication service
|
// Initialize IAM authentication service
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
|
|
@ -374,19 +300,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_metadata_backend(value: &str) -> Result<MetadataBackend, Box<dyn std::error::Error>> {
|
|
||||||
match value.trim().to_ascii_lowercase().as_str() {
|
|
||||||
"flaredb" => Ok(MetadataBackend::FlareDb),
|
|
||||||
"postgres" => Ok(MetadataBackend::Postgres),
|
|
||||||
"sqlite" => Ok(MetadataBackend::Sqlite),
|
|
||||||
other => Err(format!(
|
|
||||||
"invalid metadata backend '{}'; expected one of: flaredb, postgres, sqlite",
|
|
||||||
other
|
|
||||||
)
|
|
||||||
.into()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
|
fn metadata_backend_name(backend: MetadataBackend) -> &'static str {
|
||||||
match backend {
|
match backend {
|
||||||
MetadataBackend::FlareDb => "flaredb",
|
MetadataBackend::FlareDb => "flaredb",
|
||||||
|
|
|
||||||
|
|
@ -60,12 +60,8 @@ impl NetworkMetadataStore {
|
||||||
endpoint: Option<String>,
|
endpoint: Option<String>,
|
||||||
pd_endpoint: Option<String>,
|
pd_endpoint: Option<String>,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let endpoint = endpoint.unwrap_or_else(|| {
|
let endpoint = endpoint.unwrap_or_else(|| "127.0.0.1:2479".to_string());
|
||||||
std::env::var("PRISMNET_FLAREDB_ENDPOINT")
|
|
||||||
.unwrap_or_else(|_| "127.0.0.1:2479".to_string())
|
|
||||||
});
|
|
||||||
let pd_endpoint = pd_endpoint
|
let pd_endpoint = pd_endpoint
|
||||||
.or_else(|| std::env::var("PRISMNET_CHAINFIRE_ENDPOINT").ok())
|
|
||||||
.map(|value| normalize_transport_addr(&value))
|
.map(|value| normalize_transport_addr(&value))
|
||||||
.unwrap_or_else(|| endpoint.clone());
|
.unwrap_or_else(|| endpoint.clone());
|
||||||
|
|
||||||
|
|
@ -168,7 +164,9 @@ impl NetworkMetadataStore {
|
||||||
)
|
)
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| MetadataError::Storage(format!("Failed to initialize Postgres schema: {}", e)))?;
|
.map_err(|e| {
|
||||||
|
MetadataError::Storage(format!("Failed to initialize Postgres schema: {}", e))
|
||||||
|
})?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -181,7 +179,9 @@ impl NetworkMetadataStore {
|
||||||
)
|
)
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| MetadataError::Storage(format!("Failed to initialize SQLite schema: {}", e)))?;
|
.map_err(|e| {
|
||||||
|
MetadataError::Storage(format!("Failed to initialize SQLite schema: {}", e))
|
||||||
|
})?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -208,9 +208,7 @@ impl NetworkMetadataStore {
|
||||||
.bind(value)
|
.bind(value)
|
||||||
.execute(pool.as_ref())
|
.execute(pool.as_ref())
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| MetadataError::Storage(format!("Postgres put failed: {}", e)))?;
|
||||||
MetadataError::Storage(format!("Postgres put failed: {}", e))
|
|
||||||
})?;
|
|
||||||
}
|
}
|
||||||
SqlStorageBackend::Sqlite(pool) => {
|
SqlStorageBackend::Sqlite(pool) => {
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,12 @@
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use prismnet_types::{DhcpOptions, Port, PortId, SecurityGroupId, SecurityGroupRule, SecurityGroupRuleId, VpcId};
|
use prismnet_types::{
|
||||||
|
DhcpOptions, Port, PortId, SecurityGroupId, SecurityGroupRule, SecurityGroupRuleId, VpcId,
|
||||||
|
};
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
|
use crate::config::{OvnConfig, OvnMode as ConfigOvnMode};
|
||||||
use crate::ovn::mock::MockOvnState;
|
use crate::ovn::mock::MockOvnState;
|
||||||
|
|
||||||
/// OVN client mode
|
/// OVN client mode
|
||||||
|
|
@ -31,22 +34,19 @@ pub struct OvnClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl OvnClient {
|
impl OvnClient {
|
||||||
/// Build an OVN client from environment variables (default: mock)
|
/// Build an OVN client from configuration (default: mock)
|
||||||
/// - PRISMNET_OVN_MODE: "mock" (default) or "real"
|
pub fn from_config(config: &OvnConfig) -> OvnResult<Self> {
|
||||||
/// - PRISMNET_OVN_NB_ADDR: ovsdb northbound address (real mode only)
|
match config.mode {
|
||||||
pub fn from_env() -> OvnResult<Self> {
|
ConfigOvnMode::Mock => Ok(Self::new_mock()),
|
||||||
let mode = std::env::var("PRISMNET_OVN_MODE").unwrap_or_else(|_| "mock".to_string());
|
ConfigOvnMode::Real => {
|
||||||
match mode.to_lowercase().as_str() {
|
if config.nb_addr.trim().is_empty() {
|
||||||
"mock" => Ok(Self::new_mock()),
|
Err(OvnError::InvalidArgument(
|
||||||
"real" => {
|
"OVN nb_addr must not be empty in real mode".to_string(),
|
||||||
let nb_addr = std::env::var("PRISMNET_OVN_NB_ADDR")
|
))
|
||||||
.unwrap_or_else(|_| "tcp:127.0.0.1:6641".to_string());
|
} else {
|
||||||
Ok(Self::new_real(nb_addr))
|
Ok(Self::new_real(config.nb_addr.clone()))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
other => Err(OvnError::InvalidArgument(format!(
|
|
||||||
"Unknown OVN mode: {}",
|
|
||||||
other
|
|
||||||
))),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -305,10 +305,7 @@ impl OvnClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
if !options.dns_servers.is_empty() {
|
if !options.dns_servers.is_empty() {
|
||||||
opts.push(format!(
|
opts.push(format!("dns_server={{{}}}", options.dns_servers.join(",")));
|
||||||
"dns_server={{{}}}",
|
|
||||||
options.dns_servers.join(",")
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
opts.push(format!("lease_time={}", options.lease_time));
|
opts.push(format!("lease_time={}", options.lease_time));
|
||||||
|
|
@ -389,7 +386,10 @@ impl OvnClient {
|
||||||
|
|
||||||
if !output.status.success() {
|
if !output.status.success() {
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
return Err(OvnError::Command(format!("lr-add query failed: {}", stderr)));
|
return Err(OvnError::Command(format!(
|
||||||
|
"lr-add query failed: {}",
|
||||||
|
stderr
|
||||||
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
let uuid = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
let uuid = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||||
|
|
@ -456,12 +456,8 @@ impl OvnClient {
|
||||||
let ls_name = Self::logical_switch_name(switch_id);
|
let ls_name = Self::logical_switch_name(switch_id);
|
||||||
|
|
||||||
// ovn-nbctl lsp-add <switch> <port-name>
|
// ovn-nbctl lsp-add <switch> <port-name>
|
||||||
self.run_nbctl(vec![
|
self.run_nbctl(vec!["lsp-add".into(), ls_name, switch_port_name.clone()])
|
||||||
"lsp-add".into(),
|
.await?;
|
||||||
ls_name,
|
|
||||||
switch_port_name.clone(),
|
|
||||||
])
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
// ovn-nbctl lsp-set-type <port> router
|
// ovn-nbctl lsp-set-type <port> router
|
||||||
self.run_nbctl(vec![
|
self.run_nbctl(vec![
|
||||||
|
|
@ -698,10 +694,7 @@ mod tests {
|
||||||
let client = OvnClient::new_mock();
|
let client = OvnClient::new_mock();
|
||||||
|
|
||||||
// Create a router
|
// Create a router
|
||||||
let router_id = client
|
let router_id = client.create_logical_router("test-router").await.unwrap();
|
||||||
.create_logical_router("test-router")
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
assert!(!router_id.is_empty());
|
assert!(!router_id.is_empty());
|
||||||
assert!(router_id.starts_with("router-"));
|
assert!(router_id.starts_with("router-"));
|
||||||
|
|
@ -732,10 +725,7 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
// Create router
|
// Create router
|
||||||
let router_id = client
|
let router_id = client.create_logical_router("test-router").await.unwrap();
|
||||||
.create_logical_router("test-router")
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Add router port
|
// Add router port
|
||||||
let mac = "02:00:00:00:00:01";
|
let mac = "02:00:00:00:00:01";
|
||||||
|
|
@ -760,10 +750,7 @@ mod tests {
|
||||||
let client = OvnClient::new_mock();
|
let client = OvnClient::new_mock();
|
||||||
|
|
||||||
// Create router
|
// Create router
|
||||||
let router_id = client
|
let router_id = client.create_logical_router("test-router").await.unwrap();
|
||||||
.create_logical_router("test-router")
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Configure SNAT
|
// Configure SNAT
|
||||||
let external_ip = "203.0.113.10";
|
let external_ip = "203.0.113.10";
|
||||||
|
|
@ -791,10 +778,7 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
// Create router
|
// Create router
|
||||||
let router_id = client
|
let router_id = client.create_logical_router("test-router").await.unwrap();
|
||||||
.create_logical_router("test-router")
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Add router port
|
// Add router port
|
||||||
let mac = "02:00:00:00:00:01";
|
let mac = "02:00:00:00:00:01";
|
||||||
|
|
@ -840,10 +824,7 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
// Create router
|
// Create router
|
||||||
let router_id = client
|
let router_id = client.create_logical_router("test-router").await.unwrap();
|
||||||
.create_logical_router("test-router")
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Add router ports to both switches
|
// Add router ports to both switches
|
||||||
let port1_id = client
|
let port1_id = client
|
||||||
|
|
@ -876,10 +857,7 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
// Step 2: Create router
|
// Step 2: Create router
|
||||||
let router_id = client
|
let router_id = client.create_logical_router("vpc-router").await.unwrap();
|
||||||
.create_logical_router("vpc-router")
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Step 3: Attach router to switch
|
// Step 3: Attach router to switch
|
||||||
let mac = "02:00:00:00:00:01";
|
let mac = "02:00:00:00:00:01";
|
||||||
|
|
@ -920,10 +898,7 @@ mod tests {
|
||||||
let client = OvnClient::new_mock();
|
let client = OvnClient::new_mock();
|
||||||
|
|
||||||
// Create router
|
// Create router
|
||||||
let router_id = client
|
let router_id = client.create_logical_router("test-router").await.unwrap();
|
||||||
.create_logical_router("test-router")
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Configure multiple SNAT rules
|
// Configure multiple SNAT rules
|
||||||
client
|
client
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue