375 lines
11 KiB
Nix
375 lines
11 KiB
Nix
{ config, lib, pkgs, ... }:
|
|
|
|
let
|
|
cfg = config.services.lightningstor;
|
|
tomlFormat = pkgs.formats.toml { };
|
|
|
|
serverEnabled = cfg.mode != "data";
|
|
nodeEnabled = cfg.mode != "metadata";
|
|
|
|
serverDataDir =
|
|
if cfg.mode == "all-in-one"
|
|
then "${toString cfg.dataDir}/server"
|
|
else toString cfg.dataDir;
|
|
|
|
nodeDataDir =
|
|
if cfg.mode == "all-in-one"
|
|
then "${toString cfg.dataDir}/node"
|
|
else toString cfg.dataDir;
|
|
|
|
nodeListenPort =
|
|
if cfg.mode == "data"
|
|
then cfg.port
|
|
else cfg.nodePort;
|
|
|
|
localDependencies =
|
|
lib.optionals
|
|
(serverEnabled && (config.services ? iam) && config.services.iam.enable)
|
|
[ "iam.service" ]
|
|
++ lib.optionals
|
|
(serverEnabled && (config.services ? flaredb) && config.services.flaredb.enable)
|
|
[ "flaredb.service" ];
|
|
|
|
effectiveNodeEndpoints = lib.unique (
|
|
cfg.distributedNodeEndpoints
|
|
++ lib.optionals
|
|
(serverEnabled && nodeEnabled && cfg.objectStorageBackend == "distributed")
|
|
[ "http://127.0.0.1:${toString nodeListenPort}" ]
|
|
);
|
|
|
|
distributedRedundancy =
|
|
if cfg.redundancyMode == "replicated"
|
|
then {
|
|
type = "replicated";
|
|
replica_count = cfg.replicaCount;
|
|
read_quorum = cfg.readQuorum;
|
|
write_quorum = cfg.writeQuorum;
|
|
}
|
|
else {
|
|
type = "erasure_coded";
|
|
data_shards = cfg.dataShards;
|
|
parity_shards = cfg.parityShards;
|
|
};
|
|
|
|
serverBaseConfig = {
|
|
grpc_addr = "0.0.0.0:${toString cfg.port}";
|
|
s3_addr = "0.0.0.0:${toString cfg.s3Port}";
|
|
log_level = "info";
|
|
data_dir = serverDataDir;
|
|
sync_on_write = cfg.syncOnWrite;
|
|
object_storage_backend = cfg.objectStorageBackend;
|
|
auth = {
|
|
iam_server_addr =
|
|
if cfg.iamAddr != null
|
|
then cfg.iamAddr
|
|
else "127.0.0.1:50080";
|
|
};
|
|
distributed = {
|
|
node_endpoints = effectiveNodeEndpoints;
|
|
connection_timeout_ms = cfg.distributedConnectionTimeoutMs;
|
|
request_timeout_ms = cfg.distributedRequestTimeoutMs;
|
|
redundancy = distributedRedundancy;
|
|
}
|
|
// lib.optionalAttrs (cfg.distributedRegistryEndpoint != null) {
|
|
registry_endpoint = cfg.distributedRegistryEndpoint;
|
|
};
|
|
}
|
|
// lib.optionalAttrs (cfg.flaredbAddr != null) {
|
|
flaredb_endpoint = cfg.flaredbAddr;
|
|
}
|
|
// {
|
|
metadata_backend = cfg.metadataBackend;
|
|
single_node = cfg.singleNode;
|
|
}
|
|
// lib.optionalAttrs (cfg.databaseUrl != null) {
|
|
metadata_database_url = cfg.databaseUrl;
|
|
}
|
|
// lib.optionalAttrs (cfg.chainfireAddr != null) {
|
|
chainfire_endpoint = "http://${cfg.chainfireAddr}";
|
|
};
|
|
|
|
lightningstorConfigFile =
|
|
tomlFormat.generate "lightningstor.toml" (lib.recursiveUpdate serverBaseConfig cfg.settings);
|
|
|
|
lightningstorNodeConfigFile = tomlFormat.generate "lightningstor-node.toml" {
|
|
node_id = cfg.nodeId;
|
|
grpc_addr = "0.0.0.0:${toString nodeListenPort}";
|
|
data_dir = nodeDataDir;
|
|
zone = cfg.zone;
|
|
region = cfg.region;
|
|
log_level = "info";
|
|
max_capacity_bytes = cfg.maxCapacityBytes;
|
|
metrics_port = cfg.nodeMetricsPort;
|
|
sync_on_write = cfg.syncOnWrite;
|
|
};
|
|
|
|
serverCommand = lib.escapeShellArgs [
|
|
"${cfg.package}/bin/lightningstor-server"
|
|
"--config"
|
|
"${lightningstorConfigFile}"
|
|
];
|
|
|
|
nodeCommand = lib.escapeShellArgs [
|
|
"${cfg.nodePackage}/bin/lightningstor-node"
|
|
"--config"
|
|
"${lightningstorNodeConfigFile}"
|
|
];
|
|
|
|
allInOneLauncher = pkgs.writeShellScript "lightningstor-all-in-one" ''
|
|
set -euo pipefail
|
|
|
|
${nodeCommand} &
|
|
node_pid=$!
|
|
|
|
cleanup() {
|
|
if kill -0 "$node_pid" 2>/dev/null; then
|
|
kill "$node_pid" 2>/dev/null || true
|
|
wait "$node_pid" 2>/dev/null || true
|
|
fi
|
|
}
|
|
|
|
trap cleanup EXIT INT TERM
|
|
exec ${serverCommand}
|
|
'';
|
|
|
|
execStart =
|
|
if cfg.mode == "data"
|
|
then nodeCommand
|
|
else if cfg.mode == "metadata"
|
|
then serverCommand
|
|
else allInOneLauncher;
|
|
in
|
|
{
|
|
options.services.lightningstor = {
|
|
enable = lib.mkEnableOption "lightningstor service";
|
|
|
|
mode = lib.mkOption {
|
|
type = lib.types.enum [ "data" "metadata" "all-in-one" ];
|
|
default = "all-in-one";
|
|
description = "LightningStor operating mode: data (storage node), metadata (coordinator), or all-in-one";
|
|
};
|
|
|
|
port = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 50086;
|
|
description = "Port for lightningstor gRPC API. In mode=data this is reused as the node daemon gRPC port.";
|
|
};
|
|
|
|
nodePort = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 50090;
|
|
description = "Port for lightningstor-node when mode=all-in-one.";
|
|
};
|
|
|
|
s3Port = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 9000;
|
|
description = "Port for S3-compatible HTTP API";
|
|
};
|
|
|
|
iamAddr = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "IAM service endpoint address (host:port)";
|
|
example = "10.0.0.1:50080";
|
|
};
|
|
|
|
chainfireAddr = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "ChainFire endpoint address (host:port) for cluster coordination only";
|
|
example = "10.0.0.1:2379";
|
|
};
|
|
|
|
flaredbAddr = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "FlareDB endpoint address (host:port) for metadata/user data";
|
|
example = "10.0.0.1:2479";
|
|
};
|
|
|
|
metadataBackend = lib.mkOption {
|
|
type = lib.types.enum [ "flaredb" "postgres" "sqlite" ];
|
|
default = "flaredb";
|
|
description = "Metadata backend for LightningStor.";
|
|
};
|
|
|
|
objectStorageBackend = lib.mkOption {
|
|
type = lib.types.enum [ "local_fs" "distributed" ];
|
|
default = "local_fs";
|
|
description = "Object data backend for metadata/all-in-one modes.";
|
|
};
|
|
|
|
distributedNodeEndpoints = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [];
|
|
description = "Static LightningStor node gRPC endpoints for distributed object storage.";
|
|
example = [ "http://10.0.0.21:50086" "http://10.0.0.22:50086" ];
|
|
};
|
|
|
|
distributedRegistryEndpoint = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "Registry endpoint for future dynamic node discovery. Currently informational only.";
|
|
};
|
|
|
|
distributedConnectionTimeoutMs = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 5000;
|
|
description = "Connection timeout in milliseconds for distributed LightningStor node RPCs.";
|
|
};
|
|
|
|
distributedRequestTimeoutMs = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 300000;
|
|
description = "Request timeout in milliseconds for distributed LightningStor node RPCs.";
|
|
};
|
|
|
|
redundancyMode = lib.mkOption {
|
|
type = lib.types.enum [ "replicated" "erasure_coded" ];
|
|
default = "replicated";
|
|
description = "Redundancy strategy for distributed object storage.";
|
|
};
|
|
|
|
replicaCount = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 3;
|
|
description = "Replica count when redundancyMode=replicated.";
|
|
};
|
|
|
|
readQuorum = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 1;
|
|
description = "Read quorum when redundancyMode=replicated.";
|
|
};
|
|
|
|
writeQuorum = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 2;
|
|
description = "Write quorum when redundancyMode=replicated.";
|
|
};
|
|
|
|
dataShards = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 4;
|
|
description = "Data shard count when redundancyMode=erasure_coded.";
|
|
};
|
|
|
|
parityShards = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 2;
|
|
description = "Parity shard count when redundancyMode=erasure_coded.";
|
|
};
|
|
|
|
nodeId = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = config.networking.hostName;
|
|
description = "Node ID for lightningstor-node.";
|
|
};
|
|
|
|
zone = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "";
|
|
description = "Placement zone for lightningstor-node.";
|
|
};
|
|
|
|
region = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "";
|
|
description = "Placement region for lightningstor-node.";
|
|
};
|
|
|
|
maxCapacityBytes = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 0;
|
|
description = "Maximum capacity for lightningstor-node (0 = unlimited).";
|
|
};
|
|
|
|
syncOnWrite = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = "Whether filesystem-backed LightningStor writes are flushed before success is returned.";
|
|
};
|
|
|
|
nodeMetricsPort = lib.mkOption {
|
|
type = lib.types.port;
|
|
default = 9098;
|
|
description = "Prometheus metrics port for lightningstor-node.";
|
|
};
|
|
|
|
databaseUrl = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "SQL database URL for metadata (required when metadataBackend is postgres/sqlite).";
|
|
example = "postgres://lightningstor:secret@10.0.0.10:5432/lightningstor";
|
|
};
|
|
|
|
singleNode = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = "Enable single-node mode (required when metadata backend is SQLite)";
|
|
};
|
|
|
|
dataDir = lib.mkOption {
|
|
type = lib.types.path;
|
|
default = "/var/lib/lightningstor";
|
|
description = "Data directory for lightningstor";
|
|
};
|
|
|
|
settings = lib.mkOption {
|
|
type = lib.types.attrs;
|
|
default = {};
|
|
description = "Additional configuration settings merged into the lightningstor-server config.";
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.lightningstor-server or (throw "lightningstor-server package not found");
|
|
description = "Package to use for lightningstor-server.";
|
|
};
|
|
|
|
nodePackage = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.lightningstor-node or (throw "lightningstor-node package not found");
|
|
description = "Package to use for lightningstor-node.";
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
users.users.lightningstor = {
|
|
isSystemUser = true;
|
|
group = "lightningstor";
|
|
description = "LightningStor service user";
|
|
home = cfg.dataDir;
|
|
};
|
|
|
|
users.groups.lightningstor = { };
|
|
|
|
systemd.services.lightningstor = {
|
|
description = "LightningStor Object Storage Service";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network.target" ] ++ localDependencies;
|
|
requires = localDependencies;
|
|
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
User = "lightningstor";
|
|
Group = "lightningstor";
|
|
Restart = "on-failure";
|
|
RestartSec = "10s";
|
|
StateDirectory = "lightningstor";
|
|
StateDirectoryMode = "0750";
|
|
NoNewPrivileges = true;
|
|
PrivateTmp = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
ReadWritePaths = [ cfg.dataDir ];
|
|
ExecStart = execStart;
|
|
};
|
|
|
|
environment = {
|
|
RUST_LOG = "info";
|
|
};
|
|
};
|
|
};
|
|
}
|