photoncloud-monorepo/nix/modules/lightningstor.nix

{ config, lib, pkgs, ... }:

let
  cfg = config.services.lightningstor;
  tomlFormat = pkgs.formats.toml { };

  serverEnabled = cfg.mode != "data";
  nodeEnabled = cfg.mode != "metadata";

  serverDataDir =
    if cfg.mode == "all-in-one"
    then "${toString cfg.dataDir}/server"
    else toString cfg.dataDir;

  nodeDataDir =
    if cfg.mode == "all-in-one"
    then "${toString cfg.dataDir}/node"
    else toString cfg.dataDir;

  nodeListenPort =
    if cfg.mode == "data"
    then cfg.port
    else cfg.nodePort;

  localDependencies =
    lib.optionals
      (serverEnabled && (config.services ? iam) && config.services.iam.enable)
      [ "iam.service" ]
    ++ lib.optionals
      (serverEnabled && (config.services ? flaredb) && config.services.flaredb.enable)
      [ "flaredb.service" ];

  effectiveNodeEndpoints = lib.unique (
    cfg.distributedNodeEndpoints
    ++ lib.optionals
      (serverEnabled && nodeEnabled && cfg.objectStorageBackend == "distributed")
      [ "http://127.0.0.1:${toString nodeListenPort}" ]
  );

  distributedRedundancy =
    if cfg.redundancyMode == "replicated"
    then {
      type = "replicated";
      replica_count = cfg.replicaCount;
      read_quorum = cfg.readQuorum;
      write_quorum = cfg.writeQuorum;
    }
    else {
      type = "erasure_coded";
      data_shards = cfg.dataShards;
      parity_shards = cfg.parityShards;
    };

  serverBaseConfig = {
    grpc_addr = "0.0.0.0:${toString cfg.port}";
    s3_addr = "0.0.0.0:${toString cfg.s3Port}";
    log_level = "info";
    data_dir = serverDataDir;
    sync_on_write = cfg.syncOnWrite;
    object_storage_backend = cfg.objectStorageBackend;
    auth = {
      iam_server_addr =
        if cfg.iamAddr != null
        then cfg.iamAddr
        else "127.0.0.1:50080";
    };
    distributed = {
      node_endpoints = effectiveNodeEndpoints;
      connection_timeout_ms = cfg.distributedConnectionTimeoutMs;
      request_timeout_ms = cfg.distributedRequestTimeoutMs;
      redundancy = distributedRedundancy;
    }
    // lib.optionalAttrs (cfg.distributedRegistryEndpoint != null) {
      registry_endpoint = cfg.distributedRegistryEndpoint;
    };
  }
  // lib.optionalAttrs (cfg.flaredbAddr != null) {
    flaredb_endpoint = cfg.flaredbAddr;
  }
  // {
    metadata_backend = cfg.metadataBackend;
    single_node = cfg.singleNode;
  }
  // lib.optionalAttrs (cfg.databaseUrl != null) {
    metadata_database_url = cfg.databaseUrl;
  }
  // lib.optionalAttrs (cfg.chainfireAddr != null) {
    chainfire_endpoint = "http://${cfg.chainfireAddr}";
  };

  lightningstorConfigFile =
    tomlFormat.generate "lightningstor.toml" (lib.recursiveUpdate serverBaseConfig cfg.settings);

  lightningstorNodeConfigFile = tomlFormat.generate "lightningstor-node.toml" {
    node_id = cfg.nodeId;
    grpc_addr = "0.0.0.0:${toString nodeListenPort}";
    data_dir = nodeDataDir;
    zone = cfg.zone;
    region = cfg.region;
    log_level = "info";
    max_capacity_bytes = cfg.maxCapacityBytes;
    metrics_port = cfg.nodeMetricsPort;
    sync_on_write = cfg.syncOnWrite;
  };

  serverCommand = lib.escapeShellArgs [
    "${cfg.package}/bin/lightningstor-server"
    "--config"
    "${lightningstorConfigFile}"
  ];

  nodeCommand = lib.escapeShellArgs [
    "${cfg.nodePackage}/bin/lightningstor-node"
    "--config"
    "${lightningstorNodeConfigFile}"
  ];

  allInOneLauncher = pkgs.writeShellScript "lightningstor-all-in-one" ''
    set -euo pipefail

    ${nodeCommand} &
    node_pid=$!

    cleanup() {
      if kill -0 "$node_pid" 2>/dev/null; then
        kill "$node_pid" 2>/dev/null || true
        wait "$node_pid" 2>/dev/null || true
      fi
    }

    trap cleanup EXIT INT TERM
    exec ${serverCommand}
  '';

  execStart =
    if cfg.mode == "data"
    then nodeCommand
    else if cfg.mode == "metadata"
    then serverCommand
    else allInOneLauncher;
in
{
  options.services.lightningstor = {
    enable = lib.mkEnableOption "lightningstor service";

    mode = lib.mkOption {
      type = lib.types.enum [ "data" "metadata" "all-in-one" ];
      default = "all-in-one";
      description = "LightningStor operating mode: data (storage node), metadata (coordinator), or all-in-one";
    };

    port = lib.mkOption {
      type = lib.types.port;
      default = 50086;
      description = "Port for lightningstor gRPC API. In mode=data this is reused as the node daemon gRPC port.";
    };

    nodePort = lib.mkOption {
      type = lib.types.port;
      default = 50090;
      description = "Port for lightningstor-node when mode=all-in-one.";
    };

    s3Port = lib.mkOption {
      type = lib.types.port;
      default = 9000;
      description = "Port for S3-compatible HTTP API";
    };

    iamAddr = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "IAM service endpoint address (host:port)";
      example = "10.0.0.1:50080";
    };

    chainfireAddr = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "ChainFire endpoint address (host:port) for cluster coordination only";
      example = "10.0.0.1:2379";
    };

    flaredbAddr = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "FlareDB endpoint address (host:port) for metadata/user data";
      example = "10.0.0.1:2479";
    };

    metadataBackend = lib.mkOption {
      type = lib.types.enum [ "flaredb" "postgres" "sqlite" ];
      default = "flaredb";
      description = "Metadata backend for LightningStor.";
    };

    objectStorageBackend = lib.mkOption {
      type = lib.types.enum [ "local_fs" "distributed" ];
      default = "local_fs";
      description = "Object data backend for metadata/all-in-one modes.";
    };

    distributedNodeEndpoints = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [];
      description = "Static LightningStor node gRPC endpoints for distributed object storage.";
      example = [ "http://10.0.0.21:50086" "http://10.0.0.22:50086" ];
    };

    distributedRegistryEndpoint = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "Registry endpoint for future dynamic node discovery. Currently informational only.";
    };

    distributedConnectionTimeoutMs = lib.mkOption {
      type = lib.types.int;
      default = 5000;
      description = "Connection timeout in milliseconds for distributed LightningStor node RPCs.";
    };

    distributedRequestTimeoutMs = lib.mkOption {
      type = lib.types.int;
      default = 300000;
      description = "Request timeout in milliseconds for distributed LightningStor node RPCs.";
    };

    redundancyMode = lib.mkOption {
      type = lib.types.enum [ "replicated" "erasure_coded" ];
      default = "replicated";
      description = "Redundancy strategy for distributed object storage.";
    };

    replicaCount = lib.mkOption {
      type = lib.types.int;
      default = 3;
      description = "Replica count when redundancyMode=replicated.";
    };

    readQuorum = lib.mkOption {
      type = lib.types.int;
      default = 1;
      description = "Read quorum when redundancyMode=replicated.";
    };

    writeQuorum = lib.mkOption {
      type = lib.types.int;
      default = 2;
      description = "Write quorum when redundancyMode=replicated.";
    };

    dataShards = lib.mkOption {
      type = lib.types.int;
      default = 4;
      description = "Data shard count when redundancyMode=erasure_coded.";
    };

    parityShards = lib.mkOption {
      type = lib.types.int;
      default = 2;
      description = "Parity shard count when redundancyMode=erasure_coded.";
    };

    nodeId = lib.mkOption {
      type = lib.types.str;
      default = config.networking.hostName;
      description = "Node ID for lightningstor-node.";
    };

    zone = lib.mkOption {
      type = lib.types.str;
      default = "";
      description = "Placement zone for lightningstor-node.";
    };

    region = lib.mkOption {
      type = lib.types.str;
      default = "";
      description = "Placement region for lightningstor-node.";
    };

    maxCapacityBytes = lib.mkOption {
      type = lib.types.int;
      default = 0;
      description = "Maximum capacity for lightningstor-node (0 = unlimited).";
    };

    syncOnWrite = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = "Whether filesystem-backed LightningStor writes are flushed before success is returned.";
    };

    nodeMetricsPort = lib.mkOption {
      type = lib.types.port;
      default = 9098;
      description = "Prometheus metrics port for lightningstor-node.";
    };

    s3StreamingPutThresholdBytes = lib.mkOption {
      type = lib.types.int;
      default = 64 * 1024 * 1024;
      description = "Streaming PUT multipart threshold for the S3 frontend.";
    };

    s3InlinePutMaxBytes = lib.mkOption {
      type = lib.types.int;
      default = 128 * 1024 * 1024;
      description = "Maximum inline single-PUT size for the S3 frontend.";
    };

    s3MultipartPutConcurrency = lib.mkOption {
      type = lib.types.int;
      default = 4;
      description = "Maximum in-flight multipart PUT part uploads.";
    };

    s3MultipartFetchConcurrency = lib.mkOption {
      type = lib.types.int;
      default = 4;
      description = "Maximum concurrent multipart GET part fetches.";
    };

    databaseUrl = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "SQL database URL for metadata (required when metadataBackend is postgres/sqlite).";
      example = "postgres://lightningstor:secret@10.0.0.10:5432/lightningstor";
    };

    singleNode = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = "Enable single-node mode (required when metadata backend is SQLite)";
    };

    dataDir = lib.mkOption {
      type = lib.types.path;
      default = "/var/lib/lightningstor";
      description = "Data directory for lightningstor";
    };

    settings = lib.mkOption {
      type = lib.types.attrs;
      default = {};
      description = "Additional configuration settings merged into the lightningstor-server config.";
    };

    package = lib.mkOption {
      type = lib.types.package;
      default = pkgs.lightningstor-server or (throw "lightningstor-server package not found");
      description = "Package to use for lightningstor-server.";
    };

    nodePackage = lib.mkOption {
      type = lib.types.package;
      default = pkgs.lightningstor-node or (throw "lightningstor-node package not found");
      description = "Package to use for lightningstor-node.";
    };
  };

  config = lib.mkIf cfg.enable {
    users.users.lightningstor = {
      isSystemUser = true;
      group = "lightningstor";
      description = "LightningStor service user";
      home = cfg.dataDir;
    };

    users.groups.lightningstor = { };

    systemd.services.lightningstor = {
      description = "LightningStor Object Storage Service";
      wantedBy = [ "multi-user.target" ];
      after = [ "network.target" ] ++ localDependencies;
      requires = localDependencies;

      serviceConfig = {
        Type = "simple";
        User = "lightningstor";
        Group = "lightningstor";
        Restart = "on-failure";
        RestartSec = "10s";
        StateDirectory = "lightningstor";
        StateDirectoryMode = "0750";
        NoNewPrivileges = true;
        PrivateTmp = true;
        ProtectSystem = "strict";
        ProtectHome = true;
        ReadWritePaths = [ cfg.dataDir ];
        ExecStart = execStart;
      };

      environment = {
        RUST_LOG = "info";
        LIGHTNINGSTOR_S3_STREAMING_PUT_THRESHOLD_BYTES =
          toString cfg.s3StreamingPutThresholdBytes;
        LIGHTNINGSTOR_S3_INLINE_PUT_MAX_BYTES =
          toString cfg.s3InlinePutMaxBytes;
        LIGHTNINGSTOR_S3_MULTIPART_PUT_CONCURRENCY =
          toString cfg.s3MultipartPutConcurrency;
        LIGHTNINGSTOR_S3_MULTIPART_FETCH_CONCURRENCY =
          toString cfg.s3MultipartFetchConcurrency;
      };
    };
  };
}