photoncloud-monorepo/nix/single-node/base.nix

{ config, lib, pkgs, ... }:

let
  cfg = config.ultracloud.quickstart;

  localChainfire = "127.0.0.1:${toString config.services.chainfire.port}";
  localChainfireHttp = "http://127.0.0.1:${toString config.services.chainfire.httpPort}";
  localFlaredb = "127.0.0.1:${toString config.services.flaredb.port}";
  localIamGrpc = "127.0.0.1:${toString config.services.iam.port}";
  localIamHttp = "http://127.0.0.1:${toString config.services.iam.httpPort}";
  localPrismnetGrpc = "127.0.0.1:${toString config.services.prismnet.port}";
  localPrismnetHttp = "http://127.0.0.1:${toString config.services.prismnet.httpPort}";
  localPlasmavmcHttp = "http://127.0.0.1:${toString config.services.plasmavmc.httpPort}";
  localLightningstorGrpc = "127.0.0.1:${toString config.services.lightningstor.port}";
  localCreditserviceGrpc = "127.0.0.1:${toString config.services.creditservice.grpcPort}";
  localFiberlbHttp = "http://127.0.0.1:${toString config.services.fiberlb.port}";
  localFlashdnsHttp = "http://127.0.0.1:${toString config.services.flashdns.port}";
  localCoronafsHttp = "http://127.0.0.1:${toString config.services.coronafs.port}";

  requiredPackages = with pkgs; [
    chainfire-server
    flaredb-server
    iam-server
    prismnet-server
    plasmavmc-server
    curl
    iproute2
    jq
    qemu
  ];

  optionalPackages =
    (lib.optionals cfg.enableLightningStor [ pkgs.lightningstor-server ])
    ++ (lib.optionals cfg.enableCoronafs [ pkgs.coronafs-server ])
    ++ (lib.optionals cfg.enableFlashDNS [ pkgs.flashdns-server ])
    ++ (lib.optionals cfg.enableFiberLB [ pkgs.fiberlb-server ])
    ++ (lib.optionals cfg.enableApiGateway [ pkgs.apigateway-server ])
    ++ (lib.optionals cfg.enableNightlight [ pkgs.nightlight-server ])
    ++ (lib.optionals cfg.enableCreditService [ pkgs.creditservice-server ])
    ++ (lib.optionals cfg.enableK8sHost [ pkgs.k8shost-server ]);

  requiredUnits = [
    "chainfire.service"
    "flaredb.service"
    "iam.service"
    "prismnet.service"
    "plasmavmc.service"
  ];

  optionalUnits =
    (lib.optionals cfg.enableLightningStor [ "lightningstor.service" ])
    ++ (lib.optionals cfg.enableCoronafs [ "coronafs.service" ])
    ++ (lib.optionals cfg.enableFlashDNS [ "flashdns.service" ])
    ++ (lib.optionals cfg.enableFiberLB [ "fiberlb.service" ])
    ++ (lib.optionals cfg.enableApiGateway [ "apigateway.service" ])
    ++ (lib.optionals cfg.enableNightlight [ "nightlight.service" ])
    ++ (lib.optionals cfg.enableCreditService [ "creditservice.service" ])
    ++ (lib.optionals cfg.enableK8sHost [ "k8shost.service" ]);

  readyUnitArgs = lib.escapeShellArgs (requiredUnits ++ optionalUnits);

  healthChecks = [
    { name = "chainfire"; url = "http://127.0.0.1:8081/health"; }
    { name = "flaredb"; url = "http://127.0.0.1:8082/health"; }
    { name = "iam"; url = "http://127.0.0.1:8083/health"; }
    { name = "prismnet"; url = "http://127.0.0.1:8087/health"; }
    { name = "plasmavmc"; url = localPlasmavmcHttp + "/health"; }
  ]
  ++ lib.optionals cfg.enableCoronafs [
    { name = "coronafs"; url = localCoronafsHttp + "/healthz"; }
  ]
  ++ lib.optionals cfg.enableApiGateway [
    { name = "apigateway"; url = "http://127.0.0.1:8080/health"; }
  ]
  ++ lib.optionals cfg.enableNightlight [
    { name = "nightlight"; url = "http://127.0.0.1:9101/healthz"; }
  ]
  ++ lib.optionals cfg.enableCreditService [
    { name = "creditservice"; url = "http://127.0.0.1:3011/health"; }
  ]
  ++ lib.optionals cfg.enableK8sHost [
    { name = "k8shost"; url = "http://127.0.0.1:8085/health"; }
  ];

  healthCheckScript = lib.concatMapStringsSep "\n" (check: ''
    wait_for_health ${lib.escapeShellArg check.name} ${lib.escapeShellArg check.url}
  '') healthChecks;
in
{
  options.ultracloud.quickstart = {
    enable = lib.mkEnableOption "UltraCloud single-node quickstart profile";

    hostName = lib.mkOption {
      type = lib.types.str;
      default = "single-node-quickstart";
      description = "Hostname used by the single-node quickstart profile.";
    };

    nodeId = lib.mkOption {
      type = lib.types.str;
      default = "single01";
      description = "Node identifier for the single-node quickstart profile.";
    };

    enableLightningStor = lib.mkEnableOption "LightningStor object/image storage add-on for the quickstart";
    enableCoronafs = lib.mkEnableOption "CoronaFS shared-volume add-on for the quickstart";
    enableFlashDNS = lib.mkEnableOption "FlashDNS add-on for the quickstart";
    enableFiberLB = lib.mkEnableOption "FiberLB add-on for the quickstart";
    enableApiGateway = lib.mkEnableOption "API gateway add-on for the quickstart";
    enableNightlight = lib.mkEnableOption "Nightlight metrics add-on for the quickstart";
    enableCreditService = lib.mkEnableOption "CreditService reference add-on for the quickstart";
    enableK8sHost = lib.mkEnableOption "K8sHost add-on for the quickstart";
  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = !cfg.enableK8sHost || (cfg.enableFiberLB && cfg.enableFlashDNS);
        message = "ultracloud.quickstart.enableK8sHost requires ultracloud.quickstart.enableFiberLB and ultracloud.quickstart.enableFlashDNS.";
      }
    ];

    networking.hostName = lib.mkDefault cfg.hostName;
    networking.useDHCP = lib.mkDefault true;
    networking.vswitches = lib.mkDefault {};

    networking.firewall.allowedTCPPorts = [
      22
      2379
      2380
      2381
      2479
      2480
      50080
      50081
      50082
      8081
      8082
      8083
      8084
      8087
    ]
    ++ (lib.optionals cfg.enableLightningStor [ 50086 50090 9000 ])
    ++ (lib.optionals cfg.enableCoronafs [ 50088 ])
    ++ (lib.optionals cfg.enableFlashDNS [ 50084 ])
    ++ (lib.optionals cfg.enableFiberLB [ 50085 ])
    ++ (lib.optionals cfg.enableApiGateway [ 8080 ])
    ++ (lib.optionals cfg.enableNightlight [ 9091 9101 ])
    ++ (lib.optionals cfg.enableCreditService [ 3010 3011 ])
    ++ (lib.optionals cfg.enableK8sHost [ 50087 8085 ]);

    networking.firewall.allowedUDPPorts =
      [ 2381 4789 ]
      ++ (lib.optionals cfg.enableFlashDNS [ 5353 ]);

    boot.kernelModules = [ "kvm-intel" "kvm-amd" "tun" ];
    boot.extraModprobeConfig = ''
      options kvm_intel nested=1
      options kvm_amd nested=1
    '';
    boot.kernel.sysctl = {
      "fs.file-max" = 1000000;
      "net.core.netdev_max_backlog" = 5000;
      "net.core.rmem_max" = 134217728;
      "net.core.wmem_max" = 134217728;
      "net.ipv4.ip_forward" = 1;
      "net.ipv6.conf.all.forwarding" = 1;
      "vm.swappiness" = 10;
    };

    services.chainfire = {
      enable = true;
      nodeId = cfg.nodeId;
      initialPeers = [ "${cfg.nodeId}=127.0.0.1:2380" ];
    };

    services.flaredb = {
      enable = true;
      nodeId = cfg.nodeId;
      initialPeers = [ "${cfg.nodeId}=127.0.0.1:2479" ];
      pdAddr = localChainfire;
    };

    services.iam = {
      enable = true;
      chainfireAddr = localChainfire;
      flaredbAddr = localFlaredb;
      allowRandomSigningKey = true;
      allowUnauthenticatedAdmin = true;
    };

    services.prismnet = {
      enable = true;
      iamAddr = localIamGrpc;
      chainfireAddr = localChainfire;
      flaredbAddr = localFlaredb;
    };

    services.plasmavmc = {
      enable = true;
      mode = "all-in-one";
      prismnetAddr = localPrismnetGrpc;
      iamAddr = localIamGrpc;
      chainfireAddr = localChainfire;
      flaredbAddr = localFlaredb;
      lightningstorAddr = if cfg.enableLightningStor then localLightningstorGrpc else null;
      coronafsControllerEndpoint = if cfg.enableCoronafs then localCoronafsHttp else null;
      coronafsNodeEndpoint = if cfg.enableCoronafs then localCoronafsHttp else null;
    };

    services.lightningstor = lib.mkIf cfg.enableLightningStor {
      enable = true;
      mode = "all-in-one";
      objectStorageBackend = "local_fs";
      iamAddr = localIamGrpc;
      chainfireAddr = localChainfire;
      flaredbAddr = localFlaredb;
    };

    services.coronafs = lib.mkIf cfg.enableCoronafs {
      enable = true;
      metadataBackend = "chainfire";
      chainfireApiUrl = localChainfireHttp;
      advertiseHost = "127.0.0.1";
    };

    services.flashdns = lib.mkIf cfg.enableFlashDNS {
      enable = true;
      iamAddr = localIamGrpc;
      chainfireAddr = localChainfire;
      flaredbAddr = localFlaredb;
    };

    services.fiberlb = lib.mkIf cfg.enableFiberLB {
      enable = true;
      iamAddr = localIamGrpc;
      chainfireAddr = localChainfire;
      flaredbAddr = localFlaredb;
    };

    services.creditservice = lib.mkIf cfg.enableCreditService {
      enable = true;
      iamAddr = localIamGrpc;
      chainfireAddr = localChainfire;
      flaredbAddr = localFlaredb;
    };

    services.nightlight = lib.mkIf cfg.enableNightlight {
      enable = true;
    };

    services.apigateway = lib.mkIf cfg.enableApiGateway {
      enable = true;
      authProviders = [
        {
          name = "iam";
          providerType = "grpc";
          endpoint = localIamHttp;
        }
      ];
      creditProviders = lib.optionals cfg.enableCreditService [
        {
          name = "creditservice";
          providerType = "grpc";
          endpoint = "http://${localCreditserviceGrpc}";
        }
      ];
      routes =
        [
          {
            name = "iam-rest";
            pathPrefix = "/iam";
            upstream = localIamHttp;
            stripPrefix = true;
            auth = {
              provider = "iam";
              mode = "required";
            };
          }
        ]
        ++ lib.optionals cfg.enableCreditService [
          {
            name = "credit-rest";
            pathPrefix = "/credit";
            upstream = "http://127.0.0.1:${toString config.services.creditservice.httpPort}";
            stripPrefix = true;
            auth = {
              provider = "iam";
              mode = "required";
            };
            credit = {
              provider = "creditservice";
              mode = "optional";
              units = 1;
              commitOn = "success";
            };
          }
        ];
    };

    services.k8shost = lib.mkIf cfg.enableK8sHost {
      enable = true;
      iamAddr = localIamHttp;
      chainfireAddr = "http://${localChainfire}";
      prismnetAddr = localPrismnetHttp;
      flaredbPdAddr = localChainfire;
      flaredbDirectAddr = localFlaredb;
      fiberlbAddr = localFiberlbHttp;
      flashdnsAddr = localFlashdnsHttp;
      creditserviceAddr =
        if cfg.enableCreditService
        then "http://${localCreditserviceGrpc}"
        else null;
    };

    environment.systemPackages = requiredPackages ++ optionalPackages;

    systemd.services.ultracloud-single-node-quickstart-ready = {
      description = "Verify UltraCloud single-node quickstart readiness";
      wantedBy = [ "multi-user.target" ];
      after = requiredUnits ++ optionalUnits;
      wants = requiredUnits ++ optionalUnits;
      path = [ pkgs.coreutils pkgs.curl pkgs.qemu pkgs.systemd ];
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
      };
      script = ''
        set -euo pipefail

        wait_for_health() {
          local name="$1"
          local url="$2"
          local deadline=$((SECONDS + 180))
          while true; do
            if curl -fsS "$url" >/dev/null 2>&1; then
              return 0
            fi
            if [ "$SECONDS" -ge "$deadline" ]; then
              echo "timed out waiting for $name at $url" >&2
              return 1
            fi
            sleep 1
          done
        }

        systemctl is-active ${readyUnitArgs}
        ${healthCheckScript}
        test -x ${pkgs.qemu}/bin/qemu-system-x86_64
        test -x ${pkgs.qemu}/bin/qemu-img
        test -c /dev/net/tun
        if [ -e /dev/kvm ]; then
          test -r /dev/kvm
        fi
      '';
    };

    system.stateVersion = "24.11";
  };
}