photoncloud-monorepo/nix/nodes/vm-cluster/lib.nix

{ lib }:

let
  inherit (lib) concatStringsSep genAttrs mkIf;

  clusterName = "ultracloud-vm-cluster";
  clusterId = clusterName;
  environment = "dev";
  bgpAsn = 64512;

  bootstrapToken = "vm-cluster-bootstrap-token";
  adminToken = "vm-cluster-admin-token";

  chainfireApiPort = 2379;
  chainfireRaftPort = 2380;
  flaredbApiPort = 2479;
  flaredbRaftPort = 2480;
  iamPort = 50080;

  controlPlaneLabels = {
    tier = "control-plane";
    platform = "vm-cluster";
  };

  controlPlaneNodes = {
    node01 = {
      ip = "192.168.100.11";
      failureDomain = "rack-a";
    };
    node02 = {
      ip = "192.168.100.12";
      failureDomain = "rack-b";
    };
    node03 = {
      ip = "192.168.100.13";
      failureDomain = "rack-c";
    };
  };

  controlPlaneNodeNames = builtins.attrNames controlPlaneNodes;
  bootstrapNodeName = builtins.head controlPlaneNodeNames;

  mkEndpoint = nodeName: port:
    let
      node = controlPlaneNodes.${nodeName};
    in
    "${node.ip}:${toString port}";

  chainfireInitialPeers =
    map (nodeName: "${nodeName}=${mkEndpoint nodeName chainfireRaftPort}") controlPlaneNodeNames;

  flaredbInitialPeers =
    map (nodeName: "${nodeName}=${mkEndpoint nodeName flaredbRaftPort}") controlPlaneNodeNames;

  chainfireControlPlaneAddrs =
    concatStringsSep "," (map (nodeName: mkEndpoint nodeName chainfireApiPort) controlPlaneNodeNames);

  flaredbControlPlaneAddrs =
    concatStringsSep "," (map (nodeName: mkEndpoint nodeName flaredbApiPort) controlPlaneNodeNames);

  sharedDesiredSystem = {
    healthCheckCommand = [ "systemctl" "is-system-running" "--wait" ];
    rollbackOnFailure = true;
  };

  mkInstallPlan = nodeName: {
    nixosConfiguration = nodeName;
    diskoConfigPath = "nix/nodes/vm-cluster/${nodeName}/disko.nix";
    targetDisk = "/dev/vda";
  };

  mkClusterNode = nodeName:
    let
      node = controlPlaneNodes.${nodeName};
    in
    {
      role = "control-plane";
      ip = node.ip;
      services = [ "chainfire" "flaredb" "iam" ];
      labels = controlPlaneLabels;
      pool = "control";
      nodeClass = "control-plane";
      failureDomain = node.failureDomain;
      nixProfile = "profiles/control-plane";
      installPlan = mkInstallPlan nodeName;
      desiredSystem = sharedDesiredSystem;
      raftPort = chainfireRaftPort;
      apiPort = chainfireApiPort;
    };

  clusterNodes = genAttrs controlPlaneNodeNames mkClusterNode;

  nodeConfigurationPaths = {
    node01 = ./node01/configuration.nix;
    node02 = ./node02/configuration.nix;
    node03 = ./node03/configuration.nix;
  };

  nodeDiskoPaths = {
    node01 = ./node01/disko.nix;
    node02 = ./node02/disko.nix;
    node03 = ./node03/disko.nix;
  };
in
{
  inherit
    adminToken
    bgpAsn
    bootstrapNodeName
    bootstrapToken
    chainfireApiPort
    chainfireControlPlaneAddrs
    chainfireInitialPeers
    chainfireRaftPort
    clusterId
    clusterName
    clusterNodes
    controlPlaneLabels
    controlPlaneNodeNames
    controlPlaneNodes
    environment
    flaredbApiPort
    flaredbControlPlaneAddrs
    flaredbInitialPeers
    flaredbRaftPort
    iamPort
    nodeConfigurationPaths
    nodeDiskoPaths
    sharedDesiredSystem
    ;

  controlPlaneNodeClass = {
    description = "Control-plane VM cluster nodes";
    nixProfile = "profiles/control-plane";
    roles = [ "control-plane" ];
    labels = controlPlaneLabels;
  };

  controlPlanePool = {
    description = "VM cluster control-plane pool";
    nodeClass = "control-plane";
    labels = {
      plane = "control";
      cluster = "vm-cluster";
    };
  };

  hostDeployments = {
    control-plane-canary = {
      selector.nodeIds = [ bootstrapNodeName ];
      nixosConfiguration = bootstrapNodeName;
      flakeRef = "github:centra/cloud";
      batchSize = 1;
      maxUnavailable = 1;
      healthCheckCommand = sharedDesiredSystem.healthCheckCommand;
      switchAction = "switch";
      rollbackOnFailure = sharedDesiredSystem.rollbackOnFailure;
    };
  };

  mkControlPlaneNodeModule = nodeName:
    { lib, ... }:
    {
      imports = [
        ./cluster.nix
        nodeDiskoPaths.${nodeName}
      ];

      networking.hostName = nodeName;
      networking.useDHCP = lib.mkDefault true;

      boot.loader.grub = {
        enable = true;
        devices = [ "/dev/vda" ];
        efiSupport = true;
        efiInstallAsRemovable = true;
      };

      services.chainfire = {
        enable = true;
        nodeId = nodeName;
        apiAddr = mkEndpoint nodeName chainfireApiPort;
        raftAddr = mkEndpoint nodeName chainfireRaftPort;
        initialPeers = chainfireInitialPeers;
      };

      services.flaredb = {
        enable = true;
        nodeId = nodeName;
        apiAddr = mkEndpoint nodeName flaredbApiPort;
        raftAddr = mkEndpoint nodeName flaredbRaftPort;
        initialPeers = flaredbInitialPeers;
      };

      services.iam = {
        enable = true;
        port = iamPort;
        chainfireAddr = chainfireControlPlaneAddrs;
        flaredbAddr = flaredbControlPlaneAddrs;
      };

      services.openssh.enable = true;
      users.users.root.openssh.authorizedKeys.keys = [ ];

      system.stateVersion = "24.05";
    };

  mkBootstrapServicesModule =
    { self
    , nodeName
    , enableDeployer ? false
    ,
    }:
    { pkgs, lib, ... }:
    {
      services.nix-agent = {
        enable = true;
        chainfireEndpoint = "http://${mkEndpoint bootstrapNodeName chainfireApiPort}";
        clusterId = clusterId;
        nodeId = nodeName;
        flakeRoot = self.outPath;
        intervalSecs = 30;
        apply = true;
      };

      services.deployer = mkIf enableDeployer {
        enable = true;
        bindAddr = "0.0.0.0:8088";
        chainfireEndpoints = [ "http://${mkEndpoint bootstrapNodeName chainfireApiPort}" ];
        clusterId = clusterId;
        requireChainfire = true;
        allowUnknownNodes = false;
        allowUnauthenticated = false;
        bootstrapToken = bootstrapToken;
        adminToken = adminToken;
        bootstrapFlakeBundle = pkgs.ultracloudFlakeBundle;
        seedClusterState = true;
      };
    };
}