photoncloud-monorepo/nix/modules/fiberlb.nix

{ config, lib, pkgs, ... }:

let
  cfg = config.services.fiberlb;
  tomlFormat = pkgs.formats.toml { };
  bgpPeerType = lib.types.submodule {
    options = {
      address = lib.mkOption {
        type = lib.types.str;
        description = "BGP peer IP address or hostname.";
        example = "192.0.2.1";
      };

      port = lib.mkOption {
        type = lib.types.port;
        default = 179;
        description = "BGP peer TCP port.";
      };

      asn = lib.mkOption {
        type = lib.types.ints.positive;
        description = "Peer AS number.";
        example = 65020;
      };

      description = lib.mkOption {
        type = lib.types.str;
        default = "";
        description = "Optional description used for logs and operators.";
      };

      med = lib.mkOption {
        type = lib.types.nullOr lib.types.ints.unsigned;
        default = null;
        description = "Optional MED to attach to VIP announcements sent to this peer.";
      };

      communities = lib.mkOption {
        type = lib.types.listOf lib.types.str;
        default = [ ];
        description = "Optional standard BGP communities to attach to VIP announcements sent to this peer.";
        example = [ "65001:100" "65001:200" ];
      };

      bfd = {
        enable = lib.mkEnableOption "single-hop BFD for this BGP peer";

        desiredMinTxMillis = lib.mkOption {
          type = lib.types.ints.positive;
          default = 300;
          description = "Desired BFD transmit interval in milliseconds.";
        };

        requiredMinRxMillis = lib.mkOption {
          type = lib.types.ints.positive;
          default = 300;
          description = "Required BFD receive interval in milliseconds.";
        };

        detectMultiplier = lib.mkOption {
          type = lib.types.ints.positive;
          default = 3;
          description = "BFD detection multiplier.";
        };

        bootstrapTimeoutSecs = lib.mkOption {
          type = lib.types.ints.positive;
          default = 10;
          description = "How long FiberLB waits for the BFD session to reach Up after BGP establishment.";
        };
      };
    };
  };

  fiberlbBaseConfig = {
    grpc_addr = "0.0.0.0:${toString cfg.port}";
    log_level = "info";
    auth = {
      iam_server_addr =
        if cfg.iamAddr != null
        then cfg.iamAddr
        else "127.0.0.1:50080";
    };
    health = {
      interval_secs = cfg.healthCheckIntervalSecs;
      timeout_secs = cfg.healthCheckTimeoutSecs;
    };
    vip_advertisement = {
      interval_secs = cfg.vipCheckIntervalSecs;
      drain_file = cfg.vipDrain.filePath;
      drain_hold_time_secs = cfg.vipDrain.holdTimeSecs;
    };
    vip_ownership = {
      enabled = cfg.vipOwnership.enable;
      interface = cfg.vipOwnership.interface;
    };
  } // lib.optionalAttrs cfg.bgp.enable {
    bgp =
      {
        enabled = true;
        local_as = cfg.bgp.localAs;
        router_id =
          if cfg.bgp.routerId != null
          then cfg.bgp.routerId
          else "127.0.0.1";
        hold_time_secs = cfg.bgp.holdTimeSecs;
        keepalive_secs = cfg.bgp.keepaliveSecs;
        connect_retry_secs = cfg.bgp.connectRetrySecs;
        peers = map
          (peer: {
            inherit (peer) address port asn description;
            export_policy = {
              inherit (peer) communities;
            } // lib.optionalAttrs (peer.med != null) {
              med = peer.med;
            };
            bfd = {
              enabled = peer.bfd.enable;
              desired_min_tx_millis = peer.bfd.desiredMinTxMillis;
              required_min_rx_millis = peer.bfd.requiredMinRxMillis;
              detect_multiplier = peer.bfd.detectMultiplier;
              bootstrap_timeout_secs = peer.bfd.bootstrapTimeoutSecs;
            };
          })
          cfg.bgp.peers;
      }
      // lib.optionalAttrs (cfg.bgp.nextHop != null) {
        next_hop = cfg.bgp.nextHop;
      };
  };
  fiberlbConfigFile = tomlFormat.generate "fiberlb.toml" (lib.recursiveUpdate fiberlbBaseConfig cfg.settings);
  flaredbDependencies = lib.optional (cfg.metadataBackend == "flaredb") "flaredb.service";
  normalizedDatabaseUrl =
    let
      sqliteUrl =
        if cfg.databaseUrl != null
          && cfg.metadataBackend == "sqlite"
          && lib.hasPrefix "sqlite:/" cfg.databaseUrl
          && !(lib.hasPrefix "sqlite://" cfg.databaseUrl)
        then "sqlite://${lib.removePrefix "sqlite:" cfg.databaseUrl}"
        else cfg.databaseUrl;
    in
    if sqliteUrl != null
      && cfg.metadataBackend == "sqlite"
      && !(lib.hasInfix "?" sqliteUrl)
    then "${sqliteUrl}?mode=rwc"
    else sqliteUrl;
in
{
  options.services.fiberlb = {
    enable = lib.mkEnableOption "fiberlb service";

    port = lib.mkOption {
      type = lib.types.port;
      default = 50085;
      description = "Port for fiberlb gRPC management API";
    };

    iamAddr = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "IAM service endpoint address (host:port)";
      example = "10.0.0.1:50080";
    };

    chainfireAddr = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "ChainFire endpoint address (host:port) for cluster coordination only";
      example = "10.0.0.1:2379";
    };

    flaredbAddr = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "FlareDB endpoint address (host:port) for metadata/user data";
      example = "10.0.0.1:2479";
    };

    metadataBackend = lib.mkOption {
      type = lib.types.enum [ "flaredb" "postgres" "sqlite" ];
      default = "flaredb";
      description = "Metadata backend for FiberLB.";
    };

    databaseUrl = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "SQL database URL for metadata (required when metadataBackend is postgres/sqlite).";
      example = "postgres://fiberlb:secret@10.0.0.10:5432/fiberlb";
    };

    singleNode = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = "Enable single-node mode (required when metadata backend is SQLite)";
    };

    healthCheckIntervalSecs = lib.mkOption {
      type = lib.types.ints.positive;
      default = 5;
      description = "Interval between FiberLB backend health sweeps.";
    };

    healthCheckTimeoutSecs = lib.mkOption {
      type = lib.types.ints.positive;
      default = 5;
      description = "Timeout for each FiberLB backend health probe.";
    };

    vipCheckIntervalSecs = lib.mkOption {
      type = lib.types.ints.positive;
      default = 3;
      description = "Interval between FiberLB VIP-to-BGP reconciliation sweeps.";
    };

    vipOwnership = {
      enable = lib.mkEnableOption "FiberLB local VIP ownership";

      interface = lib.mkOption {
        type = lib.types.str;
        default = "lo";
        description = "Interface where FiberLB should claim VIP /32 addresses.";
      };
    };

    vipDrain = {
      filePath = lib.mkOption {
        type = lib.types.str;
        default = "/var/lib/fiberlb/drain";
        description = "Presence of this file puts FiberLB into node drain mode.";
      };

      holdTimeSecs = lib.mkOption {
        type = lib.types.ints.unsigned;
        default = 5;
        description = "How long FiberLB keeps a locally owned VIP after withdrawing it for drain.";
      };
    };

    bgp = {
      enable = lib.mkEnableOption "FiberLB native BGP VIP advertisement";

      localAs = lib.mkOption {
        type = lib.types.ints.positive;
        default = 65001;
        description = "Local AS number used by FiberLB's native BGP speaker.";
      };

      routerId = lib.mkOption {
        type = lib.types.nullOr lib.types.str;
        default = null;
        description = "IPv4 router ID used by FiberLB's native BGP speaker.";
        example = "192.0.2.10";
      };

      nextHop = lib.mkOption {
        type = lib.types.nullOr lib.types.str;
        default = null;
        description = "Explicit BGP NEXT_HOP address. Defaults to routerId when unset.";
        example = "192.0.2.10";
      };

      holdTimeSecs = lib.mkOption {
        type = lib.types.ints.positive;
        default = 90;
        description = "Requested BGP hold time in seconds.";
      };

      keepaliveSecs = lib.mkOption {
        type = lib.types.ints.positive;
        default = 30;
        description = "BGP keepalive interval in seconds.";
      };

      connectRetrySecs = lib.mkOption {
        type = lib.types.ints.positive;
        default = 5;
        description = "Delay before FiberLB reconnects to a failed BGP peer.";
      };

      peers = lib.mkOption {
        type = lib.types.listOf bgpPeerType;
        default = [ ];
        description = "Static BGP peers for FiberLB's native speaker.";
      };
    };

    dataDir = lib.mkOption {
      type = lib.types.path;
      default = "/var/lib/fiberlb";
      description = "Data directory for fiberlb";
    };

    settings = lib.mkOption {
      type = lib.types.attrs;
      default = {};
      description = "Additional configuration settings";
    };

    package = lib.mkOption {
      type = lib.types.package;
      default = pkgs.fiberlb-server or (throw "fiberlb-server package not found");
      description = "Package to use for fiberlb";
    };
  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = cfg.metadataBackend != "sqlite" || cfg.singleNode;
        message = "services.fiberlb.singleNode must be true when metadataBackend is sqlite";
      }
      {
        assertion = cfg.metadataBackend == "flaredb" || cfg.databaseUrl != null;
        message = "services.fiberlb.databaseUrl is required when metadataBackend is postgres or sqlite";
      }
      {
        assertion = (!cfg.bgp.enable) || cfg.bgp.routerId != null;
        message = "services.fiberlb.bgp.routerId must be set when native BGP is enabled";
      }
      {
        assertion = (!cfg.bgp.enable) || ((builtins.length cfg.bgp.peers) > 0);
        message = "services.fiberlb.bgp.peers must contain at least one peer when native BGP is enabled";
      }
      {
        assertion = (!cfg.vipOwnership.enable) || cfg.bgp.enable;
        message = "services.fiberlb.vipOwnership.enable requires services.fiberlb.bgp.enable";
      }
    ];

    # Create system user
    users.users.fiberlb = {
      isSystemUser = true;
      group = "fiberlb";
      description = "FiberLB service user";
      home = cfg.dataDir;
    };

    users.groups.fiberlb = {};

    # Create systemd service
    systemd.services.fiberlb = {
      description = "FiberLB Load Balancing Service";
      wantedBy = [ "multi-user.target" ];
      after = [ "network.target" "iam.service" ] ++ flaredbDependencies;
      requires = [ "iam.service" ] ++ flaredbDependencies;
      path = lib.optionals cfg.vipOwnership.enable [ pkgs.iproute2 ];

      serviceConfig = {
        Type = "simple";
        User = "fiberlb";
        Group = "fiberlb";
        Restart = "on-failure";
        RestartSec = "10s";

        # State directory management
        StateDirectory = "fiberlb";
        StateDirectoryMode = "0750";

        # Security hardening
        NoNewPrivileges = true;
        PrivateTmp = true;
        ProtectSystem = "strict";
        ProtectHome = true;
        ReadWritePaths = [ cfg.dataDir ];
        AmbientCapabilities = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];
        CapabilityBoundingSet = lib.optionals cfg.vipOwnership.enable [ "CAP_NET_ADMIN" ];

        # Environment variables for service endpoints
        Environment = [
          "RUST_LOG=info"
          "FIBERLB_FLAREDB_ENDPOINT=${if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479"}"
          "FIBERLB_METADATA_BACKEND=${cfg.metadataBackend}"
        ] ++ lib.optional (normalizedDatabaseUrl != null) "FIBERLB_METADATA_DATABASE_URL=${normalizedDatabaseUrl}"
          ++ lib.optional cfg.singleNode "FIBERLB_SINGLE_NODE=true"
          ++ lib.optional (cfg.chainfireAddr != null) "FIBERLB_CHAINFIRE_ENDPOINT=http://${cfg.chainfireAddr}";

        # Start command
        ExecStart = lib.concatStringsSep " " ([
          "${cfg.package}/bin/fiberlb"
          "--config ${fiberlbConfigFile}"
          "--grpc-addr 0.0.0.0:${toString cfg.port}"
          "--flaredb-endpoint ${if cfg.flaredbAddr != null then cfg.flaredbAddr else "127.0.0.1:2479"}"
        ]);
      };
    };
  };
}