Create host template config

bootstrap/variables.tf

@@ -1,6 +1,6 @@
 variable "node_count" {
   type        = number
-  default     = 1
+  default     = 3
   description = "the number of nodes to create"
 }
 

common/default.nix

@@ -0,0 +1,116 @@
+{
+  config,
+  inputs,
+  pkgs,
+  agenix,
+  ...
+}: {
+  deployment = {
+    targetHost = "<PLACEHOLDER>";
+    targetPort = 22;
+    targetUser = "root";
+    buildOnTarget = true;
+  };
+
+  networking = {
+    hostName = "<PLACEHOLDER>";
+    nftables.enable = true;
+    firewall = {
+      allowedTCPPorts = [22 80 443];
+      allowedUDPPorts = [config.services.tailscale.port];
+    };
+  };
+  imports = [
+    agenix.nixosModules.default
+    inputs.nixos-facter-modules.nixosModules.facter
+    {config.facter.reportPath = ./facter.json;}
+    inputs.disko.nixosModules.disko
+    ./hardware-configuration.nix
+    ./disko-config.nix
+    ./base-system.nix
+    ./nix.nix
+    ./nomad.nix
+  ];
+
+  boot = {
+    supportedFilesystems = ["btrfs" "ntfs" "vfat" "ext4"];
+    loader.grub = {
+      enable = true;
+    };
+  };
+
+  # age.secrets.docker_auth_json = {
+  #   file = ./secrets/docker_ghcr_auth_config_json.age;
+  #   path = "/run/containers/0/auth.json";
+  # };
+
+  age.secrets.tailscale_auth_key = {
+    file = ./secrets/tailscale_auth_key;
+  };
+
+  environment.systemPackages = with pkgs; [
+    nomad_1_9
+  ];
+
+  #
+  # TEST: authentication to private repo
+  #
+  # environment.variables = {
+  #   XDG_RUNTIME_DIR = "/var/run/user/0";
+  # };
+
+  # Define a systemd service to create files in /var/run/user/0
+  # systemd.services."create-incus-ghcr-auth-file" = {
+  #   description = "Create ghcr.io auth file for incus to retrieve private images";
+  #   after = ["default.target"];
+  #   wantedBy = ["default.target"];
+  #   serviceConfig = {
+  #     ExecStart = ''${pkgs.bashInteractive}/bin/bash -c 'echo {\\"auths\\":{ \\"ghcr.io\\": {\\"auth\\": \\"YWNhbG9pYXJvOiBnaHBfVnN4dzc4YXZ6N0lBeXQyejVrcGpZQTJXV21XbUZNMzlZdkMxIA==\\"}}} > /var/run/user/0/containers/auth.json' '';
+  #     ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p /var/run/user/0/containers";
+  #     Restart = "no";
+  #   };
+  # };
+  #
+  # /TEST: authentication to private repo
+  #
+
+  users.users = {
+    root = {
+      openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
+    };
+    sfos = {
+      isNormalUser = true;
+      extraGroups = ["wheel"];
+      # TODO Initial password and authorized keys need to come from .env, perhaps using nix-edit
+      initialPassword = "Test.123";
+      openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
+    };
+  };
+
+  services = {
+    # TODO caddy needs a plugin that watches the nomad node list for specific tags that cause them to be reverse proxied
+    # from this host system
+    caddy = {
+      enable = true;
+      # package = pkgs.caddy.withPlugins {
+      #   plugins = ["github.com/acaloiaro/caddy-incus-upstreams@v0.0.0-20241228235755-786cd3b6a9dc"];
+      #   hash = "sha256-yjqXgo7WiJPBVmN3+O8Wq9Nxd1xU8KrXW7Hv7Elcf7I=";
+      # };
+      # virtualHosts = {
+      #   "https://adrianofyi.incus.adriano.fyi:443" = {
+      #     extraConfig = ''
+      #        reverse_proxy {
+      #       	dynamic incus
+      #       }
+      #     '';
+      #   };
+      # };
+    };
+    openssh.enable = true;
+    tailscale = {
+      enable = true;
+      authKeyFile = config.age.secrets.tailscale_auth_key.path;
+    };
+  };
+  system.stateVersion = "24.05";
+}

common/nomad.nix

@@ -0,0 +1,58 @@
+{...}: {
+  services.nomad = {
+    enable = true;
+    enableDocker = true;
+    settings = {
+      server = {
+        enabled = true;
+        # Ideally the bootstrap expect value would reflect the number of servers set by `instance_count`, but in the interest
+        # of not chaning each server's "user data" every time the instance_count changes, we hard code the nomad cluster to
+        # bootstrap itself whenever 3 nodes are present
+        bootstrap_expect = 3;
+
+        server_join = {
+          retry_join = [];
+        };
+      };
+
+      acl = {
+        enabled = true;
+      };
+
+      client = {
+        enabled = true;
+      };
+
+      plugin.docker = {
+        config = {
+          gc = {
+            image = true;
+            image_delay = "3m";
+            container = true;
+
+            dangling_containers = {
+              enabled = true;
+              dry_run = false;
+              period = "5m";
+              creation_grace = "5m";
+            };
+          };
+
+          volumes = {
+            enabled = true;
+            selinuxlabel = "z";
+          };
+
+          allow_caps = ["chown" "net_raw"];
+          allow_privileged = true;
+        };
+      };
+
+      plugin.raw_exec = {
+        config = {
+          enabled = true;
+        };
+      };
+    };
+  };
+}

common/secrets/docker_ghcr_auth_config_json.age

@@ -0,0 +1,17 @@
+age-encryption.org/v1
+-> ssh-rsa QGRkYA
+dJW71AoISeNPqSMv7bsAnYQ8yFAA7SsE6Dkh+yGFp9oXAJ/AjIM+/DlaLM2AeZol
+JgekTnmRqFNZzpJ+Q7q2OAE76ROXG1rQhwuyQa2axnsHeup749QXnWam2V3SATA5
+kVaLVO4QbzvFIEUia1rPmtUPSImO0QxkzI3dUz19j3pIEG6kMQEJyuJ9FurS+rSX
+KnZxes8Q3Q8iszeufN6+1L3ENWOHwMKaSWw+cV4o1Pis8Rh/wUQwi9iyJGSZx4rT
+IORFhoJsXI6q2WvnnEcM0BBjiKylVW2e4P6fBkKg3+EEN3cHL026RjzyddjsDy0n
+ppHmB2bCNIf3PW7za8TfZ47tqKgA+mNpkXO85SkP5S5u87hBjILlzAXtevFnVrr+
+Xpjq8ZHDy7QYLmQl6i0opgxLe7JuOYP++3/6PL5T2XcEw7kymXlg2KwtQN4TBEZj
+YDnhPuh++TTikSyKeNzzpGTc/jnXV2BIAevXLQ0PmCs6k5/FrD2KS+rzRmVuGl44
+
+-> ssh-ed25519 9G9Vsg ZKyEMJVyLAr0G3GhOoqf+0+mZUa6XwbFc1PYGLRZjXQ
+CSBhHp+92cH5hu3p5oTzqxkYS1uTo3EWbqlMEtOJn7M
+-> ssh-ed25519 KKRRSg EE6TBwP0ScwxT8zsdMkEnbldb2sAaQiuQ4ILZY2/MkE
+Qdr7swRd4uQ0DV0S42WDxC+RWI8zS4nz9h1HbcQ7zME
+--- ltjYR7/VAs27ta24Y4d2UhWD7hOGBVYl4Zk3b0nXAqg
+ýb61æÙƾ+œší‡Âî¾5;Í
:!><3E>¿Aȱ2µ<32>GéR†’â<E28099>Í2Î¥–ú˜ö?–ª[ž8d³™¯Nú;Žð°´TÚy4Pü³Â<16>æ¼hõ:»Ê’,¤-Š8éO=ÝÚ"£âÀ¯óë„%:(Æbüù1_<31>öÖŒæ3ç_ÖÅíÜÅû^˜®'“ÿç4e·á

common/secrets/secrets.nix

@@ -1,9 +1,7 @@
 # Use ssh-keyscan <SERVER> to fetch the below keys for target systems (SSH server must be running on system)
 let
   localMachine = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1";
-  sfos = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJLQTZC70O2rE4cVDOWlrFLa6mxIJ+B+4gE7eVbmaw2k";
-  sfosnew = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINYjXyfiGMxgc4z8F4BtMKsTr2hSXrpoy7mAWNsWJXWS";
 in {
-  "docker_ghcr_auth_config_json.age".publicKeys = [localMachine sfos sfosnew];
+  # "docker_ghcr_auth_config_json.age".publicKeys = [localMachine sfosnew];
   tailscale_auth_key.publicKeys = [localMachine "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGrsG0K92nEWxcvZpHSJJNGnmw+WPyrrkOUDdtrffC9G"];
 }

flake.nix

@@ -60,8 +60,9 @@
           inherit (inputs) agenix;
         };
       };
-
+      sfos-0 = ./hosts/sfos-0;
-      sfos = ./hosts/sfos;
+      sfos-1 = ./hosts/sfos-1;
+      sfos-2 = ./hosts/sfos-2;
     };
     devShells = forEachSystem (system: let
       config = self.devShells.${system}.default.config;
@@ -116,26 +117,28 @@
               sfos-init = {
                 description = "Initialize cluster members";
                 exec = ''
-                  cd ./bootstrap
                   #!/run/current-system/sw/bin/bash
+                  cwd=$(pwd)
                   # tofu apply -auto-approve
-                  # Add our public key to the sfos user's authorized keys
+                  tofu -chdir="$cwd/bootstrap" output -json | jq '.cluster_ips.value | join(",")' | tr -d '"' | tr ',' "\n" | while read  host; do
-                  tofu output -json | jq '.cluster_ips.value | join(",")' | tr -d '"' | tr ',' "\n" | while read host; do
+                      cd "$cwd/hosts"
                       echo "Initializing $host"
-                      hostName=$(tofu output -json | hostName=$host jq -r ".cluster_nodes.value[] | select(.ipv4_address == \"$host\").name")
+                      hostName=$(tofu -chdir="$cwd/bootstrap" output -json | hostName=$host jq -r ".cluster_nodes.value[] | select(.ipv4_address == \"$host\").name")
                       echo "Setting host name: $hostName"
-                      nix-editor ../hosts/sfos/default.nix users.users.sfos.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
-                      nix-editor ../hosts/sfos/default.nix users.users.root.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
-                      nix-editor ../hosts/sfos/default.nix networking.hostName -i -v "\"$hostName\"\;"
                       ssh_public_key=$(ssh-keyscan $host | grep ssh-ed25519 | cut -d " " -f 2,3)
-                      nix-editor ../hosts/sfos/secrets/secrets.nix tailscale_auth_key.publicKeys -i -v "[localMachine \"$ssh_public_key\"]"
+                      mkdir -p "$hostName/secrets"
-                      # nixos-rebuild switch --flake .#sfos --target-host "root@$host"
+                      cp -rf ../common/* "$hostName/"
-                      # nixos-rebuild switch --flake .#sfos --target-host "root@$host" --build-host "root@$host"
+                      nix-editor "$hostName/default.nix" users.users.sfos.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
-                      # nixos-rebuild switch --flake .#sfos --target-host "sfos@$host" --build-host "sfos@$host" --use-remote-sudo
+                      nix-editor "$hostName/default.nix" users.users.root.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
+                      nix-editor "$hostName/default.nix" networking.hostName -i -v "\"$hostName\"\;"
+                      nix-editor "$hostName/default.nix" deployment.targetHost -i -v "\"$host\"\;"
+                      nix-editor "$hostName/secrets/secrets.nix" tailscale_auth_key.publicKeys -i -v "[localMachine \"$ssh_public_key\"]"
+                      nix-editor ../flake.nix "outputs.colmena.$hostName" -i -v "./hosts/$hostName";
+                      cd "$hostName/secrets"
+                      echo "$TF_VAR_tailscale_auth_key" | agenix -e tailscale_auth_key -i ~/.ssh/id_rsa_agenix
                   done
-                  cd ../hosts/sfos/secrets
-                  agenix -r -i ~/.ssh/id_rsa_agenix
                   cd ../../..
+                  git add hosts
                 '';
               };
             };

hosts/sfos-0/base-system.nix

@@ -0,0 +1,71 @@
+{
+  lib,
+  pkgs,
+  self,
+  ...
+}: {
+  imports = [
+    ./nix.nix
+  ];
+
+  boot = {
+    initrd.systemd.enable = true;
+    tmp = {
+      cleanOnBoot = true;
+      useTmpfs = true;
+    };
+  };
+
+  environment = {
+    systemPackages = with pkgs; [
+      lsof
+      wget
+    ];
+  };
+
+  security.sudo.wheelNeedsPassword = false;
+  i18n = let
+    locale = "en_US.UTF-8";
+  in {
+    defaultLocale = locale;
+    extraLocaleSettings = {
+      LC_ADDRESS = locale;
+      LC_IDENTIFICATION = locale;
+      LC_MEASUREMENT = locale;
+      LC_MONETARY = locale;
+      LC_NAME = locale;
+      LC_NUMERIC = locale;
+      LC_PAPER = locale;
+      LC_TELEPHONE = locale;
+      LC_TIME = locale;
+    };
+  };
+
+  networking = {
+    enableIPv6 = false;
+  };
+
+  programs = {};
+
+  services = {
+    fstrim.enable = true;
+    locate.enable = true;
+    rsyslogd.enable = true;
+    openssh = {
+      enable = true;
+      settings.PermitRootLogin = "yes";
+    };
+  };
+
+  security = {
+    rtkit.enable = true;
+  };
+
+  systemd.services.nix-daemon = {
+    environment.TMPDIR = "/var/tmp";
+  };
+
+  time.timeZone = "America/New_York";
+  users.mutableUsers = false;
+  system.configurationRevision = lib.mkIf (self ? rev) self.rev;
+}

hosts/sfos-0/default.nix

@@ -0,0 +1,116 @@
+{
+  config,
+  inputs,
+  pkgs,
+  agenix,
+  ...
+}: {
+  deployment = {
+    targetHost = "65.109.174.113";
+    targetPort = 22;
+    targetUser = "root";
+    buildOnTarget = true;
+  };
+
+  networking = {
+    hostName = "sfos-0";
+    nftables.enable = true;
+    firewall = {
+      allowedTCPPorts = [22 80 443];
+      allowedUDPPorts = [config.services.tailscale.port];
+    };
+  };
+  imports = [
+    agenix.nixosModules.default
+    inputs.nixos-facter-modules.nixosModules.facter
+    {config.facter.reportPath = ./facter.json;}
+    inputs.disko.nixosModules.disko
+    ./hardware-configuration.nix
+    ./disko-config.nix
+    ./base-system.nix
+    ./nix.nix
+    ./nomad.nix
+  ];
+
+  boot = {
+    supportedFilesystems = ["btrfs" "ntfs" "vfat" "ext4"];
+    loader.grub = {
+      enable = true;
+    };
+  };
+
+  # age.secrets.docker_auth_json = {
+  #   file = ./secrets/docker_ghcr_auth_config_json.age;
+  #   path = "/run/containers/0/auth.json";
+  # };
+
+  age.secrets.tailscale_auth_key = {
+    file = ./secrets/tailscale_auth_key;
+  };
+
+  environment.systemPackages = with pkgs; [
+    nomad_1_9
+  ];
+
+  #
+  # TEST: authentication to private repo
+  #
+  # environment.variables = {
+  #   XDG_RUNTIME_DIR = "/var/run/user/0";
+  # };
+
+  # Define a systemd service to create files in /var/run/user/0
+  # systemd.services."create-incus-ghcr-auth-file" = {
+  #   description = "Create ghcr.io auth file for incus to retrieve private images";
+  #   after = ["default.target"];
+  #   wantedBy = ["default.target"];
+  #   serviceConfig = {
+  #     ExecStart = ''${pkgs.bashInteractive}/bin/bash -c 'echo {\\"auths\\":{ \\"ghcr.io\\": {\\"auth\\": \\"YWNhbG9pYXJvOiBnaHBfVnN4dzc4YXZ6N0lBeXQyejVrcGpZQTJXV21XbUZNMzlZdkMxIA==\\"}}} > /var/run/user/0/containers/auth.json' '';
+  #     ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p /var/run/user/0/containers";
+  #     Restart = "no";
+  #   };
+  # };
+  #
+  # /TEST: authentication to private repo
+  #
+
+  users.users = {
+    root = {
+      openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
+    };
+    sfos = {
+      isNormalUser = true;
+      extraGroups = ["wheel"];
+      # TODO Initial password and authorized keys need to come from .env, perhaps using nix-edit
+      initialPassword = "Test.123";
+      openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
+    };
+  };
+
+  services = {
+    # TODO caddy needs a plugin that watches the nomad node list for specific tags that cause them to be reverse proxied
+    # from this host system
+    caddy = {
+      enable = true;
+      # package = pkgs.caddy.withPlugins {
+      #   plugins = ["github.com/acaloiaro/caddy-incus-upstreams@v0.0.0-20241228235755-786cd3b6a9dc"];
+      #   hash = "sha256-yjqXgo7WiJPBVmN3+O8Wq9Nxd1xU8KrXW7Hv7Elcf7I=";
+      # };
+      # virtualHosts = {
+      #   "https://adrianofyi.incus.adriano.fyi:443" = {
+      #     extraConfig = ''
+      #        reverse_proxy {
+      #       	dynamic incus
+      #       }
+      #     '';
+      #   };
+      # };
+    };
+    openssh.enable = true;
+    tailscale = {
+      enable = true;
+      authKeyFile = config.age.secrets.tailscale_auth_key.path;
+    };
+  };
+  system.stateVersion = "24.05";
+}

hosts/sfos-0/disko-config.nix

@@ -0,0 +1,99 @@
+{
+  disko.devices.disk.os = {
+    # device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_59633795";
+    device = "/dev/sda";
+    type = "disk";
+    content = {
+      type = "gpt";
+      partitions = {
+        boot = {
+          size = "1M";
+          type = "EF02"; # for grub MBR
+        };
+        # ESP = {
+        #   type = "EF00";
+        #   size = "500M";
+        #   content = {
+        #     type = "filesystem";
+        #     format = "vfat";
+        #     mountpoint = "/boot";
+        #     mountOptions = ["umask=0077"];
+        #   };
+        # };
+        root = {
+          size = "100%";
+          content = {
+            type = "filesystem";
+            format = "ext4";
+            mountpoint = "/";
+          };
+        };
+      };
+    };
+  };
+}
+# {
+#   disko.devices.disk.os = {
+#     type = "disk";
+#     content = {
+#       device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_59632825";
+#       type = "gpt";
+#       partitions = {
+#         boot = {
+#           type = "EF02";
+#           size = "1M";
+#         };
+#         ESP = {
+#           type = "EF00";
+#           size = "500M";
+#           content = {
+#             type = "filesystem";
+#             format = "vfat";
+#             mountpoint = "/boot";
+#             mountOptions = ["umask=0077"];
+#           };
+#         };
+#         root = {
+#           size = "100%";
+#           content = {
+#             type = "filesystem";
+#             format = "ext4";
+#             mountpoint = "/";
+#           };
+#         };
+#       };
+#     };
+#   };
+# }
+# {
+#   device = "/dev/sda";
+#   type = "disk";
+#   content = {
+#     type = "gpt";
+#     partitions = {
+#       boot = {
+#         type = "EF02";
+#         size = "1M";
+#       };
+#       ESP = {
+#         type = "EF00";
+#         size = "500M";
+#         content = {
+#           type = "filesystem";
+#           format = "vfat";
+#           mountpoint = "/boot";
+#           mountOptions = ["umask=0077"];
+#         };
+#       };
+#       root = {
+#         size = "100%";
+#         content = {
+#           type = "filesystem";
+#           format = "ext4";
+#           mountpoint = "/";
+#         };
+#       };
+#     };
+#   };
+# }
+

hosts/sfos-0/hardware-configuration.nix

@@ -0,0 +1,26 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  lib,
+  modulesPath,
+  ...
+}: {
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+
+  boot.initrd.availableKernelModules = ["virtio_pci" "virtio_scsi" "ahci" "sd_mod"];
+  boot.initrd.kernelModules = [];
+  boot.kernelModules = [];
+  boot.extraModulePackages = [];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp0s4.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/sfos-0/nix.nix

@@ -0,0 +1,33 @@
+{pkgs, ...}: {
+  nixpkgs = {
+    config = {
+      allowUnfree = true;
+    };
+  };
+
+  nix = {
+    package = pkgs.nix;
+
+    gc = {
+      automatic = true;
+      dates = "weekly";
+      options = "--delete-older-than 14d";
+    };
+
+    settings = {
+      trusted-users = [
+        "@wheel"
+      ];
+
+      experimental-features = [
+        "flakes"
+        "nix-command"
+      ];
+
+      trusted-public-keys = [
+        "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+        "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+      ];
+    };
+  };
+}

hosts/sfos-0/nomad.nix

@@ -0,0 +1,58 @@
+{...}: {
+  services.nomad = {
+    enable = true;
+    enableDocker = true;
+    settings = {
+      server = {
+        enabled = true;
+        # Ideally the bootstrap expect value would reflect the number of servers set by `instance_count`, but in the interest
+        # of not chaning each server's "user data" every time the instance_count changes, we hard code the nomad cluster to
+        # bootstrap itself whenever 3 nodes are present
+        bootstrap_expect = 3;
+
+        server_join = {
+          retry_join = [];
+        };
+      };
+
+      acl = {
+        enabled = true;
+      };
+
+      client = {
+        enabled = true;
+      };
+
+      plugin.docker = {
+        config = {
+          gc = {
+            image = true;
+            image_delay = "3m";
+            container = true;
+
+            dangling_containers = {
+              enabled = true;
+              dry_run = false;
+              period = "5m";
+              creation_grace = "5m";
+            };
+          };
+
+          volumes = {
+            enabled = true;
+            selinuxlabel = "z";
+          };
+
+          allow_caps = ["chown" "net_raw"];
+          allow_privileged = true;
+        };
+      };
+
+      plugin.raw_exec = {
+        config = {
+          enabled = true;
+        };
+      };
+    };
+  };
+}

hosts/sfos-0/secrets/docker_ghcr_auth_config_json.age

@@ -0,0 +1,17 @@
+age-encryption.org/v1
+-> ssh-rsa QGRkYA
+dJW71AoISeNPqSMv7bsAnYQ8yFAA7SsE6Dkh+yGFp9oXAJ/AjIM+/DlaLM2AeZol
+JgekTnmRqFNZzpJ+Q7q2OAE76ROXG1rQhwuyQa2axnsHeup749QXnWam2V3SATA5
+kVaLVO4QbzvFIEUia1rPmtUPSImO0QxkzI3dUz19j3pIEG6kMQEJyuJ9FurS+rSX
+KnZxes8Q3Q8iszeufN6+1L3ENWOHwMKaSWw+cV4o1Pis8Rh/wUQwi9iyJGSZx4rT
+IORFhoJsXI6q2WvnnEcM0BBjiKylVW2e4P6fBkKg3+EEN3cHL026RjzyddjsDy0n
+ppHmB2bCNIf3PW7za8TfZ47tqKgA+mNpkXO85SkP5S5u87hBjILlzAXtevFnVrr+
+Xpjq8ZHDy7QYLmQl6i0opgxLe7JuOYP++3/6PL5T2XcEw7kymXlg2KwtQN4TBEZj
+YDnhPuh++TTikSyKeNzzpGTc/jnXV2BIAevXLQ0PmCs6k5/FrD2KS+rzRmVuGl44
+
+-> ssh-ed25519 9G9Vsg ZKyEMJVyLAr0G3GhOoqf+0+mZUa6XwbFc1PYGLRZjXQ
+CSBhHp+92cH5hu3p5oTzqxkYS1uTo3EWbqlMEtOJn7M
+-> ssh-ed25519 KKRRSg EE6TBwP0ScwxT8zsdMkEnbldb2sAaQiuQ4ILZY2/MkE
+Qdr7swRd4uQ0DV0S42WDxC+RWI8zS4nz9h1HbcQ7zME
+--- ltjYR7/VAs27ta24Y4d2UhWD7hOGBVYl4Zk3b0nXAqg
+ýb61æÙƾ+œší‡Âî¾5;Í
:!><3E>¿Aȱ2µ<32>GéR†’â<E28099>Í2Î¥–ú˜ö?–ª[ž8d³™¯Nú;Žð°´TÚy4Pü³Â<16>æ¼hõ:»Ê’,¤-Š8éO=ÝÚ"£âÀ¯óë„%:(Æbüù1_<31>öÖŒæ3ç_ÖÅíÜÅû^˜®'“ÿç4e·á

hosts/sfos-0/secrets/secrets.nix

@@ -0,0 +1,7 @@
+# Use ssh-keyscan <SERVER> to fetch the below keys for target systems (SSH server must be running on system)
+let
+  localMachine = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1";
+in {
+  # "docker_ghcr_auth_config_json.age".publicKeys = [localMachine sfosnew];
+  tailscale_auth_key.publicKeys = [localMachine "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMQfiuxKgCzq6dfS1loMZl01+OeaFu2plgQ31oEjy8Cp"];
+}

hosts/sfos-0/secrets/tailscale_auth_key

@@ -0,0 +1,15 @@
+age-encryption.org/v1
+-> ssh-rsa QGRkYA
+fL4gLU9vq4o5ihDmpysJeuMP0wAaKDz2MBpjZV0o3Rgy0h8EJExBlQ9VTgu8KaKh
+4KkLDYSOlScVAkxf9c/SCZnKhdJfVJKx1tjSfAhJUyMjRQG64D6NCUm/zq/xfvtz
+tL5IwUNe6omvie8LCC+D8gbNku1JFIYavexV9NFbFpgb6lFNtXshFDUK3HxUt+g0
+9k+icS6vcLQF1wcPaDtELWWHg0YBXISDTQRsFUWPLlI223W62h+QCX9pqmXsWay3
+KHxcGOW5YiRTk6BBp4WBb68+uuiXKCbbFt//nEPom/xoyxy1C9nN1LzLwjO4opDq
+HSuNnIpAJMI1g+ghTddpODI6m9f9WYyAfyW9+zCBU+jh1L3OjN+hohWRNMBoO6ex
+n+hAEVzM+CebnWBHIWn8Y4xA3BhthtYNBZWet3t1FO2h+KAijJQvPiAtJ4Yid7y5
+Qsp+oId1k4PpecPkb4Dx0ZrUur50D6NeKH6WkbsREyp7tx2aBFP4/cZe0lvnNfum
+
+-> ssh-ed25519 nMOjaQ O12DsuoJTdp1PKjfhZjYht/8ikFQ/Aet56TabhmM6yQ
+ZTpqJgpSR8m1r566XEQa2mJwnb32AIkH8oc11cgPxQQ
+--- a7RVCWyzR5OZ2N5YxQkIugTdTIapEK/OSLQQ+ci+T6w
+‹‰(}sŠù…³*•íź¾å6Éi²SöZÂn¾û¥ŠÕëÌŒ¶ôVW<ÒVX89ã;½¬4ÌdE’Boð¡3µýD
<0A>í|óå2}ºª»®

hosts/sfos-1/base-system.nix

@@ -0,0 +1,71 @@
+{
+  lib,
+  pkgs,
+  self,
+  ...
+}: {
+  imports = [
+    ./nix.nix
+  ];
+
+  boot = {
+    initrd.systemd.enable = true;
+    tmp = {
+      cleanOnBoot = true;
+      useTmpfs = true;
+    };
+  };
+
+  environment = {
+    systemPackages = with pkgs; [
+      lsof
+      wget
+    ];
+  };
+
+  security.sudo.wheelNeedsPassword = false;
+  i18n = let
+    locale = "en_US.UTF-8";
+  in {
+    defaultLocale = locale;
+    extraLocaleSettings = {
+      LC_ADDRESS = locale;
+      LC_IDENTIFICATION = locale;
+      LC_MEASUREMENT = locale;
+      LC_MONETARY = locale;
+      LC_NAME = locale;
+      LC_NUMERIC = locale;
+      LC_PAPER = locale;
+      LC_TELEPHONE = locale;
+      LC_TIME = locale;
+    };
+  };
+
+  networking = {
+    enableIPv6 = false;
+  };
+
+  programs = {};
+
+  services = {
+    fstrim.enable = true;
+    locate.enable = true;
+    rsyslogd.enable = true;
+    openssh = {
+      enable = true;
+      settings.PermitRootLogin = "yes";
+    };
+  };
+
+  security = {
+    rtkit.enable = true;
+  };
+
+  systemd.services.nix-daemon = {
+    environment.TMPDIR = "/var/tmp";
+  };
+
+  time.timeZone = "America/New_York";
+  users.mutableUsers = false;
+  system.configurationRevision = lib.mkIf (self ? rev) self.rev;
+}

hosts/sfos-1/default.nix

@@ -9,9 +9,17 @@
     targetHost = "65.109.8.31";
     targetPort = 22;
     targetUser = "root";
-    # buildOnTarget = true;
+    buildOnTarget = true;
   };
 
+  networking = {
+    hostName = "sfos-1";
+    nftables.enable = true;
+    firewall = {
+      allowedTCPPorts = [22 80 443];
+      allowedUDPPorts = [config.services.tailscale.port];
+    };
+  };
   imports = [
     agenix.nixosModules.default
     inputs.nixos-facter-modules.nixosModules.facter
@@ -21,6 +29,7 @@
     ./disko-config.nix
     ./base-system.nix
     ./nix.nix
+    ./nomad.nix
   ];
 
   boot = {
@@ -78,15 +87,6 @@
     };
   };
 
-  networking = {
-    hostName = "sfos-0";
-    nftables.enable = true;
-    firewall = {
-      allowedTCPPorts = [22 80 443];
-      allowedUDPPorts = [config.services.tailscale.port];
-    };
-  };
-
   services = {
     # TODO caddy needs a plugin that watches the nomad node list for specific tags that cause them to be reverse proxied
     # from this host system

hosts/sfos-1/disko-config.nix

@@ -0,0 +1,99 @@
+{
+  disko.devices.disk.os = {
+    # device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_59633795";
+    device = "/dev/sda";
+    type = "disk";
+    content = {
+      type = "gpt";
+      partitions = {
+        boot = {
+          size = "1M";
+          type = "EF02"; # for grub MBR
+        };
+        # ESP = {
+        #   type = "EF00";
+        #   size = "500M";
+        #   content = {
+        #     type = "filesystem";
+        #     format = "vfat";
+        #     mountpoint = "/boot";
+        #     mountOptions = ["umask=0077"];
+        #   };
+        # };
+        root = {
+          size = "100%";
+          content = {
+            type = "filesystem";
+            format = "ext4";
+            mountpoint = "/";
+          };
+        };
+      };
+    };
+  };
+}
+# {
+#   disko.devices.disk.os = {
+#     type = "disk";
+#     content = {
+#       device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_59632825";
+#       type = "gpt";
+#       partitions = {
+#         boot = {
+#           type = "EF02";
+#           size = "1M";
+#         };
+#         ESP = {
+#           type = "EF00";
+#           size = "500M";
+#           content = {
+#             type = "filesystem";
+#             format = "vfat";
+#             mountpoint = "/boot";
+#             mountOptions = ["umask=0077"];
+#           };
+#         };
+#         root = {
+#           size = "100%";
+#           content = {
+#             type = "filesystem";
+#             format = "ext4";
+#             mountpoint = "/";
+#           };
+#         };
+#       };
+#     };
+#   };
+# }
+# {
+#   device = "/dev/sda";
+#   type = "disk";
+#   content = {
+#     type = "gpt";
+#     partitions = {
+#       boot = {
+#         type = "EF02";
+#         size = "1M";
+#       };
+#       ESP = {
+#         type = "EF00";
+#         size = "500M";
+#         content = {
+#           type = "filesystem";
+#           format = "vfat";
+#           mountpoint = "/boot";
+#           mountOptions = ["umask=0077"];
+#         };
+#       };
+#       root = {
+#         size = "100%";
+#         content = {
+#           type = "filesystem";
+#           format = "ext4";
+#           mountpoint = "/";
+#         };
+#       };
+#     };
+#   };
+# }
+

hosts/sfos-1/hardware-configuration.nix

@@ -0,0 +1,26 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  lib,
+  modulesPath,
+  ...
+}: {
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+
+  boot.initrd.availableKernelModules = ["virtio_pci" "virtio_scsi" "ahci" "sd_mod"];
+  boot.initrd.kernelModules = [];
+  boot.kernelModules = [];
+  boot.extraModulePackages = [];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp0s4.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/sfos-1/nix.nix

@@ -0,0 +1,33 @@
+{pkgs, ...}: {
+  nixpkgs = {
+    config = {
+      allowUnfree = true;
+    };
+  };
+
+  nix = {
+    package = pkgs.nix;
+
+    gc = {
+      automatic = true;
+      dates = "weekly";
+      options = "--delete-older-than 14d";
+    };
+
+    settings = {
+      trusted-users = [
+        "@wheel"
+      ];
+
+      experimental-features = [
+        "flakes"
+        "nix-command"
+      ];
+
+      trusted-public-keys = [
+        "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+        "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+      ];
+    };
+  };
+}

hosts/sfos-1/nomad.nix

@@ -0,0 +1,58 @@
+{...}: {
+  services.nomad = {
+    enable = true;
+    enableDocker = true;
+    settings = {
+      server = {
+        enabled = true;
+        # Ideally the bootstrap expect value would reflect the number of servers set by `instance_count`, but in the interest
+        # of not chaning each server's "user data" every time the instance_count changes, we hard code the nomad cluster to
+        # bootstrap itself whenever 3 nodes are present
+        bootstrap_expect = 3;
+
+        server_join = {
+          retry_join = [];
+        };
+      };
+
+      acl = {
+        enabled = true;
+      };
+
+      client = {
+        enabled = true;
+      };
+
+      plugin.docker = {
+        config = {
+          gc = {
+            image = true;
+            image_delay = "3m";
+            container = true;
+
+            dangling_containers = {
+              enabled = true;
+              dry_run = false;
+              period = "5m";
+              creation_grace = "5m";
+            };
+          };
+
+          volumes = {
+            enabled = true;
+            selinuxlabel = "z";
+          };
+
+          allow_caps = ["chown" "net_raw"];
+          allow_privileged = true;
+        };
+      };
+
+      plugin.raw_exec = {
+        config = {
+          enabled = true;
+        };
+      };
+    };
+  };
+}

hosts/sfos-1/secrets/docker_ghcr_auth_config_json.age

@@ -0,0 +1,17 @@
+age-encryption.org/v1
+-> ssh-rsa QGRkYA
+dJW71AoISeNPqSMv7bsAnYQ8yFAA7SsE6Dkh+yGFp9oXAJ/AjIM+/DlaLM2AeZol
+JgekTnmRqFNZzpJ+Q7q2OAE76ROXG1rQhwuyQa2axnsHeup749QXnWam2V3SATA5
+kVaLVO4QbzvFIEUia1rPmtUPSImO0QxkzI3dUz19j3pIEG6kMQEJyuJ9FurS+rSX
+KnZxes8Q3Q8iszeufN6+1L3ENWOHwMKaSWw+cV4o1Pis8Rh/wUQwi9iyJGSZx4rT
+IORFhoJsXI6q2WvnnEcM0BBjiKylVW2e4P6fBkKg3+EEN3cHL026RjzyddjsDy0n
+ppHmB2bCNIf3PW7za8TfZ47tqKgA+mNpkXO85SkP5S5u87hBjILlzAXtevFnVrr+
+Xpjq8ZHDy7QYLmQl6i0opgxLe7JuOYP++3/6PL5T2XcEw7kymXlg2KwtQN4TBEZj
+YDnhPuh++TTikSyKeNzzpGTc/jnXV2BIAevXLQ0PmCs6k5/FrD2KS+rzRmVuGl44
+
+-> ssh-ed25519 9G9Vsg ZKyEMJVyLAr0G3GhOoqf+0+mZUa6XwbFc1PYGLRZjXQ
+CSBhHp+92cH5hu3p5oTzqxkYS1uTo3EWbqlMEtOJn7M
+-> ssh-ed25519 KKRRSg EE6TBwP0ScwxT8zsdMkEnbldb2sAaQiuQ4ILZY2/MkE
+Qdr7swRd4uQ0DV0S42WDxC+RWI8zS4nz9h1HbcQ7zME
+--- ltjYR7/VAs27ta24Y4d2UhWD7hOGBVYl4Zk3b0nXAqg
+ýb61æÙƾ+œší‡Âî¾5;Í
:!><3E>¿Aȱ2µ<32>GéR†’â<E28099>Í2Î¥–ú˜ö?–ª[ž8d³™¯Nú;Žð°´TÚy4Pü³Â<16>æ¼hõ:»Ê’,¤-Š8éO=ÝÚ"£âÀ¯óë„%:(Æbüù1_<31>öÖŒæ3ç_ÖÅíÜÅû^˜®'“ÿç4e·á

hosts/sfos-1/secrets/secrets.nix

@@ -0,0 +1,7 @@
+# Use ssh-keyscan <SERVER> to fetch the below keys for target systems (SSH server must be running on system)
+let
+  localMachine = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1";
+in {
+  # "docker_ghcr_auth_config_json.age".publicKeys = [localMachine sfosnew];
+  tailscale_auth_key.publicKeys = [localMachine "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKEp+dbZKfMP05gwDhI2cimqJvsM1kBzIZEJ4VlEmlh/"];
+}

hosts/sfos-1/secrets/tailscale_auth_key

@@ -0,0 +1,16 @@
+age-encryption.org/v1
+-> ssh-rsa QGRkYA
+TFdKGkVbaTg3UY4Ec8rWL7yMWo98zBAMwdwnN7joJ5N5/Fii6oQWrUYmANO4gFbr
+Um4alYXABbTZ+0Kk8gbRAFJuTq299kAtEQWHJhUUCYki3Mcdc4JpJD+mgaHF52dI
+Xg6/afbwZqibtuyiELKKi9AoCIbCg1HSerIkNkjkvbJyfamT6JnHEJHKLat6Hw2K
+an51I7HaSjxO5sm4ZRWP5CZK/qFavmPtxnFe76zrfKlIMaM6UXfCnssHMOOmZ3bR
+vMt5lWrgh/cGWS6HgYupNv6fuAqIuLMOG43T5PdCn8lgO3GNZLcRJ/DuJZy7CESg
+fQkV5sBTPllwKHqMSC7hCO3MMojWmawT7ShxynWS4TdmzZtopm3R9aEDmccFA599
+6H7kaoy/rbAQNdLUEfpB459iLtHPTrFR8o9ISquqasffbmmrySh9BVDVi0mR1Ozo
+oBVkHQz3Y3YXw+DZIQz481MYWFhlX01G3SQX7OIwA9OeNuRBkfrx3zAp222SllMi
+
+-> ssh-ed25519 lAwSkg HUtVizzG7ytaUIT+3saAvKzKOFk7Pku0DU53hsZX2xI
+N2kVZPATopjyvTterxctu58cNuz8pqBXPjb7+XagjEU
+--- Aof3l4CecKF5oJA7qFHEtLajljBDr6OueVw3haf7QHs
+ËH\4«;<3B>A¸2<C2B8>0ö<30>cö‡T ”ÏÈaGE/ðº…UÌ
ÒJ#mö7T«b!k×x^æ
+4™Ñ»xç
ظÔùQ7i|’‘G¬½ˆ,±echWê²—

hosts/sfos-2/base-system.nix

@@ -0,0 +1,71 @@
+{
+  lib,
+  pkgs,
+  self,
+  ...
+}: {
+  imports = [
+    ./nix.nix
+  ];
+
+  boot = {
+    initrd.systemd.enable = true;
+    tmp = {
+      cleanOnBoot = true;
+      useTmpfs = true;
+    };
+  };
+
+  environment = {
+    systemPackages = with pkgs; [
+      lsof
+      wget
+    ];
+  };
+
+  security.sudo.wheelNeedsPassword = false;
+  i18n = let
+    locale = "en_US.UTF-8";
+  in {
+    defaultLocale = locale;
+    extraLocaleSettings = {
+      LC_ADDRESS = locale;
+      LC_IDENTIFICATION = locale;
+      LC_MEASUREMENT = locale;
+      LC_MONETARY = locale;
+      LC_NAME = locale;
+      LC_NUMERIC = locale;
+      LC_PAPER = locale;
+      LC_TELEPHONE = locale;
+      LC_TIME = locale;
+    };
+  };
+
+  networking = {
+    enableIPv6 = false;
+  };
+
+  programs = {};
+
+  services = {
+    fstrim.enable = true;
+    locate.enable = true;
+    rsyslogd.enable = true;
+    openssh = {
+      enable = true;
+      settings.PermitRootLogin = "yes";
+    };
+  };
+
+  security = {
+    rtkit.enable = true;
+  };
+
+  systemd.services.nix-daemon = {
+    environment.TMPDIR = "/var/tmp";
+  };
+
+  time.timeZone = "America/New_York";
+  users.mutableUsers = false;
+  system.configurationRevision = lib.mkIf (self ? rev) self.rev;
+}

hosts/sfos-2/default.nix

@@ -0,0 +1,116 @@
+{
+  config,
+  inputs,
+  pkgs,
+  agenix,
+  ...
+}: {
+  deployment = {
+    targetHost = "157.180.16.185";
+    targetPort = 22;
+    targetUser = "root";
+    buildOnTarget = true;
+  };
+
+  networking = {
+    hostName = "sfos-2";
+    nftables.enable = true;
+    firewall = {
+      allowedTCPPorts = [22 80 443];
+      allowedUDPPorts = [config.services.tailscale.port];
+    };
+  };
+  imports = [
+    agenix.nixosModules.default
+    inputs.nixos-facter-modules.nixosModules.facter
+    {config.facter.reportPath = ./facter.json;}
+    inputs.disko.nixosModules.disko
+    ./hardware-configuration.nix
+    ./disko-config.nix
+    ./base-system.nix
+    ./nix.nix
+    ./nomad.nix
+  ];
+
+  boot = {
+    supportedFilesystems = ["btrfs" "ntfs" "vfat" "ext4"];
+    loader.grub = {
+      enable = true;
+    };
+  };
+
+  # age.secrets.docker_auth_json = {
+  #   file = ./secrets/docker_ghcr_auth_config_json.age;
+  #   path = "/run/containers/0/auth.json";
+  # };
+
+  age.secrets.tailscale_auth_key = {
+    file = ./secrets/tailscale_auth_key;
+  };
+
+  environment.systemPackages = with pkgs; [
+    nomad_1_9
+  ];
+
+  #
+  # TEST: authentication to private repo
+  #
+  # environment.variables = {
+  #   XDG_RUNTIME_DIR = "/var/run/user/0";
+  # };
+
+  # Define a systemd service to create files in /var/run/user/0
+  # systemd.services."create-incus-ghcr-auth-file" = {
+  #   description = "Create ghcr.io auth file for incus to retrieve private images";
+  #   after = ["default.target"];
+  #   wantedBy = ["default.target"];
+  #   serviceConfig = {
+  #     ExecStart = ''${pkgs.bashInteractive}/bin/bash -c 'echo {\\"auths\\":{ \\"ghcr.io\\": {\\"auth\\": \\"YWNhbG9pYXJvOiBnaHBfVnN4dzc4YXZ6N0lBeXQyejVrcGpZQTJXV21XbUZNMzlZdkMxIA==\\"}}} > /var/run/user/0/containers/auth.json' '';
+  #     ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p /var/run/user/0/containers";
+  #     Restart = "no";
+  #   };
+  # };
+  #
+  # /TEST: authentication to private repo
+  #
+
+  users.users = {
+    root = {
+      openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
+    };
+    sfos = {
+      isNormalUser = true;
+      extraGroups = ["wheel"];
+      # TODO Initial password and authorized keys need to come from .env, perhaps using nix-edit
+      initialPassword = "Test.123";
+      openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
+    };
+  };
+
+  services = {
+    # TODO caddy needs a plugin that watches the nomad node list for specific tags that cause them to be reverse proxied
+    # from this host system
+    caddy = {
+      enable = true;
+      # package = pkgs.caddy.withPlugins {
+      #   plugins = ["github.com/acaloiaro/caddy-incus-upstreams@v0.0.0-20241228235755-786cd3b6a9dc"];
+      #   hash = "sha256-yjqXgo7WiJPBVmN3+O8Wq9Nxd1xU8KrXW7Hv7Elcf7I=";
+      # };
+      # virtualHosts = {
+      #   "https://adrianofyi.incus.adriano.fyi:443" = {
+      #     extraConfig = ''
+      #        reverse_proxy {
+      #       	dynamic incus
+      #       }
+      #     '';
+      #   };
+      # };
+    };
+    openssh.enable = true;
+    tailscale = {
+      enable = true;
+      authKeyFile = config.age.secrets.tailscale_auth_key.path;
+    };
+  };
+  system.stateVersion = "24.05";
+}

hosts/sfos-2/disko-config.nix

@@ -0,0 +1,99 @@
+{
+  disko.devices.disk.os = {
+    # device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_59633795";
+    device = "/dev/sda";
+    type = "disk";
+    content = {
+      type = "gpt";
+      partitions = {
+        boot = {
+          size = "1M";
+          type = "EF02"; # for grub MBR
+        };
+        # ESP = {
+        #   type = "EF00";
+        #   size = "500M";
+        #   content = {
+        #     type = "filesystem";
+        #     format = "vfat";
+        #     mountpoint = "/boot";
+        #     mountOptions = ["umask=0077"];
+        #   };
+        # };
+        root = {
+          size = "100%";
+          content = {
+            type = "filesystem";
+            format = "ext4";
+            mountpoint = "/";
+          };
+        };
+      };
+    };
+  };
+}
+# {
+#   disko.devices.disk.os = {
+#     type = "disk";
+#     content = {
+#       device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_59632825";
+#       type = "gpt";
+#       partitions = {
+#         boot = {
+#           type = "EF02";
+#           size = "1M";
+#         };
+#         ESP = {
+#           type = "EF00";
+#           size = "500M";
+#           content = {
+#             type = "filesystem";
+#             format = "vfat";
+#             mountpoint = "/boot";
+#             mountOptions = ["umask=0077"];
+#           };
+#         };
+#         root = {
+#           size = "100%";
+#           content = {
+#             type = "filesystem";
+#             format = "ext4";
+#             mountpoint = "/";
+#           };
+#         };
+#       };
+#     };
+#   };
+# }
+# {
+#   device = "/dev/sda";
+#   type = "disk";
+#   content = {
+#     type = "gpt";
+#     partitions = {
+#       boot = {
+#         type = "EF02";
+#         size = "1M";
+#       };
+#       ESP = {
+#         type = "EF00";
+#         size = "500M";
+#         content = {
+#           type = "filesystem";
+#           format = "vfat";
+#           mountpoint = "/boot";
+#           mountOptions = ["umask=0077"];
+#         };
+#       };
+#       root = {
+#         size = "100%";
+#         content = {
+#           type = "filesystem";
+#           format = "ext4";
+#           mountpoint = "/";
+#         };
+#       };
+#     };
+#   };
+# }
+

hosts/sfos-2/hardware-configuration.nix

@@ -0,0 +1,26 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  lib,
+  modulesPath,
+  ...
+}: {
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+
+  boot.initrd.availableKernelModules = ["virtio_pci" "virtio_scsi" "ahci" "sd_mod"];
+  boot.initrd.kernelModules = [];
+  boot.kernelModules = [];
+  boot.extraModulePackages = [];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp0s4.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/sfos-2/nix.nix

@@ -0,0 +1,33 @@
+{pkgs, ...}: {
+  nixpkgs = {
+    config = {
+      allowUnfree = true;
+    };
+  };
+
+  nix = {
+    package = pkgs.nix;
+
+    gc = {
+      automatic = true;
+      dates = "weekly";
+      options = "--delete-older-than 14d";
+    };
+
+    settings = {
+      trusted-users = [
+        "@wheel"
+      ];
+
+      experimental-features = [
+        "flakes"
+        "nix-command"
+      ];
+
+      trusted-public-keys = [
+        "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+        "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+      ];
+    };
+  };
+}

hosts/sfos-2/nomad.nix

@@ -0,0 +1,58 @@
+{...}: {
+  services.nomad = {
+    enable = true;
+    enableDocker = true;
+    settings = {
+      server = {
+        enabled = true;
+        # Ideally the bootstrap expect value would reflect the number of servers set by `instance_count`, but in the interest
+        # of not chaning each server's "user data" every time the instance_count changes, we hard code the nomad cluster to
+        # bootstrap itself whenever 3 nodes are present
+        bootstrap_expect = 3;
+
+        server_join = {
+          retry_join = [];
+        };
+      };
+
+      acl = {
+        enabled = true;
+      };
+
+      client = {
+        enabled = true;
+      };
+
+      plugin.docker = {
+        config = {
+          gc = {
+            image = true;
+            image_delay = "3m";
+            container = true;
+
+            dangling_containers = {
+              enabled = true;
+              dry_run = false;
+              period = "5m";
+              creation_grace = "5m";
+            };
+          };
+
+          volumes = {
+            enabled = true;
+            selinuxlabel = "z";
+          };
+
+          allow_caps = ["chown" "net_raw"];
+          allow_privileged = true;
+        };
+      };
+
+      plugin.raw_exec = {
+        config = {
+          enabled = true;
+        };
+      };
+    };
+  };
+}

hosts/sfos-2/secrets/docker_ghcr_auth_config_json.age

@@ -0,0 +1,17 @@
+age-encryption.org/v1
+-> ssh-rsa QGRkYA
+dJW71AoISeNPqSMv7bsAnYQ8yFAA7SsE6Dkh+yGFp9oXAJ/AjIM+/DlaLM2AeZol
+JgekTnmRqFNZzpJ+Q7q2OAE76ROXG1rQhwuyQa2axnsHeup749QXnWam2V3SATA5
+kVaLVO4QbzvFIEUia1rPmtUPSImO0QxkzI3dUz19j3pIEG6kMQEJyuJ9FurS+rSX
+KnZxes8Q3Q8iszeufN6+1L3ENWOHwMKaSWw+cV4o1Pis8Rh/wUQwi9iyJGSZx4rT
+IORFhoJsXI6q2WvnnEcM0BBjiKylVW2e4P6fBkKg3+EEN3cHL026RjzyddjsDy0n
+ppHmB2bCNIf3PW7za8TfZ47tqKgA+mNpkXO85SkP5S5u87hBjILlzAXtevFnVrr+
+Xpjq8ZHDy7QYLmQl6i0opgxLe7JuOYP++3/6PL5T2XcEw7kymXlg2KwtQN4TBEZj
+YDnhPuh++TTikSyKeNzzpGTc/jnXV2BIAevXLQ0PmCs6k5/FrD2KS+rzRmVuGl44
+
+-> ssh-ed25519 9G9Vsg ZKyEMJVyLAr0G3GhOoqf+0+mZUa6XwbFc1PYGLRZjXQ
+CSBhHp+92cH5hu3p5oTzqxkYS1uTo3EWbqlMEtOJn7M
+-> ssh-ed25519 KKRRSg EE6TBwP0ScwxT8zsdMkEnbldb2sAaQiuQ4ILZY2/MkE
+Qdr7swRd4uQ0DV0S42WDxC+RWI8zS4nz9h1HbcQ7zME
+--- ltjYR7/VAs27ta24Y4d2UhWD7hOGBVYl4Zk3b0nXAqg
+ýb61æÙƾ+œší‡Âî¾5;Í
:!><3E>¿Aȱ2µ<32>GéR†’â<E28099>Í2Î¥–ú˜ö?–ª[ž8d³™¯Nú;Žð°´TÚy4Pü³Â<16>æ¼hõ:»Ê’,¤-Š8éO=ÝÚ"£âÀ¯óë„%:(Æbüù1_<31>öÖŒæ3ç_ÖÅíÜÅû^˜®'“ÿç4e·á

hosts/sfos-2/secrets/secrets.nix

@@ -0,0 +1,7 @@
+# Use ssh-keyscan <SERVER> to fetch the below keys for target systems (SSH server must be running on system)
+let
+  localMachine = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1";
+in {
+  # "docker_ghcr_auth_config_json.age".publicKeys = [localMachine sfosnew];
+  tailscale_auth_key.publicKeys = [localMachine "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOHlumbz17LcZBbhASdTexz+qdtFQui2jGtZQQpqw3xG"];
+}

hosts/sfos-2/secrets/tailscale_auth_key

@@ -0,0 +1,16 @@
+age-encryption.org/v1
+-> ssh-rsa QGRkYA
+Sn8qFRelbxE52SjhiGJaVyOm3R7PpRk1FgZxcSKyeKt2nEftmeteO1ue6tY6FZNz
+idIW3aLJNgxwrOOOj4XkwWyDw3yTboqWUIxTmQK+wvIOLl1GyOSZRTqytHrsonlo
+Nsoz5s4QIDx7OE1sqRnbkB+NdACYTI8MNfkAr5aSx6hsLmX+wgKKI4SoOcMxpkgP
+StKk4iiSlIQlnQf6ev0yXKuvo2e7HwbabnTi9Ik8qrXU6IiKjNvd8lJyYgFBh4w7
+JrngF1pSKqud0Mld8v2mbCb83zmsqyK/mCY5VaI194ymnqQeSGEZyzfNd1JcoXj/
+cNugHhS5AE5mbT3t5UrvdsHZEOAsJb0x0r7URhMYbcm09FjZpSeUErrUwX2fORvh
+koIDN1XlKAGfeiypJN3CWfbOoQM9dbiJ2j+wyw2/hYIDfsCVDvYolfQ2cbZV6t7g
+B9UxCgUcefRuJwNsjUCDVNVAxQTRdAjDNJRZsvuu8GXNpr5qhrq45Ubv624+b49g
+
+-> ssh-ed25519 mLVJBA kaeK1dfmJd07snz6Bfs1DlBJXlQGN4hXA6wnL8e88lg
+/oSUB6IK+yRZ0diaOQxx4+v+lxINnlSYyV/uGO5PXQ0
+--- CnljCNkgqJr1tkheis1KKe+U2kAfX05Kuw3wZ5IuCKA
+*æùµ¼pè[ Æ®‹j
*Ü'[J×ëáÆ_-Î
+¦±3K‹¬•³®‘دátä¤dí»°˜"þî<C3BE>LÙ¿ûFñ#Є\峧ÏÄ
?·®@§óÉEío/¨VÉ´ø

terraform.tfstate

@@ -0,0 +1 @@
+{"version":4,"terraform_version":"1.9.0","serial":1,"lineage":"a583e282-ff90-f5dd-b2fd-af7cc28dab8c","outputs":{},"resources":[],"check_results":null}