me/ncm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: tailscale initialization and nomad (almost clustered)

Browse source
This commit is contained in:
Adriano Caloiaro 2025-04-14 11:38:16 -06:00
parent 9f4cf95e18
commit e9c74d7c9a
No known key found for this signature in database
GPG key ID: C2BC56DE73CE3F75
42 changed files with 1865 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build/systems/default/base-system.nix
View file

@ -57,8 +57,6 @@
sfos = {
isNormalUser = true;
extraGroups = ["wheel"];
# TODO Initial password and authorized keys need to come from .env, perhaps using nix-edit
initialPassword = "Test.123";
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
};
};

4
deploy/compute_providers/linode/main.tf
View file

@ -34,14 +34,14 @@ resource "linode_instance" "ncm" {
tags = ["ncm"]
region = var.compute_provider_region
type = var.compute_provider_node_type
booted = true
}
resource "linode_instance_config" "config" {
count = var.node_count
linode_id = linode_instance.ncm[count.index].id
label = "nvm-${count.index}"
kernel = "linode/grub2"
booted = true
linode_id = linode_instance.ncm[count.index].id
memory_limit = 0
root_device = "/dev/sda"
run_level = "default"

12
deploy/compute_providers/linode/outputs.tf
View file

@ -1,3 +1,11 @@
output "ip_addresses" {
value = flatten(linode_instance.ncm.*.ipv4)
output "node_details" {
value = [
for instance in linode_instance.ncm :
{
id = instance.id
label = instance.label
ipv4 = instance.ip_address
ipv6 = instance.ipv6
}
]
}

65
deploy/ncm-0/base-system.nix Normal file
View file

@ -0,0 +1,65 @@
{
lib,
pkgs,
self,
...
}: {
boot = {
initrd.systemd.enable = true;
tmp = {
cleanOnBoot = true;
useTmpfs = true;
};
};
nixpkgs = {
config = {
allowUnfree = true;
};
};
nix = {
package = pkgs.nix;
gc = {
automatic = true;
dates = "weekly";
options = "--delete-older-than 14d";
};
settings = {
trusted-users = [
"@wheel"
];
experimental-features = [
"flakes"
"nix-command"
];
trusted-public-keys = [
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
];
};
};
security.sudo.wheelNeedsPassword = true;
services = {
rsyslogd.enable = true;
openssh = {
enable = true;
settings.PermitRootLogin = "yes";
};
};
security = {
rtkit.enable = true;
};
systemd.services.nix-daemon = {
environment.TMPDIR = "/var/tmp";
};
time.timeZone = "America/New_York";
users.mutableUsers = false;
system.configurationRevision = lib.mkIf (self ? rev) self.rev;
}

118
deploy/ncm-0/default.nix Normal file
View file

@ -0,0 +1,118 @@
{
config,
inputs,
pkgs,
agenix,
...
}: {
deployment = {
targetHost = "<PLACEHOLDER>";
targetPort = 22;
targetUser = "root";
buildOnTarget = true;
};
networking = {
hostName = "<PLACEHOLDER>";
nftables.enable = true;
firewall = {
allowedTCPPorts = [22 80 443];
allowedUDPPorts = [config.services.tailscale.port];
};
};
imports = [
agenix.nixosModules.default
inputs.nixos-facter-modules.nixosModules.facter
{config.facter.reportPath = ./facter.json;}
inputs.disko.nixosModules.disko
./hardware-configuration.nix
./disko-config.nix
./base-system.nix
./nix.nix
./nomad.nix
];
boot = {
supportedFilesystems = ["btrfs" "ntfs" "vfat" "ext4"];
loader.grub = {
enable = true;
};
};
# age.secrets.docker_auth_json = {
# file = ./secrets/docker_ghcr_auth_config_json.age;
# path = "/run/containers/0/auth.json";
# };
age.secrets.tailscale_auth_key = {
file = ./secrets/tailscale_auth_key;
};
environment.systemPackages = with pkgs; [
nomad_1_9
jq
];
#
# TEST: authentication to private repo
#
# environment.variables = {
# XDG_RUNTIME_DIR = "/var/run/user/0";
# };
# Define a systemd service to create files in /var/run/user/0
# systemd.services."create-incus-ghcr-auth-file" = {
# description = "Create ghcr.io auth file for incus to retrieve private images";
# after = ["default.target"];
# wantedBy = ["default.target"];
# serviceConfig = {
# ExecStart = ''${pkgs.bashInteractive}/bin/bash -c 'echo {\\"auths\\":{ \\"ghcr.io\\": {\\"auth\\": \\"YWNhbG9pYXJvOiBnaHBfVnN4dzc4YXZ6N0lBeXQyejVrcGpZQTJXV21XbUZNMzlZdkMxIA==\\"}}} > /var/run/user/0/containers/auth.json' '';
# ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p /var/run/user/0/containers";
# Restart = "no";
# };
# };
#
# /TEST: authentication to private repo
#
users.users = {
root = {
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
};
sfos = {
isNormalUser = true;
extraGroups = ["wheel"];
# TODO Initial password and authorized keys need to come from .env, perhaps using nix-edit
initialPassword = "Test.123";
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
};
};
services = {
# TODO caddy needs a plugin that watches the nomad node list for specific tags that cause them to be reverse proxied
# from this host system
caddy = {
enable = true;
# package = pkgs.caddy.withPlugins {
# plugins = ["github.com/acaloiaro/caddy-incus-upstreams@v0.0.0-20241228235755-786cd3b6a9dc"];
# hash = "sha256-yjqXgo7WiJPBVmN3+O8Wq9Nxd1xU8KrXW7Hv7Elcf7I=";
# };
# virtualHosts = {
# "https://adrianofyi.incus.adriano.fyi:443" = {
# extraConfig = ''
# reverse_proxy {
# dynamic incus
# }
# '';
# };
# };
};
openssh.enable = true;
tailscale = {
enable = true;
authKeyFile = config.age.secrets.tailscale_auth_key.path;
extraUpFlags = ["--advertise-tags tag:sfos"];
};
};
system.stateVersion = "24.05";
}

60
deploy/ncm-0/nomad.nix Normal file
View file

@ -0,0 +1,60 @@
{...}: {
services.nomad = {
enable = true;
enableDocker = true;
dropPrivileges = false;
settings = {
data_dir = "/var/lib/private/nomad";
server = {
enabled = true;
# Ideally the bootstrap expect value would reflect the number of servers set by `instance_count`, but in the interest
# of not chaning each server's "user data" every time the instance_count changes, we hard code the nomad cluster to
# bootstrap itself whenever 3 nodes are present
bootstrap_expect = 3;
server_join = {
retry_join = [];
};
};
acl = {
enabled = true;
};
client = {
enabled = true;
};
plugin.docker = {
config = {
gc = {
image = true;
image_delay = "3m";
container = true;
dangling_containers = {
enabled = true;
dry_run = false;
period = "5m";
creation_grace = "5m";
};
};
volumes = {
enabled = true;
selinuxlabel = "z";
};
allow_caps = ["chown" "net_raw"];
allow_privileged = true;
};
};
plugin.raw_exec = {
config = {
enabled = true;
};
};
};
};
}

65
deploy/ncm-1/base-system.nix Normal file
View file

@ -0,0 +1,65 @@
{
lib,
pkgs,
self,
...
}: {
boot = {
initrd.systemd.enable = true;
tmp = {
cleanOnBoot = true;
useTmpfs = true;
};
};
nixpkgs = {
config = {
allowUnfree = true;
};
};
nix = {
package = pkgs.nix;
gc = {
automatic = true;
dates = "weekly";
options = "--delete-older-than 14d";
};
settings = {
trusted-users = [
"@wheel"
];
experimental-features = [
"flakes"
"nix-command"
];
trusted-public-keys = [
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
];
};
};
security.sudo.wheelNeedsPassword = true;
services = {
rsyslogd.enable = true;
openssh = {
enable = true;
settings.PermitRootLogin = "yes";
};
};
security = {
rtkit.enable = true;
};
systemd.services.nix-daemon = {
environment.TMPDIR = "/var/tmp";
};
time.timeZone = "America/New_York";
users.mutableUsers = false;
system.configurationRevision = lib.mkIf (self ? rev) self.rev;
}

118
deploy/ncm-1/default.nix Normal file
View file

@ -0,0 +1,118 @@
{
config,
inputs,
pkgs,
agenix,
...
}: {
deployment = {
targetHost = "<PLACEHOLDER>";
targetPort = 22;
targetUser = "root";
buildOnTarget = true;
};
networking = {
hostName = "<PLACEHOLDER>";
nftables.enable = true;
firewall = {
allowedTCPPorts = [22 80 443];
allowedUDPPorts = [config.services.tailscale.port];
};
};
imports = [
agenix.nixosModules.default
inputs.nixos-facter-modules.nixosModules.facter
{config.facter.reportPath = ./facter.json;}
inputs.disko.nixosModules.disko
./hardware-configuration.nix
./disko-config.nix
./base-system.nix
./nix.nix
./nomad.nix
];
boot = {
supportedFilesystems = ["btrfs" "ntfs" "vfat" "ext4"];
loader.grub = {
enable = true;
};
};
# age.secrets.docker_auth_json = {
# file = ./secrets/docker_ghcr_auth_config_json.age;
# path = "/run/containers/0/auth.json";
# };
age.secrets.tailscale_auth_key = {
file = ./secrets/tailscale_auth_key;
};
environment.systemPackages = with pkgs; [
nomad_1_9
jq
];
#
# TEST: authentication to private repo
#
# environment.variables = {
# XDG_RUNTIME_DIR = "/var/run/user/0";
# };
# Define a systemd service to create files in /var/run/user/0
# systemd.services."create-incus-ghcr-auth-file" = {
# description = "Create ghcr.io auth file for incus to retrieve private images";
# after = ["default.target"];
# wantedBy = ["default.target"];
# serviceConfig = {
# ExecStart = ''${pkgs.bashInteractive}/bin/bash -c 'echo {\\"auths\\":{ \\"ghcr.io\\": {\\"auth\\": \\"YWNhbG9pYXJvOiBnaHBfVnN4dzc4YXZ6N0lBeXQyejVrcGpZQTJXV21XbUZNMzlZdkMxIA==\\"}}} > /var/run/user/0/containers/auth.json' '';
# ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p /var/run/user/0/containers";
# Restart = "no";
# };
# };
#
# /TEST: authentication to private repo
#
users.users = {
root = {
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
};
sfos = {
isNormalUser = true;
extraGroups = ["wheel"];
# TODO Initial password and authorized keys need to come from .env, perhaps using nix-edit
initialPassword = "Test.123";
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
};
};
services = {
# TODO caddy needs a plugin that watches the nomad node list for specific tags that cause them to be reverse proxied
# from this host system
caddy = {
enable = true;
# package = pkgs.caddy.withPlugins {
# plugins = ["github.com/acaloiaro/caddy-incus-upstreams@v0.0.0-20241228235755-786cd3b6a9dc"];
# hash = "sha256-yjqXgo7WiJPBVmN3+O8Wq9Nxd1xU8KrXW7Hv7Elcf7I=";
# };
# virtualHosts = {
# "https://adrianofyi.incus.adriano.fyi:443" = {
# extraConfig = ''
# reverse_proxy {
# dynamic incus
# }
# '';
# };
# };
};
openssh.enable = true;
tailscale = {
enable = true;
authKeyFile = config.age.secrets.tailscale_auth_key.path;
extraUpFlags = ["--advertise-tags tag:sfos"];
};
};
system.stateVersion = "24.05";
}

60
deploy/ncm-1/nomad.nix Normal file
View file

@ -0,0 +1,60 @@
{...}: {
services.nomad = {
enable = true;
enableDocker = true;
dropPrivileges = false;
settings = {
data_dir = "/var/lib/private/nomad";
server = {
enabled = true;
# Ideally the bootstrap expect value would reflect the number of servers set by `instance_count`, but in the interest
# of not chaning each server's "user data" every time the instance_count changes, we hard code the nomad cluster to
# bootstrap itself whenever 3 nodes are present
bootstrap_expect = 3;
server_join = {
retry_join = ["exec=linode-cli linodes list --tags ncm --json | jq '.[].ipv4[]' | tr \"\n\" \",\""];
};
};
acl = {
enabled = true;
};
client = {
enabled = true;
};
plugin.docker = {
config = {
gc = {
image = true;
image_delay = "3m";
container = true;
dangling_containers = {
enabled = true;
dry_run = false;
period = "5m";
creation_grace = "5m";
};
};
volumes = {
enabled = true;
selinuxlabel = "z";
};
allow_caps = ["chown" "net_raw"];
allow_privileged = true;
};
};
plugin.raw_exec = {
config = {
enabled = true;
};
};
};
};
}

65
deploy/ncm-2/base-system.nix Normal file
View file

@ -0,0 +1,65 @@
{
lib,
pkgs,
self,
...
}: {
boot = {
initrd.systemd.enable = true;
tmp = {
cleanOnBoot = true;
useTmpfs = true;
};
};
nixpkgs = {
config = {
allowUnfree = true;
};
};
nix = {
package = pkgs.nix;
gc = {
automatic = true;
dates = "weekly";
options = "--delete-older-than 14d";
};
settings = {
trusted-users = [
"@wheel"
];
experimental-features = [
"flakes"
"nix-command"
];
trusted-public-keys = [
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
];
};
};
security.sudo.wheelNeedsPassword = true;
services = {
rsyslogd.enable = true;
openssh = {
enable = true;
settings.PermitRootLogin = "yes";
};
};
security = {
rtkit.enable = true;
};
systemd.services.nix-daemon = {
environment.TMPDIR = "/var/tmp";
};
time.timeZone = "America/New_York";
users.mutableUsers = false;
system.configurationRevision = lib.mkIf (self ? rev) self.rev;
}

118
deploy/ncm-2/default.nix Normal file
View file

@ -0,0 +1,118 @@
{
config,
inputs,
pkgs,
agenix,
...
}: {
deployment = {
targetHost = "<PLACEHOLDER>";
targetPort = 22;
targetUser = "root";
buildOnTarget = true;
};
networking = {
hostName = "<PLACEHOLDER>";
nftables.enable = true;
firewall = {
allowedTCPPorts = [22 80 443];
allowedUDPPorts = [config.services.tailscale.port];
};
};
imports = [
agenix.nixosModules.default
inputs.nixos-facter-modules.nixosModules.facter
{config.facter.reportPath = ./facter.json;}
inputs.disko.nixosModules.disko
./hardware-configuration.nix
./disko-config.nix
./base-system.nix
./nix.nix
./nomad.nix
];
boot = {
supportedFilesystems = ["btrfs" "ntfs" "vfat" "ext4"];
loader.grub = {
enable = true;
};
};
# age.secrets.docker_auth_json = {
# file = ./secrets/docker_ghcr_auth_config_json.age;
# path = "/run/containers/0/auth.json";
# };
age.secrets.tailscale_auth_key = {
file = ./secrets/tailscale_auth_key;
};
environment.systemPackages = with pkgs; [
nomad_1_9
jq
];
#
# TEST: authentication to private repo
#
# environment.variables = {
# XDG_RUNTIME_DIR = "/var/run/user/0";
# };
# Define a systemd service to create files in /var/run/user/0
# systemd.services."create-incus-ghcr-auth-file" = {
# description = "Create ghcr.io auth file for incus to retrieve private images";
# after = ["default.target"];
# wantedBy = ["default.target"];
# serviceConfig = {
# ExecStart = ''${pkgs.bashInteractive}/bin/bash -c 'echo {\\"auths\\":{ \\"ghcr.io\\": {\\"auth\\": \\"YWNhbG9pYXJvOiBnaHBfVnN4dzc4YXZ6N0lBeXQyejVrcGpZQTJXV21XbUZNMzlZdkMxIA==\\"}}} > /var/run/user/0/containers/auth.json' '';
# ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p /var/run/user/0/containers";
# Restart = "no";
# };
# };
#
# /TEST: authentication to private repo
#
users.users = {
root = {
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
};
sfos = {
isNormalUser = true;
extraGroups = ["wheel"];
# TODO Initial password and authorized keys need to come from .env, perhaps using nix-edit
initialPassword = "Test.123";
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
};
};
services = {
# TODO caddy needs a plugin that watches the nomad node list for specific tags that cause them to be reverse proxied
# from this host system
caddy = {
enable = true;
# package = pkgs.caddy.withPlugins {
# plugins = ["github.com/acaloiaro/caddy-incus-upstreams@v0.0.0-20241228235755-786cd3b6a9dc"];
# hash = "sha256-yjqXgo7WiJPBVmN3+O8Wq9Nxd1xU8KrXW7Hv7Elcf7I=";
# };
# virtualHosts = {
# "https://adrianofyi.incus.adriano.fyi:443" = {
# extraConfig = ''
# reverse_proxy {
# dynamic incus
# }
# '';
# };
# };
};
openssh.enable = true;
tailscale = {
enable = true;
authKeyFile = config.age.secrets.tailscale_auth_key.path;
extraUpFlags = ["--advertise-tags tag:sfos"];
};
};
system.stateVersion = "24.05";
}

60
deploy/ncm-2/nomad.nix Normal file
View file

@ -0,0 +1,60 @@
{...}: {
services.nomad = {
enable = true;
enableDocker = true;
dropPrivileges = false;
settings = {
data_dir = "/var/lib/private/nomad";
server = {
enabled = true;
# Ideally the bootstrap expect value would reflect the number of servers set by `instance_count`, but in the interest
# of not chaning each server's "user data" every time the instance_count changes, we hard code the nomad cluster to
# bootstrap itself whenever 3 nodes are present
bootstrap_expect = 3;
server_join = {
retry_join = [];
};
};
acl = {
enabled = true;
};
client = {
enabled = true;
};
plugin.docker = {
config = {
gc = {
image = true;
image_delay = "3m";
container = true;
dangling_containers = {
enabled = true;
dry_run = false;
period = "5m";
creation_grace = "5m";
};
};
volumes = {
enabled = true;
selinuxlabel = "z";
};
allow_caps = ["chown" "net_raw"];
allow_privileged = true;
};
};
plugin.raw_exec = {
config = {
enabled = true;
};
};
};
};
}

12
deploy/outputs.tf
View file

@ -1,9 +1,5 @@
output "cluster_ips" {
description = "the ipv4 addresses of cluster members"
value = module.compute_provider.ip_addresses
}
# output "cluster_nodes" {
# description = "the nodes in your cluster"
# value = module.compute_provider.nodes
# }
output "cluster_node_details" {
description = "details for each cluser member"
value = module.compute_provider.node_details
}

20
deploy/test.sh Normal file
View file

@ -0,0 +1,20 @@
#!/run/current-system/sw/bin/bash
cwd=$(pwd)/..
tofu output -json | jq '.cluster_node_details.value[].ipv4' | tr -d '"' | while read host; do
echo "Initializing $host"
hostName=$(tofu -chdir="$cwd/deploy" output -json | jq -r ".cluster_node_details.value[] | select(.ipv4 == \"$host\").label")
echo "Setting host name: $hostName"
ssh_public_key=$(ssh-keyscan $host | grep ssh-ed25519 | cut -d " " -f 2,3)
mkdir -p "$hostName/secrets"
cp -rf ../operate/templates/* "$hostName/"
nix-editor "$hostName/default.nix" users.users.sfos.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$hostName/default.nix" users.users.root.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$hostName/default.nix" networking.hostName -i -v "\"$hostName\"\;"
nix-editor "$hostName/default.nix" deployment.targetHost -i -v "\"$host\"\;"
nix-editor "$hostName/secrets/secrets.nix" tailscale_auth_key.publicKeys -i -v "[localMachine \"$ssh_public_key\"]"
# nix-editor "$hostName/nomad.nix" services.nomad.settings.server.server_join.retry_join -i -v "[\"exec=/run/current-system/sw/bin/tailscale status --json | jq '.Peer[] | select(has(\\\"Tags\\\")) | select( [ .Tags[] | contains(\\\"tag:hz-cluster\\\")] | any).TailscaleIPs[0]' | tr '\n' ' '\"]"
nix-editor ../flake.nix "outputs.colmena.$hostName" -i -v "./hosts/$hostName";
# cd "$hostName/secrets"
# echo "$TF_VAR_tailscale_auth_key" | agenix -e tailscale_auth_key -i ~/.ssh/id_rsa_agenix
done

47
flake.nix
View file

@ -57,9 +57,9 @@
inherit (inputs) agenix;
};
};
# sfos-0 = ./hosts/sfos-0;
# sfos-1 = ./hosts/sfos-1;
# sfos-2 = ./hosts/sfos-2;
ncm-0 = ./operate/hosts/ncm-0;
ncm-2 = ./operate/hosts/ncm-2;
ncm-1 = ./operate/hosts/ncm-1;
};
devShells = forEachSystem (system: let
config = self.devShells.${system}.default.config;
@ -117,32 +117,31 @@
ncm-init = {
description = "Initialize cluster members";
exec = ''
#!/run/current-system/sw/bin/bash
cwd=$(pwd)
mkdir -p "$cwd/deploy/hosts"
cd "$cwd/deploy"
source .env
tofu apply -auto-approve
tofu output -json | jq '.cluster_ips.value | join(",")' | tr -d '"' | tr ',' "\n" | while read host; do
cd "$cwd/deploy/hosts"
#!/run/current-system/sw/bin/bash
hosts_dir="./operate/hosts"
mkdir -p $hosts_dir
tofu -chdir="./deploy" output -json | jq '.cluster_node_details.value[].ipv4' | tr -d '"' | while read host; do
echo "Initializing $host"
hostName=$(tofu -chdir="$cwd/deploy/hosts" output -json | hostName=$host jq -r ".cluster_nodes.value[] | select(.ipv4_address == \"$host\").name")
hostName=$(tofu -chdir="./deploy" output -json | jq -r ".cluster_node_details.value[] | select(.ipv4 == \"$host\").label")
host_dir="./operate/hosts/$hostName"
echo "Setting host name: $hostName"
ssh_public_key=$(ssh-keyscan $host | grep ssh-ed25519 | cut -d " " -f 2,3)
mkdir -p "$hostName/secrets"
cp -rf ../tepmlates/* "$hostName/"
nix-editor "$hostName/default.nix" users.users.sfos.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$hostName/default.nix" users.users.root.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$hostName/default.nix" networking.hostName -i -v "\"$hostName\"\;"
nix-editor "$hostName/default.nix" deployment.targetHost -i -v "\"$host\"\;"
nix-editor "$hostName/secrets/secrets.nix" tailscale_auth_key.publicKeys -i -v "[localMachine \"$ssh_public_key\"]"
nix-editor "$hostName/nomad.nix" services.nomad.settings.server.server_join.retry_join -i -v "[\"exec=/run/current-system/sw/bin/tailscale status --json | jq '.Peer[] | select(has(\\\"Tags\\\")) | select( [ .Tags[] | contains(\\\"tag:hz-cluster\\\")] | any).TailscaleIPs[0]' | tr '\n' ' '\"]"
nix-editor ../flake.nix "outputs.colmena.$hostName" -i -v "./hosts/$hostName";
cd "$hostName/secrets"
mkdir -p "$host_dir/secrets"
cp -rf operate/templates/* "$hosts_dir/$hostName/"
nix-editor "$host_dir/default.nix" users.users.sfos.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$host_dir/default.nix" users.users.root.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$host_dir/default.nix" networking.hostName -i -v "\"$hostName\"\;"
nix-editor "$host_dir/default.nix" deployment.targetHost -i -v "\"$host\"\;"
nix-editor "$host_dir/secrets/secrets.nix" tailscale_auth_key.publicKeys -i -v "[\"$TF_VAR_admin_ssh_public_key\" \"$ssh_public_key\"]"
nix-editor "$host_dir/secrets/secrets.nix" linode_cli_pat.publicKeys -i -v "[\"$TF_VAR_admin_ssh_public_key\" \"$ssh_public_key\"]"
# nix-editor "$hostName/nomad.nix" services.nomad.settings.server.server_join.retry_join -i -v "[\"exec=/run/current-system/sw/bin/tailscale status --json | jq '.Peer[] | select(has(\\\"Tags\\\")) | select( [ .Tags[] | contains(\\\"tag:hz-cluster\\\")] | any).TailscaleIPs[0]' | tr '\n' ' '\"]"
nix-editor flake.nix "outputs.colmena.$hostName" -i -v "$host_dir";
cd "$host_dir/secrets"
echo "$TF_VAR_tailscale_auth_key" | agenix -e tailscale_auth_key -i ~/.ssh/id_rsa_agenix
echo "$TF_VAR_nodes_linode_cli_pat" | agenix -e linode_cli_pat -i ~/.ssh/id_rsa_agenix
cd ../../../..
done
cd $cwd
git add deploy/hosts
git add ./operate/hosts
'';
};
};

65
operate/hosts/ncm-0/base-system.nix Normal file
View file

@ -0,0 +1,65 @@
{
lib,
pkgs,
self,
...
}: {
boot = {
initrd.systemd.enable = true;
tmp = {
cleanOnBoot = true;
useTmpfs = true;
};
};
nixpkgs = {
config = {
allowUnfree = true;
};
};
nix = {
package = pkgs.nix;
gc = {
automatic = true;
dates = "weekly";
options = "--delete-older-than 14d";
};
settings = {
trusted-users = [
"@wheel"
];
experimental-features = [
"flakes"
"nix-command"
];
trusted-public-keys = [
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
];
};
};
security.sudo.wheelNeedsPassword = true;
services = {
rsyslogd.enable = true;
openssh = {
enable = true;
settings.PermitRootLogin = "yes";
};
};
security = {
rtkit.enable = true;
};
systemd.services.nix-daemon = {
environment.TMPDIR = "/var/tmp";
};
time.timeZone = "America/New_York";
users.mutableUsers = false;
system.configurationRevision = lib.mkIf (self ? rev) self.rev;
}

126
operate/hosts/ncm-0/default.nix Normal file
View file

@ -0,0 +1,126 @@
{
config,
pkgs,
agenix,
...
}: {
deployment = {
targetHost = "173.255.226.80";
targetPort = 22;
targetUser = "root";
buildOnTarget = false;
};
networking = {
hostName = "ncm-0";
nftables.enable = true;
firewall = {
allowedTCPPorts = [22 80 443];
allowedUDPPorts = [config.services.tailscale.port];
};
};
imports = [
agenix.nixosModules.default
./base-system.nix
./nomad.nix
./hardware-configuration.nix
];
boot = {
supportedFilesystems = ["btrfs" "ntfs" "vfat" "ext4"];
loader.grub = {
enable = true;
};
};
# age.secrets.docker_auth_json = {
# file = ./secrets/docker_ghcr_auth_config_json.age;
# path = "/run/containers/0/auth.json";
# };
age.secrets.tailscale_auth_key = {
file = ./secrets/tailscale_auth_key;
};
age.secrets.linode_cli_pat = {
file = ./secrets/linode_cli_pat;
};
environment.systemPackages = with pkgs; [
nomad_1_9
jq
(writeShellScriptBin "list-cluster-members" ''
#!/usr/bin/env bash
token_file="${config.age.secrets.linode_cli_pat.path}"
if [[ ! -r "$token_file" ]]; then
echo "Token file not found or unreadable: $token_file" >&2
exit 1
fi
export LINODE_CLI_TOKEN="$(cat $token_file)"
${pkgs.linode-cli}/bin/linode-cli linodes list --tags ncm --json | ${pkgs.jq}/bin/jq '.[].ipv4[]' | tr '\n' ' '
'')
];
#
# TEST: authentication to private repo
#
# environment.variables = {
# XDG_RUNTIME_DIR = "/var/run/user/0";
# };
# Define a systemd service to create files in /var/run/user/0
# systemd.services."create-incus-ghcr-auth-file" = {
# description = "Create ghcr.io auth file for incus to retrieve private images";
# after = ["default.target"];
# wantedBy = ["default.target"];
# serviceConfig = {
# ExecStart = ''${pkgs.bashInteractive}/bin/bash -c 'echo {\\"auths\\":{ \\"ghcr.io\\": {\\"auth\\": \\"YWNhbG9pYXJvOiBnaHBfVnN4dzc4YXZ6N0lBeXQyejVrcGpZQTJXV21XbUZNMzlZdkMxIA==\\"}}} > /var/run/user/0/containers/auth.json' '';
# ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p /var/run/user/0/containers";
# Restart = "no";
# };
# };
#
# /TEST: authentication to private repo
#
users.users = {
root = {
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1"];
};
sfos = {
isNormalUser = true;
extraGroups = ["wheel"];
# TODO Initial password and authorized keys need to come from .env, perhaps using nix-edit
initialPassword = "Test.123";
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1"];
};
};
services = {
# TODO caddy needs a plugin that watches the nomad node list for specific tags that cause them to be reverse proxied
# from this host system
caddy = {
enable = true;
# package = pkgs.caddy.withPlugins {
# plugins = ["github.com/acaloiaro/caddy-incus-upstreams@v0.0.0-20241228235755-786cd3b6a9dc"];
# hash = "sha256-yjqXgo7WiJPBVmN3+O8Wq9Nxd1xU8KrXW7Hv7Elcf7I=";
# };
# virtualHosts = {
# "https://adrianofyi.incus.adriano.fyi:443" = {
# extraConfig = ''
# reverse_proxy {
# dynamic incus
# }
# '';
# };
# };
};
openssh.enable = true;
tailscale = {
enable = true;
authKeyFile = config.age.secrets.tailscale_auth_key.path;
extraUpFlags = [];
};
};
system.stateVersion = "24.05";
}

49
operate/hosts/ncm-0/hardware-configuration.nix Normal file
View file

@ -0,0 +1,49 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{
lib,
modulesPath,
...
}: {
imports = [
(modulesPath + "/profiles/qemu-guest.nix")
];
boot = {
initrd.availableKernelModules = ["virtio_pci" "virtio_scsi" "ahci" "sd_mod"];
initrd.kernelModules = [];
kernelModules = [];
extraModulePackages = [];
kernelParams = ["console=ttyS0,19200n8"];
loader = {
grub = {
forceInstall = true;
extraConfig = ''
serial --speed=19200 --unit=0 --word=8 --parity=0 --stop=1;
terminal_input serial;
terminal_output serial
'';
device = "nodev";
};
timeout = 10;
};
};
fileSystems."/" = {
device = "/dev/sda";
fsType = "ext4";
};
swapDevices = [
{device = "/dev/sdb";}
];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp0s4.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
}

60
operate/hosts/ncm-0/nomad.nix Normal file
View file

@ -0,0 +1,60 @@
{...}: {
services.nomad = {
enable = true;
enableDocker = true;
dropPrivileges = false;
settings = {
data_dir = "/var/lib/private/nomad";
server = {
enabled = true;
# Ideally the bootstrap expect value would reflect the number of servers set by `instance_count`, but in the interest
# of not chaning each server's "user data" every time the instance_count changes, we hard code the nomad cluster to
# bootstrap itself whenever 3 nodes are present
bootstrap_expect = 3;
server_join = {
retry_join = ["exec=/run/current-system/sw/bin/list-cluster-members"];
};
};
acl = {
enabled = true;
};
client = {
enabled = true;
};
plugin.docker = {
config = {
gc = {
image = true;
image_delay = "3m";
container = true;
dangling_containers = {
enabled = true;
dry_run = false;
period = "5m";
creation_grace = "5m";
};
};
volumes = {
enabled = true;
selinuxlabel = "z";
};
allow_caps = ["chown" "net_raw"];
allow_privileged = true;
};
};
plugin.raw_exec = {
config = {
enabled = true;
};
};
};
};
}

BIN
operate/hosts/ncm-0/secrets/linode_cli_pat Normal file
View file

Binary file not shown.

6
operate/hosts/ncm-0/secrets/secrets.nix Normal file
View file

@ -0,0 +1,6 @@
# Use ssh-keyscan <SERVER> to fetch the below keys for target systems (SSH server must be running on system)
let
in {
tailscale_auth_key.publicKeys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILLnAerH/zLDsefU8J1Dxv/Q++LvBBYM5Zk0cuFJ4jBg"];
linode_cli_pat.publicKeys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILLnAerH/zLDsefU8J1Dxv/Q++LvBBYM5Zk0cuFJ4jBg"];
}

BIN
operate/hosts/ncm-0/secrets/tailscale_auth_key Normal file
View file

Binary file not shown.

65
operate/hosts/ncm-1/base-system.nix Normal file
View file

@ -0,0 +1,65 @@
{
lib,
pkgs,
self,
...
}: {
boot = {
initrd.systemd.enable = true;
tmp = {
cleanOnBoot = true;
useTmpfs = true;
};
};
nixpkgs = {
config = {
allowUnfree = true;
};
};
nix = {
package = pkgs.nix;
gc = {
automatic = true;
dates = "weekly";
options = "--delete-older-than 14d";
};
settings = {
trusted-users = [
"@wheel"
];
experimental-features = [
"flakes"
"nix-command"
];
trusted-public-keys = [
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
];
};
};
security.sudo.wheelNeedsPassword = true;
services = {
rsyslogd.enable = true;
openssh = {
enable = true;
settings.PermitRootLogin = "yes";
};
};
security = {
rtkit.enable = true;
};
systemd.services.nix-daemon = {
environment.TMPDIR = "/var/tmp";
};
time.timeZone = "America/New_York";
users.mutableUsers = false;
system.configurationRevision = lib.mkIf (self ? rev) self.rev;
}

126
operate/hosts/ncm-1/default.nix Normal file
View file

@ -0,0 +1,126 @@
{
config,
pkgs,
agenix,
...
}: {
deployment = {
targetHost = "173.255.226.115";
targetPort = 22;
targetUser = "root";
buildOnTarget = false;
};
networking = {
hostName = "ncm-1";
nftables.enable = true;
firewall = {
allowedTCPPorts = [22 80 443];
allowedUDPPorts = [config.services.tailscale.port];
};
};
imports = [
agenix.nixosModules.default
./base-system.nix
./nomad.nix
./hardware-configuration.nix
];
boot = {
supportedFilesystems = ["btrfs" "ntfs" "vfat" "ext4"];
loader.grub = {
enable = true;
};
};
# age.secrets.docker_auth_json = {
# file = ./secrets/docker_ghcr_auth_config_json.age;
# path = "/run/containers/0/auth.json";
# };
age.secrets.tailscale_auth_key = {
file = ./secrets/tailscale_auth_key;
};
age.secrets.linode_cli_pat = {
file = ./secrets/linode_cli_pat;
};
environment.systemPackages = with pkgs; [
nomad_1_9
jq
(writeShellScriptBin "list-cluster-members" ''
#!/usr/bin/env bash
token_file="${config.age.secrets.linode_cli_pat.path}"
if [[ ! -r "$token_file" ]]; then
echo "Token file not found or unreadable: $token_file" >&2
exit 1
fi
export LINODE_CLI_TOKEN="$(cat $token_file)"
${pkgs.linode-cli}/bin/linode-cli linodes list --tags ncm --json | ${pkgs.jq}/bin/jq '.[].ipv4[]' | tr '\n' ' '
'')
];
#
# TEST: authentication to private repo
#
# environment.variables = {
# XDG_RUNTIME_DIR = "/var/run/user/0";
# };
# Define a systemd service to create files in /var/run/user/0
# systemd.services."create-incus-ghcr-auth-file" = {
# description = "Create ghcr.io auth file for incus to retrieve private images";
# after = ["default.target"];
# wantedBy = ["default.target"];
# serviceConfig = {
# ExecStart = ''${pkgs.bashInteractive}/bin/bash -c 'echo {\\"auths\\":{ \\"ghcr.io\\": {\\"auth\\": \\"YWNhbG9pYXJvOiBnaHBfVnN4dzc4YXZ6N0lBeXQyejVrcGpZQTJXV21XbUZNMzlZdkMxIA==\\"}}} > /var/run/user/0/containers/auth.json' '';
# ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p /var/run/user/0/containers";
# Restart = "no";
# };
# };
#
# /TEST: authentication to private repo
#
users.users = {
root = {
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1"];
};
sfos = {
isNormalUser = true;
extraGroups = ["wheel"];
# TODO Initial password and authorized keys need to come from .env, perhaps using nix-edit
initialPassword = "Test.123";
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1"];
};
};
services = {
# TODO caddy needs a plugin that watches the nomad node list for specific tags that cause them to be reverse proxied
# from this host system
caddy = {
enable = true;
# package = pkgs.caddy.withPlugins {
# plugins = ["github.com/acaloiaro/caddy-incus-upstreams@v0.0.0-20241228235755-786cd3b6a9dc"];
# hash = "sha256-yjqXgo7WiJPBVmN3+O8Wq9Nxd1xU8KrXW7Hv7Elcf7I=";
# };
# virtualHosts = {
# "https://adrianofyi.incus.adriano.fyi:443" = {
# extraConfig = ''
# reverse_proxy {
# dynamic incus
# }
# '';
# };
# };
};
openssh.enable = true;
tailscale = {
enable = true;
authKeyFile = config.age.secrets.tailscale_auth_key.path;
extraUpFlags = [];
};
};
system.stateVersion = "24.05";
}

49
operate/hosts/ncm-1/hardware-configuration.nix Normal file
View file

@ -0,0 +1,49 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{
lib,
modulesPath,
...
}: {
imports = [
(modulesPath + "/profiles/qemu-guest.nix")
];
boot = {
initrd.availableKernelModules = ["virtio_pci" "virtio_scsi" "ahci" "sd_mod"];
initrd.kernelModules = [];
kernelModules = [];
extraModulePackages = [];
kernelParams = ["console=ttyS0,19200n8"];
loader = {
grub = {
forceInstall = true;
extraConfig = ''
serial --speed=19200 --unit=0 --word=8 --parity=0 --stop=1;
terminal_input serial;
terminal_output serial
'';
device = "nodev";
};
timeout = 10;
};
};
fileSystems."/" = {
device = "/dev/sda";
fsType = "ext4";
};
swapDevices = [
{device = "/dev/sdb";}
];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp0s4.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
}

60
operate/hosts/ncm-1/nomad.nix Normal file
View file

@ -0,0 +1,60 @@
{...}: {
services.nomad = {
enable = true;
enableDocker = true;
dropPrivileges = false;
settings = {
data_dir = "/var/lib/private/nomad";
server = {
enabled = true;
# Ideally the bootstrap expect value would reflect the number of servers set by `instance_count`, but in the interest
# of not chaning each server's "user data" every time the instance_count changes, we hard code the nomad cluster to
# bootstrap itself whenever 3 nodes are present
bootstrap_expect = 3;
server_join = {
retry_join = ["exec=/run/current-system/sw/bin/list-cluster-members"];
};
};
acl = {
enabled = true;
};
client = {
enabled = true;
};
plugin.docker = {
config = {
gc = {
image = true;
image_delay = "3m";
container = true;
dangling_containers = {
enabled = true;
dry_run = false;
period = "5m";
creation_grace = "5m";
};
};
volumes = {
enabled = true;
selinuxlabel = "z";
};
allow_caps = ["chown" "net_raw"];
allow_privileged = true;
};
};
plugin.raw_exec = {
config = {
enabled = true;
};
};
};
};
}

15
operate/hosts/ncm-1/secrets/linode_cli_pat Normal file
View file

@ -0,0 +1,15 @@
age-encryption.org/v1
-> ssh-rsa QGRkYA
rW2uHKgdVwwWwEy9wjp+NAreiE9xMYmeV2bwrk2rwHwk/T4OxhUn7Cnr8vdMEZsC
QCTjEMMHdBnZIDw0aoPifUW0uKoaMcAnQpG80DHZcVyiPQVWUqG3hEGkvIaw8xiI
sUajyey4tEtzOWgZGpyTPsdReijJz0gdQPhCowraDWtHgzXs/Yu5Fq1A5CCI/+ci
oBZ5k/l524gtByxzIMs1C23GEzmepyGRWnKkwufj6XWShb2EYHKY2X/rm5fubegV
3hruFNOVJXvXi/p6IzCxwJztqeLX7UdFFw3TrqAeWbeTNVAFZrLymgfOG97luew9
HCqYPNuUSgLTwRCIvGmsrtaaNNzxWUTn4b0QRuTvQ6XfpvYX+Q5ZVN6oEyRm0f5D
a52SwTZ0sOw8F7SiZHVx0NDXvpb5Kz1Uw9qMGLtU+xTxT2pQoFWmqpko2DYGYNga
fQRWP2tB34jyi/aZspRTcki/yjSFWu4ElTcv+NxwU4aetzYOApAuc/0l0r2cyaUf
-> ssh-ed25519 xB9J8g 2avjqyC/of+Rj79qsq747+MzcfU/ffxcXh9eu73pcSw
bUsRMDpYSW8hJqQq35Qw+50JC+wxK94ZpFuoAFxomog
--- WcyXRrwHBwSMdnMeLPbeBGuLGrzyRgUDaEOLsX48+sQ
¯MÆn©ÅzùPýïÝ!‹™¶Mõ–Q‘âûbÔñc6²¹QLm<4C>DU–>®_2†ëðùêÜ­ÝW[IŸÔjûû~$¼*jP‘hlP¤’ôVð2>wr/I¨•Á\ §.Ed ã^

6
operate/hosts/ncm-1/secrets/secrets.nix Normal file
View file

@ -0,0 +1,6 @@
# Use ssh-keyscan <SERVER> to fetch the below keys for target systems (SSH server must be running on system)
let
in {
tailscale_auth_key.publicKeys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILoJx18Mkb/nSDiU1GWLHJ8dhAdnvzh2JboKwmALZ40K"];
linode_cli_pat.publicKeys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILoJx18Mkb/nSDiU1GWLHJ8dhAdnvzh2JboKwmALZ40K"];
}

15
operate/hosts/ncm-1/secrets/tailscale_auth_key Normal file
View file

@ -0,0 +1,15 @@
age-encryption.org/v1
-> ssh-rsa QGRkYA
WFw/YdgBDiqGPi0gnHtDVWJGO//BI26ddSQlPOU6RzloymrYRQCzTpsKz7Goi271
lnXR8/7l/e5AFwe8Bm8lWiglfy3cvO07cFbRQX3MGCfzW0JoHqUgNPjKefqIrQpn
e8n2etFY2t9cC1XsIzr58dMCQH4J9uoBr9nf8lR1sJxyo5+2dcqB0/Fi0zZxh+xz
UQQQBUnJ+ucHJehQUvWRqgjqdClK1HQTVi6JJ5cvUqxkLLzYCT/zD9EODghj9t8i
+BJ9GzpVS42kv0Mw9aq2e85/NTiP4SMCNpuFSvVRmQLLrKggKW6NHjY7+dVu82wb
w2JhAUpjTjGoEmevKSXgxc5sfcQpY8tnWr+ZL3xoZS1LdqNrBhD0+kdtd8m/u3V5
GahlKQsA0LWLIlxs/sY0+Ns78LurktUvL5cGskgsrPoS7/ElFtcfN8qD54xBmQ8e
IZ/ylMidTCO2K0EWnIZWpXEg8rOs45rVrd8osxWvl3Va9K5QulFJFNQ0JgzVV0D+
-> ssh-ed25519 xB9J8g IguiJ0DskNsMbf/P0KakS36p9Z71pdRMrlj9lvfTjB0
Q3oGTJzBlYm4We1nB6ReCgMtNJ8gGZBnKPQUQUFuNOg
--- 6V3Qupw69F1YtUjPxQTvYdcaU/dWzUbT6XI+Ep5DcQI
ÂPZ¼Ä[}÷Ío@%¾Ûì{ÕOòåì€×Èñ§§<.­(صWN(|E¬Ü>€çÅ8X0Ê¿oÿqî‚5cóM-sA¾y'¾£œ<N6T gè?"=4Ž4ˆ¾ÒÐp:}©§ñR

65
operate/hosts/ncm-2/base-system.nix Normal file
View file

@ -0,0 +1,65 @@
{
lib,
pkgs,
self,
...
}: {
boot = {
initrd.systemd.enable = true;
tmp = {
cleanOnBoot = true;
useTmpfs = true;
};
};
nixpkgs = {
config = {
allowUnfree = true;
};
};
nix = {
package = pkgs.nix;
gc = {
automatic = true;
dates = "weekly";
options = "--delete-older-than 14d";
};
settings = {
trusted-users = [
"@wheel"
];
experimental-features = [
"flakes"
"nix-command"
];
trusted-public-keys = [
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
];
};
};
security.sudo.wheelNeedsPassword = true;
services = {
rsyslogd.enable = true;
openssh = {
enable = true;
settings.PermitRootLogin = "yes";
};
};
security = {
rtkit.enable = true;
};
systemd.services.nix-daemon = {
environment.TMPDIR = "/var/tmp";
};
time.timeZone = "America/New_York";
users.mutableUsers = false;
system.configurationRevision = lib.mkIf (self ? rev) self.rev;
}

126
operate/hosts/ncm-2/default.nix Normal file
View file

@ -0,0 +1,126 @@
{
config,
pkgs,
agenix,
...
}: {
deployment = {
targetHost = "173.255.226.87";
targetPort = 22;
targetUser = "root";
buildOnTarget = false;
};
networking = {
hostName = "ncm-2";
nftables.enable = true;
firewall = {
allowedTCPPorts = [22 80 443];
allowedUDPPorts = [config.services.tailscale.port];
};
};
imports = [
agenix.nixosModules.default
./base-system.nix
./nomad.nix
./hardware-configuration.nix
];
boot = {
supportedFilesystems = ["btrfs" "ntfs" "vfat" "ext4"];
loader.grub = {
enable = true;
};
};
# age.secrets.docker_auth_json = {
# file = ./secrets/docker_ghcr_auth_config_json.age;
# path = "/run/containers/0/auth.json";
# };
age.secrets.tailscale_auth_key = {
file = ./secrets/tailscale_auth_key;
};
age.secrets.linode_cli_pat = {
file = ./secrets/linode_cli_pat;
};
environment.systemPackages = with pkgs; [
nomad_1_9
jq
(writeShellScriptBin "list-cluster-members" ''
#!/usr/bin/env bash
token_file="${config.age.secrets.linode_cli_pat.path}"
if [[ ! -r "$token_file" ]]; then
echo "Token file not found or unreadable: $token_file" >&2
exit 1
fi
export LINODE_CLI_TOKEN="$(cat $token_file)"
${pkgs.linode-cli}/bin/linode-cli linodes list --tags ncm --json | ${pkgs.jq}/bin/jq '.[].ipv4[]' | tr '\n' ' '
'')
];
#
# TEST: authentication to private repo
#
# environment.variables = {
# XDG_RUNTIME_DIR = "/var/run/user/0";
# };
# Define a systemd service to create files in /var/run/user/0
# systemd.services."create-incus-ghcr-auth-file" = {
# description = "Create ghcr.io auth file for incus to retrieve private images";
# after = ["default.target"];
# wantedBy = ["default.target"];
# serviceConfig = {
# ExecStart = ''${pkgs.bashInteractive}/bin/bash -c 'echo {\\"auths\\":{ \\"ghcr.io\\": {\\"auth\\": \\"YWNhbG9pYXJvOiBnaHBfVnN4dzc4YXZ6N0lBeXQyejVrcGpZQTJXV21XbUZNMzlZdkMxIA==\\"}}} > /var/run/user/0/containers/auth.json' '';
# ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p /var/run/user/0/containers";
# Restart = "no";
# };
# };
#
# /TEST: authentication to private repo
#
users.users = {
root = {
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1"];
};
sfos = {
isNormalUser = true;
extraGroups = ["wheel"];
# TODO Initial password and authorized keys need to come from .env, perhaps using nix-edit
initialPassword = "Test.123";
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1"];
};
};
services = {
# TODO caddy needs a plugin that watches the nomad node list for specific tags that cause them to be reverse proxied
# from this host system
caddy = {
enable = true;
# package = pkgs.caddy.withPlugins {
# plugins = ["github.com/acaloiaro/caddy-incus-upstreams@v0.0.0-20241228235755-786cd3b6a9dc"];
# hash = "sha256-yjqXgo7WiJPBVmN3+O8Wq9Nxd1xU8KrXW7Hv7Elcf7I=";
# };
# virtualHosts = {
# "https://adrianofyi.incus.adriano.fyi:443" = {
# extraConfig = ''
# reverse_proxy {
# dynamic incus
# }
# '';
# };
# };
};
openssh.enable = true;
tailscale = {
enable = true;
authKeyFile = config.age.secrets.tailscale_auth_key.path;
extraUpFlags = [];
};
};
system.stateVersion = "24.05";
}

49
operate/hosts/ncm-2/hardware-configuration.nix Normal file
View file

@ -0,0 +1,49 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{
lib,
modulesPath,
...
}: {
imports = [
(modulesPath + "/profiles/qemu-guest.nix")
];
boot = {
initrd.availableKernelModules = ["virtio_pci" "virtio_scsi" "ahci" "sd_mod"];
initrd.kernelModules = [];
kernelModules = [];
extraModulePackages = [];
kernelParams = ["console=ttyS0,19200n8"];
loader = {
grub = {
forceInstall = true;
extraConfig = ''
serial --speed=19200 --unit=0 --word=8 --parity=0 --stop=1;
terminal_input serial;
terminal_output serial
'';
device = "nodev";
};
timeout = 10;
};
};
fileSystems."/" = {
device = "/dev/sda";
fsType = "ext4";
};
swapDevices = [
{device = "/dev/sdb";}
];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp0s4.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
}

60
operate/hosts/ncm-2/nomad.nix Normal file
View file

@ -0,0 +1,60 @@
{...}: {
services.nomad = {
enable = true;
enableDocker = true;
dropPrivileges = false;
settings = {
data_dir = "/var/lib/private/nomad";
server = {
enabled = true;
# Ideally the bootstrap expect value would reflect the number of servers set by `instance_count`, but in the interest
# of not chaning each server's "user data" every time the instance_count changes, we hard code the nomad cluster to
# bootstrap itself whenever 3 nodes are present
bootstrap_expect = 3;
server_join = {
retry_join = ["exec=/run/current-system/sw/bin/list-cluster-members"];
};
};
acl = {
enabled = true;
};
client = {
enabled = true;
};
plugin.docker = {
config = {
gc = {
image = true;
image_delay = "3m";
container = true;
dangling_containers = {
enabled = true;
dry_run = false;
period = "5m";
creation_grace = "5m";
};
};
volumes = {
enabled = true;
selinuxlabel = "z";
};
allow_caps = ["chown" "net_raw"];
allow_privileged = true;
};
};
plugin.raw_exec = {
config = {
enabled = true;
};
};
};
};
}

15
operate/hosts/ncm-2/secrets/linode_cli_pat Normal file
View file

@ -0,0 +1,15 @@
age-encryption.org/v1
-> ssh-rsa QGRkYA
BNxv6kF4vaTVLtnX56x/QbYzLNTs1X+UPNElRBN4NE6bX7go/FM1u1498UB9+4tI
o4sVZKrs3t2D8c49fCsv1VgP8PSed4Vv3NyKKeUZP2Rj42VZ5LUXxxIwVnQbEbT5
3yS8ulH95/ELb/tx2UcYN+raZ/p7Y+1S6FY8azl9RwOBBzOBYIpehqKEVtz0CoUW
/+o8aq3KwH4e5ZnyBKhjFcaaEqYYXr3QrULRqVAk4bKVKCzv+ateR9q5z1AZkD2h
CQADo6HkblI2V3Jk6LiO7j/hf1VDm6D6DKJck8aPZnd74j+wqdWPyQg3YzYIFatm
4lHaluwAkwolkrQkuPuAbG1mgXD8aY7oi9tG9fH57iNIAEXxMk7+3pIb/iJfnUo8
ImcxbiVBT7CyOGhO8gYspcykDBvem9Pt0HxSbwAFEOoA90N+UvXl1gdU3FJ/WNB7
ECZyaEhS2GCmEhdO/Zk7j4V4j2qknKIu3XMrVGyuuSjDAMkheWoQIJc8f6hC7msk
-> ssh-ed25519 swqv7w A3atuKJxY2RdbLAAxdS9lnnBMpryn1cwBut17xA++RE
z6KjFGnjUZdP0UKPILHYGuQAe/P3y9OvaXNup953NJI
--- L3rbEjxs57P3b0EvzuD6tR5Vvc53Xy10SzTRVbbgv/8
· Cœ„,2_6ÈÀÿ¤˃%<25>‰úŸŽûB´Zß<5A>B¼¢6©2úÔ´Bõ;|.üÜH}Ý„Ü] Fcs÷èætdNGö2hp<68>9eü¶Ç øóºGJÊðŽ=×FṈ(Á#W

6
operate/hosts/ncm-2/secrets/secrets.nix Normal file
View file

@ -0,0 +1,6 @@
# Use ssh-keyscan <SERVER> to fetch the below keys for target systems (SSH server must be running on system)
let
in {
tailscale_auth_key.publicKeys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP2kHzhXZuo/li6kIKemqvDDWkY+1RjVyvtEehAw2yud"];
linode_cli_pat.publicKeys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx/tvR11RUMbYaauuHPxR3j79vm1Hxg+Y3LXR+rLgZ7N/2ulvIzLLZgVuIUbmJHDqkp9Au5zZPMZjsKbaqQ8V75wSEcyaSMbaxC9PaNDCwEHMJWaz8XPQe5IjVRE5O+4sTyh7ZBx+NMQmPcQbrNInoKuy4EjFmwTW/t0xYo/sCrC0NX0cCyeBwii2JdFXytXfxF+RMzGNXw1xfcOFJe6F7JdS/Cpf/0fe+VmNg8d0nic4Obcb/djYsRLAAC6Cvb+4i3EBZWl+9Ih9hId8bFCRKhI6TmGT2z4YUTa+v+3j/JZUh1gD5n4vRGf8QjLj9N6DrBnKcbywwqyqnLhTLgBBN35rcLdU1k3n0NorRRDCU0Lg/ejsFe3oi2FmOwNmmd8zqBNZHjJTi5Wy63EXMFwHltEY2M+hAhgWsQ5U4zuVPgv1HfD6LYPodRwhdZivwTNr2IClAiVVxR//O0WtrRXrrEM5uudj+Y30/ah7bn9Mje86UV0TqfS2tdjtMkySHL+M= adriano@z1" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP2kHzhXZuo/li6kIKemqvDDWkY+1RjVyvtEehAw2yud"];
}

15
operate/hosts/ncm-2/secrets/tailscale_auth_key Normal file
View file

@ -0,0 +1,15 @@
age-encryption.org/v1
-> ssh-rsa QGRkYA
r3NaZFH8FGrpQrbRX0ejy+OtuncaYLvFB5JsX8ZxVGy2nl/jNjBnpPMQZeyDbfQU
Hs+IbZt5Kp6i8ucOAgnr6P3VXywf7Gm0DS9v00gqkRX45yIfPI0CR+kKqfRR7RWg
U6Jfz1l5Jr4CEY5ACrPBk9nhr99sOhJFyqsJUnJudqVSc6WDdRh1Wwrb8myzMcpx
vCyQDFhIjJAFNl2Aohnzj9ViqFnixxUy6ArOwQlipWkULJjVxhIF6h5h/jkSrOdF
dSbKsz2Ij4CMrn14QgPYX5/ODkR1ejLPfI95xtsjeJeIV8dwkW5w+ePfCdeJLNib
/yrzYs0HhBEK2vFBQYnLh8vxy6XGutwqEu4Lr8bGbcYWFABTuOZrwdEegZ87EbDi
b1WNWObLpVDtAQsHwpdEEJVk56ACmWHvB/7VsHrkT5XJklYSl0JAjq36CZF1HmD7
W2bgEKG9+yjLjoOOCHSa6HF2rSP0qKQ68uQav1us1sqNukmGCPPWpsiw/WBcGN1Q
-> ssh-ed25519 swqv7w nL/N8NzrXky1vyus6bJU/74f3eNONdpcxg8DN8igyFE
9reaz1YtyMeFTZVbqi9BBF/Ww7Vss2rB8ZznUPrHeTA
--- SikS98A5Xm6QoVdXGbWDKtTjVAV5RqdPXTrJC2RkgP8
V‰€EÛS%UÇàô0$ <>nÜ‘rP`cau<61>~IŒ¦w£S4òî×ÇÓÍäÜÍÂœ=ÇÖÈÙÒvZÒ{›µ¹ÛÌå'}Ã*:Í1uu`Ð ƒó)øVh}‹Ñ%ñ/Á

30
operate/templates/default.nix
View file

@ -1,6 +1,5 @@
{
config,
inputs,
pkgs,
agenix,
...
@ -9,7 +8,7 @@
targetHost = "<PLACEHOLDER>";
targetPort = 22;
targetUser = "root";
buildOnTarget = true;
buildOnTarget = false;
};
networking = {
@ -22,14 +21,9 @@
};
imports = [
agenix.nixosModules.default
inputs.nixos-facter-modules.nixosModules.facter
{config.facter.reportPath = ./facter.json;}
inputs.disko.nixosModules.disko
./hardware-configuration.nix
./disko-config.nix
./base-system.nix
./nix.nix
./nomad.nix
./hardware-configuration.nix
];
boot = {
@ -48,9 +42,23 @@
file = ./secrets/tailscale_auth_key;
};
age.secrets.linode_cli_pat = {
file = ./secrets/linode_cli_pat;
};
environment.systemPackages = with pkgs; [
nomad_1_9
jq
(writeShellScriptBin "list-cluster-members" ''
#!/usr/bin/env bash
token_file="${config.age.secrets.linode_cli_pat.path}"
if [[ ! -r "$token_file" ]]; then
echo "Token file not found or unreadable: $token_file" >&2
exit 1
fi
export LINODE_CLI_TOKEN="$(cat $token_file)"
${pkgs.linode-cli}/bin/linode-cli linodes list --tags ncm --json | ${pkgs.jq}/bin/jq '.[].ipv4[]' | tr '\n' ' '
'')
];
#
@ -77,14 +85,14 @@
users.users = {
root = {
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
openssh.authorizedKeys.keys = [];
};
sfos = {
isNormalUser = true;
extraGroups = ["wheel"];
# TODO Initial password and authorized keys need to come from .env, perhaps using nix-edit
initialPassword = "Test.123";
openssh.authorizedKeys.keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1LwyUmY8yaaIfPKn9aUIsbm8NkcLvx8MOILtKubMxOvnJ+ZkOQnqve/KE+VNdvOzlZgnnLA24ZAeM5fD8n/WFVjDRsKqXVAfZOIygm2/P1RzEK5+AoVOeIC25DhizNGJ0pE8F4aSVTmTtOq5kOf1bTSuVhv3p/k6ZusrzBI2HOEOUg/sfs3Q1L7wHDHTA5qxqYACLebGocq0KqWPW4GTJ67XEMiNIENBh4EEEDTaeQZjRomeeR0ssDlrNAabf+vp+dxEtyHXS9dPznCFUIh7KyCx1oKLBl/O3B2NuVycXdo2yGpPGF6iKC6HW6lBHkYWfmgunQ4NOZWpbFFF0nT7K/kbFjmQKn3h7xuH3wXqs+iGXlDCQ1c/7YKarrD/JOsyWN/qHj9nto5QE40GZZRqhO1i16jCgMTyk0VLwZ5Eq6+zAKBKBQ2t/aFov4i05LuM3geg3LO4BoyQnP/ikuDb4ENRb1+wlJp9kCk2YKZeLwcgBXYg9xkXpX5ZnQl9E26s= adriano@zenity"];
openssh.authorizedKeys.keys = [];
};
};
@ -111,7 +119,7 @@
tailscale = {
enable = true;
authKeyFile = config.age.secrets.tailscale_auth_key.path;
extraUpFlags = ["--advertise-tags tag:sfos"];
extraUpFlags = ["--advertise-tags 'tag:ncm'"];
};
};
system.stateVersion = "24.05";

49
operate/templates/hardware-configuration.nix Normal file
View file

@ -0,0 +1,49 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{
lib,
modulesPath,
...
}: {
imports = [
(modulesPath + "/profiles/qemu-guest.nix")
];
boot = {
initrd.availableKernelModules = ["virtio_pci" "virtio_scsi" "ahci" "sd_mod"];
initrd.kernelModules = [];
kernelModules = [];
extraModulePackages = [];
kernelParams = ["console=ttyS0,19200n8"];
loader = {
grub = {
forceInstall = true;
extraConfig = ''
serial --speed=19200 --unit=0 --word=8 --parity=0 --stop=1;
terminal_input serial;
terminal_output serial
'';
device = "nodev";
};
timeout = 10;
};
};
fileSystems."/" = {
device = "/dev/sda";
fsType = "ext4";
};
swapDevices = [
{device = "/dev/sdb";}
];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp0s4.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
}

2
operate/templates/nomad.nix
View file

@ -13,7 +13,7 @@
bootstrap_expect = 3;
server_join = {
retry_join = [];
retry_join = ["exec=/run/current-system/sw/bin/list-cluster-members"];
};
};

6
operate/templates/secrets/secrets.nix Normal file
View file

@ -0,0 +1,6 @@
# Use ssh-keyscan <SERVER> to fetch the below keys for target systems (SSH server must be running on system)
let
in {
tailscale_auth_key.publicKeys = [];
linode_cli_pat.publicKeys = [];
}

1
terraform.tfstate Normal file
View file

@ -0,0 +1 @@
{"version":4,"terraform_version":"1.9.0","serial":1,"lineage":"62cca95b-cc45-f212-ab57-48d59d0ae763","outputs":{},"resources":[],"check_results":null}

23
test.sh Executable file
View file

@ -0,0 +1,23 @@
#!/run/current-system/sw/bin/bash
hosts_dir="./operate/hosts"
mkdir -p $hosts_dir
tofu -chdir="./deploy" output -json | jq '.cluster_node_details.value[].ipv4' | tr -d '"' | while read host; do
echo "Initializing $host"
hostName=$(tofu -chdir="./deploy" output -json | jq -r ".cluster_node_details.value[] | select(.ipv4 == \"$host\").label")
host_dir="./operate/hosts/$hostName"
echo "Setting host name: $hostName"
ssh_public_key=$(ssh-keyscan $host | grep ssh-ed25519 | cut -d " " -f 2,3)
mkdir -p "$host_dir/secrets"
cp -rf operate/templates/* "$hosts_dir/$hostName/"
nix-editor "$host_dir/default.nix" users.users.sfos.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$host_dir/default.nix" users.users.root.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$host_dir/default.nix" networking.hostName -i -v "\"$hostName\"\;"
nix-editor "$host_dir/default.nix" deployment.targetHost -i -v "\"$host\"\;"
nix-editor "$host_dir/secrets/secrets.nix" tailscale_auth_key.publicKeys -i -v "[\"$TF_VAR_admin_ssh_public_key\" \"$ssh_public_key\"]"
# nix-editor "$hostName/nomad.nix" services.nomad.settings.server.server_join.retry_join -i -v "[\"exec=/run/current-system/sw/bin/tailscale status --json | jq '.Peer[] | select(has(\\\"Tags\\\")) | select( [ .Tags[] | contains(\\\"tag:hz-cluster\\\")] | any).TailscaleIPs[0]' | tr '\n' ' '\"]"
nix-editor flake.nix "outputs.colmena.$hostName" -i -v "$host_dir";
cd "$host_dir/secrets"
echo "$TF_VAR_tailscale_auth_key" | agenix -e tailscale_auth_key -i ~/.ssh/id_rsa_agenix
cd ../../../..
done