me/ncm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: ncm-prepare refactor

Browse source
This commit is contained in:
Adriano Caloiaro 2026-07-09 03:22:50 -06:00
parent e61b22f55a
commit fa21dd0758
No known key found for this signature in database
1 changed files with 101 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

167
flake.nix
View file

@ -166,6 +166,7 @@
linode-cli
opentofu
pre-commit
ssh-to-age
];
enterShell =
@ -199,88 +200,122 @@
'';
};
ncm-prepare = {
description = "Prepares ./operate and ./deploy changes to be deployed with colmena";
exec = ''
#!/run/current-system/sw/bin/bash
hosts_dir="./operate/hosts"
mkdir -p $hosts_dir
tofu -chdir="./deploy" output -json | jq '.cluster_node_details.value[].ipv4' | tr -d '"' | while read host; do
echo "Initializing $host"
hostName=$(tofu -chdir="./deploy" output -json | jq -r ".cluster_node_details.value[] | select(.ipv4 == \"$host\").label")
private_ip=$(tofu -chdir="./deploy" output -json | jq -r ".cluster_node_details.value[] | select(.ipv4 == \"$host\").private_ipv4")
host_dir="./operate/hosts/$hostName"
echo "Setting host name: $hostName"
ssh_public_key=$(ssh-keyscan $host | grep ssh-ed25519 | cut -d " " -f 2,3)
cp -rf operate/templates/* "$hosts_dir/$hostName/"
nix-editor "$host_dir/default.nix" users.users.sfos.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$host_dir/default.nix" users.users.root.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$host_dir/default.nix" networking.hostName -i -v "\"$hostName\""
nix-editor "$host_dir/default.nix" deployment.targetHost -i -v "\"$host\""
nix-editor flake.nix "outputs.colmena.$hostName" -i -v "$host_dir"
cd ../../../..
done
git add ./operate/hosts
'';
};
ncm-add-nodes = {
description = "Add new nodes without touching existing ones (for blue/green deployments)";
description = "Prepare cluster nodes for colmena: SSH keys, agenix-rekey secrets, and host configs. Skips existing nodes by default; pass --overwrite [node1,node2] to re-initialize specific nodes, or --overwrite alone for all.";
exec = ''
#!/run/current-system/sw/bin/bash
set -e
set -euo pipefail
: "''${TF_VAR_admin_ssh_public_key:?TF_VAR_admin_ssh_public_key must be set}"
overwrite_all=false
overwrite_nodes=""
expect_nodes=false
for arg in "$@"; do
if [ "$expect_nodes" = "true" ]; then
case "$arg" in
--*) overwrite_all=true ;;
*) overwrite_nodes="$arg" ;;
esac
expect_nodes=false
fi
[ "$arg" = "--overwrite" ] && expect_nodes=true
done
[ "$expect_nodes" = "true" ] && overwrite_all=true
should_overwrite() {
local hostName="$1"
[ "$overwrite_all" = "true" ] && return 0
[ -n "$overwrite_nodes" ] || return 1
case ",$overwrite_nodes," in
*",$hostName,"*) return 0 ;;
*) return 1 ;;
esac
}
hosts_dir="./operate/hosts"
mkdir -p "$hosts_dir"
echo "Checking for new nodes..."
new_count=0
echo "Fetching node details from OpenTofu..."
node_details=$(tofu -chdir="./deploy" output -json | jq '.cluster_node_details.value')
tofu -chdir="./deploy" output -json | jq -r '.cluster_node_details.value[] | "\(.label)|\(.ipv4)|\(.private_ipv4)|\(.generation)"' | while IFS='|' read -r hostName host private_ip generation; do
host_dir="./operate/hosts/$hostName"
failed_hosts=""
prepared_hosts=""
skipped_hosts=""
prepared_count=0
# Skip if host directory already exists
if [ -d "$host_dir" ]; then
echo " ⏭ Skipping $hostName (already exists)"
continue
fi
prepare_host() {
local hostName="$1" host="$2"
local tmp_dir
tmp_dir=$(mktemp -d)
trap "rm -rf '$tmp_dir'" RETURN
echo " ✨ Adding new node: $hostName (generation: $generation)"
new_count=$((new_count + 1))
echo " Scanning SSH host key from $host..."
local ssh_public_key
ssh_public_key=$(ssh-keyscan -T 15 "$host" 2>/dev/null | grep ssh-ed25519 | head -1 | cut -d " " -f 2,3)
[ -n "$ssh_public_key" ] || { echo " ✗ No ed25519 key returned from $host"; return 1; }
echo " - Getting SSH key from $host..."
ssh_public_key=$(ssh-keyscan "$host" 2>/dev/null | grep ssh-ed25519 | cut -d " " -f 2,3)
echo " Converting SSH key to age format..."
local age_pubkey
age_pubkey=$(echo "$ssh_public_key" | ssh-to-age 2>/dev/null)
[ -n "$age_pubkey" ] || { echo " ✗ ssh-to-age conversion failed"; return 1; }
if [ -z "$ssh_public_key" ]; then
echo " ⚠ Warning: Could not get SSH key from $host"
echo " ℹ Make sure the node is accessible via SSH"
continue
fi
echo " Building config..."
cp -rf operate/templates/. "$tmp_dir/"
echo " - Creating config directory..."
cp -rf operate/templates/* "$host_dir/"
nix-editor "$tmp_dir/default.nix" users.users.sfos.openssh.authorizedKeys.keys \
-i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$tmp_dir/default.nix" users.users.root.openssh.authorizedKeys.keys \
-i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$tmp_dir/default.nix" networking.hostName -i -v "\"$hostName\""
nix-editor "$tmp_dir/default.nix" deployment.targetHost -i -v "\"$host\""
nix-editor "$tmp_dir/default.nix" age.rekey.hostPubkey -i -v "\"$age_pubkey\""
nix-editor "$tmp_dir/default.nix" age.rekey.storageMode -i -v "\"local\""
nix-editor "$tmp_dir/default.nix" age.rekey.localStorageDir \
-i -v "../../../secrets/rekeyed/$hostName"
echo " - Configuring NixOS settings..."
nix-editor "$host_dir/default.nix" users.users.sfos.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$host_dir/default.nix" users.users.root.openssh.authorizedKeys.keys -i -v "[\"$TF_VAR_admin_ssh_public_key\"]"
nix-editor "$host_dir/default.nix" networking.hostName -i -v "\"$hostName\""
nix-editor "$host_dir/default.nix" deployment.targetHost -i -v "\"$host\""
nix-editor "$host_dir/default.nix" age.rekey.localStorageDir -i -v "../../../secrets/rekeyed/$hostName"
local host_dir="$hosts_dir/$hostName"
rm -rf "$host_dir"
cp -r "$tmp_dir/." "$host_dir"
echo " - Updating flake.nix..."
nix-editor flake.nix "outputs.colmena.$hostName" -i -v "$host_dir"
echo " ✅ Node $hostName configured successfully"
done
echo " ✓ Configured (age: $age_pubkey)"
}
while IFS='|' read -r hostName host _private_ip _generation; do
echo ""
echo "Preparing $hostName ($host)..."
host_dir="$hosts_dir/$hostName"
if [ -d "$host_dir" ] && ! should_overwrite "$hostName"; then
echo " Skipping (already configured; use --overwrite $hostName to force)"
skipped_hosts="$skipped_hosts $hostName"
continue
fi
rc=0
prepare_host "$hostName" "$host" || rc=$?
if [ "$rc" -eq 0 ]; then
prepared_hosts="$prepared_hosts $hostName"
prepared_count=$((prepared_count + 1))
else
failed_hosts="$failed_hosts $hostName"
fi
done < <(echo "$node_details" | jq -r 'to_entries[] | "\(.key)|\(.value.ipv4)|\(.value.private_ipv4)|\(.value.generation)"')
echo ""
if [ "$new_count" -eq 0 ]; then
echo "✨ No new nodes to add. All nodes are already configured."
else
echo "✅ Added $new_count new node(s)"
echo ""
echo "Next steps:"
echo " 1. Review the new configs in operate/hosts/"
echo " 2. Deploy with: colmena apply --on <node-names> --impure"
echo " 3. Verify: nomad node status"
git add ./operate/hosts
[ -n "$skipped_hosts" ] && echo "⏭ Already configured (skipped):$skipped_hosts"
if [ -n "$failed_hosts" ]; then
echo "⚠ Failed to prepare:$failed_hosts"
echo " Check SSH connectivity and retry."
fi
if [ "$prepared_count" -gt 0 ]; then
echo "Rekeying secrets for all configured hosts..."
nix run .#agenix-rekey.x86_64-linux.rekey
git add ./operate/hosts ./secrets/rekeyed
echo ""
echo "✓ Prepared $prepared_count host(s):$prepared_hosts"
echo " Next: colmena apply --impure"
fi
'';
};