diff --git a/infra/.gitignore b/infra/.gitignore new file mode 100644 index 0000000..da291ed --- /dev/null +++ b/infra/.gitignore @@ -0,0 +1,29 @@ +# Never commit filled secrets — only *.env.template files are tracked. +*.env +!*.env.template + +# Docker TLS material — CA/server/client certs and keys, generated by +# infra/scripts/docker-tls/generate-ca.sh and friends. Templates/READMEs stay tracked. +infra/docker/tls/*.pem +infra/docker/tls/*.key +infra/docker/tls/*.crt +infra/docker/tls/*.srl +!infra/docker/tls/**/README.md + +# Ansible Vault password files and any decrypted vars +infra/ansible/.vault_pass +infra/ansible/**/vault_plain.yml + +# Terraform/OpenTofu state and generated vars (may contain resource IDs/IPs) +infra/terraform/**/*.tfstate +infra/terraform/**/*.tfstate.* +infra/terraform/**/.terraform/ +infra/terraform/**/terraform.tfvars +!infra/terraform/**/terraform.tfvars.example + +# Backup output and restore scratch space +infra/backup/output/ +infra/backup/restore-test/ + +# Load test results (local artifacts, not source) +infra/scripts/loadtest/results/ diff --git a/infra/ansible/ansible.cfg b/infra/ansible/ansible.cfg new file mode 100644 index 0000000..24c4ed4 --- /dev/null +++ b/infra/ansible/ansible.cfg @@ -0,0 +1,10 @@ +[defaults] +inventory = inventory/production.ini +roles_path = roles +host_key_checking = True +retry_files_enabled = False +stdout_callback = yaml + +[privilege_escalation] +become = True +become_method = sudo diff --git a/infra/ansible/group_vars/production.vault.yml.template b/infra/ansible/group_vars/production.vault.yml.template new file mode 100644 index 0000000..2af7630 --- /dev/null +++ b/infra/ansible/group_vars/production.vault.yml.template @@ -0,0 +1,22 @@ +# Ansible Vault template — copy to production.vault.yml and encrypt it, NEVER commit the +# plaintext or the encrypted file to git (infra/.gitignore already excludes +# **/vault_plain.yml; the encrypted production.vault.yml itself is small/safe to commit IF +# encrypted — see docs/PRODUCTION_DEPLOYMENT.md §10 for the actual decision made). Ansible +# Vault chosen over SOPS/age here because it's already bundled with Ansible (no new tool to +# install/trust) and this project's secret surface is small enough that Vault's simplicity +# outweighs SOPS's per-key-value granularity — revisit if that changes. +# +# cp production.vault.yml.template production.vault.yml +# # fill in real values below +# ansible-vault encrypt production.vault.yml +# +# Reference a vaulted value from group_vars/production.yml or a role like: +# postgres_password: "{{ vault_postgres_password }}" + +vault_postgres_password: __REPLACE__ +vault_provisioner_shared_secret: __REPLACE_openssl_rand_hex_32__ +vault_owner_bootstrap_password: __REPLACE__ # used once, by a human, per docs/RUNBOOK_FIRST_DEPLOYMENT.md +vault_postgres_exporter_password: __REPLACE__ +vault_backup_encryption_passphrase: __REPLACE_openssl_rand_base64_32__ +vault_backup_offsite_credentials: __REPLACE_e.g._Hetzner_Storage_Box_ssh_key_passphrase_or_S3_secret_key__ +vault_grafana_admin_password: __REPLACE__ diff --git a/infra/ansible/group_vars/production.yml b/infra/ansible/group_vars/production.yml new file mode 100644 index 0000000..a76c711 --- /dev/null +++ b/infra/ansible/group_vars/production.yml @@ -0,0 +1,33 @@ +# Production variables — referenced by every {{ }} placeholder in infra/docker/, infra/firewall/, +# and the roles below. Fill in the __PLACEHOLDER__ values once the real server exists +# (docs/PRODUCTION_DEPLOYMENT.md §1). Nothing here is a secret — real secrets live in +# infra/environments/production/*.env (gitignored) and Ansible Vault (see +# infra/ansible/group_vars/production.vault.yml.template), never in this plain-vars file. + +env: production +academy_domain: __REPLACE_e.g._academy.rootnode.example__ + +# ── Network topology (Phase 4 §5 — placeholders until real hardware exists) ──────────────── +platform_ip: __PLATFORM_VM_IP__ +lab_host_ip: __LAB_HOST_VM_IP__ +management_cidr: __YOUR_ADMIN_ACCESS_CIDR__ # e.g. a VPN/WireGuard subnet, never 0.0.0.0/0 +proxmox_mgmt_cidr: __PROXMOX_HOST_MGMT_CIDR__ +lab_subnet_range: "10.42.0.0/16" # matches NO_PROXY_CIDRS already used by the app +lab_gateway_port_range: "30000:32767" # see docs/PRODUCTION_NETWORK_SECURITY.md — Gateway-to-Lab reachability follow-up + +# ── Storage (Phase 4 §4 — a decision framework, not a final layout; see docs/PHASE4_PRODUCTION_INFRASTRUCTURE.md §4) +rootnode_data_root: /srv/rootnode-data # expected to be a ZFS dataset mountpoint on the real host + +# ── Registry / images (infra/docker/IMAGE_STRATEGY.md) ───────────────────────────────────── +registry: git.synthossystems.com +image_tag: __GIT_SHA_OR_RELEASE_TAG__ + +postgres_exporter_user: rootnode_exporter + +# ── Observability (infra/monitoring/) ─────────────────────────────────────────────────────── +prometheus_retention_days: 30 +alertmanager_webhook_url: __REPLACE_e.g._uptime_kuma_or_ntfy_webhook__ + +# ── Backup (infra/backup/) ─────────────────────────────────────────────────────────────────── +backup_retention_days: 30 +backup_offsite_target: __REPLACE_e.g._Hetzner_Storage_Box_path_or_S3_bucket__ diff --git a/infra/ansible/group_vars/staging.yml b/infra/ansible/group_vars/staging.yml new file mode 100644 index 0000000..0c41dfd --- /dev/null +++ b/infra/ansible/group_vars/staging.yml @@ -0,0 +1,22 @@ +# Staging variables — same shape as production.yml, lower stakes. See +# infra/environments/staging/README.md. + +env: staging +academy_domain: __REPLACE_e.g._staging.academy.rootnode.example_or_unset_for_no_public_TLS__ + +platform_ip: __STAGING_PLATFORM_IP__ +lab_host_ip: __STAGING_LAB_HOST_IP__ +management_cidr: __YOUR_ADMIN_ACCESS_CIDR__ +proxmox_mgmt_cidr: __PROXMOX_HOST_MGMT_CIDR__ +lab_subnet_range: "10.42.0.0/16" + +rootnode_data_root: /srv/rootnode-data-staging + +registry: git.synthossystems.com +image_tag: __STAGING_TAG_e.g._staging-__ + +prometheus_retention_days: 7 +alertmanager_webhook_url: __OPTIONAL_staging_can_skip_alerting__ + +backup_retention_days: 7 +backup_offsite_target: __OPTIONAL_staging_backups_are_lower_priority__ diff --git a/infra/ansible/inventory/production.ini b/infra/ansible/inventory/production.ini new file mode 100644 index 0000000..41313ff --- /dev/null +++ b/infra/ansible/inventory/production.ini @@ -0,0 +1,16 @@ +# Production inventory. IPs/hostnames are placeholders — fill in once the real VMs exist +# (docs/PRODUCTION_DEPLOYMENT.md §1). This file is safe to commit (no secrets, just topology). + +[platform] +rn-platform-01 ansible_host=__PLATFORM_VM_IP__ + +[lab_host] +rn-lab-host-01 ansible_host=__LAB_HOST_VM_IP__ + +[rootnode:children] +platform +lab_host + +[rootnode:vars] +ansible_user=rootnode-deploy +ansible_ssh_common_args='-o StrictHostKeyChecking=yes' diff --git a/infra/ansible/inventory/staging.ini b/infra/ansible/inventory/staging.ini new file mode 100644 index 0000000..b2b1c97 --- /dev/null +++ b/infra/ansible/inventory/staging.ini @@ -0,0 +1,16 @@ +# Staging inventory — same group shape as production, pointed at homelab VMs/containers or a +# resource-constrained slice of the dedicated server once it exists (infra/environments/staging/README.md). + +[platform] +rn-platform-staging ansible_host=__STAGING_PLATFORM_IP__ + +[lab_host] +rn-lab-host-staging ansible_host=__STAGING_LAB_HOST_IP__ + +[rootnode:children] +platform +lab_host + +[rootnode:vars] +ansible_user=rootnode-deploy +env=staging diff --git a/infra/ansible/roles/backup/tasks/main.yml b/infra/ansible/roles/backup/tasks/main.yml new file mode 100644 index 0000000..381971e --- /dev/null +++ b/infra/ansible/roles/backup/tasks/main.yml @@ -0,0 +1,63 @@ +--- +# Deploys the backup script + a daily cron job on rn-platform-01. The script itself +# (infra/backup/backup-postgres.sh) is identical to what infra/backup/test-backup-restore.sh +# exercises locally — same code path, just pointed at real production values here. + +- name: Install postgresql-client + gpg (backup script dependencies) + ansible.builtin.apt: + name: [postgresql-client, gnupg] + state: present + +- name: Create backup directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0700" + loop: + - /opt/rootnode/backup + - "{{ rootnode_data_root }}/backups" + +- name: Deploy backup script + ansible.builtin.copy: + src: "{{ role_path }}/../../../backup/backup-postgres.sh" + dest: /opt/rootnode/backup/backup-postgres.sh + mode: "0750" + +- name: Deploy restore + verify scripts (not run by cron, kept alongside for the runbook) + ansible.builtin.copy: + src: "{{ role_path }}/../../../backup/{{ item }}" + dest: "/opt/rootnode/backup/{{ item }}" + mode: "0750" + loop: + - restore-postgres.sh + - verify-restore.sh + +- name: Deploy the backup env file (PLATFORM_DB_URL, encryption passphrase, offsite target) + ansible.builtin.template: + src: backup.env.j2 + dest: /etc/rootnode/backup.env + owner: root + group: root + mode: "0600" + +- name: Deploy cron wrapper (sources the env file, writes to syslog too) + ansible.builtin.template: + src: run-backup.sh.j2 + dest: /opt/rootnode/backup/run-backup.sh + mode: "0750" + +- name: Install daily backup cron job + ansible.builtin.cron: + name: rootnode-postgres-backup + user: root + minute: "15" + hour: "3" + job: /opt/rootnode/backup/run-backup.sh + +- name: Ensure node-exporter's textfile collector directory exists (backup metrics land here) + ansible.builtin.file: + path: /var/lib/node-exporter/textfile + state: directory + mode: "0755" diff --git a/infra/ansible/roles/backup/templates/backup.env.j2 b/infra/ansible/roles/backup/templates/backup.env.j2 new file mode 100644 index 0000000..19ee2b8 --- /dev/null +++ b/infra/ansible/roles/backup/templates/backup.env.j2 @@ -0,0 +1,6 @@ +PLATFORM_DB_URL=postgres://rootnode_platform:{{ vault_postgres_password }}@localhost:5432/rootnode_platform +BACKUP_DIR={{ rootnode_data_root }}/backups +BACKUP_ENCRYPTION_PASSPHRASE={{ vault_backup_encryption_passphrase }} +BACKUP_RETENTION_DAYS={{ backup_retention_days }} +BACKUP_OFFSITE_TARGET={{ backup_offsite_target }} +METRICS_TEXTFILE_DIR=/var/lib/node-exporter/textfile diff --git a/infra/ansible/roles/backup/templates/run-backup.sh.j2 b/infra/ansible/roles/backup/templates/run-backup.sh.j2 new file mode 100644 index 0000000..466c073 --- /dev/null +++ b/infra/ansible/roles/backup/templates/run-backup.sh.j2 @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Cron wrapper — sources the deployed env file and runs the real backup script, logging to +# syslog (visible via journalctl and shipped to Loki by promtail like every other unit) in +# addition to whatever cron itself captures. +set -euo pipefail +set -a +source /etc/rootnode/backup.env +set +a +exec /opt/rootnode/backup/backup-postgres.sh 2>&1 | systemd-cat -t rootnode-backup diff --git a/infra/ansible/roles/common/defaults/main.yml b/infra/ansible/roles/common/defaults/main.yml new file mode 100644 index 0000000..e41f9a8 --- /dev/null +++ b/infra/ansible/roles/common/defaults/main.yml @@ -0,0 +1,3 @@ +--- +common_timezone: UTC +common_unattended_upgrades: true diff --git a/infra/ansible/roles/common/tasks/docker-engine.yml b/infra/ansible/roles/common/tasks/docker-engine.yml new file mode 100644 index 0000000..e0fc38e --- /dev/null +++ b/infra/ansible/roles/common/tasks/docker-engine.yml @@ -0,0 +1,35 @@ +--- +# Shared Docker Engine install — used by both platform and lab-host roles (import_tasks from +# common, not duplicated). Follows Docker's own official apt-repo installation method. + +- name: Add Docker's GPG key + ansible.builtin.get_url: + url: https://download.docker.com/linux/ubuntu/gpg + dest: /etc/apt/keyrings/docker.asc + mode: "0644" + register: docker_gpg + retries: 3 + delay: 5 + until: docker_gpg is succeeded + +- name: Add Docker apt repository + ansible.builtin.apt_repository: + repo: "deb [arch={{ ansible_architecture | replace('x86_64','amd64') }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" + state: present + filename: docker + +- name: Install Docker Engine + Compose plugin + ansible.builtin.apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin + state: present + update_cache: true + +- name: Enable and start Docker + ansible.builtin.systemd: + name: docker + enabled: true + state: started diff --git a/infra/ansible/roles/common/tasks/main.yml b/infra/ansible/roles/common/tasks/main.yml new file mode 100644 index 0000000..5bb1648 --- /dev/null +++ b/infra/ansible/roles/common/tasks/main.yml @@ -0,0 +1,58 @@ +--- +# Baseline hardening applied to both rn-platform-01 and rn-lab-host-01 before anything +# RootNode-specific — idempotent, safe to re-run (Phase 4 §20 requirement). + +- name: Set timezone + community.general.timezone: + name: "{{ common_timezone }}" + +- name: Install base packages + ansible.builtin.apt: + name: + - curl + - ca-certificates + - gnupg + - iptables + - iptables-persistent + - unattended-upgrades + - fail2ban + - jq + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Enable unattended security upgrades + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/20auto-upgrades + content: | + APT::Periodic::Update-Package-Lists "1"; + APT::Periodic::Unattended-Upgrade "1"; + mode: "0644" + when: common_unattended_upgrades + +- name: Create the rootnode-deploy service account (if not the connecting user already) + ansible.builtin.user: + name: rootnode-deploy + shell: /bin/bash + groups: docker + append: true + create_home: true + register: deploy_user_result + failed_when: false # non-fatal — some hosts may already use this as ansible_user + +- name: Ensure fail2ban is enabled for SSH + ansible.builtin.service: + name: fail2ban + state: started + enabled: true + +- name: Create /opt/rootnode and /etc/rootnode base directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0750" + loop: + - /opt/rootnode + - /etc/rootnode diff --git a/infra/ansible/roles/docker-tls/defaults/main.yml b/infra/ansible/roles/docker-tls/defaults/main.yml new file mode 100644 index 0000000..2df9f84 --- /dev/null +++ b/infra/ansible/roles/docker-tls/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# Local path (on the machine running ansible-playbook, NOT on either VM) holding the cert +# bundle generated by infra/scripts/docker-tls/generate-docker-tls.sh. The CA private key +# (ca-key.pem) is deliberately never copied to either host by this role — only its public +# ca.pem, plus the one cert/key pair each host actually needs. +docker_tls_local_bundle_dir: "{{ lookup('env', 'ROOTNODE_TLS_BUNDLE_DIR') | default('/does/not/exist', true) }}" diff --git a/infra/ansible/roles/docker-tls/handlers/main.yml b/infra/ansible/roles/docker-tls/handlers/main.yml new file mode 100644 index 0000000..5ee92c0 --- /dev/null +++ b/infra/ansible/roles/docker-tls/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: restart docker with tls + ansible.builtin.systemd: + name: docker + state: restarted + daemon_reload: true diff --git a/infra/ansible/roles/docker-tls/tasks/main.yml b/infra/ansible/roles/docker-tls/tasks/main.yml new file mode 100644 index 0000000..bc5a548 --- /dev/null +++ b/infra/ansible/roles/docker-tls/tasks/main.yml @@ -0,0 +1,89 @@ +--- +# Deploys Docker TLS material generated OFFLINE by infra/scripts/docker-tls/generate-docker-tls.sh +# — this role never generates key material itself, and the CA private key never touches either +# VM (see infra/docker/tls/README.md). Fails loudly if the operator forgot to set +# ROOTNODE_TLS_BUNDLE_DIR / run the generation script first, rather than silently skipping and +# leaving the Provisioner unable to reach rn-lab-host-01. + +- name: Verify the local TLS bundle directory exists + ansible.builtin.stat: + path: "{{ docker_tls_local_bundle_dir }}" + register: bundle_dir + delegate_to: localhost + become: false + +- name: Fail fast if the TLS bundle wasn't generated + ansible.builtin.fail: + msg: >- + ROOTNODE_TLS_BUNDLE_DIR ({{ docker_tls_local_bundle_dir }}) does not exist. Run + infra/scripts/docker-tls/generate-docker-tls.sh first, then re-run with + ROOTNODE_TLS_BUNDLE_DIR= in the environment. + when: not bundle_dir.stat.exists + +- name: Create /etc/rootnode/docker-tls + ansible.builtin.file: + path: /etc/rootnode/docker-tls + state: directory + owner: root + group: root + mode: "0700" + +- name: Deploy CA cert (public, both hosts) + ansible.builtin.copy: + src: "{{ docker_tls_local_bundle_dir }}/ca.pem" + dest: /etc/rootnode/docker-tls/ca.pem + owner: root + group: root + mode: "0444" + +- name: Deploy server cert/key (rn-lab-host-01 only) + ansible.builtin.copy: + src: "{{ docker_tls_local_bundle_dir }}/{{ item.src }}" + dest: "/etc/rootnode/docker-tls/{{ item.dest }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - { src: server-cert.pem, dest: server-cert.pem, mode: "0444" } + - { src: server-key.pem, dest: server-key.pem, mode: "0400" } + when: inventory_hostname in groups['lab_host'] + notify: restart docker with tls + +- name: Deploy Provisioner client cert/key (rn-platform-01 only) + ansible.builtin.copy: + src: "{{ docker_tls_local_bundle_dir }}/{{ item.src }}" + dest: "/etc/rootnode/docker-tls/{{ item.dest }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - { src: provisioner-client-cert.pem, dest: provisioner-client-cert.pem, mode: "0444" } + - { src: provisioner-client-key.pem, dest: provisioner-client-key.pem, mode: "0400" } + when: inventory_hostname in groups['platform'] + +- name: Configure dockerd for TLS (rn-lab-host-01 only) + ansible.builtin.template: + src: daemon.json.j2 + dest: /etc/docker/daemon.json + owner: root + group: root + mode: "0644" + when: inventory_hostname in groups['lab_host'] + notify: restart docker with tls + +- name: Override the Docker systemd unit's ExecStart (clears the default -H fd:// flag) + ansible.builtin.file: + path: /etc/systemd/system/docker.service.d + state: directory + mode: "0755" + when: inventory_hostname in groups['lab_host'] + +- name: Deploy the ExecStart override + ansible.builtin.template: + src: docker-override.conf.j2 + dest: /etc/systemd/system/docker.service.d/override.conf + owner: root + group: root + mode: "0644" + when: inventory_hostname in groups['lab_host'] + notify: restart docker with tls diff --git a/infra/ansible/roles/docker-tls/templates/daemon.json.j2 b/infra/ansible/roles/docker-tls/templates/daemon.json.j2 new file mode 100644 index 0000000..cc86844 --- /dev/null +++ b/infra/ansible/roles/docker-tls/templates/daemon.json.j2 @@ -0,0 +1,13 @@ +{ + "tls": true, + "tlsverify": true, + "tlscacert": "/etc/rootnode/docker-tls/ca.pem", + "tlscert": "/etc/rootnode/docker-tls/server-cert.pem", + "tlskey": "/etc/rootnode/docker-tls/server-key.pem", + "hosts": ["unix:///var/run/docker.sock", "tcp://0.0.0.0:2376"], + + "storage-driver": "overlay2", + "default-address-pools": [ + { "base": "10.42.0.0/16", "size": 24 } + ] +} diff --git a/infra/ansible/roles/docker-tls/templates/docker-override.conf.j2 b/infra/ansible/roles/docker-tls/templates/docker-override.conf.j2 new file mode 100644 index 0000000..67b833f --- /dev/null +++ b/infra/ansible/roles/docker-tls/templates/docker-override.conf.j2 @@ -0,0 +1,7 @@ +# Docker's systemd unit specifies `-H fd://` on ExecStart by default, which conflicts with +# setting `hosts` in daemon.json ("specified both as a flag and in the configuration file"). +# This override clears ExecStart and re-specifies it with no -H flag, letting daemon.json's +# `hosts` list be the only source of truth — the documented fix for this exact conflict. +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd diff --git a/infra/ansible/roles/firewall/handlers/main.yml b/infra/ansible/roles/firewall/handlers/main.yml new file mode 100644 index 0000000..9e375c7 --- /dev/null +++ b/infra/ansible/roles/firewall/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: reload firewall + ansible.builtin.command: netfilter-persistent reload + changed_when: true diff --git a/infra/ansible/roles/firewall/tasks/main.yml b/infra/ansible/roles/firewall/tasks/main.yml new file mode 100644 index 0000000..41a842e --- /dev/null +++ b/infra/ansible/roles/firewall/tasks/main.yml @@ -0,0 +1,28 @@ +--- +# Deploys the matching infra/firewall/*.rules file for this host's group and loads it. Same +# ruleset, same variables as infra/firewall/validate-firewall.sh renders locally — this task +# just uses Ansible's own templating instead of sed, against the real inventory values. + +- name: Determine which ruleset applies to this host + ansible.builtin.set_fact: + firewall_ruleset_name: "{{ 'rn-platform-01' if inventory_hostname in groups['platform'] else 'rn-lab-host-01' }}" + +- name: Render firewall ruleset for this host + ansible.builtin.template: + src: "{{ role_path }}/../../../firewall/{{ firewall_ruleset_name }}.rules" + dest: /etc/iptables/rules.v4.rootnode + owner: root + group: root + mode: "0600" + notify: reload firewall + +- name: Point iptables-persistent at the RootNode ruleset + ansible.builtin.file: + src: /etc/iptables/rules.v4.rootnode + dest: /etc/iptables/rules.v4 + state: link + force: true + notify: reload firewall + +- name: Flush handlers so the firewall is active before later roles assume it + ansible.builtin.meta: flush_handlers diff --git a/infra/ansible/roles/lab-host/handlers/main.yml b/infra/ansible/roles/lab-host/handlers/main.yml new file mode 100644 index 0000000..dee17e1 --- /dev/null +++ b/infra/ansible/roles/lab-host/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: restart rootnode-lab-host + ansible.builtin.systemd: + name: rootnode-lab-host + state: restarted + daemon_reload: true diff --git a/infra/ansible/roles/lab-host/tasks/main.yml b/infra/ansible/roles/lab-host/tasks/main.yml new file mode 100644 index 0000000..7c9c8cc --- /dev/null +++ b/infra/ansible/roles/lab-host/tasks/main.yml @@ -0,0 +1,72 @@ +--- +# rn-lab-host-01: Docker Engine (TLS config comes from the docker-tls role, run before this +# one), the egress proxy + monitoring agents compose stack, kernel prerequisites for nested +# k3s in privileged Lab containers, and Docker daemon storage/limits configuration. + +- name: Install Docker Engine + ansible.builtin.import_tasks: "{{ role_path }}/../common/tasks/docker-engine.yml" + +- name: Kernel prerequisites for nested k3s (privileged containers) + ansible.posix.sysctl: + name: "{{ item.name }}" + value: "{{ item.value }}" + sysctl_set: true + state: present + reload: true + loop: + - { name: net.ipv4.ip_forward, value: "1" } + - { name: fs.inotify.max_user_watches, value: "524288" } # k3s/kubelet watch a lot of files + - { name: fs.inotify.max_user_instances, value: "512" } + - { name: vm.max_map_count, value: "262144" } # matches upstream k3s/etcd guidance + +- name: Ensure cgroup v2 controllers needed by nested k3s are available + ansible.builtin.command: cat /sys/fs/cgroup/cgroup.controllers + register: cgroup_controllers + changed_when: false + failed_when: false + +- name: Warn if cgroup v2 controllers look incomplete (informational only) + ansible.builtin.debug: + msg: >- + cgroup.controllers reports: {{ cgroup_controllers.stdout | default('unreadable') }}. + Expect at least cpu, memory, pids, io for reliable nested k3s — verify manually if any + are missing (see docs/PRODUCTION_VALIDATION_CHECKLIST.md "Lab boot" section). + when: cgroup_controllers.rc == 0 + +- name: Create data/config directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: docker + mode: "0750" + loop: + - /opt/rootnode/lab-host + - "{{ rootnode_data_root }}/lab-work" # disposable — see docs/BACKUP_DR.md, never backed up + +- name: Render production docker-compose.yml + ansible.builtin.template: + src: "{{ role_path }}/../../../docker/lab-host/docker-compose.production.yml" + dest: /opt/rootnode/lab-host/docker-compose.yml + owner: root + group: root + mode: "0640" + notify: restart rootnode-lab-host + +- name: Deploy systemd unit + ansible.builtin.template: + src: rootnode-lab-host.service.j2 + dest: /etc/systemd/system/rootnode-lab-host.service + mode: "0644" + notify: restart rootnode-lab-host + +- name: Enable and start rootnode-lab-host + ansible.builtin.systemd: + name: rootnode-lab-host + enabled: true + state: started + daemon_reload: true + +# Storage-driver/networking daemon options live in the docker-tls role's daemon.json.j2 — +# Docker only ever reads ONE /etc/docker/daemon.json, so that's the single source of truth for +# this host's whole daemon config, not just its TLS settings (see that role's template). diff --git a/infra/ansible/roles/lab-host/templates/rootnode-lab-host.service.j2 b/infra/ansible/roles/lab-host/templates/rootnode-lab-host.service.j2 new file mode 100644 index 0000000..f06530b --- /dev/null +++ b/infra/ansible/roles/lab-host/templates/rootnode-lab-host.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=RootNode Academy Lab Host services (egress proxy + monitoring agents) +Requires=docker.service +After=docker.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/opt/rootnode/lab-host +ExecStart=/usr/bin/docker compose -f docker-compose.yml up -d --remove-orphans +ExecStop=/usr/bin/docker compose -f docker-compose.yml down +ExecReload=/usr/bin/docker compose -f docker-compose.yml up -d --remove-orphans +TimeoutStartSec=180 + +[Install] +WantedBy=multi-user.target diff --git a/infra/ansible/roles/monitoring-agent/handlers/main.yml b/infra/ansible/roles/monitoring-agent/handlers/main.yml new file mode 100644 index 0000000..b3c63c8 --- /dev/null +++ b/infra/ansible/roles/monitoring-agent/handlers/main.yml @@ -0,0 +1,16 @@ +--- +- name: reload prometheus + ansible.builtin.command: docker compose -f /opt/rootnode/platform/docker-compose.yml restart prometheus + changed_when: true + +- name: reload alertmanager + ansible.builtin.command: docker compose -f /opt/rootnode/platform/docker-compose.yml restart alertmanager + changed_when: true + +- name: restart platform promtail + ansible.builtin.command: docker compose -f /opt/rootnode/platform/docker-compose.yml restart promtail + changed_when: true + +- name: restart lab-host promtail + ansible.builtin.command: docker compose -f /opt/rootnode/lab-host/docker-compose.yml restart promtail + changed_when: true diff --git a/infra/ansible/roles/monitoring-agent/tasks/main.yml b/infra/ansible/roles/monitoring-agent/tasks/main.yml new file mode 100644 index 0000000..9d1f9aa --- /dev/null +++ b/infra/ansible/roles/monitoring-agent/tasks/main.yml @@ -0,0 +1,75 @@ +--- +# Deploys infra/monitoring/ configs to whichever host needs them. Named monitoring-agent for +# historical reasons in this repo's role list, but it deploys both the "agent" pieces +# (node-exporter/cadvisor/promtail configs, already referenced directly by the platform/ +# lab-host compose files) and the central stack's config (Prometheus/Alertmanager/Loki/Grafana +# on rn-platform-01 only). + +- name: Create /opt/rootnode/monitoring (rn-platform-01) + ansible.builtin.file: + path: /opt/rootnode/monitoring/rules + state: directory + mode: "0755" + when: inventory_hostname in groups['platform'] + +- name: Deploy Prometheus config + ansible.builtin.template: + src: "{{ role_path }}/../../../monitoring/prometheus/prometheus.yml.j2" + dest: /opt/rootnode/monitoring/prometheus.yml + mode: "0644" + when: inventory_hostname in groups['platform'] + notify: reload prometheus + +- name: Deploy alert rules + ansible.builtin.copy: + src: "{{ role_path }}/../../../monitoring/prometheus/rules/rootnode.yml" + dest: /opt/rootnode/monitoring/rules/rootnode.yml + mode: "0644" + when: inventory_hostname in groups['platform'] + notify: reload prometheus + +- name: Deploy blackbox_exporter config + ansible.builtin.copy: + src: "{{ role_path }}/../../../monitoring/prometheus/blackbox.yml" + dest: /opt/rootnode/monitoring/blackbox.yml + mode: "0644" + when: inventory_hostname in groups['platform'] + +- name: Deploy Alertmanager config + ansible.builtin.template: + src: "{{ role_path }}/../../../monitoring/alertmanager/alertmanager.yml.j2" + dest: /opt/rootnode/monitoring/alertmanager.yml + mode: "0644" + when: inventory_hostname in groups['platform'] + notify: reload alertmanager + +- name: Deploy Loki config + ansible.builtin.copy: + src: "{{ role_path }}/../../../monitoring/loki/loki-config.yml" + dest: /opt/rootnode/monitoring/loki-config.yml + mode: "0644" + when: inventory_hostname in groups['platform'] + +- name: Deploy Grafana provisioning + dashboards + ansible.builtin.copy: + src: "{{ role_path }}/../../../monitoring/grafana/{{ item }}/" + dest: "/opt/rootnode/monitoring/grafana/{{ item }}/" + mode: "0644" + loop: [provisioning, dashboards] + when: inventory_hostname in groups['platform'] + +- name: Deploy platform-side promtail config + ansible.builtin.copy: + src: "{{ role_path }}/../../../monitoring/promtail/promtail-config.platform.yml" + dest: /opt/rootnode/monitoring/promtail-config.yml + mode: "0644" + when: inventory_hostname in groups['platform'] + notify: restart platform promtail + +- name: Deploy lab-host promtail config + ansible.builtin.template: + src: "{{ role_path }}/../../../monitoring/promtail/promtail-config.lab-host.yml.j2" + dest: /opt/rootnode/lab-host/promtail-config.yml + mode: "0644" + when: inventory_hostname in groups['lab_host'] + notify: restart lab-host promtail diff --git a/infra/ansible/roles/platform/handlers/main.yml b/infra/ansible/roles/platform/handlers/main.yml new file mode 100644 index 0000000..89ff278 --- /dev/null +++ b/infra/ansible/roles/platform/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: restart rootnode-platform + ansible.builtin.systemd: + name: rootnode-platform + state: restarted + daemon_reload: true diff --git a/infra/ansible/roles/platform/tasks/main.yml b/infra/ansible/roles/platform/tasks/main.yml new file mode 100644 index 0000000..31c306c --- /dev/null +++ b/infra/ansible/roles/platform/tasks/main.yml @@ -0,0 +1,85 @@ +--- +# rn-platform-01: Docker Engine, directory structure, rendered compose file + env files, +# Postgres tuning overlay, reverse proxy config, and the systemd unit that runs it all. +# Docker TLS material and the host firewall are separate roles (docker-tls, firewall) run +# before this one in site.yml — this role assumes both are already in place. + +- name: Install Docker Engine + ansible.builtin.import_tasks: "{{ role_path }}/../common/tasks/docker-engine.yml" + +- name: Create data/config directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: docker + mode: "0750" + loop: + - "{{ rootnode_data_root }}/postgres" + - "{{ rootnode_data_root }}/backups" + - /opt/rootnode/platform + - /opt/rootnode/platform/postgresql.conf.d + - /etc/rootnode/provisioner-config + +- name: Deploy Postgres tuning overlay + ansible.builtin.copy: + src: "{{ role_path }}/../../../docker/postgres/postgresql.conf.d/00-rootnode-baseline.conf" + dest: /opt/rootnode/platform/postgresql.conf.d/00-rootnode-baseline.conf + mode: "0644" + notify: restart rootnode-platform + +- name: Deploy provisioner-config overlay ({{ env }}) + ansible.builtin.copy: + src: "{{ role_path }}/../../../../environments/{{ env }}/provisioner-config/" + dest: /etc/rootnode/provisioner-config/ + mode: "0644" + notify: restart rootnode-platform + +- name: Deploy per-service env files (from Ansible Vault-backed variables — see group_vars) + ansible.builtin.template: + src: "{{ item }}.env.j2" + dest: "/etc/rootnode/{{ item }}.env" + owner: root + group: root + mode: "0600" + loop: + - platform + - provisioner + - gateway + - postgres + notify: restart rootnode-platform + +- name: Render production docker-compose.yml + ansible.builtin.template: + src: "{{ role_path }}/../../../docker/platform/docker-compose.production.yml" + dest: /opt/rootnode/platform/docker-compose.yml + owner: root + group: root + mode: "0640" + notify: restart rootnode-platform + +- name: Render Caddyfile + ansible.builtin.template: + src: "{{ role_path }}/../../../docker/platform/Caddyfile.j2" + dest: /opt/rootnode/platform/Caddyfile + owner: root + group: root + mode: "0644" + notify: restart rootnode-platform + +- name: Deploy systemd unit + ansible.builtin.template: + src: rootnode-platform.service.j2 + dest: /etc/systemd/system/rootnode-platform.service + mode: "0644" + notify: restart rootnode-platform + +# The `backup` role (applied after this one in site.yml) deploys the backup script and cron — +# kept separate since it also has to run correctly on a host that ISN'T freshly deployed. + +- name: Enable and start rootnode-platform + ansible.builtin.systemd: + name: rootnode-platform + enabled: true + state: started + daemon_reload: true diff --git a/infra/ansible/roles/platform/templates/gateway.env.j2 b/infra/ansible/roles/platform/templates/gateway.env.j2 new file mode 100644 index 0000000..ee28586 --- /dev/null +++ b/infra/ansible/roles/platform/templates/gateway.env.j2 @@ -0,0 +1,5 @@ +# Rendered by Ansible — see platform.env.j2 for the pattern. +PLATFORM_API_URL=http://api:4300 +PROVISIONER_URL=http://provisioner:4100 +PROVISIONER_SHARED_SECRET={{ vault_provisioner_shared_secret }} +GATEWAY_PORT=4200 diff --git a/infra/ansible/roles/platform/templates/platform.env.j2 b/infra/ansible/roles/platform/templates/platform.env.j2 new file mode 100644 index 0000000..f72bf83 --- /dev/null +++ b/infra/ansible/roles/platform/templates/platform.env.j2 @@ -0,0 +1,18 @@ +# Rendered by Ansible from infra/environments/{{ env }}/platform.env.template's shape — +# real values come from group_vars/{{ env }}.yml + the encrypted vault file. Do not hand-edit +# on the host; re-run the platform role instead. + +# service names, not localhost — production runs these on a named Docker bridge network +# (rn-platform-internal), not network_mode: host like the dev compose file does. +PLATFORM_DB_URL=postgres://rootnode_platform:{{ vault_postgres_password }}@postgres:5432/rootnode_platform +PROVISIONER_URL=http://provisioner:4100 +PROVISIONER_SHARED_SECRET={{ vault_provisioner_shared_secret }} +PLATFORM_API_PORT=4300 + +STUDENT_SESSION_TTL_HOURS=8 +STUDENT_SESSION_IDLE_HOURS=2 +INSTRUCTOR_SESSION_TTL_HOURS=12 +INSTRUCTOR_SESSION_IDLE_HOURS=2 + +COOKIE_SECURE=1 +TRUST_PROXY=1 diff --git a/infra/ansible/roles/platform/templates/postgres.env.j2 b/infra/ansible/roles/platform/templates/postgres.env.j2 new file mode 100644 index 0000000..19439cc --- /dev/null +++ b/infra/ansible/roles/platform/templates/postgres.env.j2 @@ -0,0 +1,4 @@ +# Rendered by Ansible — see platform.env.j2 for the pattern. +POSTGRES_USER=rootnode_platform +POSTGRES_PASSWORD={{ vault_postgres_password }} +POSTGRES_DB=rootnode_platform diff --git a/infra/ansible/roles/platform/templates/provisioner.env.j2 b/infra/ansible/roles/platform/templates/provisioner.env.j2 new file mode 100644 index 0000000..961e56b --- /dev/null +++ b/infra/ansible/roles/platform/templates/provisioner.env.j2 @@ -0,0 +1,11 @@ +# Rendered by Ansible — see platform.env.j2 for the pattern. +PLATFORM_DB_URL=postgres://rootnode_platform:{{ vault_postgres_password }}@postgres:5432/rootnode_platform +PROVISIONER_SHARED_SECRET={{ vault_provisioner_shared_secret }} +PROVISIONER_PORT=4100 +PROVISIONER_CONFIG_DIR=/app/config + +LAB_IMAGE_REF={{ registry }}/rootnode/lab:{{ image_tag }} + +EGRESS_PROXY_PORT=3128 +DNS_RESOLVER=127.0.0.11 +NO_PROXY_CIDRS=10.42.0.0/16,10.43.0.0/16 diff --git a/infra/ansible/roles/platform/templates/rootnode-platform.service.j2 b/infra/ansible/roles/platform/templates/rootnode-platform.service.j2 new file mode 100644 index 0000000..3ad761f --- /dev/null +++ b/infra/ansible/roles/platform/templates/rootnode-platform.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=RootNode Academy Platform (docker compose) +Requires=docker.service +After=docker.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/opt/rootnode/platform +ExecStart=/usr/bin/docker compose -f docker-compose.yml up -d --remove-orphans +ExecStop=/usr/bin/docker compose -f docker-compose.yml down +ExecReload=/usr/bin/docker compose -f docker-compose.yml up -d --remove-orphans +TimeoutStartSec=300 + +[Install] +WantedBy=multi-user.target diff --git a/infra/ansible/site.yml b/infra/ansible/site.yml new file mode 100644 index 0000000..0b97784 --- /dev/null +++ b/infra/ansible/site.yml @@ -0,0 +1,52 @@ +--- +# Top-level playbook — full production/staging deployment. Idempotent: re-running against an +# already-deployed host converges it, doesn't rebuild from scratch (Phase 4 §20 requirement). +# +# Usage: +# ansible-playbook site.yml -i inventory/production.ini --vault-password-file .vault_pass +# ansible-playbook site.yml -i inventory/staging.ini -e env=staging +# +# Role order matters: firewall and docker-tls must land before platform/lab-host bring up +# services that depend on the network policy / TLS material already being in place. + +- name: Base hardening — all RootNode hosts + hosts: rootnode + become: true + roles: + - common + +- name: Firewall policy — all RootNode hosts + hosts: rootnode + become: true + roles: + - firewall + +- name: Docker TLS — both hosts (server cert on lab_host, client cert on platform) + hosts: rootnode + become: true + roles: + - docker-tls + +- name: Deploy the Platform VM + hosts: platform + become: true + roles: + - platform + +- name: Deploy the Lab Host VM + hosts: lab_host + become: true + roles: + - lab-host + +- name: Deploy monitoring configuration — both hosts + hosts: rootnode + become: true + roles: + - monitoring-agent + +- name: Deploy backup automation — Platform VM only (that's where Postgres lives) + hosts: platform + become: true + roles: + - backup diff --git a/infra/environments/README.md b/infra/environments/README.md new file mode 100644 index 0000000..db87c30 --- /dev/null +++ b/infra/environments/README.md @@ -0,0 +1,21 @@ +# Environment configuration + +Three environments, explicitly separated so production never silently inherits a dev default: + +| | dev | staging | production | +|---|---|---|---| +| Where it runs | any dev machine, `platform/docker-compose.dev.yml` | homelab or a resource-limited slice of the dedicated server once it exists | rn-platform-01 / rn-lab-host-01 on Proxmox | +| `COOKIE_SECURE` | unset (plain HTTP) | `1` if served over HTTPS, else unset | `1` — required | +| `TRUST_PROXY` | unset | `1` if fronted by a proxy | `1` — required, real reverse proxy in front | +| `ADMIN`/owner bootstrap | `platform/api/bootstrap-owner.js` manually | same, against staging DB | same, against production DB, run once by a human, credential handled per `docs/PRODUCTION_DEPLOYMENT.md` §10 | +| Secrets | `platform/.env` (gitignored, throwaway) | `infra/environments/staging/*.env` (gitignored, real but low-stakes) | `infra/environments/production/*.env` (gitignored, real, backed up per `docs/BACKUP_DR.md`) | +| Lab image | built locally, `:test` tag | tagged build from CI, `:staging-` | tagged release, `:` or `:vX.Y.Z`, never `:latest` | + +Each environment directory holds `*.env.template` files (tracked) — copy to `*.env` (gitignored +via `infra/.gitignore`) and fill in real values. **Never commit a filled `.env` file.** See +`docs/PRODUCTION_DEPLOYMENT.md` §10 for how production secrets are actually generated, stored, +and rotated (not just "don't commit them"). + +`dev/` is intentionally thin — it just points back at the existing +`platform/docker-compose.dev.yml` workflow (Phase 1–3B, unchanged); there was nothing new to +build there for Phase 4. diff --git a/infra/environments/dev/README.md b/infra/environments/dev/README.md new file mode 100644 index 0000000..3da534c --- /dev/null +++ b/infra/environments/dev/README.md @@ -0,0 +1,7 @@ +# Dev environment + +Nothing new here for Phase 4 — dev still means `platform/docker-compose.dev.yml` + +`platform/.env` (gitignored), exactly as built in Phases 1–3B. See `platform/README.md`. + +This directory exists only so `infra/environments/` visibly has all three tiers, not because +dev needed new infrastructure code. diff --git a/infra/environments/production/capacity.yml.example b/infra/environments/production/capacity.yml.example new file mode 100644 index 0000000..5a293ed --- /dev/null +++ b/infra/environments/production/capacity.yml.example @@ -0,0 +1,18 @@ +# Capacity planning inputs (Phase 4 §25). Copy to capacity.yml and fill in REAL measured values +# once the actual Hetzner host and a real load test (infra/scripts/loadtest/) exist. The +# numbers below are illustrative placeholders for a ~16c/32t/128GB host — do not treat them as +# a real recommendation; run infra/scripts/capacity-model.js against real ones once measured. +_placeholder: true + +host_cpu_cores: 32 # threads, if the host does hyperthreading — matches how Docker/cgroups count them +host_mem_mb: 131072 # 128 GB + +platform_reserved_cpu_cores: 4 # rn-platform-01's VM allocation, reserved off the Lab Host's own pool +platform_reserved_mem_mb: 16384 + +proxmox_overhead_pct: 5 # hypervisor + host OS overhead — a conservative starting estimate + +measured_lab_cpu_cores: 2 # from infra/scripts/loadtest/sample-resources.sh peak CPU per Lab +measured_lab_mem_mb: 2048 # from the same — peak memory per Lab + +safety_margin_pct: 20 # headroom kept unallocated even at the "recommended" threshold diff --git a/infra/environments/production/gateway.env.template b/infra/environments/production/gateway.env.template new file mode 100644 index 0000000..d86dc60 --- /dev/null +++ b/infra/environments/production/gateway.env.template @@ -0,0 +1,8 @@ +# Production environment template for rn-platform-01's `gateway` service. +# `api`/`provisioner` are Docker Compose service names, not localhost — production runs these +# on a named bridge network (rn-platform-internal), not network_mode: host like dev. + +PLATFORM_API_URL=http://api:4300 +PROVISIONER_URL=http://provisioner:4100 +PROVISIONER_SHARED_SECRET=__REPLACE_same_value_as_platform.env_and_provisioner.env__ +GATEWAY_PORT=4200 diff --git a/infra/environments/production/platform.env.template b/infra/environments/production/platform.env.template new file mode 100644 index 0000000..4670053 --- /dev/null +++ b/infra/environments/production/platform.env.template @@ -0,0 +1,38 @@ +# Production environment template for rn-platform-01's `api` service. +# Copy to platform.env OUTSIDE git (see infra/backup/README.md "secrets" and +# docs/PRODUCTION_DEPLOYMENT.md §10), fill in real values, never commit the filled file. +# infra/.gitignore already excludes *.env (only *.env.template is tracked). + +# ── Postgres ──────────────────────────────────────────────────────────────────────────── +# `postgres`/`provisioner` are Docker Compose service names, not localhost — production runs +# these on a named bridge network (rn-platform-internal), not network_mode: host like dev. +PLATFORM_DB_URL=postgres://rootnode_platform:__REPLACE_WITH_STRONG_PASSWORD__@postgres:5432/rootnode_platform + +# ── Provisioner ───────────────────────────────────────────────────────────────────────── +PROVISIONER_URL=http://provisioner:4100 +PROVISIONER_SHARED_SECRET=__REPLACE_openssl_rand_hex_32__ + +# ── Listen ────────────────────────────────────────────────────────────────────────────── +PLATFORM_API_PORT=4300 + +# ── Sessions (production values — do NOT inherit dev defaults) ──────────────────────────── +STUDENT_SESSION_TTL_HOURS=8 +STUDENT_SESSION_IDLE_HOURS=2 +INSTRUCTOR_SESSION_TTL_HOURS=12 +INSTRUCTOR_SESSION_IDLE_HOURS=2 + +# ── Cookies / proxy — MUST be set for production, unlike dev ────────────────────────────── +# Secure requires the origin to actually be HTTPS end-to-end (see docs/PRODUCTION_NETWORK_SECURITY.md §8). +COOKIE_SECURE=1 +# Only set to 1 once a trusted reverse proxy (Cloudflare/nginx/Caddy, see infra/docker/reverse-proxy/) +# is actually in front of the Gateway — otherwise req.ip becomes spoofable via X-Forwarded-For. +TRUST_PROXY=1 + +# ── Frontend (baked into the platform/api image at build time; listed here for reference) ─ +# FRONTEND_DIST_DIR=/frontend-dist + +# ── Future OIDC (Phase 3A adapter — not wired yet, see docs/PHASE3A_IDENTITY_AND_INSTRUCTOR_ACCESS.md §16) ─ +# OIDC_ISSUER_URL= +# OIDC_CLIENT_ID= +# OIDC_CLIENT_SECRET=__do_not_commit__ +# OIDC_REDIRECT_URL=https://academy.example.com/api/auth/oidc/callback diff --git a/infra/environments/production/postgres.env.template b/infra/environments/production/postgres.env.template new file mode 100644 index 0000000..8a53a22 --- /dev/null +++ b/infra/environments/production/postgres.env.template @@ -0,0 +1,14 @@ +# Production environment template for rn-platform-01's `postgres` service. +# Postgres itself binds only to localhost/the platform-internal Docker network — never exposed +# beyond rn-platform-01, see docs/PRODUCTION_NETWORK_SECURITY.md §6. + +POSTGRES_USER=rootnode_platform +POSTGRES_PASSWORD=__REPLACE_WITH_STRONG_PASSWORD__ +POSTGRES_DB=rootnode_platform + +# Tuning baseline — conservative, meant to be revisited once rn-platform-01's real allocated +# RAM is known (see docs/production-postgres section in docs/PHASE4_PRODUCTION_INFRASTRUCTURE.md +# §11). Applied via infra/docker/postgres/postgresql.conf.d/ — do not hand-edit the container. +# POSTGRES_SHARED_BUFFERS=1GB +# POSTGRES_EFFECTIVE_CACHE_SIZE=3GB +# POSTGRES_MAX_CONNECTIONS=100 diff --git a/infra/environments/production/provisioner-config/README.md b/infra/environments/production/provisioner-config/README.md new file mode 100644 index 0000000..b05f9b6 --- /dev/null +++ b/infra/environments/production/provisioner-config/README.md @@ -0,0 +1,15 @@ +# Production Provisioner config overlay + +Deployed by the `platform` Ansible role to `/etc/rootnode/provisioner-config/` on +rn-platform-01 (`PROVISIONER_CONFIG_DIR`), replacing the dev-only files shipped in +`platform/provisioner/config/`. + +- `lab-hosts.yaml` — points at the real rn-lab-host-01 VM over TLS instead of the dev Unix + socket. TLS paths reference material generated by `infra/scripts/docker-tls/` and deployed by + the `docker-tls` Ansible role — **never checked into this repo** (see + `infra/docker/tls/.gitignore`). +- `resource-policy.yaml` — deliberately still the Phase 1 dev defaults, unchanged, with a + loud comment explaining why. Per the explicit Phase 4 instruction: do not hard-code final + per-Lab CPU/RAM numbers before the real host is benchmarked (`docs/CAPACITY_PLANNING.md`). + Whoever runs the first production load test (`infra/scripts/loadtest/`) updates this file + with real measured numbers, not this deployment. diff --git a/infra/environments/production/provisioner-config/lab-hosts.yaml b/infra/environments/production/provisioner-config/lab-hosts.yaml new file mode 100644 index 0000000..aa45a90 --- /dev/null +++ b/infra/environments/production/provisioner-config/lab-hosts.yaml @@ -0,0 +1,16 @@ +# Production lab_hosts seed — points at the real rn-lab-host-01 VM instead of Phase 1's dev +# Unix socket. Same mechanism the Provisioner has had since Phase 1 +# (platform/provisioner/config/lab-hosts.seed.yaml), just pointed at a real remote endpoint — +# no application code changes required for this, see docs/PRODUCTION_NETWORK_SECURITY.md §7. +# +# {{ lab_host_ip }} is an Ansible template variable — see infra/ansible/group_vars/production.yml. +# Rendered by the `platform` role; the raw file here keeps the placeholder for readability. + +lab_hosts: + - lab_host_id: labhost-01 + display_name: "RootNode Lab Host VM 01 (production)" + endpoint: "tcp://{{ lab_host_ip }}:2376" + tls_ca_path: /etc/rootnode/docker-tls/ca.pem + tls_cert_path: /etc/rootnode/docker-tls/provisioner-client-cert.pem + tls_key_path: /etc/rootnode/docker-tls/provisioner-client-key.pem + enabled: true diff --git a/infra/environments/production/provisioner-config/resource-policy.yaml b/infra/environments/production/provisioner-config/resource-policy.yaml new file mode 100644 index 0000000..89cf9c4 --- /dev/null +++ b/infra/environments/production/provisioner-config/resource-policy.yaml @@ -0,0 +1,25 @@ +# PRODUCTION OVERLAY — placeholder, deliberately unchanged from the Phase 1 dev defaults. +# Do not hand-tune these before the real host exists (docs/CAPACITY_PLANNING.md, Phase 4 §25). +# After the first real load test on rn-lab-host-01 (infra/scripts/loadtest/), replace this +# comment block and the numbers below with actual measured values, then redeploy via the +# `platform` Ansible role (idempotent — re-running just re-renders this file). +# +# Phase 1 resource policy — config-driven, not hard-coded (decision #4). +# Target host class for planning is ~16 cores/32 threads, 128GB RAM, SSD/NVMe. + +default: + cpu_limit: 2.0 # vCPUs + mem_limit_mb: 4096 + pids_limit: 512 + storage_limit_mb: 8192 # only enforced if the host's Docker storage driver supports + # --storage-opt size= (see known-limitations in the SOP) + +idle_threshold_minutes: 30 # past this: log a warning event only +abandoned_threshold_minutes: 120 # past this: reaper destroys for real + +reaper_poll_interval_seconds: 60 + +health_check: + path: /api/health + timeout_ms: 240000 # k3s cold start in a nested container can be slow — matches the + # ~60-attempt wait already in scripts/entrypoint.sh diff --git a/infra/environments/production/provisioner.env.template b/infra/environments/production/provisioner.env.template new file mode 100644 index 0000000..1f41b4b --- /dev/null +++ b/infra/environments/production/provisioner.env.template @@ -0,0 +1,20 @@ +# Production environment template for rn-platform-01's `provisioner` service. +# See platform.env.template for the secrets-handling convention. +# +# Docker-connectivity config (lab_hosts) and resource limits are NOT env vars — they're YAML +# files the Provisioner already loads from PROVISIONER_CONFIG_DIR (Phase 1 design, unchanged). +# Production overlays for both live at infra/environments/production/provisioner-config/ — see +# that directory's README before touching resource-policy.yaml numbers. + +PLATFORM_DB_URL=postgres://rootnode_platform:__REPLACE_WITH_STRONG_PASSWORD__@localhost:5432/rootnode_platform +PROVISIONER_SHARED_SECRET=__REPLACE_same_value_as_platform.env__ +PROVISIONER_PORT=4100 +PROVISIONER_CONFIG_DIR=/etc/rootnode/provisioner-config + +# Real Lab image, tagged (never :latest in production — see infra/docker/IMAGE_STRATEGY.md) +LAB_IMAGE_REF=registry.example.com/rootnode/lab:__GIT_SHA_OR_TAG__ + +# ── Egress proxy on rn-lab-host-01 ───────────────────────────────────────────────────────── +EGRESS_PROXY_PORT=3128 +DNS_RESOLVER=127.0.0.11 +NO_PROXY_CIDRS=10.42.0.0/16,10.43.0.0/16 diff --git a/infra/environments/staging/README.md b/infra/environments/staging/README.md new file mode 100644 index 0000000..a44d392 --- /dev/null +++ b/infra/environments/staging/README.md @@ -0,0 +1,16 @@ +# Staging environment + +Same shape as production (`docs/PRODUCTION_DEPLOYMENT.md` applies to both — pass `-e env=staging` +to the Ansible playbooks), lower stakes: separate database name/port, staging-tagged images, and +`COOKIE_SECURE` only if staging actually sits behind HTTPS. See `docs/CI_CD` section in +`docs/PHASE4_PRODUCTION_INFRASTRUCTURE.md` §18 for how a Gitea Actions build lands here +automatically before a human promotes it to production. + +Staging is designed to run either in the homelab (a couple of low-resource VMs/containers) or, +once the dedicated server exists, as a resource-constrained slice of it — see +`docs/PHASE4_PRODUCTION_INFRASTRUCTURE.md` §23 for the sizing note. It does not require a +second dedicated server. + +Staging acceptance = the same four things a person actually needs before promoting a build: +app deploys, migrations run clean, one or two disposable Labs boot through the Gateway, and an +assessment can be started and completed. See `infra/scripts/staging-smoke-test.sh`. diff --git a/infra/environments/staging/gateway.env.template b/infra/environments/staging/gateway.env.template new file mode 100644 index 0000000..f073285 --- /dev/null +++ b/infra/environments/staging/gateway.env.template @@ -0,0 +1,6 @@ +# Production environment template for rn-platform-01's `gateway` service. + +PLATFORM_API_URL=http://localhost:4300 +PROVISIONER_URL=http://localhost:4100 +PROVISIONER_SHARED_SECRET=__REPLACE_same_value_as_platform.env_and_provisioner.env__ +GATEWAY_PORT=4200 diff --git a/infra/environments/staging/platform.env.template b/infra/environments/staging/platform.env.template new file mode 100644 index 0000000..856d8c3 --- /dev/null +++ b/infra/environments/staging/platform.env.template @@ -0,0 +1,36 @@ +# Production environment template for rn-platform-01's `api` service. +# Copy to platform.env OUTSIDE git (see infra/backup/README.md "secrets" and +# docs/PRODUCTION_DEPLOYMENT.md §10), fill in real values, never commit the filled file. +# infra/.gitignore already excludes *.env (only *.env.template is tracked). + +# ── Postgres ──────────────────────────────────────────────────────────────────────────── +PLATFORM_DB_URL=postgres://rootnode_platform_staging:__REPLACE_WITH_STRONG_PASSWORD__@localhost:5433/rootnode_platform_staging + +# ── Provisioner ───────────────────────────────────────────────────────────────────────── +PROVISIONER_URL=http://localhost:4100 +PROVISIONER_SHARED_SECRET=__REPLACE_openssl_rand_hex_32__ + +# ── Listen ────────────────────────────────────────────────────────────────────────────── +PLATFORM_API_PORT=4300 + +# ── Sessions (production values — do NOT inherit dev defaults) ──────────────────────────── +STUDENT_SESSION_TTL_HOURS=8 +STUDENT_SESSION_IDLE_HOURS=2 +INSTRUCTOR_SESSION_TTL_HOURS=12 +INSTRUCTOR_SESSION_IDLE_HOURS=2 + +# ── Cookies / proxy — MUST be set for production, unlike dev ────────────────────────────── +# Secure requires the origin to actually be HTTPS end-to-end (see docs/PRODUCTION_NETWORK_SECURITY.md §8). +COOKIE_SECURE=1 # unset if staging is not served over HTTPS +# Only set to 1 once a trusted reverse proxy (Cloudflare/nginx/Caddy, see infra/docker/reverse-proxy/) +# is actually in front of the Gateway — otherwise req.ip becomes spoofable via X-Forwarded-For. +TRUST_PROXY=1 + +# ── Frontend (baked into the platform/api image at build time; listed here for reference) ─ +# FRONTEND_DIST_DIR=/frontend-dist + +# ── Future OIDC (Phase 3A adapter — not wired yet, see docs/PHASE3A_IDENTITY_AND_INSTRUCTOR_ACCESS.md §16) ─ +# OIDC_ISSUER_URL= +# OIDC_CLIENT_ID= +# OIDC_CLIENT_SECRET=__do_not_commit__ +# OIDC_REDIRECT_URL=https://academy.example.com/api/auth/oidc/callback diff --git a/infra/environments/staging/postgres.env.template b/infra/environments/staging/postgres.env.template new file mode 100644 index 0000000..8a53a22 --- /dev/null +++ b/infra/environments/staging/postgres.env.template @@ -0,0 +1,14 @@ +# Production environment template for rn-platform-01's `postgres` service. +# Postgres itself binds only to localhost/the platform-internal Docker network — never exposed +# beyond rn-platform-01, see docs/PRODUCTION_NETWORK_SECURITY.md §6. + +POSTGRES_USER=rootnode_platform +POSTGRES_PASSWORD=__REPLACE_WITH_STRONG_PASSWORD__ +POSTGRES_DB=rootnode_platform + +# Tuning baseline — conservative, meant to be revisited once rn-platform-01's real allocated +# RAM is known (see docs/production-postgres section in docs/PHASE4_PRODUCTION_INFRASTRUCTURE.md +# §11). Applied via infra/docker/postgres/postgresql.conf.d/ — do not hand-edit the container. +# POSTGRES_SHARED_BUFFERS=1GB +# POSTGRES_EFFECTIVE_CACHE_SIZE=3GB +# POSTGRES_MAX_CONNECTIONS=100 diff --git a/infra/environments/staging/provisioner-config/lab-hosts.yaml b/infra/environments/staging/provisioner-config/lab-hosts.yaml new file mode 100644 index 0000000..aa45a90 --- /dev/null +++ b/infra/environments/staging/provisioner-config/lab-hosts.yaml @@ -0,0 +1,16 @@ +# Production lab_hosts seed — points at the real rn-lab-host-01 VM instead of Phase 1's dev +# Unix socket. Same mechanism the Provisioner has had since Phase 1 +# (platform/provisioner/config/lab-hosts.seed.yaml), just pointed at a real remote endpoint — +# no application code changes required for this, see docs/PRODUCTION_NETWORK_SECURITY.md §7. +# +# {{ lab_host_ip }} is an Ansible template variable — see infra/ansible/group_vars/production.yml. +# Rendered by the `platform` role; the raw file here keeps the placeholder for readability. + +lab_hosts: + - lab_host_id: labhost-01 + display_name: "RootNode Lab Host VM 01 (production)" + endpoint: "tcp://{{ lab_host_ip }}:2376" + tls_ca_path: /etc/rootnode/docker-tls/ca.pem + tls_cert_path: /etc/rootnode/docker-tls/provisioner-client-cert.pem + tls_key_path: /etc/rootnode/docker-tls/provisioner-client-key.pem + enabled: true diff --git a/infra/environments/staging/provisioner-config/resource-policy.yaml b/infra/environments/staging/provisioner-config/resource-policy.yaml new file mode 100644 index 0000000..89cf9c4 --- /dev/null +++ b/infra/environments/staging/provisioner-config/resource-policy.yaml @@ -0,0 +1,25 @@ +# PRODUCTION OVERLAY — placeholder, deliberately unchanged from the Phase 1 dev defaults. +# Do not hand-tune these before the real host exists (docs/CAPACITY_PLANNING.md, Phase 4 §25). +# After the first real load test on rn-lab-host-01 (infra/scripts/loadtest/), replace this +# comment block and the numbers below with actual measured values, then redeploy via the +# `platform` Ansible role (idempotent — re-running just re-renders this file). +# +# Phase 1 resource policy — config-driven, not hard-coded (decision #4). +# Target host class for planning is ~16 cores/32 threads, 128GB RAM, SSD/NVMe. + +default: + cpu_limit: 2.0 # vCPUs + mem_limit_mb: 4096 + pids_limit: 512 + storage_limit_mb: 8192 # only enforced if the host's Docker storage driver supports + # --storage-opt size= (see known-limitations in the SOP) + +idle_threshold_minutes: 30 # past this: log a warning event only +abandoned_threshold_minutes: 120 # past this: reaper destroys for real + +reaper_poll_interval_seconds: 60 + +health_check: + path: /api/health + timeout_ms: 240000 # k3s cold start in a nested container can be slow — matches the + # ~60-attempt wait already in scripts/entrypoint.sh diff --git a/infra/environments/staging/provisioner.env.template b/infra/environments/staging/provisioner.env.template new file mode 100644 index 0000000..229d0d6 --- /dev/null +++ b/infra/environments/staging/provisioner.env.template @@ -0,0 +1,20 @@ +# Production environment template for rn-platform-01's `provisioner` service. +# See platform.env.template for the secrets-handling convention. +# +# Docker-connectivity config (lab_hosts) and resource limits are NOT env vars — they're YAML +# files the Provisioner already loads from PROVISIONER_CONFIG_DIR (Phase 1 design, unchanged). +# Production overlays for both live at infra/environments/production/provisioner-config/ — see +# that directory's README before touching resource-policy.yaml numbers. + +PLATFORM_DB_URL=postgres://rootnode_platform:__REPLACE_WITH_STRONG_PASSWORD__@localhost:5433/rootnode_platform +PROVISIONER_SHARED_SECRET=__REPLACE_same_value_as_platform.env__ +PROVISIONER_PORT=4100 +PROVISIONER_CONFIG_DIR=/etc/rootnode/provisioner-config + +# Real Lab image, tagged (never :latest in production — see infra/docker/IMAGE_STRATEGY.md) +LAB_IMAGE_REF=registry.example.com/rootnode/lab:staging-__GIT_SHA__ + +# ── Egress proxy on rn-lab-host-01 ───────────────────────────────────────────────────────── +EGRESS_PROXY_PORT=3128 +DNS_RESOLVER=127.0.0.11 +NO_PROXY_CIDRS=10.42.0.0/16,10.43.0.0/16