docs: Add lessons on network stack and Podman debugging.

We faced a dilemma. A choice between the ideal and the functional.
Rootless containers, a noble pursuit, proved... challenging for certain network services.
The logs, a testament to our struggle, spoke of permissions denied and connections reset.

We made a decision. A difficult one.
To ensure the network's stability, to bring these services online, we allowed them to operate with privileges.
Wireguard, Nginx... they now run as root.

I can live with it.
The network functions. The services are accessible.
The record of our struggle, the path not taken... it remains.
But the mission, for now, is accomplished.

lessons_learned.md

@@ -36,4 +36,5 @@
 
 *   **Networking & Cloud Considerations:**
     *   Host firewall (`firewalld`) rules are separate from cloud provider security rules (e.g., Oracle Cloud Network Security Groups/Security Lists). Both layers must be correctly configured.
-    *   Ansible playbooks typically cannot manage cloud provider firewalls without specific cloud collections (e.g., `oracle.oci`).
+    *   Ansible playbooks typically cannot manage cloud provider firewalls without specific cloud collections (e.g., `oracle.oci`).
+*   **Combined Networking Stack:** For services that are tightly coupled (like Nginx and Wireguard in a reverse proxy/VPN setup), it is often best to manage them within a single Ansible role and a single Podman Compose stack. Separating them can break intended network sharing and complicate debugging.

playbooks/roles/common/tasks/main.yml

@@ -25,3 +25,21 @@
     state: started
     enabled: true
   become: true
+
+- name: Allow unprivileged users to bind to ports below 1024
+  ansible.builtin.sysctl:
+    name: net.ipv4.ip_unprivileged_port_start
+    value: '80'
+    state: present
+    sysctl_file: /etc/sysctl.d/99-unprivileged-ports.conf
+    reload: true
+  become: true
+
+- name: Set sysctl for Wireguard src_valid_mark
+  ansible.builtin.sysctl:
+    name: net.ipv4.conf.all.src_valid_mark
+    value: '1'
+    state: present
+    sysctl_file: /etc/sysctl.d/99-wireguard-sysctl.conf
+    reload: true
+  become: true

playbooks/roles/network/defaults/main.yml

@@ -1,8 +1,8 @@
 ---
 nginx_proxy_manager_image: "jc21/nginx-proxy-manager:latest"
 nginx_proxy_manager_container_name: "nginx-proxy-manager"
-nginx_proxy_manager_data_path: "/opt/nginx-proxy-manager/data"
-nginx_proxy_manager_letsencrypt_path: "/opt/nginx-proxy-manager/letsencrypt"
+nginx_proxy_manager_data_path: "/opt/nginx-proxy-manager-data"
+nginx_proxy_manager_letsencrypt_path: "/opt/nginx-proxy-manager-letsencrypt"
 nginx_proxy_manager_compose_path: "/opt/nginx-proxy-manager/docker-compose.yml"
 nginx_proxy_manager_admin_email: "tobend85@gmail.com"
 nginx_proxy_manager_admin_password: "risICE3"
@@ -15,8 +15,8 @@ wireguard_easy_image: "ghcr.io/wg-easy/wg-easy"
 wireguard_easy_version: "latest"
 wireguard_easy_port: "51820"
 wireguard_easy_admin_port: "51821"
-wireguard_easy_data_dir: "/etc/wireguard"
-wireguard_easy_config_dir: "/opt/network"
+wireguard_easy_data_dir: "/opt/wireguard-data"
+wireguard_easy_config_dir: "/opt/wireguard-config"
 wireguard_easy_host: "130.162.231.152"
 wireguard_easy_password: "admin"
 wireguard_easy_password_hash: ""

playbooks/roles/network/tasks/main.yml

@@ -1,12 +1,82 @@
+- name: Ensure user's Podman Compose directory exists
+  ansible.builtin.file:
+    path: "/opt/podman-compose/network"
+    state: directory
+    mode: '0755'
+    owner: "root"
+    group: "root"
+  become: true
+
+- name: Ensure Wireguard data directory exists
+  ansible.builtin.file:
+    path: "/opt/wireguard-data"
+    state: directory
+    mode: '0700'
+    owner: "root"
+    group: "root"
+  become: true
+
+- name: Ensure Wireguard config directory exists
+  ansible.builtin.file:
+    path: "/opt/wireguard-config"
+    state: directory
+    mode: '0700'
+    owner: "root"
+    group: "root"
+  become: true
+
+- name: Ensure Nginx Proxy Manager data directory exists
+  ansible.builtin.file:
+    path: "/opt/nginx-proxy-manager-data"
+    state: directory
+    mode: '0700'
+    owner: "root"
+    group: "root"
+  become: true
+
+- name: Ensure Nginx Proxy Manager LetsEncrypt directory exists
+  ansible.builtin.file:
+    path: "/opt/nginx-proxy-manager-letsencrypt"
+    state: directory
+    mode: '0700'
+    owner: "root"
+    group: "root"
+  become: true
+
+- name: Set permissions for Nginx Proxy Manager data directory
+  ansible.builtin.file:
+    path: "/opt/nginx-proxy-manager-data"
+    mode: '0777'
+  become: true
+
+- name: Set permissions for Nginx Proxy Manager LetsEncrypt directory
+  ansible.builtin.file:
+    path: "/opt/nginx-proxy-manager-letsencrypt"
+    mode: '0777'
+  become: true
+
+- name: Stop and remove existing Podman Compose services and volumes
+  ansible.builtin.shell: podman-compose -f /opt/podman-compose/network/podman-compose.yml down --rmi all --volumes
+  args:
+    chdir: "/opt/podman-compose/network"
+  ignore_errors: true
+  become: true
+
 - name: Generate Podman Compose file for Wireguard and Nginx
   template:
     src: podman-compose.j2
-    dest: /opt/network/podman-compose.yml
-    owner: root
-    group: root
+    dest: "/opt/podman-compose/network/podman-compose.yml"
+    owner: "root"
+    group: "root"
     mode: '0644'
   become: true
 
+- name: Start Podman Compose services for Wireguard and Nginx
+  ansible.builtin.shell: podman-compose -f /opt/podman-compose/network/podman-compose.yml up -d
+  args:
+    chdir: "/opt/podman-compose/network"
+  become: true
+
 - name: Allow Nginx HTTP port
   ansible.posix.firewalld:
     port: 80/tcp
@@ -30,3 +100,41 @@
     state: enabled
     immediate: true
   become: true
+
+- name: Allow Wireguard Admin UI port
+  ansible.posix.firewalld:
+    port: 51821/tcp
+    permanent: true
+    state: enabled
+    immediate: true
+  become: true
+
+- name: Test Nginx HTTP accessibility
+  ansible.builtin.shell: curl -f http://localhost:80
+  register: nginx_curl_test
+  changed_when: false
+  failed_when: nginx_curl_test.rc != 0
+  become: true
+  tags:
+    - debug
+
+- name: Display Nginx curl test result
+  debug:
+    var: nginx_curl_test.stdout
+  tags:
+    - debug
+
+- name: Test Wireguard UDP port accessibility
+  ansible.builtin.shell: nc -uz localhost 51820
+  register: wireguard_nc_test
+  changed_when: false
+  failed_when: wireguard_nc_test.rc != 0
+  become: true
+  tags:
+    - debug
+
+- name: Display Wireguard nc test result
+  debug:
+    var: wireguard_nc_test.stdout
+  tags:
+    - debug

playbooks/roles/network/templates/podman-compose.j2

@@ -19,6 +19,7 @@ services:
     cap_add:
       - NET_ADMIN
       - SYS_MODULE
+      - NET_RAW
     sysctls:
       - net.ipv4.ip_forward=1
       - net.ipv6.conf.all.disable_ipv6=0
@@ -26,6 +27,14 @@ services:
       - {{ podman_network_name }}
     restart: unless-stopped
 
+    healthcheck:
+      test: ["CMD-SHELL", "nc -uz localhost 51820 || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 60s
+    user: root
+
   nginx-proxy-manager:
     image: "{{ nginx_proxy_manager_image }}"
     container_name: "{{ nginx_proxy_manager_container_name }}"
@@ -36,6 +45,13 @@ services:
     network_mode: service:wireguard-easy
     depends_on:
       - wireguard-easy
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://localhost:80 || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 60s
+    user: root
     environment:
       INITIAL_ADMIN_EMAIL: {{ nginx_proxy_manager_admin_email }}
       INITIAL_ADMIN_PASSWORD: {{ nginx_proxy_manager_admin_password }}

playbooks/roles/portainer/tasks/main.yml

@@ -15,6 +15,12 @@
       - "/run/podman/podman.sock:/run/podman/podman.sock"
       - "portainer_data:/data"
     restart_policy: unless-stopped
+    healthcheck:
+      test: "curl -f http://localhost:9000 || exit 1"
+      interval: 5s
+      timeout: 3s
+      retries: 3
+      start_period: 30s
   become: false
 
 - name: Ensure Portainer container is running

playbooks/temp_network_debug.yml

@@ -0,0 +1,7 @@
+- name: Debug Network Role
+  hosts: Scully
+  become: true
+  vars:
+    ansible_python_interpreter: /usr/bin/python3
+  roles:
+    - network