Fix fresh-install failures in the mirror playbooks (task ordering, SELinux, certbot failure handling)

ansible/combined-mirror/cloudlinux-complete-mirror.service.j2

@@ -6,7 +6,7 @@ Wants=network-online.target
 [Service]
 Type=oneshot
 SuccessExitStatus=24
-ExecStart=/bin/bash -c '/usr/bin/rsync -av --delete {{ cloudlinux_rsync_source }} {{ cloudlinux_mirror_path }}/; /usr/bin/rsync -av --delete {{ swng_rsync_source }} {{ swng_mirror_path }}/'
+ExecStart=/bin/bash -c '/usr/bin/rsync -av --no-owner --no-group --delete {{ cloudlinux_rsync_source }} {{ cloudlinux_mirror_path }}/; /usr/bin/rsync -av --no-owner --no-group --delete {{ swng_rsync_source }} {{ swng_mirror_path }}/'
 ExecStartPost=/usr/bin/python3 /opt/healthcheck/healthcheck_update.py --service sync --field repo --value {{ healthcheck_source_name }} --status OK
 StandardOutput=append:{{ combined_sync_log }}
 StandardError=append:{{ combined_sync_log }}

ansible/combined-mirror/cloudlinux-complete-mirror.timer.j2

@@ -1,6 +1,5 @@
 [Unit]
 Description=Run Complete CloudLinux and SWNG Mirror Sync Every {{ sync_interval_hours }} Hours
-Requires={{ service_name }}.service
 
 [Timer]
 OnCalendar={{ timer_schedule }}

ansible/combined-mirror/cloudlinux-mirror.service.j2

@@ -6,6 +6,6 @@ Wants=network-online.target
 [Service]
 Type=oneshot
 SuccessExitStatus=24
-ExecStart=/usr/bin/rsync -av --delete {{ cloudlinux_rsync_source }} {{ cloudlinux_mirror_path }}/
+ExecStart=/usr/bin/rsync -av --no-owner --no-group --delete {{ cloudlinux_rsync_source }} {{ cloudlinux_mirror_path }}/
 StandardOutput=append:{{ cloudlinux_sync_log }}
 StandardError=append:{{ cloudlinux_sync_log }}

ansible/combined-mirror/cloudlinux-mirror.timer.j2

@@ -1,6 +1,5 @@
 [Unit]
 Description=Run CloudLinux Mirror Sync Every {{ sync_interval_hours }} Hours
-Requires=cloudlinux-mirror.service
 
 [Timer]
 OnCalendar={{ timer_schedule }}

ansible/combined-mirror/playbook.yml

@@ -6,15 +6,6 @@
     - defaults/main.yml
 
   tasks:
-    - name: Check available disk space
-      command: df -h {{ mirror_base_path }}
-      register: disk_space
-      changed_when: false
-
-    - name: Display available disk space
-      debug:
-        msg: "{{ disk_space.stdout_lines }}"
-
     - name: Create CloudLinux mirror directory
       file:
         path: "{{ cloudlinux_mirror_path }}"
@@ -31,14 +22,36 @@
         owner: root
         group: root
 
+    - name: Check available disk space
+      command: df -h {{ mirror_base_path }}
+      register: disk_space
+      changed_when: false
+
+    - name: Display available disk space
+      debug:
+        msg: "{{ disk_space.stdout_lines }}"
+
     - name: Install rsync if not present
       package:
         name: rsync
         state: present
 
+    - name: Detect mirror files not owned by root (from earlier owner-preserving syncs)
+      command: find {{ item }} -not -user root -print -quit
+      loop:
+        - "{{ cloudlinux_mirror_path }}"
+        - "{{ swng_mirror_path }}"
+      register: mirror_nonroot
+      changed_when: false
+
+    - name: Normalize mirror tree ownership to root (one-time migration)
+      command: chown -R root:root {{ item.item }}
+      loop: "{{ mirror_nonroot.results }}"
+      when: item.stdout | length > 0
+
     - name: Perform initial CloudLinux repository sync
       command: >
-        rsync -av --delete
+        rsync -av --no-owner --no-group --delete
         --progress
         --log-file={{ cloudlinux_sync_log }}
         {{ cloudlinux_rsync_source }}
@@ -50,7 +63,7 @@
 
     - name: Perform initial SWNG repository sync
       command: >
-        rsync -av --delete
+        rsync -av --no-owner --no-group --delete
         --progress
         --log-file={{ swng_sync_log }}
         {{ swng_rsync_source }}
@@ -198,15 +211,22 @@
       ansible.builtin.shell: |
         semanage fcontext -a -t public_content_rw_t '/var/www(/.*)?' 2>/dev/null \
           || semanage fcontext -m -t public_content_rw_t '/var/www(/.*)?'
-        restorecon -Fv /var/www {{ healthcheck_file }} {{ healthcheck_json_file }} 2>/dev/null \
-          || restorecon -Fv /var/www
+        restorecon -RFv /var/www {{ healthcheck_file }} {{ healthcheck_json_file }} 2>/dev/null \
+          || restorecon -RFv /var/www
       register: selinux_relabel
       changed_when: "'Relabeled' in selinux_relabel.stdout"
       when:
         - ansible_facts['os_family'] == 'RedHat'
         - ansible_facts['selinux'] is defined
         - ansible_facts['selinux'].status | default('disabled') == 'enabled'
 
+    - name: Allow systemd-run rsync to act as a network client and write mirror data (SELinux enforcing)
+      ansible.builtin.command: setsebool -P rsync_client=1 rsync_anon_write=1
+      when:
+        - ansible_facts['os_family'] == 'RedHat'
+        - ansible_facts['selinux'] is defined
+        - ansible_facts['selinux'].status | default('disabled') == 'enabled'
+
     - name: Generate initial /healthcheck so it is available before first sync completes
       tags: healthcheck
       ansible.builtin.command:
@@ -311,14 +331,20 @@
         state: started
       when: certbot_enabled | bool and certbot_authenticator == 'standalone'
 
+    - name: Check whether the Let's Encrypt certificate exists
+      stat:
+        path: "/etc/letsencrypt/live/{{ mirror_domain }}/fullchain.pem"
+      register: letsencrypt_cert
+      when: certbot_enabled | bool
+
     - name: Create Nginx HTTPS configuration for combined mirror
       tags: healthcheck
       template:
         src: nginx-https.conf.j2
         dest: /etc/nginx/conf.d/combined-mirror-https.conf
         mode: '0644'
       notify: restart nginx
-      when: certbot_enabled | bool
+      when: certbot_enabled | bool and letsencrypt_cert.stat.exists
 
     - name: Update HTTP Nginx configuration to redirect to HTTPS
       tags: healthcheck
@@ -327,7 +353,15 @@
         dest: /etc/nginx/conf.d/combined-mirror.conf
         mode: '0644'
       notify: restart nginx
-      when: certbot_enabled | bool
+      when: certbot_enabled | bool and letsencrypt_cert.stat.exists
+
+    - name: Warn when certificate could not be obtained (HTTPS config skipped)
+      debug:
+        msg: >-
+          Let's Encrypt certificate for {{ mirror_domain }} was not obtained.
+          The HTTPS vhost was NOT deployed so nginx keeps serving over HTTP.
+          Check /var/log/letsencrypt/letsencrypt.log and re-run the playbook.
+      when: certbot_enabled | bool and not letsencrypt_cert.stat.exists
 
     - name: Ensure certbot renew cron job exists
       cron:

ansible/combined-mirror/swng-mirror.service.j2

@@ -6,7 +6,7 @@ Wants=network-online.target
 [Service]
 Type=oneshot
 SuccessExitStatus=24
-ExecStart=/usr/bin/rsync -av --delete {{ swng_rsync_source }} {{ swng_mirror_path }}/
+ExecStart=/usr/bin/rsync -av --no-owner --no-group --delete {{ swng_rsync_source }} {{ swng_mirror_path }}/
 ExecStartPost=/usr/bin/python3 /opt/healthcheck/healthcheck_update.py --service sync --field repo --value {{ healthcheck_source_name }} --status OK
 StandardOutput=append:{{ swng_sync_log }}
 StandardError=append:{{ swng_sync_log }}

ansible/combined-mirror/swng-mirror.timer.j2

@@ -1,6 +1,5 @@
 [Unit]
 Description=Run SWNG Mirror Sync Every {{ sync_interval_hours }} Hours
-Requires=swng-mirror.service
 
 [Timer]
 OnCalendar={{ timer_schedule }}

ansible/combined-mirror/verify.yml

@@ -17,9 +17,9 @@
         msg: "OK: {{ mirror_base_path }} is mounted"
       when: mountpoint_check.rc == 0
 
-    - name: Fail if storage not mounted
-      fail:
-        msg: "ERROR: {{ mirror_base_path }} is not mounted"
+    - name: Warn if storage is not a dedicated mountpoint
+      debug:
+        msg: "WARNING: {{ mirror_base_path }} is not a dedicated mountpoint - the mirror lives on the root filesystem"
       when: mountpoint_check.rc != 0
 
     - name: Check CloudLinux mirror directory exists

ansible/complete-swng-rsync/playbook.yml

@@ -6,6 +6,14 @@
     - defaults/main.yml
 
   tasks:
+    - name: Create mirror directory
+      file:
+        path: "{{ swng_mirror_path }}"
+        state: directory
+        mode: '0755'
+        owner: root
+        group: root
+
     - name: Check available disk space
       command: df -h {{ mirror_base_path }}
       register: disk_space
@@ -15,22 +23,23 @@
       debug:
         msg: "{{ disk_space.stdout_lines }}"
 
-    - name: Create mirror directory
-      file:
-        path: "{{ swng_mirror_path }}"
-        state: directory
-        mode: '0755'
-        owner: root
-        group: root
-
     - name: Install rsync if not present
       package:
         name: rsync
         state: present
 
+    - name: Detect mirror files not owned by root (from earlier owner-preserving syncs)
+      command: find {{ swng_mirror_path }} -not -user root -print -quit
+      register: swng_nonroot
+      changed_when: false
+
+    - name: Normalize mirror tree ownership to root (one-time migration)
+      command: chown -R root:root {{ swng_mirror_path }}
+      when: swng_nonroot.stdout | length > 0
+
     - name: Perform initial SWNG repository sync
       command: >
-        rsync -av --delete
+        rsync -av --no-owner --no-group --delete
         --progress
         --log-file={{ sync_log_file }}
         {{ rsync_source }}
@@ -129,15 +138,22 @@
       ansible.builtin.shell: |
         semanage fcontext -a -t public_content_rw_t '/var/www(/.*)?' 2>/dev/null \
           || semanage fcontext -m -t public_content_rw_t '/var/www(/.*)?'
-        restorecon -Fv /var/www {{ healthcheck_file }} {{ healthcheck_json_file }} 2>/dev/null \
-          || restorecon -Fv /var/www
+        restorecon -RFv /var/www {{ healthcheck_file }} {{ healthcheck_json_file }} 2>/dev/null \
+          || restorecon -RFv /var/www
       register: selinux_relabel
       changed_when: "'Relabeled' in selinux_relabel.stdout"
       when:
         - ansible_facts['os_family'] == 'RedHat'
         - ansible_facts['selinux'] is defined
         - ansible_facts['selinux'].status | default('disabled') == 'enabled'
 
+    - name: Allow systemd-run rsync to act as a network client and write mirror data (SELinux enforcing)
+      ansible.builtin.command: setsebool -P rsync_client=1 rsync_anon_write=1
+      when:
+        - ansible_facts['os_family'] == 'RedHat'
+        - ansible_facts['selinux'] is defined
+        - ansible_facts['selinux'].status | default('disabled') == 'enabled'
+
     - name: Generate initial /healthcheck so it is available before first sync completes
       tags: healthcheck
       ansible.builtin.command:
@@ -242,14 +258,20 @@
         state: started
       when: certbot_enabled | bool and certbot_authenticator == 'standalone'
 
+    - name: Check whether the Let's Encrypt certificate exists
+      stat:
+        path: "/etc/letsencrypt/live/{{ mirror_domain }}/fullchain.pem"
+      register: letsencrypt_cert
+      when: certbot_enabled | bool
+
     - name: Create Nginx HTTPS configuration for SWNG mirror
       tags: healthcheck
       template:
         src: nginx-https.conf.j2
         dest: /etc/nginx/conf.d/swng-mirror-https.conf
         mode: '0644'
       notify: restart nginx
-      when: certbot_enabled | bool
+      when: certbot_enabled | bool and letsencrypt_cert.stat.exists
 
     - name: Update HTTP Nginx configuration to redirect to HTTPS
       tags: healthcheck
@@ -258,7 +280,15 @@
         dest: /etc/nginx/conf.d/swng-mirror.conf
         mode: '0644'
       notify: restart nginx
-      when: certbot_enabled | bool
+      when: certbot_enabled | bool and letsencrypt_cert.stat.exists
+
+    - name: Warn when certificate could not be obtained (HTTPS config skipped)
+      debug:
+        msg: >-
+          Let's Encrypt certificate for {{ mirror_domain }} was not obtained.
+          The HTTPS vhost was NOT deployed so nginx keeps serving over HTTP.
+          Check /var/log/letsencrypt/letsencrypt.log and re-run the playbook.
+      when: certbot_enabled | bool and not letsencrypt_cert.stat.exists
 
     - name: Ensure certbot renew cron job exists
       cron:

ansible/complete-swng-rsync/swng-mirror.service.j2

@@ -6,7 +6,7 @@ Wants=network-online.target
 [Service]
 Type=oneshot
 SuccessExitStatus=24
-ExecStart=/usr/bin/rsync -av --delete {{ rsync_source }} {{ swng_mirror_path }}/
+ExecStart=/usr/bin/rsync -av --no-owner --no-group --delete {{ rsync_source }} {{ swng_mirror_path }}/
 ExecStartPost=/usr/bin/python3 /opt/healthcheck/healthcheck_update.py --service sync --field repo --value {{ healthcheck_source_name }} --status OK
 StandardOutput=append:{{ sync_log_file }}
 StandardError=append:{{ sync_log_file }}

ansible/complete-swng-rsync/swng-mirror.timer.j2

@@ -1,6 +1,5 @@
 [Unit]
 Description=Run Complete SWNG Mirror Sync Every {{ sync_interval_hours }} Hours
-Requires=swng-mirror.service
 
 [Timer]
 OnCalendar={{ timer_schedule }}

ansible/complete-swng-rsync/verify.yml

@@ -17,9 +17,9 @@
         msg: "OK: {{ mirror_base_path }} is mounted"
       when: mountpoint_check.rc == 0
 
-    - name: Fail if storage not mounted
-      fail:
-        msg: "ERROR: {{ mirror_base_path }} is not mounted"
+    - name: Warn if storage is not a dedicated mountpoint
+      debug:
+        msg: "WARNING: {{ mirror_base_path }} is not a dedicated mountpoint - the mirror lives on the root filesystem"
       when: mountpoint_check.rc != 0
 
     - name: Check mirror directory exists