diff --git a/roles/pfsense_upgrade/tasks/carp.yml b/roles/pfsense_upgrade/tasks/carp.yml index bfdfd42..2714ae8 100644 --- a/roles/pfsense_upgrade/tasks/carp.yml +++ b/roles/pfsense_upgrade/tasks/carp.yml @@ -1,4 +1,3 @@ ---- # roles/pfsense_upgrade/tasks/carp.yml # Handles CARP/HA awareness during upgrades. # Only runs when ha_peer is defined. @@ -15,19 +14,26 @@ register: _peer_reachable failed_when: _peer_reachable.rc != 0 when: ha_role == 'backup' + ### Not altered ### -- name: "[CARP/backup] Check current CARP state on this node" - ansible.builtin.raw: > - php -r "require_once('/etc/inc/interfaces.inc'); - \$carp = get_carp_status(); - echo \$carp;" +# --- REWRITTEN: Check CARP state on backup using native pfSense function --- +- name: "[CARP/backup] Verify backup is in BACKUP state (all VIPs)" + ansible.builtin.raw: | + php -r 'require_once("/etc/inc/config.inc"); require_once("/etc/inc/interfaces.inc"); $ready = true; foreach(config_get_path("virtualip/vip", []) as $vip) { if ($vip["mode"] != "carp") continue; if (get_carp_interface_status("_vip" . $vip["uniqid"]) != "BACKUP") { $ready = false; break; } } echo $ready ? "BACKUP_READY" : "NOT_READY";' register: _backup_carp_state changed_when: false when: ha_role == 'backup' +- name: "[CARP/backup] Fail if backup is not fully in BACKUP state" + ansible.builtin.fail: + msg: "Backup node {{ inventory_hostname }} is not in BACKUP state for all CARP VIPs. State: {{ _backup_carp_state.stdout }}" + when: + - ha_role == 'backup' + - _backup_carp_state.stdout != "BACKUP_READY" + - name: "[CARP/backup] Display CARP state" ansible.builtin.debug: - msg: "CARP state on {{ inventory_hostname }} (backup): {{ _backup_carp_state.stdout | trim }}" + msg: "CARP state on {{ inventory_hostname }} (backup): All VIPs are BACKUP" when: ha_role == 'backup' # --------------------------------------------------------------------------- @@ -43,6 +49,7 @@ when: - ha_role == 'primary' - ha_peer is defined + ### Not altered ### # --- Step 2: Verify peer is on the upgraded version --- - name: "[CARP/primary] Read version on backup peer {{ ha_peer }}" @@ -53,6 +60,7 @@ when: - ha_role == 'primary' - ha_peer is defined + ### Not altered ### - name: "[CARP/primary] Set peer version fact" ansible.builtin.set_fact: @@ -60,6 +68,7 @@ when: - ha_role == 'primary' - ha_peer is defined + ### Not altered ### - name: "[CARP/primary] Fail if backup peer is not on a newer version than primary" ansible.builtin.fail: @@ -71,35 +80,33 @@ - ha_role == 'primary' - ha_peer is defined - ha_peer_version == pfsense_current_version + ### Not altered ### -# --- Step 3: Verify backup peer is in MASTER CARP state --- -- name: "[CARP/primary] Check CARP state on backup peer {{ ha_peer }}" - ansible.builtin.raw: > - ifconfig | grep -o 'carp: [A-Z]*' | awk '{print $2}' | sort -u - delegate_to: "{{ ha_peer }}" +# --- Step 3: Verify backup peer is in MASTER CARP state (rewritten) --- +- name: "[CARP/primary] Verify backup peer is MASTER for all CARP VIPs" + ansible.builtin.raw: | + ssh {{ ha_peer }} 'php -r "require_once(\"/etc/inc/config.inc\"); require_once(\"/etc/inc/interfaces.inc\"); $all_master = true; foreach(config_get_path(\"virtualip/vip\", []) as $vip) { if ($vip[\"mode\"] != \"carp\") continue; if (get_carp_interface_status(\"_vip\" . $vip[\"uniqid\"]) != \"MASTER\") { $all_master = false; break; } } echo $all_master ? \"ALL_MASTER\" : \"NOT_ALL_MASTER\";"' register: _peer_carp_state_raw changed_when: false when: - ha_role == 'primary' - ha_peer is defined -- name: "[CARP/primary] Fail if backup peer is not MASTER" +- name: "[CARP/primary] Fail if backup peer is not MASTER for all VIPs" ansible.builtin.fail: msg: > - Backup peer {{ ha_peer }} CARP state is '{{ _peer_carp_state_raw.stdout | trim }}'. - Expected MASTER. Resolve CARP state before upgrading the primary. + Backup peer {{ ha_peer }} is not MASTER for all CARP VIPs. + State: {{ _peer_carp_state_raw.stdout }}. + Resolve CARP state before upgrading the primary. when: - ha_role == 'primary' - ha_peer is defined - - "'MASTER' not in _peer_carp_state_raw.stdout" + - _peer_carp_state_raw.stdout != "ALL_MASTER" -# --- Step 4: Force demotion of primary --- -- name: "[CARP/primary] Force CARP demotion on this node (set advskew to 254)" - ansible.builtin.raw: > - sudo php -r " - require_once('/etc/inc/interfaces.inc'); - interfaces_carp_set_maintenancemode(true); - " +# --- Step 4: Force demotion of primary (rewritten for tcsh safety) --- +- name: "[CARP/primary] Force CARP demotion on this node (enter maintenance mode)" + ansible.builtin.raw: | + php -r 'require_once("/etc/inc/interfaces.inc"); interfaces_carp_set_maintenancemode(true);' register: _carp_demotion changed_when: true when: @@ -108,13 +115,13 @@ - name: "[CARP/primary] Wait for CARP failover to settle" ansible.builtin.pause: - seconds: 15 + seconds: 30 + ### Not altered ### -# --- Step 5: Re-verify backup peer has taken MASTER --- -- name: "[CARP/primary] Re-check CARP state on backup peer after demotion" - ansible.builtin.raw: > - ifconfig | grep -o 'carp: [A-Z]*' | awk '{print $2}' | sort -u - delegate_to: "{{ ha_peer }}" +# --- Step 5: Re-verify backup peer has taken MASTER (rewritten) --- +- name: "[CARP/primary] Re-check backup peer MASTER state after demotion" + ansible.builtin.raw: | + ssh {{ ha_peer }} 'php -r "require_once(\"/etc/inc/config.inc\"); require_once(\"/etc/inc/interfaces.inc\"); $all_master = true; foreach(config_get_path(\"virtualip/vip\", []) as $vip) { if ($vip[\"mode\"] != \"carp\") continue; if (get_carp_interface_status(\"_vip\" . $vip[\"uniqid\"]) != \"MASTER\") { $all_master = false; break; } } echo $all_master ? \"ALL_MASTER\" : \"NOT_ALL_MASTER\";"' register: _peer_carp_recheck changed_when: false when: @@ -124,32 +131,30 @@ - name: "[CARP/primary] Fail if backup peer did not take MASTER after demotion" ansible.builtin.fail: msg: > - Backup peer {{ ha_peer }} did not take MASTER after primary demotion. - CARP state is '{{ _peer_carp_recheck.stdout | trim }}'. + Backup peer {{ ha_peer }} did not become MASTER for all VIPs after primary demotion. + State: {{ _peer_carp_recheck.stdout }}. Investigate CARP before proceeding — primary has been demoted but backup is not MASTER. when: - ha_role == 'primary' - ha_peer is defined - - "'MASTER' not in _peer_carp_recheck.stdout" + - _peer_carp_recheck.stdout != "ALL_MASTER" - name: "[CARP/primary] CARP demotion confirmed — backup is MASTER, safe to upgrade primary" ansible.builtin.debug: msg: > - {{ ha_peer }} is MASTER. {{ inventory_hostname }} is demoted. + {{ ha_peer }} is MASTER for all VIPs. {{ inventory_hostname }} is demoted. Proceeding with primary upgrade. when: - ha_role == 'primary' - ha_peer is defined + ### Not altered ### # --------------------------------------------------------------------------- # Post-upgrade: restore CARP on primary and verify state # --------------------------------------------------------------------------- -- name: "[CARP/primary] Re-enable CARP maintenance mode off (restore advskew)" - ansible.builtin.raw: > - sudo php -r " - require_once('/etc/inc/interfaces.inc'); - interfaces_carp_set_maintenancemode(false); - " +- name: "[CARP/primary] Re-enable CARP (exit maintenance mode)" + ansible.builtin.raw: | + php -r 'require_once("/etc/inc/interfaces.inc"); interfaces_carp_set_maintenancemode(false);' register: _carp_restore changed_when: true when: @@ -159,10 +164,11 @@ - name: "[CARP/primary] Wait for CARP state to stabilize after restore" ansible.builtin.pause: seconds: 20 + ### Not altered ### -- name: "[CARP/primary] Verify primary has reclaimed MASTER" - ansible.builtin.raw: > - ifconfig | grep -o 'carp: [A-Z]*' | awk '{print $2}' | sort -u +- name: "[CARP/primary] Verify primary has reclaimed MASTER for all VIPs" + ansible.builtin.raw: | + php -r 'require_once("/etc/inc/config.inc"); require_once("/etc/inc/interfaces.inc"); $all_master = true; foreach(config_get_path("virtualip/vip", []) as $vip) { if ($vip["mode"] != "carp") continue; if (get_carp_interface_status("_vip" . $vip["uniqid"]) != "MASTER") { $all_master = false; break; } } echo $all_master ? "ALL_MASTER" : "NOT_ALL_MASTER";' register: _primary_carp_final changed_when: false when: @@ -172,17 +178,17 @@ - name: "[CARP/primary] Warn if primary did not reclaim MASTER" ansible.builtin.debug: msg: > - WARNING: Primary CARP state is '{{ _primary_carp_final.stdout | trim }}' — expected MASTER. + WARNING: Primary CARP state is '{{ _primary_carp_final.stdout }}' — expected ALL_MASTER. This may resolve on its own. Check CARP status on both nodes manually. when: - ha_role == 'primary' - ha_peer is defined - - "'MASTER' not in _primary_carp_final.stdout" + - _primary_carp_final.stdout != "ALL_MASTER" - name: "[CARP/primary] CARP state confirmed restored" ansible.builtin.debug: - msg: "{{ inventory_hostname }} has reclaimed MASTER. HA pair is fully operational." + msg: "{{ inventory_hostname }} has reclaimed MASTER for all VIPs. HA pair is fully operational." when: - ha_role == 'primary' - ha_peer is defined - - "'MASTER' in _primary_carp_final.stdout" \ No newline at end of file + - _primary_carp_final.stdout == "ALL_MASTER" \ No newline at end of file