all: Complete overhaul with new Interactive Mode, variable fragments, publishing automation and updated playbooks

.ansible-lint

@@ -5,7 +5,9 @@ exclude_paths:
   - .cache/
   - .github/
   #- docs/
-  #- roles/sap_vm_provision
+  # Ansible requirements are not recognized as valid yaml file
+  - deploy_scenarios/.*/ansible_requirements\.yml$
+  - common_fragments/vars/ansible_requirements.yml
 
 enable_list:
   - yaml
@@ -32,7 +34,7 @@ skip_list:
   # Used for /etc/hosts edits
   - no-tabs
 
-  # Incorrectly shows error for nested {%- endfor -%} already correct idententation
+  # Incorrectly shows error for nested {%- endfor -%} already correct indent
   - jinja[spacing]
 
   # Shell commands are required
@@ -45,7 +47,7 @@ skip_list:
   - yaml[empty-lines]
 
   # Ignore ha_cluster vars used within sap_ha_pacemaker_cluster
-  - var-naming[no-role-prefix]
+  # - var-naming[no-role-prefix]
 
   # Ignore dynamic variable population for Scale-Out
   - var-naming[no-jinja]

.github/workflows/ansible-lint.yml

@@ -1,13 +1,19 @@
 ---
 
-# Workflow for ansible-lint of the collection
-
-name: ansible-lint of all Ansible Playbooks for SAP
+name: ansible-lint of the ansible playbooks for sap
 
 on:
+  # Runs at 10:10 on every Monday
   schedule:
     - cron: '10 10 * * 1'
 
+  # Runs on push to dev and main branches
+  push:
+    branches:
+      - dev
+      - main
+
+  # Enable manual start button
   workflow_dispatch:
 
 jobs:
@@ -17,6 +23,8 @@ jobs:
     steps:
       - name: Check out the code
         uses: actions/checkout@v4
+        with:
+          path: repo_playbooks
 
       - name: Set up Python 3
         uses: actions/setup-python@v5
@@ -25,14 +33,39 @@ jobs:
 
       - name: Install dependencies for Python
         run: |
-          pip3 install ansible==9.1.0
-          pip3 install ansible-compat==4.1.10
-          pip3 install ansible-core==2.16.2
-          pip3 install ansible-lint==6.22.1
+          pip3 install --upgrade pip
+          pip3 install ansible==11.4.0
+          pip3 install ansible-compat==4.1.11
+          pip3 install ansible-core==2.18.4
+          pip3 install ansible-lint==25.2.1
 
-      - name: Install dependencies for Ansible
-        run: ansible-galaxy collection install -r /home/runner/work/ansible.playbooks_for_sap/ansible.playbooks_for_sap/deploy_scenarios/sap_hana/ansible_requirements.yml
+        # dev branch is using shared fragments
+      - name: Install dependencies for Ansible (dev)
+        working-directory: repo_playbooks
+        if: github.ref == 'refs/heads/dev'
+        run: |
+          ansible-galaxy collection install -r ./common_fragments/ansible_requirements.yml
+
+        # main branch is using:
+        # - deploy_scenarios/sap_hana if file exists
+        # - else first scenario with ansible_requirements.yml
+      - name: Install dependencies for Ansible (main)
+        working-directory: repo_playbooks
+        if: github.ref == 'refs/heads/main'
+        run: |
+          if [ -f ./deploy_scenarios/sap_hana/ansible_requirements.yml ]; then
+            ansible-galaxy collection install -r ./deploy_scenarios/sap_hana/ansible_requirements.yml
+          else
+            first_file=$(find ./deploy_scenarios/ -name ansible_requirements.yml -print -quit)
+            if [ -n "$first_file" ]; then
+              echo "Using $first_file"
+              ansible-galaxy collection install -r "$first_file"
+            else
+              echo "No ansible_requirements.yml found in deploy_scenarios"
+              exit 1
+            fi
+          fi
 
       - name: Run ansible-lint
-        working-directory: /home/runner/work/ansible.playbooks_for_sap/
+        working-directory: repo_playbooks
         run: ansible-lint

.github/workflows/codespell.yml

@@ -1,21 +1,31 @@
+---
 name: CodeSpell
 
 on:
+  # Runs on push and pull request to dev
   push:
     branches:
       - dev
   pull_request:
     branches:
       - dev
 
+  # Enable manual start button
+  workflow_dispatch:
+
 jobs:
   codespell:
     name: Check for spelling errors
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Check out the code
+        uses: actions/checkout@v4
+        with:
+          path: repo_playbooks
+
       - uses: codespell-project/actions-codespell@v2
         with:
+          path: repo_playbooks
           # lowercase only
           ignore_words_list: aas,iterm,daa

.github/workflows/publish-stage-to-main.yml

@@ -0,0 +1,168 @@
+---
+name: Publish Stage branch to Main
+
+on:
+  # Runs on merged pull_request: to stage branch
+  pull_request:
+    branches:
+      - stage
+    types:
+      - closed
+
+  # Enable manual start button
+  workflow_dispatch:
+
+jobs:
+  sync:
+    name: 'Publish Stage branch to Main'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout branch stage
+        uses: actions/checkout@v4
+        with:
+          path: repo_playbooks_stage
+          ref: stage
+
+      - name: Checkout branch main
+        uses: actions/checkout@v4
+        with:
+          path: repo_playbooks_main
+          ref: main
+
+      - name: Remove internal scenarios from publishing
+        working-directory: repo_playbooks_stage
+        run: |
+          shopt -s nullglob # Ensures that empty glob results are empty
+
+          # Loop through all deploy_scenarios and supported special_actions
+          for dir in ./deploy_scenarios/* ./special_actions/*; do
+            extravars_file="$dir/ansible_extravars.yml"
+
+            # Check if the ansible_extravars.yml file exists
+            if [ -f "$extravars_file" ]; then
+              # Check if the file contains "sap_playbook_internal: true"
+              if grep -q "sap_playbook_internal: true" "$extravars_file"; then
+                rm -rf "$dir"
+              fi  # string is present
+            fi  # extravars file exists
+          done
+
+          shopt -u nullglob # Revert nullglob to its default behavior
+
+      - name: Copy shared files to scenarios
+        working-directory: repo_playbooks_stage
+        run: |
+          shopt -s nullglob  # Ensures that empty results are empty
+
+          # Update path in interactive tasks
+          sed -i 's/..\/..\/common_fragments\/tasks\//optional\//' ./common_fragments/tasks/interactive/*.yml ./common_fragments/tasks/interactive/*/*.yml
+
+          # Loop through all deploy_scenarios and supported special_actions
+          for dir in ./deploy_scenarios/* ./special_actions/*; do
+            # Copy interactive tasks only if optional/ansible_extravars_interactive.yml is present.
+            interactive_vars="$dir"/optional/ansible_extravars_interactive.yml
+            if [ -f "$interactive_vars" ]; then
+              # Copy tasks
+              cp -r ./common_fragments/tasks/* $dir/optional/
+
+              # Update path to interactive tasks in playbooks
+              sed -i 's/..\/..\/common_fragments\/tasks\//optional\//' $dir/ansible_playbook.yml
+            fi  # interactive_vars exists
+
+            # Copy ansible_requirements if it does not exist
+            ansible_requirements="$dir"/ansible_requirements.yml
+            if [ ! -f "$ansible_requirements" ]; then
+              cp ./common_fragments/ansible_requirements.yml $dir/
+            fi  # ansible_requirements not exists
+          done
+
+          shopt -u nullglob
+
+      - name: Append shared fragments to deploy_scenarios files
+        working-directory: repo_playbooks_stage
+        run: |
+          # Loop through all deploy_scenarios and supported special_actions
+          for dir in ./deploy_scenarios/* ./special_actions/*; do
+
+            # Loop through all valid extravars files
+            for file in "$dir"/ansible_extravars_*_base.yml; do
+              if [[ -f "$file" ]]; then
+                # Extract platform name using basename of file
+                platform=$(basename "$file" | sed 's/ansible_extravars_\(.*\)_base\.yml/\1/')
+
+                # Define files for appending
+                fragment_vars="common_fragments/vars/platform_vars_${platform}.yml"
+                fragment_images="common_fragments/vars/platform_images_${platform}.yml"
+                scenario_spec="$dir"/ansible_extravars_"$platform"_spec.yml
+                interactive_vars="$dir"/optional/ansible_extravars_interactive.yml
+
+                # Append shared platform vars fragment when it exists
+                if [ -f "$fragment_vars" ]; then
+                  sed 's/^---/\n/g' "$fragment_vars" >> "$file"
+                fi  # if fragment_vars exists
+
+                # Append scenario spec vars when it exists
+                # Spec is also appended to interactive vars
+                if [ -f "$scenario_spec" ]; then
+                  sed 's/^---/\n/g' "$scenario_spec" >> "$file"
+                  if [ -f "$interactive_vars" ]; then
+                    sed 's/^---/\n/g' "$scenario_spec" >> "$interactive_vars"
+                  fi  # interactive_vars exists
+                  rm -f "$scenario_spec"
+                fi  # if scenario_spec exists
+
+                # Append shared platform image fragment when it exists
+                # Images are also appended to interactive vars
+                if [ -f "$fragment_images" ]; then
+                  sed 's/^---/\n/g' "$fragment_images" >> "$file"
+                  if [ -f "$interactive_vars" ]; then
+                    sed 's/^---/\n/g' "$fragment_images" >> "$interactive_vars"
+                  fi  # interactive_vars exists
+                fi  # if fragment_images exists
+
+                # Remove base files
+                new_file="${file%_base.yml}.yml"
+                mv "$file" "$new_file"
+              fi  # if file exists
+
+            done  # file loop
+          done  # dir loop
+
+          # Remove fragments after copying them
+          git rm -rf ./common_fragments
+
+      - name: Synchronize main with transformed stage
+        run: |
+          rsync --archive --delete --exclude '.git' repo_playbooks_stage/ repo_playbooks_main/
+
+
+      - name: Check for changes
+        working-directory: repo_playbooks_main
+        id: check_changes
+        run: |
+          git add .
+          if [[ -n "$(git status --porcelain)" ]]; then
+            echo "::set-output name=has_changes::true"
+          else
+            echo "::set-output name=has_changes::false"
+          fi
+
+      - name: Commit changes
+        working-directory: repo_playbooks_main
+        if: steps.check_changes.outputs.has_changes == 'true'
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            PR_NUMBER=$(echo ${{ github.event.pull_request.number }})
+          else
+            PR_NUMBER="manual"
+          fi
+          git commit -m "Merge changes from stage to main branch (PR #$PR_NUMBER)."
+
+      - name: Push changes
+        working-directory: repo_playbooks_main
+        if: steps.check_changes.outputs.has_changes == 'true'
+        run: |
+          git push origin main --force