[version-3-4] docs: DOC-1426 Version Branches Automation Improvements (…

…#4314) (#4317)

* docs: DOC-1426 Version Branches Automation Improvements (#4314)

* ci: create CI YAML

* chore: update config updater

* docs: DOC-1426

* chore: save

* chore: fix path

* ci: ready for test

* chore: fix CI name

* ci: version logic+

(cherry picked from commit 6c05221)

* ci: adding missing robots.txt or updating noindex in docusaurus.config.js

---------

Co-authored-by: Karl Cardenas <29551334+karl-cardenas-coding@users.noreply.github.com>
Co-authored-by: vault-token-factory-spectrocloud[bot] <vault-token-factory-spectrocloud[bot]@users.noreply.github.com>

.github/workflows/versions_robot.yaml

@@ -0,0 +1,105 @@
+name: Versions Robot.txt Check
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    branches: ["version-*", "backport/version-*"]
+
+
+concurrency:
+  group: versions-robot-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_KEY }}
+  AWS_DEFAULT_REGION: us-east-1
+  APPZI_TOKEN: ${{ secrets.APPZI_TOKEN }}
+  FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
+  ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
+  ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
+  ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
+  ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
+  GITHUB_BRANCH: ${{ github.ref_name }} 
+
+jobs:
+  run-ci:
+    # runs-on: ubuntu-latest
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    if: ${{ !github.event.pull_request.draft && github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' }}
+    steps:
+      # If the condition above is not met, aka, the PR is not in draft status, then this step is skipped.
+      # Because this step is part of the critical path, omission of this step will result in remaining CI steps not gettinge executed.
+      # As of 8/8/2022 there is now way to enforce this beahvior in GitHub Actions CI.
+      - run: exit 0
+
+  robots-txt:
+    name: Robots.txt Check
+    needs: [run-ci]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - run: npm ci
+
+      - name: Determine branch name
+        id: extract_branch
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            echo "GITHUB_BRANCH=${{ github.head_ref }}" >> $GITHUB_ENV
+          else
+            echo "GITHUB_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+
+      - name: Check if robots.txt exists and create if missing
+        shell: bash
+        run: |
+          if [ ! -f "static/robots.txt" ]; then
+            echo "Does not Exist"
+            echo "User-agent: *" > static/robots.txt
+            echo "Disallow: /" >> static/robots.txt
+          else
+            echo "Robots.txt file exists"
+          fi
+      - name: Ensure noIndex is set
+        id: check_noindex
+        run: node scripts/noindex_docusaurus_config.js $RUNNER_TEMP $PWD && mv $RUNNER_TEMP/temp.docusaurus.config.js $PWD/docusaurus.config.js
+
+      - name: Commit Changes
+        id: commit
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          file_pattern: '*.js *.txt **.txt **.js **/*.txt **/*.js'
+          disable_globbing: true
+          commit_message: "ci: adding missing robots.txt or updating noindex in docusaurus.config.js"
+
+      - name: Slack Notification
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_PRIVATE_TEAM_WEBHOOK }}
+          SLACK_USERNAME: "spectromate"
+          SLACK_ICON_EMOJI: ":robot_panic:"
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: ' The PR for version branch ${{env.GITHUB_BRANCH}} failed when attempting add a robot.txt file or update the docusaurus.config.js. Review the GitHub Actions logs for more details.'
+
+      - name: Post Netlify progress
+        if: ${{ steps.commit.conclusion == 'success' && steps.commit.outputs.changes_detected == 'true' }}
+        uses: mshick/add-pr-comment@v2
+        with:
+          message: |
+              🤖 A robot.txt file was auto-generated or the docusaurus.config.js was updated and commited to the branch. All version braches require a robot.txt file.
+          refresh-message-position: false
+