[version-3-4] docs: Version Branches Robots.txt Update (#4324) (#4326)

* docs: Version Branches Robots.txt Update (#4324)

* chore: sligh bash modification

* chore: Apply suggestions from code review

Co-authored-by: Lenny Chen <55669665+lennessyy@users.noreply.github.com>

---------

Co-authored-by: Lenny Chen <55669665+lennessyy@users.noreply.github.com>
(cherry picked from commit 7927729)

* ci: adding missing robots.txt or updating noindex in docusaurus.config.js

---------

Co-authored-by: Karl Cardenas <29551334+karl-cardenas-coding@users.noreply.github.com>
Co-authored-by: vault-token-factory-spectrocloud[bot] <vault-token-factory-spectrocloud[bot]@users.noreply.github.com>

.github/workflows/versions_robot.yaml

@@ -66,13 +66,21 @@ jobs:
       - name: Check if robots.txt exists and create if missing
         shell: bash
         run: |
-          if [ ! -f "static/robots.txt" ]; then
-            echo "Does not Exist"
-            echo "User-agent: *" > static/robots.txt
-            echo "Disallow: /" >> static/robots.txt
-          else
-            echo "Robots.txt file exists"
-          fi
+            if [ ! -f "static/robots.txt" ]; then
+              echo "Does not Exist. Creating file with expected content..."
+              echo "User-agent: *" > static/robots.txt
+              echo "Disallow: /" >> static/robots.txt
+            else
+              echo "Robots.txt file exists"
+              if ! grep -Fxq "User-agent: *" static/robots.txt || ! grep -Fxq "Disallow: /" static/robots.txt; then
+                echo "Robots.txt file does not have the expected content. Updating..."
+                echo "User-agent: *" > static/robots.txt
+                echo "Disallow: /" >> static/robots.txt
+              else
+                echo "Robots.txt file has the expected content."
+              fi
+            fi
+            
       - name: Ensure noIndex is set
         id: check_noindex
         run: node scripts/noindex_docusaurus_config.js $RUNNER_TEMP $PWD && mv $RUNNER_TEMP/temp.docusaurus.config.js $PWD/docusaurus.config.js

static/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /