[soil] Set up ci.oilshell.org

New directory structure to avoid SSH DoS of Dreamhost - ci.oilshell.org/uuu/ is managed by wwup.cgi - we upload results via HTTP - ci.oilshell.org/code is managed by SSH - Remove OILS_GITHUB_KEY in a bunch of places - Tweak front page
oils-for-unix · Aug 26, 2024 · c287f55 · c287f55
1 parent 44f3c7f
commit c287f55
Show file tree

Hide file tree

Showing 8 changed files with 27 additions and 42 deletions.
diff --git a/.github/workflows/all-builds.yml b/.github/workflows/all-builds.yml
@@ -45,9 +45,6 @@ jobs:
           soil/worker.sh JOB-raw-vm
 
       - name: publish-html
-        env:
-          # for deploying to dashboard
-          OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }}
         run: |
           soil/github-actions.sh publish-and-exit raw-vm T
 
@@ -85,9 +82,6 @@ jobs:
           soil/github-actions.sh run-job dummy podman
 
       - name: publish-html
-        env:
-          # for deploying to dashboard
-          OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }}
         run: |
           soil/github-actions.sh publish-and-exit dummy T
 
@@ -412,9 +406,6 @@ jobs:
           soil/worker.sh JOB-maybe-merge
 
       - name: publish-html
-        env:
-          # for deploying to dashboard
-          OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }}
         run: |
           # NOTE: does not publish to status API
           soil/github-actions.sh publish-and-exit maybe-merge
diff --git a/.github/workflows/fast-subset.yml_DISABLED b/.github/workflows/fast-subset.yml_DISABLED
@@ -45,9 +45,6 @@ jobs:
           soil/worker.sh JOB-raw-vm
 
       - name: publish-html
-        env:
-          # for deploying to dashboard
-          OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }}
         run: |
           soil/github-actions.sh publish-and-exit raw-vm T
 
@@ -66,9 +63,6 @@ jobs:
           soil/github-actions.sh run-job dummy podman
 
       - name: publish-html
-        env:
-          # for deploying to dashboard
-          OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }}
         run: |
           soil/github-actions.sh publish-and-exit dummy T
 
@@ -123,9 +117,6 @@ jobs:
           soil/worker.sh JOB-dummy
 
       - name: publish-html
-        env:
-          # for deploying to dashboard
-          OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }}
         run: |
           # NOTE: does not publish to status API
           soil/github-actions.sh publish-and-exit maybe-merge
diff --git a/soil/common.sh b/soil/common.sh
@@ -22,9 +22,9 @@ dump-env() {
 
 if true; then
   readonly SOIL_USER='travis_admin'
-  readonly SOIL_HOST='uuu.oilshell.org'
-  readonly SOIL_HOST_DIR=~/uuu.oilshell.org  # used on server
-  readonly SOIL_REMOTE_DIR=uuu.oilshell.org  # used on client
+  readonly SOIL_HOST='ci.oilshell.org'
+  readonly SOIL_HOST_DIR=~/ci.oilshell.org  # used on server
+  readonly SOIL_REMOTE_DIR=ci.oilshell.org  # used on client
 elif false; then
   readonly SOIL_USER='oils'
   readonly SOIL_HOST='mb.oils.pub'
@@ -40,7 +40,7 @@ fi
 
 readonly SOIL_USER_HOST="$SOIL_USER@$SOIL_HOST"
 
-readonly WWUP_URL="https://$SOIL_HOST/wwup.cgi"
+readonly WWUP_URL="https://$SOIL_HOST/uuu/wwup.cgi"
 
 html-head() {
   # TODO: Shebang line should change too
@@ -75,7 +75,7 @@ git-commit-dir() {
   local commit_hash
   commit_hash=$(cat _tmp/soil/commit-hash.txt)
 
-  local git_commit_dir="$SOIL_REMOTE_DIR/${prefix}jobs/git-$commit_hash"
+  local git_commit_dir="$SOIL_REMOTE_DIR/code/${prefix}jobs/git-$commit_hash"
 
   echo $git_commit_dir
 }
@@ -88,7 +88,7 @@ git-commit-url() {
   commit_hash=$(cat _tmp/soil/commit-hash.txt)
 
   # https:// not working on Github Actions because of cert issues?
-  local url="http://$SOIL_HOST/${prefix}jobs/git-$commit_hash"
+  local url="http://$SOIL_HOST/code/${prefix}jobs/git-$commit_hash"
 
   echo $url
 }
diff --git a/soil/github-actions.sh b/soil/github-actions.sh
@@ -88,7 +88,7 @@ load-secret-key() {
 publish-html() {
   ### Publish job HTML, and optionally status-api
 
-  load-secret-key
+  #load-secret-key
 
   set -x
   # $1 can be the job name

diff --git a/soil/maybe-merge.sh b/soil/maybe-merge.sh
@@ -125,7 +125,7 @@ soil-run() {
   mkdir -p $dir
 
   # These tiny files are written by each Soil task
-  local url_base="http://$SOIL_HOST/status-api/github/$run_id"
+  local url_base="http://$SOIL_HOST/uuu/status-api/github/$run_id"
 
   #local jobs='dummy pea other-tests'  # minimal set of jobs to wait for
   local jobs=$(soil/worker.sh list-jobs)

diff --git a/soil/web-init.sh b/soil/web-init.sh
@@ -53,7 +53,7 @@ home-page() {
 
       <tr>
         <td>
-          <a href="sourcehut-jobs/">sr.ht</a> 
+          <a href="uuu/sourcehut-jobs/">sr.ht</a> 
         </td>
         <td>
           <a href="https://builds.sr.ht/~andyc">builds.sr.ht</a>
@@ -65,7 +65,7 @@ home-page() {
 
       <tr>
         <td>
-          <a href="github-jobs/">Github Actions</a> 
+          <a href="uuu/github-jobs/">Github Actions</a> 
         </td>
         <td>
           <a href="https://github.com/oilshell/oil/actions/workflows/all-builds.yml">github.com</a>
@@ -117,7 +117,10 @@ EOF
 
     <ul>
       <li>
-        <a href="status-api/github/">static-api/github/</a>
+        <a href="code/github-jobs/">code/github-jobs/</a> - tarballs at every commit
+      </li>
+      <li>
+        <a href="uuu/status-api/github/">uuu/static-api/github/</a> - files used by the CI
       </li>
     </ul>
 
@@ -130,20 +133,20 @@ deploy-data() {
   local user=${1:-$SOIL_USER}
   local host=${2:-$SOIL_HOST}
 
-  # www/ prefix for Mythic beasts
-  local host_dir=$SOIL_REMOTE_DIR
+  local host_dir=$SOIL_REMOTE_DIR/uuu
 
   # TODO: Better to put HTML in www/$host/uuu/github-jobs, etc.
   ssh $user@$host mkdir -v -p \
-    $host_dir/{travis-jobs,sourcehut-jobs,github-jobs,circle-jobs,cirrus-jobs,web,status-api/github} \
+    $host_dir/{sourcehut-jobs,github-jobs,status-api/github} \
     $host_dir/web/table
 
-  home-page "$host" > _tmp/index.html
-
   # note: duplicating CSS
-  scp _tmp/index.html $user@$host:$host_dir/
   scp web/{base.css,soil.css,ajax.js} $user@$host:$host_dir/web
   scp web/table/*.{js,css} $user@$host:$host_dir/web/table
+
+  home-page "$host" > _tmp/index.html
+  # Home page goes in the domain root
+  scp _tmp/index.html $user@$host:$SOIL_REMOTE_DIR/
 }
 
 soil-web-manifest() {

diff --git a/soil/web-worker.sh b/soil/web-worker.sh
@@ -352,7 +352,7 @@ publish-cpp-tarball() {
 
     log 'Tarball:'
     log ''
-    log "http://$SOIL_HOST/github-jobs/git-$commit_hash/"
+    log "http://$SOIL_HOST/code/github-jobs/git-$commit_hash/"
 
   else
     # Fix subtle problem here !!!

diff --git a/soil/web.sh b/soil/web.sh
@@ -48,9 +48,9 @@ rewrite-jobs-index() {
   local prefix=$1
   local run_id=$2   # pass GITHUB_RUN_NUMBER or git-$hash
 
-  local dir=$SOIL_HOST_DIR/${prefix}jobs
+  local dir=$SOIL_HOST_DIR/uuu/${prefix}jobs
 
-  log "soil-web: Rewriting ${prefix}jobs/index.html"
+  log "soil-web: Rewriting uuu/${prefix}jobs/index.html"
 
   # Fix for bug #1169: don't create the temp file on a different file system,
   # which /tmp may be.
@@ -82,7 +82,7 @@ cleanup-jobs-index() {
   local prefix=$1
   local dry_run=${2:-true}
 
-  local dir=$SOIL_HOST_DIR/${prefix}jobs
+  local dir=$SOIL_HOST_DIR/uuu/${prefix}jobs
 
   # Pass it all JSON, and then it figures out what files to delete (TSV, etc.)
   case $dry_run in
@@ -120,7 +120,7 @@ cleanup-status-api() {
 
   local dry_run=${1:-true}
 
-  local dir=$SOIL_HOST_DIR/status-api/github
+  local dir=$SOIL_HOST_DIR/uuu/status-api/github
 
   pushd $dir
   case $dry_run in
@@ -137,7 +137,7 @@ cleanup-status-api() {
   popd
 }
 
-_event-job-done() {
+event-job-done() {
   ### "Server side" handler
 
   local prefix=$1  # 'github-' or 'sourcehut-'
@@ -149,7 +149,7 @@ _event-job-done() {
   cleanup-jobs-index $prefix false
 }
 
-event-job-done() {
+DISABLED-event-job-done() {
   ### Hook for wwup.cgi to execute
 
   # As long as the CGI script shows output, I don't think we need any wrappers