customize value of TMPDIR to simplify cleanup

Signed-off-by: Jakub Sokołowski <jakub@status.im>
status-im · Oct 17, 2019 · 9d73a1b · 9d73a1b
1 parent bf088de
commit 9d73a1b
Show file tree

Hide file tree

Showing 10 changed files with 130 additions and 98 deletions.
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: nix-add-gcroots clean nix-clean disable-githooks react-native-android react-native-ios react-native-desktop test release _list _fix-node-perms
+.PHONY: nix-add-gcroots clean nix-clean disable-githooks react-native-android react-native-ios react-native-desktop test release _list _fix-node-perms _tmpdir-mk _tmpdir-rm
 
 help: SHELL := /bin/sh
 help: ##@other Show this help
@@ -27,13 +27,18 @@ HELP_FUN = \
 		   }
 HOST_OS := $(shell uname | tr '[:upper:]' '[:lower:]')
 
+# This can come from Jenkins
+ifndef BUILD_TAG
+export BUILD_TAG = $(shell git rev-parse --short HEAD)
+endif
+
 # Defines which variables will be kept for Nix pure shell, use semicolon as divider
-export _NIX_KEEP ?= BUILD_ENV
+export _NIX_KEEP ?= TMPDIR,BUILD_ENV
 export NIX_CONF_DIR = $(PWD)/nix
 # We don't want to use /run/user/$UID because it runs out of space too easilly
-export TMPDIR = /tmp
-
-export REACT_SERVER_PORT ?= 5001 # any value different from default 5000 will work; this has to be specified for both the Node.JS server process and the Qt process
+export TMPDIR = /tmp/tmp-status-react-$(BUILD_TAG)
+# this has to be specified for both the Node.JS server process and the Qt process
+export REACT_SERVER_PORT ?= 5001
 
 #----------------
 # Nix targets
@@ -89,19 +94,19 @@ _fix-node-perms: ##@prepare Fix permissions so that directory can be cleaned
 	$(shell test -d node_modules && chmod -R 744 node_modules)
 	$(shell test -d node_modules.tmp && chmod -R 744 node_modules.tmp)
 
+_tmpdir-mk: SHELL := /bin/sh
+_tmpdir-mk: ##@prepare Create a TMPDIR for temporary files
+	@mkdir -p "$(TMPDIR)"
+# Make sure TMPDIR exists every time make is called
+-include _tmpdir-mk
+
+_tmpdir-rm: SHELL := /bin/sh
+_tmpdir-rm: ##@prepare Remove TMPDIR
+	rm -fr "$(TMPDIR)"
+
 clean: SHELL := /bin/sh
-clean: _fix-node-perms ##@prepare Remove all output folders
-	echo "Cleaning status-react repo ..."; \
-	git clean -dxf -f; \
-	echo "Cleaning Yarn/Metro caches in temp dirs ..."; \
-	for d in "$$TMPDIR" "/tmp"; do \
-	if [ -d "$$d" ]; then \
-	pushd $$TMPDIR; \
-	rm -rf ./*metro*; \
-	rm -rf ./*yarn*; \
-	popd; \
-	fi; \
-	done
+clean: _fix-node-perms _tmpdir-rm ##@prepare Remove all output folders
+	git clean -dxf
 
 watchman-clean: export _NIX_ATTR := targets.watchman.shell
 watchman-clean: ##@prepare Delete repo directory from watchman 

diff --git a/ci/Jenkinsfile.android b/ci/Jenkinsfile.android
@@ -61,7 +61,7 @@ pipeline {
         script { cmn.nix.shell('lein deps :tree', attr: 'targets.leiningen.shell') }
       }
     }
-    stage('Parallel') {
+    stage('Parallel Assemble') {
       parallel {
         stage('Checks') { stages {
           stage('Lint') {
@@ -74,11 +74,6 @@ pipeline {
               script { cmn.nix.shell('lein test-cljs', attr: 'targets.leiningen.shell') }
             }
           }
-          /* stage('Coverage') {
-            steps {
-              script { android.coverage() }
-            }
-          } */
         } }
         stage('Build') { stages {
           stage('JSBundle') {
@@ -96,35 +91,41 @@ pipeline {
         } }
       }
     }
-    stage('Archive') {
-      steps { script {
-        apks.each { archiveArtifacts it }
-      } }
-    }
-    stage('Upload') {
-      steps {
-        script {
-          def urls = apks.collect { cmn.uploadArtifact(it) }
-          /* return only the universal APK */
-          if (urls.size() > 1) { 
-            env.PKG_URL = urls.find { it.contains('universal') }
-          } else { /* if no universal is available pick first */
-            env.PKG_URL = urls.first()
-          }
-          /* build type specific */
-          switch (btype) {
-            case 'release':
-              android.uploadToPlayStore(); break;
-            case 'nightly':
-              env.DIAWI_URL = android.uploadToDiawi(); break;
-            case 'e2e':
-              env.SAUCE_URL = android.uploadToSauceLabs(); break;
+    stage('Parallel Upload') {
+      parallel {
+        stage('Archive') {
+          steps { script {
+            apks.each { archiveArtifacts it }
+          } }
+        }
+        stage('Upload') {
+          steps {
+            script {
+              def urls = apks.collect { cmn.uploadArtifact(it) }
+              /* return only the universal APK */
+              if (urls.size() > 1) { 
+                env.PKG_URL = urls.find { it.contains('universal') }
+              } else { /* if no universal is available pick first */
+                env.PKG_URL = urls.first()
+              }
+              /* build type specific */
+              switch (btype) {
+                case 'release':
+                  android.uploadToPlayStore(); break;
+                case 'nightly':
+                  env.DIAWI_URL = android.uploadToDiawi(); break;
+                case 'e2e':
+                  env.SAUCE_URL = android.uploadToSauceLabs(); break;
+              }
+            }
           }
         }
       }
     }
     stage('Cleanup') {
-      steps { sh 'make clean' }
+      steps {
+        sh 'make clean'
+      }
     }
   }
   post {

diff --git a/ci/Jenkinsfile.combined b/ci/Jenkinsfile.combined
@@ -31,7 +31,7 @@ pipeline {
           osx    = cmn.ci.Build('status-react/combined/desktop-macos')
         } } }
         stage('Linux') { steps { script {
-          nix    = cmn.ci.Build('status-react/combined/desktop-linux')
+          tux    = cmn.ci.Build('status-react/combined/desktop-linux')
         } } }
         stage('Windows') { steps { script {
           win    = cmn.ci.Build('status-react/combined/desktop-windows')
@@ -55,7 +55,7 @@ pipeline {
         sh('rm -f pkg/*')
         if (btype != 'release') {
           cmn.ci.copyArts(osx)
-          cmn.ci.copyArts(nix)
+          cmn.ci.copyArts(tux)
           cmn.ci.copyArts(win)
         }
         cmn.ci.copyArts(ios)
@@ -78,7 +78,7 @@ pipeline {
           iOS: cmn.pkgUrl(ios), /*iOSe2e: cmn.pkgUrl(iose2e),*/
           Diawi: cmn.utils.getEnv(ios, 'DIAWI_URL'),
           /* desktop */
-          App: cmn.pkgUrl(nix), Mac: cmn.pkgUrl(osx), Win: cmn.pkgUrl(win),
+          App: cmn.pkgUrl(tux), Mac: cmn.pkgUrl(osx), Win: cmn.pkgUrl(win),
           /* upload the sha256 checksums file too */
           SHA: cmn.uploadArtifact(cmn.utils.pkgFind('sha256')),
         ]

diff --git a/ci/Jenkinsfile.fastlane.clean b/ci/Jenkinsfile.fastlane.clean
@@ -7,6 +7,8 @@ pipeline {
     LC_ALL    = 'en_US.UTF-8'
     TARGET_OS = 'ios'
     FASTLANE_DISABLE_COLORS = 1
+    /* See nix/README.md */
+    NIX_IGNORE_SYMLINK_STORE = 1
   }
 
   options {

diff --git a/ci/Jenkinsfile.ios b/ci/Jenkinsfile.ios
@@ -27,7 +27,8 @@ pipeline {
     LANGUAGE = "en_US.UTF-8"
     TARGET_OS = 'ios'
     NIX_CONF_DIR = "${env.WORKSPACE}/nix"
-    NIX_IGNORE_SYMLINK_STORE = 1 // https://github.com/NixOS/nix/issues/2925#issuecomment-499544039
+    /* See nix/README.md */
+    NIX_IGNORE_SYMLINK_STORE = 1
     FASTLANE_DISABLE_COLORS = 1
     BUNDLE_PATH = "${HOME}/.bundle"
     /* We use EXECUTOR_NUMBER to avoid multiple instances clashing */
@@ -49,7 +50,7 @@ pipeline {
         }
       }
     }
-    stage('Parallel') {
+    stage('Parallel Assemble') {
       parallel {
         stage('Checks') { stages {
           stage('Lint') {
@@ -77,20 +78,24 @@ pipeline {
         } }
       }
     }
-    stage('Archive') {
-      steps {
-        archiveArtifacts api
-      }
-    }
-    stage('Upload') {
-      steps {
-        script {
-          env.PKG_URL = cmn.uploadArtifact(api)
-          /* e2e builds get tested in SauceLabs */
-          if (btype == 'e2e') {
-            env.SAUCE_URL = ios.uploadToSauceLabs()
-          } else {
-            env.DIAWI_URL = ios.uploadToDiawi()
+    stage('Parallel Upload') {
+      parallel {
+        stage('Archive') {
+          steps {
+            archiveArtifacts api
+          }
+        }
+        stage('Upload') {
+          steps {
+            script {
+              env.PKG_URL = cmn.uploadArtifact(api)
+              /* e2e builds get tested in SauceLabs */
+              if (btype == 'e2e') {
+                env.SAUCE_URL = ios.uploadToSauceLabs()
+              } else {
+                env.DIAWI_URL = ios.uploadToDiawi()
+              }
+            }
           }
         }
       }

diff --git a/ci/Jenkinsfile.linux b/ci/Jenkinsfile.linux
@@ -53,7 +53,7 @@ pipeline {
         }
       }
     }
-    stage('Parallel') {
+    stage('Parallel Assemble') {
       parallel {
         stage('Checks') { stages {
           stage('Lint') {
@@ -86,14 +86,18 @@ pipeline {
         } }
       }
     }
-    stage('Archive') {
-      steps {
-        archiveArtifacts app
-      }
-    }
-    stage('Upload') {
-      steps {
-        script { env.PKG_URL = cmn.uploadArtifact(app) }
+    stage('Parallel Upload') {
+      parallel {
+        stage('Archive') {
+          steps {
+            archiveArtifacts app
+          }
+        }
+        stage('Upload') {
+          steps {
+            script { env.PKG_URL = cmn.uploadArtifact(app) }
+          }
+        }
       }
     }
     stage('Cleanup') {

diff --git a/ci/Jenkinsfile.macos b/ci/Jenkinsfile.macos
@@ -27,7 +27,8 @@ pipeline {
     LANGUAGE = "en_US.UTF-8"
     TARGET_OS = 'macos'
     NIX_CONF_DIR = "${env.WORKSPACE}/nix"
-    NIX_IGNORE_SYMLINK_STORE = 1 // https://github.com/NixOS/nix/issues/2925#issuecomment-499544039
+    /* See nix/README.md */
+    NIX_IGNORE_SYMLINK_STORE = 1
     VERBOSE_LEVEL = '3'
     /* We use EXECUTOR_NUMBER to avoid multiple instances clashing */
     LEIN_HOME         = "/var/tmp/lein-${EXECUTOR_NUMBER}"
@@ -50,7 +51,7 @@ pipeline {
         }
       }
     }
-    stage('Parallel') {
+    stage('Parallel Assemble') {
       parallel {
         stage('Checks') { stages {
           stage('Lint') {
@@ -83,14 +84,18 @@ pipeline {
         } }
       }
     }
-    stage('Archive') {
-      steps {
-        archiveArtifacts dmg
-      }
-    }
-    stage('Upload') {
-      steps {
-        script { env.PKG_URL = cmn.uploadArtifact(dmg) }
+    stage('Parallel Upload') {
+      parallel {
+        stage('Archive') {
+          steps {
+            archiveArtifacts dmg
+          }
+        }
+        stage('Upload') {
+          steps {
+            script { env.PKG_URL = cmn.uploadArtifact(dmg) }
+          }
+        }
       }
     }
     stage('Cleanup') {

diff --git a/ci/Jenkinsfile.windows b/ci/Jenkinsfile.windows
@@ -56,7 +56,7 @@ pipeline {
         }
       }
     }
-    stage('Parallel') {
+    stage('Parallel Assemble') {
       parallel {
         stage('Checks') { stages {
           stage('Lint') {
@@ -89,14 +89,18 @@ pipeline {
         } }
       }
     }
-    stage('Archive') {
-      steps {
-        archiveArtifacts app
-      }
-    }
-    stage('Upload') {
-      steps {
-        script { env.PKG_URL = cmn.uploadArtifact(app) }
+    stage('Parallel Upload') {
+      parallel {
+        stage('Archive') {
+          steps {
+            archiveArtifacts app
+          }
+        }
+        stage('Upload') {
+          steps {
+            script { env.PKG_URL = cmn.uploadArtifact(app) }
+          }
+        }
       }
     }
     stage('Cleanup') {

diff --git a/ci/common.groovy b/ci/common.groovy
@@ -59,11 +59,11 @@ def prep(type = 'nightly') {
 
   if (env.TARGET_OS == 'macos' || env.TARGET_OS == 'linux' || env.TARGET_OS == 'windows') {
     /* node deps, pods, and status-go download */
-    utils.nix.shell('scripts/prepare-for-desktop-platform.sh', pure: false)
+    nix.shell('scripts/prepare-for-desktop-platform.sh', pure: false)
     sh('scripts/copy-translations.sh')
   } else if (env.TARGET_OS != 'android') {
     // run script in the nix shell so that node_modules gets instantiated before attempting the copies
-    utils.nix.shell('scripts/copy-translations.sh chmod')
+    nix.shell('scripts/copy-translations.sh chmod')
   }
 }
 
@@ -87,8 +87,8 @@ def uploadArtifact(path) {
     passwordVariable: 'DO_SECRET_KEY'
   )]) {
     nix.shell("""
-      s3cmd \\
-        --acl-public ${customOpts} \\
+      s3cmd ${customOpts} \\
+        --acl-public \\
         --host="${domain}" \\
         --host-bucket="%(bucket)s.${domain}" \\
         --access_key=${DO_ACCESS_KEY} \\

diff --git a/nix/build.sh b/nix/build.sh
@@ -55,6 +55,12 @@ nixOpts=(
   "default.nix"
 )
 
+# This variable allows specifying which env vars to keep for Nix pure shell
+# The separator is a semicolon
+if [[ -n "${_NIX_KEEP}" ]]; then
+  nixOpts+=("--keep ${_NIX_KEEP//;/ --keep }")
+fi
+
 # Run the actual build
 echo "Running: nix-build ${nixOpts[@]}"
 nixResultPath=$(nix-build ${nixOpts[@]})