From 438a2e4b10bf86ddbcdc6f93a7eba5ced8ea53c8 Mon Sep 17 00:00:00 2001 From: guru-aot Date: Tue, 12 Dec 2023 10:50:50 -0700 Subject: [PATCH] 2535 - Crash Loop Back offs Error (#2577) Note: The branch name is created without a # in the front, to test my build and deployments from my local. - [x] Investigate the issue and apply the fix on all our custom Dockerfile (Api, web, workers, queue-consumers, load-test-gateway & db-migration) - [x] Most updated image in node has to be deployed What does EACCES error means in node - https://betterstack.com/community/guides/scaling-nodejs/nodejs-errors/#13-eacces On Analysing the issue, i was able to understand that this issue happens in certain version of node & npm libraries, some related bugs reported in the years are given below for reference. - https://github.com/npm/cli/issues/5114 - https://stackoverflow.com/questions/71450392/npm-cache-issue - https://stackoverflow.com/questions/67163194/jenkins-build-issue-npm-err-your-cache-folder-contains-root-owned-files The possible solutions given by the people are alwauys these 2 - give sudo permission and access for the user 1001 for the /.npm directory/ - which is not always right to have a sudo persmission for a user in dockerfile - upgrade the node version as it states to a newer version that this issue is not happening - going with this route as we have a newer version of redhat that contains the new version from the one deplyoed already with some security issues. https://catalog.redhat.com/software/containers/ubi8/nodejs-18/6278e5c078709f5277f26998?architecture=amd64&image=65302e01ec5935b621691d22&container-tabs=packages ![image](https://github.com/bcgov/SIMS/assets/62901416/1f7136b4-3daf-43d3-9af3-6232865be1e3) https://catalog.redhat.com/software/containers/ubi8/nodejs-18/6278e5c078709f5277f26998?architecture=amd64&image=6543c3d67371c4bd3014291a&container-tabs=packages image Analyzing the changelogs of npm js, there has been bugs related to cache that has been fixed. https://docs.npmjs.com/cli/v9/using-npm/changelog#981-2023-07-18 https://docs.npmjs.com/cli/v9/using-npm/changelog#967-2023-05-17 ![image](https://github.com/bcgov/SIMS/assets/62901416/e35782a3-28f3-4c5c-982b-58b06ac6404b) https://github.com/npm/cli/pull/6464 The bug reported may not state the same issue we are facing but its related to the cache error that is happening in the version we were using. So updating the redhat image to the latest version is what was taken as an action to solve this issue. Note: https://app.zenhub.com/workspaces/student-information-management-system-5fce9df5aa1b45000e937014/issues/gh/bcgov/sims/2453 is also done as part of this PR. - [x] Remove variables from Makefile - [x] Remove references from docker-build.yml As suggested by @andrewsignori-aot https://github.com/bcgov/SIMS/pull/2577#discussion_r1421062313 changed the permission of users in the group 0 for the folder ./.npm to have write access Container before assigning the write permission image Container after assigning the write permission image Addded only the permissions for the ./.npm folder as in the past month the failed logs show this error happening only in the ./.npm folder. https://kibana-openshift-logging.apps.silver.devops.gov.bc.ca/app/kibana#/discover?_g=(refreshInterval:(pause:!t,value:0),time:(from:now-30d,mode:quick,to:now))&_a=(columns:!(message),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'8841c680-a15b-11eb-a4dc-e5bf19f04239',key:kubernetes.namespace_name,negate:!f,params:(query:'0c27fb-test',type:phrase),type:phrase,value:'0c27fb-test'),query:(match:(kubernetes.namespace_name:(query:'0c27fb-test',type:phrase))))),index:'8841c680-a15b-11eb-a4dc-e5bf19f04239',interval:auto,query:(language:lucene,query:'%22sudo%20chown%20-R%20%22'),sort:!('@timestamp',desc)) image --- devops/Makefile | 12 ++++----- devops/openshift/docker-build.yml | 27 ------------------- sources/packages/backend/apps/api/Dockerfile | 7 ++++- .../backend/apps/db-migrations/Dockerfile | 7 ++++- .../backend/apps/load-test-gateway/Dockerfile | 7 ++++- .../backend/apps/queue-consumers/Dockerfile | 7 ++++- .../packages/backend/apps/workers/Dockerfile | 7 ++++- sources/packages/web/Dockerfile | 2 +- 8 files changed, 37 insertions(+), 39 deletions(-) diff --git a/devops/Makefile b/devops/Makefile index 5c722933e5..1996ee7d4b 100644 --- a/devops/Makefile +++ b/devops/Makefile @@ -243,7 +243,7 @@ build-db-migrations: test -n "$(BUILD_REF)" test -n "$(DB_MIGRATIONS_BUILD_REF)" @echo "+\n++ BUILDING DB migrations with tag: $(BUILD_REF)\n+" - @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p BASE_IMAGE_NAME="nodejs-18" -p BASE_IMAGE_TAG="1-71.1697652955" -p BASE_IMAGE_REPO="artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/" -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)backend -p DOCKER_FILE_PATH=apps/db-migrations/Dockerfile -p NAME=$(DB_MIGRATIONS_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - + @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)backend -p DOCKER_FILE_PATH=apps/db-migrations/Dockerfile -p NAME=$(DB_MIGRATIONS_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - @oc -n $(BUILD_NAMESPACE) start-build bc/$(DB_MIGRATIONS_BUILD_REF) --wait build-api: @@ -251,7 +251,7 @@ build-api: test -n "$(BUILD_REF)" test -n "$(API_BUILD_REF)" @echo "+\n++ BUILDING API with tag: $(BUILD_REF)\n+" - @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p BASE_IMAGE_NAME="nodejs-18" -p BASE_IMAGE_TAG="1-71.1697652955" -p BASE_IMAGE_REPO="artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/" -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)backend -p DOCKER_FILE_PATH=apps/api/Dockerfile -p NAME=$(API_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - + @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)backend -p DOCKER_FILE_PATH=apps/api/Dockerfile -p NAME=$(API_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - @oc -n $(BUILD_NAMESPACE) start-build bc/$(API_BUILD_REF) --wait build-workers: @@ -259,7 +259,7 @@ build-workers: test -n "$(BUILD_REF)" test -n "$(WORKERS_BUILD_REF)" @echo "+\n++ BUILDING WORKERS with tag: $(BUILD_REF)\n+" - @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p BASE_IMAGE_NAME="nodejs-18" -p BASE_IMAGE_TAG="1-71.1697652955" -p BASE_IMAGE_REPO="artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/" -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)backend -p DOCKER_FILE_PATH=apps/workers/Dockerfile -p NAME=$(WORKERS_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - + @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)backend -p DOCKER_FILE_PATH=apps/workers/Dockerfile -p NAME=$(WORKERS_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - @oc -n $(BUILD_NAMESPACE) start-build bc/$(WORKERS_BUILD_REF) --wait build-queue-consumers: @@ -267,7 +267,7 @@ build-queue-consumers: test -n "$(BUILD_REF)" test -n "$(QUEUE_CONSUMERS_BUILD_REF)" @echo "+\n++ BUILDING QUEUE_CONSUMERS with tag: $(BUILD_REF)\n+" - @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p BASE_IMAGE_NAME="nodejs-18" -p BASE_IMAGE_TAG="1-71.1697652955" -p BASE_IMAGE_REPO="artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/" -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)backend -p DOCKER_FILE_PATH=apps/queue-consumers/Dockerfile -p NAME=$(QUEUE_CONSUMERS_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - + @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)backend -p DOCKER_FILE_PATH=apps/queue-consumers/Dockerfile -p NAME=$(QUEUE_CONSUMERS_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - @oc -n $(BUILD_NAMESPACE) start-build bc/$(QUEUE_CONSUMERS_BUILD_REF) --wait build-load-test-gateway: @@ -275,7 +275,7 @@ build-load-test-gateway: test -n "$(BUILD_REF)" test -n "$(LOAD_TEST_GATEWAY_BUILD_REF)" @echo "+\n++ BUILDING LOAD_TEST_GATEWAY with tag: $(BUILD_REF)\n+" - @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p BASE_IMAGE_NAME="nodejs-18" -p BASE_IMAGE_TAG="1-71.1697652955" -p BASE_IMAGE_REPO="artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/" -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)backend -p DOCKER_FILE_PATH=apps/load-test-gateway/Dockerfile -p NAME=$(LOAD_TEST_GATEWAY_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - + @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)backend -p DOCKER_FILE_PATH=apps/load-test-gateway/Dockerfile -p NAME=$(LOAD_TEST_GATEWAY_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - @oc -n $(BUILD_NAMESPACE) start-build bc/$(LOAD_TEST_GATEWAY_BUILD_REF) --wait build-web: @@ -283,7 +283,7 @@ build-web: test -n "$(BUILD_REF)" test -n "$(WEB_BUILD_REF)" @echo "+\n++ BUILDING WEB with tag: $(BUILD_REF)\n+" - @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p BASE_IMAGE_NAME="nginx-122" -p BASE_IMAGE_TAG="1" -p BASE_IMAGE_REPO="artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/" -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)web -p NAME=$(WEB_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - + @oc -n $(BUILD_NAMESPACE) process -f $(BUILD_TEMPLATE_PATH) -p TAG=$(BUILD_REF) -p SOURCE_REPOSITORY_REF=$(BUILD_REF) -p SOURCE_CONTEXT_DIR=$(SOURCE_CONTEXT_DIR)web -p NAME=$(WEB_BUILD_REF) | oc -n $(BUILD_NAMESPACE) apply -f - @oc -n $(BUILD_NAMESPACE) start-build bc/$(WEB_BUILD_REF) --wait init-patroni: diff --git a/devops/openshift/docker-build.yml b/devops/openshift/docker-build.yml index 40f312152b..0717eb97d1 100644 --- a/devops/openshift/docker-build.yml +++ b/devops/openshift/docker-build.yml @@ -5,28 +5,6 @@ kind: Template metadata: name: ${NAME} objects: - - apiVersion: image.openshift.io/v1 - kind: ImageStream - metadata: - name: ${BASE_IMAGE_NAME} - spec: - lookupPolicy: - local: false - - apiVersion: v1 - kind: ImageStreamTag - lookupPolicy: - local: false - metadata: - name: ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} - tag: - annotations: null - from: - kind: DockerImage - name: ${BASE_IMAGE_REPO}${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} - importPolicy: - scheduled: true - referencePolicy: - type: Source - kind: ImageStream apiVersion: v1 metadata: @@ -88,11 +66,6 @@ parameters: value: Dockerfile - name: TAG value: "latest" - - name: BASE_IMAGE_REPO - - name: BASE_IMAGE_NAME - required: true - - name: BASE_IMAGE_TAG - required: true - name: CPU_LIMIT value: "2" - name: MEMORY_LIMIT diff --git a/sources/packages/backend/apps/api/Dockerfile b/sources/packages/backend/apps/api/Dockerfile index 9e1ee6730e..e71d5bd71b 100644 --- a/sources/packages/backend/apps/api/Dockerfile +++ b/sources/packages/backend/apps/api/Dockerfile @@ -1,5 +1,5 @@ # Base Image -FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-71.1697652955 +FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-81 LABEL maintainer="BC GOV" @@ -23,5 +23,10 @@ RUN npm run build api # Exposing application port EXPOSE ${PORT} +# Grant access to group 0 to allow npm v9 to work +# See: https://docs.openshift.com/container-platform/4.11/openshift_images/create-images.html#use-uid_create-images +RUN chgrp -R 0 ./.npm && \ + chmod -R g=u ./.npm + # Entry point CMD [ "npm", "run", "start:prod:api" ] \ No newline at end of file diff --git a/sources/packages/backend/apps/db-migrations/Dockerfile b/sources/packages/backend/apps/db-migrations/Dockerfile index a20c2e8a5d..f66fc6f289 100644 --- a/sources/packages/backend/apps/db-migrations/Dockerfile +++ b/sources/packages/backend/apps/db-migrations/Dockerfile @@ -1,5 +1,5 @@ # Base Image -FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-71.1697652955 +FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-81 LABEL maintainer="BC GOV" @@ -15,5 +15,10 @@ COPY tsconfig.json ./ COPY tsconfig.build.json ./ COPY env-setup.js ./ +# Grant access to group 0 to allow npm v9 to work +# See: https://docs.openshift.com/container-platform/4.11/openshift_images/create-images.html#use-uid_create-images +RUN chgrp -R 0 ./.npm && \ + chmod -R g=u ./.npm + # Entry point CMD [ "npm", "run", "start:prod:db-migrations" ] diff --git a/sources/packages/backend/apps/load-test-gateway/Dockerfile b/sources/packages/backend/apps/load-test-gateway/Dockerfile index 3b358ce4bb..d8d4cc97fd 100644 --- a/sources/packages/backend/apps/load-test-gateway/Dockerfile +++ b/sources/packages/backend/apps/load-test-gateway/Dockerfile @@ -1,5 +1,5 @@ # Base Image -FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-71.1697652955 +FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-81 LABEL maintainer="BC GOV" @@ -16,5 +16,10 @@ COPY ./libs ./libs # Building app RUN npm run build load-test-gateway +# Grant access to group 0 to allow npm v9 to work +# See: https://docs.openshift.com/container-platform/4.11/openshift_images/create-images.html#use-uid_create-images +RUN chgrp -R 0 ./.npm && \ + chmod -R g=u ./.npm + # Entry point CMD [ "npm", "run", "start:prod:load-test-gateway" ] \ No newline at end of file diff --git a/sources/packages/backend/apps/queue-consumers/Dockerfile b/sources/packages/backend/apps/queue-consumers/Dockerfile index 7864a08e7b..331ed9c856 100644 --- a/sources/packages/backend/apps/queue-consumers/Dockerfile +++ b/sources/packages/backend/apps/queue-consumers/Dockerfile @@ -1,5 +1,5 @@ # Base Image -FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-71.1697652955 +FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-81 LABEL maintainer="BC GOV" @@ -16,5 +16,10 @@ COPY ./libs ./libs # Building app RUN npm run build queue-consumers +# Grant access to group 0 to allow npm v9 to work +# See: https://docs.openshift.com/container-platform/4.11/openshift_images/create-images.html#use-uid_create-images +RUN chgrp -R 0 ./.npm && \ + chmod -R g=u ./.npm + # Entry point CMD [ "npm", "run", "start:prod:queue-consumers" ] \ No newline at end of file diff --git a/sources/packages/backend/apps/workers/Dockerfile b/sources/packages/backend/apps/workers/Dockerfile index 976ab01e8a..d68ac1c0f3 100644 --- a/sources/packages/backend/apps/workers/Dockerfile +++ b/sources/packages/backend/apps/workers/Dockerfile @@ -1,5 +1,5 @@ # Base Image -FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-71.1697652955 +FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-81 LABEL maintainer="BC GOV" @@ -16,5 +16,10 @@ COPY ./libs ./libs # Building app RUN npm run build workers +# Grant access to group 0 to allow npm v9 to work +# See: https://docs.openshift.com/container-platform/4.11/openshift_images/create-images.html#use-uid_create-images +RUN chgrp -R 0 ./.npm && \ + chmod -R g=u ./.npm + # Entry point CMD [ "npm", "run", "start:prod:workers" ] \ No newline at end of file diff --git a/sources/packages/web/Dockerfile b/sources/packages/web/Dockerfile index 2db8bc2110..8a57ddd13e 100644 --- a/sources/packages/web/Dockerfile +++ b/sources/packages/web/Dockerfile @@ -1,4 +1,4 @@ -FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-71.1697652955 AS builder +FROM artifacts.developer.gov.bc.ca/redhat-access-docker-remote/ubi8/nodejs-18:1-81 AS builder # Application Port. ENV PORT 3030