Merge branch 'develop'

intranda · Sep 2, 2021 · 156ce88 · 156ce88
2 parents d5cb30c + 113b13b
commit 156ce88
Show file tree

Hide file tree

Showing 35 changed files with 1,401 additions and 848 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,26 @@
+**/.classpath
+**/.dockerignore
+**/.env
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+README.md
+
+# .git is not ignored, as it is needed during maven run
diff --git a/.github/workflows/default.yml b/.github/workflows/default.yml
@@ -8,7 +8,7 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     steps:
     - uses: actions/checkout@v2
@@ -22,10 +22,11 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-maven-
 
-    - name: Set up JDK 1.8
-      uses: actions/setup-java@v1
+    - name: Set up Java 11
+      uses: actions/setup-java@v2
       with:
-        java-version: 1.8
+        distribution: 'zulu'
+        java-version: '11'
 
     - name: Compile Goobi viewer Indexer
       run: mvn -f goobi-viewer-indexer/pom.xml clean package -U -B

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -8,7 +8,7 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     steps:
     - uses: actions/checkout@v2
@@ -22,10 +22,11 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-maven-
 
-    - name: Set up JDK 1.8
-      uses: actions/setup-java@v1
+    - name: Set up Java 11
+      uses: actions/setup-java@v2
       with:
-        java-version: 1.8
+        distribution: 'zulu'
+        java-version: '11'
 
     - name: Create solrIndexer.jar
       run: mvn -f goobi-viewer-indexer/pom.xml clean package -U -B

diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,26 @@
+FROM maven:3.6-jdk-11 AS BUILD
+# build indexer jar
+
+COPY ./ /indexer
+WORKDIR /indexer
+RUN mvn -f goobi-viewer-indexer clean package
+
+
+# start assembling the final image
+FROM openjdk:11-jdk AS ASSEMBLE
+LABEL org.opencontainers.image.authors="Matthias Geerdsen <matthias.geerdsen@intranda.com>"
+
+
+ENV SOLR_URL http://solr:8983/solr/collection1
+ENV VIEWER_URL http://viewer:8080/viewer
+
+RUN mkdir -p /opt/digiverso/indexer
+
+COPY --from=BUILD  /indexer/goobi-viewer-indexer/target/solr-Indexer.jar /usr/local/bin/solrIndexer.jar
+COPY --from=BUILD  /indexer/goobi-viewer-indexer/src/main/resources/indexerconfig_solr.xml /opt/digiverso/indexer/solr_indexerconfig.xml
+COPY ./docker/* /
+RUN sed -e "s|<solrUrl>.*</solrUrl>|<solrUrl>${SOLR_URL}</solrUrl>|" -e 's|C:|/opt|g' -e "s|<viewerUrl>.*</viewerUrl>|<viewerUrl>${VIEWER_URL}</viewerUrl>|" -i /opt/digiverso/indexer/solr_indexerconfig.xml
+
+# TODO: check for solr availability before start (wait-for-solr from solr image?)
+
+CMD ["/run.sh"]
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,25 +1,27 @@
 pipeline {
 
-  agent {
-    docker {
-      image 'nexus.intranda.com:4443/goobi-viewer-testing-index:latest'
-      args '-v $HOME/.m2:/var/maven/.m2:z -v $HOME/.config:/var/maven/.config -v $HOME/.sonar:/var/maven/.sonar -u 1000 -ti -e _JAVA_OPTIONS=-Duser.home=/var/maven -e MAVEN_CONFIG=/var/maven/.m2'
-      registryUrl 'https://nexus.intranda.com:4443/'
-      registryCredentialsId 'jenkins-docker'
-    }
-  }
+  agent none
 
   options {
     buildDiscarder logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '15', daysToKeepStr: '90', numToKeepStr: '')
   }
 
   stages {
     stage('prepare') {
+      agent any
       steps {
         sh 'git clean -fdx'
       }
     }
     stage('build') {
+      agent {
+        docker {
+          image 'nexus.intranda.com:4443/goobi-viewer-testing-index:latest'
+          args '-v $HOME/.m2:/var/maven/.m2:z -v $HOME/.config:/var/maven/.config -v $HOME/.sonar:/var/maven/.sonar -u 1000 -ti -e _JAVA_OPTIONS=-Duser.home=/var/maven -e MAVEN_CONFIG=/var/maven/.m2'
+          registryUrl 'https://nexus.intranda.com:4443/'
+          registryCredentialsId 'jenkins-docker'
+        }
+      }
       steps {
               sh 'mvn -f goobi-viewer-indexer/pom.xml -DskipTests=false clean install -U'
               recordIssues enabledForFailure: true, aggregatingResults: true, tools: [java(), javaDoc()]
@@ -31,13 +33,29 @@ pipeline {
           branch 'sonar_*'
         }
       }
+      agent {
+        docker {
+          image 'nexus.intranda.com:4443/goobi-viewer-testing-index:latest'
+          args '-v $HOME/.m2:/var/maven/.m2:z -v $HOME/.config:/var/maven/.config -v $HOME/.sonar:/var/maven/.sonar -u 1000 -ti -e _JAVA_OPTIONS=-Duser.home=/var/maven -e MAVEN_CONFIG=/var/maven/.m2'
+          registryUrl 'https://nexus.intranda.com:4443/'
+          registryCredentialsId 'jenkins-docker'
+        }
+      }
       steps {
         withCredentials([string(credentialsId: 'jenkins-sonarcloud', variable: 'TOKEN')]) {
           sh 'mvn -f goobi-viewer-indexer/pom.xml verify sonar:sonar -Dsonar.login=$TOKEN'
         }
       }
     }
     stage('deployment to maven repository') {
+      agent {
+        docker {
+          image 'nexus.intranda.com:4443/goobi-viewer-testing-index:latest'
+          args '-v $HOME/.m2:/var/maven/.m2:z -v $HOME/.config:/var/maven/.config -v $HOME/.sonar:/var/maven/.sonar -u 1000 -ti -e _JAVA_OPTIONS=-Duser.home=/var/maven -e MAVEN_CONFIG=/var/maven/.m2'
+          registryUrl 'https://nexus.intranda.com:4443/'
+          registryCredentialsId 'jenkins-docker'
+        }
+      }
       when {
         anyOf {
         branch 'master'
@@ -48,20 +66,95 @@ pipeline {
         sh 'mvn -f goobi-viewer-indexer/pom.xml deploy'
       }
     }
+    stage('build docker image') {
+      agent any
+      steps {
+        script{
+          docker.withRegistry('https://nexus.intranda.com:4443','jenkins-docker'){
+            indexerimage = docker.build("goobi-viewer-indexer:${BRANCH_NAME}-${env.BUILD_ID}_${env.GIT_COMMIT}")
+            indexerimage_public = docker.build("intranda/goobi-viewer-indexer:${BRANCH_NAME}-${env.BUILD_ID}_${env.GIT_COMMIT}")
+          }
+        }
+      }
+    }
+    stage('basic tests'){
+      agent any
+      steps{
+        script {
+          indexerimage.inside {
+            sh 'test -f  /opt/digiverso/indexer/solrIndexer.jar || echo "/opt/digiverso/indexer/solrIndexer.jar missing"'
+          }
+        }
+      }
+    }
+    stage('publish docker devel image to internal repository'){
+      agent any
+      steps{
+        script {
+          docker.withRegistry('https://nexus.intranda.com:4443','jenkins-docker'){
+            indexerimage.push("${env.BRANCH_NAME}-${env.BUILD_ID}_${env.GIT_COMMIT}")
+            indexerimage.push("${env.BRANCH_NAME}")
+          }
+        }
+      }
+    }
+    stage('publish develop image to Docker Hub'){
+      agent any
+      when {
+        branch 'develop'
+      }
+      steps{
+        script{
+          docker.withRegistry('','0b13af35-a2fb-41f7-8ec7-01eaddcbe99d'){
+            indexerimage_public.push("${env.BRANCH_NAME}")
+          }
+        }
+      }
+    }
+    stage('publish docker production image to internal repository'){
+      agent any
+      when { branch 'master' }
+      steps{
+        script {
+          docker.withRegistry('https://nexus.intranda.com:4443','jenkins-docker'){
+            indexerimage.push("${env.TAG_NAME}-${env.BUILD_ID}_${env.GIT_COMMIT}")
+            indexerimage.push("latest")
+          }
+        }
+      }
+    }
+    stage('publish production image to Docker Hub'){
+      agent any
+      when {
+        branch 'master'
+      }
+      steps{
+        script{
+          docker.withRegistry('','0b13af35-a2fb-41f7-8ec7-01eaddcbe99d'){
+            indexerimage_public.push("${env.TAG_NAME}")
+            indexerimage_public.push("latest")
+          }
+        }
+      }
+    }
   }
   post {
     always {
-      junit "**/target/surefire-reports/*.xml"
-      step([
-        $class           : 'JacocoPublisher',
-        execPattern      : 'goobi-viewer-indexer/target/jacoco.exec',
-        classPattern     : 'goobi-viewer-indexer/target/classes/',
-        sourcePattern    : 'goobi-viewer-indexer/src/main/java',
-        exclusionPattern : '**/*Test.class'
-      ])
+      node(null) {
+        junit "**/target/surefire-reports/*.xml"
+        step([
+          $class           : 'JacocoPublisher',
+          execPattern      : 'goobi-viewer-indexer/target/jacoco.exec',
+          classPattern     : 'goobi-viewer-indexer/target/classes/',
+          sourcePattern    : 'goobi-viewer-indexer/src/main/java',
+          exclusionPattern : '**/*Test.class'
+        ])
+      }
     }
     success {
-      archiveArtifacts artifacts: '**/target/*.jar, */src/main/resources/indexerconfig_solr.xml, */src/main/resources/other/schema.xml, */src/main/resources/other/solrindexer.service', fingerprint: true
+      node(null){
+        archiveArtifacts artifacts: '**/target/*.jar, */src/main/resources/indexerconfig_solr.xml, */src/main/resources/other/schema.xml, */src/main/resources/other/solrindexer.service', fingerprint: true
+      }
     }
     changed {
       emailext(

diff --git a/docker/log4j2.xml b/docker/log4j2.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration>
+
+	<Appenders>
+		<Console name="stdout">
+			<PatternLayout pattern="%-5level %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] (%F\:%M\:%L)%n        %msg%n" />
+		</Console>
+	</Appenders>
+
+	<Loggers>
+		<Logger name="io.goobi" level="INFO" additivity="true" />
+		<Logger name="de.intranda.digiverso.ocr" level="WARN" additivity="false" />
+		<Logger name="de.intranda.digiverso.normdataimporter" level="DEBUG" additivity="false" />
+		<Logger name="org.apache" level="ERROR" additivity="true" />
+		<Root level="info">
+			<AppenderRef ref="stdout" />
+		</Root>
+	</Loggers>
+
+</Configuration>
diff --git a/docker/run.sh b/docker/run.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+set -e
+
+[ -z "$CONFIGSOURCE" ] && CONFIGSOURCE="default"
+
+set -u
+
+#if [ -n "${WORKING_STORAGE:-}" ]
+#then
+  #CATALINA_TMPDIR="${WORKING_STORAGE}/goobi/jvmtemp"
+  #mkdir -p "${CATALINA_TMPDIR}"
+  #echo >> /usr/local/tomcat/bin/setenv.sh
+  #echo "CATALINA_TMPDIR=${CATALINA_TMPDIR}" >> /usr/local/tomcat/bin/setenv.sh
+#fi
+
+case $CONFIGSOURCE in
+  # s3)
+  #   if [ -z "$AWS_S3_BUCKET" ]
+  #   then
+  #     echo "AWS_S3_BUCKET is required"
+  #     exit 1
+  #   fi
+  #   echo "Pulling configuration from s3 bucket"
+  #   aws s3 cp s3://$AWS_S3_BUCKET/viewer/config/ /opt/digiverso/viewer/config/ --recursive
+  #   ;;
+  folder)
+    if [ -z "$CONFIG_FOLDER" ]
+    then
+      echo "CONFIG_FOLDER is required"
+      exit 1
+    fi
+
+    if ! [ -d "$CONFIG_FOLDER" ]
+    then
+      echo "CONFIG_FOLDER: $CONFIG_FOLDER does not exists or is not a folder"
+      exit 1
+    fi
+
+    echo "Copying configuration from local folder"
+    [ -d "$CONFIG_FOLDER" ] && cp -arv "$CONFIG_FOLDER"/* /opt/digiverso/indexer/
+    ;;
+
+  *)
+    echo "Keeping configuration"
+    # set solrUrl and viewerUrl from environment variables (defaults are given in Dockerfile)
+    [ -w /opt/digiverso/indexer/solr_indexerconfig.xml ] && sed -e "s|<solrUrl>.*</solrUrl>|<solrUrl>${SOLR_URL}</solrUrl>|" -e "s|<viewerUrl>.*</viewerUrl>|<viewerUrl>${VIEWER_URL}</viewerUrl>|" -i /opt/digiverso/indexer/solr_indexerconfig.xml || echo "Did not modify indexer configuration from environment."
+
+    ;;
+esac
+
+
+echo "Starting application"
+exec java -Dlog4j.configurationFile=/log4j2.xml -jar /usr/local/bin/solrIndexer.jar /opt/digiverso/indexer/solr_indexerconfig.xml