Merge branch 'integration' into feature/fiReindex-integration

NationalSecurityAgency · Dec 3, 2024 · fc4e826 · fc4e826
2 parents 0ebab44 + 3b31c16
commit fc4e826
Show file tree

Hide file tree

Showing 621 changed files with 219,319 additions and 10,450 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
diff --git a/.github/workflows/build-accumulo.yml b/.github/workflows/build-accumulo.yml
@@ -0,0 +1,99 @@
+name: Build Accumulo snapshot and update DataWave to use
+
+on:
+  workflow_dispatch:
+    inputs:
+      accumuloBranch:
+        required: true
+        default: "2.1"
+        description: "Branch name to build. Will be used as image tag."
+      accumuloRepo:
+        required: true
+        default: "apache/accumulo"
+        description: "Accumulo Repo to use. Expected to be at Github. Example: apache/accumulo"
+      deployAccumulo:
+        required: true
+        default: "false"
+        description: "Set to false if this accumulo version has already been pushed to Github Packages"
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository.lowercase }}
+  ACCUMULO_JAVA_VERSION: '17'
+  DATAWAVE_JAVA_VERSION: '11'
+  JAVA_DISTRIBUTION: 'zulu' #This is the default on v1 of the action for 1.8
+  USER_NAME: ${{ secrets.GHCR_WRITE_USER_NAME }}
+  ACCESS_TOKEN: ${{ secrets.GHCR_WRITE_ACCESS_TOKEN }}
+
+jobs:
+  build-and-deploy-accumulo:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      #
+    steps:
+    - name: Checkout DataWave
+      uses: actions/checkout@v4
+      with:
+        path: datawave
+
+    - name: Checkout Accumulo
+      uses: actions/checkout@v4
+      id: accumuloCheckout
+      with:
+        repository: ${{ github.event.inputs.accumuloRepo }}
+        path: accumulo
+        ref: ${{ github.event.inputs.accumuloBranch }}
+
+    - name: Set up JDK ${{env.ACCUMULO_JAVA_VERSION}}
+      uses: actions/setup-java@v4
+      with:
+        distribution: ${{env.JAVA_DISTRIBUTION}}
+        java-version: ${{env.ACCUMULO_JAVA_VERSION}}
+        cache: 'maven'
+    - run: echo "ACCUMULO_JAVA=$JAVA_HOME" >> $GITHUB_ENV
+    - name: Set up JDK ${{env.DATAWAVE_JAVA_VERSION}}
+      uses: actions/setup-java@v4
+      with:
+        distribution: ${{env.JAVA_DISTRIBUTION}}
+        java-version: ${{env.DATAWAVE_JAVA_VERSION}}
+        cache: 'maven'
+    - run: echo "DATAWAVE_JAVA=$JAVA_HOME" >> $GITHUB_ENV
+
+    - name: Get Accumulo Version
+      id: get-accumulo-version
+      run: |
+        export JAVA_HOME="$ACCUMULO_JAVA"
+        cd "$GITHUB_WORKSPACE/accumulo"
+        mvn build-helper:parse-version versions:set -DgenerateBackupPoms=false -DnewVersion=\${parsedVersion.majorVersion}.\${parsedVersion.minorVersion}.\${parsedVersion.incrementalVersion}-dwv-$(git rev-parse --short HEAD)
+        export newVersion=$(mvn -q help:evaluate -DforceStdout -Dexpression=project.version)
+        echo accumuloVersion=$newVersion >> $GITHUB_OUTPUT
+    - name: Deploy Accumulo
+      if: ${{ github.event.inputs.deployAccumulo == 'true'}}
+      run: | 
+        export JAVA_HOME="$ACCUMULO_JAVA"
+        cd "$GITHUB_WORKSPACE/accumulo"
+        mvn -DaltDeploymentRepository=github-datawave::https://maven.pkg.github.com/NationalSecurityAgency/datawave -V -B -e -ntp "-Dstyle.color=always" -DskipTests -T1C clean source:jar deploy -s "$GITHUB_WORKSPACE/datawave/.github/workflows/settings.xml"
+    - name: Log in to the Container registry
+      uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ env.USER_NAME }}
+        password: ${{ env.ACCESS_TOKEN }}
+
+    - name: Update DataWave Dependency Version
+      run: |
+
+        export JAVA_HOME="$DATAWAVE_JAVA"
+        cd "$GITHUB_WORKSPACE/datawave"
+        mvn -s "$GITHUB_WORKSPACE/datawave/.github/workflows/settings.xml" versions:set-property -Dproperty=version.accumulo -DnewVersion=${{ steps.get-accumulo-version.outputs.accumuloVersion }} -DgenerateBackupPoms=false
+    - name: Build Web and Ingest Docker Images (Maven)
+      run: |
+        export JAVA_HOME="$DATAWAVE_JAVA"
+        cd "$GITHUB_WORKSPACE/datawave"
+        mvn -s "$GITHUB_WORKSPACE/datawave/.github/workflows/settings.xml" clean install -Prpm,kubernetes,assemble,deploy-ws -Ddist -Pdocker -DpushImage -Ddocker-release -DskipTests -Ddocker.image.accumulo.tag=${{ steps.get-accumulo-version.outputs.accumuloVersion }}
+
+ 
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -32,9 +32,9 @@ jobs:
         username: ${{ env.USER_NAME }}
         password: ${{ env.ACCESS_TOKEN }}
     - name: Checkout Code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Set up JDK ${{env.JAVA_VERSION}}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: ${{env.JAVA_DISTRIBUTION}}
         java-version: ${{env.JAVA_VERSION}}

diff --git a/.github/workflows/microservice-build-image.yaml b/.github/workflows/microservice-build-image.yaml
@@ -0,0 +1,41 @@
+#
+name: Create and publish a Docker image
+
+on:
+  workflow_call:
+    secrets:
+      USER_NAME:
+        description: "User Name for maven pulls"
+        required: true
+      ACCESS_TOKEN:
+        description: "Access token for maven pulls"
+        required: true
+
+
+jobs:
+  build-and-push-datawave-images:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Log in to the Container registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ secrets.USER_NAME }}
+        password: ${{ secrets.ACCESS_TOKEN }}
+    - name: Checkout Code
+      uses: actions/checkout@v4
+    - name: Set up JDK 11
+      uses: actions/setup-java@v4
+      with:
+        distribution: "zulu"
+        java-version: 11
+        cache: 'maven'
+    - name: Build And Push Docker Image (Maven)
+      env:
+        MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
+        USER_NAME: ${{ secrets.USER_NAME }}
+        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+      run: |
+        mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e clean install -Pdocker,exec -Ddocker.image.prefix=ghcr.io/nationalsecurityagency/ -DpushImage
+
+ 
diff --git a/.github/workflows/microservice-maven-tests.yaml b/.github/workflows/microservice-maven-tests.yaml
@@ -0,0 +1,72 @@
+name: Tests
+
+on:
+  workflow_call:
+    secrets:
+      USER_NAME:
+        description: "User Name for maven pulls"
+        required: true
+      ACCESS_TOKEN:
+        description: "Access token for maven pulls"
+        required: true
+
+env:
+  MAVEN_OPTS: "-Djansi.force=true -Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true -XX:ThreadStackSize=1m"
+
+jobs:
+  # Runs the pom sorter and code formatter to ensure that the code
+  # is formatted and poms are sorted according to project rules. This
+  # will fail if the formatter makes any changes.
+  check-code-formatting:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4
+    - name: Set up JDK 11
+      uses: actions/setup-java@v4
+      with:
+        java-version: 11
+        distribution: 'zulu'
+    - uses: actions/cache@v4
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-format-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-format-
+          ${{ runner.os }}-maven-
+    - name: Format code
+      run: |
+        mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e clean formatter:format sortpom:sort -Pautoformat
+        git status
+        git diff-index --quiet HEAD || (echo "Error! There are modified files after formatting." && false)
+      env:
+        MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
+        USER_NAME: ${{ secrets.USER_NAME }}
+        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+
+  # Build the code and run the unit/integration tests.
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4
+    - name: Set up JDK 11
+      uses: actions/setup-java@v4
+      with:
+        java-version: 11
+        distribution: 'zulu'
+    - uses: actions/cache@v4
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-build-
+          ${{ runner.os }}-maven-format-
+          ${{ runner.os }}-maven-
+    - name: Build and Run Unit Tests
+      run: mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Ddist clean verify
+      env:
+        MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
+        USER_NAME: ${{ secrets.USER_NAME }}
+        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+