johnynek · johnynek · May 3, 2022 · May 1, 2022
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,212 @@
+# This file was automatically generated by sbt-github-actions using the
+# githubWorkflowGenerate task. You should add and commit this file to
+# your git repository. It goes without saying that you shouldn't edit
+# this file by hand! Instead, if you wish to make changes, you should
+# change your sbt build configuration to revise the workflow description
+# to meet your needs, then regenerate this file.
+
+name: Continuous Integration
+
+on:
+  pull_request:
+    branches: ['**']
+  push:
+    branches: ['**']
+    tags: [v*]
+
+env:
+  PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
+  SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
+  SONATYPE_CREDENTIAL_HOST: ${{ secrets.SONATYPE_CREDENTIAL_HOST }}
+  SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+  PGP_SECRET: ${{ secrets.PGP_SECRET }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  build:
+    name: Build and Test
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        scala: [2.13.3]
+        java: [temurin@8]
+        project: [rootJVM]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout current branch (full)
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Download Java (temurin@8)
+        id: download-java-temurin-8
+        if: matrix.java == 'temurin@8'
+        uses: typelevel/download-java@v1
+        with:
+          distribution: temurin
+          java-version: 8
+
+      - name: Setup Java (temurin@8)
+        if: matrix.java == 'temurin@8'
+        uses: actions/setup-java@v2
+        with:
+          distribution: jdkfile
+          java-version: 8
+          jdkFile: ${{ steps.download-java-temurin-8.outputs.jdkFile }}
+
+      - name: Cache sbt
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.sbt
+            ~/.ivy2/cache
+            ~/.coursier/cache/v1
+            ~/.cache/coursier/v1
+            ~/AppData/Local/Coursier/Cache/v1
+            ~/Library/Caches/Coursier/v1
+          key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }}
+
+      - name: Check that workflows are up to date
+        run: sbt 'project ${{ matrix.project }}' '++${{ matrix.scala }}' 'project /' githubWorkflowCheck
+
+      - name: Check headers and formatting
+        if: matrix.java == 'temurin@8'
+        run: sbt 'project ${{ matrix.project }}' '++${{ matrix.scala }}' headerCheckAll scalafmtCheckAll 'project /' scalafmtSbtCheck
+
+      - name: Test
+        run: sbt 'project ${{ matrix.project }}' '++${{ matrix.scala }}' test
+
+      - name: Check binary compatibility
+        if: matrix.java == 'temurin@8'
+        run: sbt 'project ${{ matrix.project }}' '++${{ matrix.scala }}' mimaReportBinaryIssues
+
+      - name: Generate API documentation
+        if: matrix.java == 'temurin@8'
+        run: sbt 'project ${{ matrix.project }}' '++${{ matrix.scala }}' doc
+
+      - name: Make target directories
+        if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master')
+        run: mkdir -p aws/target target .js/target docs/target .jvm/target .native/target jobs/target core/target bench/target project/target
+
+      - name: Compress target directories
+        if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master')
+        run: tar cf targets.tar aws/target target .js/target docs/target .jvm/target .native/target jobs/target core/target bench/target project/target
+
+      - name: Upload target directories
+        if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master')
+        uses: actions/upload-artifact@v2
+        with:
+          name: target-${{ matrix.os }}-${{ matrix.java }}-${{ matrix.scala }}-${{ matrix.project }}
+          path: targets.tar
+
+  publish:
+    name: Publish Artifacts
+    needs: [build]
+    if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master')
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        scala: [2.13.3]
+        java: [temurin@8]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout current branch (full)
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Download Java (temurin@8)
+        id: download-java-temurin-8
+        if: matrix.java == 'temurin@8'
+        uses: typelevel/download-java@v1
+        with:
+          distribution: temurin
+          java-version: 8
+
+      - name: Setup Java (temurin@8)
+        if: matrix.java == 'temurin@8'
+        uses: actions/setup-java@v2
+        with:
+          distribution: jdkfile
+          java-version: 8
+          jdkFile: ${{ steps.download-java-temurin-8.outputs.jdkFile }}
+
+      - name: Cache sbt
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.sbt
+            ~/.ivy2/cache
+            ~/.coursier/cache/v1
+            ~/.cache/coursier/v1
+            ~/AppData/Local/Coursier/Cache/v1
+            ~/Library/Caches/Coursier/v1
+          key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }}
+
+      - name: Download target directories (2.13.3, rootJVM)
+        uses: actions/download-artifact@v2
+        with:
+          name: target-${{ matrix.os }}-${{ matrix.java }}-2.13.3-rootJVM
+
+      - name: Inflate target directories (2.13.3, rootJVM)
+        run: |
+          tar xf targets.tar
+          rm targets.tar
+
+      - name: Import signing key
+        if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE == ''
+        run: echo $PGP_SECRET | base64 -di | gpg --import
+
+      - name: Import signing key and strip passphrase
+        if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE != ''
+        run: |
+          echo "$PGP_SECRET" | base64 -di > /tmp/signing-key.gpg
+          echo "$PGP_PASSPHRASE" | gpg --pinentry-mode loopback --passphrase-fd 0 --import /tmp/signing-key.gpg
+          (echo "$PGP_PASSPHRASE"; echo; echo) | gpg --command-fd 0 --pinentry-mode loopback --change-passphrase $(gpg --list-secret-keys --with-colons 2> /dev/null | grep '^sec:' | cut --delimiter ':' --fields 5 | tail -n 1)
+
+      - name: Publish
+        run: sbt '++${{ matrix.scala }}' tlRelease
+
+  coverage:
+    name: Generate coverage report
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        scala: [2.13.3]
+        java: [temurin@11]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout current branch (fast)
+        uses: actions/checkout@v2
+
+      - name: Download Java (temurin@8)
+        id: download-java-temurin-8
+        if: matrix.java == 'temurin@8'
+        uses: typelevel/download-java@v1
+        with:
+          distribution: temurin
+          java-version: 8
+
+      - name: Setup Java (temurin@8)
+        if: matrix.java == 'temurin@8'
+        uses: actions/setup-java@v2
+        with:
+          distribution: jdkfile
+          java-version: 8
+          jdkFile: ${{ steps.download-java-temurin-8.outputs.jdkFile }}
+
+      - name: Cache sbt
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.sbt
+            ~/.ivy2/cache
+            ~/.coursier/cache/v1
+            ~/.cache/coursier/v1
+            ~/AppData/Local/Coursier/Cache/v1
+            ~/Library/Caches/Coursier/v1
+          key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }}
+
+      - run: sbt '++${{ matrix.scala }}' coverage root/test coverageAggregate
+
+      - run: 'bash <(curl -s https://codecov.io/bash)'
diff --git a/.github/workflows/clean.yml b/.github/workflows/clean.yml
@@ -0,0 +1,59 @@
+# This file was automatically generated by sbt-github-actions using the
+# githubWorkflowGenerate task. You should add and commit this file to
+# your git repository. It goes without saying that you shouldn't edit
+# this file by hand! Instead, if you wish to make changes, you should
+# change your sbt build configuration to revise the workflow description
+# to meet your needs, then regenerate this file.
+
+name: Clean
+
+on: push
+
+jobs:
+  delete-artifacts:
+    name: Delete Artifacts
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Delete artifacts
+        run: |
+          # Customize those three lines with your repository and credentials:
+          REPO=${GITHUB_API_URL}/repos/${{ github.repository }}
+
+          # A shortcut to call GitHub API.
+          ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; }
+
+          # A temporary file which receives HTTP response headers.
+          TMPFILE=/tmp/tmp.$$
+
+          # An associative array, key: artifact name, value: number of artifacts of that name.
+          declare -A ARTCOUNT
+
+          # Process all artifacts on this repository, loop on returned "pages".
+          URL=$REPO/actions/artifacts
+          while [[ -n "$URL" ]]; do
+
+            # Get current page, get response headers in a temporary file.
+            JSON=$(ghapi --dump-header $TMPFILE "$URL")
+
+            # Get URL of next page. Will be empty if we are at the last page.
+            URL=$(grep '^Link:' "$TMPFILE" | tr ',' '\n' | grep 'rel="next"' | head -1 | sed -e 's/.*<//' -e 's/>.*//')
+            rm -f $TMPFILE
+
+            # Number of artifacts on this page:
+            COUNT=$(( $(jq <<<$JSON -r '.artifacts | length') ))
+
+            # Loop on all artifacts on this page.
+            for ((i=0; $i < $COUNT; i++)); do
+
+              # Get name of artifact and count instances of this name.
+              name=$(jq <<<$JSON -r ".artifacts[$i].name?")
+              ARTCOUNT[$name]=$(( $(( ${ARTCOUNT[$name]} )) + 1))
+
+              id=$(jq <<<$JSON -r ".artifacts[$i].id?")
+              size=$(( $(jq <<<$JSON -r ".artifacts[$i].size_in_bytes?") ))
+              printf "Deleting '%s' #%d, %'d bytes\n" $name ${ARTCOUNT[$name]} $size
+              ghapi -X DELETE $REPO/actions/artifacts/$id
+            done
+          done
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,7 @@ project/target/
 project/metals.sbt
 data/
 .bloop/
+.bsp/
 .metals/
 *.jar
 aws_config/out
diff --git a/aws/src/it/scala/dev/posco/hiona/aws/PuaAwsPostgresTest.scala b/aws/src/it/scala/dev/posco/hiona/aws/PuaAwsPostgresTest.scala
@@ -16,7 +16,7 @@ class PuaAwsPostgresTest extends munit.ScalaCheckSuite {
         20
       ) // a bit slow, but locally, this passes with more
 
-  //override val scalaCheckInitialSeed = "gV_zHGkPkmQY_dODNFYtlHXPSnq6eea0NHyGt-9VNpK="
+  // override val scalaCheckInitialSeed = "gV_zHGkPkmQY_dODNFYtlHXPSnq6eea0NHyGt-9VNpK="
 
   property("postgres puaaws matches what we expect") {
 
@@ -28,26 +28,26 @@ class PuaAwsPostgresTest extends munit.ScalaCheckSuite {
     ) = {
       val computation =
         Blocker[IO]
-        .flatMap { blocker =>
-          PostgresDBControl("sbt_it_tests", "sbtester", "sbtpw", blocker)
-        }
-        .use(H2DBControl.makeComputation(_, pua, dir0, inputs))
+          .flatMap { blocker =>
+            PostgresDBControl("sbt_it_tests", "sbtester", "sbtpw", blocker)
+          }
+          .use(H2DBControl.makeComputation(_, pua, dir0, inputs))
 
       H2DBControl.await30 {
-      (computation.attempt, inputs.traverse(fn).attempt)
-        .mapN {
-          case (Right(a), Right(b)) =>
-            assertEquals(a, b)
-          case (Left(_), Left(_)) =>
-            assert(true)
-          case (left, right) =>
-            // this will fail, but we want to see what we get
-            left match {
-              case Left(err) => err.printStackTrace
-              case Right(_)  => ()
-            }
-            assertEquals(left, right)
-        }
+        (computation.attempt, inputs.traverse(fn).attempt)
+          .mapN {
+            case (Right(a), Right(b)) =>
+              assertEquals(a, b)
+            case (Left(_), Left(_)) =>
+              assert(true)
+            case (left, right) =>
+              // this will fail, but we want to see what we get
+              left match {
+                case Left(err) => err.printStackTrace
+                case Right(_)  => ()
+              }
+              assertEquals(left, right)
+          }
       }
     }
 

diff --git a/aws/src/main/scala/dev/posco/hiona/aws/AWSIO.scala b/aws/src/main/scala/dev/posco/hiona/aws/AWSIO.scala
@@ -1,8 +1,24 @@
+/*
+ * Copyright 2022 devposco
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package dev.posco.hiona.aws
 
 import alex.mojaki.s3upload.StreamTransferManager
-import cats.effect.{Blocker, ContextShift, IO, Resource, Sync}
 import cats.effect.concurrent.Ref
+import cats.effect.{Blocker, ContextShift, IO, Resource, Sync}
 import com.amazonaws.services.s3
 import com.amazonaws.services.s3.model.GetObjectRequest
 import dev.posco.hiona.Row

diff --git a/aws/src/main/scala/dev/posco/hiona/aws/ContentStore.scala b/aws/src/main/scala/dev/posco/hiona/aws/ContentStore.scala
@@ -1,8 +1,24 @@
+/*
+ * Copyright 2022 devposco
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package dev.posco.hiona.aws
 
 import cats.effect.{Blocker, ContextShift, Sync}
-import java.nio.file.Path
 import com.amazonaws.services.s3
+import java.nio.file.Path
 import org.apache.commons.codec.binary.Base32
 import org.slf4j.LoggerFactory