add public + shared fields to ext_index.json

neondatabase · Jul 26, 2023 · df69b9c · df69b9c
1 parent 5f8fd64
commit df69b9c
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 5 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -944,9 +944,9 @@ jobs:
         run: rm -rf ~/.ecr
 
   upload-postgres-extensions-to-s3:
-    if: |
-      (github.ref_name == 'main' || github.ref_name == 'release') &&
-       github.event_name != 'workflow_dispatch'
+    # if: |
+    #   (github.ref_name == 'main' || github.ref_name == 'release') &&
+    #    github.event_name != 'workflow_dispatch'
     runs-on: ${{ github.ref_name == 'release' && fromJSON('["self-hosted", "prod", "x64"]') || fromJSON('["self-hosted", "gen3", "small"]') }}
     needs: [ tag, promote-images ]
     strategy:

diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node
@@ -779,7 +779,7 @@ COPY --from=kq-imcx-pg-build /extensions/ /extensions/
 COPY --from=pg-anon-pg-build /extensions/ /extensions/
 COPY --from=postgis-build /extensions/ /extensions/
 COPY scripts/combine_control_files.py ./combine_control_files.py
-RUN python3 ./combine_control_files.py ${PG_VERSION} ${BUILD_TAG}
+RUN python3 ./combine_control_files.py ${PG_VERSION} ${BUILD_TAG} --public_extensions="postgis,anon"
 
 FROM scratch AS postgres-extensions
 # After the transition this layer will include all extensitons.

diff --git a/scripts/combine_control_files.py b/scripts/combine_control_files.py
@@ -6,15 +6,46 @@
 import json
 from pathlib import Path
 
+"""
+# ext_index.json example:
+{
+    "public_extensions": [
+        "anon"
+    ],
+    "library_index": {
+        "anon": "anon",
+        "kq_imcx": "kq_imcx"
+        // would be more complicated for something like postgis where multiple library names all map to postgis
+    },
+    "extension_data": {
+        "kq_imcx": {
+            "control_data": {
+                "kq_imcx.control": "# This file is generated content from add_postgresql_extension.\n# No point in modifying it, it will be overwritten anyway.\n\n# Default version, always set\ndefault_version = '0.1'\n\n# Module pathname generated from target shared library name. Use\n# MODULE_PATHNAME in script file.\nmodule_pathname = '$libdir/kq_imcx.so'\n\n# Comment for extension. Set using COMMENT option. Can be set in\n# script file as well.\ncomment = 'ketteQ In-Memory Calendar Extension (IMCX)'\n\n# Encoding for script file. Set using ENCODING option.\n#encoding = ''\n\n# Required extensions. Set using REQUIRES option (multi-valued).\n#requires = ''\ntrusted = true\n"
+            },
+            "archive_path": "5648391853/v15/extensions/kq_imcx.tar.zst"
+        },
+        "anon": {
+            "control_data": {
+                "anon.control": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n"
+            },
+            "archive_path": "5648391853/v15/extensions/anon.tar.zst"
+        }
+    }
+}
+"""
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="generate ext_index.json")
     parser.add_argument("pg_version", type=str, choices=["v14", "v15"], help="pg_version")
     parser.add_argument("BUILD_TAG", type=str, help="BUILD_TAG for this compute image")
+    parser.add_argument("--public_extensions", type=str, help="list of public extensions")
     args = parser.parse_args()
     pg_version = args.pg_version
     BUILD_TAG = args.BUILD_TAG
+    public_ext_list = args.public_extensions.split(',')
 
     ext_index = {}
+    library_index = {}
     EXT_PATH = Path("extensions")
     for extension in EXT_PATH.iterdir():
         if extension.is_dir():
@@ -28,6 +59,22 @@
                 "control_data": control_data,
                 "archive_path": f"{BUILD_TAG}/{pg_version}/extensions/{extension.name}.tar.zst",
             }
+        elif extension.suffix == ".tar.zst":
+            import subprocess
+
+            file_list = str(
+                subprocess.check_output(["tar" "tf" "{extension.name}.tar.zst"]), "utf-8"
+            ).strip().split("\n")
+            for file in file_list:
+                if ".so" in file:
+                    lib_name = file.replace("lib/", "").replace(".so", "")
+                    library_index[lib_name] = extension.name
 
+    all_data = {
+        "public_extensions": public_ext_list,
+        "library_index": library_index,
+        "extension_data": ext_index,
+    }
     with open("ext_index.json", "w") as f:
-        json.dump(ext_index, f)
+        json.dump(all_data, f)
+