Merge pull request #3542 from magda-io/issue/3541-indexer-doc

Issue/3541 & 3543 region files & region mappings

CHANGES.md

@@ -3,6 +3,8 @@
 ## v4.0.1
 
 - Add helm chart configuration option allows users to disable gateway auto-gzip response feature
+- #3541 Update default region files url to [Github repo release](https://github.com/magda-io/magda-regions)
+- #3543 Make the region mapping served by search api configurable
 
 ## v4.0.0
 

deploy/helm/internal-charts/indexer/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-description: A Helm chart for Kubernetes
+description: A Helm chart for Magda's Indexer service.
 name: indexer
 version: 4.0.0
 kubeVersion: ">= 1.21.0"

deploy/helm/internal-charts/indexer/README.md

@@ -2,7 +2,13 @@
 
 ![Version: 4.0.0](https://img.shields.io/badge/Version-4.0.0-informational?style=flat-square)
 
-A Helm chart for Kubernetes
+A Helm chart for Magda's Indexer service.
+
+The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-regions/releases).
+
+For production deployment, you might want to host those region mapping files yourself in a more reliable way (e.g. put into a storage bucket).
+
+To config region mapping files, please refer to the this repo: https://github.com/magda-io/magda-regions
 
 ## Requirements
 
@@ -31,6 +37,3 @@ Kubernetes: `>= 1.21.0`
 | resources.limits.cpu | string | `"250m"` |  |
 | resources.requests.cpu | string | `"100m"` |  |
 | resources.requests.memory | string | `"250Mi"` |  |
-
-----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)

deploy/helm/internal-charts/indexer/README.md.gotmpl

@@ -0,0 +1,24 @@
+{{ template "chart.header" . }}
+{{ template "chart.deprecationWarning" . }}
+
+{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }}
+
+{{ template "chart.description" . }}
+
+The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-regions/releases).
+
+For production deployment, you might want to host those region mapping files yourself in a more reliable way (e.g. put into a storage bucket).
+
+To config region mapping files, please refer to the this repo: https://github.com/magda-io/magda-regions
+
+{{ template "chart.homepageLine" . }}
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.sourcesSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+{{ template "chart.valuesHeader" . }}
+
+{{ template "chart.valuesTable" . }}

docs/docs/index.md

@@ -22,5 +22,6 @@
 - [How to recover with continuous archive backup](./how-to-recover-with-continuous-archive-backup.md)
 - [Ports used when running locally](./local-ports.md)
 - [Windows Setup Instructions](./windows-instructions.md)
+- [Regions Config](./region-files.md)
 
 More documentation, please check [here](https://github.com/magda-io/magda/tree/master/docs/docs)

docs/docs/region-files.md

@@ -0,0 +1,9 @@
+### Region Mapping Files
+
+Magda allow users to supply a list of region mapping files that contains a list of regions available as region search filter options. The region mapping files will be fetched via HTTP protocol by Magda's [indexer](https://github.com/magda-io/magda/tree/main/deploy/helm/internal-charts/indexer) module when it's required to create region index in search engine (e.g. for the first deployment).
+
+The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-regions/releases).
+
+For production deployment, you might want to host those region mapping files yourself in a more reliable way (e.g. put into a storage bucket).
+
+To config indexer to use different region mapping files or if you want to host region mapping files in your own facility, please check this repo for more details: https://github.com/magda-io/magda-regions

magda-elastic-search/package.json

@@ -19,49 +19,49 @@
     },
     "regionSources": {
       "COUNTRY": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/country.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson",
         "idField": "id",
         "nameField": "name",
         "order": 9
       },
       "OFFSHORE_TERRITORIES": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/off-shore-territories.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/off-shore-territories.geojson",
         "idField": "id",
         "nameField": "name",
         "order": 11
       },
       "SA4": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA4.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA4.geojson",
         "idField": "SA4_CODE11",
         "nameField": "SA4_NAME11",
         "order": 30
       },
       "SA3": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA3.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA3.geojson",
         "idField": "SA3_CODE11",
         "nameField": "SA3_NAME11",
         "order": 40
       },
       "SA2": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA2.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA2.geojson",
         "idField": "SA2_MAIN11",
         "nameField": "SA2_NAME11",
         "order": 50
       },
       "LGA": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/LGA.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/LGA.geojson",
         "idField": "LGA_CODE15",
         "nameField": "LGA_NAME15",
         "order": 20
       },
       "COM_ELB_ID_2016": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/COM_ELB_ID_2016.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/COM_ELB_ID_2016.geojson",
         "idField": "DIV_ID",
         "nameField": "SORTNAME",
         "order": 80
       },
       "STE": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/STE.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/STE.geojson",
         "idField": "STE_CODE11",
         "nameField": "STE_NAME11",
         "shortNameField": "STE_ABBREV",

magda-elastic-search/regionSynonyms.txt

@@ -1,14 +1,14 @@
 
 
 # Region Type: COUNTRY
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/country.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson
 
 australia (mainland), australia => country/1
 australian offshore remote territories => country/2
 
 
 # Region Type: OFFSHORE_TERRITORIES
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/off-shore-territories.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/off-shore-territories.geojson
 
 norfolk island => offshore_territories/21
 christmas island => offshore_territories/22
@@ -21,7 +21,7 @@ heard island and mcdonald islands => offshore_territories/28
 
 
 # Region Type: SA4
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA4.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA4.geojson
 
 capital region => sa4/101
 central coast => sa4/102
@@ -132,7 +132,7 @@ special purpose codes sa4 (ot), special purpose codes sa4 => sa4/999
 
 
 # Region Type: SA3
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA3.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA3.geojson
 
 goulburn - yass => sa3/10101
 queanbeyan => sa3/10102
@@ -488,7 +488,7 @@ special purpose codes sa3 (ot), special purpose codes sa3 => sa3/99999
 
 
 # Region Type: SA2
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA2.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA2.geojson
 
 goulburn => sa2/101011001
 goulburn region => sa2/101011002
@@ -2707,7 +2707,7 @@ no usual address (ot), no usual address => sa2/999999499
 
 
 # Region Type: LGA
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/LGA.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/LGA.geojson
 
 albury (c), albury => lga/10050
 armidale dumaresq (a), armidale dumaresq => lga/10110
@@ -3293,7 +3293,7 @@ no usual address (ot), no usual address => lga/99499
 
 
 # Region Type: COM_ELB_ID_2016
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/COM_ELB_ID_2016.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/COM_ELB_ID_2016.geojson
 
 lingiari => com_elb_id_2016/306
 solomon => com_elb_id_2016/307
@@ -3448,7 +3448,7 @@ tangney => com_elb_id_2016/248
 
 
 # Region Type: STE
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/STE.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/STE.geojson
 
 new south wales, nsw => ste/1
 victoria, vic => ste/2

magda-int-test-ts/indexer-setup.conf

@@ -7,7 +7,7 @@ registry.registerForWebhooks = false
 # only enable / use one region file for speeding up the initialisation
 regionSources = {
     COUNTRY {
-        url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/country.geojson"
+        url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson"
         idField = "id"
         nameField = "name"
         order = 9

magda-opensearch/package.json

@@ -19,49 +19,49 @@
     },
     "regionSources": {
       "COUNTRY": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/country.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson",
         "idField": "id",
         "nameField": "name",
         "order": 9
       },
       "OFFSHORE_TERRITORIES": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/off-shore-territories.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/off-shore-territories.geojson",
         "idField": "id",
         "nameField": "name",
         "order": 11
       },
       "SA4": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA4.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA4.geojson",
         "idField": "SA4_CODE11",
         "nameField": "SA4_NAME11",
         "order": 30
       },
       "SA3": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA3.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA3.geojson",
         "idField": "SA3_CODE11",
         "nameField": "SA3_NAME11",
         "order": 40
       },
       "SA2": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA2.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA2.geojson",
         "idField": "SA2_MAIN11",
         "nameField": "SA2_NAME11",
         "order": 50
       },
       "LGA": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/LGA.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/LGA.geojson",
         "idField": "LGA_CODE15",
         "nameField": "LGA_NAME15",
         "order": 20
       },
       "COM_ELB_ID_2016": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/COM_ELB_ID_2016.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/COM_ELB_ID_2016.geojson",
         "idField": "DIV_ID",
         "nameField": "SORTNAME",
         "order": 80
       },
       "STE": {
-        "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/STE.geojson",
+        "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/STE.geojson",
         "idField": "STE_CODE11",
         "nameField": "STE_NAME11",
         "shortNameField": "STE_ABBREV",

magda-opensearch/regionSynonyms.txt

@@ -1,14 +1,14 @@
 
 
 # Region Type: COUNTRY
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/country.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson
 
 australia (mainland), australia => country/1
 australian offshore remote territories => country/2
 
 
 # Region Type: OFFSHORE_TERRITORIES
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/off-shore-territories.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/off-shore-territories.geojson
 
 norfolk island => offshore_territories/21
 christmas island => offshore_territories/22
@@ -21,7 +21,7 @@ heard island and mcdonald islands => offshore_territories/28
 
 
 # Region Type: SA4
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA4.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA4.geojson
 
 capital region => sa4/101
 central coast => sa4/102
@@ -132,7 +132,7 @@ special purpose codes sa4 (ot), special purpose codes sa4 => sa4/999
 
 
 # Region Type: SA3
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA3.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA3.geojson
 
 goulburn - yass => sa3/10101
 queanbeyan => sa3/10102
@@ -488,7 +488,7 @@ special purpose codes sa3 (ot), special purpose codes sa3 => sa3/99999
 
 
 # Region Type: SA2
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA2.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA2.geojson
 
 goulburn => sa2/101011001
 goulburn region => sa2/101011002
@@ -2707,7 +2707,7 @@ no usual address (ot), no usual address => sa2/999999499
 
 
 # Region Type: LGA
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/LGA.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/LGA.geojson
 
 albury (c), albury => lga/10050
 armidale dumaresq (a), armidale dumaresq => lga/10110
@@ -3293,7 +3293,7 @@ no usual address (ot), no usual address => lga/99499
 
 
 # Region Type: COM_ELB_ID_2016
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/COM_ELB_ID_2016.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/COM_ELB_ID_2016.geojson
 
 lingiari => com_elb_id_2016/306
 solomon => com_elb_id_2016/307
@@ -3448,7 +3448,7 @@ tangney => com_elb_id_2016/248
 
 
 # Region Type: STE
-# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/STE.geojson
+# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/STE.geojson
 
 new south wales, nsw => ste/1
 victoria, vic => ste/2