From 001446b694fb3cd1e5e585ac3aa2cd6f56dbef55 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 19 Jun 2024 17:39:47 +1000 Subject: [PATCH 1/7] related to #3541: update the default region mapping files urls to --- CHANGES.md | 1 + .../src/main/resources/common.conf | 23 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6b112e102b..25e49e9991 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,7 @@ ## v4.0.1 - Add helm chart configuration option allows users to disable gateway auto-gzip response feature +- Update default region mapping files url to [Github repo release](https://github.com/magda-io/magda-region-mappings) ## v4.0.0 diff --git a/magda-scala-common/src/main/resources/common.conf b/magda-scala-common/src/main/resources/common.conf index 387bc7c914..f4cfc8282d 100755 --- a/magda-scala-common/src/main/resources/common.conf +++ b/magda-scala-common/src/main/resources/common.conf @@ -88,23 +88,24 @@ logging { level = "INFO" } - +# default region mapping files. +# See repo here: https://github.com/magda-io/magda-region-mappings regionSources = { COUNTRY { - url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/country.geojson" + url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson" idField = "id" nameField = "name" order = 9 } OFFSHORE_TERRITORIES { - url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/off-shore-territories.geojson" + url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/off-shore-territories.geojson" idField = "id" nameField = "name" lv1Id = "2" order = 11 } SA4 { - url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA4.geojson" + url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA4.geojson" idField = "SA4_CODE11" nameField = "SA4_NAME11" lv1Id = "1" @@ -112,7 +113,7 @@ regionSources = { order = 30 } SA3 { - url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA3.geojson" + url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA3.geojson" idField = "SA3_CODE11" nameField = "SA3_NAME11" lv1Id = "1" @@ -121,7 +122,7 @@ regionSources = { order = 40 } SA2 { - url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA2.geojson" + url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA2.geojson" idField = "SA2_MAIN11" nameField = "SA2_NAME11" lv1Id = "1" @@ -131,7 +132,7 @@ regionSources = { order = 50 } SA1 { - url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA1.geojson" + url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA1.geojson" idField = "SA1_MAIN11" nameField = "SA1_MAIN11", lv1Id = "1" @@ -143,7 +144,7 @@ regionSources = { order = 60 } LGA { - url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/LGA.geojson" + url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/LGA.geojson" idField = "LGA_CODE15" nameField = "LGA_NAME15" lv1Id = "1" @@ -151,21 +152,21 @@ regionSources = { order = 20 } POA { - url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/POA.geojson" + url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/POA.geojson" idField = "POA_CODE" nameField = "POA_NAME" lv1Id = "1" order = 70 } COM_ELB_ID_2016 { - url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/COM_ELB_ID_2016.geojson" + url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/COM_ELB_ID_2016.geojson" idField = "DIV_ID" nameField = "SORTNAME" lv1Id = "1" order = 80 } STE { - url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/STE.simplified.geojson" + url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/STE.simplified.geojson" idField = "STE_CODE11" nameField = "STE_NAME11" shortNameField = "STE_ABBREV" From 41d0472ef708b33516d970530e34ef296134fc31 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 19 Jun 2024 21:39:30 +1000 Subject: [PATCH 2/7] update indexer docs --- .../helm/internal-charts/indexer/Chart.yaml | 2 +- deploy/helm/internal-charts/indexer/README.md | 7 +++---- .../internal-charts/indexer/README.md.gotmpl | 20 +++++++++++++++++++ docs/docs/index.md | 1 + docs/docs/region-mapping-files.md | 5 +++++ .../src/main/resources/common.conf | 10 ++++++++++ 6 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 deploy/helm/internal-charts/indexer/README.md.gotmpl create mode 100644 docs/docs/region-mapping-files.md diff --git a/deploy/helm/internal-charts/indexer/Chart.yaml b/deploy/helm/internal-charts/indexer/Chart.yaml index ee6116e83e..3000437315 100644 --- a/deploy/helm/internal-charts/indexer/Chart.yaml +++ b/deploy/helm/internal-charts/indexer/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -description: A Helm chart for Kubernetes +description: A Helm chart for Magda's Indexer service name: indexer version: 4.0.0 kubeVersion: ">= 1.21.0" diff --git a/deploy/helm/internal-charts/indexer/README.md b/deploy/helm/internal-charts/indexer/README.md index 8c01ca180f..9f8e5bc85a 100644 --- a/deploy/helm/internal-charts/indexer/README.md +++ b/deploy/helm/internal-charts/indexer/README.md @@ -2,7 +2,9 @@ ![Version: 4.0.0](https://img.shields.io/badge/Version-4.0.0-informational?style=flat-square) -A Helm chart for Kubernetes +A Helm chart for Magda's Indexer service + +To config region mapping files, please refer to the this repo: https://github.com/magda-io/magda-region-mappings ## Requirements @@ -31,6 +33,3 @@ Kubernetes: `>= 1.21.0` | resources.limits.cpu | string | `"250m"` | | | resources.requests.cpu | string | `"100m"` | | | resources.requests.memory | string | `"250Mi"` | | - ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1) diff --git a/deploy/helm/internal-charts/indexer/README.md.gotmpl b/deploy/helm/internal-charts/indexer/README.md.gotmpl new file mode 100644 index 0000000000..2b352ac1de --- /dev/null +++ b/deploy/helm/internal-charts/indexer/README.md.gotmpl @@ -0,0 +1,20 @@ +{{ template "chart.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +{{ template "chart.description" . }} + +To config region mapping files, please refer to the this repo: https://github.com/magda-io/magda-region-mappings + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesHeader" . }} + +{{ template "chart.valuesTable" . }} diff --git a/docs/docs/index.md b/docs/docs/index.md index 878e3d4c21..80b802e729 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -22,5 +22,6 @@ - [How to recover with continuous archive backup](./how-to-recover-with-continuous-archive-backup.md) - [Ports used when running locally](./local-ports.md) - [Windows Setup Instructions](./windows-instructions.md) +- [Region Mapping Files Config](./region-mapping-files.md) More documentation, please check [here](https://github.com/magda-io/magda/tree/master/docs/docs) diff --git a/docs/docs/region-mapping-files.md b/docs/docs/region-mapping-files.md new file mode 100644 index 0000000000..b70153a801 --- /dev/null +++ b/docs/docs/region-mapping-files.md @@ -0,0 +1,5 @@ +### Region Mapping Files + +Magda allow users to supply a list of region mapping files that contains a list of regions available as region search filter options. The region mapping files will be fetched via HTTP protocol by Magda's [indexer](https://github.com/magda-io/magda/tree/main/deploy/helm/internal-charts/indexer) module when it's required to create region index in search engine (e.g. for the first deployment). + +To config indexer to use different region mapping files or if you want to host region mapping files in your own facility, please check this repo for more details: https://github.com/magda-io/magda-region-mappings diff --git a/magda-scala-common/src/main/resources/common.conf b/magda-scala-common/src/main/resources/common.conf index f4cfc8282d..e29339cabc 100755 --- a/magda-scala-common/src/main/resources/common.conf +++ b/magda-scala-common/src/main/resources/common.conf @@ -91,12 +91,14 @@ logging { # default region mapping files. # See repo here: https://github.com/magda-io/magda-region-mappings regionSources = { + # Australia (Mainland) and all offshore territories as a whole COUNTRY { url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson" idField = "id" nameField = "name" order = 9 } + # Regions for each of Australia offshore territories OFFSHORE_TERRITORIES { url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/off-shore-territories.geojson" idField = "id" @@ -104,6 +106,7 @@ regionSources = { lv1Id = "2" order = 11 } + # ABS Statistical Area Level 4 SA4 { url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA4.geojson" idField = "SA4_CODE11" @@ -112,6 +115,7 @@ regionSources = { lv2IdField = "STE_CODE11" order = 30 } + # ABS Statistical Area Level 3 SA3 { url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA3.geojson" idField = "SA3_CODE11" @@ -121,6 +125,7 @@ regionSources = { lv3IdField = "SA4_CODE11" order = 40 } + # ABS Statistical Area Level 2 SA2 { url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA2.geojson" idField = "SA2_MAIN11" @@ -131,6 +136,7 @@ regionSources = { lv4IdField = "SA3_CODE11" order = 50 } + # ABS Statistical Area Level 1 SA1 { url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA1.geojson" idField = "SA1_MAIN11" @@ -143,6 +149,7 @@ regionSources = { includeIdInName = false, order = 60 } + # Australia Local Government Areas LGA { url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/LGA.geojson" idField = "LGA_CODE15" @@ -151,6 +158,7 @@ regionSources = { steIdField = "STE_CODE11" order = 20 } + # Australia Postal Areas POA { url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/POA.geojson" idField = "POA_CODE" @@ -158,6 +166,7 @@ regionSources = { lv1Id = "1" order = 70 } + # Australia Commonwealth electoral boundaries COM_ELB_ID_2016 { url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/COM_ELB_ID_2016.geojson" idField = "DIV_ID" @@ -165,6 +174,7 @@ regionSources = { lv1Id = "1" order = 80 } + # Australia State and Territory STE { url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/STE.simplified.geojson" idField = "STE_CODE11" From 046e0212429cb8990d293afdd9ccc5bb18d28b26 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 19 Jun 2024 21:44:14 +1000 Subject: [PATCH 3/7] update indexer docs --- deploy/helm/internal-charts/indexer/Chart.yaml | 2 +- deploy/helm/internal-charts/indexer/README.md | 6 +++++- deploy/helm/internal-charts/indexer/README.md.gotmpl | 4 ++++ docs/docs/region-mapping-files.md | 4 ++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/deploy/helm/internal-charts/indexer/Chart.yaml b/deploy/helm/internal-charts/indexer/Chart.yaml index 3000437315..8e5114bbed 100644 --- a/deploy/helm/internal-charts/indexer/Chart.yaml +++ b/deploy/helm/internal-charts/indexer/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -description: A Helm chart for Magda's Indexer service +description: A Helm chart for Magda's Indexer service. name: indexer version: 4.0.0 kubeVersion: ">= 1.21.0" diff --git a/deploy/helm/internal-charts/indexer/README.md b/deploy/helm/internal-charts/indexer/README.md index 9f8e5bc85a..4bbdb3caaf 100644 --- a/deploy/helm/internal-charts/indexer/README.md +++ b/deploy/helm/internal-charts/indexer/README.md @@ -2,7 +2,11 @@ ![Version: 4.0.0](https://img.shields.io/badge/Version-4.0.0-informational?style=flat-square) -A Helm chart for Magda's Indexer service +A Helm chart for Magda's Indexer service. + +The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-region-mappings/releases). + +For production deployment, you might want to host those region mapping files yourself in a more reliable way (e.g. put into a storage bucket). To config region mapping files, please refer to the this repo: https://github.com/magda-io/magda-region-mappings diff --git a/deploy/helm/internal-charts/indexer/README.md.gotmpl b/deploy/helm/internal-charts/indexer/README.md.gotmpl index 2b352ac1de..9233a961ab 100644 --- a/deploy/helm/internal-charts/indexer/README.md.gotmpl +++ b/deploy/helm/internal-charts/indexer/README.md.gotmpl @@ -5,6 +5,10 @@ {{ template "chart.description" . }} +The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-region-mappings/releases). + +For production deployment, you might want to host those region mapping files yourself in a more reliable way (e.g. put into a storage bucket). + To config region mapping files, please refer to the this repo: https://github.com/magda-io/magda-region-mappings {{ template "chart.homepageLine" . }} diff --git a/docs/docs/region-mapping-files.md b/docs/docs/region-mapping-files.md index b70153a801..e41bfd8317 100644 --- a/docs/docs/region-mapping-files.md +++ b/docs/docs/region-mapping-files.md @@ -2,4 +2,8 @@ Magda allow users to supply a list of region mapping files that contains a list of regions available as region search filter options. The region mapping files will be fetched via HTTP protocol by Magda's [indexer](https://github.com/magda-io/magda/tree/main/deploy/helm/internal-charts/indexer) module when it's required to create region index in search engine (e.g. for the first deployment). +The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-region-mappings/releases). + +For production deployment, you might want to host those region mapping files yourself in a more reliable way (e.g. put into a storage bucket). + To config indexer to use different region mapping files or if you want to host region mapping files in your own facility, please check this repo for more details: https://github.com/magda-io/magda-region-mappings From 6906875dc317af7de98cc0a5fb984cf0558f516b Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Thu, 20 Jun 2024 14:49:53 +1000 Subject: [PATCH 4/7] updated region mapping file RLs --- magda-elastic-search/package.json | 16 ++++++++-------- magda-elastic-search/regionSynonyms.txt | 16 ++++++++-------- magda-int-test-ts/indexer-setup.conf | 2 +- magda-opensearch/package.json | 16 ++++++++-------- magda-opensearch/regionSynonyms.txt | 16 ++++++++-------- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/magda-elastic-search/package.json b/magda-elastic-search/package.json index 26321b7941..1198625c76 100644 --- a/magda-elastic-search/package.json +++ b/magda-elastic-search/package.json @@ -19,49 +19,49 @@ }, "regionSources": { "COUNTRY": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/country.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson", "idField": "id", "nameField": "name", "order": 9 }, "OFFSHORE_TERRITORIES": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/off-shore-territories.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/off-shore-territories.geojson", "idField": "id", "nameField": "name", "order": 11 }, "SA4": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA4.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA4.geojson", "idField": "SA4_CODE11", "nameField": "SA4_NAME11", "order": 30 }, "SA3": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA3.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA3.geojson", "idField": "SA3_CODE11", "nameField": "SA3_NAME11", "order": 40 }, "SA2": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA2.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA2.geojson", "idField": "SA2_MAIN11", "nameField": "SA2_NAME11", "order": 50 }, "LGA": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/LGA.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/LGA.geojson", "idField": "LGA_CODE15", "nameField": "LGA_NAME15", "order": 20 }, "COM_ELB_ID_2016": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/COM_ELB_ID_2016.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/COM_ELB_ID_2016.geojson", "idField": "DIV_ID", "nameField": "SORTNAME", "order": 80 }, "STE": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/STE.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/STE.geojson", "idField": "STE_CODE11", "nameField": "STE_NAME11", "shortNameField": "STE_ABBREV", diff --git a/magda-elastic-search/regionSynonyms.txt b/magda-elastic-search/regionSynonyms.txt index 120a004b04..6870ca4904 100644 --- a/magda-elastic-search/regionSynonyms.txt +++ b/magda-elastic-search/regionSynonyms.txt @@ -1,14 +1,14 @@ # Region Type: COUNTRY -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/country.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson australia (mainland), australia => country/1 australian offshore remote territories => country/2 # Region Type: OFFSHORE_TERRITORIES -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/off-shore-territories.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/off-shore-territories.geojson norfolk island => offshore_territories/21 christmas island => offshore_territories/22 @@ -21,7 +21,7 @@ heard island and mcdonald islands => offshore_territories/28 # Region Type: SA4 -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA4.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA4.geojson capital region => sa4/101 central coast => sa4/102 @@ -132,7 +132,7 @@ special purpose codes sa4 (ot), special purpose codes sa4 => sa4/999 # Region Type: SA3 -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA3.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA3.geojson goulburn - yass => sa3/10101 queanbeyan => sa3/10102 @@ -488,7 +488,7 @@ special purpose codes sa3 (ot), special purpose codes sa3 => sa3/99999 # Region Type: SA2 -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA2.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA2.geojson goulburn => sa2/101011001 goulburn region => sa2/101011002 @@ -2707,7 +2707,7 @@ no usual address (ot), no usual address => sa2/999999499 # Region Type: LGA -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/LGA.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/LGA.geojson albury (c), albury => lga/10050 armidale dumaresq (a), armidale dumaresq => lga/10110 @@ -3293,7 +3293,7 @@ no usual address (ot), no usual address => lga/99499 # Region Type: COM_ELB_ID_2016 -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/COM_ELB_ID_2016.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/COM_ELB_ID_2016.geojson lingiari => com_elb_id_2016/306 solomon => com_elb_id_2016/307 @@ -3448,7 +3448,7 @@ tangney => com_elb_id_2016/248 # Region Type: STE -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/STE.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/STE.geojson new south wales, nsw => ste/1 victoria, vic => ste/2 diff --git a/magda-int-test-ts/indexer-setup.conf b/magda-int-test-ts/indexer-setup.conf index 685a62dc79..87ca4e041e 100644 --- a/magda-int-test-ts/indexer-setup.conf +++ b/magda-int-test-ts/indexer-setup.conf @@ -7,7 +7,7 @@ registry.registerForWebhooks = false # only enable / use one region file for speeding up the initialisation regionSources = { COUNTRY { - url = "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/country.geojson" + url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson" idField = "id" nameField = "name" order = 9 diff --git a/magda-opensearch/package.json b/magda-opensearch/package.json index dd526864b1..a1d1ee2919 100644 --- a/magda-opensearch/package.json +++ b/magda-opensearch/package.json @@ -19,49 +19,49 @@ }, "regionSources": { "COUNTRY": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/country.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson", "idField": "id", "nameField": "name", "order": 9 }, "OFFSHORE_TERRITORIES": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/off-shore-territories.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/off-shore-territories.geojson", "idField": "id", "nameField": "name", "order": 11 }, "SA4": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA4.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA4.geojson", "idField": "SA4_CODE11", "nameField": "SA4_NAME11", "order": 30 }, "SA3": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA3.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA3.geojson", "idField": "SA3_CODE11", "nameField": "SA3_NAME11", "order": 40 }, "SA2": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/SA2.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA2.geojson", "idField": "SA2_MAIN11", "nameField": "SA2_NAME11", "order": 50 }, "LGA": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/LGA.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/LGA.geojson", "idField": "LGA_CODE15", "nameField": "LGA_NAME15", "order": 20 }, "COM_ELB_ID_2016": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/COM_ELB_ID_2016.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/COM_ELB_ID_2016.geojson", "idField": "DIV_ID", "nameField": "SORTNAME", "order": 80 }, "STE": { - "url": "https://s3-ap-southeast-2.amazonaws.com/magda-files/STE.geojson", + "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/STE.geojson", "idField": "STE_CODE11", "nameField": "STE_NAME11", "shortNameField": "STE_ABBREV", diff --git a/magda-opensearch/regionSynonyms.txt b/magda-opensearch/regionSynonyms.txt index 120a004b04..6870ca4904 100644 --- a/magda-opensearch/regionSynonyms.txt +++ b/magda-opensearch/regionSynonyms.txt @@ -1,14 +1,14 @@ # Region Type: COUNTRY -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/country.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson australia (mainland), australia => country/1 australian offshore remote territories => country/2 # Region Type: OFFSHORE_TERRITORIES -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/custom/off-shore-territories.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/off-shore-territories.geojson norfolk island => offshore_territories/21 christmas island => offshore_territories/22 @@ -21,7 +21,7 @@ heard island and mcdonald islands => offshore_territories/28 # Region Type: SA4 -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA4.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA4.geojson capital region => sa4/101 central coast => sa4/102 @@ -132,7 +132,7 @@ special purpose codes sa4 (ot), special purpose codes sa4 => sa4/999 # Region Type: SA3 -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA3.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA3.geojson goulburn - yass => sa3/10101 queanbeyan => sa3/10102 @@ -488,7 +488,7 @@ special purpose codes sa3 (ot), special purpose codes sa3 => sa3/99999 # Region Type: SA2 -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/SA2.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA2.geojson goulburn => sa2/101011001 goulburn region => sa2/101011002 @@ -2707,7 +2707,7 @@ no usual address (ot), no usual address => sa2/999999499 # Region Type: LGA -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/LGA.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/LGA.geojson albury (c), albury => lga/10050 armidale dumaresq (a), armidale dumaresq => lga/10110 @@ -3293,7 +3293,7 @@ no usual address (ot), no usual address => lga/99499 # Region Type: COM_ELB_ID_2016 -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/COM_ELB_ID_2016.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/COM_ELB_ID_2016.geojson lingiari => com_elb_id_2016/306 solomon => com_elb_id_2016/307 @@ -3448,7 +3448,7 @@ tangney => com_elb_id_2016/248 # Region Type: STE -# From: https://s3-ap-southeast-2.amazonaws.com/magda-files/STE.geojson +# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/STE.geojson new south wales, nsw => ste/1 victoria, vic => ste/2 From 3ca5d209c80d69b84f7e6a341c7c5b35e2e97eb2 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Thu, 20 Jun 2024 17:56:51 +1000 Subject: [PATCH 5/7] Make regionLoader follows HTTP redirections --- .../data61/magda/spatial/RegionLoader.scala | 60 +++++++++++++++---- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionLoader.scala b/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionLoader.scala index 9ea4112eab..09e7c26cd0 100644 --- a/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionLoader.scala +++ b/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionLoader.scala @@ -4,14 +4,16 @@ import java.io.File import akka.actor.ActorSystem import akka.http.scaladsl.Http import akka.http.scaladsl.client.RequestBuilding +import akka.http.scaladsl.model._ import akka.stream.Materializer import akka.stream.scaladsl.FileIO import akka.stream.scaladsl.JsonFraming -import akka.stream.scaladsl.Source +import akka.stream.scaladsl.{Sink, Source} import spray.json.JsObject import spray.json._ import scala.concurrent.Future import com.typesafe.config.Config +import scala.util.{Failure, Success} trait RegionLoader { def setupRegions(): Source[(RegionSource, JsObject), _] @@ -33,7 +35,48 @@ class FileRegionLoader(regionSources: List[RegionSource])( implicit val ec = system.dispatcher val pool = Http().superPool[Int]() + def sendRequestWithRedirects( + uri: Uri, + maxRedirects: Int = 5 + ): Future[HttpResponse] = { + def requestLoop( + currentUri: Uri, + remainingRedirects: Int + ): Future[HttpResponse] = { + val request = RequestBuilding.Get(currentUri.toString) + Source + .single((request, 0)) + .via(pool) + .runWith(Sink.head) + .flatMap { + case (Success(response), _) => + response.status match { + case StatusCodes.MovedPermanently | StatusCodes.Found | + StatusCodes.SeeOther | StatusCodes.TemporaryRedirect | + StatusCodes.PermanentRedirect if remainingRedirects > 0 => + response.header[headers.Location] match { + case Some(locationHeader) => + val newUri = locationHeader.uri + response.discardEntityBytes() // Clean up response entity + requestLoop(newUri, remainingRedirects - 1) + case None => + Future.failed( + new RuntimeException( + "Redirect response missing Location header" + ) + ) + } + case _ => Future.successful(response) + } + case (Failure(exception), _) => + Future.failed(exception) + } + } + requestLoop(uri, maxRedirects) + } + def loadABSRegions(regionSource: RegionSource): Future[File] = { + val file = new File( s"${config.getString("regionLoading.cachePath")}/${regionSource.name}.json" ) @@ -56,18 +99,13 @@ class FileRegionLoader(regionSources: List[RegionSource])( file.getParentFile.mkdirs() file.createNewFile() - val request = RequestBuilding.Get(regionSource.url.toString) - system.log.info("Indexing regions from {}", regionSource.url) - Source - .single((request, 0)) - .via(pool) - .flatMapConcat { - case (response, _) => - response.get.entity.withoutSizeLimit().dataBytes + sendRequestWithRedirects(regionSource.url.toString) + .flatMap { response => + val fileSink = FileIO.toPath(file.toPath()) + response.entity.withoutSizeLimit().dataBytes.runWith(fileSink) } - .runWith(FileIO.toPath(file.toPath())) .recover { case e: Throwable => system.log.info( @@ -85,7 +123,7 @@ class FileRegionLoader(regionSources: List[RegionSource])( override def setupRegions(): Source[(RegionSource, JsObject), _] = { Source(regionSources) - .mapAsync(4)( + .mapAsync(3)( regionSource => loadABSRegions(regionSource).map((_, regionSource)) ) .flatMapConcat { From 3dbd8ed71b67c26b19c544f819d24c63721ed936 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Thu, 20 Jun 2024 22:25:45 +1000 Subject: [PATCH 6/7] update magda-regions repo url --- CHANGES.md | 2 +- deploy/helm/internal-charts/indexer/README.md | 4 ++-- .../internal-charts/indexer/README.md.gotmpl | 4 ++-- docs/docs/index.md | 2 +- ...egion-mapping-files.md => region-files.md} | 4 ++-- magda-elastic-search/package.json | 16 ++++++------- magda-elastic-search/regionSynonyms.txt | 16 ++++++------- magda-int-test-ts/indexer-setup.conf | 2 +- magda-opensearch/package.json | 16 ++++++------- magda-opensearch/regionSynonyms.txt | 16 ++++++------- .../src/main/resources/common.conf | 23 +++++++++---------- .../au/csiro/data61/magda/api/SearchApi.scala | 2 +- 12 files changed, 53 insertions(+), 54 deletions(-) rename docs/docs/{region-mapping-files.md => region-files.md} (93%) diff --git a/CHANGES.md b/CHANGES.md index 25e49e9991..7b8d5072cd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,7 +3,7 @@ ## v4.0.1 - Add helm chart configuration option allows users to disable gateway auto-gzip response feature -- Update default region mapping files url to [Github repo release](https://github.com/magda-io/magda-region-mappings) +- Update default region files url to [Github repo release](https://github.com/magda-io/magda-regions) ## v4.0.0 diff --git a/deploy/helm/internal-charts/indexer/README.md b/deploy/helm/internal-charts/indexer/README.md index 4bbdb3caaf..3c3c404165 100644 --- a/deploy/helm/internal-charts/indexer/README.md +++ b/deploy/helm/internal-charts/indexer/README.md @@ -4,11 +4,11 @@ A Helm chart for Magda's Indexer service. -The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-region-mappings/releases). +The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-regions/releases). For production deployment, you might want to host those region mapping files yourself in a more reliable way (e.g. put into a storage bucket). -To config region mapping files, please refer to the this repo: https://github.com/magda-io/magda-region-mappings +To config region mapping files, please refer to the this repo: https://github.com/magda-io/magda-regions ## Requirements diff --git a/deploy/helm/internal-charts/indexer/README.md.gotmpl b/deploy/helm/internal-charts/indexer/README.md.gotmpl index 9233a961ab..ec648dc671 100644 --- a/deploy/helm/internal-charts/indexer/README.md.gotmpl +++ b/deploy/helm/internal-charts/indexer/README.md.gotmpl @@ -5,11 +5,11 @@ {{ template "chart.description" . }} -The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-region-mappings/releases). +The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-regions/releases). For production deployment, you might want to host those region mapping files yourself in a more reliable way (e.g. put into a storage bucket). -To config region mapping files, please refer to the this repo: https://github.com/magda-io/magda-region-mappings +To config region mapping files, please refer to the this repo: https://github.com/magda-io/magda-regions {{ template "chart.homepageLine" . }} diff --git a/docs/docs/index.md b/docs/docs/index.md index 80b802e729..a61dfe5350 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -22,6 +22,6 @@ - [How to recover with continuous archive backup](./how-to-recover-with-continuous-archive-backup.md) - [Ports used when running locally](./local-ports.md) - [Windows Setup Instructions](./windows-instructions.md) -- [Region Mapping Files Config](./region-mapping-files.md) +- [Regions Config](./region-files.md) More documentation, please check [here](https://github.com/magda-io/magda/tree/master/docs/docs) diff --git a/docs/docs/region-mapping-files.md b/docs/docs/region-files.md similarity index 93% rename from docs/docs/region-mapping-files.md rename to docs/docs/region-files.md index e41bfd8317..d26b32a081 100644 --- a/docs/docs/region-mapping-files.md +++ b/docs/docs/region-files.md @@ -2,8 +2,8 @@ Magda allow users to supply a list of region mapping files that contains a list of regions available as region search filter options. The region mapping files will be fetched via HTTP protocol by Magda's [indexer](https://github.com/magda-io/magda/tree/main/deploy/helm/internal-charts/indexer) module when it's required to create region index in search engine (e.g. for the first deployment). -The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-region-mappings/releases). +The default config will pull region mapping files from [the `magda-region-mappings` repo release download area](https://github.com/magda-io/magda-regions/releases). For production deployment, you might want to host those region mapping files yourself in a more reliable way (e.g. put into a storage bucket). -To config indexer to use different region mapping files or if you want to host region mapping files in your own facility, please check this repo for more details: https://github.com/magda-io/magda-region-mappings +To config indexer to use different region mapping files or if you want to host region mapping files in your own facility, please check this repo for more details: https://github.com/magda-io/magda-regions diff --git a/magda-elastic-search/package.json b/magda-elastic-search/package.json index 1198625c76..a6eabdb6b7 100644 --- a/magda-elastic-search/package.json +++ b/magda-elastic-search/package.json @@ -19,49 +19,49 @@ }, "regionSources": { "COUNTRY": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson", "idField": "id", "nameField": "name", "order": 9 }, "OFFSHORE_TERRITORIES": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/off-shore-territories.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/off-shore-territories.geojson", "idField": "id", "nameField": "name", "order": 11 }, "SA4": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA4.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA4.geojson", "idField": "SA4_CODE11", "nameField": "SA4_NAME11", "order": 30 }, "SA3": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA3.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA3.geojson", "idField": "SA3_CODE11", "nameField": "SA3_NAME11", "order": 40 }, "SA2": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA2.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA2.geojson", "idField": "SA2_MAIN11", "nameField": "SA2_NAME11", "order": 50 }, "LGA": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/LGA.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/LGA.geojson", "idField": "LGA_CODE15", "nameField": "LGA_NAME15", "order": 20 }, "COM_ELB_ID_2016": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/COM_ELB_ID_2016.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/COM_ELB_ID_2016.geojson", "idField": "DIV_ID", "nameField": "SORTNAME", "order": 80 }, "STE": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/STE.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/STE.geojson", "idField": "STE_CODE11", "nameField": "STE_NAME11", "shortNameField": "STE_ABBREV", diff --git a/magda-elastic-search/regionSynonyms.txt b/magda-elastic-search/regionSynonyms.txt index 6870ca4904..0161e3de94 100644 --- a/magda-elastic-search/regionSynonyms.txt +++ b/magda-elastic-search/regionSynonyms.txt @@ -1,14 +1,14 @@ # Region Type: COUNTRY -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson australia (mainland), australia => country/1 australian offshore remote territories => country/2 # Region Type: OFFSHORE_TERRITORIES -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/off-shore-territories.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/off-shore-territories.geojson norfolk island => offshore_territories/21 christmas island => offshore_territories/22 @@ -21,7 +21,7 @@ heard island and mcdonald islands => offshore_territories/28 # Region Type: SA4 -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA4.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA4.geojson capital region => sa4/101 central coast => sa4/102 @@ -132,7 +132,7 @@ special purpose codes sa4 (ot), special purpose codes sa4 => sa4/999 # Region Type: SA3 -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA3.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA3.geojson goulburn - yass => sa3/10101 queanbeyan => sa3/10102 @@ -488,7 +488,7 @@ special purpose codes sa3 (ot), special purpose codes sa3 => sa3/99999 # Region Type: SA2 -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA2.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA2.geojson goulburn => sa2/101011001 goulburn region => sa2/101011002 @@ -2707,7 +2707,7 @@ no usual address (ot), no usual address => sa2/999999499 # Region Type: LGA -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/LGA.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/LGA.geojson albury (c), albury => lga/10050 armidale dumaresq (a), armidale dumaresq => lga/10110 @@ -3293,7 +3293,7 @@ no usual address (ot), no usual address => lga/99499 # Region Type: COM_ELB_ID_2016 -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/COM_ELB_ID_2016.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/COM_ELB_ID_2016.geojson lingiari => com_elb_id_2016/306 solomon => com_elb_id_2016/307 @@ -3448,7 +3448,7 @@ tangney => com_elb_id_2016/248 # Region Type: STE -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/STE.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/STE.geojson new south wales, nsw => ste/1 victoria, vic => ste/2 diff --git a/magda-int-test-ts/indexer-setup.conf b/magda-int-test-ts/indexer-setup.conf index 87ca4e041e..2fefedeec4 100644 --- a/magda-int-test-ts/indexer-setup.conf +++ b/magda-int-test-ts/indexer-setup.conf @@ -7,7 +7,7 @@ registry.registerForWebhooks = false # only enable / use one region file for speeding up the initialisation regionSources = { COUNTRY { - url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson" idField = "id" nameField = "name" order = 9 diff --git a/magda-opensearch/package.json b/magda-opensearch/package.json index a1d1ee2919..49ae837e55 100644 --- a/magda-opensearch/package.json +++ b/magda-opensearch/package.json @@ -19,49 +19,49 @@ }, "regionSources": { "COUNTRY": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson", "idField": "id", "nameField": "name", "order": 9 }, "OFFSHORE_TERRITORIES": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/off-shore-territories.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/off-shore-territories.geojson", "idField": "id", "nameField": "name", "order": 11 }, "SA4": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA4.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA4.geojson", "idField": "SA4_CODE11", "nameField": "SA4_NAME11", "order": 30 }, "SA3": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA3.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA3.geojson", "idField": "SA3_CODE11", "nameField": "SA3_NAME11", "order": 40 }, "SA2": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA2.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA2.geojson", "idField": "SA2_MAIN11", "nameField": "SA2_NAME11", "order": 50 }, "LGA": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/LGA.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/LGA.geojson", "idField": "LGA_CODE15", "nameField": "LGA_NAME15", "order": 20 }, "COM_ELB_ID_2016": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/COM_ELB_ID_2016.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/COM_ELB_ID_2016.geojson", "idField": "DIV_ID", "nameField": "SORTNAME", "order": 80 }, "STE": { - "url": "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/STE.geojson", + "url": "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/STE.geojson", "idField": "STE_CODE11", "nameField": "STE_NAME11", "shortNameField": "STE_ABBREV", diff --git a/magda-opensearch/regionSynonyms.txt b/magda-opensearch/regionSynonyms.txt index 6870ca4904..0161e3de94 100644 --- a/magda-opensearch/regionSynonyms.txt +++ b/magda-opensearch/regionSynonyms.txt @@ -1,14 +1,14 @@ # Region Type: COUNTRY -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson australia (mainland), australia => country/1 australian offshore remote territories => country/2 # Region Type: OFFSHORE_TERRITORIES -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/off-shore-territories.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/off-shore-territories.geojson norfolk island => offshore_territories/21 christmas island => offshore_territories/22 @@ -21,7 +21,7 @@ heard island and mcdonald islands => offshore_territories/28 # Region Type: SA4 -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA4.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA4.geojson capital region => sa4/101 central coast => sa4/102 @@ -132,7 +132,7 @@ special purpose codes sa4 (ot), special purpose codes sa4 => sa4/999 # Region Type: SA3 -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA3.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA3.geojson goulburn - yass => sa3/10101 queanbeyan => sa3/10102 @@ -488,7 +488,7 @@ special purpose codes sa3 (ot), special purpose codes sa3 => sa3/99999 # Region Type: SA2 -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA2.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA2.geojson goulburn => sa2/101011001 goulburn region => sa2/101011002 @@ -2707,7 +2707,7 @@ no usual address (ot), no usual address => sa2/999999499 # Region Type: LGA -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/LGA.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/LGA.geojson albury (c), albury => lga/10050 armidale dumaresq (a), armidale dumaresq => lga/10110 @@ -3293,7 +3293,7 @@ no usual address (ot), no usual address => lga/99499 # Region Type: COM_ELB_ID_2016 -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/COM_ELB_ID_2016.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/COM_ELB_ID_2016.geojson lingiari => com_elb_id_2016/306 solomon => com_elb_id_2016/307 @@ -3448,7 +3448,7 @@ tangney => com_elb_id_2016/248 # Region Type: STE -# From: https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/STE.geojson +# From: https://github.com/magda-io/magda-regions/releases/download/v1.0.0/STE.geojson new south wales, nsw => ste/1 victoria, vic => ste/2 diff --git a/magda-scala-common/src/main/resources/common.conf b/magda-scala-common/src/main/resources/common.conf index e29339cabc..c67e1aeb0f 100755 --- a/magda-scala-common/src/main/resources/common.conf +++ b/magda-scala-common/src/main/resources/common.conf @@ -89,18 +89,18 @@ logging { } # default region mapping files. -# See repo here: https://github.com/magda-io/magda-region-mappings +# See repo here: https://github.com/magda-io/magda-regions regionSources = { # Australia (Mainland) and all offshore territories as a whole COUNTRY { - url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/country.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson" idField = "id" nameField = "name" order = 9 } # Regions for each of Australia offshore territories OFFSHORE_TERRITORIES { - url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/off-shore-territories.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/off-shore-territories.geojson" idField = "id" nameField = "name" lv1Id = "2" @@ -108,7 +108,7 @@ regionSources = { } # ABS Statistical Area Level 4 SA4 { - url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA4.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA4.geojson" idField = "SA4_CODE11" nameField = "SA4_NAME11" lv1Id = "1" @@ -117,7 +117,7 @@ regionSources = { } # ABS Statistical Area Level 3 SA3 { - url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA3.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA3.geojson" idField = "SA3_CODE11" nameField = "SA3_NAME11" lv1Id = "1" @@ -127,7 +127,7 @@ regionSources = { } # ABS Statistical Area Level 2 SA2 { - url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA2.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA2.geojson" idField = "SA2_MAIN11" nameField = "SA2_NAME11" lv1Id = "1" @@ -138,7 +138,7 @@ regionSources = { } # ABS Statistical Area Level 1 SA1 { - url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/SA1.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA1.geojson" idField = "SA1_MAIN11" nameField = "SA1_MAIN11", lv1Id = "1" @@ -146,12 +146,11 @@ regionSources = { lv3IdField = "SA4_CODE11" lv4IdField = "SA3_CODE11" lv5IdField = "SA2_MAIN11" - includeIdInName = false, order = 60 } # Australia Local Government Areas LGA { - url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/LGA.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/LGA.geojson" idField = "LGA_CODE15" nameField = "LGA_NAME15" lv1Id = "1" @@ -160,7 +159,7 @@ regionSources = { } # Australia Postal Areas POA { - url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/POA.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/POA.geojson" idField = "POA_CODE" nameField = "POA_NAME" lv1Id = "1" @@ -168,7 +167,7 @@ regionSources = { } # Australia Commonwealth electoral boundaries COM_ELB_ID_2016 { - url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/COM_ELB_ID_2016.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/COM_ELB_ID_2016.geojson" idField = "DIV_ID" nameField = "SORTNAME" lv1Id = "1" @@ -176,7 +175,7 @@ regionSources = { } # Australia State and Territory STE { - url = "https://github.com/magda-io/magda-region-mappings/releases/download/v1.0.0/STE.simplified.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/STE.simplified.geojson" idField = "STE_CODE11" nameField = "STE_NAME11" shortNameField = "STE_ABBREV" diff --git a/magda-search-api/src/main/scala/au/csiro/data61/magda/api/SearchApi.scala b/magda-search-api/src/main/scala/au/csiro/data61/magda/api/SearchApi.scala index f917ba0f25..afd01cbec5 100755 --- a/magda-search-api/src/main/scala/au/csiro/data61/magda/api/SearchApi.scala +++ b/magda-search-api/src/main/scala/au/csiro/data61/magda/api/SearchApi.scala @@ -407,7 +407,7 @@ class SearchApi( * @apiDescription Returns a list of region types * * @apiSuccess {string} comments Notes for programmers. - * @apiSuccess {RegionWMSMap[]} regionWmsMap A mepping of string to WMS layer metadata. + * @apiSuccess {RegionWMSMap[]} regionWmsMap A mapping of string to WMS layer metadata. * * @apiSuccessExample {any} 200 * { From 4007a628000bc80e0d7de58839fbc47328cde2e2 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Thu, 20 Jun 2024 23:31:45 +1000 Subject: [PATCH 7/7] #3543 Make the region mapping served by search api configurable --- CHANGES.md | 3 +- .../src/main/resources/common.conf | 205 ++++++++++++++++++ .../au/csiro/data61/magda/api/SearchApi.scala | 29 ++- 3 files changed, 234 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7b8d5072cd..bdc3574a97 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,7 +3,8 @@ ## v4.0.1 - Add helm chart configuration option allows users to disable gateway auto-gzip response feature -- Update default region files url to [Github repo release](https://github.com/magda-io/magda-regions) +- #3541 Update default region files url to [Github repo release](https://github.com/magda-io/magda-regions) +- #3543 Make the region mapping served by search api configurable ## v4.0.0 diff --git a/magda-scala-common/src/main/resources/common.conf b/magda-scala-common/src/main/resources/common.conf index c67e1aeb0f..01efed7e36 100755 --- a/magda-scala-common/src/main/resources/common.conf +++ b/magda-scala-common/src/main/resources/common.conf @@ -183,3 +183,208 @@ regionSources = { order = 10 } } + +# default region mapping setting +# Only by frontend region filter map view. +# Served via search API [region-types](https://dev.magda.io/api/v0/apidocs/index.html#api-Search-GetV0SearchRegionTypes) endpoint +# See repo here: https://github.com/magda-io/magda-regions +# If not provided, the [default region mapping file](https://github.com/magda-io/magda/blob/main/magda-search-api/src/main/resources/regionMapping.json) will be used. +# regionMapping = { +# "comments": "Matching takes place in the order defined in this file. Place code matches before name matches, and smaller regions before larger ones.", +# "regionWmsMap": { +# "STE": { +# "layerName": "FID_STE_2011_AUST", +# "server": "https://vector-tiles.terria.io/FID_STE_2011_AUST/{z}/{x}/{y}.pbf", +# "regionProp": "STE_CODE11", +# "aliases": ["ste_code", "ste_code_2011", "ste"], +# "digits": 1, +# "description": "States and Territories (STE)", +# "regionIdsFile": "data/regionids/region_map-FID_STE_2011_AUST_STE_CODE11.json", +# "serverType": "MVT", +# "serverSubdomains": [], +# "serverMinZoom": 0, +# "serverMaxNativeZoom": 12, +# "serverMaxZoom": 28, +# "bbox": [ +# 96.81694140799998, +# -43.74050960300003, +# 159.10921900799997, +# -9.142175976999999 +# ], +# "nameProp": "STE_NAME11" +# }, +# "SA4": { +# "layerName": "FID_SA4_2011_AUST", +# "server": "https://vector-tiles.terria.io/FID_SA4_2011_AUST/{z}/{x}/{y}.pbf", +# "regionProp": "SA4_CODE11", +# "aliases": ["sa4_code_2011", "sa4_code", "sa4"], +# "digits": 3, +# "description": "Statistical Area Level 4 (SA4)", +# "regionIdsFile": "data/regionids/region_map-FID_SA4_2011_AUST_SA4_CODE11.json", +# "serverType": "MVT", +# "serverSubdomains": [], +# "serverMinZoom": 0, +# "serverMaxNativeZoom": 12, +# "serverMaxZoom": 28, +# "bbox": [ +# 96.81694140799998, +# -43.74050960300003, +# 159.10921900799997, +# -9.142175976999999 +# ], +# "nameProp": "SA4_NAME11" +# }, +# "SA3": { +# "layerName": "FID_SA3_2011_AUST", +# "server": "https://vector-tiles.terria.io/FID_SA3_2011_AUST/{z}/{x}/{y}.pbf", +# "regionProp": "SA3_CODE11", +# "aliases": ["sa3_code_2011", "sa3_code", "sa3"], +# "digits": 5, +# "description": "Statistical Area Level 3 (SA3)", +# "regionIdsFile": "data/regionids/region_map-FID_SA3_2011_AUST_SA3_CODE11.json", +# "serverType": "MVT", +# "serverSubdomains": [], +# "serverMinZoom": 0, +# "serverMaxNativeZoom": 12, +# "serverMaxZoom": 28, +# "bbox": [ +# 96.81694140799998, +# -43.74050960300003, +# 159.10921900799997, +# -9.142175976999999 +# ], +# "nameProp": "SA3_NAME11" +# }, +# "SA2": { +# "layerName": "FID_SA2_2011_AUST", +# "server": "https://vector-tiles.terria.io/FID_SA2_2011_AUST/{z}/{x}/{y}.pbf", +# "regionProp": "SA2_MAIN11", +# "aliases": ["sa2_code_2011", "sa2_code", "sa2"], +# "digits": 9, +# "description": "Statistical Area Level 2 (SA2)", +# "regionIdsFile": "data/regionids/region_map-FID_SA2_2011_AUST_SA2_MAIN11.json", +# "serverType": "MVT", +# "serverSubdomains": [], +# "serverMinZoom": 0, +# "serverMaxNativeZoom": 12, +# "serverMaxZoom": 28, +# "bbox": [ +# 96.81694140799998, +# -43.74050960300003, +# 159.10921900799997, +# -9.142175976999999 +# ], +# "nameProp": "SA2_NAME11" +# }, +# "SA1": { +# "layerName": "FID_SA1_2011_AUST", +# "server": "https://vector-tiles.terria.io/FID_SA1_2011_AUST/{z}/{x}/{y}.pbf", +# "regionProp": "SA1_MAIN11", +# "aliases": [ +# "sa1_code_2011", +# "sa1_maincode_2011", +# "sa1_code", +# "sa1" +# ], +# "digits": 11, +# "description": "Statistical Area Level 1 (SA1)", +# "regionIdsFile": "data/regionids/region_map-FID_SA1_2011_AUST_SA1_MAIN11.json", +# "serverType": "MVT", +# "serverSubdomains": [], +# "serverMinZoom": 0, +# "serverMaxNativeZoom": 12, +# "serverMaxZoom": 28, +# "bbox": [ +# 96.81694140799998, +# -43.74050960300003, +# 159.10921900799997, +# -9.142175976999999 +# ], +# "nameProp": "SA2_NAME11" +# }, +# "LGA": { +# "layerName": "FID_LGA_2015_AUST", +# "server": "https://vector-tiles.terria.io/FID_LGA_2015_AUST/{z}/{x}/{y}.pbf", +# "regionProp": "LGA_CODE15", +# "aliases": [ +# "lga_code_2015", +# "lga_code", +# "lga", +# "lga_code_2014", +# "lga_code_2012", +# "lga_code_2010" +# ], +# "digits": 5, +# "description": "Local Government Area (LGA)", +# "regionIdsFile": "data/regionids/region_map-FID_LGA_2015_AUST_LGA_CODE15.json", +# "serverType": "MVT", +# "serverSubdomains": [], +# "serverMinZoom": 0, +# "serverMaxNativeZoom": 12, +# "serverMaxZoom": 28, +# "bbox": [ +# 96.81694140799998, +# -43.740509602999985, +# 159.10921900799997, +# -9.142175976999999 +# ], +# "nameProp": "LGA_NAME15" +# }, +# "POA": { +# "layerName": "FID_POA_2011_AUST", +# "server": "https://vector-tiles.terria.io/FID_POA_2011_AUST/{z}/{x}/{y}.pbf", +# "regionProp": "POA_CODE", +# "aliases": [ +# "poa_2011", +# "postcode_2011", +# "poa", +# "poa_code", +# "poa_code_2011", +# "postcode", +# "postcode_2015" +# ], +# "digits": 4, +# "dataReplacements": [["^(?=\\d\\d\\d$)", "0"]], +# "description": "Postal Area (POA)", +# "regionIdsFile": "data/regionids/region_map-FID_POA_2011_AUST_POA_CODE.json", +# "serverType": "MVT", +# "serverSubdomains": [], +# "serverMinZoom": 0, +# "serverMaxNativeZoom": 12, +# "serverMaxZoom": 28, +# "bbox": [ +# 96.81694140799998, +# -43.59821500299999, +# 159.10921900799997, +# -9.142175976999999 +# ], +# "nameProp": "POA_NAME" +# }, +# "COM_ELB_ID_2016": { +# "layerName": "FID_COM20160509_ELB", +# "server": "https://vector-tiles.terria.io/FID_COM20160509_ELB/{z}/{x}/{y}.pbf", +# "regionProp": "DIV_ID", +# "aliases": [ +# "divisionid", +# "com_elb_id_2016", +# "com_elb_id", +# "com_elb" +# ], +# "digits": 3, +# "description": "Commonwealth Electoral District", +# "regionIdsFile": "data/regionids/region_map-FID_COM20160509_ELB_DIV_ID.json", +# "serverType": "MVT", +# "serverSubdomains": [], +# "serverMinZoom": 0, +# "serverMaxNativeZoom": 12, +# "serverMaxZoom": 28, +# "bbox": [ +# 96.81676599999997, +# -43.740509999999986, +# 159.1092189999999, +# -9.142175999999996 +# ], +# "nameProp": "SORTNAME" +# } +# } +# } diff --git a/magda-search-api/src/main/scala/au/csiro/data61/magda/api/SearchApi.scala b/magda-search-api/src/main/scala/au/csiro/data61/magda/api/SearchApi.scala index afd01cbec5..9b99e9a293 100755 --- a/magda-search-api/src/main/scala/au/csiro/data61/magda/api/SearchApi.scala +++ b/magda-search-api/src/main/scala/au/csiro/data61/magda/api/SearchApi.scala @@ -5,13 +5,14 @@ import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.StatusCodes._ import akka.http.scaladsl.server.Directives._ +import akka.http.scaladsl.model._ import au.csiro.data61.magda.model.misc import au.csiro.data61.magda.model.misc._ import au.csiro.data61.magda.api.{model => apimodel} import au.csiro.data61.magda.client.AuthApiClient import au.csiro.data61.magda.directives.TenantDirectives.requiresTenantId import au.csiro.data61.magda.search.SearchQueryer -import com.typesafe.config.Config +import com.typesafe.config.{Config, ConfigRenderOptions} import au.csiro.data61.magda.search.Directives.withDatasetReadAuthDecision /** @@ -32,6 +33,14 @@ class SearchApi( with apimodel.Protocols { override def getLogger = logger + val regionMappingConfig = if (config.hasPath("regionMapping")) { + Some( + config.getObject("regionMapping").render(ConfigRenderOptions.concise()) + ) + } else { + None + } + val routes = magdaRoute { pathPrefix("v0") { @@ -423,7 +432,23 @@ class SearchApi( * } * } */ - path("region-types") { get { getFromResource("regionMapping.json") } } ~ + path("region-types") { + get { + + if (regionMappingConfig.isEmpty) { + // Serve the default region mapping + getFromResource("regionMapping.json") + } else { + // Serve the region mapping from config + val entity = HttpEntity( + ContentTypes.`application/json`, + regionMappingConfig.get + ) + complete(HttpResponse(StatusCodes.OK, entity = entity)) + } + + } + } ~ /** * @apiGroup Search * @api {get} /v0/search/regions Get Regions