diff --git a/CHANGES.md b/CHANGES.md index bdc3574a97..a6c84ff93c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,14 @@ # CHANGELOG +## v4.1.0 + +- #3487: Update to use ASGS 2021 and latest LGAs (2023) boundaries +- Added `indexed` field to region index & Bump region index version to `27` + - Migration instruction can be found from [the migration doc](./docs/docs/migration/4.1.0.md) +- Adjusted & improved region config entry skip logic +- Improved the region loader performance +- Fixed: region loader failed to convert number type id property into a string + ## v4.0.1 - Add helm chart configuration option allows users to disable gateway auto-gzip response feature diff --git a/docs/docs/migration/4.1.0.md b/docs/docs/migration/4.1.0.md new file mode 100644 index 0000000000..efc499c7b3 --- /dev/null +++ b/docs/docs/migration/4.1.0.md @@ -0,0 +1,217 @@ +## v4.1.0 + +## Search Engine Region Index Upgrade + +We upgrade region index mapping version to `27` and upgrade region data to use ASGS 2021 and latest LGAs (2023) boundaries. During the upgrade, indexer will auto-recreate the region index. Depends on your database size, this might take some time (5-30 mins depends on resources allocated). + +> To build your own region files & MVT tile service, please refer to this repo: https://github.com/magda-io/magda-regions + +To avoid serving incomplete region data during the re-indexing process, you can manually set the region index version that search API uses to the existing versions via [helm chart options](https://github.com/magda-io/magda/blob/944ae887842b98c51698d567435003be2e9dbefd/deploy/helm/internal-charts/search-api/values.yaml#L29). + +Moreover, we also need to set the region mapping served from search API [Get Region Types +](https://dev.magda.io/api/v0/apidocs/index.html#api-Search-GetV0SearchRegionTypes) endpoint to ensure it matches the old region index version we set. + +e.g. you can set the following in your helm deployment values file / config to make search api still queries the previous region index version (`26`) while indexer's creating the new indexes: + +``` +search-api: + # set region index version used to previous version: 26 + regionsIndexVersion: 26 + appConfig: + # Set to previous region mapping for regions in index version 26 or older + # You can find region mapping from the repo here: https://github.com/magda-io/magda/blob/main/magda-search-api/src/main/resources/regionMapping.json + regionMapping: + regionWmsMap: + STE: + layerName: FID_STE_2011_AUST + server: https://tiles.magda.io/FID_STE_2011_AUST/{z}/{x}/{y}.pbf + regionProp: STE_CODE11 + aliases: + - ste_code + - ste_code_2011 + - ste + digits: 1 + description: States and Territories (STE) + regionIdsFile: data/regionids/region_map-FID_STE_2011_AUST_STE_CODE11.json + serverType: MVT + serverSubdomains: [] + serverMinZoom: 0 + serverMaxNativeZoom: 12 + serverMaxZoom: 28 + bbox: + - 96.81694140799998 + - -43.74050960300003 + - 159.10921900799997 + - -9.142175976999999 + nameProp: STE_NAME11 + SA4: + layerName: FID_SA4_2011_AUST + server: https://tiles.magda.io/FID_SA4_2011_AUST/{z}/{x}/{y}.pbf + regionProp: SA4_CODE11 + aliases: + - sa4_code_2011 + - sa4_code + - sa4 + digits: 3 + description: Statistical Area Level 4 (SA4) + regionIdsFile: data/regionids/region_map-FID_SA4_2011_AUST_SA4_CODE11.json + serverType: MVT + serverSubdomains: [] + serverMinZoom: 0 + serverMaxNativeZoom: 12 + serverMaxZoom: 28 + bbox: + - 96.81694140799998 + - -43.74050960300003 + - 159.10921900799997 + - -9.142175976999999 + nameProp: SA4_NAME11 + SA3: + layerName: FID_SA3_2011_AUST + server: https://tiles.magda.io/FID_SA3_2011_AUST/{z}/{x}/{y}.pbf + regionProp: SA3_CODE11 + aliases: + - sa3_code_2011 + - sa3_code + - sa3 + digits: 5 + description: Statistical Area Level 3 (SA3) + regionIdsFile: data/regionids/region_map-FID_SA3_2011_AUST_SA3_CODE11.json + serverType: MVT + serverSubdomains: [] + serverMinZoom: 0 + serverMaxNativeZoom: 12 + serverMaxZoom: 28 + bbox: + - 96.81694140799998 + - -43.74050960300003 + - 159.10921900799997 + - -9.142175976999999 + nameProp: SA3_NAME11 + SA2: + layerName: FID_SA2_2011_AUST + server: https://tiles.magda.io/FID_SA2_2011_AUST/{z}/{x}/{y}.pbf + regionProp: SA2_MAIN11 + aliases: + - sa2_code_2011 + - sa2_code + - sa2 + digits: 9 + description: Statistical Area Level 2 (SA2) + regionIdsFile: data/regionids/region_map-FID_SA2_2011_AUST_SA2_MAIN11.json + serverType: MVT + serverSubdomains: [] + serverMinZoom: 0 + serverMaxNativeZoom: 12 + serverMaxZoom: 28 + bbox: + - 96.81694140799998 + - -43.74050960300003 + - 159.10921900799997 + - -9.142175976999999 + nameProp: SA2_NAME11 + SA1: + layerName: FID_SA1_2011_AUST + server: https://tiles.magda.io/FID_SA1_2011_AUST/{z}/{x}/{y}.pbf + regionProp: SA1_MAIN11 + aliases: + - sa1_code_2011 + - sa1_maincode_2011 + - sa1_code + - sa1 + digits: 11 + description: Statistical Area Level 1 (SA1) + regionIdsFile: data/regionids/region_map-FID_SA1_2011_AUST_SA1_MAIN11.json + serverType: MVT + serverSubdomains: [] + serverMinZoom: 0 + serverMaxNativeZoom: 12 + serverMaxZoom: 28 + bbox: + - 96.81694140799998 + - -43.74050960300003 + - 159.10921900799997 + - -9.142175976999999 + nameProp: SA2_NAME11 + LGA: + layerName: FID_LGA_2015_AUST + server: https://tiles.magda.io/FID_LGA_2015_AUST/{z}/{x}/{y}.pbf + regionProp: LGA_CODE15 + aliases: + - lga_code_2015 + - lga_code + - lga + - lga_code_2014 + - lga_code_2012 + - lga_code_2010 + digits: 5 + description: Local Government Area (LGA) + regionIdsFile: data/regionids/region_map-FID_LGA_2015_AUST_LGA_CODE15.json + serverType: MVT + serverSubdomains: [] + serverMinZoom: 0 + serverMaxNativeZoom: 12 + serverMaxZoom: 28 + bbox: + - 96.81694140799998 + - -43.740509602999985 + - 159.10921900799997 + - -9.142175976999999 + nameProp: LGA_NAME15 + POA: + layerName: FID_POA_2011_AUST + server: https://tiles.magda.io/FID_POA_2011_AUST/{z}/{x}/{y}.pbf + regionProp: POA_CODE + aliases: + - poa_2011 + - postcode_2011 + - poa + - poa_code + - poa_code_2011 + - postcode + - postcode_2015 + digits: 4 + dataReplacements: + - - "^(?=\\d\\d\\d$)" + - '0' + description: Postal Area (POA) + regionIdsFile: data/regionids/region_map-FID_POA_2011_AUST_POA_CODE.json + serverType: MVT + serverSubdomains: [] + serverMinZoom: 0 + serverMaxNativeZoom: 12 + serverMaxZoom: 28 + bbox: + - 96.81694140799998 + - -43.59821500299999 + - 159.10921900799997 + - -9.142175976999999 + nameProp: POA_NAME + COM_ELB_ID_2016: + layerName: FID_COM20160509_ELB + server: https://tiles.magda.io/FID_COM20160509_ELB/{z}/{x}/{y}.pbf + regionProp: DIV_ID + aliases: + - divisionid + - com_elb_id_2016 + - com_elb_id + - com_elb + digits: 3 + description: Commonwealth Electoral District + regionIdsFile: data/regionids/region_map-FID_COM20160509_ELB_DIV_ID.json + serverType: MVT + serverSubdomains: [] + serverMinZoom: 0 + serverMaxNativeZoom: 12 + serverMaxZoom: 28 + bbox: + - 96.81676599999997 + - -43.740509999999986 + - 159.1092189999999 + - -9.142175999999996 + nameProp: SORTNAME +``` + +Once indexer fully completes the new region index creation / indexing, you can remove the above config from deployment config and deploy again to make search API uses the latest version index. + +You can check the indexing progress via Indexer module logs. diff --git a/magda-int-test-ts/indexer-setup.conf b/magda-int-test-ts/indexer-setup.conf index 2fefedeec4..8914b9b58c 100644 --- a/magda-int-test-ts/indexer-setup.conf +++ b/magda-int-test-ts/indexer-setup.conf @@ -7,7 +7,7 @@ registry.registerForWebhooks = false # only enable / use one region file for speeding up the initialisation regionSources = { COUNTRY { - url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v2.0.0/country.ndjson" idField = "id" nameField = "name" order = 9 diff --git a/magda-scala-common/src/main/resources/common.conf b/magda-scala-common/src/main/resources/common.conf index 01efed7e36..02f86ba9d4 100755 --- a/magda-scala-common/src/main/resources/common.conf +++ b/magda-scala-common/src/main/resources/common.conf @@ -47,7 +47,7 @@ elasticSearch { indices { regions { - version = 26 + version = 27 } datasets { @@ -93,14 +93,14 @@ logging { regionSources = { # Australia (Mainland) and all offshore territories as a whole COUNTRY { - url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/country.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v2.0.0/country.ndjson" idField = "id" nameField = "name" order = 9 } # Regions for each of Australia offshore territories OFFSHORE_TERRITORIES { - url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/off-shore-territories.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v2.0.0/off-shore-territories.ndjson" idField = "id" nameField = "name" lv1Id = "2" @@ -108,74 +108,74 @@ regionSources = { } # ABS Statistical Area Level 4 SA4 { - url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA4.geojson" - idField = "SA4_CODE11" - nameField = "SA4_NAME11" + url = "https://github.com/magda-io/magda-regions/releases/download/v2.0.0/SA4_2021.ndjson" + idField = "SA4_CODE_2021" + nameField = "SA4_NAME_2021" lv1Id = "1" - lv2IdField = "STE_CODE11" + lv2IdField = "STATE_CODE_2021" order = 30 } # ABS Statistical Area Level 3 SA3 { - url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA3.geojson" - idField = "SA3_CODE11" - nameField = "SA3_NAME11" + url = "https://github.com/magda-io/magda-regions/releases/download/v2.0.0/SA3_2021.ndjson" + idField = "SA3_CODE_2021" + nameField = "SA3_NAME_2021" lv1Id = "1" - lv2IdField = "STE_CODE11" - lv3IdField = "SA4_CODE11" + lv2IdField = "STATE_CODE_2021" + lv3IdField = "SA4_CODE_2021" order = 40 } # ABS Statistical Area Level 2 SA2 { - url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA2.geojson" - idField = "SA2_MAIN11" - nameField = "SA2_NAME11" + url = "https://github.com/magda-io/magda-regions/releases/download/v2.0.0/SA2_2021.ndjson" + idField = "SA2_CODE_2021" + nameField = "SA2_NAME_2021" lv1Id = "1" - lv2IdField = "STE_CODE11" - lv3IdField = "SA4_CODE11" - lv4IdField = "SA3_CODE11" + lv2IdField = "STATE_CODE_2021" + lv3IdField = "SA4_CODE_2021" + lv4IdField = "SA3_CODE_2021" order = 50 } # ABS Statistical Area Level 1 SA1 { - url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/SA1.geojson" - idField = "SA1_MAIN11" - nameField = "SA1_MAIN11", + url = "https://github.com/magda-io/magda-regions/releases/download/v2.0.0/SA1_2021.ndjson" + idField = "SA1_CODE_2021" + nameField = "SA1_CODE_2021" lv1Id = "1" - lv2IdField = "STE_CODE11" - lv3IdField = "SA4_CODE11" - lv4IdField = "SA3_CODE11" - lv5IdField = "SA2_MAIN11" + lv2IdField = "STATE_CODE_2021" + lv3IdField = "SA4_CODE_2021" + lv4IdField = "SA3_CODE_2021" + lv5IdField = "SA2_CODE_2021" order = 60 } # Australia Local Government Areas LGA { - url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/LGA.geojson" - idField = "LGA_CODE15" - nameField = "LGA_NAME15" + url = "https://github.com/magda-io/magda-regions/releases/download/v2.0.0/LGA_2023.ndjson" + idField = "LGA_CODE_2023" + nameField = "LGA_NAME_2023" lv1Id = "1" - steIdField = "STE_CODE11" + lv2IdField = "STATE_CODE_2021" order = 20 } # Australia Postal Areas POA { - url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/POA.geojson" - idField = "POA_CODE" - nameField = "POA_NAME" + url = "https://github.com/magda-io/magda-regions/releases/download/v2.0.0/POA_2021.ndjson" + idField = "POA_CODE_2021" + nameField = "POA_NAME_2021" lv1Id = "1" order = 70 } # Australia Commonwealth electoral boundaries - COM_ELB_ID_2016 { - url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/COM_ELB_ID_2016.geojson" - idField = "DIV_ID" - nameField = "SORTNAME" + ELB { + url = "https://github.com/magda-io/magda-regions/releases/download/v2.0.0/ELB_2021.ndjson" + idField = "Elect_div" + nameField = "Elect_div" lv1Id = "1" order = 80 } # Australia State and Territory STE { - url = "https://github.com/magda-io/magda-regions/releases/download/v1.0.0/STE.simplified.geojson" + url = "https://github.com/magda-io/magda-regions/releases/download/v2.0.0/STE.simplified.ndjson" idField = "STE_CODE11" nameField = "STE_NAME11" shortNameField = "STE_ABBREV" @@ -189,202 +189,4 @@ regionSources = { # Served via search API [region-types](https://dev.magda.io/api/v0/apidocs/index.html#api-Search-GetV0SearchRegionTypes) endpoint # See repo here: https://github.com/magda-io/magda-regions # If not provided, the [default region mapping file](https://github.com/magda-io/magda/blob/main/magda-search-api/src/main/resources/regionMapping.json) will be used. -# regionMapping = { -# "comments": "Matching takes place in the order defined in this file. Place code matches before name matches, and smaller regions before larger ones.", -# "regionWmsMap": { -# "STE": { -# "layerName": "FID_STE_2011_AUST", -# "server": "https://vector-tiles.terria.io/FID_STE_2011_AUST/{z}/{x}/{y}.pbf", -# "regionProp": "STE_CODE11", -# "aliases": ["ste_code", "ste_code_2011", "ste"], -# "digits": 1, -# "description": "States and Territories (STE)", -# "regionIdsFile": "data/regionids/region_map-FID_STE_2011_AUST_STE_CODE11.json", -# "serverType": "MVT", -# "serverSubdomains": [], -# "serverMinZoom": 0, -# "serverMaxNativeZoom": 12, -# "serverMaxZoom": 28, -# "bbox": [ -# 96.81694140799998, -# -43.74050960300003, -# 159.10921900799997, -# -9.142175976999999 -# ], -# "nameProp": "STE_NAME11" -# }, -# "SA4": { -# "layerName": "FID_SA4_2011_AUST", -# "server": "https://vector-tiles.terria.io/FID_SA4_2011_AUST/{z}/{x}/{y}.pbf", -# "regionProp": "SA4_CODE11", -# "aliases": ["sa4_code_2011", "sa4_code", "sa4"], -# "digits": 3, -# "description": "Statistical Area Level 4 (SA4)", -# "regionIdsFile": "data/regionids/region_map-FID_SA4_2011_AUST_SA4_CODE11.json", -# "serverType": "MVT", -# "serverSubdomains": [], -# "serverMinZoom": 0, -# "serverMaxNativeZoom": 12, -# "serverMaxZoom": 28, -# "bbox": [ -# 96.81694140799998, -# -43.74050960300003, -# 159.10921900799997, -# -9.142175976999999 -# ], -# "nameProp": "SA4_NAME11" -# }, -# "SA3": { -# "layerName": "FID_SA3_2011_AUST", -# "server": "https://vector-tiles.terria.io/FID_SA3_2011_AUST/{z}/{x}/{y}.pbf", -# "regionProp": "SA3_CODE11", -# "aliases": ["sa3_code_2011", "sa3_code", "sa3"], -# "digits": 5, -# "description": "Statistical Area Level 3 (SA3)", -# "regionIdsFile": "data/regionids/region_map-FID_SA3_2011_AUST_SA3_CODE11.json", -# "serverType": "MVT", -# "serverSubdomains": [], -# "serverMinZoom": 0, -# "serverMaxNativeZoom": 12, -# "serverMaxZoom": 28, -# "bbox": [ -# 96.81694140799998, -# -43.74050960300003, -# 159.10921900799997, -# -9.142175976999999 -# ], -# "nameProp": "SA3_NAME11" -# }, -# "SA2": { -# "layerName": "FID_SA2_2011_AUST", -# "server": "https://vector-tiles.terria.io/FID_SA2_2011_AUST/{z}/{x}/{y}.pbf", -# "regionProp": "SA2_MAIN11", -# "aliases": ["sa2_code_2011", "sa2_code", "sa2"], -# "digits": 9, -# "description": "Statistical Area Level 2 (SA2)", -# "regionIdsFile": "data/regionids/region_map-FID_SA2_2011_AUST_SA2_MAIN11.json", -# "serverType": "MVT", -# "serverSubdomains": [], -# "serverMinZoom": 0, -# "serverMaxNativeZoom": 12, -# "serverMaxZoom": 28, -# "bbox": [ -# 96.81694140799998, -# -43.74050960300003, -# 159.10921900799997, -# -9.142175976999999 -# ], -# "nameProp": "SA2_NAME11" -# }, -# "SA1": { -# "layerName": "FID_SA1_2011_AUST", -# "server": "https://vector-tiles.terria.io/FID_SA1_2011_AUST/{z}/{x}/{y}.pbf", -# "regionProp": "SA1_MAIN11", -# "aliases": [ -# "sa1_code_2011", -# "sa1_maincode_2011", -# "sa1_code", -# "sa1" -# ], -# "digits": 11, -# "description": "Statistical Area Level 1 (SA1)", -# "regionIdsFile": "data/regionids/region_map-FID_SA1_2011_AUST_SA1_MAIN11.json", -# "serverType": "MVT", -# "serverSubdomains": [], -# "serverMinZoom": 0, -# "serverMaxNativeZoom": 12, -# "serverMaxZoom": 28, -# "bbox": [ -# 96.81694140799998, -# -43.74050960300003, -# 159.10921900799997, -# -9.142175976999999 -# ], -# "nameProp": "SA2_NAME11" -# }, -# "LGA": { -# "layerName": "FID_LGA_2015_AUST", -# "server": "https://vector-tiles.terria.io/FID_LGA_2015_AUST/{z}/{x}/{y}.pbf", -# "regionProp": "LGA_CODE15", -# "aliases": [ -# "lga_code_2015", -# "lga_code", -# "lga", -# "lga_code_2014", -# "lga_code_2012", -# "lga_code_2010" -# ], -# "digits": 5, -# "description": "Local Government Area (LGA)", -# "regionIdsFile": "data/regionids/region_map-FID_LGA_2015_AUST_LGA_CODE15.json", -# "serverType": "MVT", -# "serverSubdomains": [], -# "serverMinZoom": 0, -# "serverMaxNativeZoom": 12, -# "serverMaxZoom": 28, -# "bbox": [ -# 96.81694140799998, -# -43.740509602999985, -# 159.10921900799997, -# -9.142175976999999 -# ], -# "nameProp": "LGA_NAME15" -# }, -# "POA": { -# "layerName": "FID_POA_2011_AUST", -# "server": "https://vector-tiles.terria.io/FID_POA_2011_AUST/{z}/{x}/{y}.pbf", -# "regionProp": "POA_CODE", -# "aliases": [ -# "poa_2011", -# "postcode_2011", -# "poa", -# "poa_code", -# "poa_code_2011", -# "postcode", -# "postcode_2015" -# ], -# "digits": 4, -# "dataReplacements": [["^(?=\\d\\d\\d$)", "0"]], -# "description": "Postal Area (POA)", -# "regionIdsFile": "data/regionids/region_map-FID_POA_2011_AUST_POA_CODE.json", -# "serverType": "MVT", -# "serverSubdomains": [], -# "serverMinZoom": 0, -# "serverMaxNativeZoom": 12, -# "serverMaxZoom": 28, -# "bbox": [ -# 96.81694140799998, -# -43.59821500299999, -# 159.10921900799997, -# -9.142175976999999 -# ], -# "nameProp": "POA_NAME" -# }, -# "COM_ELB_ID_2016": { -# "layerName": "FID_COM20160509_ELB", -# "server": "https://vector-tiles.terria.io/FID_COM20160509_ELB/{z}/{x}/{y}.pbf", -# "regionProp": "DIV_ID", -# "aliases": [ -# "divisionid", -# "com_elb_id_2016", -# "com_elb_id", -# "com_elb" -# ], -# "digits": 3, -# "description": "Commonwealth Electoral District", -# "regionIdsFile": "data/regionids/region_map-FID_COM20160509_ELB_DIV_ID.json", -# "serverType": "MVT", -# "serverSubdomains": [], -# "serverMinZoom": 0, -# "serverMaxNativeZoom": 12, -# "serverMaxZoom": 28, -# "bbox": [ -# 96.81676599999997, -# -43.740509999999986, -# 159.1092189999999, -# -9.142175999999996 -# ], -# "nameProp": "SORTNAME" -# } -# } -# } +# regionMapping = {} diff --git a/magda-scala-common/src/main/scala/au/csiro/data61/magda/search/elasticsearch/IndexDefinition.scala b/magda-scala-common/src/main/scala/au/csiro/data61/magda/search/elasticsearch/IndexDefinition.scala index fc1d9ee70d..d093680934 100644 --- a/magda-scala-common/src/main/scala/au/csiro/data61/magda/search/elasticsearch/IndexDefinition.scala +++ b/magda-scala-common/src/main/scala/au/csiro/data61/magda/search/elasticsearch/IndexDefinition.scala @@ -22,6 +22,7 @@ import org.locationtech.spatial4j.context.jts.JtsSpatialContext import spray.json._ import au.csiro.data61.magda.util.RichConfig._ +import java.time.OffsetDateTime import scala.concurrent.Future case class IndexDefinition( @@ -390,7 +391,7 @@ object IndexDefinition extends DefaultJsonProtocol { val regions: IndexDefinition = new IndexDefinition( name = "regions", - version = 26, + version = 27, indicesIndex = Indices.RegionsIndex, definition = (indices, config) => { val createIdxReq = @@ -413,7 +414,8 @@ object IndexDefinition extends DefaultJsonProtocol { .searchAnalyzer("regionSearchIdInput"), magdaGeoShapeField("boundingBox"), magdaGeoShapeField("geometry"), - intField("order") + intField("order"), + dateField("indexed") ) ) .analysis( @@ -611,7 +613,14 @@ object IndexDefinition extends DefaultJsonProtocol { .map { case (regionSource, jsonRegion) => val properties = jsonRegion.fields("properties").asJsObject - val id = properties.fields(regionSource.idProperty).convertTo[String] + val id = properties.fields(regionSource.idProperty) match { + case JsString(s) => s + case JsNumber(n) => n.toString() + case _ => + throw new Exception( + "Invalid id property value type found for region file: " + regionSource.name + ) + } val name = if (regionSource.includeIdInName) { JsString( properties @@ -743,7 +752,8 @@ object IndexDefinition extends DefaultJsonProtocol { 5, jsonRegion, regionSource - ) + ), + "indexed" -> JsString(OffsetDateTime.now.toString) ).toJson ) ) diff --git a/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionLoader.scala b/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionLoader.scala index 09e7c26cd0..6284fffd6a 100644 --- a/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionLoader.scala +++ b/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionLoader.scala @@ -106,6 +106,17 @@ class FileRegionLoader(regionSources: List[RegionSource])( val fileSink = FileIO.toPath(file.toPath()) response.entity.withoutSizeLimit().dataBytes.runWith(fileSink) } + .flatMap { ioResult => + if (ioResult.wasSuccessful) { + Future.successful(file) + } else { + Future.failed( + new RuntimeException( + s"Failed to write file: ${ioResult.getError.getMessage}" + ) + ) + } + } .recover { case e: Throwable => system.log.info( @@ -117,13 +128,12 @@ class FileRegionLoader(regionSources: List[RegionSource])( file.delete() throw e } - .map(_ => file) } } override def setupRegions(): Source[(RegionSource, JsObject), _] = { Source(regionSources) - .mapAsync(3)( + .mapAsyncUnordered(3)( regionSource => loadABSRegions(regionSource).map((_, regionSource)) ) .flatMapConcat { @@ -136,6 +146,13 @@ class FileRegionLoader(regionSources: List[RegionSource])( .map(byteString => byteString.decodeString("UTF-8")) .map(string => string.parseJson) .map(jsValue => (regionSource, jsValue.asJsObject)) + .recover { + case ex: Exception => + system.log.error( + s"region file processing failed: ${ex.getMessage}" + ) + throw ex + } } } } diff --git a/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionSource.scala b/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionSource.scala index 98452beb87..be1805c665 100644 --- a/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionSource.scala +++ b/magda-scala-common/src/main/scala/au/csiro/data61/magda/spatial/RegionSource.scala @@ -53,41 +53,54 @@ class RegionSources(config: Config) { .map { case (name: String, config: ConfigObject) => val regionSourceConfig = config.toConfig() - RegionSource( - name = name, - url = new URL(regionSourceConfig.getString("url")), - idProperty = regionSourceConfig.getString("idField"), - nameProperty = regionSourceConfig.getString("nameField"), - shortNameProperty = - regionSourceConfig.getOptionalString("shortNameField"), - includeIdInName = - if (regionSourceConfig.hasPath("includeIdInName")) - regionSourceConfig.getBoolean("includeIdInName") - else false, - disabled = regionSourceConfig - .hasPath("disabled") && regionSourceConfig.getBoolean("disabled"), - order = regionSourceConfig.getInt("order"), - simplifyToleranceRatio = - if (regionSourceConfig.hasPath("simplifyToleranceRatio")) - regionSourceConfig.getDouble("simplifyToleranceRatio") - else 0.01, - requireSimplify = - if (regionSourceConfig.hasPath("requireSimplify")) - regionSourceConfig.getBoolean("requireSimplify") - else true, - lv1IdField = regionSourceConfig.getOptionalString("lv1IdField"), - lv2IdField = regionSourceConfig.getOptionalString("lv2IdField"), - lv3IdField = regionSourceConfig.getOptionalString("lv3IdField"), - lv4IdField = regionSourceConfig.getOptionalString("lv4IdField"), - lv5IdField = regionSourceConfig.getOptionalString("lv5IdField"), - lv1Id = regionSourceConfig.getOptionalString("lv1Id"), - lv2Id = regionSourceConfig.getOptionalString("lv2Id"), - lv3Id = regionSourceConfig.getOptionalString("lv3Id"), - lv4Id = regionSourceConfig.getOptionalString("lv4Id"), - lv5Id = regionSourceConfig.getOptionalString("lv5Id") - ) + + val disabled = regionSourceConfig + .hasPath("disabled") && regionSourceConfig.getBoolean("disabled") + + if (disabled) { + println(s"""Region ${name} file is disabled and ignored.""") + None + } else { + Some( + RegionSource( + name = name, + url = new URL(regionSourceConfig.getString("url")), + idProperty = regionSourceConfig.getString("idField"), + nameProperty = regionSourceConfig.getString("nameField"), + shortNameProperty = + regionSourceConfig.getOptionalString("shortNameField"), + includeIdInName = + if (regionSourceConfig.hasPath("includeIdInName")) + regionSourceConfig.getBoolean("includeIdInName") + else false, + disabled = regionSourceConfig + .hasPath("disabled") && regionSourceConfig + .getBoolean("disabled"), + order = regionSourceConfig.getInt("order"), + simplifyToleranceRatio = + if (regionSourceConfig.hasPath("simplifyToleranceRatio")) + regionSourceConfig.getDouble("simplifyToleranceRatio") + else 0.01, + requireSimplify = + if (regionSourceConfig.hasPath("requireSimplify")) + regionSourceConfig.getBoolean("requireSimplify") + else true, + lv1IdField = regionSourceConfig.getOptionalString("lv1IdField"), + lv2IdField = regionSourceConfig.getOptionalString("lv2IdField"), + lv3IdField = regionSourceConfig.getOptionalString("lv3IdField"), + lv4IdField = regionSourceConfig.getOptionalString("lv4IdField"), + lv5IdField = regionSourceConfig.getOptionalString("lv5IdField"), + lv1Id = regionSourceConfig.getOptionalString("lv1Id"), + lv2Id = regionSourceConfig.getOptionalString("lv2Id"), + lv3Id = regionSourceConfig.getOptionalString("lv3Id"), + lv4Id = regionSourceConfig.getOptionalString("lv4Id"), + lv5Id = regionSourceConfig.getOptionalString("lv5Id") + ) + ) + } } .toSeq - .filterNot(_.disabled) + .filterNot(_.isEmpty) + .map(_.get) } } diff --git a/magda-search-api/src/main/resources/regionMapping.json b/magda-search-api/src/main/resources/regionMapping.json index 13fa7d10e6..a09aa7b8ee 100644 --- a/magda-search-api/src/main/resources/regionMapping.json +++ b/magda-search-api/src/main/resources/regionMapping.json @@ -2,198 +2,150 @@ "comments": "Matching takes place in the order defined in this file. Place code matches before name matches, and smaller regions before larger ones.", "regionWmsMap": { "STE": { - "layerName": "FID_STE_2011_AUST", - "server": "https://vector-tiles.terria.io/FID_STE_2011_AUST/{z}/{x}/{y}.pbf", - "regionProp": "STE_CODE11", - "aliases": ["ste_code", "ste_code_2011", "ste"], - "digits": 1, - "description": "States and Territories (STE)", - "regionIdsFile": "data/regionids/region_map-FID_STE_2011_AUST_STE_CODE11.json", + "layerName": "STE_2021", + "server": "https://tiles.magda.io/STE_2021/{z}/{x}/{y}.pbf", "serverType": "MVT", - "serverSubdomains": [], - "serverMinZoom": 0, "serverMaxNativeZoom": 12, + "serverMinZoom": 0, "serverMaxZoom": 28, - "bbox": [ - 96.81694140799998, - -43.74050960300003, - 159.10921900799997, - -9.142175976999999 - ], - "nameProp": "STE_NAME11" + "regionIdsFile": "build/TerriaJS/data/regionids/region_map-STE_2021.json", + "uniqueIdProp": "FID", + "regionProp": "STATE_CODE_2021", + "nameProp": "STATE_NAME_2021", + "aliases": ["ste_code_2021", "ste_code", "ste"], + "description": "States and Territories 2021", + "bbox": [96.81, -43.74, 168, -9.14] }, - "SA4": { - "layerName": "FID_SA4_2011_AUST", - "server": "https://vector-tiles.terria.io/FID_SA4_2011_AUST/{z}/{x}/{y}.pbf", - "regionProp": "SA4_CODE11", - "aliases": ["sa4_code_2011", "sa4_code", "sa4"], - "digits": 3, - "description": "Statistical Area Level 4 (SA4)", - "regionIdsFile": "data/regionids/region_map-FID_SA4_2011_AUST_SA4_CODE11.json", + "SA1": { + "layerName": "SA1_2021", + "server": "https://tiles.magda.io/SA1_2021/{z}/{x}/{y}.pbf", "serverType": "MVT", - "serverSubdomains": [], - "serverMinZoom": 0, "serverMaxNativeZoom": 12, + "serverMinZoom": 0, "serverMaxZoom": 28, - "bbox": [ - 96.81694140799998, - -43.74050960300003, - 159.10921900799997, - -9.142175976999999 + "regionIdsFile": "build/TerriaJS/data/regionids/region_map-SA1_2021.json", + "uniqueIdProp": "FID", + "regionProp": "SA1_CODE_2021", + "nameProp": "SA1_CODE_2021", + "aliases": [ + "sa1_code_2021", + "sa1_maincode_2021", + "sa1", + "sa1_code" ], - "nameProp": "SA4_NAME11" + "description": "Statistical Area Level 1 2021 (ABS)", + "bbox": [96.81, -43.75, 159.11, -9.14] }, - "SA3": { - "layerName": "FID_SA3_2011_AUST", - "server": "https://vector-tiles.terria.io/FID_SA3_2011_AUST/{z}/{x}/{y}.pbf", - "regionProp": "SA3_CODE11", - "aliases": ["sa3_code_2011", "sa3_code", "sa3"], - "digits": 5, - "description": "Statistical Area Level 3 (SA3)", - "regionIdsFile": "data/regionids/region_map-FID_SA3_2011_AUST_SA3_CODE11.json", + "SA2": { + "layerName": "SA2_2021", + "server": "https://tiles.magda.io/SA2_2021/{z}/{x}/{y}.pbf", "serverType": "MVT", - "serverSubdomains": [], - "serverMinZoom": 0, "serverMaxNativeZoom": 12, + "serverMinZoom": 0, "serverMaxZoom": 28, - "bbox": [ - 96.81694140799998, - -43.74050960300003, - 159.10921900799997, - -9.142175976999999 + "regionIdsFile": "build/TerriaJS/data/regionids/region_map-SA2_2021.json", + "uniqueIdProp": "FID", + "regionProp": "SA2_CODE_2021", + "nameProp": "SA2_NAME_2021", + "aliases": [ + "sa2_code_2021", + "sa2_maincode_2021", + "sa2", + "sa2_code" ], - "nameProp": "SA3_NAME11" + "description": "Statistical Area Level 2 2021 by code (ABS)", + "bbox": [96.81, -43.75, 168, -9.14] }, - "SA2": { - "layerName": "FID_SA2_2011_AUST", - "server": "https://vector-tiles.terria.io/FID_SA2_2011_AUST/{z}/{x}/{y}.pbf", - "regionProp": "SA2_MAIN11", - "aliases": ["sa2_code_2011", "sa2_code", "sa2"], - "digits": 9, - "description": "Statistical Area Level 2 (SA2)", - "regionIdsFile": "data/regionids/region_map-FID_SA2_2011_AUST_SA2_MAIN11.json", + "SA3": { + "layerName": "SA3_2021", + "server": "https://tiles.magda.io/SA3_2021/{z}/{x}/{y}.pbf", "serverType": "MVT", - "serverSubdomains": [], - "serverMinZoom": 0, "serverMaxNativeZoom": 12, + "serverMinZoom": 0, "serverMaxZoom": 28, - "bbox": [ - 96.81694140799998, - -43.74050960300003, - 159.10921900799997, - -9.142175976999999 - ], - "nameProp": "SA2_NAME11" - }, - "SA1": { - "layerName": "FID_SA1_2011_AUST", - "server": "https://vector-tiles.terria.io/FID_SA1_2011_AUST/{z}/{x}/{y}.pbf", - "regionProp": "SA1_MAIN11", + "regionIdsFile": "build/TerriaJS/data/regionids/region_map-SA3_2021.json", + "uniqueIdProp": "FID", + "regionProp": "SA3_CODE_2021", + "nameProp": "SA3_NAME_2021", "aliases": [ - "sa1_code_2011", - "sa1_maincode_2011", - "sa1_code", - "sa1" + "sa3_code_2021", + "sa3_maincode_2021", + "sa3", + "sa3_code" ], - "digits": 11, - "description": "Statistical Area Level 1 (SA1)", - "regionIdsFile": "data/regionids/region_map-FID_SA1_2011_AUST_SA1_MAIN11.json", + "description": "Statistical Area Level 3 2021 by code (ABS)", + "bbox": [96.81, -43.75, 168, -9.14] + }, + "SA4": { + "layerName": "SA4_2021", + "server": "https://tiles.magda.io/SA4_2021/{z}/{x}/{y}.pbf", "serverType": "MVT", - "serverSubdomains": [], - "serverMinZoom": 0, "serverMaxNativeZoom": 12, + "serverMinZoom": 0, "serverMaxZoom": 28, - "bbox": [ - 96.81694140799998, - -43.74050960300003, - 159.10921900799997, - -9.142175976999999 + "regionIdsFile": "build/TerriaJS/data/regionids/region_map-SA4_2021.json", + "uniqueIdProp": "FID", + "regionProp": "SA4_CODE_2021", + "nameProp": "SA4_NAME_2021", + "aliases": [ + "sa4_code_2021", + "sa4_maincode_2021", + "sa4", + "sa4_code" ], - "nameProp": "SA2_NAME11" + "description": "Statistical Area Level 4 2021 by code (ABS)", + "bbox": [96.81, -43.75, 168, -9.14] }, "LGA": { - "layerName": "FID_LGA_2015_AUST", - "server": "https://vector-tiles.terria.io/FID_LGA_2015_AUST/{z}/{x}/{y}.pbf", - "regionProp": "LGA_CODE15", - "aliases": [ - "lga_code_2015", - "lga_code", - "lga", - "lga_code_2014", - "lga_code_2012", - "lga_code_2010" - ], - "digits": 5, - "description": "Local Government Area (LGA)", - "regionIdsFile": "data/regionids/region_map-FID_LGA_2015_AUST_LGA_CODE15.json", + "layerName": "LGA_2023", + "server": "https://tiles.magda.io/LGA_2023/{z}/{x}/{y}.pbf", "serverType": "MVT", - "serverSubdomains": [], - "serverMinZoom": 0, "serverMaxNativeZoom": 12, + "serverMinZoom": 0, "serverMaxZoom": 28, - "bbox": [ - 96.81694140799998, - -43.740509602999985, - 159.10921900799997, - -9.142175976999999 - ], - "nameProp": "LGA_NAME15" + "regionIdsFile": "build/TerriaJS/data/regionids/region_map-LGA_2023.json", + "uniqueIdProp": "FID", + "regionProp": "LGA_CODE_2023", + "nameProp": "LGA_NAME_2023", + "aliases": ["lga_code_2023", "lga_code", "lga"], + "description": "Local Government Areas 2023", + "bbox": [96.81, -43.74, 168, -9.14] }, - "POA": { - "layerName": "FID_POA_2011_AUST", - "server": "https://vector-tiles.terria.io/FID_POA_2011_AUST/{z}/{x}/{y}.pbf", - "regionProp": "POA_CODE", - "aliases": [ - "poa_2011", - "postcode_2011", - "poa", - "poa_code", - "poa_code_2011", - "postcode", - "postcode_2015" - ], - "digits": 4, - "dataReplacements": [["^(?=\\d\\d\\d$)", "0"]], - "description": "Postal Area (POA)", - "regionIdsFile": "data/regionids/region_map-FID_POA_2011_AUST_POA_CODE.json", + "ELB": { + "layerName": "ELB_2021", + "server": "https://tiles.magda.io/ELB_2021/{z}/{x}/{y}.pbf", "serverType": "MVT", - "serverSubdomains": [], - "serverMinZoom": 0, "serverMaxNativeZoom": 12, + "serverMinZoom": 0, "serverMaxZoom": 28, - "bbox": [ - 96.81694140799998, - -43.59821500299999, - 159.10921900799997, - -9.142175976999999 - ], - "nameProp": "POA_NAME" + "regionIdsFile": "build/TerriaJS/data/regionids/region_map-ELB_NAME_2021.json", + "uniqueIdProp": "FID", + "regionProp": "Elect_div", + "nameProp": "Elect_div", + "aliases": ["com_elb_name_2021", "com_elb_name"], + "description": "Commonwealth Electoral Divisions as at 2 August 2021 (AEC)", + "bbox": [96.81, -43.73, 168, -9.1] }, - "COM_ELB_ID_2016": { - "layerName": "FID_COM20160509_ELB", - "server": "https://vector-tiles.terria.io/FID_COM20160509_ELB/{z}/{x}/{y}.pbf", - "regionProp": "DIV_ID", - "aliases": [ - "divisionid", - "com_elb_id_2016", - "com_elb_id", - "com_elb" - ], - "digits": 3, - "description": "Commonwealth Electoral District", - "regionIdsFile": "data/regionids/region_map-FID_COM20160509_ELB_DIV_ID.json", + "POA": { + "layerName": "POA_2021", + "server": "https://tiles.magda.io/POA_2021/{z}/{x}/{y}.pbf", "serverType": "MVT", - "serverSubdomains": [], - "serverMinZoom": 0, "serverMaxNativeZoom": 12, + "serverMinZoom": 0, "serverMaxZoom": 28, - "bbox": [ - 96.81676599999997, - -43.740509999999986, - 159.1092189999999, - -9.142175999999996 + "regionIdsFile": "build/TerriaJS/data/regionids/region_map-POA_2021.json", + "uniqueIdProp": "FID", + "regionProp": "POA_CODE_2021", + "nameProp": "POA_CODE_2021", + "aliases": [ + "poa_code_2021", + "poa_code", + "poa", + "postcode_2021", + "postcode" ], - "nameProp": "SORTNAME" + "description": "Postal Areas 2021 (ABS)", + "bbox": [96.81, -43.74, 168, -9.14] } } } diff --git a/magda-web-client/src/Components/Dataset/Search/Facets/FacetRegion.js b/magda-web-client/src/Components/Dataset/Search/Facets/FacetRegion.js index 50c565eeae..ad60bc15fd 100644 --- a/magda-web-client/src/Components/Dataset/Search/Facets/FacetRegion.js +++ b/magda-web-client/src/Components/Dataset/Search/Facets/FacetRegion.js @@ -50,12 +50,12 @@ class FacetRegion extends Component { } onFeatureClick(feature) { - let regionMapping = this.props.regionMapping; + const regionMapping = this.props.regionMapping; let regionType = this.state._activeRegion.regionType; if (!regionType) regionType = "STE"; - let regionProp = regionMapping[regionType].regionProp; - let nameProp = regionMapping[regionType].nameProp; + const regionProp = regionMapping[regionType].regionProp; + const nameProp = regionMapping[regionType].nameProp; const region = { regionType: regionType, regionId: feature.properties[regionProp],