diff --git a/interfaces/IBF-dashboard/package-lock.json b/interfaces/IBF-dashboard/package-lock.json index 2ea9f334a..00214b141 100644 --- a/interfaces/IBF-dashboard/package-lock.json +++ b/interfaces/IBF-dashboard/package-lock.json @@ -2341,9 +2341,9 @@ "dev": true }, "@types/cors": { - "version": "2.8.9", - "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.9.tgz", - "integrity": "sha512-zurD1ibz21BRlAOIKP8yhrxlqKx6L9VCwkB5kMiP6nZAhoF5MvC7qS1qPA7nRcr1GJolfkQC7/EAL4hdYejLtg==", + "version": "2.8.10", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.10.tgz", + "integrity": "sha512-C7srjHiVG3Ey1nR6d511dtDkCEjxuN9W1HWAEjGq8kpcwmNM6JJkpC0xvabM7BXTG2wDq8Eu33iH9aQKa7IvLQ==", "dev": true }, "@types/geojson": { @@ -8387,9 +8387,9 @@ } }, "karma": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/karma/-/karma-6.1.0.tgz", - "integrity": "sha512-QmRZ6HdKe6mHd89Az6yb85URRyDbXmn2IBo3lic7cwkkLjDWpjrMJxPAKlwNa5dFM1iHdT+kjtNJM0J5YAH90A==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/karma/-/karma-6.2.0.tgz", + "integrity": "sha512-pCB8eNxGgdIdZeC885rbhZ/VyuOPNHUIDNL9EaaMf1NVzpvTjMO8a7zRTn51ZJhOOOxCSpalUdT1A8x76LyVqg==", "dev": true, "requires": { "body-parser": "^1.19.0", @@ -14053,9 +14053,9 @@ "dev": true }, "rfdc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.2.0.tgz", - "integrity": "sha512-ijLyszTMmUrXvjSooucVQwimGUk84eRcmCuLV8Xghe3UO85mjUtRAHRyoMM6XtyqbECaXuBWx18La3523sXINA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", + "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==", "dev": true }, "rgb-regex": { @@ -14744,14 +14744,14 @@ } }, "socket.io": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-3.1.0.tgz", - "integrity": "sha512-Aqg2dlRh6xSJvRYK31ksG65q4kmBOqU4g+1ukhPcoT6wNGYoIwSYPlCPuRwOO9pgLUajojGFztl6+V2opmKcww==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-3.1.2.tgz", + "integrity": "sha512-JubKZnTQ4Z8G4IZWtaAZSiRP3I/inpy8c/Bsx2jrwGrTbKeVU5xd6qkKMHpChYeM3dWZSO0QACiGK+obhBNwYw==", "dev": true, "requires": { "@types/cookie": "^0.4.0", "@types/cors": "^2.8.8", - "@types/node": "^14.14.10", + "@types/node": ">=10.0.0", "accepts": "~1.3.4", "base64id": "~2.0.0", "debug": "~4.3.1", @@ -16168,9 +16168,9 @@ "dev": true }, "ua-parser-js": { - "version": "0.7.23", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.23.tgz", - "integrity": "sha512-m4hvMLxgGHXG3O3fQVAyyAQpZzDOvwnhOTjYz5Xmr7r/+LpkNy3vJXdVRWgd1TkAb7NGROZuSy96CrlNVjA7KA==", + "version": "0.7.24", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.24.tgz", + "integrity": "sha512-yo+miGzQx5gakzVK3QFfN0/L9uVhosXBBO7qmnk7c2iw1IhL212wfA3zbnI54B0obGwC/5NWub/iT9sReMx+Fw==", "dev": true }, "unicode-canonical-property-names-ecmascript": { diff --git a/services/IBF-pipeline/pipeline/lib/cronJob/rainfalldata.py b/services/IBF-pipeline/pipeline/lib/cronJob/rainfalldata.py index bfaae0237..46d0da716 100644 --- a/services/IBF-pipeline/pipeline/lib/cronJob/rainfalldata.py +++ b/services/IBF-pipeline/pipeline/lib/cronJob/rainfalldata.py @@ -16,6 +16,7 @@ from datetime import datetime from bs4 import BeautifulSoup import requests +from geocube.api.core import make_geocube class RainfallData: @@ -24,10 +25,10 @@ def __init__(self, fcStep, days, country_code): self.fcStep = fcStep self.days = days self.inputPath = PIPELINE_DATA + 'input/rainfall/' - # self.extractedGlofasPath = PIPELINE_OUTPUT + \ - # 'glofas_extraction/glofas_forecast_' + self.fcStep + '_' + country_code + '.json' ## create name of outputs, stick to json - self.triggersPerStationPath = PIPELINE_OUTPUT + \ - 'triggers_rp_per_station/triggers_rp_' + self.fcStep + '_' + country_code + '.json' + self.rainrasterPath = PIPELINE_OUTPUT + \ + 'triggers_rp_per_station/rain_rp_' + self.fcStep + '_' + country_code + '.tif' + # self.triggersPerStationPath = PIPELINE_OUTPUT + \ + # 'triggers_rp_per_station/triggers_rp_' + self.fcStep + '_' + country_code + '.json' self.WATERSTATIONS_TRIGGERS = PIPELINE_INPUT + SETTINGS[country_code]['trigger_levels'] self.TRIGGER_RP_COLNAME = SETTINGS[country_code]['trigger_colname'] self.ADMIN_BOUNDARIES = PIPELINE_INPUT + SETTINGS[country_code]['admin_boundaries']['filename'] @@ -131,7 +132,7 @@ def download_GFS_forecast(self): all_url = self.listFD(GFS_SOURCE, ext='') gfs_url = sorted([i for i in all_url if i.split('/')[-2].startswith('gfs.')], reverse=True) # url_date = [] - fc_hrs = np.arange(3, 267, 3) + fc_hrs = np.arange(3, 195, 3) for url_date in gfs_url: # latest_day_url = gfs_url[-1] @@ -231,7 +232,7 @@ def findTrigger(self): threshold_gdf = gpd.GeoDataFrame(df_leadtime, geometry=geometry).set_crs("EPSG:4326") ## forecast (.5 degree) - fc_by_day = mean_by_day.sel(fc_day=mean_by_day.fc_day.values[self.days - 1]).to_dataframe().reset_index() + fc_by_day = mean_by_day.sel(fc_day=mean_by_day.fc_day.values[self.days]).to_dataframe().reset_index() geometry = [Point(xy) for xy in zip(fc_by_day.longitude.astype(float), fc_by_day.latitude.astype(float))] fc_gdf = gpd.GeoDataFrame(fc_by_day, geometry=geometry).set_crs("EPSG:4326") @@ -251,13 +252,13 @@ def findTrigger(self): compare_gdf['fc_day'] = compare_gdf['fc_day'].astype(str) df_trigger = compare_gdf.filter(['latitude', 'longitude', 'geometry', str(str(self.fcStep) + '_pred')]) - out = df_trigger.to_json() + # out = df_trigger.to_json() # output_name = '%s_%sday_'%(runcycle_day, self.fcStep) +self.TRIGGER_RP_COLNAME - with open(self.triggersPerStationPath, 'w') as fp: - fp.write(out) - print('Processed Glofas data - File saved') - - # cube = make_geocube(vector_data=compare_gdf, measurements=[str(str(self.fcStep)+'_pred')], resolution=(0.5, -0.5), output_crs="EPSG:4326") - # cube.rio.to_raster(PIPELINE_OUTPUT + '/' + output_name + '.tif') - # compare_gdf.to_file(outpath + 'option2/' + output_name + '.shp') + # with open(self.triggersPerStationPath, 'w') as fp: + # fp.write(out) + + cube = make_geocube(vector_data=df_trigger, measurements=[str(str(self.fcStep)+'_pred')], resolution=(-0.5, 0.5), align=(0.25, 0.25), output_crs="EPSG:4326") + cube.rio.to_raster(self.rainrasterPath) + + print('Processed Glofas data - File saved') diff --git a/services/IBF-pipeline/pipeline/settings.py b/services/IBF-pipeline/pipeline/settings.py index 373a0fc29..f526fb132 100644 --- a/services/IBF-pipeline/pipeline/settings.py +++ b/services/IBF-pipeline/pipeline/settings.py @@ -123,7 +123,10 @@ }, "trigger_levels": 'Rainfall_station_locations_with_trigger_levels.csv', 'district_mapping': 'Rainfall_station_per_district_uga.csv', - 'admin_boundaries': 'vector/egy_admbnda_adm1_capmas_20170421.shp', + 'admin_boundaries': { + 'filename': 'vector/egy_admbnda_adm1_capmas_20170421.shp', + 'pcode_colname': 'ADM1_PCODE' + }, 'flood_extent_admin_boundaries': '', 'exposure_admin_boundaries': '', 'trigger_colname': '5yr_threshold',