From 9192b420866fc4c20d64c83758a806668871773a Mon Sep 17 00:00:00 2001 From: Chris Kirney Date: Fri, 1 Dec 2023 14:24:33 -0500 Subject: [PATCH 1/2] Making code to expand zip files it's own method in NECB2011. Adjusting btap_cli.rb to support external weather directories containing weather zip files. Adding code to btap_cli_local_test.rb to support this too. Adjusting btap_datapoint.rb to accept, and pass on, external weather data directories. Modifying necb_2011.rb to check if weather files not in the openstudio-standards/data/weather/ directory are contained in a zip file in the external weather data directory. If they are, then the weather zip file is copied to the openstudio-standards/data/weather folder and expanded there. If no appropriate weather data zip files are in the external weather directory (or it doesn't exist) then it moves on to trying to download it from the btap_weather repository. --- .../standards/necb/NECB2011/necb_2011.rb | 128 +++++++++++++----- .../standards/necb/common/btap_datapoint.rb | 7 +- utilities/btap_cli/btap_cli.rb | 2 + .../btap_cli/tests/btap_cli_local_test.rb | 3 +- 4 files changed, 102 insertions(+), 38 deletions(-) diff --git a/lib/openstudio-standards/standards/necb/NECB2011/necb_2011.rb b/lib/openstudio-standards/standards/necb/NECB2011/necb_2011.rb index 1cc450b344..72cf04c0fe 100644 --- a/lib/openstudio-standards/standards/necb/NECB2011/necb_2011.rb +++ b/lib/openstudio-standards/standards/necb/NECB2011/necb_2011.rb @@ -181,6 +181,7 @@ def get_necb_hdd18(model) def model_create_prototype_model(template:, building_type:, epw_file:, + custom_weather_folder: nil, debug: false, sizing_run_dir: Dir.pwd, primary_heating_fuel: 'Electricity', @@ -241,6 +242,7 @@ def model_create_prototype_model(template:, tbd_option: tbd_option, tbd_interpolate: tbd_interpolate, epw_file: epw_file, + custom_weather_folder: custom_weather_folder, sizing_run_dir: sizing_run_dir, primary_heating_fuel: primary_heating_fuel, dcv_type: dcv_type, # Four options: (1) 'NECB_Default', (2) 'No_DCV', (3) 'Occupancy_based_DCV' , (4) 'CO2_based_DCV' @@ -313,6 +315,7 @@ def model_apply_standard(model:, tbd_option: nil, tbd_interpolate: nil, epw_file:, + custom_weather_folder: nil, sizing_run_dir: Dir.pwd, necb_reference_hp: false, necb_reference_hp_supp_fuel: 'DefaultFuel', @@ -378,7 +381,7 @@ def model_apply_standard(model:, space.autocalculateVolume end - apply_weather_data(model: model, epw_file: epw_file) + apply_weather_data(model: model, epw_file: epw_file, custom_weather_folder: custom_weather_folder) apply_loads(model: model, lights_type: lights_type, lights_scale: lights_scale, @@ -623,14 +626,17 @@ def apply_loads(model:, ecm.scale_oa_loads(model: model, scale: oa_scale) end - def apply_weather_data(model:, epw_file:) + def apply_weather_data(model:, epw_file:, custom_weather_folder: nil) # Create full path to weather file weather_files = File.absolute_path(File.join(__FILE__, '..', '..', '..', '..', '..' , '..', "data/weather")) weather_file = File.join(weather_files, epw_file) # Check if the weather file exists. If it does continue as normal, otherwise try to dowload it from the # canmet-energy/btap_weather repository unless File.exist?(weather_file) - get_weather_file_from_repo(epw_file: epw_file) + # Check if btap_batch transferred the weather file + weather_transfer = check_datapoint_weather_folder(epw_file: epw_file, weather_folder: weather_files, custom_weather_folder: custom_weather_folder) + # If btap_batch didn't transfer the weather file, download it. + get_weather_file_from_repo(epw_file: epw_file) unless weather_transfer end climate_zone = 'NECB HDD Method' # Fix EMS references. Temporary workaround for OS issue #2598 @@ -2214,28 +2220,14 @@ def get_weather_file_from_repo(epw_file:) historic_git_folder = @standards_data['constants']['historic_weather_folder_url']['value'].to_s # Get the files from the repository success_flag = download_and_save_file(weather_list_url: historic_weather_files_loc, weather_loc: weather_loc, git_folder: historic_git_folder) - return if success_flag == true + return if success_flag # If the file could not be found in the historical data look for it with the future weather data. puts "Could not find #{epw_file} in historical weather data files, looking in future weather data files." future_weather_files_loc = @standards_data['constants']['future_weather_file_list']['value'].to_s future_git_folder = @standards_data['constants']['future_weather_folder_url']['value'].to_s success_flag = download_and_save_file(weather_list_url: future_weather_files_loc, weather_loc: weather_loc, git_folder: future_git_folder) - if success_flag == true - # Rename the non-ASHRAE.ddy as '_non_ASHRAE.ddy' and save the '_ASHRAE.ddy' as the regular '.ddy' file. This is - # because the ASHRAE .ddy file includes sizing information not included in the regular .ddy file for future - # weather data files. Unfortunately, openstudio-standards just looks for the regular .ddy file for sizing - # information which is why the switch is done. - puts "Renaming #{weather_loc}.ddy as #{weather_loc}_orig.ddy and #{weather_loc}_ASHRAE.ddy as #{weather_loc}.ddy." - puts "This is so that the design weather information in the #{weather_loc}_ASHRAE.ddy file is used." - weather_dir = File.absolute_path(File.join(__FILE__, '..', '..', '..', '..', '..' , '..', "data/weather")) - orig_ddy_name = File.join(weather_dir, (weather_loc + ".ddy")) - ashrae_ddy_name = File.join(weather_dir, (weather_loc + "_ASHRAE.ddy")) - rev_ddy_name = File.join(weather_dir, (weather_loc + "_orig.ddy")) - FileUtils.cp(orig_ddy_name, rev_ddy_name) - FileUtils.cp(ashrae_ddy_name, orig_ddy_name) - return - end - raise("Could not locate the following file in the canmet/btap_weather repository: #{epw_file}. Please check the spelling of the file or visit https://github.com/canmet-energy/btap_weather to see if the file exists.") + return if success_flag + raise("Could not locate the following file in the canmet/btap_weather repository or could not extract the data: #{epw_file}. Please check the spelling of the file or visit https://github.com/canmet-energy/btap_weather to see if the file exists.") end # This method actually looks for and downloads the zip file from the https://github.com/canmet-energy/btap_weather @@ -2268,7 +2260,6 @@ def download_and_save_file(weather_list_url:, weather_loc:, git_folder:) # If the weather file is on the list proceed, otherwise report that it could not be found unless zip_name.nil? # Found the weather file on the list - status = true # Define the full url of the weather zip file we want to download save_file_url = git_folder + zip_name # Define the local location of where the weather zip file will be saved @@ -2288,21 +2279,7 @@ def download_and_save_file(weather_list_url:, weather_loc:, git_folder:) File.open(save_file, 'wb') { |f| f.write(file_url.read) } puts "Downloaded #{save_file_url} to #{save_file}" # Extract the individual weother files from the zip file - Zip::File.open(save_file) do |zip_file| - puts "Expanding #{save_file}" - # Cycle through each file in the zip file - zip_file.each do |entry| - # Define the location of where the file will be saved locally - curr_save_file = File.join(weather_dir, entry.name.to_s) - puts "Extracting #{entry.name} to #{curr_save_file}" - # entry.extract # This was required before but now it isn't. I'm confused so am saving this comment to - # remind me if there are ploblems later - # Read the data from the file - content = entry.get_input_stream.read - # Save the data locally - File.open(curr_save_file, 'wb') { |save_f| save_f.write(content) } - end - end + status = extract_weather_data(zipped_file: save_file, weather_dir: weather_dir) end attemptb = 10 rescue @@ -2321,4 +2298,83 @@ def download_and_save_file(weather_list_url:, weather_loc:, git_folder:) return status end + # This method checks if a zip file containing weather data is stored in an external directory. If it is, then it + # checks if the name of the zip file (without extension) matches the name of the desired epw file (without extension). + # If it does then it copies the zip file to the openstudio-standards weather directory and expands the file. + # Presumably the zip file contains the .ddy, .epw, and .stat files needed by the rest of BTAP. If no appropriate + # weather zip files are present in the external directory then the method returns false. + # Arguments: + # epw_file (string): The name of the .epw file that BTAP will use. + # weather_folder (string): The path to the openstudio-standards weather folder. + # custom_weather_folder (string): The path to the external folder presumably containing the weather data zip file. + def check_datapoint_weather_folder(epw_file:, weather_folder:, custom_weather_folder: nil) + # Check if the external weather directory exists and return false if there isn't one + return false if custom_weather_folder.nil? + # Check if there are any zip files in the external weather directory and return false if there isn't any. + zip_search_term = File.join(custom_weather_folder.to_s, '*.zip') + zip_files = Dir[zip_search_term] + return false if zip_files.empty? + # Look for a zip file in the external directory named after the .epw file. If there isn't one return false. + weather_loc = epw_file[0..-5] + weather_zip = weather_loc + '.zip' + zip_find = zip_files.select{ |check_file | File.basename(check_file).to_s.downcase == weather_zip.to_s.downcase } + return false if zip_find.empty? + # Copy the zip file we want from the external weather directory to the openstudio-standards weather directory and + # extract the weather data in the file. + puts "Copying: #{zip_find[0]} from the btap_cli weather folder to the openstudio-standards weather folder: #{weather_folder}" + FileUtils.cp(zip_find[0], weather_folder) + dest_zip = File.join(weather_folder, weather_zip) + # Return true if everything goes well. + return extract_weather_data(zipped_file: dest_zip, weather_dir: weather_folder) + end + + # This method extracts data from a zip file. The name implies it is to be used for zip files containing weather + # data but really it can be used to extract any zip file. + # Arguments: + # zipped_file (string): As the name implies, this is the name and path to the zip file we want to expand. + # weather_dir (string): This is the folder where the zipped files will be extracted to. Don't let the name fool you. + # it can be any folder not just a weather directory. + def extract_weather_data(zipped_file:, weather_dir:) + # Set a flag to check if the weather data is for future weather. + future_file = false + # Start expanding the data. + Zip::File.open(zipped_file) do |zip_file| + puts "Expanding #{zipped_file}" + # Cycle through each file in the zip file + zip_file.each do |entry| + # Define the location of where the file will be saved locally + curr_save_file = File.join(weather_dir, entry.name.to_s) + puts "Extracting #{entry.name} to #{curr_save_file}" + # Check if the file includes a '_ASHRAE.ddy' term. If there is, later on we will rename the '_ASHRAE.ddy' file + # to just '.ddy'. This is so the rest of BTAP uses the _ASHRAE.ddy file. + entry_name = entry.name.to_s + if entry_name.length > 11 + future_file = true if entry_name[-11..-1] == '_ASHRAE.ddy' + end + # entry.extract # This was required before but now it isn't. I'm confused so am saving this comment to + # remind me if there are problems later. + # Read the data from the file + content = entry.get_input_stream.read + # Save the data locally + File.open(curr_save_file, 'wb') { |save_f| save_f.write(content) } + end + end + if future_file + # Rename the non-ASHRAE.ddy as '_non_ASHRAE.ddy' and save the '_ASHRAE.ddy' as the regular '.ddy' file. This is + # because the ASHRAE .ddy file includes sizing information not included in the regular .ddy file for future + # weather data files. Unfortunately, openstudio-standards just looks for the regular .ddy file for sizing + # information which is why the switch is done. + weather_loc = (File.basename(zipped_file).to_s)[0..-5] + puts "Renaming #{weather_loc}.ddy as #{weather_loc}_orig.ddy and #{weather_loc}_ASHRAE.ddy as #{weather_loc}.ddy." + puts "This is so that the design weather information in the #{weather_loc}_ASHRAE.ddy file is used." + orig_ddy_name = File.join(weather_dir, (weather_loc + ".ddy")) + ashrae_ddy_name = File.join(weather_dir, (weather_loc + "_ASHRAE.ddy")) + rev_ddy_name = File.join(weather_dir, (weather_loc + "_orig.ddy")) + FileUtils.cp(orig_ddy_name, rev_ddy_name) + FileUtils.cp(ashrae_ddy_name, orig_ddy_name) + end + # Return true if everything worked out + return true + end + end diff --git a/lib/openstudio-standards/standards/necb/common/btap_datapoint.rb b/lib/openstudio-standards/standards/necb/common/btap_datapoint.rb index 5ee2f21f58..1e4e28d87a 100644 --- a/lib/openstudio-standards/standards/necb/common/btap_datapoint.rb +++ b/lib/openstudio-standards/standards/necb/common/btap_datapoint.rb @@ -9,6 +9,7 @@ class BTAPDatapoint def initialize(input_folder: nil, output_folder: nil, + weather_folder: nil, input_folder_cache: File.join(__dir__, 'input_cache')) @failed = false @@ -19,7 +20,10 @@ def initialize(input_folder: nil, if output_folder.nil? output_folder = File.join(__dir__, 'output') end - + # Create an empty weather folder if one doesn't exist. This is to avoid issues later. + if weather_folder.nil? + weather_folder = File.join(__dir__, 'weather') + end puts("INPUT FOLDER:#{input_folder}") puts("OUTPUT FOLDER:#{output_folder}") @@ -115,6 +119,7 @@ def initialize(input_folder: nil, # Otherwise modify osm input with options. @standard.model_apply_standard(model: model, epw_file: @options[:epw_file], + custom_weather_folder: weather_folder, sizing_run_dir: File.join(@dp_temp_folder, 'sizing_folder'), primary_heating_fuel: @options[:primary_heating_fuel], necb_reference_hp: @options[:necb_reference_hp], diff --git a/utilities/btap_cli/btap_cli.rb b/utilities/btap_cli/btap_cli.rb index 6e08f659ae..e129da6a5f 100644 --- a/utilities/btap_cli/btap_cli.rb +++ b/utilities/btap_cli/btap_cli.rb @@ -4,6 +4,7 @@ # Default folders if not using s3 @options[:input_folder] = File.join(__dir__, 'input') @options[:output_folder] = File.join(__dir__, 'output') +@options[:weather_folder] = File.join(__dir__, 'weather') optparse = OptionParser.new do |opts| opts.banner = "Usage: #{$0} -s NAME id ..." opts.on('--help', 'Display this screen') do @@ -16,6 +17,7 @@ optparse.parse! BTAPDatapoint.new(input_folder: @options[:input_folder], output_folder: @options[:output_folder], + weather_folder: @options[:weather_folder], input_folder_cache: File.join(__dir__, 'input_cache')) # Example command using s3 urls for input/output diff --git a/utilities/btap_cli/tests/btap_cli_local_test.rb b/utilities/btap_cli/tests/btap_cli_local_test.rb index 44e0d11da5..65b9c4f0b1 100644 --- a/utilities/btap_cli/tests/btap_cli_local_test.rb +++ b/utilities/btap_cli/tests/btap_cli_local_test.rb @@ -4,12 +4,13 @@ def test_cli input_folder = File.join(__dir__, '..', 'input') input_folder_cache = File.join(__dir__, '..', 'input_cache') output_folder = File.join(__dir__, '..', 'output') + weather_folder = File.join(__dir__, ['..', 'weather']) # Make sure temp folder is always clean. FileUtils.rm_rf(input_folder) if Dir.exist?(input_folder) FileUtils.rm_rf(input_folder_cache) if Dir.exist?(input_folder_cache) FileUtils.rm_rf(output_folder) if Dir.exist?(output_folder) FileUtils.mkdir_p(input_folder) FileUtils.cp(File.join(__dir__, 'run_options.yml'), input_folder) - BTAPDatapoint.new(input_folder: input_folder, output_folder: output_folder, input_folder_cache: File.join(__dir__, 'input_cache')) + BTAPDatapoint.new(input_folder: input_folder, output_folder: output_folder, weather_folder: weather_folder, input_folder_cache: File.join(__dir__, 'input_cache')) end end From d6fe46530e8ea159fc585c941e3e2150616437ad Mon Sep 17 00:00:00 2001 From: Chris Kirney Date: Fri, 1 Dec 2023 14:38:29 -0500 Subject: [PATCH 2/2] Adding support for btap_cli weather data folders to btap_cli_local_osm_test.rb. --- utilities/btap_cli/tests/btap_cli_local_osm_test.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utilities/btap_cli/tests/btap_cli_local_osm_test.rb b/utilities/btap_cli/tests/btap_cli_local_osm_test.rb index ea2262337a..2dd729a058 100644 --- a/utilities/btap_cli/tests/btap_cli_local_osm_test.rb +++ b/utilities/btap_cli/tests/btap_cli_local_osm_test.rb @@ -4,6 +4,7 @@ def test_cli_local_osm input_folder = File.join(__dir__, '..', 'input') input_folder_cache = File.join(__dir__, '..', 'input_cache') output_folder = File.join(__dir__, '..', 'output') + weather_folder = File.join(__dir__, ['..', 'weather']) # Make sure temp folder is always clean. FileUtils.rm_rf(input_folder) if Dir.exist?(input_folder) FileUtils.rm_rf(input_folder_cache) if Dir.exist?(input_folder_cache) @@ -12,6 +13,6 @@ def test_cli_local_osm # Run options and local osm file locations FileUtils.cp(File.join(__dir__, 'run_options_local_osm.yml'), File.join(input_folder, 'run_options.yml')) FileUtils.cp(File.join(__dir__, 'LocalCompleteModel.osm'), File.join(input_folder, 'LocalCompleteModel.osm')) - BTAPDatapoint.new(input_folder: input_folder, output_folder: output_folder, input_folder_cache: File.join(__dir__, 'input_cache')) + BTAPDatapoint.new(input_folder: input_folder, output_folder: output_folder, weather_folder: weather_folder, input_folder_cache: File.join(__dir__, 'input_cache')) end end