From 4f68b8df93d12fe9a69b62f7218757eef75b12b2 Mon Sep 17 00:00:00 2001 From: gaelforget Date: Tue, 14 May 2024 22:52:56 -0400 Subject: [PATCH 1/5] skip build if :exe is specified --- src/ModelConfigurations.jl | 31 ++++++++++++++++++----------- src/ModelSteps.jl | 40 ++++++++++++++++++++------------------ 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/src/ModelConfigurations.jl b/src/ModelConfigurations.jl index 939f6292..7c1aaf86 100644 --- a/src/ModelConfigurations.jl +++ b/src/ModelConfigurations.jl @@ -14,6 +14,9 @@ folder=joinpath(pwd(),"tmp1") MC=MITgcm_config(inputs=params,folder=folder) +#providing executable (optional) +#push!(MC.inputs[:setup][:main],(:exe => "./mitgcmuv")) + #modifying run time options (optional) #MC.inputs[:pkg][:PACKAGES][:useECCO]=false @@ -29,17 +32,23 @@ launch(MC) """ function setup_ECCO4!(config::MITgcm_config) if !haskey(config.inputs[:setup],:build) - println("downloading MITgcm ... ") - u0="https://github.com/MITgcm/MITgcm"; p0=joinpath(config,"MITgcm") - @suppress run(`$(git()) clone --depth 1 --branch checkpoint68o $(u0) $(p0)`) - println("downloading code folder ... ") - u0="https://github.com/gaelforget/ECCOv4"; p0=joinpath(config,"ECCOv4") - @suppress run(`$(git()) clone $(u0) $(p0)`) - p1=joinpath(config,"MITgcm","mysetups") - p2=joinpath(p1,"ECCOv4") - mkdir(p1); mv(p0,p2) - p3=joinpath(p2,"build") - P=OrderedDict(:path=>p3,:options=>"-mods=../code -mpi",:exe=>"mitgcmuv") + if !haskey(config.inputs[:setup][:main],:exe) + println("downloading MITgcm ... ") + u0="https://github.com/MITgcm/MITgcm"; p0=joinpath(config,"MITgcm") + @suppress run(`$(git()) clone --depth 1 --branch checkpoint68o $(u0) $(p0)`) + println("downloading code folder ... ") + u0="https://github.com/gaelforget/ECCOv4"; p0=joinpath(config,"ECCOv4") + @suppress run(`$(git()) clone $(u0) $(p0)`) + p1=joinpath(config,"MITgcm","mysetups") + p2=joinpath(p1,"ECCOv4") + mkdir(p1); mv(p0,p2) + p3=joinpath(p2,"build") + n3="mitgcmuv" + else + p3=dirname(config.inputs[:setup][:main][:exe]) + n3=basename(config.inputs[:setup][:main][:exe]) + end + P=OrderedDict(:path=>p3,:options=>"-mods=../code -mpi",:exe=>n3) push!(config.inputs[:setup],(:build => P)) #push!(config.inputs[:setup][:main],(:command => "mpirun -np 96 mitgcmuv")) push!(config.inputs[:setup][:main],(:command => "qsub job.csh")) diff --git a/src/ModelSteps.jl b/src/ModelSteps.jl index 51173176..d42b0677 100644 --- a/src/ModelSteps.jl +++ b/src/ModelSteps.jl @@ -24,30 +24,32 @@ end build(config::MITgcm_config) Build the model using `genmake2`, `make depend`, and `make`. The first two link all -code files, headers, etc in the `build/` folder before compiling the model +code files, headers, etc in the `build/` folder before compiling the model. -(part of the climate model interface as specialized for `MITgcm`) +Note : this is skipped if `config.inputs[:setup][:main][:exe]` is specified. """ function build(config::MITgcm_config) - try + if !haskey(config.inputs[:setup][:main],:exe) + try + pth=pwd() + catch e + cd() + end pth=pwd() - catch e - cd() - end - pth=pwd() - cd(config.inputs[:setup][:build][:path]) - opt=config.inputs[:setup][:build][:options] - opt=Cmd(convert(Vector{String}, split(opt))) - try - @suppress run(`../../../tools/genmake2 $(opt)`) - @suppress run(`make clean`) - @suppress run(`make depend`) - @suppress run(`make -j 4`) - catch e - println("model compilation may have failed") + cd(config.inputs[:setup][:build][:path]) + opt=config.inputs[:setup][:build][:options] + opt=Cmd(convert(Vector{String}, split(opt))) + try + @suppress run(`../../../tools/genmake2 $(opt)`) + @suppress run(`make clean`) + @suppress run(`make depend`) + @suppress run(`make -j 4`) + catch e + println("model compilation may have failed") + end + cd(pth) + return true end - cd(pth) - return true end """ From 917cef49f8896ac7268c989fce4c3e80dce00eea Mon Sep 17 00:00:00 2001 From: gaelforget Date: Wed, 15 May 2024 00:21:00 -0400 Subject: [PATCH 2/5] add input_folder, as option or download --- src/ModelConfigurations.jl | 108 ++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 36 deletions(-) diff --git a/src/ModelConfigurations.jl b/src/ModelConfigurations.jl index 7c1aaf86..ccb48498 100644 --- a/src/ModelConfigurations.jl +++ b/src/ModelConfigurations.jl @@ -17,6 +17,9 @@ MC=MITgcm_config(inputs=params,folder=folder) #providing executable (optional) #push!(MC.inputs[:setup][:main],(:exe => "./mitgcmuv")) +#providing input folder (optional) +#push!(MC.inputs[:setup][:main],(:input_folder => "./input_folder")) + #modifying run time options (optional) #MC.inputs[:pkg][:PACKAGES][:useECCO]=false @@ -31,32 +34,41 @@ launch(MC) ``` """ function setup_ECCO4!(config::MITgcm_config) - if !haskey(config.inputs[:setup],:build) - if !haskey(config.inputs[:setup][:main],:exe) - println("downloading MITgcm ... ") - u0="https://github.com/MITgcm/MITgcm"; p0=joinpath(config,"MITgcm") - @suppress run(`$(git()) clone --depth 1 --branch checkpoint68o $(u0) $(p0)`) - println("downloading code folder ... ") - u0="https://github.com/gaelforget/ECCOv4"; p0=joinpath(config,"ECCOv4") - @suppress run(`$(git()) clone $(u0) $(p0)`) - p1=joinpath(config,"MITgcm","mysetups") - p2=joinpath(p1,"ECCOv4") - mkdir(p1); mv(p0,p2) - p3=joinpath(p2,"build") - n3="mitgcmuv" - else - p3=dirname(config.inputs[:setup][:main][:exe]) - n3=basename(config.inputs[:setup][:main][:exe]) - end - P=OrderedDict(:path=>p3,:options=>"-mods=../code -mpi",:exe=>n3) - push!(config.inputs[:setup],(:build => P)) - #push!(config.inputs[:setup][:main],(:command => "mpirun -np 96 mitgcmuv")) - push!(config.inputs[:setup][:main],(:command => "qsub job.csh")) - println("creating job submission script ...") - p=joinpath(pathof(config),"run") - f=joinpath(p,"submit.csh") - create_script(p,f) + if !haskey(config.inputs[:setup][:main],:exe) + println("downloading MITgcm ... ") + u0="https://github.com/MITgcm/MITgcm"; p0=joinpath(config,"MITgcm") + @suppress run(`$(git()) clone --depth 1 --branch checkpoint68o $(u0) $(p0)`) + println("downloading code folder ... ") + u0="https://github.com/gaelforget/ECCOv4"; p0=joinpath(config,"ECCOv4") + @suppress run(`$(git()) clone $(u0) $(p0)`) + p1=joinpath(config,"MITgcm","mysetups") + p2=joinpath(p1,"ECCOv4") + mkdir(p1); mv(p0,p2) + p3=joinpath(p2,"build") + n3="mitgcmuv" + else + p3=dirname(config.inputs[:setup][:main][:exe]) + n3=basename(config.inputs[:setup][:main][:exe]) end + P=OrderedDict(:path=>p3,:options=>"-mods=../code -mpi",:exe=>n3) + push!(config.inputs[:setup],(:build => P)) + #push!(config.inputs[:setup][:main],(:command => "mpirun -np 96 mitgcmuv")) + push!(config.inputs[:setup][:main],(:command => "qsub job.csh")) + println("creating job submission script ...") + p=joinpath(pathof(config),"run") + f=joinpath(p,"submit.csh") + create_script(p,f) + + if haskey(config.inputs[:setup][:main],:input_folder) + p=config.inputs[:setup][:main][:input_folder] + else + p=ECCO4_inputs.download_input_folder(config) + end + println(p) + f=readdir(p) + pth_run=joinpath(pathof(config),"run") + [symlink(joinpath(p,f[i]),joinpath(pth_run,f[i])) for i in 1:length(f)] + return true end @@ -65,21 +77,22 @@ end module ECCO4_inputs using Dataverse, ClimateModels.DataFrames, ClimateModels.CSV -export get_list, get_files +import MITgcm: MITgcm_config +export get_list, get_files, download_input_folder ## list0=[ - "doi:10.7910/DVN/PICCRE,documentation,inputs_baseline2", + "doi:10.7910/DVN/PICCRE,documentation,", "doi:10.7910/DVN/9WYSZF,surface forcing fields,forcing_baseline2", - "doi:10.7910/DVN/7XYXSF,model initialization,inputs_baseline2", - "doi:10.7910/DVN/GNOREE,in situ T-S profiles,inputs_baseline2", - "doi:10.7910/DVN/MEDQWY,sea level anomaly,inputs_baseline2", - "doi:10.7910/DVN/L3OQT0,sea surface temperature,inputs_baseline2", - "doi:10.7910/DVN/DKXQHO,ice cover fraction,inputs_baseline2", - "doi:10.7910/DVN/F8BCRF,surface wind stress,inputs_baseline2", - "doi:10.7910/DVN/SYZMUX,bottom pressure,inputs_baseline2", - "doi:10.7910/DVN/H2Q1ND,miscellaneous,inputs_baseline2"]; + "doi:10.7910/DVN/7XYXSF,model initialization,", + "doi:10.7910/DVN/GNOREE,in situ T-S profiles,", + "doi:10.7910/DVN/MEDQWY,sea level anomaly,", + "doi:10.7910/DVN/L3OQT0,sea surface temperature,", + "doi:10.7910/DVN/DKXQHO,ice cover fraction,", + "doi:10.7910/DVN/F8BCRF,surface wind stress,", + "doi:10.7910/DVN/SYZMUX,bottom pressure,", + "doi:10.7910/DVN/H2Q1ND,miscellaneous,"]; fil0=joinpath(tempdir(),"Dataverse_list.csv") @@ -123,7 +136,7 @@ function get_list(list1::DataFrame,name::String) end """ - get_files(list1::DataFrame,nam1::String) + get_files(list1::DataFrame,nam1::String,path1::String) Create a list of Dataverse files from folder with specified `name`. @@ -147,6 +160,29 @@ function get_files(list1::DataFrame,nam1::String,path1::String) path3 end +""" + download_input_folder(config::MITgcm_config) +""" +function download_input_folder(config::MITgcm_config) + p=joinpath(config,"input_folder") + mkdir(p) + list1=ECCO4_inputs.get_list() + nam1="model initialization" + ECCO4_inputs.get_files(list1,nam1,p) + nam1="surface forcing fields" + ECCO4_inputs.get_files(list1,nam1,p) + if config.inputs[:pkg][:PACKAGES][:useECCO] + list2=["sea level anomaly","sea surface temperature", + "ice cover fraction","surface wind stress","bottom pressure"] + [ECCO4_inputs.get_files(list1,nam2,p) for nam2 in list2] + end + if config.inputs[:pkg][:PACKAGES][:useProfiles] + nam1="in situ T-S profiles" + ECCO4_inputs.get_files(list1,nam1,p) + end + p +end + end module ECCO4_testreport From 52c87a999c282e7219d83d15245b566c1dfb2467 Mon Sep 17 00:00:00 2001 From: gaelforget Date: Wed, 15 May 2024 00:27:13 -0400 Subject: [PATCH 3/5] full path names --- src/ModelConfigurations.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ModelConfigurations.jl b/src/ModelConfigurations.jl index ccb48498..97e90b86 100644 --- a/src/ModelConfigurations.jl +++ b/src/ModelConfigurations.jl @@ -15,10 +15,10 @@ folder=joinpath(pwd(),"tmp1") MC=MITgcm_config(inputs=params,folder=folder) #providing executable (optional) -#push!(MC.inputs[:setup][:main],(:exe => "./mitgcmuv")) +#push!(MC.inputs[:setup][:main],(:exe => joinpath(pwd(),"mitgcmuv"))) #providing input folder (optional) -#push!(MC.inputs[:setup][:main],(:input_folder => "./input_folder")) +#push!(MC.inputs[:setup][:main],(:input_folder => joinpath(pwd(),"input_folder"))) #modifying run time options (optional) #MC.inputs[:pkg][:PACKAGES][:useECCO]=false From e2e7564baaa2ce1fb988fcf5826092d07a07b088 Mon Sep 17 00:00:00 2001 From: gaelforget Date: Wed, 15 May 2024 00:52:24 -0400 Subject: [PATCH 4/5] fix test, skip download --- src/ModelConfigurations.jl | 9 +++++---- test/runtests.jl | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/ModelConfigurations.jl b/src/ModelConfigurations.jl index 97e90b86..596f800c 100644 --- a/src/ModelConfigurations.jl +++ b/src/ModelConfigurations.jl @@ -64,10 +64,11 @@ function setup_ECCO4!(config::MITgcm_config) else p=ECCO4_inputs.download_input_folder(config) end - println(p) - f=readdir(p) - pth_run=joinpath(pathof(config),"run") - [symlink(joinpath(p,f[i]),joinpath(pth_run,f[i])) for i in 1:length(f)] + if isdir(p) + f=readdir(p) + pth_run=joinpath(pathof(config),"run") + [symlink(joinpath(p,f[i]),joinpath(pth_run,f[i])) for i in 1:length(f)] + end return true end diff --git a/test/runtests.jl b/test/runtests.jl index c3d83e16..fb312781 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -92,15 +92,15 @@ MITgcm_download() @test isa(Γ,NamedTuple) # - fil=joinpath(@__DIR__,"..","examples","configurations","OCCA2.toml") - MC=MITgcm_config(inputs=read_toml(fil)) + MC=MITgcm_config(inputs=read_toml(:OCCA2)) + push!(MC.inputs[:setup][:main],(:exe => tempname())) + push!(MC.inputs[:setup][:main],(:input_folder => tempname())) @suppress setup(MC) - @test MC.inputs[:setup][:build][:exe]=="mitgcmuv" list1=ECCO4_inputs.get_list() nam1="documentation" - @suppress ECCO4_inputs.get_files(list1,nam1,pathof(MC)) - fil=joinpath(MC,"inputs_baseline2","README.pdf") + @suppress ECCO4_inputs.get_files(list1,nam1,joinpath(MC,"run")) + fil=joinpath(MC,"run","README.pdf") @test isfile(fil) ref_file=joinpath(MC,"MITgcm","mysetups","ECCOv4","test","testreport_baseline2.csv") From d85d4805d053b987e04fd1ccdcd0e9101e552a04 Mon Sep 17 00:00:00 2001 From: gaelforget Date: Wed, 15 May 2024 01:10:45 -0400 Subject: [PATCH 5/5] fix test --- test/runtests.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index fb312781..a0086185 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -93,7 +93,6 @@ MITgcm_download() # MC=MITgcm_config(inputs=read_toml(:OCCA2)) - push!(MC.inputs[:setup][:main],(:exe => tempname())) push!(MC.inputs[:setup][:main],(:input_folder => tempname())) @suppress setup(MC)