-
Notifications
You must be signed in to change notification settings - Fork 1
/
iterate_blockdata.jl
136 lines (108 loc) · 3.61 KB
/
iterate_blockdata.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
"""
GKSwstype=nul /opt/julia-1.8.5/bin/julia --threads 24 iterate_blockdata.jl
"""
#include("/net/home/lschulz/scripts/toolbox.jl")
#include("/net/home/lschulz/scripts/struct_blockdata.jl")
#include("/net/home/lschulz/scripts/lSSA_deseasonalization.jl")
include("/net/home/lschulz/reduce-daemensionality/toolbox.jl")
include("/net/home/lschulz/reduce-daemensionality/struct_blockdata.jl")
include("/net/home/lschulz/reduce-daemensionality/old/lSSA_deseasonalization.jl")
#big parameters
#N = 11322
#W = Int(floor((365.25*11)))
N = 5844
k = 48
"""
running raw at the moment
"""
#deseasonalization
#windowlength = 30
#overlap=10
#fluxnetfullset
savedirname = "/net/scratch/lschulz/fluxdata_march23/"*"fluxdata_raw.jld2"
wholedata = SharedArray{Float32}(load(savedirname)["data"])
#wholedata = SharedArray{Float32}(load("/net/scratch/lschulz/fluxnetfullset/fullset_15a_gpp_nee_reco_ts_ta_vpd.jld2")["wholedata"])
#LAIF
#wholedata = SharedArray{Float32}(Matrix(load("/net/scratch/lschulz/LAIF/data.jld2")["data"]'))
N,spots,vars = size(wholedata)
"""
OUTDIR NEEDS TO HAVE /
"""
outdir = "/net/scratch/lschulz/fluxfullset_march23/"
#outdir="/net/scratch/lschulz/LAIF/"
#spots=71
#varlist = ["TA_ERA"]#,"VPD_ERA","PA_ERA"]
#wholedata = reshape(hcat([Float32.(load("/net/scratch/lschulz/FLUXERAI/F32_$(flux).jld2")["data"]) for flux in varlist]...),(N,spots,length(varlist)))
#outdir = "/net/scratch/lschulz/ta_erai_11a/"
"""
put EVERYTHING variable TO KNOW in the package
"""
#loc_ind var_ind
struct package
outdir::String
loc_ind::Int64
var_ind::Int64
function package(outdir,loc_ind,var_ind)
return new(outdir,loc_ind,var_ind)
end
end
function packaging(outdir)
if isdir(outdir)==false
mkdir(outdir)
end
P = package[]
for spot in 1:spots,vari in 1:vars
P = push!(P,package(outdir,spot,vari))
end
return P
end
#iterate the packages of all specified parameters
function doitall(d::matrixholder,wholedata::SharedArray{Float32},p::package)
outdir = p.outdir
loc_ind= p.loc_ind
var_ind= p.var_ind
#preproc year manual
#sampleyear = 24
sampleyear = 365
put_data_raw(d,wholedata[:,loc_ind,var_ind])
put_epsilon(d)
put_EOF_diff(d)
calculate(d)
save_results(d,outdir*"diff_$(d.W)_$(loc_ind)_$(var_ind)_raw")
put_EOF_ssa(d)
calculate(d)
save_results(d,outdir*"ssa_$(d.W)_$(loc_ind)_$(var_ind)_raw")
#put_data_gSSA(d,wholedata[:,loc_ind,var_ind]::Vector{Float32},sampleyear)
#put_epsilon(d)
#put_EOF_diff(d)
#calculate(d)
#save_results(d,outdir*"diff_$(d.W)_$(loc_ind)_$(var_ind)_gSSA")
#put_EOF_ssa(d)
#calculate(d)
#save_results(d,outdir*"ssa_$(d.W)_$(loc_ind)_$(var_ind)_gSSA")
#put_data_lSSA(d,wholedata[:,loc_ind,var_ind]::Vector{Float32},sampleyear)
#put_epsilon(d)
#put_EOF_diff(d)
#calculate(d)
##save_results(d,outdir*"diff_$(d.W)_$(loc_ind)_$(var_ind)_lSSA")
#put_EOF_ssa(d)
#calculate(d)
#save_results(d,outdir*"ssa_$(d.W)_$(loc_ind)_$(var_ind)_lSSA")
#put_data_window(d,wholedata[:,loc_ind,var_ind]::Vector{Float32},sampleyear)
#put_epsilon(d)
#put_EOF_diff(d)
#calculate(d)
#save_results(d,outdir*"diff_$(d.W)_$(loc_ind)_$(var_ind)_win")
#put_EOF_ssa(d)
#calculate(d)
#save_results(d,outdir*"ssa_$(d.W)_$(loc_ind)_$(var_ind)_win")
end
#need to manually put the sampleyear in here...
#Walist = [5,5+1/4,5+1/3,5+1/2,5+2/3,5+3/4,6,6+1/4,6+1/3,6+1/2,6+2/3,6+3/4,7]
#Walist = [15,16,17]
Threads.@threads for p=packaging(outdir)
for W = Int.(floor.(365.25 .* [2,3,4,5,6,7,8]))
d = matrixholder(N,W,k)
doitall(d,wholedata,p)
end
end