-
Notifications
You must be signed in to change notification settings - Fork 0
/
.dvc-rc
335 lines (305 loc) · 7.98 KB
/
.dvc-rc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
alias dv=dvc
defn dva dvc add
defn da dvc add
# Config
alias dvcg="dvc config"
alias dvcgct="dvc config cache.type"
alias dvcgctl="dvc config cache.type reflink,symlink,hardlink,copy"
alias dvcgg="dvc config --global"
alias dvcggct="dvc config --global cache.type"
alias dvcggctl="dvc config --global cache.type reflink,symlink,hardlink,copy"
# Checkout
defn dvco dvc checkout
defn dvcf dvc checkout -f
defn dvcof dvc checkout -f
defn dvcor dvc checkout -R
defn dvcr dvc checkout -R
defn dcrf dvc checkout -R -f
defn dvcrf dvc checkout -R -f
defn dvcorl dvc checkout --relink
# Fetch
alias dvf="dvc fetch"
# Unprotect
alias dvup="dvc unprotect"
dvc_mv_to_git() {
for path in "$@"; do
path="${path%.dvc}"
dvc_path="${path}.dvc"
dvc checkout "$path"
dvc unprotect "$path"
git rm "$dvc_path"
git add -f "$path"
done
}
export -f dvc_mv_to_git
defn dv2g dvc_mv_to_git
# GC
alias dvgc="dvc gc"
alias dvgcw="dvc gc -w"
alias dvgcwf="dvc gc -w -f"
alias dvgcwn="dvc gc -w --dry -f"
alias dvgcwr="dvc gc -w --not-in-remote"
alias dvgcwrf="dvc gc -w --not-in-remote -f"
alias dvgcwrn="dvc gc -w --not-in-remote --dry -f"
dvc_to_md5() {
if [ $# -eq 0 ]; then
cat | yq -r '.outs[0].md5'
else
local spec
for spec in "$@"; do
spec="${spec%.dvc}.dvc"
if [[ "$spec" == *:* ]]; then
local cmd=(git show "$spec")
else
local cmd=(cat "$spec")
fi
num_outs="$("${cmd[@]}" | yq -r '.outs | length')"
if [ "$num_outs" != "1" ]; then
echo "$spec: found $num_outs outs, expected 1" >&2
return 1
fi
"${cmd[@]}" | yq -r '.outs[0].md5'
done
fi
}
export -f dvc_to_md5
defn dv2m dvc_to_md5
defn dv5 dvc_to_md5
defn dpm dvc_to_md5
# Diff
defn dvd dvc-diff
defn ddc dvc-diff -c
defn ddcu1 dvc-diff -c -U1
defn ddc1 dvc-diff -c -U1
defn ddc1r dvc-diff -c -U1 -r
defn ddcz dvc-diff -c -x '"gunzip -c"'
defn ddcr dvc-diff -c -r
defn ddr dvc-diff -r
defn dvdc dvc-diff -c
defn dvdr dvc-diff -r
defn dvdv dvc-diff -v
defn dvdh dvc-diff -r HEAD
defn dvdph dvc-diff -r 'HEAD^..HEAD'
defn dvdw dvc-diff -w
defn dvdwh dvc-diff -w -r HEAD
defn dvdwph dvc-diff -w -r 'HEAD^..HEAD'
defn dvud dvc-diff
# ls
alias dvl="dvc ls"
alias dvlr="dvc ls -R"
alias dvm="dvc commit"
alias dvmv="dvc mv"
alias dvpl="dvc pull"
alias dvplr="dvc pull -r"
alias dvpr="dvc push -r"
alias dvps="dvc push"
alias dvpt="dvc status"
alias dvr="dvc remote"
alias dvra="dvc remote add"
alias dra="dvc remote add"
alias dvrd="dvc remote default"
alias drd="dvc remote default"
alias dvrl="dvc remote list"
alias drl="dvc remote list"
alias dvrm="dvc remote modify"
alias dcas="dvc config core.autostage"
alias dcast="dvc config core.autostage true"
dvc_remote_get_url() {
dvc remote list | grep "^$1" | awk '{print $2}' | perl -pe 's/s3:\//s3/'
}
export -f dvc_remote_get_url
defn dvrgu dvc_remote_get_url
dvc_remote_set_url() {
if [ $# -ne 2 ]; then
echo "Usage: $0 <remote> <url>" >&2
return 1
fi
dvc remote modify "$1" url "$2"
}
export -f dvc_remote_set_url
defn dvrsu dvc_remote_set_url
dvc_remote_url() {
if [ $# -eq 1 ]; then
dvc_remote_get_url "$@"
elif [ $# -eq 2 ]; then
dvc_remote_set_url "$@"
else
echo "Usage: $0 <remote> [url]" >&2
return 1
fi
}
export -f dvc_remote_url
defn dvru dvc_remote_url
# Status
alias ds="dvc data status"
alias dvs="dvc data status"
alias dsg="dvc data status --granular"
alias dvsg="dvc data status --granular"
alias dvscj="dvc data status --json"
dvc_data_status_jq() {
dvc data status --json | jq "$@"
}
export -f dvc_data_status_jq
defn dvscjq dvc_data_status_jq
alias dvsj="dvc data status --unchanged --json"
dvc_data_status_unchanged_jq() {
dvc data status --unchanged --json | jq "$@"
}
export -f dvc_data_status_unchanged_jq
defn dvsjq dvc_data_status_unchanged_jq
alias dvsa="dvc data status --unchanged"
alias dvsu="dvc data status --unchanged"
dvc_relpath() {
relpath="$(realpath "--relative-to=`dvc root`" .)"
if [ "$relpath" = "." ]; then
relpath=
else
relpath="$relpath/"
fi
echo "$relpath"
}
export -f dvc_relpath
defn drp dvc_relpath
defn dvrlp dvc_relpath
dvc_list_category() {
if [ $# -eq 0 ]; then
echo "Usage: dvc_list_category <unchanged|uncommitted.modified|uncommitted.deleted|...> [patterns...]" >&2
return 1
fi
category="$1"; shift
relpath="$(dvc_relpath)"
dvc data status --unchanged --json | \
jq -r "(.$category // [])[]" | \
grep "^$relpath" | \
sed "s|^$relpath||" | \
sort | \
if [ $# -gt 0 ]; then
while read -r line; do
for pattern in "$@"; do
if [[ $line == $pattern ]]; then
echo "$line"
break
fi
done
done
else
cat
fi
}
export -f dvc_list_category
defn dla dvc_list_category committed.added
defn dvla dvc_list_category committed.added
defn dlu dvc_list_category unchanged
defn dvlu dvc_list_category unchanged
defn dld dvc_list_category uncommitted.deleted
defn dvld dvc_list_category uncommitted.deleted
defn dlm dvc_list_category uncommitted.modified
defn dvlm dvc_list_category uncommitted.modified
defn dlt dvc_list_category not_in_cache
defn dvlt dvc_list_category not_in_cache
dvc_help() {
dvc "$@" --help
}
export -f dvc_help
defn dvh dvc_help
defn dvcd dvc cache dir
dvc_mdf_cache_path() {
cache="$(dvc cache dir)"
md5_dir="$cache/files/md5"
for md5 in "$@"; do
prefix="${md5:0:2}"
suffix="${md5:2}"
local_cache_path="$md5_dir/$prefix/$suffix"
echo "$local_cache_path"
done
}
export -f dvc_mdf_cache_path
defn dvmcp dvc_mdf_cache_path
dvc_local_cache_path() {
if [ $# -eq 0 ]; then
md5="$(dvc_to_md5)"
dvc_mdf_cache_path "$md5"
else
for d in "$@"; do
d="${d%.dvc}.dvc"
md5="$(dvc_to_md5 "$d")"
dvc_mdf_cache_path "$md5"
done
fi
}
export -f dvc_local_cache_path
defn dlp dvc_local_cache_path
defn dvlp dvc_local_cache_path
defn dvlcp dvc_local_cache_path
dvc_local_cache_path_ref() {
if [ $# -ne 1 ]; then
echo "Usage: dvc_local_cache_path_ref <ref>:<path>" >&2
return 1
fi
ref="${1%:*}"
path="${1#*:}"
if [ "$ref" == "$path" ]; then
ref="HEAD"
fi
path="${path%.dvc}.dvc"
path="$(git relpath "$path")"
git show "$ref:$path" | dvc_local_cache_path
}
export -f dvc_local_cache_path_ref
defn dvlpr dvc_local_cache_path_ref
defn dpr dvc_local_cache_path_ref
dvc_size() {
if [ $# -eq 0 ]; then
yq '.outs | map(.size)[] | add'
elif [ $# -eq 1 ] ; then
yq '.outs | map(.size) | add' "$1"
else
for path in "$@"; do
dvc_size "$path"
done | jq -n '[ inputs ] | add'
fi
}
export -f dvc_size
defn dvz dvc_size
dvc_size_human() {
dvc_size "$@" | numfmt --to=iec
}
export -f dvc_size_human
defn dvzh dvc_size_human
dvc_find() {
git ls-files | grep '.dvc$'
}
export -f dvc_find
defn dvfn dvc_find
dvc_find_suffix() {
git ls-files | grep "${1}.dvc\$"
}
export -f dvc_find_suffix
defn dvfns dvc_find_suffix
dvc_remote_root() {
dvc remote list "$@" | awk '{ print $2 }'
}
export -f dvc_remote_root
defn dvrr dvc_remote_root
dvc_remote_blob_url() {
if [ $# -eq 2 ]; then
remote="$1"; shift
path="$1"; shift
elif [ $# -eq 1 ]; then
remote=
path="$1"; shift
else
echo "Usage: $0 [remote] <path>" >&2
return 1
fi
remote_root="$(dvc_remote_root $remote)"
local_cache_path="$(dvc_local_cache_path "$path")"
dir="$(basename "$(dirname "$local_cache_path")")"
base="$(basename "$local_cache_path")"
echo "$remote_root/files/md5/$dir/$base"
}
export -f dvc_remote_blob_url
defn dvrbu dvc_remote_blob_url
defn dvrp dvc_remote_blob_url
# Used by dvc-utils (https://pypi.org/project/dvc-utils)
export DVC_UTILS_CACHE_DIR=.dvc/cache