diff --git a/README.md b/README.md index c52dd43..5885247 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,18 @@ pip install outpack ``` +## Development + +Useful hatch commands + +``` +hatch shell +hatch run test +hatch run cov +hatch run lint:style +hatch run lint:fmt +``` + ## License `outpack` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license. diff --git a/example/.outpack/config.json b/example/.outpack/config.json new file mode 100644 index 0000000..2abf1f2 --- /dev/null +++ b/example/.outpack/config.json @@ -0,0 +1 @@ +{"schema_version":"0.1.0","core":{"path_archive":"archive","use_file_store":true,"require_complete_tree":false,"hash_algorithm":"sha256"},"logging":{"console":false,"threshold":"info"},"location":[{"name":"local","type":"local","args":{}}]} diff --git a/example/.outpack/files/sha256/2a/85eb5a027c8d2255e672d1592cc38c82cc0b08279b545a573ceccce9eb27cd b/example/.outpack/files/sha256/2a/85eb5a027c8d2255e672d1592cc38c82cc0b08279b545a573ceccce9eb27cd new file mode 100644 index 0000000..efc9762 --- /dev/null +++ b/example/.outpack/files/sha256/2a/85eb5a027c8d2255e672d1592cc38c82cc0b08279b545a573ceccce9eb27cd @@ -0,0 +1,2 @@ +"","a","b" +"1",1,"x" diff --git a/example/.outpack/files/sha256/2d/0edf86db715c06ed0392f2f4adff16b292eb8a8d170f135d6c8c10ad033571 b/example/.outpack/files/sha256/2d/0edf86db715c06ed0392f2f4adff16b292eb8a8d170f135d6c8c10ad033571 new file mode 100644 index 0000000..6642c12 --- /dev/null +++ b/example/.outpack/files/sha256/2d/0edf86db715c06ed0392f2f4adff16b292eb8a8d170f135d6c8c10ad033571 @@ -0,0 +1 @@ +[{"topic":"name","detail":"data","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.1074},{"topic":"id","detail":"20230807-152345-1ad02157","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.10742},{"topic":"parameter","detail":["a: 2","b: y"],"caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.10745},{"topic":"start","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.10751},{"topic":"result","detail":"success","caller":"orderly2::orderly_run","log_level":"info","time":1691421825.10862},{"topic":"end","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.11338},{"topic":"elapsed","detail":"0.005954027 secs","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.11348}] diff --git a/example/.outpack/files/sha256/57/5b62d07fdf3fdf08c7272ae11a6322e5cbc907ee1dcb5d198bf9f900020619 b/example/.outpack/files/sha256/57/5b62d07fdf3fdf08c7272ae11a6322e5cbc907ee1dcb5d198bf9f900020619 new file mode 100644 index 0000000..a01b20f --- /dev/null +++ b/example/.outpack/files/sha256/57/5b62d07fdf3fdf08c7272ae11a6322e5cbc907ee1dcb5d198bf9f900020619 @@ -0,0 +1,2 @@ +"","a","b" +"1",2,"x" diff --git a/example/.outpack/files/sha256/6e/e430041e1b83b72bac79a2e548a0450117dc763cc6ca71aca3bd2dbda6f520 b/example/.outpack/files/sha256/6e/e430041e1b83b72bac79a2e548a0450117dc763cc6ca71aca3bd2dbda6f520 new file mode 100644 index 0000000..021489c --- /dev/null +++ b/example/.outpack/files/sha256/6e/e430041e1b83b72bac79a2e548a0450117dc763cc6ca71aca3bd2dbda6f520 @@ -0,0 +1,3 @@ +orderly2::orderly_parameters(a = NULL, b = NULL) +d <- data.frame(a = a, b = b) +write.csv(d, "data.csv") diff --git a/example/.outpack/files/sha256/70/95e31a5371a4fcf0e105e638d9a2d4229bb8d4253c426874511ec1ed64d685 b/example/.outpack/files/sha256/70/95e31a5371a4fcf0e105e638d9a2d4229bb8d4253c426874511ec1ed64d685 new file mode 100644 index 0000000..3334195 --- /dev/null +++ b/example/.outpack/files/sha256/70/95e31a5371a4fcf0e105e638d9a2d4229bb8d4253c426874511ec1ed64d685 @@ -0,0 +1,2 @@ +"","a","b" +"1",1,"y" diff --git a/example/.outpack/files/sha256/78/f16fdefa2e7a921fadec936a9b076d853d1f0ae8135396944af4e650ff4968 b/example/.outpack/files/sha256/78/f16fdefa2e7a921fadec936a9b076d853d1f0ae8135396944af4e650ff4968 new file mode 100644 index 0000000..2b5b397 --- /dev/null +++ b/example/.outpack/files/sha256/78/f16fdefa2e7a921fadec936a9b076d853d1f0ae8135396944af4e650ff4968 @@ -0,0 +1 @@ +[{"topic":"name","detail":"data","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.08179},{"topic":"id","detail":"20230807-152345-144cf26c","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.08181},{"topic":"parameter","detail":["a: 2","b: x"],"caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.08184},{"topic":"start","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.0819},{"topic":"result","detail":"success","caller":"orderly2::orderly_run","log_level":"info","time":1691421825.08294},{"topic":"end","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.08773},{"topic":"elapsed","detail":"0.005899668 secs","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.08782}] diff --git a/example/.outpack/files/sha256/a3/f318a59d7801e2ab1e403c65f4b225887a3ed4c5b7c97826d0d093ca5af309 b/example/.outpack/files/sha256/a3/f318a59d7801e2ab1e403c65f4b225887a3ed4c5b7c97826d0d093ca5af309 new file mode 100644 index 0000000..074d430 --- /dev/null +++ b/example/.outpack/files/sha256/a3/f318a59d7801e2ab1e403c65f4b225887a3ed4c5b7c97826d0d093ca5af309 @@ -0,0 +1 @@ +[{"topic":"name","detail":"data","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.05724},{"topic":"id","detail":"20230807-152345-0e0662d0","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.05725},{"topic":"parameter","detail":["a: 1","b: y"],"caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.05728},{"topic":"start","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.05734},{"topic":"result","detail":"success","caller":"orderly2::orderly_run","log_level":"info","time":1691421825.05834},{"topic":"end","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.06294},{"topic":"elapsed","detail":"0.005679131 secs","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.06303}] diff --git a/example/.outpack/files/sha256/a6/66a5bc8fa211cfcd1561b051a1f32f4d7730f9fd1295d4ca893009ffba6e98 b/example/.outpack/files/sha256/a6/66a5bc8fa211cfcd1561b051a1f32f4d7730f9fd1295d4ca893009ffba6e98 new file mode 100644 index 0000000..13e087f --- /dev/null +++ b/example/.outpack/files/sha256/a6/66a5bc8fa211cfcd1561b051a1f32f4d7730f9fd1295d4ca893009ffba6e98 @@ -0,0 +1,2 @@ +"","a","b" +"1",2,"y" diff --git a/example/.outpack/files/sha256/cf/b4d4931dcbaef0e0edb2f77b8ed75a15e8b98eb4ce935909fe3c0f277440c9 b/example/.outpack/files/sha256/cf/b4d4931dcbaef0e0edb2f77b8ed75a15e8b98eb4ce935909fe3c0f277440c9 new file mode 100644 index 0000000..ce62c1b --- /dev/null +++ b/example/.outpack/files/sha256/cf/b4d4931dcbaef0e0edb2f77b8ed75a15e8b98eb4ce935909fe3c0f277440c9 @@ -0,0 +1 @@ +[{"topic":"name","detail":"data","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421824.93776},{"topic":"id","detail":"20230807-152344-ee606dce","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421824.93779},{"topic":"parameter","detail":["a: 1","b: x"],"caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421824.93785},{"topic":"start","detail":"2023-08-07 16:23:44","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421824.93795},{"topic":"result","detail":"success","caller":"orderly2::orderly_run","log_level":"info","time":1691421824.94152},{"topic":"end","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.03063},{"topic":"elapsed","detail":"0.09281063 secs","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.03146}] diff --git a/example/.outpack/index/outpack.rds b/example/.outpack/index/outpack.rds new file mode 100644 index 0000000..c68b0fa Binary files /dev/null and b/example/.outpack/index/outpack.rds differ diff --git a/example/.outpack/location/local/20230807-152344-ee606dce b/example/.outpack/location/local/20230807-152344-ee606dce new file mode 100644 index 0000000..d213c9b --- /dev/null +++ b/example/.outpack/location/local/20230807-152344-ee606dce @@ -0,0 +1 @@ +{"schema_version":"0.1.0","packet":"20230807-152344-ee606dce","time":1691421825.0517,"hash":"sha256:f46ff9faa7558a53e4144aaf2694912c15f8ca9fab3da050b320fa31cd9cbe15"} diff --git a/example/.outpack/location/local/20230807-152345-0e0662d0 b/example/.outpack/location/local/20230807-152345-0e0662d0 new file mode 100644 index 0000000..066653f --- /dev/null +++ b/example/.outpack/location/local/20230807-152345-0e0662d0 @@ -0,0 +1 @@ +{"schema_version":"0.1.0","packet":"20230807-152345-0e0662d0","time":1691421825.0762,"hash":"sha256:81a8bd4c765760d148cfae6fdc4b2029727fbb0c14008d6518be7a20ef948f1e"} diff --git a/example/.outpack/location/local/20230807-152345-144cf26c b/example/.outpack/location/local/20230807-152345-144cf26c new file mode 100644 index 0000000..c2c9d7d --- /dev/null +++ b/example/.outpack/location/local/20230807-152345-144cf26c @@ -0,0 +1 @@ +{"schema_version":"0.1.0","packet":"20230807-152345-144cf26c","time":1691421825.1015,"hash":"sha256:58cbb8f71f82684e9185891400a740901a504d1fff420b2c0ea5b6f44ce0c4d3"} diff --git a/example/.outpack/location/local/20230807-152345-1ad02157 b/example/.outpack/location/local/20230807-152345-1ad02157 new file mode 100644 index 0000000..a076e5b --- /dev/null +++ b/example/.outpack/location/local/20230807-152345-1ad02157 @@ -0,0 +1 @@ +{"schema_version":"0.1.0","packet":"20230807-152345-1ad02157","time":1691421825.1291,"hash":"sha256:35f6dc5060f774f26135d3a2f093118e6b31ee1f87aceb01a12d211a656d3085"} diff --git a/example/.outpack/metadata/20230807-152344-ee606dce b/example/.outpack/metadata/20230807-152344-ee606dce new file mode 100644 index 0000000..74506d7 --- /dev/null +++ b/example/.outpack/metadata/20230807-152344-ee606dce @@ -0,0 +1 @@ +{"schema_version":"0.1.0","name":"data","id":"20230807-152344-ee606dce","time":{"start":1691421824.9377,"end":1691421825.0305},"parameters":{"a":1,"b":"x"},"files":[{"path":"data.csv","size":21,"hash":"sha256:2a85eb5a027c8d2255e672d1592cc38c82cc0b08279b545a573ceccce9eb27cd"},{"path":"log.json","size":896,"hash":"sha256:cfb4d4931dcbaef0e0edb2f77b8ed75a15e8b98eb4ce935909fe3c0f277440c9"},{"path":"orderly.R","size":104,"hash":"sha256:6ee430041e1b83b72bac79a2e548a0450117dc763cc6ca71aca3bd2dbda6f520"}],"depends":[],"git":{"sha":"407c7343fdfc1ff4d949b6719bd6977b96cf4fe6","branch":"config","url":["git@github.com:reside-ic/outpack-py.git"]},"custom":{"orderly":{"artefacts":[],"role":[{"path":"orderly.R","role":"orderly"},{"path":"log.json","role":"log"}],"description":{"display":null,"long":null,"custom":null},"shared":[],"session":{"platform":{"version":"R version 4.3.1 (2023-06-16)","os":"Ubuntu 20.04.6 LTS","system":"x86_64, linux-gnu"},"packages":[{"package":"orderly2","version":"1.99.2","attached":false},{"package":"utf8","version":"1.2.3","attached":false},{"package":"R6","version":"2.5.1","attached":false},{"package":"gert","version":"1.9000","attached":false},{"package":"magrittr","version":"2.0.3","attached":false},{"package":"glue","version":"1.6.2","attached":false},{"package":"tibble","version":"3.2.1","attached":false},{"package":"pkgconfig","version":"2.0.3","attached":false},{"package":"lifecycle","version":"1.0.3","attached":false},{"package":"cli","version":"3.6.1","attached":false},{"package":"askpass","version":"1.1","attached":false},{"package":"fansi","version":"1.0.4","attached":false},{"package":"openssl","version":"2.1.0","attached":false},{"package":"vctrs","version":"0.6.3","attached":false},{"package":"withr","version":"2.5.0","attached":false},{"package":"compiler","version":"4.3.1","attached":false},{"package":"sys","version":"3.4.2","attached":false},{"package":"credentials","version":"1.3.2","attached":false},{"package":"pillar","version":"1.9.0","attached":false},{"package":"yaml","version":"2.3.7","attached":false},{"package":"rlang","version":"1.1.1","attached":false},{"package":"jsonlite","version":"1.8.7","attached":false},{"package":"fs","version":"1.6.3","attached":false}]}}}} diff --git a/example/.outpack/metadata/20230807-152345-0e0662d0 b/example/.outpack/metadata/20230807-152345-0e0662d0 new file mode 100644 index 0000000..c81a1f3 --- /dev/null +++ b/example/.outpack/metadata/20230807-152345-0e0662d0 @@ -0,0 +1 @@ +{"schema_version":"0.1.0","name":"data","id":"20230807-152345-0e0662d0","time":{"start":1691421825.0572,"end":1691421825.0629},"parameters":{"a":1,"b":"y"},"files":[{"path":"data.csv","size":21,"hash":"sha256:7095e31a5371a4fcf0e105e638d9a2d4229bb8d4253c426874511ec1ed64d685"},{"path":"log.json","size":897,"hash":"sha256:a3f318a59d7801e2ab1e403c65f4b225887a3ed4c5b7c97826d0d093ca5af309"},{"path":"orderly.R","size":104,"hash":"sha256:6ee430041e1b83b72bac79a2e548a0450117dc763cc6ca71aca3bd2dbda6f520"}],"depends":[],"git":{"sha":"407c7343fdfc1ff4d949b6719bd6977b96cf4fe6","branch":"config","url":["git@github.com:reside-ic/outpack-py.git"]},"custom":{"orderly":{"artefacts":[],"role":[{"path":"orderly.R","role":"orderly"},{"path":"log.json","role":"log"}],"description":{"display":null,"long":null,"custom":null},"shared":[],"session":{"platform":{"version":"R version 4.3.1 (2023-06-16)","os":"Ubuntu 20.04.6 LTS","system":"x86_64, linux-gnu"},"packages":[{"package":"orderly2","version":"1.99.2","attached":false},{"package":"utf8","version":"1.2.3","attached":false},{"package":"R6","version":"2.5.1","attached":false},{"package":"gert","version":"1.9000","attached":false},{"package":"magrittr","version":"2.0.3","attached":false},{"package":"glue","version":"1.6.2","attached":false},{"package":"tibble","version":"3.2.1","attached":false},{"package":"pkgconfig","version":"2.0.3","attached":false},{"package":"lifecycle","version":"1.0.3","attached":false},{"package":"cli","version":"3.6.1","attached":false},{"package":"askpass","version":"1.1","attached":false},{"package":"fansi","version":"1.0.4","attached":false},{"package":"openssl","version":"2.1.0","attached":false},{"package":"vctrs","version":"0.6.3","attached":false},{"package":"withr","version":"2.5.0","attached":false},{"package":"compiler","version":"4.3.1","attached":false},{"package":"sys","version":"3.4.2","attached":false},{"package":"credentials","version":"1.3.2","attached":false},{"package":"pillar","version":"1.9.0","attached":false},{"package":"yaml","version":"2.3.7","attached":false},{"package":"rlang","version":"1.1.1","attached":false},{"package":"jsonlite","version":"1.8.7","attached":false},{"package":"fs","version":"1.6.3","attached":false}]}}}} diff --git a/example/.outpack/metadata/20230807-152345-144cf26c b/example/.outpack/metadata/20230807-152345-144cf26c new file mode 100644 index 0000000..075d75b --- /dev/null +++ b/example/.outpack/metadata/20230807-152345-144cf26c @@ -0,0 +1 @@ +{"schema_version":"0.1.0","name":"data","id":"20230807-152345-144cf26c","time":{"start":1691421825.0818,"end":1691421825.0877},"parameters":{"a":2,"b":"x"},"files":[{"path":"data.csv","size":21,"hash":"sha256:575b62d07fdf3fdf08c7272ae11a6322e5cbc907ee1dcb5d198bf9f900020619"},{"path":"log.json","size":896,"hash":"sha256:78f16fdefa2e7a921fadec936a9b076d853d1f0ae8135396944af4e650ff4968"},{"path":"orderly.R","size":104,"hash":"sha256:6ee430041e1b83b72bac79a2e548a0450117dc763cc6ca71aca3bd2dbda6f520"}],"depends":[],"git":{"sha":"407c7343fdfc1ff4d949b6719bd6977b96cf4fe6","branch":"config","url":["git@github.com:reside-ic/outpack-py.git"]},"custom":{"orderly":{"artefacts":[],"role":[{"path":"orderly.R","role":"orderly"},{"path":"log.json","role":"log"}],"description":{"display":null,"long":null,"custom":null},"shared":[],"session":{"platform":{"version":"R version 4.3.1 (2023-06-16)","os":"Ubuntu 20.04.6 LTS","system":"x86_64, linux-gnu"},"packages":[{"package":"orderly2","version":"1.99.2","attached":false},{"package":"utf8","version":"1.2.3","attached":false},{"package":"R6","version":"2.5.1","attached":false},{"package":"gert","version":"1.9000","attached":false},{"package":"magrittr","version":"2.0.3","attached":false},{"package":"glue","version":"1.6.2","attached":false},{"package":"tibble","version":"3.2.1","attached":false},{"package":"pkgconfig","version":"2.0.3","attached":false},{"package":"lifecycle","version":"1.0.3","attached":false},{"package":"cli","version":"3.6.1","attached":false},{"package":"askpass","version":"1.1","attached":false},{"package":"fansi","version":"1.0.4","attached":false},{"package":"openssl","version":"2.1.0","attached":false},{"package":"vctrs","version":"0.6.3","attached":false},{"package":"withr","version":"2.5.0","attached":false},{"package":"compiler","version":"4.3.1","attached":false},{"package":"sys","version":"3.4.2","attached":false},{"package":"credentials","version":"1.3.2","attached":false},{"package":"pillar","version":"1.9.0","attached":false},{"package":"yaml","version":"2.3.7","attached":false},{"package":"rlang","version":"1.1.1","attached":false},{"package":"jsonlite","version":"1.8.7","attached":false},{"package":"fs","version":"1.6.3","attached":false}]}}}} diff --git a/example/.outpack/metadata/20230807-152345-1ad02157 b/example/.outpack/metadata/20230807-152345-1ad02157 new file mode 100644 index 0000000..05f0319 --- /dev/null +++ b/example/.outpack/metadata/20230807-152345-1ad02157 @@ -0,0 +1 @@ +{"schema_version":"0.1.0","name":"data","id":"20230807-152345-1ad02157","time":{"start":1691421825.1074,"end":1691421825.1133},"parameters":{"a":2,"b":"y"},"files":[{"path":"data.csv","size":21,"hash":"sha256:a666a5bc8fa211cfcd1561b051a1f32f4d7730f9fd1295d4ca893009ffba6e98"},{"path":"log.json","size":896,"hash":"sha256:2d0edf86db715c06ed0392f2f4adff16b292eb8a8d170f135d6c8c10ad033571"},{"path":"orderly.R","size":104,"hash":"sha256:6ee430041e1b83b72bac79a2e548a0450117dc763cc6ca71aca3bd2dbda6f520"}],"depends":[],"git":{"sha":"407c7343fdfc1ff4d949b6719bd6977b96cf4fe6","branch":"config","url":["git@github.com:reside-ic/outpack-py.git"]},"custom":{"orderly":{"artefacts":[],"role":[{"path":"orderly.R","role":"orderly"},{"path":"log.json","role":"log"}],"description":{"display":null,"long":null,"custom":null},"shared":[],"session":{"platform":{"version":"R version 4.3.1 (2023-06-16)","os":"Ubuntu 20.04.6 LTS","system":"x86_64, linux-gnu"},"packages":[{"package":"orderly2","version":"1.99.2","attached":false},{"package":"utf8","version":"1.2.3","attached":false},{"package":"R6","version":"2.5.1","attached":false},{"package":"gert","version":"1.9000","attached":false},{"package":"magrittr","version":"2.0.3","attached":false},{"package":"glue","version":"1.6.2","attached":false},{"package":"tibble","version":"3.2.1","attached":false},{"package":"pkgconfig","version":"2.0.3","attached":false},{"package":"lifecycle","version":"1.0.3","attached":false},{"package":"cli","version":"3.6.1","attached":false},{"package":"askpass","version":"1.1","attached":false},{"package":"fansi","version":"1.0.4","attached":false},{"package":"openssl","version":"2.1.0","attached":false},{"package":"vctrs","version":"0.6.3","attached":false},{"package":"withr","version":"2.5.0","attached":false},{"package":"compiler","version":"4.3.1","attached":false},{"package":"sys","version":"3.4.2","attached":false},{"package":"credentials","version":"1.3.2","attached":false},{"package":"pillar","version":"1.9.0","attached":false},{"package":"yaml","version":"2.3.7","attached":false},{"package":"rlang","version":"1.1.1","attached":false},{"package":"jsonlite","version":"1.8.7","attached":false},{"package":"fs","version":"1.6.3","attached":false}]}}}} diff --git a/example/archive/data/20230807-152344-ee606dce/data.csv b/example/archive/data/20230807-152344-ee606dce/data.csv new file mode 100644 index 0000000..efc9762 --- /dev/null +++ b/example/archive/data/20230807-152344-ee606dce/data.csv @@ -0,0 +1,2 @@ +"","a","b" +"1",1,"x" diff --git a/example/archive/data/20230807-152344-ee606dce/log.json b/example/archive/data/20230807-152344-ee606dce/log.json new file mode 100644 index 0000000..ce62c1b --- /dev/null +++ b/example/archive/data/20230807-152344-ee606dce/log.json @@ -0,0 +1 @@ +[{"topic":"name","detail":"data","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421824.93776},{"topic":"id","detail":"20230807-152344-ee606dce","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421824.93779},{"topic":"parameter","detail":["a: 1","b: x"],"caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421824.93785},{"topic":"start","detail":"2023-08-07 16:23:44","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421824.93795},{"topic":"result","detail":"success","caller":"orderly2::orderly_run","log_level":"info","time":1691421824.94152},{"topic":"end","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.03063},{"topic":"elapsed","detail":"0.09281063 secs","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.03146}] diff --git a/example/archive/data/20230807-152344-ee606dce/orderly.R b/example/archive/data/20230807-152344-ee606dce/orderly.R new file mode 100644 index 0000000..021489c --- /dev/null +++ b/example/archive/data/20230807-152344-ee606dce/orderly.R @@ -0,0 +1,3 @@ +orderly2::orderly_parameters(a = NULL, b = NULL) +d <- data.frame(a = a, b = b) +write.csv(d, "data.csv") diff --git a/example/archive/data/20230807-152345-0e0662d0/data.csv b/example/archive/data/20230807-152345-0e0662d0/data.csv new file mode 100644 index 0000000..3334195 --- /dev/null +++ b/example/archive/data/20230807-152345-0e0662d0/data.csv @@ -0,0 +1,2 @@ +"","a","b" +"1",1,"y" diff --git a/example/archive/data/20230807-152345-0e0662d0/log.json b/example/archive/data/20230807-152345-0e0662d0/log.json new file mode 100644 index 0000000..074d430 --- /dev/null +++ b/example/archive/data/20230807-152345-0e0662d0/log.json @@ -0,0 +1 @@ +[{"topic":"name","detail":"data","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.05724},{"topic":"id","detail":"20230807-152345-0e0662d0","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.05725},{"topic":"parameter","detail":["a: 1","b: y"],"caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.05728},{"topic":"start","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.05734},{"topic":"result","detail":"success","caller":"orderly2::orderly_run","log_level":"info","time":1691421825.05834},{"topic":"end","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.06294},{"topic":"elapsed","detail":"0.005679131 secs","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.06303}] diff --git a/example/archive/data/20230807-152345-0e0662d0/orderly.R b/example/archive/data/20230807-152345-0e0662d0/orderly.R new file mode 100644 index 0000000..021489c --- /dev/null +++ b/example/archive/data/20230807-152345-0e0662d0/orderly.R @@ -0,0 +1,3 @@ +orderly2::orderly_parameters(a = NULL, b = NULL) +d <- data.frame(a = a, b = b) +write.csv(d, "data.csv") diff --git a/example/archive/data/20230807-152345-144cf26c/data.csv b/example/archive/data/20230807-152345-144cf26c/data.csv new file mode 100644 index 0000000..a01b20f --- /dev/null +++ b/example/archive/data/20230807-152345-144cf26c/data.csv @@ -0,0 +1,2 @@ +"","a","b" +"1",2,"x" diff --git a/example/archive/data/20230807-152345-144cf26c/log.json b/example/archive/data/20230807-152345-144cf26c/log.json new file mode 100644 index 0000000..2b5b397 --- /dev/null +++ b/example/archive/data/20230807-152345-144cf26c/log.json @@ -0,0 +1 @@ +[{"topic":"name","detail":"data","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.08179},{"topic":"id","detail":"20230807-152345-144cf26c","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.08181},{"topic":"parameter","detail":["a: 2","b: x"],"caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.08184},{"topic":"start","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.0819},{"topic":"result","detail":"success","caller":"orderly2::orderly_run","log_level":"info","time":1691421825.08294},{"topic":"end","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.08773},{"topic":"elapsed","detail":"0.005899668 secs","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.08782}] diff --git a/example/archive/data/20230807-152345-144cf26c/orderly.R b/example/archive/data/20230807-152345-144cf26c/orderly.R new file mode 100644 index 0000000..021489c --- /dev/null +++ b/example/archive/data/20230807-152345-144cf26c/orderly.R @@ -0,0 +1,3 @@ +orderly2::orderly_parameters(a = NULL, b = NULL) +d <- data.frame(a = a, b = b) +write.csv(d, "data.csv") diff --git a/example/archive/data/20230807-152345-1ad02157/data.csv b/example/archive/data/20230807-152345-1ad02157/data.csv new file mode 100644 index 0000000..13e087f --- /dev/null +++ b/example/archive/data/20230807-152345-1ad02157/data.csv @@ -0,0 +1,2 @@ +"","a","b" +"1",2,"y" diff --git a/example/archive/data/20230807-152345-1ad02157/log.json b/example/archive/data/20230807-152345-1ad02157/log.json new file mode 100644 index 0000000..6642c12 --- /dev/null +++ b/example/archive/data/20230807-152345-1ad02157/log.json @@ -0,0 +1 @@ +[{"topic":"name","detail":"data","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.1074},{"topic":"id","detail":"20230807-152345-1ad02157","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.10742},{"topic":"parameter","detail":["a: 2","b: y"],"caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.10745},{"topic":"start","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_start","log_level":"info","time":1691421825.10751},{"topic":"result","detail":"success","caller":"orderly2::orderly_run","log_level":"info","time":1691421825.10862},{"topic":"end","detail":"2023-08-07 16:23:45","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.11338},{"topic":"elapsed","detail":"0.005954027 secs","caller":"orderly2::outpack_packet_end","log_level":"info","time":1691421825.11348}] diff --git a/example/archive/data/20230807-152345-1ad02157/orderly.R b/example/archive/data/20230807-152345-1ad02157/orderly.R new file mode 100644 index 0000000..021489c --- /dev/null +++ b/example/archive/data/20230807-152345-1ad02157/orderly.R @@ -0,0 +1,3 @@ +orderly2::orderly_parameters(a = NULL, b = NULL) +d <- data.frame(a = a, b = b) +write.csv(d, "data.csv") diff --git a/example/orderly_config.yml b/example/orderly_config.yml new file mode 100644 index 0000000..9a03358 --- /dev/null +++ b/example/orderly_config.yml @@ -0,0 +1 @@ +minimum_orderly_version: "1.99.0" diff --git a/example/src/data/orderly.R b/example/src/data/orderly.R new file mode 100644 index 0000000..021489c --- /dev/null +++ b/example/src/data/orderly.R @@ -0,0 +1,3 @@ +orderly2::orderly_parameters(a = NULL, b = NULL) +d <- data.frame(a = a, b = b) +write.csv(d, "data.csv") diff --git a/pyproject.toml b/pyproject.toml index c743454..dbbaea7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,9 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] -dependencies = [] +dependencies = [ + "dataclasses-json" +] [project.urls] Documentation = "https://github.com/mrc-ide/outpack#readme" diff --git a/scripts/generate_examples b/scripts/generate_examples new file mode 100755 index 0000000..3dce0bb --- /dev/null +++ b/scripts/generate_examples @@ -0,0 +1,23 @@ +#!/usr/bin/env Rscript + +# Until we implement enough of outpack-py, we won't be able to +# bootstrap our own example code to read, so this generates a little +# example set. +root <- gert::git_find() +path <- file.path(root, "example") +unlink(path, recursive = TRUE) +orderly2::orderly_init(path, logging_console = FALSE, use_file_store = TRUE) + +path_src <- file.path(path, "src", "data") +fs::dir_create(path_src) +writeLines( + c("orderly2::orderly_parameters(a = NULL, b = NULL)", + "d <- data.frame(a = a, b = b)", + 'write.csv(d, "data.csv")'), + file.path(path_src, "orderly.R")) + +for (a in 1:2) { + for (b in c("x", "y")) { + orderly2::orderly_run("data", list(a = a, b = b), root = path) + } +} diff --git a/src/outpack/config.py b/src/outpack/config.py new file mode 100644 index 0000000..d1cc40b --- /dev/null +++ b/src/outpack/config.py @@ -0,0 +1,61 @@ +import os.path +from dataclasses import dataclass, field +from typing import Dict, Optional + +from dataclasses_json import config, dataclass_json + + +def read_config(root_path): + with open(_config_path(root_path)) as f: + s = f.read() + return Config.from_json(s.strip()) + + +def _encode_location_dict(d): + return [x.to_dict() for x in d.values()] + + +def _decode_location_dict(d): + return {x["name"]: Location.from_dict(x) for x in d} + + +@dataclass_json() +@dataclass +class ConfigCore: + hash_algorithm: str + path_archive: Optional[str] + use_file_store: bool + require_complete_tree: bool + + +# Note, using A002 and A003 noqa here to allow 'type' to be used as a +# field name and argument; this keeps the class close to the json +# names, and means that things read nicely (location.type rather than +# location.location_type) +@dataclass_json +@dataclass +class Location: + name: str + type: str # noqa: A003 + args: Optional[dict] = None + + def __init__(self, name, type, args=None): # noqa: A002 + self.name = name + self.type = type + self.args = args + + +@dataclass_json() +@dataclass +class Config: + schema_version: str + core: ConfigCore + location: Dict[str, Location] = field( + metadata=config( + encoder=_encode_location_dict, decoder=_decode_location_dict + ) + ) + + +def _config_path(root_path): + return os.path.join(root_path, ".outpack", "config.json") diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..6c09b9e --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,21 @@ +from outpack.config import Config, Location, read_config + + +def test_can_read_config(): + cfg = read_config("example") + assert isinstance(cfg, Config) + assert cfg.core.path_archive == "archive" + assert cfg.core.use_file_store + assert cfg.core.require_complete_tree is False + assert len(cfg.location) == 1 + assert "local" in cfg.location + local = cfg.location["local"] + assert isinstance(local, Location) + assert local.name == "local" + assert local.type == "local" + assert local.args == {} + + +def test_can_write_json(): + cfg = read_config("example") + assert Config.from_json(cfg.to_json()) == cfg