From 84cc498355640315107626dc20350a756711b67e Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 May 2024 10:41:49 -0700 Subject: [PATCH 01/24] add mongoDB 7.4 to GHA --- .github/workflows/test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1a391d8..13d5ce7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -24,6 +24,9 @@ jobs: - python: '3.9.19' mongo: 'mongodb-linux-x86_64-3.6.23' wired_tiger: 'false' + - python: '3.9.19' + mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4' + wired_tiger: 'true' steps: - name: Check out GitHub repo From 299402c67829c1563268f4196f94b79af07130bd Mon Sep 17 00:00:00 2001 From: Sijie Xiang Date: Tue, 28 May 2024 17:53:49 +0000 Subject: [PATCH 02/24] upgrade pymango to 4.7.2 --- Pipfile | 2 +- Pipfile.lock | 210 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 125 insertions(+), 87 deletions(-) diff --git a/Pipfile b/Pipfile index 89cc37b..172b74e 100644 --- a/Pipfile +++ b/Pipfile @@ -6,7 +6,7 @@ name = "pypi" [packages] cachetools = "==4.2.2" jsonrpcbase = "==0.2.0" -pymongo = "==3.8.0" +pymongo = "==4.7.2" requests = "==2.31.0" uwsgi = "==2.0.22" diff --git a/Pipfile.lock b/Pipfile.lock index ec7b322..307b9a1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ffdc3fe16bfaca74a9e7b5f036dd7f1ec8e64208a7e3733c23de520e0d2f31f7" + "sha256": "a57becc1f2cf1d6afab94c65419fffd73caaf839782d9d5f220e3effb89fc9c4" }, "pipfile-spec": 6, "requires": { @@ -129,6 +129,14 @@ "markers": "python_full_version >= '3.7.0'", "version": "==3.3.2" }, + "dnspython": { + "hashes": [ + "sha256:5ef3b9680161f6fa89daf8ad451b5f1a33b18ae8a1c6778cdf4b43f08c0a6e50", + "sha256:e8f0f9c23a7b7cb99ded64e6c3a6f3e701d78f50c55e002b839dea7225cff7cc" + ], + "markers": "python_version >= '3.8'", + "version": "==2.6.1" + }, "idna": { "hashes": [ "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc", @@ -146,40 +154,70 @@ }, "pymongo": { "hashes": [ - "sha256:32421df60d06f479d71b6b539642e410ece3006e8910688e68df962c8eb40a21", - "sha256:324b22a8443e11faca44c96b20e7ec8a9e59a1e664457edeeb4f796080b31cde", - "sha256:3b6336b1d2a1ac2fcc8f629070016f3c76ad7dc969f269232471953d6dd17c0d", - "sha256:4505ff8b7923dd7a8bed1bf25c9c4d0df5ab0b8b2821f2296533f2149a55f401", - "sha256:460b224681ea711e48e3638d15be2249024031b7dcb9622ba19c2e85bd5a26cc", - "sha256:47473b70c5f3cd5ddd2c49ab3b9ceafdafbbed5bc963f147df22a9343d7978f5", - "sha256:49375839af76834e9c5c3cc78c78386873fd0b2ad9a0860a7dc4ec9fe73af9dd", - "sha256:4a65f0f71ece86c860d30a1436b646db8ea32aec518845ef2903ca569faec32e", - "sha256:530621906c5dd6d27305b39c4e017701e5f4299aa68b93cde70eb985f94ca26f", - "sha256:54f4770b5810e8dc3cbeed675874195f02bb2bc4e95a9d665068edfb3baff4f7", - "sha256:5ed9382410e938b0ff76041c34018210504729a83bcf4f6a70c7092c28169f6f", - "sha256:61cad83637ae12c1c825130d7f9325cd6c162e3a64e8747a8144866020be3ff4", - "sha256:61e8e1c58b4fdf47ab79b7c7db8bb022c1e40b3b5fcbbaeea5fc94dc5c75638d", - "sha256:6e04e496af7d156b66cce70460011c621ecbadf5dcdce325c7acbb3cd6ea245d", - "sha256:74838f04da0b3995b830fe1f00f9b200831582cbc42a22b77e04dfb717cb0d56", - "sha256:7ef89ec435e89da902451dde6845066fe2770befaf0301fe2a1ac426b51fced3", - "sha256:854e8425e5eb775ccfffad04ecd094c99923d60a2c2d49babb5c435e836a91fa", - "sha256:9569796d48498e4db4e1d56284b626a8ed15f641ce3a8b2085f06bb03f4c2c88", - "sha256:9d50c99c6388863cbfdc5db9bad62e3a7c2e5fc151554a07c7f3c2530334a34f", - "sha256:9ea016c2c011df21f77c1f806ce45129a344ba2d414bd50f9e065b13a4a134be", - "sha256:a8421f0823174888fb12a5fa675322e756499d71e77ff712b4412d4b8f3c6503", - "sha256:aef7d88384ada699976350a285c7a333f96ebc959e98e7d2c98589f47bbf3b7f", - "sha256:b4d7ff9957ee770cf03bd7156a68a2f2e838e60712d9608eadc8741c15d01e72", - "sha256:c1db85c39e6a60588f855dbc7bd68fb0dab796096148ab5aa4abecaff19e1c6e", - "sha256:c3e813b1bd0b883639e30170dc9daccb9b6ef7e81836188b88d3fc7364892b35", - "sha256:cee2fc0b94e66e7230da12fc4b3d34793c49957e16ee04f6468a94e264a1e41d", - "sha256:cf1dea28379a16b23e47db312883f07b3ba8d9d6abc1c59e51d4c8ae1820ab43", - "sha256:d1cd175df7c8b5fc976bade78bf4d9fb5aa7ab465c0f59931e380bbe188ef8fc", - "sha256:d48a94edf3cdd34524936a72ea01b352682b337f33a42db10ba29a96c37147d3", - "sha256:d9cc103a4e97f78bc77a1d72759ab3722f6cdf0374ad4fb4b0c53bd3238bdf98", - "sha256:fcb9ae8aa9158106c5d98a4349ec0d90b68f052d620b2d24622ba03b91e4d81d" + "sha256:02efd1bb3397e24ef2af45923888b41a378ce00cb3a4259c5f4fc3c70497a22f", + "sha256:0d833651f1ba938bb7501f13e326b96cfbb7d98867b2d545ca6d69c7664903e0", + "sha256:12c466e02133b7f8f4ff1045c6b5916215c5f7923bc83fd6e28e290cba18f9f6", + "sha256:12d1fef77d25640cb78893d07ff7d2fac4c4461d8eec45bd3b9ad491a1115d6e", + "sha256:194065c9d445017b3c82fb85f89aa2055464a080bde604010dc8eb932a6b3c95", + "sha256:1c78f156edc59b905c80c9003e022e1a764c54fd40ac4fea05b0764f829790e2", + "sha256:1e37faf298a37ffb3e0809e77fbbb0a32b6a2d18a83c59cfc2a7b794ea1136b0", + "sha256:25eeb2c18ede63891cbd617943dd9e6b9cbccc54f276e0b2e693a0cc40f243c5", + "sha256:268d8578c0500012140c5460755ea405cbfe541ef47c81efa9d6744f0f99aeca", + "sha256:2cb77d09bd012cb4b30636e7e38d00b5f9be5eb521c364bde66490c45ee6c4b4", + "sha256:347c49cf7f0ba49ea87c1a5a1984187ecc5516b7c753f31938bf7b37462824fd", + "sha256:35b3f0c7d49724859d4df5f0445818d525824a6cd55074c42573d9b50764df67", + "sha256:37e9ea81fa59ee9274457ed7d59b6c27f6f2a5fe8e26f184ecf58ea52a019cb8", + "sha256:47a1a4832ef2f4346dcd1a10a36ade7367ad6905929ddb476459abb4fd1b98cb", + "sha256:4bdb5ffe1cd3728c9479671a067ef44dacafc3743741d4dc700c377c4231356f", + "sha256:4ffd1519edbe311df73c74ec338de7d294af535b2748191c866ea3a7c484cd15", + "sha256:5239776633f7578b81207e5646245415a5a95f6ae5ef5dff8e7c2357e6264bfc", + "sha256:5239ef7e749f1326ea7564428bf861d5250aa39d7f26d612741b1b1273227062", + "sha256:56bf8b706946952acdea0fe478f8e44f1ed101c4b87f046859e6c3abe6c0a9f4", + "sha256:65b4c00dedbd333698b83cd2095a639a6f0d7c4e2a617988f6c65fb46711f028", + "sha256:6a87eef394039765679f75c6a47455a4030870341cb76eafc349c5944408c882", + "sha256:727ad07952c155cd20045f2ce91143c7dc4fb01a5b4e8012905a89a7da554b0c", + "sha256:730778b6f0964b164c187289f906bbc84cb0524df285b7a85aa355bbec43eb21", + "sha256:743552033c63f0afdb56b9189ab04b5c1dbffd7310cf7156ab98eebcecf24621", + "sha256:7e9d9d2c0aae73aa4369bd373ac2ac59f02c46d4e56c4b6d6e250cfe85f76802", + "sha256:82102e353be13f1a6769660dd88115b1da382447672ba1c2662a0fbe3df1d861", + "sha256:827611beb6c483260d520cfa6a49662d980dfa5368a04296f65fa39e78fccea7", + "sha256:84bc00200c3cbb6c98a2bb964c9e8284b641e4a33cf10c802390552575ee21de", + "sha256:87032f818bf5052ab742812c715eff896621385c43f8f97cdd37d15b5d394e95", + "sha256:87832d6076c2c82f42870157414fd876facbb6554d2faf271ffe7f8f30ce7bed", + "sha256:87bb453ac3eb44db95cb6d5a616fbc906c1c00661eec7f55696253a6245beb8a", + "sha256:9024e1661c6e40acf468177bf90ce924d1bc681d2b244adda3ed7b2f4c4d17d7", + "sha256:9349f0bb17a31371d4cacb64b306e4ca90413a3ad1fffe73ac7cd495570d94b5", + "sha256:9385654f01a90f73827af4db90c290a1519f7d9102ba43286e187b373e9a78e9", + "sha256:9a8bd37f5dabc86efceb8d8cbff5969256523d42d08088f098753dba15f3b37a", + "sha256:9d892fb91e81cccb83f507cdb2ea0aa026ec3ced7f12a1d60f6a5bf0f20f9c1f", + "sha256:a754e366c404d19ff3f077ddeed64be31e0bb515e04f502bf11987f1baa55a16", + "sha256:b48a5650ee5320d59f6d570bd99a8d5c58ac6f297a4e9090535f6561469ac32e", + "sha256:bcf337d1b252405779d9c79978d6ca15eab3cdaa2f44c100a79221bddad97c8a", + "sha256:c44efab10d9a3db920530f7bcb26af8f408b7273d2f0214081d3891979726328", + "sha256:c72d16fede22efe7cdd1f422e8da15760e9498024040429362886f946c10fe95", + "sha256:cb6e00a79dff22c9a72212ad82021b54bdb3b85f38a85f4fc466bde581d7d17a", + "sha256:ce1a374ea0e49808e0380ffc64284c0ce0f12bd21042b4bef1af3eb7bdf49054", + "sha256:cecd2df037249d1c74f0af86fb5b766104a5012becac6ff63d85d1de53ba8b98", + "sha256:cf17ea9cea14d59b0527403dd7106362917ced7c4ec936c4ba22bd36c912c8e0", + "sha256:cf28430ec1924af1bffed37b69a812339084697fd3f3e781074a0148e6475803", + "sha256:d1bcd58669e56c08f1e72c5758868b5df169fe267501c949ee83c418e9df9155", + "sha256:d275596f840018858757561840767b39272ac96436fcb54f5cac6d245393fd97", + "sha256:d2dcf608d35644e8d276d61bf40a93339d8d66a0e5f3e3f75b2c155a421a1b71", + "sha256:d4d59776f435564159196d971aa89422ead878174aff8fe18e06d9a0bc6d648c", + "sha256:d9b6cbc037108ff1a0a867e7670d8513c37f9bcd9ee3d2464411bfabf70ca002", + "sha256:db4380d1e69fdad1044a4b8f3bb105200542c49a0dde93452d938ff9db1d6d29", + "sha256:e004527ea42a6b99a8b8d5b42b42762c3bdf80f88fbdb5c3a9d47f3808495b86", + "sha256:e6eab12c6385526d386543d6823b07187fefba028f0da216506e00f0e1855119", + "sha256:eb0642e5f0dd7e86bb358749cc278e70b911e617f519989d346f742dc9520dfb", + "sha256:f91073049c43d14e66696970dd708d319b86ee57ef9af359294eee072abaac79", + "sha256:fadc6e8db7707c861ebe25b13ad6aca19ea4d2c56bf04a26691f46c23dadf6e4", + "sha256:fc5af24fcf5fc6f7f40d65446400d45dd12bea933d0299dc9e90c5b22197f1e9", + "sha256:fcaf8c911cb29316a02356f89dbc0e0dfcc6a712ace217b6b543805690d2aefd", + "sha256:ffd4d7cb2e6c6e100e2b39606d38a9ffc934e18593dc9bb326196afc7d93ce3d" ], "index": "pypi", - "version": "==3.8.0" + "markers": "python_version >= '3.7'", + "version": "==4.7.2" }, "requests": { "hashes": [ @@ -220,61 +258,61 @@ "toml" ], "hashes": [ - "sha256:0646599e9b139988b63704d704af8e8df7fa4cbc4a1f33df69d97f36cb0a38de", - "sha256:0cdcbc320b14c3e5877ee79e649677cb7d89ef588852e9583e6b24c2e5072661", - "sha256:0d0a0f5e06881ecedfe6f3dd2f56dcb057b6dbeb3327fd32d4b12854df36bf26", - "sha256:1434e088b41594baa71188a17533083eabf5609e8e72f16ce8c186001e6b8c41", - "sha256:16db7f26000a07efcf6aea00316f6ac57e7d9a96501e990a36f40c965ec7a95d", - "sha256:1cc0fe9b0b3a8364093c53b0b4c0c2dd4bb23acbec4c9240b5f284095ccf7981", - "sha256:1fc81d5878cd6274ce971e0a3a18a8803c3fe25457165314271cf78e3aae3aa2", - "sha256:2ec92012fefebee89a6b9c79bc39051a6cb3891d562b9270ab10ecfdadbc0c34", - "sha256:39afcd3d4339329c5f58de48a52f6e4e50f6578dd6099961cf22228feb25f38f", - "sha256:4a7b0ceee8147444347da6a66be737c9d78f3353b0681715b668b72e79203e4a", - "sha256:4a9ca3f2fae0088c3c71d743d85404cec8df9be818a005ea065495bedc33da35", - "sha256:4bf0655ab60d754491004a5efd7f9cccefcc1081a74c9ef2da4735d6ee4a6223", - "sha256:4cc37def103a2725bc672f84bd939a6fe4522310503207aae4d56351644682f1", - "sha256:4fc84a37bfd98db31beae3c2748811a3fa72bf2007ff7902f68746d9757f3746", - "sha256:5037f8fcc2a95b1f0e80585bd9d1ec31068a9bcb157d9750a172836e98bc7a90", - "sha256:54de9ef3a9da981f7af93eafde4ede199e0846cd819eb27c88e2b712aae9708c", - "sha256:556cf1a7cbc8028cb60e1ff0be806be2eded2daf8129b8811c63e2b9a6c43bca", - "sha256:57e0204b5b745594e5bc14b9b50006da722827f0b8c776949f1135677e88d0b8", - "sha256:5a5740d1fb60ddf268a3811bcd353de34eb56dc24e8f52a7f05ee513b2d4f596", - "sha256:5c3721c2c9e4c4953a41a26c14f4cef64330392a6d2d675c8b1db3b645e31f0e", - "sha256:5fa567e99765fe98f4e7d7394ce623e794d7cabb170f2ca2ac5a4174437e90dd", - "sha256:5fd215c0c7d7aab005221608a3c2b46f58c0285a819565887ee0b718c052aa4e", - "sha256:6175d1a0559986c6ee3f7fccfc4a90ecd12ba0a383dcc2da30c2b9918d67d8a3", - "sha256:61c4bf1ba021817de12b813338c9be9f0ad5b1e781b9b340a6d29fc13e7c1b5e", - "sha256:6537e7c10cc47c595828b8a8be04c72144725c383c4702703ff4e42e44577312", - "sha256:68f962d9b72ce69ea8621f57551b2fa9c70509af757ee3b8105d4f51b92b41a7", - "sha256:7352b9161b33fd0b643ccd1f21f3a3908daaddf414f1c6cb9d3a2fd618bf2572", - "sha256:796a79f63eca8814ca3317a1ea443645c9ff0d18b188de470ed7ccd45ae79428", - "sha256:79afb6197e2f7f60c4824dd4b2d4c2ec5801ceb6ba9ce5d2c3080e5660d51a4f", - "sha256:7a588d39e0925f6a2bff87154752481273cdb1736270642aeb3635cb9b4cad07", - "sha256:8748731ad392d736cc9ccac03c9845b13bb07d020a33423fa5b3a36521ac6e4e", - "sha256:8fe7502616b67b234482c3ce276ff26f39ffe88adca2acf0261df4b8454668b4", - "sha256:9314d5678dcc665330df5b69c1e726a0e49b27df0461c08ca12674bcc19ef136", - "sha256:9735317685ba6ec7e3754798c8871c2f49aa5e687cc794a0b1d284b2389d1bd5", - "sha256:9981706d300c18d8b220995ad22627647be11a4276721c10911e0e9fa44c83e8", - "sha256:9e78295f4144f9dacfed4f92935fbe1780021247c2fabf73a819b17f0ccfff8d", - "sha256:b016ea6b959d3b9556cb401c55a37547135a587db0115635a443b2ce8f1c7228", - "sha256:b6cf3764c030e5338e7f61f95bd21147963cf6aa16e09d2f74f1fa52013c1206", - "sha256:beccf7b8a10b09c4ae543582c1319c6df47d78fd732f854ac68d518ee1fb97fa", - "sha256:c0884920835a033b78d1c73b6d3bbcda8161a900f38a488829a83982925f6c2e", - "sha256:c3e757949f268364b96ca894b4c342b41dc6f8f8b66c37878aacef5930db61be", - "sha256:ca498687ca46a62ae590253fba634a1fe9836bc56f626852fb2720f334c9e4e5", - "sha256:d1d0d98d95dd18fe29dc66808e1accf59f037d5716f86a501fc0256455219668", - "sha256:d21918e9ef11edf36764b93101e2ae8cc82aa5efdc7c5a4e9c6c35a48496d601", - "sha256:d7fed867ee50edf1a0b4a11e8e5d0895150e572af1cd6d315d557758bfa9c057", - "sha256:db66fc317a046556a96b453a58eced5024af4582a8dbdc0c23ca4dbc0d5b3146", - "sha256:dde0070c40ea8bb3641e811c1cfbf18e265d024deff6de52c5950677a8fb1e0f", - "sha256:df4e745a81c110e7446b1cc8131bf986157770fa405fe90e15e850aaf7619bc8", - "sha256:e2213def81a50519d7cc56ed643c9e93e0247f5bbe0d1247d15fa520814a7cd7", - "sha256:ef48e2707fb320c8f139424a596f5b69955a85b178f15af261bab871873bb987", - "sha256:f152cbf5b88aaeb836127d920dd0f5e7edff5a66f10c079157306c4343d86c19", - "sha256:fc0b4d8bfeabd25ea75e94632f5b6e047eef8adaed0c2161ada1e922e7f7cece" + "sha256:015eddc5ccd5364dcb902eaecf9515636806fa1e0d5bef5769d06d0f31b54523", + "sha256:04aefca5190d1dc7a53a4c1a5a7f8568811306d7a8ee231c42fb69215571944f", + "sha256:05ac5f60faa0c704c0f7e6a5cbfd6f02101ed05e0aee4d2822637a9e672c998d", + "sha256:0bbddc54bbacfc09b3edaec644d4ac90c08ee8ed4844b0f86227dcda2d428fcb", + "sha256:1d2a830ade66d3563bb61d1e3c77c8def97b30ed91e166c67d0632c018f380f0", + "sha256:239a4e75e09c2b12ea478d28815acf83334d32e722e7433471fbf641c606344c", + "sha256:244f509f126dc71369393ce5fea17c0592c40ee44e607b6d855e9c4ac57aac98", + "sha256:25a5caf742c6195e08002d3b6c2dd6947e50efc5fc2c2205f61ecb47592d2d83", + "sha256:296a7d9bbc598e8744c00f7a6cecf1da9b30ae9ad51c566291ff1314e6cbbed8", + "sha256:2e079c9ec772fedbade9d7ebc36202a1d9ef7291bc9b3a024ca395c4d52853d7", + "sha256:33ca90a0eb29225f195e30684ba4a6db05dbef03c2ccd50b9077714c48153cac", + "sha256:33fc65740267222fc02975c061eb7167185fef4cc8f2770267ee8bf7d6a42f84", + "sha256:341dd8f61c26337c37988345ca5c8ccabeff33093a26953a1ac72e7d0103c4fb", + "sha256:34d6d21d8795a97b14d503dcaf74226ae51eb1f2bd41015d3ef332a24d0a17b3", + "sha256:3538d8fb1ee9bdd2e2692b3b18c22bb1c19ffbefd06880f5ac496e42d7bb3884", + "sha256:38a3b98dae8a7c9057bd91fbf3415c05e700a5114c5f1b5b0ea5f8f429ba6614", + "sha256:3d5a67f0da401e105753d474369ab034c7bae51a4c31c77d94030d59e41df5bd", + "sha256:50084d3516aa263791198913a17354bd1dc627d3c1639209640b9cac3fef5807", + "sha256:55f689f846661e3f26efa535071775d0483388a1ccfab899df72924805e9e7cd", + "sha256:5bc5a8c87714b0c67cfeb4c7caa82b2d71e8864d1a46aa990b5588fa953673b8", + "sha256:62bda40da1e68898186f274f832ef3e759ce929da9a9fd9fcf265956de269dbc", + "sha256:705f3d7c2b098c40f5b81790a5fedb274113373d4d1a69e65f8b68b0cc26f6db", + "sha256:75e3f4e86804023e991096b29e147e635f5e2568f77883a1e6eed74512659ab0", + "sha256:7b2a19e13dfb5c8e145c7a6ea959485ee8e2204699903c88c7d25283584bfc08", + "sha256:7cec2af81f9e7569280822be68bd57e51b86d42e59ea30d10ebdbb22d2cb7232", + "sha256:8383a6c8cefba1b7cecc0149415046b6fc38836295bc4c84e820872eb5478b3d", + "sha256:8c836309931839cca658a78a888dab9676b5c988d0dd34ca247f5f3e679f4e7a", + "sha256:8e317953bb4c074c06c798a11dbdd2cf9979dbcaa8ccc0fa4701d80042d4ebf1", + "sha256:923b7b1c717bd0f0f92d862d1ff51d9b2b55dbbd133e05680204465f454bb286", + "sha256:990fb20b32990b2ce2c5f974c3e738c9358b2735bc05075d50a6f36721b8f303", + "sha256:9aad68c3f2566dfae84bf46295a79e79d904e1c21ccfc66de88cd446f8686341", + "sha256:a5812840d1d00eafae6585aba38021f90a705a25b8216ec7f66aebe5b619fb84", + "sha256:a6519d917abb15e12380406d721e37613e2a67d166f9fb7e5a8ce0375744cd45", + "sha256:ab0b028165eea880af12f66086694768f2c3139b2c31ad5e032c8edbafca6ffc", + "sha256:aea7da970f1feccf48be7335f8b2ca64baf9b589d79e05b9397a06696ce1a1ec", + "sha256:b1196e13c45e327d6cd0b6e471530a1882f1017eb83c6229fc613cd1a11b53cd", + "sha256:b368e1aee1b9b75757942d44d7598dcd22a9dbb126affcbba82d15917f0cc155", + "sha256:bde997cac85fcac227b27d4fb2c7608a2c5f6558469b0eb704c5726ae49e1c52", + "sha256:c4c2872b3c91f9baa836147ca33650dc5c172e9273c808c3c3199c75490e709d", + "sha256:c59d2ad092dc0551d9f79d9d44d005c945ba95832a6798f98f9216ede3d5f485", + "sha256:d1da0a2e3b37b745a2b2a678a4c796462cf753aebf94edcc87dcc6b8641eae31", + "sha256:d8b7339180d00de83e930358223c617cc343dd08e1aa5ec7b06c3a121aec4e1d", + "sha256:dd4b3355b01273a56b20c219e74e7549e14370b31a4ffe42706a8cda91f19f6d", + "sha256:e08c470c2eb01977d221fd87495b44867a56d4d594f43739a8028f8646a51e0d", + "sha256:f5102a92855d518b0996eb197772f5ac2a527c0ec617124ad5242a3af5e25f85", + "sha256:f542287b1489c7a860d43a7d8883e27ca62ab84ca53c965d11dac1d3a1fab7ce", + "sha256:f78300789a708ac1f17e134593f577407d52d0417305435b134805c4fb135adb", + "sha256:f81bc26d609bf0fbc622c7122ba6307993c83c795d2d6f6f6fd8c000a770d974", + "sha256:f836c174c3a7f639bded48ec913f348c4761cbf49de4a20a956d3431a7c9cb24", + "sha256:fa21a04112c59ad54f69d80e376f7f9d0f5f9123ab87ecd18fbb9ec3a2beed56", + "sha256:fcf7d1d6f5da887ca04302db8e0e0cf56ce9a5e05f202720e49b3e8157ddb9a9", + "sha256:fd27d8b49e574e50caa65196d908f80e4dff64d7e592d0c59788b45aad7e8b35" ], "markers": "python_version >= '3.8'", - "version": "==7.5.1" + "version": "==7.5.3" }, "exceptiongroup": { "hashes": [ From cb4485d0b1713045271c0eefb7017147d5852ba6 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 May 2024 11:03:58 -0700 Subject: [PATCH 03/24] fix mongocontroller --- test/mongo_controller.py | 41 +++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/test/mongo_controller.py b/test/mongo_controller.py index 7c2f082..7f4185d 100644 --- a/test/mongo_controller.py +++ b/test/mongo_controller.py @@ -50,6 +50,9 @@ def __init__(self, mongoexe: Path, root_temp_dir: Path, use_wired_tiger: bool = if not root_temp_dir: raise ValueError('root_temp_dir is None') + # semver parser + s = semver.VersionInfo.parse + # make temp dirs root_temp_dir = root_temp_dir.absolute() os.makedirs(root_temp_dir, exist_ok=True) @@ -58,9 +61,12 @@ def __init__(self, mongoexe: Path, root_temp_dir: Path, use_wired_tiger: bool = os.makedirs(data_dir) self.port = _find_free_port() + mongodb_ver = self.get_mongodb_version(mongoexe) + command = [str(mongoexe), '--port', str(self.port), '--dbpath', str(data_dir)] + + if s(mongodb_ver) < s('6.1.0'): + command.extend(['--nojournal']) - command = [str(mongoexe), '--port', str(self.port), '--dbpath', str(data_dir), - '--nojournal'] if use_wired_tiger: command.extend(['--storageEngine', 'wiredTiger']) @@ -68,18 +74,35 @@ def __init__(self, mongoexe: Path, root_temp_dir: Path, use_wired_tiger: bool = self._proc = subprocess.Popen(command, stdout=self._outfile, stderr=subprocess.STDOUT) time.sleep(1) # wait for server to start up - self.client: MongoClient = MongoClient('localhost', self.port) - # check that the server is up. See - # https://api.mongodb.com/python/3.7.0/api/pymongo/mongo_client.html - # #pymongo.mongo_client.MongoClient - self.client.admin.command('ismaster') + + try: + self.client = MongoClient('localhost', self.port) + # This line will raise an exception if the server is down + server_info = self.client.server_info() + except Exception as e: + raise ValueError("MongoDB server is down") from e # get some info about the db - self.db_version = self.client.server_info()['version'] - s = semver.VersionInfo.parse + self.db_version = server_info['version'] self.index_version = 2 if (s(self.db_version) >= s('3.4.0')) else 1 self.includes_system_indexes = (s(self.db_version) < s('3.2.0') and not use_wired_tiger) + def get_mongodb_version(self, mongoexe: Path) -> str: + try: + process = subprocess.Popen( + [str(mongoexe), '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + stdout, stderr = process.communicate() + + if process.returncode == 0: + version_line = stdout.decode().split('\n')[0] + mongodb_version = version_line.split()[2][1:] + return mongodb_version.strip() + else: + raise ValueError(f"Error: {stderr.decode()}") + except Exception as e: + raise ValueError("Failed to get MongoDB version") from e + def destroy(self, delete_temp_files: bool) -> None: """ Shut down the MongoDB server. From dcd2318f8b14b482a062d075d54fa8b31b5203bf Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 May 2024 11:38:35 -0700 Subject: [PATCH 04/24] fix 'Cursor' object has no attribute 'count' error --- lib/AbstractHandle/Utils/MongoUtil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/AbstractHandle/Utils/MongoUtil.py b/lib/AbstractHandle/Utils/MongoUtil.py index 0c470c2..083a16d 100644 --- a/lib/AbstractHandle/Utils/MongoUtil.py +++ b/lib/AbstractHandle/Utils/MongoUtil.py @@ -104,7 +104,7 @@ def find_in(self, elements, field_name, projection={'_id': False}, batch_size=10 ''.join(traceback.format_exception(None, e, e.__traceback__))) raise ValueError(error_msg) - logging.info('returned {} results'.format(result.count())) + logging.info('returned {} results'.format(len(list(result.clone())))) return result From ff3a5e2716c2dae49a207fb16e3835318453aa46 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 May 2024 11:51:26 -0700 Subject: [PATCH 05/24] fix get_hid_counter function --- lib/AbstractHandle/Utils/MongoUtil.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/AbstractHandle/Utils/MongoUtil.py b/lib/AbstractHandle/Utils/MongoUtil.py index 083a16d..9c66630 100644 --- a/lib/AbstractHandle/Utils/MongoUtil.py +++ b/lib/AbstractHandle/Utils/MongoUtil.py @@ -81,10 +81,11 @@ def get_hid_counter(self): """ get current handle id counter """ - counter = self.hid_counter_collection.find({'_id': {'$eq': self._HID_COUNTER_ID}}) + cursor = self.hid_counter_collection.find({'_id': {'$eq': self._HID_COUNTER_ID}}) + counter = list(cursor) - if counter.count(): - return counter.next().get('hid_counter') + if len(counter): + return counter[0].get('hid_counter') else: return 0 From 72d10ce82242e403357a2a36aac9a15a0438ea4a Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 May 2024 12:34:26 -0700 Subject: [PATCH 06/24] fix test_delete_one_ok --- test/MongoUtil_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/MongoUtil_test.py b/test/MongoUtil_test.py index 8e028d0..20843e0 100644 --- a/test/MongoUtil_test.py +++ b/test/MongoUtil_test.py @@ -193,20 +193,20 @@ def test_delete_one_ok(self): self.start_test() mongo_util = self.getMongoUtil() docs = mongo_util.handle_collection.find() - self.assertEqual(docs.count(), 10) + self.assertEqual(len(list(docs.clone())), 10) doc = docs.next() hid = doc.get('hid') mongo_util.delete_one(doc) - self.assertEqual(mongo_util.handle_collection.find().count(), 9) + self.assertEqual(mongo_util.handle_collection.count_documents(), 9) docs = mongo_util.find_in([hid], 'hid', projection=None) - self.assertEqual(docs.count(), 0) + self.assertEqual(len(list(docs)), 0) mongo_util.insert_one(doc) - self.assertEqual(mongo_util.handle_collection.find().count(), 10) + self.assertEqual(mongo_util.handle_collection.count_documents(), 10) docs = mongo_util.find_in([hid], 'hid', projection=None) - self.assertEqual(docs.count(), 1) + self.assertEqual(len(list(docs)), 1) def test_delete_many_ok(self): self.start_test() From bfef14d29a432d1631448e2188454c01d39c53a9 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 May 2024 12:38:16 -0700 Subject: [PATCH 07/24] add missing filter --- test/MongoUtil_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/MongoUtil_test.py b/test/MongoUtil_test.py index 20843e0..afed286 100644 --- a/test/MongoUtil_test.py +++ b/test/MongoUtil_test.py @@ -198,13 +198,13 @@ def test_delete_one_ok(self): doc = docs.next() hid = doc.get('hid') mongo_util.delete_one(doc) - self.assertEqual(mongo_util.handle_collection.count_documents(), 9) + self.assertEqual(mongo_util.handle_collection.count_documents({}), 9) docs = mongo_util.find_in([hid], 'hid', projection=None) self.assertEqual(len(list(docs)), 0) mongo_util.insert_one(doc) - self.assertEqual(mongo_util.handle_collection.count_documents(), 10) + self.assertEqual(mongo_util.handle_collection.count_documents({}), 10) docs = mongo_util.find_in([hid], 'hid', projection=None) self.assertEqual(len(list(docs)), 1) From 258c118a47029b7c06798d39e3573b0c64bd78bb Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 May 2024 12:43:05 -0700 Subject: [PATCH 08/24] fix test_insert_one_ok --- test/MongoUtil_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/MongoUtil_test.py b/test/MongoUtil_test.py index afed286..fa44d17 100644 --- a/test/MongoUtil_test.py +++ b/test/MongoUtil_test.py @@ -133,7 +133,7 @@ def test_update_one_ok(self): def test_insert_one_ok(self): self.start_test() mongo_util = self.getMongoUtil() - self.assertEqual(mongo_util.handle_collection.find().count(), 10) + self.assertEqual(mongo_util.handle_collection.count_documents({}), 10) doc = {'_id': 9999, 'hid': 9999, 'file_name': 'fake_file'} counter = mongo_util.get_hid_counter() @@ -141,16 +141,16 @@ def test_insert_one_ok(self): new_counter = mongo_util.get_hid_counter() self.assertEqual(new_counter, counter) - self.assertEqual(mongo_util.handle_collection.find().count(), 11) + self.assertEqual(mongo_util.handle_collection.count_documents({}), 11) elements = [9999] docs = mongo_util.find_in(elements, 'hid', projection=None) - self.assertEqual(docs.count(), 1) + self.assertEqual(len(list(docs.clone())), 1) doc = docs.next() self.assertEqual(doc.get('hid'), 9999) self.assertEqual(doc.get('file_name'), 'fake_file') mongo_util.delete_one(doc) - self.assertEqual(mongo_util.handle_collection.find().count(), 10) + self.assertEqual(mongo_util.handle_collection.count_documents({}), 10) def test_increase_counter_with_multi_threads(self): From 957394e9b2c3548d0074790111b717d2115ab5b5 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 May 2024 12:50:06 -0700 Subject: [PATCH 09/24] fix test_find_in_ok && test_update_one_ok --- test/MongoUtil_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/MongoUtil_test.py b/test/MongoUtil_test.py index fa44d17..b94453e 100644 --- a/test/MongoUtil_test.py +++ b/test/MongoUtil_test.py @@ -85,17 +85,17 @@ def test_find_in_ok(self): # test query 'hid' field elements = [68020, 68022, 0] docs = mongo_util.find_in(elements, 'hid') - self.assertEqual(docs.count(), 2) + self.assertEqual(len(list(docs)), 2) # test query 'hid' field with empty data elements = [0] docs = mongo_util.find_in(elements, 'hid') - self.assertEqual(docs.count(), 0) + self.assertEqual(len(list(docs)), 0) # test query 'id' field elements = ['b753774f-0bbd-4b96-9202-89b0c70bf31c'] docs = mongo_util.find_in(elements, 'id') - self.assertEqual(docs.count(), 1) + self.assertEqual(len(list(docs.clone())), 1) doc = docs.next() self.assertFalse('_id' in doc.keys()) self.assertEqual(doc.get('hid'), 68020) @@ -103,7 +103,7 @@ def test_find_in_ok(self): # test null projection elements = ['b753774f-0bbd-4b96-9202-89b0c70bf31c'] docs = mongo_util.find_in(elements, 'id', projection=None) - self.assertEqual(docs.count(), 1) + self.assertEqual(len(list(docs.clone())), 1) doc = docs.next() self.assertEqual(doc.get('_id'), 68020) self.assertEqual(doc.get('hid'), 68020) @@ -114,7 +114,7 @@ def test_update_one_ok(self): elements = ['b753774f-0bbd-4b96-9202-89b0c70bf31c'] docs = mongo_util.find_in(elements, 'id', projection=None) - self.assertEqual(docs.count(), 1) + self.assertEqual(len(list(docs.clone())), 1) doc = docs.next() self.assertEqual(doc.get('created_by'), 'tgu2') From 3bfe573b0d3375d7a421647d87336ed2caa6b549 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 May 2024 12:57:36 -0700 Subject: [PATCH 10/24] fix test_delete_many_ok --- test/MongoUtil_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/MongoUtil_test.py b/test/MongoUtil_test.py index b94453e..461a578 100644 --- a/test/MongoUtil_test.py +++ b/test/MongoUtil_test.py @@ -212,7 +212,7 @@ def test_delete_many_ok(self): self.start_test() mongo_util = self.getMongoUtil() docs = mongo_util.handle_collection.find() - self.assertEqual(docs.count(), 10) + self.assertEqual(len(list(docs.clone())), 10) docs_to_delete = list() docs_to_delete.append(docs.next()) @@ -220,15 +220,15 @@ def test_delete_many_ok(self): docs_to_delete = docs_to_delete * 2 # test delete duplicate items deleted_count = mongo_util.delete_many(docs_to_delete) self.assertEqual(deleted_count, 2) - self.assertEqual(mongo_util.handle_collection.find().count(), 8) + self.assertEqual(mongo_util.handle_collection.count_documents({}), 8) docs = mongo_util.find_in([doc.get('hid') for doc in docs_to_delete], 'hid') - self.assertEqual(docs.count(), 0) + self.assertEqual(len(list(docs)), 0) for doc in docs_to_delete: try: mongo_util.insert_one(doc) except Exception: pass - self.assertEqual(mongo_util.handle_collection.find().count(), 10) + self.assertEqual(mongo_util.handle_collection.count_documents({}), 10) docs = mongo_util.find_in([doc.get('hid') for doc in docs_to_delete], 'hid') - self.assertEqual(docs.count(), 2) + self.assertEqual(len(list(docs)), 2) From eca64711cb7bc8543002cee042d01f6157496aec Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 29 May 2024 13:19:28 -0700 Subject: [PATCH 11/24] move get_hid_counter to test_helper.py --- lib/AbstractHandle/Utils/MongoUtil.py | 12 ------------ test/AbstractHandle_server_test.py | 12 +++++++----- test/Handler_test.py | 5 +++-- test/MongoUtil_test.py | 10 ++++++---- test/test_helper.py | 9 +++++++++ 5 files changed, 25 insertions(+), 23 deletions(-) create mode 100644 test/test_helper.py diff --git a/lib/AbstractHandle/Utils/MongoUtil.py b/lib/AbstractHandle/Utils/MongoUtil.py index 9c66630..8044484 100644 --- a/lib/AbstractHandle/Utils/MongoUtil.py +++ b/lib/AbstractHandle/Utils/MongoUtil.py @@ -77,18 +77,6 @@ def __init__(self, config): logging.basicConfig(format='%(created)s %(levelname)s: %(message)s', level=logging.INFO) - def get_hid_counter(self): - """ - get current handle id counter - """ - cursor = self.hid_counter_collection.find({'_id': {'$eq': self._HID_COUNTER_ID}}) - counter = list(cursor) - - if len(counter): - return counter[0].get('hid_counter') - else: - return 0 - def find_in(self, elements, field_name, projection={'_id': False}, batch_size=1000): """ return cursor that contains docs which field column is in elements diff --git a/test/AbstractHandle_server_test.py b/test/AbstractHandle_server_test.py index 51e3763..f1ad9aa 100644 --- a/test/AbstractHandle_server_test.py +++ b/test/AbstractHandle_server_test.py @@ -17,6 +17,7 @@ import mongo_util from mongo_controller import MongoController +from test_helper import get_hid_counter class handle_serviceTest(unittest.TestCase): @@ -59,6 +60,7 @@ def setUpClass(cls): mongo_util.create_test_db(cls.mongo_controller, db=cls.cfg['mongo-database']) cls.serviceImpl = AbstractHandle(cls.cfg) cls.mongo_util = MongoUtil.MongoUtil(cls.cfg) + cls.hid_counter_collection = cls.mongo_util.hid_counter_collection cls.shock_ids_to_delete = list() @classmethod @@ -169,9 +171,9 @@ def test_persist_handle_ok(self): 'type': 'shock', 'url': 'http://ci.kbase.us:7044/'} # testing persist_handle with non-existing handle (inserting a handle) - counter = self.mongo_util.get_hid_counter() + counter = get_hid_counter(self.hid_counter_collection) hid = handler.persist_handle(self.ctx, handle)[0] - new_counter = self.mongo_util.get_hid_counter() + new_counter = get_hid_counter(self.hid_counter_collection) self.assertEqual(new_counter, counter + 1) handles = handler.fetch_handles_by(self.ctx, {'elements': [hid], 'field_name': 'hid'})[0] self.assertEqual(len(handles), 1) @@ -184,7 +186,7 @@ def test_persist_handle_ok(self): # testing persist_handle a second handle hid = handler.persist_handle(self.ctx, handle)[0] - new_counter = self.mongo_util.get_hid_counter() + new_counter = get_hid_counter(self.hid_counter_collection) self.assertEqual(hid, 'KBH_' + str(new_counter)) self.assertEqual(new_counter, counter + 2) @@ -206,7 +208,7 @@ def test_persist_multiple_handles_ok(self): 'type': 'shock', 'url': 'http://ci.kbase.us:7044/'} - counter = self.mongo_util.get_hid_counter() + counter = get_hid_counter(self.hid_counter_collection) thread_count = 257 @@ -226,7 +228,7 @@ def test_persist_multiple_handles_ok(self): result = que.get() hids.append(int(result[0].split('KBH_')[-1])) - new_counter = self.mongo_util.get_hid_counter() + new_counter = get_hid_counter(self.hid_counter_collection) self.assertEqual(counter + thread_count, new_counter) self.assertEqual(len(set(hids)), thread_count) diff --git a/test/Handler_test.py b/test/Handler_test.py index 2e7e13a..cefe3fc 100644 --- a/test/Handler_test.py +++ b/test/Handler_test.py @@ -13,6 +13,7 @@ import mongo_util from mongo_controller import MongoController +from test_helper import get_hid_counter class HandlerTest(unittest.TestCase): @@ -174,9 +175,9 @@ def test_persist_handle_okay(self): 'type': 'shock', 'url': 'http://ci.kbase.us:7044/'} # testing persist_handle with non-existing handle (inserting a handle) - counter = handler.mongo_util.get_hid_counter() + counter = get_hid_counter(handler.mongo_util.hid_counter_collection) hid = handler.persist_handle(handle, self.user_id) - new_counter = handler.mongo_util.get_hid_counter() + new_counter = get_hid_counter(handler.mongo_util.hid_counter_collection) self.assertEqual(new_counter, counter + 1) # counter should increment handles = handler.fetch_handles_by({'elements': [hid], 'field_name': 'hid'}) self.assertEqual(len(handles), 1) diff --git a/test/MongoUtil_test.py b/test/MongoUtil_test.py index 461a578..322ee78 100644 --- a/test/MongoUtil_test.py +++ b/test/MongoUtil_test.py @@ -11,6 +11,7 @@ import mongo_util from mongo_controller import MongoController +from test_helper import get_hid_counter # TODO switch all tests to pytest, see sample_service for an example @@ -34,6 +35,7 @@ def setUpClass(cls): cls.cfg["mongo-port"] = cls.mongo_controller.port mongo_util.create_test_db(cls.mongo_controller, db=cls.cfg['mongo-database']) cls.mongo_util = MongoUtil(cls.cfg) + cls.hid_counter_collection = cls.mongo_util.hid_counter_collection @classmethod def tearDownClass(cls): @@ -136,9 +138,9 @@ def test_insert_one_ok(self): self.assertEqual(mongo_util.handle_collection.count_documents({}), 10) doc = {'_id': 9999, 'hid': 9999, 'file_name': 'fake_file'} - counter = mongo_util.get_hid_counter() + counter = get_hid_counter(self.hid_counter_collection) mongo_util.insert_one(doc) - new_counter = mongo_util.get_hid_counter() + new_counter = get_hid_counter(self.hid_counter_collection) self.assertEqual(new_counter, counter) self.assertEqual(mongo_util.handle_collection.count_documents({}), 11) @@ -155,7 +157,7 @@ def test_insert_one_ok(self): def test_increase_counter_with_multi_threads(self): mongo_util = self.getMongoUtil() - counter = mongo_util.get_hid_counter() + counter = get_hid_counter(self.hid_counter_collection) thread_count = 329 @@ -173,7 +175,7 @@ def test_increase_counter_with_multi_threads(self): while not que.empty(): hids.append(que.get()) - new_counter = mongo_util.get_hid_counter() + new_counter = get_hid_counter(self.hid_counter_collection) self.assertEqual(counter + thread_count, new_counter) self.assertEqual(len(set(hids)), thread_count) diff --git a/test/test_helper.py b/test/test_helper.py new file mode 100644 index 0000000..8983c3f --- /dev/null +++ b/test/test_helper.py @@ -0,0 +1,9 @@ +def get_hid_counter(hid_counter_collection): + """ + get current handle id counter + """ + counter = hid_counter_collection.find_one({'_id': {'$eq': 'hid_counter'}}) + + if counter != None: + return counter.next().get('hid_counter') + return 0 \ No newline at end of file From c0b9970c7ac9683f2d93e18f87fef6711d76a2df Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 29 May 2024 13:27:53 -0700 Subject: [PATCH 12/24] remove log in find_in && fix lintly --- lib/AbstractHandle/Utils/MongoUtil.py | 2 -- test/test_helper.py | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/AbstractHandle/Utils/MongoUtil.py b/lib/AbstractHandle/Utils/MongoUtil.py index 8044484..96d679e 100644 --- a/lib/AbstractHandle/Utils/MongoUtil.py +++ b/lib/AbstractHandle/Utils/MongoUtil.py @@ -93,8 +93,6 @@ def find_in(self, elements, field_name, projection={'_id': False}, batch_size=10 ''.join(traceback.format_exception(None, e, e.__traceback__))) raise ValueError(error_msg) - logging.info('returned {} results'.format(len(list(result.clone())))) - return result def insert_one(self, doc): diff --git a/test/test_helper.py b/test/test_helper.py index 8983c3f..0935101 100644 --- a/test/test_helper.py +++ b/test/test_helper.py @@ -4,6 +4,6 @@ def get_hid_counter(hid_counter_collection): """ counter = hid_counter_collection.find_one({'_id': {'$eq': 'hid_counter'}}) - if counter != None: - return counter.next().get('hid_counter') - return 0 \ No newline at end of file + if counter is not None: + return counter.get('hid_counter') + return 0 From 1d35215ab76bb29d56bfdaf3fa754229d9afd0be Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 30 May 2024 00:13:31 -0700 Subject: [PATCH 13/24] add retryWrites --- .../conf/.templates/deployment.cfg.templ | 1 + lib/AbstractHandle/Utils/MongoUtil.py | 15 ++- test.cfg.example | 6 +- test/MongoUtil_test.py | 1 - test/config_test.py | 112 ++++++++++++++++++ test/mongo_util.py | 6 +- 6 files changed, 133 insertions(+), 8 deletions(-) create mode 100644 test/config_test.py diff --git a/deployment/conf/.templates/deployment.cfg.templ b/deployment/conf/.templates/deployment.cfg.templ index 15edbe3..a687abf 100644 --- a/deployment/conf/.templates/deployment.cfg.templ +++ b/deployment/conf/.templates/deployment.cfg.templ @@ -5,6 +5,7 @@ mongo-database = {{ default .Env.mongo_database "handle_db" }} mongo-authmechanism = {{ default .Env.mongo_authmechanism "DEFAULT" }} mongo-user = {{ default .Env.mongo_user "" }} mongo-password = {{ default .Env.mongo_password "" }} +mongo-retrywrites={{ default .Env.mongo_retrywrites "false" }} start-local-mongo = {{ default .Env.start_local_mongo "0" }} diff --git a/lib/AbstractHandle/Utils/MongoUtil.py b/lib/AbstractHandle/Utils/MongoUtil.py index 96d679e..bcae78f 100644 --- a/lib/AbstractHandle/Utils/MongoUtil.py +++ b/lib/AbstractHandle/Utils/MongoUtil.py @@ -13,7 +13,8 @@ class MongoUtil: _HID_COUNTER_ID = 'hid_counter' def _get_collection(self, mongo_host, mongo_port, mongo_database, mongo_collection, - mongo_user=None, mongo_password=None, mongo_authmechanism='DEFAULT'): + mongo_user=None, mongo_password=None, mongo_authmechanism='DEFAULT', + mongo_retrywrites=False): """ connect Mongo server and return a collection """ @@ -23,10 +24,11 @@ def _get_collection(self, mongo_host, mongo_port, mongo_database, mongo_collecti my_client = MongoClient(mongo_host, mongo_port, username=mongo_user, password=mongo_password, authSource=mongo_database, - authMechanism=mongo_authmechanism) + authMechanism=mongo_authmechanism, + retry_writes=mongo_retrywrites) else: logging.info('no mongo-user found in config file, connecting without auth') - my_client = MongoClient(mongo_host, mongo_port) + my_client = MongoClient(mongo_host, mongo_port, retry_writes=mongo_retrywrites) try: my_client.server_info() # force a call to server @@ -60,11 +62,13 @@ def __init__(self, config): self.mongo_user = config['mongo-user'] self.mongo_pass = config['mongo-password'] self.mongo_authmechanism = config['mongo-authmechanism'] + self.mongo_retrywrites = config['mongo-retrywrites'] self.handle_collection = self._get_collection(self.mongo_host, self.mongo_port, self.mongo_database, MONGO_COLLECTION, self.mongo_user, self.mongo_pass, - self.mongo_authmechanism) + self.mongo_authmechanism, + self.mongo_retrywrites) # create index on startup to speed up fetching self.handle_collection.create_index('hid', unique=True) @@ -72,7 +76,8 @@ def __init__(self, config): self.mongo_database, MONGO_HID_COUNTER_COLLECTION, self.mongo_user, self.mongo_pass, - self.mongo_authmechanism) + self.mongo_authmechanism, + self.mongo_retrywrites) logging.basicConfig(format='%(created)s %(levelname)s: %(message)s', level=logging.INFO) diff --git a/test.cfg.example b/test.cfg.example index d98440e..cd6ea1c 100644 --- a/test.cfg.example +++ b/test.cfg.example @@ -42,4 +42,8 @@ mongo-database=handle_db mongo-user= # If the mongo database is authenticated, the password for the given username. -mongo-password= \ No newline at end of file +mongo-password= + +# Whether to enable ('true') the MongoDB retryWrites parameter or not (anything other than 'true'). +# See https://www.mongodb.com/docs/manual/core/retryable-writes/ +mongo-retrywrites=false \ No newline at end of file diff --git a/test/MongoUtil_test.py b/test/MongoUtil_test.py index 322ee78..b848a43 100644 --- a/test/MongoUtil_test.py +++ b/test/MongoUtil_test.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import os import unittest import inspect import copy diff --git a/test/config_test.py b/test/config_test.py new file mode 100644 index 0000000..9b6f010 --- /dev/null +++ b/test/config_test.py @@ -0,0 +1,112 @@ +import os +import tempfile +import unittest + +from pathlib import Path + +import mango_util + +class ConfigTest(unittest.TestCase): + + def test_minimal_config(self): + data_string = """[HandleService2]\n + test.mongo.exe=mongod\n + test.temp.dir=temp_test_dir\n + auth-service-url=https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login\n + auth-url=https://ci.kbase.us/services/auth\n + shock-url=https://ci.kbase.us/services/shock-api\n + admin-token=998\n + test-token=998\n + admin-roles=HANDLE_ADMIN, KBASE_ADMIN\n + namespace=KBH\n + mongo-database=handle_db\n + mongo-retrywrites=false + """ + try: + cfg_path = self._create_temp_file(data_string) + default_path = os.environ[mango_util.TEST_CONFIG_ENV_PATH] + os.environ[mango_util.TEST_CONFIG_ENV_PATH] = cfg_path + mongo_cfg, deploy_cfg = mongo_util.get_config() + finally: + os.environ[mango_util.TEST_CONFIG_ENV_PATH] = default_path + + # check mongo config + self.assertEqual(mongo_cfg.mongo_exe, Path("mongod")) + self.assertEqual(mongo_cfg.mongo_temp, Path("temp_test_dir")) + self.assertEqual(mongo_cfg.use_wired_tiger, False) + self.assertEqual(mongo_cfg.delete_temp_dir, True) + + # check deploy config + self.assertEqual(deploy_cfg["auth-service-url"], 'https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login') + self.assertEqual(deploy_cfg["auth-url"], 'https://ci.kbase.us/services/auth') + self.assertEqual(deploy_cfg["shock-url"], 'https://ci.kbase.us/services/shock-api') + self.assertEqual(deploy_cfg["admin-token"], '998') + self.assertEqual(deploy_cfg["test-token"], '998') + self.assertEqual(deploy_cfg["admin-roles"], 'HANDLE_ADMIN, KBASE_ADMIN') + self.assertEqual(deploy_cfg["namespace"], 'KBH') + self.assertEqual(deploy_cfg["mongo-database"], 'handle_db') + self.assertEqual(deploy_cfg["mongo-user"], '') + self.assertEqual(deploy_cfg["mongo-password"], '') + self.assertEqual(deploy_cfg["mongo-retrywrites"], False) + + # remove temp file + os.remove(cfg_path) + + def test_maximal_config(self): + data_string = """[HandleService2]\n + test.mongo.exe=mongod\n + test.temp.dir=temp_test_dir\n + test.mongo.wired_tiger=true\n + test.delete.temp.dir=false\n + auth-service-url=https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login\n + auth-url=https://ci.kbase.us/services/auth\n + shock-url=https://ci.kbase.us/services/shock-api\n + admin-token=998\n + test-token=998\n + admin-roles=HANDLE_ADMIN, KBASE_ADMIN\n + namespace=KBH\n + mongo-database=handle_db\n + mongo-user=mongouser\n + mongo-password=mongopassword\n + mongo-retrywrites=true + """ + try: + cfg_path = self._create_temp_file(data_string) + default_path = os.environ[mango_util.TEST_CONFIG_ENV_PATH] + os.environ[mango_util.TEST_CONFIG_ENV_PATH] = cfg_path + mongo_cfg, deploy_cfg = mongo_util.get_config() + finally: + os.environ[mango_util.TEST_CONFIG_ENV_PATH] = default_path + + # check mongo config + self.assertEqual(mongo_cfg.mongo_exe, Path("mongod")) + self.assertEqual(mongo_cfg.mongo_temp, Path("temp_test_dir")) + self.assertEqual(mongo_cfg.use_wired_tiger, True) + self.assertEqual(mongo_cfg.delete_temp_dir, False) + + # check deploy config + self.assertEqual(deploy_cfg["auth-service-url"], 'https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login') + self.assertEqual(deploy_cfg["auth-url"], 'https://ci.kbase.us/services/auth') + self.assertEqual(deploy_cfg["shock-url"], 'https://ci.kbase.us/services/shock-api') + self.assertEqual(deploy_cfg["admin-token"], '998') + self.assertEqual(deploy_cfg["test-token"], '998') + self.assertEqual(deploy_cfg["admin-roles"], 'HANDLE_ADMIN, KBASE_ADMIN') + self.assertEqual(deploy_cfg["namespace"], 'KBH') + self.assertEqual(deploy_cfg["mongo-database"], 'handle_db') + self.assertEqual(deploy_cfg["mongo-user"], 'mongouser') + self.assertEqual(deploy_cfg["mongo-password"], 'mongopassword') + self.assertEqual(deploy_cfg["mongo-retrywrites"], True) + + # remove temp file + os.remove(cfg_path) + + def _create_temp_file(data_string): + try: + with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file: + temp_file.write(data_string) + file_path = temp_file.name + return file_path + + except Exception as e: + raise ValueError("An error occurred:") from e + diff --git a/test/mongo_util.py b/test/mongo_util.py index 8b8f536..bc15137 100644 --- a/test/mongo_util.py +++ b/test/mongo_util.py @@ -29,6 +29,7 @@ TEST_DATABASE_NAME = "mongo-database" TEST_DATABASE_USER = "mongo-user" TEST_DATABASE_PWD = "mongo-password" +TEST_MONGO_RETRYWRITES = "mongo-retrywrites" MongoConfigTuple = namedtuple( "MongoConfigTuple", @@ -42,7 +43,8 @@ def get_config() -> Tuple[MongoConfigTuple, dict[str, str]]: Mongo config that stores mongo executable, temporary directory, wired_tiger, and delete_temp_dir Deploy config that stores auther_serice_url, auth_url, shock_url, admin_token - test_token, admin_roles, namespace, db_name, and mongo_user, and mongo_pwd + test_token, admin_roles, namespace, db_name, mongo_user, mongo_pwd, + and mongo_retrywrites """ config_path = _get_config_file_path() section = _get_test_config(config_path) @@ -62,6 +64,7 @@ def get_config() -> Tuple[MongoConfigTuple, dict[str, str]]: db_name = _get_value(section, TEST_DATABASE_NAME, config_path, True) mongo_user = _get_value(section, TEST_DATABASE_USER, config_path, False) mongo_pwd = _get_value(section, TEST_DATABASE_PWD, config_path, False) + mongo_retrywrites = _get_value(section, TEST_MONGO_RETRYWRITES, config_path, True) mongo_config = MongoConfigTuple( Path(mongo_exe_path), @@ -81,6 +84,7 @@ def get_config() -> Tuple[MongoConfigTuple, dict[str, str]]: TEST_DATABASE_NAME: db_name, TEST_DATABASE_USER: mongo_user, TEST_DATABASE_PWD: mongo_pwd, + TEST_MONGO_RETRYWRITES: mongo_retrywrites == "true", } return mongo_config, deploy_config From 0cf06ea0fc57bb8f5de7f9f59b0ef5005b068718 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 30 May 2024 00:19:11 -0700 Subject: [PATCH 14/24] fix linely --- test/config_test.py | 68 ++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/test/config_test.py b/test/config_test.py index 9b6f010..9537755 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -4,7 +4,8 @@ from pathlib import Path -import mango_util +import mongo_util + class ConfigTest(unittest.TestCase): @@ -24,11 +25,11 @@ def test_minimal_config(self): """ try: cfg_path = self._create_temp_file(data_string) - default_path = os.environ[mango_util.TEST_CONFIG_ENV_PATH] - os.environ[mango_util.TEST_CONFIG_ENV_PATH] = cfg_path + default_path = os.environ[mongo_util.TEST_CONFIG_ENV_PATH] + os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = cfg_path mongo_cfg, deploy_cfg = mongo_util.get_config() finally: - os.environ[mango_util.TEST_CONFIG_ENV_PATH] = default_path + os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = default_path # check mongo config self.assertEqual(mongo_cfg.mongo_exe, Path("mongod")) @@ -37,16 +38,21 @@ def test_minimal_config(self): self.assertEqual(mongo_cfg.delete_temp_dir, True) # check deploy config - self.assertEqual(deploy_cfg["auth-service-url"], 'https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login') - self.assertEqual(deploy_cfg["auth-url"], 'https://ci.kbase.us/services/auth') - self.assertEqual(deploy_cfg["shock-url"], 'https://ci.kbase.us/services/shock-api') - self.assertEqual(deploy_cfg["admin-token"], '998') - self.assertEqual(deploy_cfg["test-token"], '998') - self.assertEqual(deploy_cfg["admin-roles"], 'HANDLE_ADMIN, KBASE_ADMIN') - self.assertEqual(deploy_cfg["namespace"], 'KBH') - self.assertEqual(deploy_cfg["mongo-database"], 'handle_db') - self.assertEqual(deploy_cfg["mongo-user"], '') - self.assertEqual(deploy_cfg["mongo-password"], '') + self.assertEqual( + deploy_cfg["auth-service-url"], + "https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login", + ) + self.assertEqual(deploy_cfg["auth-url"], "https://ci.kbase.us/services/auth") + self.assertEqual( + deploy_cfg["shock-url"], "https://ci.kbase.us/services/shock-api" + ) + self.assertEqual(deploy_cfg["admin-token"], "998") + self.assertEqual(deploy_cfg["test-token"], "998") + self.assertEqual(deploy_cfg["admin-roles"], "HANDLE_ADMIN, KBASE_ADMIN") + self.assertEqual(deploy_cfg["namespace"], "KBH") + self.assertEqual(deploy_cfg["mongo-database"], "handle_db") + self.assertEqual(deploy_cfg["mongo-user"], "") + self.assertEqual(deploy_cfg["mongo-password"], "") self.assertEqual(deploy_cfg["mongo-retrywrites"], False) # remove temp file @@ -72,11 +78,11 @@ def test_maximal_config(self): """ try: cfg_path = self._create_temp_file(data_string) - default_path = os.environ[mango_util.TEST_CONFIG_ENV_PATH] - os.environ[mango_util.TEST_CONFIG_ENV_PATH] = cfg_path + default_path = os.environ[mongo_util.TEST_CONFIG_ENV_PATH] + os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = cfg_path mongo_cfg, deploy_cfg = mongo_util.get_config() finally: - os.environ[mango_util.TEST_CONFIG_ENV_PATH] = default_path + os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = default_path # check mongo config self.assertEqual(mongo_cfg.mongo_exe, Path("mongod")) @@ -85,16 +91,21 @@ def test_maximal_config(self): self.assertEqual(mongo_cfg.delete_temp_dir, False) # check deploy config - self.assertEqual(deploy_cfg["auth-service-url"], 'https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login') - self.assertEqual(deploy_cfg["auth-url"], 'https://ci.kbase.us/services/auth') - self.assertEqual(deploy_cfg["shock-url"], 'https://ci.kbase.us/services/shock-api') - self.assertEqual(deploy_cfg["admin-token"], '998') - self.assertEqual(deploy_cfg["test-token"], '998') - self.assertEqual(deploy_cfg["admin-roles"], 'HANDLE_ADMIN, KBASE_ADMIN') - self.assertEqual(deploy_cfg["namespace"], 'KBH') - self.assertEqual(deploy_cfg["mongo-database"], 'handle_db') - self.assertEqual(deploy_cfg["mongo-user"], 'mongouser') - self.assertEqual(deploy_cfg["mongo-password"], 'mongopassword') + self.assertEqual( + deploy_cfg["auth-service-url"], + "https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login", + ) + self.assertEqual(deploy_cfg["auth-url"], "https://ci.kbase.us/services/auth") + self.assertEqual( + deploy_cfg["shock-url"], "https://ci.kbase.us/services/shock-api" + ) + self.assertEqual(deploy_cfg["admin-token"], "998") + self.assertEqual(deploy_cfg["test-token"], "998") + self.assertEqual(deploy_cfg["admin-roles"], "HANDLE_ADMIN, KBASE_ADMIN") + self.assertEqual(deploy_cfg["namespace"], "KBH") + self.assertEqual(deploy_cfg["mongo-database"], "handle_db") + self.assertEqual(deploy_cfg["mongo-user"], "mongouser") + self.assertEqual(deploy_cfg["mongo-password"], "mongopassword") self.assertEqual(deploy_cfg["mongo-retrywrites"], True) # remove temp file @@ -102,11 +113,10 @@ def test_maximal_config(self): def _create_temp_file(data_string): try: - with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file: + with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: temp_file.write(data_string) file_path = temp_file.name return file_path except Exception as e: raise ValueError("An error occurred:") from e - From 9eff64380d755dcaa0d26a7d044406c3f707aab0 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 30 May 2024 00:24:46 -0700 Subject: [PATCH 15/24] fix retryWrites keyerror --- lib/AbstractHandle/Utils/MongoUtil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/AbstractHandle/Utils/MongoUtil.py b/lib/AbstractHandle/Utils/MongoUtil.py index bcae78f..fc86dd6 100644 --- a/lib/AbstractHandle/Utils/MongoUtil.py +++ b/lib/AbstractHandle/Utils/MongoUtil.py @@ -25,10 +25,10 @@ def _get_collection(self, mongo_host, mongo_port, mongo_database, mongo_collecti username=mongo_user, password=mongo_password, authSource=mongo_database, authMechanism=mongo_authmechanism, - retry_writes=mongo_retrywrites) + retryWrites=mongo_retrywrites) else: logging.info('no mongo-user found in config file, connecting without auth') - my_client = MongoClient(mongo_host, mongo_port, retry_writes=mongo_retrywrites) + my_client = MongoClient(mongo_host, mongo_port, retryWrites=mongo_retrywrites) try: my_client.server_info() # force a call to server From e4e811a3572fa24ff799a5892141223159af6c22 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 30 May 2024 00:34:52 -0700 Subject: [PATCH 16/24] move create_temp_file to test_helper --- test/config_test.py | 16 +++------------- test/test_helper.py | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/test/config_test.py b/test/config_test.py index 9537755..77a4bdb 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -1,10 +1,10 @@ import os -import tempfile import unittest from pathlib import Path import mongo_util +from test_helper import create_temp_file class ConfigTest(unittest.TestCase): @@ -24,7 +24,7 @@ def test_minimal_config(self): mongo-retrywrites=false """ try: - cfg_path = self._create_temp_file(data_string) + cfg_path = create_temp_file(data_string) default_path = os.environ[mongo_util.TEST_CONFIG_ENV_PATH] os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = cfg_path mongo_cfg, deploy_cfg = mongo_util.get_config() @@ -77,7 +77,7 @@ def test_maximal_config(self): mongo-retrywrites=true """ try: - cfg_path = self._create_temp_file(data_string) + cfg_path = create_temp_file(data_string) default_path = os.environ[mongo_util.TEST_CONFIG_ENV_PATH] os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = cfg_path mongo_cfg, deploy_cfg = mongo_util.get_config() @@ -110,13 +110,3 @@ def test_maximal_config(self): # remove temp file os.remove(cfg_path) - - def _create_temp_file(data_string): - try: - with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: - temp_file.write(data_string) - file_path = temp_file.name - return file_path - - except Exception as e: - raise ValueError("An error occurred:") from e diff --git a/test/test_helper.py b/test/test_helper.py index 0935101..7360ee3 100644 --- a/test/test_helper.py +++ b/test/test_helper.py @@ -1,9 +1,23 @@ +import tempfile + + def get_hid_counter(hid_counter_collection): """ get current handle id counter """ - counter = hid_counter_collection.find_one({'_id': {'$eq': 'hid_counter'}}) + counter = hid_counter_collection.find_one({"_id": {"$eq": "hid_counter"}}) if counter is not None: - return counter.get('hid_counter') + return counter.get("hid_counter") return 0 + + +def create_temp_file(data_string): + try: + with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: + temp_file.write(data_string) + file_path = temp_file.name + return file_path + + except Exception as e: + raise ValueError("An error occurred:") from e From 7dd51c0c5e47093bf4061c5424455cfe24308339 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 30 May 2024 01:47:29 -0700 Subject: [PATCH 17/24] move DEFAULT_TEST_CONFIG_PATH into global scope --- test/config_test.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/config_test.py b/test/config_test.py index 77a4bdb..c29188d 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -7,6 +7,9 @@ from test_helper import create_temp_file +DEFAULT_TEST_CONFIG_PATH = os.environ[mongo_util.TEST_CONFIG_ENV_PATH] + + class ConfigTest(unittest.TestCase): def test_minimal_config(self): @@ -25,11 +28,10 @@ def test_minimal_config(self): """ try: cfg_path = create_temp_file(data_string) - default_path = os.environ[mongo_util.TEST_CONFIG_ENV_PATH] os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = cfg_path mongo_cfg, deploy_cfg = mongo_util.get_config() finally: - os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = default_path + os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = DEFAULT_TEST_CONFIG_PATH # check mongo config self.assertEqual(mongo_cfg.mongo_exe, Path("mongod")) @@ -78,11 +80,10 @@ def test_maximal_config(self): """ try: cfg_path = create_temp_file(data_string) - default_path = os.environ[mongo_util.TEST_CONFIG_ENV_PATH] os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = cfg_path mongo_cfg, deploy_cfg = mongo_util.get_config() finally: - os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = default_path + os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = DEFAULT_TEST_CONFIG_PATH # check mongo config self.assertEqual(mongo_cfg.mongo_exe, Path("mongod")) From c68cf7e64f764f9401f7ad7940d9a1c6363eb5de Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 30 May 2024 13:08:39 -0700 Subject: [PATCH 18/24] move retryWrites processing outside of test --- lib/AbstractHandle/Utils/MongoUtil.py | 2 +- test/config_test.py | 15 +++++++-------- test/mongo_util.py | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/lib/AbstractHandle/Utils/MongoUtil.py b/lib/AbstractHandle/Utils/MongoUtil.py index fc86dd6..6246a56 100644 --- a/lib/AbstractHandle/Utils/MongoUtil.py +++ b/lib/AbstractHandle/Utils/MongoUtil.py @@ -62,7 +62,7 @@ def __init__(self, config): self.mongo_user = config['mongo-user'] self.mongo_pass = config['mongo-password'] self.mongo_authmechanism = config['mongo-authmechanism'] - self.mongo_retrywrites = config['mongo-retrywrites'] + self.mongo_retrywrites = config.get('mongo-retrywrites') == "true" self.handle_collection = self._get_collection(self.mongo_host, self.mongo_port, self.mongo_database, MONGO_COLLECTION, diff --git a/test/config_test.py b/test/config_test.py index c29188d..d5d08d8 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -20,11 +20,10 @@ def test_minimal_config(self): auth-url=https://ci.kbase.us/services/auth\n shock-url=https://ci.kbase.us/services/shock-api\n admin-token=998\n - test-token=998\n + test-token=999\n admin-roles=HANDLE_ADMIN, KBASE_ADMIN\n namespace=KBH\n - mongo-database=handle_db\n - mongo-retrywrites=false + mongo-database=handle_db """ try: cfg_path = create_temp_file(data_string) @@ -49,13 +48,13 @@ def test_minimal_config(self): deploy_cfg["shock-url"], "https://ci.kbase.us/services/shock-api" ) self.assertEqual(deploy_cfg["admin-token"], "998") - self.assertEqual(deploy_cfg["test-token"], "998") + self.assertEqual(deploy_cfg["test-token"], "999") self.assertEqual(deploy_cfg["admin-roles"], "HANDLE_ADMIN, KBASE_ADMIN") self.assertEqual(deploy_cfg["namespace"], "KBH") self.assertEqual(deploy_cfg["mongo-database"], "handle_db") self.assertEqual(deploy_cfg["mongo-user"], "") self.assertEqual(deploy_cfg["mongo-password"], "") - self.assertEqual(deploy_cfg["mongo-retrywrites"], False) + self.assertEqual(deploy_cfg["mongo-retrywrites"], "") # remove temp file os.remove(cfg_path) @@ -70,7 +69,7 @@ def test_maximal_config(self): auth-url=https://ci.kbase.us/services/auth\n shock-url=https://ci.kbase.us/services/shock-api\n admin-token=998\n - test-token=998\n + test-token=999\n admin-roles=HANDLE_ADMIN, KBASE_ADMIN\n namespace=KBH\n mongo-database=handle_db\n @@ -101,13 +100,13 @@ def test_maximal_config(self): deploy_cfg["shock-url"], "https://ci.kbase.us/services/shock-api" ) self.assertEqual(deploy_cfg["admin-token"], "998") - self.assertEqual(deploy_cfg["test-token"], "998") + self.assertEqual(deploy_cfg["test-token"], "999") self.assertEqual(deploy_cfg["admin-roles"], "HANDLE_ADMIN, KBASE_ADMIN") self.assertEqual(deploy_cfg["namespace"], "KBH") self.assertEqual(deploy_cfg["mongo-database"], "handle_db") self.assertEqual(deploy_cfg["mongo-user"], "mongouser") self.assertEqual(deploy_cfg["mongo-password"], "mongopassword") - self.assertEqual(deploy_cfg["mongo-retrywrites"], True) + self.assertEqual(deploy_cfg["mongo-retrywrites"], "true") # remove temp file os.remove(cfg_path) diff --git a/test/mongo_util.py b/test/mongo_util.py index bc15137..a08ab59 100644 --- a/test/mongo_util.py +++ b/test/mongo_util.py @@ -64,7 +64,7 @@ def get_config() -> Tuple[MongoConfigTuple, dict[str, str]]: db_name = _get_value(section, TEST_DATABASE_NAME, config_path, True) mongo_user = _get_value(section, TEST_DATABASE_USER, config_path, False) mongo_pwd = _get_value(section, TEST_DATABASE_PWD, config_path, False) - mongo_retrywrites = _get_value(section, TEST_MONGO_RETRYWRITES, config_path, True) + mongo_retrywrites = _get_value(section, TEST_MONGO_RETRYWRITES, config_path, False) mongo_config = MongoConfigTuple( Path(mongo_exe_path), @@ -84,7 +84,7 @@ def get_config() -> Tuple[MongoConfigTuple, dict[str, str]]: TEST_DATABASE_NAME: db_name, TEST_DATABASE_USER: mongo_user, TEST_DATABASE_PWD: mongo_pwd, - TEST_MONGO_RETRYWRITES: mongo_retrywrites == "true", + TEST_MONGO_RETRYWRITES: mongo_retrywrites, } return mongo_config, deploy_config From 1fb6481827e3ff12e03bbb7b855377e0c41b5c09 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 30 May 2024 15:03:19 -0700 Subject: [PATCH 19/24] Make a config dict, pass it into MongoUtil, and assert on the instance variables --- test/config_test.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/config_test.py b/test/config_test.py index d5d08d8..f7abdd2 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -5,6 +5,7 @@ import mongo_util from test_helper import create_temp_file +from AbstractHandle.Utils.MongoUtil import MongoUtil DEFAULT_TEST_CONFIG_PATH = os.environ[mongo_util.TEST_CONFIG_ENV_PATH] @@ -56,6 +57,22 @@ def test_minimal_config(self): self.assertEqual(deploy_cfg["mongo-password"], "") self.assertEqual(deploy_cfg["mongo-retrywrites"], "") + # make a config and pass it into MongoUtil + cfg = deploy_cfg + cfg["mongo-host"] = "localhost" + cfg["mongo-port"] = 8888 + cfg["mongo-authmechanism"] = "DEFAULT" + mu = MongoUtil(cfg) + + # assert on the instance variables + self.assertEqual(mu.mongo_host, "localhost") + self.assertEqual(mu.mongo_port, 8888) + self.assertEqual(mu.mongo_database, "handle_db") + self.assertEqual(mu.mongo_user, "") + self.assertEqual(mu.mongo_pass, "") + self.assertEqual(mu.mongo_authmechanism, "DEFAULT") + self.assertEqual(mu.mongo_retrywrites, False) + # remove temp file os.remove(cfg_path) @@ -108,5 +125,21 @@ def test_maximal_config(self): self.assertEqual(deploy_cfg["mongo-password"], "mongopassword") self.assertEqual(deploy_cfg["mongo-retrywrites"], "true") + # make a config and pass it into MongoUtil + cfg = deploy_cfg + cfg["mongo-host"] = "localhost" + cfg["mongo-port"] = 8888 + cfg["mongo-authmechanism"] = "DEFAULT" + mu = MongoUtil(cfg) + + # assert on the instance variables + self.assertEqual(mu.mongo_host, "localhost") + self.assertEqual(mu.mongo_port, 8888) + self.assertEqual(mu.mongo_database, "handle_db") + self.assertEqual(mu.mongo_user, "mongouser") + self.assertEqual(mu.mongo_pass, "mongopassword") + self.assertEqual(mu.mongo_authmechanism, "DEFAULT") + self.assertEqual(mu.mongo_retrywrites, True) + # remove temp file os.remove(cfg_path) From 179d7b6bb8e2b498d661e70f580632a2ead42ed2 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 30 May 2024 16:47:44 -0700 Subject: [PATCH 20/24] remove tests from config_test --- test/config_test.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/test/config_test.py b/test/config_test.py index f7abdd2..d5d08d8 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -5,7 +5,6 @@ import mongo_util from test_helper import create_temp_file -from AbstractHandle.Utils.MongoUtil import MongoUtil DEFAULT_TEST_CONFIG_PATH = os.environ[mongo_util.TEST_CONFIG_ENV_PATH] @@ -57,22 +56,6 @@ def test_minimal_config(self): self.assertEqual(deploy_cfg["mongo-password"], "") self.assertEqual(deploy_cfg["mongo-retrywrites"], "") - # make a config and pass it into MongoUtil - cfg = deploy_cfg - cfg["mongo-host"] = "localhost" - cfg["mongo-port"] = 8888 - cfg["mongo-authmechanism"] = "DEFAULT" - mu = MongoUtil(cfg) - - # assert on the instance variables - self.assertEqual(mu.mongo_host, "localhost") - self.assertEqual(mu.mongo_port, 8888) - self.assertEqual(mu.mongo_database, "handle_db") - self.assertEqual(mu.mongo_user, "") - self.assertEqual(mu.mongo_pass, "") - self.assertEqual(mu.mongo_authmechanism, "DEFAULT") - self.assertEqual(mu.mongo_retrywrites, False) - # remove temp file os.remove(cfg_path) @@ -125,21 +108,5 @@ def test_maximal_config(self): self.assertEqual(deploy_cfg["mongo-password"], "mongopassword") self.assertEqual(deploy_cfg["mongo-retrywrites"], "true") - # make a config and pass it into MongoUtil - cfg = deploy_cfg - cfg["mongo-host"] = "localhost" - cfg["mongo-port"] = 8888 - cfg["mongo-authmechanism"] = "DEFAULT" - mu = MongoUtil(cfg) - - # assert on the instance variables - self.assertEqual(mu.mongo_host, "localhost") - self.assertEqual(mu.mongo_port, 8888) - self.assertEqual(mu.mongo_database, "handle_db") - self.assertEqual(mu.mongo_user, "mongouser") - self.assertEqual(mu.mongo_pass, "mongopassword") - self.assertEqual(mu.mongo_authmechanism, "DEFAULT") - self.assertEqual(mu.mongo_retrywrites, True) - # remove temp file os.remove(cfg_path) From d14293cca8270d1fa3ca536256ac2bbac88276c8 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 30 May 2024 21:12:14 -0700 Subject: [PATCH 21/24] add check for retryWrites --- test/MongoUtil_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/MongoUtil_test.py b/test/MongoUtil_test.py index b848a43..cfcf282 100644 --- a/test/MongoUtil_test.py +++ b/test/MongoUtil_test.py @@ -65,6 +65,7 @@ def test_init_ok(self): ] mongo_util = self.getMongoUtil() self.assertTrue(set(class_attri) <= set(mongo_util.__dict__.keys())) + self.assertEqual(mongo_util.mongo_retrywrites, False) handle_collection = mongo_util.handle_collection self.assertEqual(handle_collection.name, 'handle') From ddf6f55edde4f1366e725bac6e8b19b4d7a1d1cf Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 31 May 2024 10:26:50 -0700 Subject: [PATCH 22/24] remove test/config_test.py --- test/config_test.py | 112 -------------------------------------------- test/test_helper.py | 14 ------ 2 files changed, 126 deletions(-) delete mode 100644 test/config_test.py diff --git a/test/config_test.py b/test/config_test.py deleted file mode 100644 index d5d08d8..0000000 --- a/test/config_test.py +++ /dev/null @@ -1,112 +0,0 @@ -import os -import unittest - -from pathlib import Path - -import mongo_util -from test_helper import create_temp_file - - -DEFAULT_TEST_CONFIG_PATH = os.environ[mongo_util.TEST_CONFIG_ENV_PATH] - - -class ConfigTest(unittest.TestCase): - - def test_minimal_config(self): - data_string = """[HandleService2]\n - test.mongo.exe=mongod\n - test.temp.dir=temp_test_dir\n - auth-service-url=https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login\n - auth-url=https://ci.kbase.us/services/auth\n - shock-url=https://ci.kbase.us/services/shock-api\n - admin-token=998\n - test-token=999\n - admin-roles=HANDLE_ADMIN, KBASE_ADMIN\n - namespace=KBH\n - mongo-database=handle_db - """ - try: - cfg_path = create_temp_file(data_string) - os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = cfg_path - mongo_cfg, deploy_cfg = mongo_util.get_config() - finally: - os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = DEFAULT_TEST_CONFIG_PATH - - # check mongo config - self.assertEqual(mongo_cfg.mongo_exe, Path("mongod")) - self.assertEqual(mongo_cfg.mongo_temp, Path("temp_test_dir")) - self.assertEqual(mongo_cfg.use_wired_tiger, False) - self.assertEqual(mongo_cfg.delete_temp_dir, True) - - # check deploy config - self.assertEqual( - deploy_cfg["auth-service-url"], - "https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login", - ) - self.assertEqual(deploy_cfg["auth-url"], "https://ci.kbase.us/services/auth") - self.assertEqual( - deploy_cfg["shock-url"], "https://ci.kbase.us/services/shock-api" - ) - self.assertEqual(deploy_cfg["admin-token"], "998") - self.assertEqual(deploy_cfg["test-token"], "999") - self.assertEqual(deploy_cfg["admin-roles"], "HANDLE_ADMIN, KBASE_ADMIN") - self.assertEqual(deploy_cfg["namespace"], "KBH") - self.assertEqual(deploy_cfg["mongo-database"], "handle_db") - self.assertEqual(deploy_cfg["mongo-user"], "") - self.assertEqual(deploy_cfg["mongo-password"], "") - self.assertEqual(deploy_cfg["mongo-retrywrites"], "") - - # remove temp file - os.remove(cfg_path) - - def test_maximal_config(self): - data_string = """[HandleService2]\n - test.mongo.exe=mongod\n - test.temp.dir=temp_test_dir\n - test.mongo.wired_tiger=true\n - test.delete.temp.dir=false\n - auth-service-url=https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login\n - auth-url=https://ci.kbase.us/services/auth\n - shock-url=https://ci.kbase.us/services/shock-api\n - admin-token=998\n - test-token=999\n - admin-roles=HANDLE_ADMIN, KBASE_ADMIN\n - namespace=KBH\n - mongo-database=handle_db\n - mongo-user=mongouser\n - mongo-password=mongopassword\n - mongo-retrywrites=true - """ - try: - cfg_path = create_temp_file(data_string) - os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = cfg_path - mongo_cfg, deploy_cfg = mongo_util.get_config() - finally: - os.environ[mongo_util.TEST_CONFIG_ENV_PATH] = DEFAULT_TEST_CONFIG_PATH - - # check mongo config - self.assertEqual(mongo_cfg.mongo_exe, Path("mongod")) - self.assertEqual(mongo_cfg.mongo_temp, Path("temp_test_dir")) - self.assertEqual(mongo_cfg.use_wired_tiger, True) - self.assertEqual(mongo_cfg.delete_temp_dir, False) - - # check deploy config - self.assertEqual( - deploy_cfg["auth-service-url"], - "https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login", - ) - self.assertEqual(deploy_cfg["auth-url"], "https://ci.kbase.us/services/auth") - self.assertEqual( - deploy_cfg["shock-url"], "https://ci.kbase.us/services/shock-api" - ) - self.assertEqual(deploy_cfg["admin-token"], "998") - self.assertEqual(deploy_cfg["test-token"], "999") - self.assertEqual(deploy_cfg["admin-roles"], "HANDLE_ADMIN, KBASE_ADMIN") - self.assertEqual(deploy_cfg["namespace"], "KBH") - self.assertEqual(deploy_cfg["mongo-database"], "handle_db") - self.assertEqual(deploy_cfg["mongo-user"], "mongouser") - self.assertEqual(deploy_cfg["mongo-password"], "mongopassword") - self.assertEqual(deploy_cfg["mongo-retrywrites"], "true") - - # remove temp file - os.remove(cfg_path) diff --git a/test/test_helper.py b/test/test_helper.py index 7360ee3..0a35362 100644 --- a/test/test_helper.py +++ b/test/test_helper.py @@ -1,6 +1,3 @@ -import tempfile - - def get_hid_counter(hid_counter_collection): """ get current handle id counter @@ -10,14 +7,3 @@ def get_hid_counter(hid_counter_collection): if counter is not None: return counter.get("hid_counter") return 0 - - -def create_temp_file(data_string): - try: - with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: - temp_file.write(data_string) - file_path = temp_file.name - return file_path - - except Exception as e: - raise ValueError("An error occurred:") from e From 4186345843e80aa5773eb4047e123ea8270150fb Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 31 May 2024 11:15:58 -0700 Subject: [PATCH 23/24] add test_retryWrites == true --- test/MongoUtil_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/MongoUtil_test.py b/test/MongoUtil_test.py index cfcf282..d6ad603 100644 --- a/test/MongoUtil_test.py +++ b/test/MongoUtil_test.py @@ -49,6 +49,12 @@ def start_test(self): testname = inspect.stack()[1][3] print('\n*** starting test: ' + testname + ' **') + def test_retryWrites(self): + new_cfg = dict(self.cfg) + new_cfg['mongo-retrywrites'] = "true" + mu = MongoUtil(new_cfg) + self.assertCountEqual(mu.mongo_retrywrites, True) + def test_get_collection(self): self.start_test() mongo_util = self.getMongoUtil() From e2ef52ea0872139b1200a7ba72e0c9a3043644ef Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 31 May 2024 11:18:35 -0700 Subject: [PATCH 24/24] fix assert bug --- test/MongoUtil_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/MongoUtil_test.py b/test/MongoUtil_test.py index d6ad603..0b96b05 100644 --- a/test/MongoUtil_test.py +++ b/test/MongoUtil_test.py @@ -53,7 +53,7 @@ def test_retryWrites(self): new_cfg = dict(self.cfg) new_cfg['mongo-retrywrites'] = "true" mu = MongoUtil(new_cfg) - self.assertCountEqual(mu.mongo_retrywrites, True) + self.assertEqual(mu.mongo_retrywrites, True) def test_get_collection(self): self.start_test()