From 27c96a42eaea84c8c890a4f6ddc700f8c1d765ed Mon Sep 17 00:00:00 2001 From: Andreas Heger Date: Tue, 29 Mar 2022 18:49:15 +0100 Subject: [PATCH 1/2] upload in dseparate sessions --- all-tests.sh | 6 --- poetry.lock | 72 ++++++++++++++++------------------ pyproject.toml | 3 +- src/daisy/storage.py | 60 ++++++++++++++--------------- src/daisy/table_cache.py | 81 +++++++++++++++++++++++++++++++-------- src/tests/test_storage.py | 7 ++-- 6 files changed, 133 insertions(+), 96 deletions(-) delete mode 100755 all-tests.sh diff --git a/all-tests.sh b/all-tests.sh deleted file mode 100755 index db0dee2..0000000 --- a/all-tests.sh +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/bash - -set -xue - -pycodestyle -python tests diff --git a/poetry.lock b/poetry.lock index 039df44..be1a469 100644 --- a/poetry.lock +++ b/poetry.lock @@ -67,7 +67,7 @@ python-versions = "*" name = "colorama" version = "0.4.4" description = "Cross-platform colored terminal text." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -151,14 +151,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "numpy" -version = "1.21.1" -description = "NumPy is the fundamental package for array computing with Python." -category = "main" -optional = false -python-versions = ">=3.7" - [[package]] name = "numpy" version = "1.21.4" @@ -421,6 +413,22 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "tqdm" +version = "4.63.0" +description = "Fast, Extensible Progress Meter" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +telegram = ["requests"] + [[package]] name = "zope.event" version = "4.5.0" @@ -449,7 +457,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "dfcb9952a246be28c614d1b4698f484dbec5ab1b1d0cb415b7fba2b7d39909b8" +content-hash = "625fb09805560d48dc4fdcaaf506fa184c3cd0795d506d1fb1e28f3e153bbf61" [metadata.files] apsw = [ @@ -468,10 +476,13 @@ attrs = [ {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, ] bcrypt = [ + {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:b589229207630484aefe5899122fb938a5b017b0f4349f769b8c13e78d99a8fd"}, {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, {file = "bcrypt-3.2.0-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7"}, {file = "bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1"}, {file = "bcrypt-3.2.0-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a0584a92329210fcd75eb8a3250c5a941633f8bfaf2a18f81009b097732839b7"}, + {file = "bcrypt-3.2.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:56e5da069a76470679f312a7d3d23deb3ac4519991a0361abc11da837087b61d"}, {file = "bcrypt-3.2.0-cp36-abi3-win32.whl", hash = "sha256:a67fb841b35c28a59cebed05fbd3e80eea26e6d75851f0574a9273c80f3e9b55"}, {file = "bcrypt-3.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:81fec756feff5b6818ea7ab031205e1d323d8943d237303baca2c5f9c7846f34"}, {file = "bcrypt-3.2.0.tar.gz", hash = "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29"}, @@ -601,6 +612,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97e5306482182170ade15c4b0d8386ded995a07d7cc2ca8f27958d34d6736497"}, {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6a36bb9474218c7a5b27ae476035497a6990e21d04c279884eb10d9b290f1b1"}, {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abb7a75ed8b968f3061327c433a0fbd17b729947b400747c334a9c29a9af6c58"}, + {file = "greenlet-1.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b336501a05e13b616ef81ce329c0e09ac5ed8c732d9ba7e3e983fcc1a9e86965"}, {file = "greenlet-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:14d4f3cd4e8b524ae9b8aa567858beed70c392fdec26dbdb0a8a418392e71708"}, {file = "greenlet-1.1.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:17ff94e7a83aa8671a25bf5b59326ec26da379ace2ebc4411d690d80a7fbcf23"}, {file = "greenlet-1.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9f3cba480d3deb69f6ee2c1825060177a22c7826431458c697df88e6aeb3caee"}, @@ -613,6 +625,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9d29ca8a77117315101425ec7ec2a47a22ccf59f5593378fc4077ac5b754fce"}, {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21915eb821a6b3d9d8eefdaf57d6c345b970ad722f856cd71739493ce003ad08"}, {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eff9d20417ff9dcb0d25e2defc2574d10b491bf2e693b4e491914738b7908168"}, + {file = "greenlet-1.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b8c008de9d0daba7b6666aa5bbfdc23dcd78cafc33997c9b7741ff6353bafb7f"}, {file = "greenlet-1.1.2-cp36-cp36m-win32.whl", hash = "sha256:32ca72bbc673adbcfecb935bb3fb1b74e663d10a4b241aaa2f5a75fe1d1f90aa"}, {file = "greenlet-1.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f0214eb2a23b85528310dad848ad2ac58e735612929c8072f6093f3585fd342d"}, {file = "greenlet-1.1.2-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:b92e29e58bef6d9cfd340c72b04d74c4b4e9f70c9fa7c78b674d1fec18896dc4"}, @@ -621,6 +634,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e12bdc622676ce47ae9abbf455c189e442afdde8818d9da983085df6312e7a1"}, {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c790abda465726cfb8bb08bd4ca9a5d0a7bd77c7ac1ca1b839ad823b948ea28"}, {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f276df9830dba7a333544bd41070e8175762a7ac20350786b322b714b0e654f5"}, + {file = "greenlet-1.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c5d5b35f789a030ebb95bff352f1d27a93d81069f2adb3182d99882e095cefe"}, {file = "greenlet-1.1.2-cp37-cp37m-win32.whl", hash = "sha256:64e6175c2e53195278d7388c454e0b30997573f3f4bd63697f88d855f7a6a1fc"}, {file = "greenlet-1.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b11548073a2213d950c3f671aa88e6f83cda6e2fb97a8b6317b1b5b33d850e06"}, {file = "greenlet-1.1.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:9633b3034d3d901f0a46b7939f8c4d64427dfba6bbc5a36b1a67364cf148a1b0"}, @@ -629,6 +643,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e859fcb4cbe93504ea18008d1df98dee4f7766db66c435e4882ab35cf70cac43"}, {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00e44c8afdbe5467e4f7b5851be223be68adb4272f44696ee71fe46b7036a711"}, {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec8c433b3ab0419100bd45b47c9c8551248a5aee30ca5e9d399a0b57ac04651b"}, + {file = "greenlet-1.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2bde6792f313f4e918caabc46532aa64aa27a0db05d75b20edfc5c6f46479de2"}, {file = "greenlet-1.1.2-cp38-cp38-win32.whl", hash = "sha256:288c6a76705dc54fba69fbcb59904ae4ad768b4c768839b8ca5fdadec6dd8cfd"}, {file = "greenlet-1.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:8d2f1fb53a421b410751887eb4ff21386d119ef9cde3797bf5e7ed49fb51a3b3"}, {file = "greenlet-1.1.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:166eac03e48784a6a6e0e5f041cfebb1ab400b394db188c48b3a84737f505b67"}, @@ -637,6 +652,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1692f7d6bc45e3200844be0dba153612103db241691088626a33ff1f24a0d88"}, {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7227b47e73dedaa513cdebb98469705ef0d66eb5a1250144468e9c3097d6b59b"}, {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ff61ff178250f9bb3cd89752df0f1dd0e27316a8bd1465351652b1b4a4cdfd3"}, + {file = "greenlet-1.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0051c6f1f27cb756ffc0ffbac7d2cd48cb0362ac1736871399a739b2885134d3"}, {file = "greenlet-1.1.2-cp39-cp39-win32.whl", hash = "sha256:f70a9e237bb792c7cc7e44c531fd48f5897961701cdaa06cf22fc14965c496cf"}, {file = "greenlet-1.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:013d61294b6cd8fe3242932c1c5e36e5d1db2c8afb58606c5a67efce62c1f5fd"}, {file = "greenlet-1.1.2.tar.gz", hash = "sha256:e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a"}, @@ -650,34 +666,6 @@ mccabe = [ {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] numpy = [ - {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, - {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, - {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, - {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, - {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, - {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, - {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, - {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, - {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, {file = "numpy-1.21.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8890b3360f345e8360133bc078d2dacc2843b6ee6059b568781b15b97acbe39f"}, {file = "numpy-1.21.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:69077388c5a4b997442b843dbdc3a85b420fb693ec8e33020bb24d647c164fa5"}, {file = "numpy-1.21.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e89717274b41ebd568cd7943fc9418eeb49b1785b66031bc8a7f6300463c5898"}, @@ -854,6 +842,10 @@ pyyaml = [ {file = "ruamel.yaml-0.17.17.tar.gz", hash = "sha256:9751de4cbb57d4bfbf8fc394e125ed4a2f170fbff3dc3d78abf50be85924f8be"}, ] "ruamel.yaml.clib" = [ + {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6e7be2c5bcb297f5b82fee9c665eb2eb7001d1050deaba8471842979293a80b0"}, + {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:221eca6f35076c6ae472a531afa1c223b9c29377e62936f61bc8e6e8bdc5f9e7"}, + {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-win32.whl", hash = "sha256:1070ba9dd7f9370d0513d649420c3b362ac2d687fe78c6e888f5b12bf8bc7bee"}, + {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:77df077d32921ad46f34816a9a16e6356d8100374579bc35e15bab5d4e9377de"}, {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751"}, {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7b2927e92feb51d830f531de4ccb11b320255ee95e791022555971c466af4527"}, {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win32.whl", hash = "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5"}, @@ -925,6 +917,10 @@ toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] +tqdm = [ + {file = "tqdm-4.63.0-py2.py3-none-any.whl", hash = "sha256:e643e071046f17139dea55b880dc9b33822ce21613b4a4f5ea57f202833dbc29"}, + {file = "tqdm-4.63.0.tar.gz", hash = "sha256:1d9835ede8e394bb8c9dcbffbca02d717217113adc679236873eeaac5bc0b3cd"}, +] "zope.event" = [ {file = "zope.event-4.5.0-py2.py3-none-any.whl", hash = "sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42"}, {file = "zope.event-4.5.0.tar.gz", hash = "sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330"}, diff --git a/pyproject.toml b/pyproject.toml index c4d3703..abff856 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cgat-daisy" -version = "0.1.10" +version = "0.1.11" description = "A benchmarking framework" authors = ["Andreas Heger "] packages = [ @@ -19,6 +19,7 @@ paramiko = "^2.8.0" PyYAML = "^6.0" gevent = "^21.8.0" apsw = "^3.36.0" +tqdm = "^4.63.0" [tool.poetry.dev-dependencies] pytest = "^6.2.5" diff --git a/src/daisy/storage.py b/src/daisy/storage.py index 0839d56..5d7deea 100644 --- a/src/daisy/storage.py +++ b/src/daisy/storage.py @@ -42,7 +42,7 @@ import cgatcore.experiment as E from daisy.toolkit import touch, read_data, hash -from daisy.table_cache import TableCache +from daisy.table_cache import TableCache, create_engine ###################################################### @@ -497,11 +497,12 @@ def generate_metric(infile): class Resource(object): - def __init__(self, args): - self.args = args + def __init__(self, database_url: str, schema: str): + self.database_url = database_url + self.schema = schema def __enter__(self): - table_cache = TableCache(*self.args) + table_cache = TableCache(self.database_url, self.schema) E.debug(f"{os.getpid()}: created resource={id(self)}: cache={id(table_cache)}") self.table_cache = table_cache return self @@ -515,8 +516,8 @@ def __exit__(self, *args, **kwargs): f"for cache={id(self.table_cache)} completed") -def open_resource(args): - return Resource(args) +def open_resource(url, schema): + return Resource(url, schema) def upload_metric(args): @@ -545,13 +546,13 @@ def setup_worker(f, *args): global resource - engine, schema, is_sqlite = args - engine.dispose() + url, schema = args + f.schema = schema - Session = sessionmaker(bind=engine) + Session = sessionmaker() f.session = Session() - resource_cm = open_resource(args) + resource_cm = open_resource(url, schema) E.debug(f"{os.getpid()}: setting up worker for resource={id(resource)}") old_resource = resource resource = resource_cm.__enter__() @@ -565,12 +566,18 @@ def setup_worker(f, *args): f.table_cache = resource.table_cache -def upload_metrics_tables(infiles: list, run_id: int, schema, session, engine, +def upload_metrics_tables(infiles: list, + run_id: int, + schema, + url: str, max_workers: int = 10): - gis_sqlite3 = True - logger = P.get_logger() + + engine = create_engine(url) + + Session = sessionmaker(bind=engine) + session = Session() logger.info(f"{os.getpid()}: collecting upload items for {len(infiles)} input files") metric_f = generate_metric @@ -585,7 +592,7 @@ def upload_metrics_tables(infiles: list, run_id: int, schema, session, engine, logger.info(f"{os.getpid()}: uploading {len(data)} items") upload_f = upload_metric - initargs = (upload_f, engine, schema, is_sqlite3) + initargs = (upload_f, url, schema) if max_workers == 1: setup_worker(*initargs) result = list(map(upload_f, data)) @@ -638,22 +645,13 @@ def upload_result(infiles, outfile, *extras): config = extras[0] url = config["database"]["url"] - is_sqlite3 = url.startswith("sqlite") - max_workers = config["database"].get("cores", 1) - if is_sqlite3: - connect_args = {'check_same_thread': False} - else: - connect_args = {} - schema = config["database"].get("schema", None) # TODO: check if schema exists to avoid incomplete # transaction. - engine = sqlalchemy.create_engine( - url, - connect_args=connect_args) + engine = create_engine(url) # Catch exceptions until database access on thame available try: @@ -668,7 +666,7 @@ def upload_result(infiles, outfile, *extras): # Create schema if not exists if schema is not None: engine.execute( - text("CREATE SCHEMA IF NOT EXISTS {}".format(schema))) + str(text("CREATE SCHEMA IF NOT EXISTS {}".format(schema)))) pipeline_name = os.path.basename(sys.argv[0]) logger.debug("uploading data to {}, schema={}".format(url, schema)) @@ -710,9 +708,13 @@ def upload_result(infiles, outfile, *extras): session.add(benchmark_tag) session.commit() + engine.dispose() + del engine upload_metrics_tables(infiles, - benchmark_run.id, schema, session, engine, + benchmark_run.id, + schema, + url, max_workers=max_workers) # upload table sizes @@ -727,11 +729,9 @@ def upload_result(infiles, outfile, *extras): # schema=None, # is_sqlite3=is_sqlite3) - benchmark_run.status = "complete" - session.commit() + # benchmark_run.status = "complete" + # session.commit() - engine.dispose() - del engine logger.info("uploaded results under run_id {}".format(benchmark_run.id)) touch(outfile) diff --git a/src/daisy/table_cache.py b/src/daisy/table_cache.py index 3469011..7e52834 100644 --- a/src/daisy/table_cache.py +++ b/src/daisy/table_cache.py @@ -24,6 +24,18 @@ def get_columns(tablename, engine): return [x.name for x in tb.columns] +def create_engine(url: str): + is_sqlite3 = url.startswith("sqlite") + if is_sqlite3: + connect_args = {'check_same_thread': False} + else: + connect_args = {} + engine = sqlalchemy.create_engine( + url, + connect_args=connect_args) + return engine + + def sql_sanitize_columns(columns): # special chars @@ -42,19 +54,41 @@ class TableExistsException(Exception): pass +class TableSchemaChangedException(Exception): + pass + + +class IndexExistsException(Exception): + pass + + def retry_table_to_sql(table, *args, **kwargs): """retry SQL statements retrying when database is locked""" while True: try: table.to_sql(*args, **kwargs) except (sqlalchemy.exc.OperationalError, sqlite3.OperationalError) as ex: - if "database is locked" in str(ex): + msg = str(ex) + if "database is locked" in msg: E.debug("database is locked") time.sleep(1) continue - elif "already exists" in str(ex): + elif "database schema has changed" in msg: + E.debug("schema has changed") + time.sleep(1) + continue + elif "already exists" in msg: raise TableExistsException(str(ex)) raise + except pandas.io.sql.DatabaseError as ex: + msg = str(ex) + if "database schema has changed" in msg: + E.debug("schema has changed") + time.sleep(1) + continue + # raise TableSchemeChangedException( + # f"database schema changed: args={args}, kwargs={kwargs}") + raise except ValueError as ex: # pandas throws ValueError if "already exists" in str(ex): @@ -70,10 +104,17 @@ def retry_sql_execute(engine, statement): try: engine.execute(statement) except (sqlalchemy.exc.OperationalError, sqlite3.OperationalError) as ex: - if "database is locked" in str(ex): + msg = str(ex) + if "database is locked" in msg: E.debug("database is locked") time.sleep(1) continue + elif "database schema has changed" in msg: + E.debug("schema has changed") + time.sleep(1) + continue + elif "index" in msg and "already exists" in msg: + raise IndexExistsException(msg) else: raise except Exception as ex: @@ -136,18 +177,19 @@ def reconcile_columns(tablename, engine, table): def save_table(table: pandas.DataFrame, - engine, - tablename, + url: str, + tablename: str, schema: str = None, - is_sqlite3: bool = False, dtypes=None, indices=["instance_id"]): logger = P.get_logger() table.columns = sql_sanitize_columns(table.columns) + engine = create_engine(url) + # pandas/sqlite3 prefers the raw connection, otherwise error: # AttributeError: 'Engine' object has no attribute 'rollback' - if is_sqlite3: + if url.startswith("sqlite"): _engine = engine.raw_connection() # In pandas >= 0.23 and using sqlite as a backend, the # pandas.DataFrame.to_sql command fails with "OperationalError: @@ -186,11 +228,14 @@ def save_table(table: pandas.DataFrame, for field in indices: E.debug(f"creating index on {field} for {tablename}") try: - _engine.execute( - text("CREATE INDEX {} ON {} ({})".format( + retry_sql_execute( + _engine, + str(text("CREATE INDEX {} ON {} ({})".format( re.sub("[-.]", "_", tablename) + "_" + field, tablename, - field))) + field)))) + except IndexExistsException: + pass except TypeError as ex: logger.warn("could not create index: {}".format(str(ex))) except sqlalchemy.exc.ProgrammingError as ex: @@ -214,12 +259,11 @@ class TableCache(): # 50 Mb max_table_bytes = 5e7 - def __init__(self, engine, schema, is_sqlite3): - self.engine = engine + def __init__(self, database_url, schema): + self.database_url = database_url self.schema = schema self.cache = {} - self.is_sqlite3 = is_sqlite3 self.total_size = 0 self.sizes = collections.defaultdict(int) self.uploaded_sizes = collections.defaultdict(int) @@ -233,12 +277,11 @@ def flush_table(self, tablename): table = self.cache[tablename] self.logger.debug(f"{os.getpid()}: uploading table {tablename} from cache={id(self)} " - f"to engine={id(self.engine)}: {self.sizes[tablename]} bytes") + f"with {self.sizes[tablename]} bytes") save_table(table, - self.engine, + self.database_url, tablename, self.schema, - is_sqlite3=self.is_sqlite3, dtypes=self.dtypes.get(tablename, None)) del table del self.cache[tablename] @@ -300,13 +343,17 @@ def add_table(self, table, tablename, dtypes=None): def close(self): self.flush_all() + if self.database_url is None: + return + + engine = create_engine(self.database_url) if not self.have_created_indices: for index_name, info in self.indices.items(): table_name, fields = info self.logger.debug("creating index {} on {} and fields {}".format( index_name, table_name, fields)) try: - self.engine.execute("CREATE INDEX {} ON {} ({})".format( + engine.execute("CREATE INDEX {} ON {} ({})".format( index_name, table_name, fields)) except sqlalchemy.exc.OperationalError as ex: self.logger.warn("could not create index: {}".format(str(ex))) diff --git a/src/tests/test_storage.py b/src/tests/test_storage.py index e82df49..739a423 100644 --- a/src/tests/test_storage.py +++ b/src/tests/test_storage.py @@ -10,7 +10,7 @@ def benchmark_layout(tmp_path): ntools = 2 - nmetrics = 4000 + nmetrics = 2000 tools = [f"tool{x}" for x in range(ntools)] metrics = [f"metric{x}" for x in range(nmetrics)] @@ -52,7 +52,7 @@ def generate_info(d, name): return outfiles, tools, metrics -@pytest.mark.parametrize("max_workers", [1, 5]) +@pytest.mark.parametrize("max_workers", [1, 2, 5]) def test_upload(benchmark_layout, tmp_path, max_workers): outfiles, tools, metrics = benchmark_layout @@ -62,8 +62,7 @@ def test_upload(benchmark_layout, tmp_path, max_workers): {"title": "test", "description": "test", "tags": [], - "database": {"url": db_path}}, - max_workers=max_workers) + "database": {"url": db_path, "cores": max_workers}}) db_engine = sqlalchemy.create_engine(db_path) From cc94282f810fe7f6353ef6bfdc139abc205e6af7 Mon Sep 17 00:00:00 2001 From: Andreas Heger Date: Tue, 29 Mar 2022 19:17:37 +0100 Subject: [PATCH 2/2] ignore error in __del__, set status to complete --- src/daisy/storage.py | 12 +++++++++--- src/daisy/table_cache.py | 7 ++++++- src/tests/test_storage.py | 4 +++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/daisy/storage.py b/src/daisy/storage.py index 5d7deea..07b41b7 100644 --- a/src/daisy/storage.py +++ b/src/daisy/storage.py @@ -606,6 +606,14 @@ def upload_metrics_tables(infiles: list, pool.join() +def mark_upload_complete(url: str, run_id: int): + engine = create_engine(url) + session = sessionmaker(bind=engine)() + benchmark_run = session.query(BenchmarkRun).filter_by(id=run_id).first() + benchmark_run.status = "complete" + session.commit() + + def upload_result(infiles, outfile, *extras): """upload results into database. @@ -729,9 +737,7 @@ def upload_result(infiles, outfile, *extras): # schema=None, # is_sqlite3=is_sqlite3) - # benchmark_run.status = "complete" - # session.commit() - + mark_upload_complete(url, benchmark_run.id) logger.info("uploaded results under run_id {}".format(benchmark_run.id)) touch(outfile) diff --git a/src/daisy/table_cache.py b/src/daisy/table_cache.py index 7e52834..debceef 100644 --- a/src/daisy/table_cache.py +++ b/src/daisy/table_cache.py @@ -346,7 +346,12 @@ def close(self): if self.database_url is None: return - engine = create_engine(self.database_url) + try: + engine = create_engine(self.database_url) + except AttributeError: + # do not create indices if sqlalchemy already shut down + return + if not self.have_created_indices: for index_name, info in self.indices.items(): table_name, fields = info diff --git a/src/tests/test_storage.py b/src/tests/test_storage.py index 739a423..007e0b0 100644 --- a/src/tests/test_storage.py +++ b/src/tests/test_storage.py @@ -10,7 +10,7 @@ def benchmark_layout(tmp_path): ntools = 2 - nmetrics = 2000 + nmetrics = 1000 tools = [f"tool{x}" for x in range(ntools)] metrics = [f"metric{x}" for x in range(nmetrics)] @@ -76,6 +76,8 @@ def test_upload(benchmark_layout, tmp_path, max_workers): run_df = pandas.read_sql("SELECT * FROM run", conn) assert len(run_df) == 1 + assert run_df.status[0] == "complete" + with db_engine.connect() as conn: ins_df = pandas.read_sql("SELECT * FROM instance", conn) assert list(ins_df.run_id.unique()) == [1]