diff --git a/libdvid/DVIDNodeService.h b/libdvid/DVIDNodeService.h index c0baf2a..0871e1e 100644 --- a/libdvid/DVIDNodeService.h +++ b/libdvid/DVIDNodeService.h @@ -193,7 +193,7 @@ class DVIDNodeService { */ Labels3D get_labels3D(std::string datatype_instance, Dims_t dims, std::vector offset, bool throttle=true, - bool compress=false); + bool compress=true); /*! * Retrive a 3D 8-byte label volume with the specified @@ -218,7 +218,7 @@ class DVIDNodeService { Labels3D get_labels3D(std::string datatype_instance, Dims_t dims, std::vector offset, std::vector channels, bool throttle=true, - bool compress=false); + bool compress=true); /*! * Put a 3D 1-byte grayscale volume to DVID with the specified @@ -262,7 +262,7 @@ class DVIDNodeService { */ void put_labels3D(std::string datatype_instance, Labels3D& volume, std::vector offset, bool throttle=true, - bool compress=false); + bool compress=true); /************** API to access DVID blocks directly **************/ // This API is probably most relevant for bulk transfers to and @@ -272,8 +272,7 @@ class DVIDNodeService { * Fetch grayscale blocks from DVID. The call will fetch * a series of contiguous blocks along the first dimension (X). * The number of blocks fetched is encoded in the GrayscaleBlocks - * returned structure. This number can be less than the - * requested span if those blocks do not exist on DVID. + * returned structure. * TODO: support compression and throttling. * \param datatype instance name of grayscale type instance * \param block_coords location of first block in span (block coordinates) @@ -287,8 +286,7 @@ class DVIDNodeService { * Fetch label blocks from DVID. The call will fetch * a series of contiguous blocks along the first dimension (X). * The number of blocks fetched is encoded in the LabelBlocks - * returned structure. This number can be less than the - * requested span if those blocks do not exist on DVID. + * returned structure. * TODO: support compression and throttling. * \param datatype instance name of labelblk type instance * \param block_coords location of first block in span (block coordinates) @@ -515,11 +513,10 @@ class DVIDNodeService { * \param datatype_instance name of datatype instance * \param block_coords starting block in DVID block coordinates * \param span number of blocks to attempt to get - * \param return number of actual blocks retrieved * \return binary data corresponding to an array of blocks */ BinaryDataPtr get_blocks(std::string datatype_instance, - std::vector block_coords, int span, int& extracted_span); + std::vector block_coords, int span); /*! * Helper to put blocks from DVID for labels and grayscale. diff --git a/load_tests/inputs/segmenation.binary.lz4 b/load_tests/inputs/segmenation.binary.lz4 new file mode 100644 index 0000000..6fea871 Binary files /dev/null and b/load_tests/inputs/segmenation.binary.lz4 differ diff --git a/load_tests/loadtest_labelblk.cpp b/load_tests/loadtest_labelblk.cpp index 4648649..d5843b6 100644 --- a/load_tests/loadtest_labelblk.cpp +++ b/load_tests/loadtest_labelblk.cpp @@ -11,10 +11,12 @@ #include "ScopeTime.h" #include +#include using std::cerr; using std::cout; using std::endl; using namespace libdvid; using std::vector; +using std::ifstream; using std::string; @@ -31,11 +33,13 @@ int NUM_FETCHES = 100; int SMALLFETCH = 128; /*! - * Exercises the nD GETs and PUTs for the labelblk datatype. + * Exercises the nD GETs and PUTs for the labelblk datatype. Assume input + * binary is VOLDIMxVOLDIMxVOLDIM 64bit numbers compressed with lz4 (example + * in the inputs directory of a segmentation of an EM dataset with 8nm voxels). */ int main(int argc, char** argv) { - if (argc != 2) { + if (argc != 3) { cout << "Usage: " << endl; return -1; } @@ -56,27 +60,23 @@ int main(int argc, char** argv) return -1; } - // ** Write and read labelblk data ** - // create label 64 image volume - unsigned long long * img_labels = new unsigned long long [VOLDIM * VOLDIM * VOLDIM]; - for (int i = 0; i < (VOLDIM*VOLDIM*VOLDIM); ++i) { - img_labels[i] = rand() % 1000000; - } - // create binary data string wrapper (64 bits per pixel) // calculate time to post volume { Dims_t dims; dims.push_back(VOLDIM); dims.push_back(VOLDIM); dims.push_back(VOLDIM); - Labels3D labelbin = Labels3D(img_labels, - VOLDIM * VOLDIM * VOLDIM, dims); - delete []img_labels; + + // read a representative segmentation + ifstream fin(argv[2]); + BinaryDataPtr binary = BinaryData::create_binary_data(fin); + binary = BinaryData::decompress_lz4(binary, VOLDIM*VOLDIM*VOLDIM*sizeof(uint64)); + Labels3D labelbin = Labels3D(binary, dims); vector start; start.push_back(0); start.push_back(0); start.push_back(0); ScopeTime post_timer(false); - dvid_node.put_labels3D(label_datatype_name, labelbin, start); + dvid_node.put_labels3D(label_datatype_name, labelbin, start, true, true); double post_time = post_timer.getElapsed(); cout << "Time to POST large label volume: " << post_time << " seconds" << endl; cout << int(VOLDIM * VOLDIM * VOLDIM * sizeof(unsigned long long) / post_time) @@ -92,7 +92,7 @@ int main(int argc, char** argv) lsizes.push_back(VOLDIM); lsizes.push_back(VOLDIM); lsizes.push_back(VOLDIM); ScopeTime get_timer(false); - Labels3D labelcomp = dvid_node.get_labels3D(label_datatype_name, lsizes, start); + Labels3D labelcomp = dvid_node.get_labels3D(label_datatype_name, lsizes, start, true, true); double read_time = get_timer.getElapsed(); cout << "Time to GET large label volume: " << read_time << " seconds" << endl; cout << int(VOLDIM * VOLDIM * VOLDIM * sizeof(unsigned long long)/ read_time) @@ -116,7 +116,7 @@ int main(int argc, char** argv) lsizes.push_back(SMALLFETCH); Labels3D labelcomp = - dvid_node.get_labels3D(label_datatype_name, lsizes, start); + dvid_node.get_labels3D(label_datatype_name, lsizes, start, true, true); double read_time = get_timer.getElapsed(); cout << "Time to GET small label volume: " << read_time << " seconds" << endl; diff --git a/load_tests/results/labelblk.emdata1.results b/load_tests/results/labelblk.emdata1.results deleted file mode 100644 index 8ad7ed5..0000000 --- a/load_tests/results/labelblk.emdata1.results +++ /dev/null @@ -1,114 +0,0 @@ -# write took a long time -- probably because of network card -# why do some GETs take a lot longer -- might be some issue -- notice GETs do take ~2x longer -- is it because of the network card -# it might take longer because of network, not because of DVID?? -# multiply everthing by 8 because I forgot to take into account that these are 64 bit -Large volume dimension: 512 -Small volume dimension: 128 -Time to POST large label volume: 182.178 seconds -736740 bytes posted per second for large label volume -Time to GET large label volume: 13.3619 seconds -10044816 bytes read per second for large label volume -Time to GET small label volume: 0.228918 seconds -Time to GET small label volume: 0.229508 seconds -Time to GET small label volume: 0.227732 seconds -Time to GET small label volume: 0.233251 seconds -Time to GET small label volume: 0.236797 seconds -Time to GET small label volume: 0.229143 seconds -Time to GET small label volume: 0.233194 seconds -Time to GET small label volume: 0.21965 seconds -Time to GET small label volume: 0.231477 seconds -Time to GET small label volume: 0.234772 seconds -Time to GET small label volume: 0.219164 seconds -Time to GET small label volume: 0.235076 seconds -Time to GET small label volume: 0.229866 seconds -Time to GET small label volume: 0.234989 seconds -Time to GET small label volume: 0.79065 seconds -Time to GET small label volume: 0.218566 seconds -Time to GET small label volume: 0.229637 seconds -Time to GET small label volume: 0.319641 seconds -Time to GET small label volume: 0.652377 seconds -Time to GET small label volume: 1.21311 seconds -Time to GET small label volume: 1.12829 seconds -Time to GET small label volume: 2.15353 seconds -Time to GET small label volume: 1.32795 seconds -Time to GET small label volume: 1.30306 seconds -Time to GET small label volume: 0.532994 seconds -Time to GET small label volume: 1.35912 seconds -Time to GET small label volume: 1.43429 seconds -Time to GET small label volume: 0.801441 seconds -Time to GET small label volume: 0.665405 seconds -Time to GET small label volume: 0.23179 seconds -Time to GET small label volume: 0.231847 seconds -Time to GET small label volume: 0.23062 seconds -Time to GET small label volume: 0.241093 seconds -Time to GET small label volume: 0.231063 seconds -Time to GET small label volume: 0.233762 seconds -Time to GET small label volume: 0.229751 seconds -Time to GET small label volume: 0.246158 seconds -Time to GET small label volume: 0.226995 seconds -Time to GET small label volume: 0.245697 seconds -Time to GET small label volume: 0.228585 seconds -Time to GET small label volume: 0.230911 seconds -Time to GET small label volume: 0.246079 seconds -Time to GET small label volume: 0.229254 seconds -Time to GET small label volume: 0.241283 seconds -Time to GET small label volume: 0.234132 seconds -Time to GET small label volume: 0.226847 seconds -Time to GET small label volume: 0.23092 seconds -Time to GET small label volume: 0.234882 seconds -Time to GET small label volume: 0.234135 seconds -Time to GET small label volume: 0.228356 seconds -Time to GET small label volume: 0.231321 seconds -Time to GET small label volume: 0.226998 seconds -Time to GET small label volume: 0.230663 seconds -Time to GET small label volume: 0.231901 seconds -Time to GET small label volume: 0.229032 seconds -Time to GET small label volume: 0.228551 seconds -Time to GET small label volume: 0.230357 seconds -Time to GET small label volume: 0.225531 seconds -Time to GET small label volume: 0.237238 seconds -Time to GET small label volume: 0.227391 seconds -Time to GET small label volume: 0.231002 seconds -Time to GET small label volume: 0.22821 seconds -Time to GET small label volume: 0.228265 seconds -Time to GET small label volume: 0.232546 seconds -Time to GET small label volume: 0.231554 seconds -Time to GET small label volume: 0.228556 seconds -Time to GET small label volume: 0.218626 seconds -Time to GET small label volume: 0.231571 seconds -Time to GET small label volume: 0.228095 seconds -Time to GET small label volume: 0.226441 seconds -Time to GET small label volume: 0.233941 seconds -Time to GET small label volume: 0.232886 seconds -Time to GET small label volume: 0.226279 seconds -Time to GET small label volume: 0.232186 seconds -Time to GET small label volume: 0.227938 seconds -Time to GET small label volume: 0.226381 seconds -Time to GET small label volume: 0.22634 seconds -Time to GET small label volume: 0.23152 seconds -Time to GET small label volume: 0.231374 seconds -Time to GET small label volume: 0.256702 seconds -Time to GET small label volume: 0.233515 seconds -Time to GET small label volume: 0.225563 seconds -Time to GET small label volume: 0.225157 seconds -Time to GET small label volume: 0.22646 seconds -Time to GET small label volume: 0.216632 seconds -Time to GET small label volume: 0.227423 seconds -Time to GET small label volume: 0.234499 seconds -Time to GET small label volume: 0.225649 seconds -Time to GET small label volume: 0.238958 seconds -Time to GET small label volume: 0.231167 seconds -Time to GET small label volume: 0.228502 seconds -Time to GET small label volume: 0.225671 seconds -Time to GET small label volume: 0.230271 seconds -Time to GET small label volume: 0.227892 seconds -Time to GET small label volume: 0.225502 seconds -Time to GET small label volume: 0.227827 seconds -Time to GET small label volume: 0.230066 seconds -Time to GET small label volume: 0.23284 seconds -Time to GET small label volume: 0.228857 seconds -Time to GET small label volume: 0.2334 seconds -100 fetches performed in 33.753 seconds -0.33753 seconds per fetch -6213229 bytes read per second -Time Elapsed: 232.712 seconds diff --git a/load_tests/results/labelblk.emrecon42.results b/load_tests/results/labelblk.emrecon42.results new file mode 100644 index 0000000..c56f633 --- /dev/null +++ b/load_tests/results/labelblk.emrecon42.results @@ -0,0 +1,216 @@ +# using lz4 compression really improves performance (especially +# when using real segmentation data, i.e., not random) +# GET requests are still slow because there appears to be an +# issue with LZ4 compression on DVID +Large volume dimension: 512 +Small volume dimension: 128 +Volume size sent: 32570374 +Time to POST large label volume: 1.61717 seconds +663964715 bytes posted per second for large label volume +Volume size received: 1077952592 +Time to GET large label volume: 13.0792 seconds +82095375 bytes read per second for large label volume +Volume size received: 16843025 +Time to GET small label volume: 0.198198 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.191843 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.197929 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.198815 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.202637 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.200879 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.190546 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.189863 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.201195 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.198938 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.197877 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.191257 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.189809 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.193442 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.195505 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199242 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.197654 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.195176 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.196706 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.19665 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.190129 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.194021 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.194939 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.196573 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.197707 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.190428 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.188568 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.197974 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.187289 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.195907 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.195333 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.192371 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.194286 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.19939 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199325 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.195019 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.196229 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.201799 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.20004 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.195592 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.1961 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.191816 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.202969 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199009 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.202007 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.197088 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.193735 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199741 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.202005 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.190223 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.20284 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.194824 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.194573 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.201355 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.201652 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.197031 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199802 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199741 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199613 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199187 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.202597 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.196597 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.197864 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.201588 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199608 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.198803 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.202597 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.192226 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.194587 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199442 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.196732 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.194057 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.201749 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.201519 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.191357 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.201477 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.195549 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.200968 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.202969 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.201019 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.200994 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199418 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.198947 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.202951 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.200399 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.192201 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.193768 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.202805 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.201637 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.193883 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.19213 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199168 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.19926 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.193223 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.193375 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.203052 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.193842 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.199219 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.200611 seconds +Volume size received: 16843025 +Time to GET small label volume: 0.195812 seconds +100 fetches performed in 19.7224 seconds +0.197224 seconds per fetch +85066846 bytes read per second +Time Elapsed: 35.2015 seconds diff --git a/load_tests/results/labelblk.local.results b/load_tests/results/labelblk.local.results deleted file mode 100644 index 16bfeb9..0000000 --- a/load_tests/results/labelblk.local.results +++ /dev/null @@ -1,111 +0,0 @@ -# 14 seconds, 5.5 second for first two fetches on DVID -- not much overhead on loading side -Large volume dimension: 512 -Small volume dimension: 128 -Time to POST large label volume: 17.8208 seconds -60252333 bytes posted per second for large label volume -Time to GET large label volume: 6.78539 seconds -158243274 bytes read per second for large label volume -Time to GET small label volume: 0.126293 seconds -Time to GET small label volume: 0.124154 seconds -Time to GET small label volume: 0.118299 seconds -Time to GET small label volume: 0.119718 seconds -Time to GET small label volume: 0.122405 seconds -Time to GET small label volume: 0.132044 seconds -Time to GET small label volume: 0.125187 seconds -Time to GET small label volume: 0.119775 seconds -Time to GET small label volume: 0.168811 seconds -Time to GET small label volume: 0.141873 seconds -Time to GET small label volume: 0.12166 seconds -Time to GET small label volume: 0.120328 seconds -Time to GET small label volume: 0.126159 seconds -Time to GET small label volume: 0.119118 seconds -Time to GET small label volume: 0.128439 seconds -Time to GET small label volume: 0.124711 seconds -Time to GET small label volume: 0.122763 seconds -Time to GET small label volume: 0.126626 seconds -Time to GET small label volume: 0.127207 seconds -Time to GET small label volume: 0.125743 seconds -Time to GET small label volume: 0.123577 seconds -Time to GET small label volume: 0.129011 seconds -Time to GET small label volume: 0.129219 seconds -Time to GET small label volume: 0.12766 seconds -Time to GET small label volume: 0.12551 seconds -Time to GET small label volume: 0.12182 seconds -Time to GET small label volume: 0.12062 seconds -Time to GET small label volume: 0.118749 seconds -Time to GET small label volume: 0.12486 seconds -Time to GET small label volume: 0.125945 seconds -Time to GET small label volume: 0.126915 seconds -Time to GET small label volume: 0.127734 seconds -Time to GET small label volume: 0.126141 seconds -Time to GET small label volume: 0.119316 seconds -Time to GET small label volume: 0.12169 seconds -Time to GET small label volume: 0.129363 seconds -Time to GET small label volume: 0.129049 seconds -Time to GET small label volume: 0.120517 seconds -Time to GET small label volume: 0.125165 seconds -Time to GET small label volume: 0.125823 seconds -Time to GET small label volume: 0.125474 seconds -Time to GET small label volume: 0.126694 seconds -Time to GET small label volume: 0.129909 seconds -Time to GET small label volume: 0.11968 seconds -Time to GET small label volume: 0.124998 seconds -Time to GET small label volume: 0.126648 seconds -Time to GET small label volume: 0.12681 seconds -Time to GET small label volume: 0.12696 seconds -Time to GET small label volume: 0.123129 seconds -Time to GET small label volume: 0.129451 seconds -Time to GET small label volume: 0.122457 seconds -Time to GET small label volume: 0.126319 seconds -Time to GET small label volume: 0.125669 seconds -Time to GET small label volume: 0.128742 seconds -Time to GET small label volume: 0.127654 seconds -Time to GET small label volume: 0.120309 seconds -Time to GET small label volume: 0.123649 seconds -Time to GET small label volume: 0.122151 seconds -Time to GET small label volume: 0.126259 seconds -Time to GET small label volume: 0.122544 seconds -Time to GET small label volume: 0.130589 seconds -Time to GET small label volume: 0.128242 seconds -Time to GET small label volume: 0.130355 seconds -Time to GET small label volume: 0.125933 seconds -Time to GET small label volume: 0.124135 seconds -Time to GET small label volume: 0.123453 seconds -Time to GET small label volume: 0.11988 seconds -Time to GET small label volume: 0.125625 seconds -Time to GET small label volume: 0.125171 seconds -Time to GET small label volume: 0.12255 seconds -Time to GET small label volume: 0.123544 seconds -Time to GET small label volume: 0.121263 seconds -Time to GET small label volume: 0.124825 seconds -Time to GET small label volume: 0.118783 seconds -Time to GET small label volume: 0.119305 seconds -Time to GET small label volume: 0.121826 seconds -Time to GET small label volume: 0.118868 seconds -Time to GET small label volume: 0.122759 seconds -Time to GET small label volume: 0.126883 seconds -Time to GET small label volume: 0.129785 seconds -Time to GET small label volume: 0.126415 seconds -Time to GET small label volume: 0.120543 seconds -Time to GET small label volume: 0.119114 seconds -Time to GET small label volume: 0.122632 seconds -Time to GET small label volume: 0.114996 seconds -Time to GET small label volume: 0.11912 seconds -Time to GET small label volume: 0.119748 seconds -Time to GET small label volume: 0.121889 seconds -Time to GET small label volume: 0.120943 seconds -Time to GET small label volume: 0.119208 seconds -Time to GET small label volume: 0.129049 seconds -Time to GET small label volume: 0.126353 seconds -Time to GET small label volume: 0.119176 seconds -Time to GET small label volume: 0.123867 seconds -Time to GET small label volume: 0.124201 seconds -Time to GET small label volume: 0.120295 seconds -Time to GET small label volume: 0.123553 seconds -Time to GET small label volume: 0.122911 seconds -Time to GET small label volume: 0.124973 seconds -Time to GET small label volume: 0.12322 seconds -100 fetches performed in 12.4715 seconds -0.124715 seconds per fetch -134524627 bytes read per second -Time Elapsed: 40.3014 seconds diff --git a/src/DVIDNodeService.cpp b/src/DVIDNodeService.cpp index 6e169dc..c07ddcc 100644 --- a/src/DVIDNodeService.cpp +++ b/src/DVIDNodeService.cpp @@ -171,6 +171,7 @@ Labels3D DVIDNodeService::get_labels3D(string datatype_instance, Dims_t sizes, data = BinaryData::decompress_lz4(data, decomp_size); } + Labels3D labels(data, sizes); return labels; } @@ -205,13 +206,11 @@ GrayscaleBlocks DVIDNodeService::get_grayblocks(string datatype_instance, vector block_coords, unsigned int span) { int ret_span = span; - BinaryDataPtr data = get_blocks(datatype_instance, block_coords, - span, ret_span); - assert(ret_span <= span); + BinaryDataPtr data = get_blocks(datatype_instance, block_coords, span); // make sure this data encodes blocks of grayscale if (data->length() != - (DEFBLOCKSIZE*DEFBLOCKSIZE*DEFBLOCKSIZE*sizeof(uint8)*ret_span)) { + (DEFBLOCKSIZE*DEFBLOCKSIZE*DEFBLOCKSIZE*sizeof(uint8)*span)) { throw ErrMsg("Expected 1-byte values from " + datatype_instance); } @@ -222,9 +221,7 @@ LabelBlocks DVIDNodeService::get_labelblocks(string datatype_instance, vector block_coords, unsigned int span) { int ret_span = span; - BinaryDataPtr data = get_blocks(datatype_instance, block_coords, - span, ret_span); - assert(ret_span <= span); + BinaryDataPtr data = get_blocks(datatype_instance, block_coords, span); // make sure this data encodes blocks of grayscale if (data->length() != @@ -773,7 +770,7 @@ void DVIDNodeService::put_volume(string datatype_instance, BinaryDataPtr volume, } BinaryDataPtr DVIDNodeService::get_blocks(string datatype_instance, - vector block_coords, int span, int& extracted_span) + vector block_coords, int span) { string prefix = "/" + datatype_instance + "/blocks/"; stringstream sstr; @@ -782,15 +779,8 @@ BinaryDataPtr DVIDNodeService::get_blocks(string datatype_instance, sstr << "/" << span; string endpoint = prefix + sstr.str(); - // first 4 bytes encodes the number of returned blocks + // first 4 bytes no longer include span (always grab what the user wants) BinaryDataPtr blockbinary = custom_request(endpoint, BinaryDataPtr(), GET); - const unsigned int* blocks_retrieved = - (const unsigned int*) blockbinary->get_raw(); - extracted_span = *blocks_retrieved; - - // remove first 4 bytes from binary - string& data_ref = blockbinary->get_data(); - data_ref.erase(0,4); return blockbinary; } diff --git a/tests/test_blocks.cpp b/tests/test_blocks.cpp index c35773c..cc6cf5e 100644 --- a/tests/test_blocks.cpp +++ b/tests/test_blocks.cpp @@ -124,14 +124,15 @@ int main(int argc, char** argv) dvid_node.put_grayblocks(gray_name, gray_blocks, offset_blocks); //dvid_node.put_labelblocks(label_name, label_blocks, offset_blocks); - // ask to read more blocks than are there (should return 2 blocks) + // ask to read more blocks than are there (should return 5 blocks with + // the last 3 blank) GrayscaleBlocks gray_blocks_comp = dvid_node.get_grayblocks(gray_name, offset_blocks, 5); /*LabelBlocks label_blocks_comp = dvid_node.get_labelblocks(label_name, offset_blocks, 5);*/ - if (gray_blocks_comp.get_num_blocks() != 2) { - throw ErrMsg("Retrieved more than 2 grayscale blocks"); + if (gray_blocks_comp.get_num_blocks() != 5) { + throw ErrMsg("Retrieve incorrect number of grayscale blocks"); } /*if (label_blocks_comp.get_num_blocks() != 2) { diff --git a/todo b/todo index 3ac21da..da9e773 100644 --- a/todo +++ b/todo @@ -1,4 +1,6 @@ --fix c++ warnings and look for any other memory leaks +-make lz4 r126 version + +-fix c++ warnings and look for any other memory leaks -- appears to be some, not sure if important -add compression for blocks when available -add load test for volume gets using compression (dependent on Bill) -improve test coverage