changes all offsets specification to signed int to allow negative coo…

…rdinates; updates tests; will break interfaces for users; regression tests verified
janelia-flyem · May 8, 2015 · 92fb526 · 92fb526
1 parent 0c8f2e8
commit 92fb526
Show file tree

Hide file tree

Showing 10 changed files with 60 additions and 57 deletions.
diff --git a/libdvid/DVIDNodeService.h b/libdvid/DVIDNodeService.h
@@ -125,7 +125,7 @@ class DVIDNodeService {
      * \return 2D grayscale object that wraps a byte buffer
     */ 
     Grayscale2D get_tile_slice(std::string datatype_instance, Slice2D slice,
-            unsigned int scaling, std::vector<unsigned int> tile_loc);
+            unsigned int scaling, std::vector<int> tile_loc);
 
     /*!
      * Retrive the raw pre-computed tile (no decompression) from
@@ -139,7 +139,7 @@ class DVIDNodeService {
      * \return byte buffer for the raw compressed data stored (e.g, JPEG or PNG) 
     */ 
     BinaryDataPtr get_tile_slice_binary(std::string datatype_instance, Slice2D slice,
-            unsigned int scaling, std::vector<unsigned int> tile_loc);
+            unsigned int scaling, std::vector<int> tile_loc);
 
     /*!
      * Retrive a 3D 1-byte grayscale volume with the specified
@@ -161,7 +161,7 @@ class DVIDNodeService {
      * \return 3D grayscale object that wraps a byte buffer
     */
     Grayscale3D get_gray3D(std::string datatype_instance, Dims_t dims,
-            std::vector<unsigned int> offset, bool throttle=true,
+            std::vector<int> offset, bool throttle=true,
             bool compress=false, std::string roi="");
 
     /*!
@@ -186,7 +186,7 @@ class DVIDNodeService {
      * \return 3D grayscale object that wraps a byte buffer
     */
     Grayscale3D get_gray3D(std::string datatype_instance, Dims_t dims,
-            std::vector<unsigned int> offset,
+            std::vector<int> offset,
             std::vector<unsigned int> channels, bool throttle=true,
             bool compress=false, std::string roi="");
 
@@ -210,7 +210,7 @@ class DVIDNodeService {
      * \return 3D label object that wraps a byte buffer
     */
     Labels3D get_labels3D(std::string datatype_instance, Dims_t dims,
-            std::vector<unsigned int> offset, bool throttle=true,
+            std::vector<int> offset, bool throttle=true,
             bool compress=true, std::string roi="");
 
     /*!
@@ -235,7 +235,7 @@ class DVIDNodeService {
      * \return 3D label object that wraps a byte buffer
     */
     Labels3D get_labels3D(std::string datatype_instance, Dims_t dims,
-            std::vector<unsigned int> offset,
+            std::vector<int> offset,
             std::vector<unsigned int> channels, bool throttle=true,
             bool compress=true, std::string roi="");
 
@@ -258,7 +258,7 @@ class DVIDNodeService {
      * \param compress enable lz4 compression
     */
     void put_gray3D(std::string datatype_instance, Grayscale3D const & volume,
-            std::vector<unsigned int> offset, bool throttle=true,
+            std::vector<int> offset, bool throttle=true,
             bool compress=false);
 
     /*!
@@ -281,7 +281,7 @@ class DVIDNodeService {
      * \param compress enable lz4 compression
     */
     void put_labels3D(std::string datatype_instance, Labels3D const & volume,
-            std::vector<unsigned int> offset, bool throttle=true,
+            std::vector<int> offset, bool throttle=true,
             bool compress=true, std::string roi="");
 
     /************** API to access DVID blocks directly **************/
@@ -300,7 +300,7 @@ class DVIDNodeService {
      * \return grayscale blocks
     */
     GrayscaleBlocks get_grayblocks(std::string datatype_instance,
-           std::vector<unsigned int> block_coords, unsigned int span); 
+           std::vector<int> block_coords, unsigned int span); 
 
     /*!
      * Fetch label blocks from DVID.  The call will fetch
@@ -314,7 +314,7 @@ class DVIDNodeService {
      * \return grayscale blocks
     */
     LabelBlocks get_labelblocks(std::string datatype_instance,
-           std::vector<unsigned int> block_coords, unsigned int span);
+           std::vector<int> block_coords, unsigned int span);
 
     /*!
      * Put grayscale blocks to DVID.   The call will put
@@ -326,7 +326,7 @@ class DVIDNodeService {
      * \param block_coords location of first block in span (block coordinates)
     */
     void put_grayblocks(std::string datatype_instance,
-            GrayscaleBlocks blocks, std::vector<unsigned int> block_coords);
+            GrayscaleBlocks blocks, std::vector<int> block_coords);
 
     /*!
      * Put label blocks to DVID.   The call will put
@@ -339,7 +339,7 @@ class DVIDNodeService {
      * \param block_coords location of first block in span (block coordinates)
     */
     void put_labelblocks(std::string datatype_instance,
-            LabelBlocks blocks, std::vector<unsigned int> block_coords);
+            LabelBlocks blocks, std::vector<int> block_coords);
 
     /*************** API to access keyvalue interface ***************/
 
@@ -616,7 +616,7 @@ class DVIDNodeService {
      * \param roi specify DVID roi to mask PUT operation (default: empty)
     */
     void put_volume(std::string datatype_instance, BinaryDataPtr volume,
-            std::vector<unsigned int> sizes, std::vector<unsigned int> offset,
+            std::vector<unsigned int> sizes, std::vector<int> offset,
             bool throttle, bool compress, std::string roi);
 
     /*!
@@ -627,7 +627,7 @@ class DVIDNodeService {
      * \return binary data corresponding to an array of blocks
     */
     BinaryDataPtr get_blocks(std::string datatype_instance,
-        std::vector<unsigned int> block_coords, int span);
+        std::vector<int> block_coords, int span);
 
     /*!
      * Helper to put blocks from DVID for labels and grayscale.
@@ -637,7 +637,7 @@ class DVIDNodeService {
      * \param block_coord starting block in DVID block coordinates
     */
     void put_blocks(std::string datatype_instance, BinaryDataPtr binary,
-            int span, std::vector<unsigned int> block_coords);
+            int span, std::vector<int> block_coords);
 
     /*!
      * Helper function to create an instance of the specified type.
@@ -668,7 +668,7 @@ class DVIDNodeService {
      * \return byte buffer corresponding to volume
     */
     BinaryDataPtr get_volume3D(std::string datatype_inst, Dims_t sizes,
-        std::vector<unsigned int> offset, std::vector<unsigned int> channels,
+        std::vector<int> offset, std::vector<unsigned int> channels,
         bool throttle, bool compress, std::string roi);
 
     /*!
@@ -683,7 +683,7 @@ class DVIDNodeService {
      * \param roi specify DVID roi to mask operation (default: empty)
     */
     std::string construct_volume_uri(std::string datatype_inst, Dims_t sizes,
-            std::vector<unsigned int> offset,
+            std::vector<int> offset,
             std::vector<unsigned int> channels, bool throttle, bool compress,
             std::string roi);
 };

diff --git a/load_tests/loadtest_labelblk.cpp b/load_tests/loadtest_labelblk.cpp
@@ -72,7 +72,7 @@ int main(int argc, char** argv)
             binary = BinaryData::decompress_lz4(binary, VOLDIM*VOLDIM*VOLDIM*sizeof(uint64));
             Labels3D labelbin = Labels3D(binary, dims);
 
-            vector<unsigned int> start;
+            vector<int> start;
             start.push_back(0); start.push_back(0); start.push_back(0);
 
             ScopeTime post_timer(false);
@@ -86,7 +86,7 @@ int main(int argc, char** argv)
         // retrieve the image volume
         // this might be artifically fast as it might be completely in DVID's cache
         {
-            vector<unsigned int> start;
+            vector<int> start;
             start.push_back(0); start.push_back(0); start.push_back(0);
             Dims_t lsizes;
             lsizes.push_back(VOLDIM); lsizes.push_back(VOLDIM); lsizes.push_back(VOLDIM);
@@ -106,7 +106,7 @@ int main(int argc, char** argv)
             int max_val = VOLDIM - SMALLFETCH + 1;
 
             // grab random small cube
-            vector<unsigned int> start;
+            vector<int> start;
             start.push_back(rand()%max_val);
             start.push_back(rand()%max_val);
             start.push_back(rand()%max_val);

diff --git a/load_tests/loadtest_tile.cpp b/load_tests/loadtest_tile.cpp
@@ -83,7 +83,7 @@ int main(int argc, char** argv)
 
         // assuming a 1024x1024 window, 9 tiles could be needed if the window
         // is not completely aligned to tile space
-        vector<unsigned int> tilepos;
+        vector<int> tilepos;
         tilepos.push_back(0);
         tilepos.push_back(0);
         tilepos.push_back(zstart);
@@ -145,7 +145,7 @@ int main(int argc, char** argv)
         cout << "Frame rate: " << total_read_time / NUM_FETCHES * 1000 << " milliseconds" << endl;
 
         // size for gray and label fetch
-        vector<unsigned int> start;
+        vector<int> start;
         start.push_back(xstartm); start.push_back(ystartm); start.push_back(zstart);
         Dims_t lsizes;
         lsizes.push_back(FETCHSIZE); lsizes.push_back(FETCHSIZE); lsizes.push_back(1);

diff --git a/load_tests/results/labelgraph.local.results b/load_tests/results/labelgraph.local.results
@@ -1,11 +1,13 @@
-# edges are slower to write because vertices get modified too
-# read rates seem similar to what I get off a bigger server
+# big performance increases in general with new deserialization/serialization of graph
+# slightly slower vertex reads
+# faster edge writes
+# much faster reads
 Total vertices: 75002
 Total edges: 496383
-Time to write vertices: 7.75302
-Vertices written per second: 9673.91
-Time to write edges: 258.427
-Edges written per second: 1920.79
-Time to read graph: 55.6595
-Vertices and edges read per second: 10265.7
-Time Elapsed: 319.034 seconds
+Time to write vertices: 8.00077
+Vertices written per second: 9374.34
+Time to write edges: 192.044
+Edges written per second: 2584.73
+Time to read graph: 9.62582
+Vertices and edges read per second: 59359.6
+Time Elapsed: 203.791 seconds
diff --git a/python/src/libdvid_python.cpp b/python/src/libdvid_python.cpp
@@ -111,6 +111,7 @@ namespace libdvid { namespace python {
         import_array();
 
         // Register custom Python -> C++ converters.
+        std_vector_from_python_iterable<int>();
         std_vector_from_python_iterable<unsigned int>();
         std_vector_from_python_iterable<BlockXYZ>();
         std_vector_from_python_iterable<SubstackXYZ>();
@@ -164,10 +165,10 @@ namespace libdvid { namespace python {
         // For overloaded functions, boost::python needs help figuring out which one we're aiming for.
         // These function pointers specify the ones we want.
         void (DVIDNodeService::*put_binary)(std::string, std::string, BinaryDataPtr) = &DVIDNodeService::put;
-        Grayscale3D (DVIDNodeService::*get_gray3D)(std::string, Dims_t, std::vector<unsigned int>, bool, bool, std::string) = &DVIDNodeService::get_gray3D;
-        Labels3D (DVIDNodeService::*get_labels3D)(std::string, Dims_t, std::vector<unsigned int>, bool, bool, std::string) = &DVIDNodeService::get_labels3D;
-        void (DVIDNodeService::*put_gray3D)(std::string, Grayscale3D const&, std::vector<unsigned int>, bool, bool) = &DVIDNodeService::put_gray3D;
-        void (DVIDNodeService::*put_labels3D)(std::string, Labels3D const&, std::vector<unsigned int>, bool, bool, std::string) = &DVIDNodeService::put_labels3D;
+        Grayscale3D (DVIDNodeService::*get_gray3D)(std::string, Dims_t, std::vector<int>, bool, bool, std::string) = &DVIDNodeService::get_gray3D;
+        Labels3D (DVIDNodeService::*get_labels3D)(std::string, Dims_t, std::vector<int>, bool, bool, std::string) = &DVIDNodeService::get_labels3D;
+        void (DVIDNodeService::*put_gray3D)(std::string, Grayscale3D const&, std::vector<int>, bool, bool) = &DVIDNodeService::put_gray3D;
+        void (DVIDNodeService::*put_labels3D)(std::string, Labels3D const&, std::vector<int>, bool, bool, std::string) = &DVIDNodeService::put_labels3D;
         bool (DVIDNodeService::*create_labelblk)(std::string, std::string) = &DVIDNodeService::create_labelblk;
 
         // DVIDNodeService python class definition

diff --git a/src/DVIDNodeService.cpp b/src/DVIDNodeService.cpp
@@ -92,7 +92,7 @@ bool DVIDNodeService::create_roi(string name)
 }
 
 Grayscale2D DVIDNodeService::get_tile_slice(string datatype_instance,
-        Slice2D slice, unsigned int scaling, vector<unsigned int> tile_loc)
+        Slice2D slice, unsigned int scaling, vector<int> tile_loc)
 {
     BinaryDataPtr binary_response = get_tile_slice_binary(datatype_instance,
             slice, scaling, tile_loc);
@@ -117,7 +117,7 @@ Grayscale2D DVIDNodeService::get_tile_slice(string datatype_instance,
 }
 
 BinaryDataPtr DVIDNodeService::get_tile_slice_binary(string datatype_instance,
-        Slice2D slice, unsigned int scaling, vector<unsigned int> tile_loc)
+        Slice2D slice, unsigned int scaling, vector<int> tile_loc)
 {
     if (tile_loc.size() != 3) {
         throw ErrMsg("Tile identification requires 3 numbers");
@@ -143,7 +143,7 @@ BinaryDataPtr DVIDNodeService::get_tile_slice_binary(string datatype_instance,
 }
 
 Grayscale3D DVIDNodeService::get_gray3D(string datatype_instance, Dims_t sizes,
-        vector<unsigned int> offset, vector<unsigned int> channels,
+        vector<int> offset, vector<unsigned int> channels,
         bool throttle, bool compress, string roi)
 {
     BinaryDataPtr data = get_volume3D(datatype_instance,
@@ -161,7 +161,7 @@ Grayscale3D DVIDNodeService::get_gray3D(string datatype_instance, Dims_t sizes,
 }
 
 Grayscale3D DVIDNodeService::get_gray3D(string datatype_instance, Dims_t sizes,
-        vector<unsigned int> offset, bool throttle, bool compress, string roi)
+        vector<int> offset, bool throttle, bool compress, string roi)
 {
     vector<unsigned int> channels;
     channels.push_back(0); channels.push_back(1); channels.push_back(2);
@@ -170,7 +170,7 @@ Grayscale3D DVIDNodeService::get_gray3D(string datatype_instance, Dims_t sizes,
 }
 
 Labels3D DVIDNodeService::get_labels3D(string datatype_instance, Dims_t sizes,
-        vector<unsigned int> offset, vector<unsigned int> channels,
+        vector<int> offset, vector<unsigned int> channels,
         bool throttle, bool compress, string roi)
 {
     BinaryDataPtr data = get_volume3D(datatype_instance,
@@ -189,7 +189,7 @@ Labels3D DVIDNodeService::get_labels3D(string datatype_instance, Dims_t sizes,
 }
 
 Labels3D DVIDNodeService::get_labels3D(string datatype_instance, Dims_t sizes,
-        vector<unsigned int> offset, bool throttle, bool compress, string roi)
+        vector<int> offset, bool throttle, bool compress, string roi)
 {
     vector<unsigned int> channels;
     channels.push_back(0); channels.push_back(1); channels.push_back(2);
@@ -198,15 +198,15 @@ Labels3D DVIDNodeService::get_labels3D(string datatype_instance, Dims_t sizes,
 }
 
 void DVIDNodeService::put_labels3D(string datatype_instance, Labels3D const & volume,
-            vector<unsigned int> offset, bool throttle, bool compress, string roi)
+            vector<int> offset, bool throttle, bool compress, string roi)
 {
     Dims_t sizes = volume.get_dims();
     put_volume(datatype_instance, volume.get_binary(), sizes,
             offset, throttle, compress, roi);
 }
 
 void DVIDNodeService::put_gray3D(string datatype_instance, Grayscale3D const & volume,
-            vector<unsigned int> offset, bool throttle, bool compress)
+            vector<int> offset, bool throttle, bool compress)
 {
     Dims_t sizes = volume.get_dims();
     put_volume(datatype_instance, volume.get_binary(), sizes,
@@ -215,7 +215,7 @@ void DVIDNodeService::put_gray3D(string datatype_instance, Grayscale3D const & v
 
 
 GrayscaleBlocks DVIDNodeService::get_grayblocks(string datatype_instance,
-        vector<unsigned int> block_coords, unsigned int span)
+        vector<int> block_coords, unsigned int span)
 {
     int ret_span = span;
     BinaryDataPtr data = get_blocks(datatype_instance, block_coords, span);
@@ -230,7 +230,7 @@ GrayscaleBlocks DVIDNodeService::get_grayblocks(string datatype_instance,
 } 
 
 LabelBlocks DVIDNodeService::get_labelblocks(string datatype_instance,
-           vector<unsigned int> block_coords, unsigned int span)
+           vector<int> block_coords, unsigned int span)
 {
     int ret_span = span;
     BinaryDataPtr data = get_blocks(datatype_instance, block_coords, span);
@@ -245,15 +245,15 @@ LabelBlocks DVIDNodeService::get_labelblocks(string datatype_instance,
 }
 
 void DVIDNodeService::put_grayblocks(string datatype_instance,
-            GrayscaleBlocks blocks, vector<unsigned int> block_coords)
+            GrayscaleBlocks blocks, vector<int> block_coords)
 {
     put_blocks(datatype_instance, blocks.get_binary(),
             blocks.get_num_blocks(), block_coords);
 }
 
 
 void DVIDNodeService::put_labelblocks(string datatype_instance,
-            LabelBlocks blocks, vector<unsigned int> block_coords)
+            LabelBlocks blocks, vector<int> block_coords)
 {
     put_blocks(datatype_instance, blocks.get_binary(),
             blocks.get_num_blocks(), block_coords);
@@ -1026,7 +1026,7 @@ bool DVIDNodeService::get_coarse_body(string labelvol_name, uint64 bodyid,
 // ******************** PRIVATE HELPER FUNCTIONS *******************************
 
 void DVIDNodeService::put_volume(string datatype_instance, BinaryDataPtr volume,
-            vector<unsigned int> sizes, vector<unsigned int> offset,
+            vector<unsigned int> sizes, vector<int> offset,
             bool throttle, bool compress, string roi)
 {
     // make sure volume specified is legal and block aligned
@@ -1089,7 +1089,7 @@ void DVIDNodeService::put_volume(string datatype_instance, BinaryDataPtr volume,
 }
 
 BinaryDataPtr DVIDNodeService::get_blocks(string datatype_instance,
-        vector<unsigned int> block_coords, int span)
+        vector<int> block_coords, int span)
 {
     string prefix = "/" + datatype_instance + "/blocks/";
     stringstream sstr;
@@ -1105,7 +1105,7 @@ BinaryDataPtr DVIDNodeService::get_blocks(string datatype_instance,
 }
 
 void DVIDNodeService::put_blocks(string datatype_instance,
-        BinaryDataPtr binary, int span, vector<unsigned int> block_coords)
+        BinaryDataPtr binary, int span, vector<int> block_coords)
 {
     string prefix = "/" + datatype_instance + "/blocks/";
     stringstream sstr;
@@ -1167,7 +1167,7 @@ bool DVIDNodeService::exists(string datatype_endpoint)
 }
 
 BinaryDataPtr DVIDNodeService::get_volume3D(string datatype_inst, Dims_t sizes,
-        vector<unsigned int> offset, vector<unsigned int> channels,
+        vector<int> offset, vector<unsigned int> channels,
         bool throttle, bool compress, string roi)
 {
     bool waiting = true;
@@ -1215,7 +1215,7 @@ BinaryDataPtr DVIDNodeService::get_volume3D(string datatype_inst, Dims_t sizes,
 }
 
 string DVIDNodeService::construct_volume_uri(string datatype_inst, Dims_t sizes,
-        vector<unsigned int> offset, vector<unsigned int> channels,
+        vector<int> offset, vector<unsigned int> channels,
         bool throttle, bool compress, string roi)
 {
     string uri = "/node/" + uuid + "/"