fix: only install psutil and posix_ipc on non-windows platforms (#234)

* fix: only install psutil and posix_ipc on non-windows platforms

Addresses #225

* fix: don't import psutil or posix_ipc unless we're not on windows

* fix: only import posix_ipc in required functions

* fix: remove posix_ipc and psutil from mandatory installs on windows

* fix: make fetching user home directory platform independent

* fix: URIs should always be handled like posix paths

* fix: handle '.' and '..' in toabs again

* fix: only use posixpath for joining URIs in non file Storage

* fix: only check provenance cache validity if cache is enabled

* fix: make compute_data_locations in cache aware of non-posix paths

* fix: chunknames must be protocol aware to join file paths correctly

* fix: make bucket_path_extraction work with Windows

* docs: described Windows support policy

* fix: update setversion.py to account for setup.py manual versioning

* chore: bump version to match current deployed version on pypi

README.md

@@ -24,7 +24,7 @@ CloudVolume can be used in single or multi-process capacity and can be optimized
 
 ## Setup
 
-Cloud-volume is regularly tested on Ubuntu with Python 2.7, 3.4, 3.5, and 3.6 (we've noticed it's faster on Python 3). Some people have used it with Python 3.7. We support Linux and OS X. Windows is currently unsupported. After installation, you'll also need to set up your cloud credentials if you're planning on writing files or reading from a private dataset. Once you're finished setting up, you can try [reading from a public dataset](https://github.com/seung-lab/cloud-volume/wiki/Reading-Public-Data-Examples).  
+Cloud-volume is regularly tested on Ubuntu with Python 2.7, 3.4, 3.5, and 3.6 (we've noticed it's faster on Python 3). Some people have used it with Python 3.7. We officially support Linux and Mac OS. Windows is community supported. After installation, you'll also need to set up your cloud credentials if you're planning on writing files or reading from a private dataset. Once you're finished setting up, you can try [reading from a public dataset](https://github.com/seung-lab/cloud-volume/wiki/Reading-Public-Data-Examples).  
 
 #### `pip` Binary Installation
 
@@ -439,6 +439,7 @@ Thanks to Yann Leprince for providing a [pure Python codec](https://github.com/H
 Thanks to Jeremy Maitin-Shepard and Stephen Plaza for their C++ code defining the compression and decompression (respectively) protocol for [compressed_segmentation](https://github.com/janelia-flyem/compressedseg).  
 Thanks to Peter Lindstrom et al. for [their work](https://computation.llnl.gov/projects/floating-point-compression) on fpzip, the C++ code, and assistance.  
 Thanks to Nico Kemnitz for his work on the "Kempression" protocol that builds on fpzip (we named it, not him).   
+Thanks to Dan Bumbarger for contributing code and information helpful for getting CloudVolume working on Windows.
 
 ## Mailing List 
 

cloudvolume/cacheservice.py

@@ -1,4 +1,5 @@
 import os
+import posixpath
 import shutil
 
 from .provenance import DataLayerProvenance
@@ -384,24 +385,26 @@ def compute_data_locations(self, cloudpaths):
     if not self.enabled:
       return { 'local': [], 'remote': cloudpaths }
 
+    pathmodule = posixpath if self.meta.path.protocol != 'file' else os.path
+
     def noextensions(fnames):
-      return [ os.path.splitext(fname)[0] for fname in fnames ]
+      return [ pathmodule.splitext(fname)[0] for fname in fnames ]
 
-    list_dirs = set([ os.path.dirname(pth) for pth in cloudpaths ])
+    list_dirs = set([ pathmodule.dirname(pth) for pth in cloudpaths ])
     filenames = []
 
     for list_dir in list_dirs:
       list_dir = os.path.join(self.path, list_dir)
       filenames += noextensions(os.listdir(mkdir(list_dir)))
 
-    basepathmap = { os.path.basename(path): os.path.dirname(path) for path in cloudpaths }
+    basepathmap = { pathmodule.basename(path): pathmodule.dirname(path) for path in cloudpaths }
 
     # check which files are already cached, we only want to download ones not in cache
-    requested = set([ os.path.basename(path) for path in cloudpaths ])
+    requested = set([ pathmodule.basename(path) for path in cloudpaths ])
     already_have = requested.intersection(set(filenames))
     to_download = requested.difference(already_have)
 
-    download_paths = [ os.path.join(basepathmap[fname], fname) for fname in to_download ]    
+    download_paths = [ pathmodule.join(basepathmap[fname], fname) for fname in to_download ]    
     already_have = [ os.path.join(basepathmap[fname], fname) for fname in already_have ]
 
     return { 'local': already_have, 'remote': download_paths }

cloudvolume/datasource/precomputed/common.py

@@ -1,9 +1,10 @@
+import concurrent.futures
 from functools import partial
 import itertools
 import math
 import multiprocessing as mp
-import concurrent.futures
 import os
+import posixpath
 import signal
 
 import numpy as np
@@ -32,15 +33,17 @@ def cleanup(signum, frame):
   signal.signal(signal.SIGINT, prevsigint)
   signal.signal(signal.SIGTERM, prevsigterm)
 
-def chunknames(bbox, volume_bbox, key, chunk_size):
+def chunknames(bbox, volume_bbox, key, chunk_size, protocol=None):
+  path = posixpath if protocol != 'file' else os.path
+
   for x,y,z in xyzrange( bbox.minpt, bbox.maxpt, chunk_size ):
     highpt = min2(Vec(x,y,z) + chunk_size, volume_bbox.maxpt)
     filename = "{}-{}_{}-{}_{}-{}".format(
       x, highpt.x,
       y, highpt.y, 
       z, highpt.z
     )
-    yield os.path.join(key, filename)
+    yield path.join(key, filename)
 
 def shade(dest_img, dest_bbox, src_img, src_bbox):
   """

cloudvolume/datasource/precomputed/image.py

@@ -147,7 +147,9 @@ def exists(self, bbox, mip=None):
     realized_bbox = Bbox.clamp(realized_bbox, self.meta.bounds(mip))
 
     cloudpaths = list(chunknames(
-      realized_bbox, self.meta.bounds(mip), self.meta.key(mip), self.meta.chunk_size(mip)
+      realized_bbox, self.meta.bounds(mip), 
+      self.meta.key(mip), self.meta.chunk_size(mip),
+      protocol=self.meta.path.protocol
     ))
 
     with Storage(self.meta.cloudpath, progress=self.config.progress) as storage:
@@ -171,7 +173,9 @@ def delete(self, bbox, mip=None):
       ))
 
     cloudpaths = list(chunknames(
-      realized_bbox, self.meta.bounds(mip), self.meta.key(mip), self.meta.chunk_size(mip)
+      realized_bbox, self.meta.bounds(mip), 
+      self.meta.key(mip), self.meta.chunk_size(mip),
+      protocol=self.meta.path.protocol
     ))
 
     with Storage(self.meta.cloudpath, progress=self.config.progress) as storage:
@@ -246,7 +250,11 @@ def transfer_to(self, cloudpath, bbox, mip, block_size=None, compress=True):
     num_blocks = np.ceil(self.meta.bounds(mip).volume() / self.meta.chunk_size(mip).rectVolume()) / step
     num_blocks = int(np.ceil(num_blocks))
 
-    cloudpaths = chunknames(bbox, self.meta.bounds(mip), self.meta.key(mip), self.meta.chunk_size(mip))
+    cloudpaths = chunknames(
+      bbox, self.meta.bounds(mip), 
+      self.meta.key(mip), self.meta.chunk_size(mip),
+      protocol=self.meta.path.protocol
+    )
 
     pbar = tqdm(
       desc='Transferring Blocks of {} Chunks'.format(step), 

cloudvolume/datasource/precomputed/metadata.py

@@ -1,5 +1,6 @@
 import json
 import os
+import posixpath
 
 import json5
 import multiprocessing as mp
@@ -49,7 +50,7 @@ def __init__(self, cloudpath, cache=None, info=None, provenance=None):
     if provenance is None:
       self.provenance = None
       self.refresh_provenance()
-      if self.cache:
+      if self.cache.enabled:
         self.cache.check_provenance_validity()
     else:
       self.provenance = self._cast_provenance(provenance)
@@ -339,23 +340,27 @@ def scale(self, mip):
 
   @property
   def basepath(self):
-    return os.path.join(self.path.bucket, self.path.intermediate_path, self.dataset)
-
+    path = os.path if self.path.protocol == 'file' else posixpath
+    return path.join(self.path.bucket, self.path.intermediate_path, self.dataset)
+
   @property 
   def layerpath(self):
-    return os.path.join(self.basepath, self.layer)
+    path = os.path if self.path.protocol == 'file' else posixpath
+    return path.join(self.basepath, self.layer)
 
   @property
   def base_cloudpath(self):
     return self.path.protocol + "://" + self.basepath
 
   @property 
   def cloudpath(self):
-    return os.path.join(self.base_cloudpath, self.layer)
+    path = os.path if self.path.protocol == 'file' else posixpath
+    return path.join(self.base_cloudpath, self.layer)
 
   @property
   def infopath(self):
-    return os.path.join(self.cloudpath, 'info')
+    path = os.path if self.path.protocol == 'file' else posixpath
+    return path.join(self.cloudpath, 'info')
 
   @property
   def skeletons(self):

cloudvolume/datasource/precomputed/rx.py

@@ -43,7 +43,9 @@ def download(
   )
   full_bbox = Bbox.clamp(full_bbox, meta.bounds(mip))
   cloudpaths = list(chunknames(
-    full_bbox, meta.bounds(mip), meta.key(mip), meta.chunk_size(mip)
+    full_bbox, meta.bounds(mip), 
+    meta.key(mip), meta.chunk_size(mip), 
+    protocol=meta.path.protocol
   ))
   shape = list(requested_bbox.size3()) + [ meta.num_channels ]
 

cloudvolume/lib.py

@@ -74,18 +74,28 @@ def colorize(color, text):
   ('protocol', 'intermediate_path', 'bucket', 'dataset','layer')
 )
 
-def extract_bucket_path(cloudpath):
+def extract_bucket_path(cloudpath, windows=None, disable_toabs=False):
   protocol_re = r'^(gs|file|s3|boss|matrix|https?)://'
-  bucket_re = r'^(/?[~\d\w_\.\-]+)/'
+  windows_file_re = r'((?:\w:\\)[\d\w_\.\-]+(?:\\)?)' # for C:\what\a\great\path
+  bucket_re = r'^(/?[~\d\w_\.\-]+)/' # posix /what/a/great/path
+
+  if windows is None:
+    windows = sys.platform == 'win32'
+
+  if disable_toabs:
+    abspath = lambda x: x # can't prepend linux paths when force testing windows
+  else:
+    abspath = toabs    
 
   error = UnsupportedProtocolError("""
     Cloud path must conform to PROTOCOL://BUCKET/PATH
     Example: gs://test_bucket/em
 
     Supported protocols: gs, s3, file, matrix, boss, http, https
+    Windows Mode: {}
 
     Received: {}
-    """.format(cloudpath))
+    """.format(windows, cloudpath))
 
   match = re.match(protocol_re, cloudpath)
 
@@ -96,7 +106,9 @@ def extract_bucket_path(cloudpath):
   cloudpath = re.sub(protocol_re, '', cloudpath)
 
   if protocol == 'file':
-    cloudpath = toabs(cloudpath)
+    cloudpath = abspath(cloudpath)
+    if windows:
+      bucket_re = windows_file_re
 
   match = re.match(bucket_re, cloudpath)
   if not match:
@@ -152,8 +164,7 @@ def extract_path(cloudpath):
   return ExtractedPath(protocol, intermediate_path, bucket, dataset, layer)
 
 def toabs(path):
-  home = os.path.join(os.environ['HOME'], '')
-  path = re.sub('^~%c?' % os.path.sep, home, path)
+  path = os.path.expanduser(path)
   return os.path.abspath(path)
 
 def mkdir(path):

cloudvolume/secrets.py

@@ -8,7 +8,8 @@
 
 from .lib import mkdir, colorize
 
-CLOUD_VOLUME_DIR = mkdir(os.path.join(os.environ['HOME'], '.cloudvolume'))
+HOME = os.path.expanduser('~')
+CLOUD_VOLUME_DIR = mkdir(os.path.join(HOME, '.cloudvolume'))
 
 def secretpath(filepath):
   preferred = os.path.join(CLOUD_VOLUME_DIR, filepath)
@@ -17,7 +18,7 @@ def secretpath(filepath):
     return preferred
 
   backcompat = [
-    os.path.join(os.environ['HOME'], '.neuroglancer'), # older
+    os.path.join(HOME, '.neuroglancer'), # older
     '/' # original
   ]
 

cloudvolume/sharedmemory.py

@@ -4,16 +4,12 @@
 import mmap
 import os
 import sys
+import time
 
 import multiprocessing as mp
 
 from six.moves import range
-import posix_ipc
-from posix_ipc import O_CREAT
 import numpy as np
-import psutil
-
-import time
 
 from .lib import Bbox, Vec, mkdir
 
@@ -121,6 +117,10 @@ def ndarray_fs(
 
 def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs):
   """Create a shared memory numpy array. Requires /dev/shm to exist."""
+  import posix_ipc
+  from posix_ipc import O_CREAT
+  import psutil
+
   nbytes = Vec(*shape).rectVolume() * np.dtype(dtype).itemsize
   available = psutil.virtual_memory().available
 
@@ -182,6 +182,7 @@ def unlink(location):
   return unlink_shm(location)
 
 def unlink_shm(location):
+  import posix_ipc
   try:
     posix_ipc.unlink_shared_memory(location)
   except posix_ipc.ExistentialError:

cloudvolume/storage/storage.py

@@ -3,6 +3,7 @@
 from collections import defaultdict
 import json
 import os.path
+import posixpath
 import re
 from functools import partial
 
@@ -48,7 +49,7 @@ def get_connection(self):
     return self._interface_cls(self._path)
 
   def get_path_to_file(self, file_path):
-    return os.path.join(self._layer_path, file_path)
+    return posixpath.join(self._layer_path, file_path)
 
   def put_json(self, file_path, content, content_type='application/json', *args, **kwargs):
     if type(content) != str:
@@ -453,7 +454,7 @@ def layer_path(self):
     return self._layer_path
 
   def get_path_to_file(self, file_path):
-    return os.path.join(self._layer_path, file_path)
+    return posixpath.join(self._layer_path, file_path)
 
   def put_json(self, file_path, content, content_type='application/json', *args, **kwargs):
     if type(content) != str:

cloudvolume/storage/storage_interfaces.py

@@ -2,6 +2,7 @@
 from collections import defaultdict
 import json
 import os.path
+import posixpath
 import re
 
 import boto3
@@ -170,7 +171,7 @@ def __init__(self, path):
     self._bucket = GC_POOL[path.bucket].get_connection()
 
   def get_path_to_file(self, file_path):
-    return os.path.join(self._path.path, file_path)
+    return posixpath.join(self._path.path, file_path)
 
   @retry
   def put_file(self, file_path, content, content_type, compress, cache_control=None):
@@ -251,7 +252,7 @@ def list_files(self, prefix, flat=False):
     prefix.
     """
     layer_path = self.get_path_to_file("")        
-    path = os.path.join(layer_path, prefix)
+    path = posixpath.join(layer_path, prefix)
     for blob in self._bucket.list_blobs(prefix=path):
       filename = blob.name.replace(layer_path, '')
       if not flat and filename[-1] != '/':
@@ -269,7 +270,7 @@ def __init__(self, path):
     self._path = path
 
   def get_path_to_file(self, file_path):
-    path = os.path.join(
+    path = posixpath.join(
       self._path.bucket, self._path.path, file_path
     )
     return self._path.protocol + '://' + path
@@ -315,7 +316,7 @@ def __init__(self, path):
     self._conn = S3_POOL[path.protocol][path.bucket].get_connection()
 
   def get_path_to_file(self, file_path):
-    return os.path.join(self._path.path, file_path)
+    return posixpath.join(self._path.path, file_path)
 
   @retry
   def put_file(self, file_path, content, content_type, compress, cache_control=None):
@@ -410,7 +411,7 @@ def list_files(self, prefix, flat=False):
     """
 
     layer_path = self.get_path_to_file("")        
-    path = os.path.join(layer_path, prefix)
+    path = posixpath.join(layer_path, prefix)
 
     @retry
     def s3lst(continuation_token=None):

requirements.txt

@@ -9,14 +9,12 @@ google-cloud-storage>=1.10.0
 intern>=0.9.11
 json5==0.5.1
 numpy>=1.13.3
-posix_ipc==1.0.4
-psutil==5.4.3
 pytest>=3.3.1 
 python-jsonschema-objects>=0.3.3
 Pillow>=4.2.1
 protobuf>=3.3.0
 requests>=2.18.4
 six>=1.10.0
 tenacity>=4.10.0
--e git+https://github.com/seung-lab/tqdm.git#egg=tqdm
+tqdm
 urllib3[secure]==1.24.2