Allow working with remote files in CWL and WDL workflows

Makefile

@@ -135,6 +135,10 @@ test: check_venv check_build_reqs
 	TOIL_OWNER_TAG="shared" \
 	    python -m pytest --durations=0 --strict-markers --log-level DEBUG --log-cli-level INFO -r s $(cov) -n $(threads) --dist loadscope $(tests) -m "$(marker)"
 
+test_debug: check_venv check_build_reqs
+	TOIL_OWNER_TAG="$(whoami)" \
+	    python -m pytest --durations=0 --strict-markers --log-level DEBUG -s -o log_cli=true --log-cli-level DEBUG -r s $(tests) -m "$(marker)" --tb=native --maxfail=1
+
 
 # This target will skip building docker and all docker based tests
 # these are our travis tests; rename?

src/toil/cwl/utils.py

@@ -16,6 +16,9 @@
 
 import logging
 import os
+from pathlib import PurePosixPath
+import posixpath
+import stat
 from typing import (
     Any,
     Callable,
@@ -31,6 +34,7 @@
 
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
+from toil.jobStores.abstractJobStore import AbstractJobStore
 
 logger = logging.getLogger(__name__)
 
@@ -128,6 +132,32 @@ def visit_cwl_class_and_reduce(
 
 DirectoryStructure = Dict[str, Union[str, "DirectoryStructure"]]
 
+def get_from_structure(dir_dict: DirectoryStructure, path: str) -> Union[str, DirectoryStructure, None]:
+    """
+    Given a relative path, follow it in the given directory structure.
+
+    Return the string URI for files, the directory dict for
+    subdirectories, or None for nonexistent things.
+    """
+
+    # Resolve .. and split into path components
+    parts = PurePosixPath(posixpath.normpath(path)).parts
+    if len(parts) == 0:
+        return dir_dict
+    if parts[0] in ('..', '/'):
+        raise RuntimeError(f"Path {path} not resolvable in virtual directory")
+    found: Union[str, DirectoryStructure] = dir_dict
+    for part in parts:
+        # Go down by each path component in turn
+        if isinstance(found, str):
+            # Looking for a subdirectory of a file, which doesn't exist
+            return None
+        if part not in found:
+            return None
+        found = found[part]
+    # Now we're at the place we want to be.
+    return found
+
 
 def download_structure(
     file_store: AbstractFileStore,
@@ -140,11 +170,12 @@ def download_structure(
     Download nested dictionary from the Toil file store to a local path.
 
     Guaranteed to fill the structure with real files, and not symlinks out of
-    it to elsewhere.
+    it to elsewhere. File URIs may be toilfile: URIs or any other URI that
+    Toil's job store system can read.
 
     :param file_store: The Toil file store to download from.
 
-    :param index: Maps from downloaded file path back to input Toil URI.
+    :param index: Maps from downloaded file path back to input URI.
 
     :param existing: Maps from file_store_id URI to downloaded file path.
 
@@ -171,16 +202,24 @@ def download_structure(
             # This must be a file path uploaded to Toil.
             if not isinstance(value, str):
                 raise RuntimeError(f"Did not find a file at {value}.")
-            if not value.startswith("toilfile:"):
-                raise RuntimeError(f"Did not find a filestore file at {value}")
+
             logger.debug("Downloading contained file '%s'", name)
             dest_path = os.path.join(into_dir, name)
-            # So download the file into place.
-            # Make sure to get a real copy of the file because we may need to
-            # mount the directory into a container as a whole.
-            file_store.readGlobalFile(
-                FileID.unpack(value[len("toilfile:") :]), dest_path, symlink=False
-            )
+
+            if value.startswith("toilfile:"):
+                # So download the file into place.
+                # Make sure to get a real copy of the file because we may need to
+                # mount the directory into a container as a whole.
+                file_store.readGlobalFile(
+                    FileID.unpack(value[len("toilfile:") :]), dest_path, symlink=False
+                )
+            else:
+                # We need to download from some other kind of URL.
+                size, executable = AbstractJobStore.read_from_url(value, open(dest_path, 'wb'))
+                if executable:
+                    # Make the written file executable
+                    os.chmod(dest_path, os.stat(dest_path).st_mode | stat.S_IXUSR)
+
             # Update the index dicts
             # TODO: why?
             index[dest_path] = value

src/toil/fileStores/abstractFileStore.py

@@ -26,12 +26,14 @@
                     Generator,
                     Iterator,
                     List,
+                    Literal,
                     Optional,
                     Set,
                     Tuple,
                     Type,
                     Union,
-                    cast)
+                    cast,
+                    overload)
 
 import dill
 
@@ -413,6 +415,21 @@ def readGlobalFile(
         """
         raise NotImplementedError()
 
+    @overload
+    def readGlobalFileStream(
+        self,
+        fileStoreID: str,
+        encoding: Literal[None] = None,
+        errors: Optional[str] = None,
+    ) -> ContextManager[IO[bytes]]:
+        ...
+
+    @overload
+    def readGlobalFileStream(
+        self, fileStoreID: str, encoding: str, errors: Optional[str] = None
+    ) -> ContextManager[IO[str]]:
+        ...
+
     @abstractmethod
     def readGlobalFileStream(
         self,