+from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple
+
+from megfile.fs_path import FSPath, StatResult, _make_stat, fs_cwd, fs_glob, fs_glob_stat, fs_home, fs_iglob, fs_lstat, fs_makedirs, fs_move, fs_path_join, fs_readlink, fs_rename, fs_resolve, is_fs
+from megfile.interfaces import Access, FileEntry, PathLike, StatResult
+
+__all__ = [
+ 'is_fs',
+ 'StatResult',
+ 'fs_path_join',
+ '_make_stat',
+ 'fs_readlink',
+ 'fs_cwd',
+ 'fs_home',
+ 'fs_iglob',
+ 'fs_glob',
+ 'fs_glob_stat',
+ 'fs_rename',
+ 'fs_resolve',
+ 'fs_move',
+ 'fs_makedirs',
+ 'fs_lstat',
+ 'fs_isabs',
+ 'fs_abspath',
+ 'fs_access',
+ 'fs_exists',
+ 'fs_getmtime',
+ 'fs_getsize',
+ 'fs_expanduser',
+ 'fs_isdir',
+ 'fs_isfile',
+ 'fs_listdir',
+ 'fs_load_from',
+ 'fs_realpath',
+ 'fs_relpath',
+ 'fs_remove',
+ 'fs_scan',
+ 'fs_scan_stat',
+ 'fs_scandir',
+ 'fs_stat',
+ 'fs_unlink',
+ 'fs_walk',
+ 'fs_getmd5',
+ 'fs_copy',
+ 'fs_sync',
+ 'fs_symlink',
+ 'fs_islink',
+ 'fs_ismount',
+ 'fs_save_as',
+]
+
+
+[docs]def fs_isabs(path: PathLike) -> bool:
+ '''Test whether a path is absolute
+
+ :param path: Given path
+ :returns: True if a path is absolute, else False
+ '''
+ return FSPath(path).is_absolute()
+
+
+[docs]def fs_abspath(path: PathLike) -> str:
+ '''Return the absolute path of given path
+
+ :param path: Given path
+ :returns: Absolute path of given path
+ '''
+ return FSPath(path).abspath()
+
+
+[docs]def fs_access(path: PathLike, mode: Access = Access.READ) -> bool:
+ '''
+ Test if path has access permission described by mode
+ Using ``os.access``
+
+ :param path: Given path
+ :param mode: access mode
+ :returns: Access: Enum, the read/write access that path has.
+ '''
+ return FSPath(path).access(mode)
+
+
+[docs]def fs_exists(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if the path exists
+
+ .. note::
+
+ The difference between this function and ``os.path.exists`` is that this function regard symlink as file.
+ In other words, this function is equal to ``os.path.lexists``
+
+ :param path: Given path
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path exists, else False
+
+ '''
+ return FSPath(path).exists(followlinks)
+
+
+[docs]def fs_getmtime(path: PathLike, follow_symlinks: bool = False) -> float:
+ '''
+ Get last-modified time of the file on the given path (in Unix timestamp format).
+ If the path is an existent directory, return the latest modified time of all file in it.
+
+ :param path: Given path
+ :returns: last-modified time
+ '''
+ return FSPath(path).getmtime(follow_symlinks)
+
+
+[docs]def fs_getsize(path: PathLike, follow_symlinks: bool = False) -> int:
+ '''
+ Get file size on the given file path (in bytes).
+ If the path in a directory, return the sum of all file size in it, including file in subdirectories (if exist).
+ The result excludes the size of directory itself. In other words, return 0 Byte on an empty directory path.
+
+ :param path: Given path
+ :returns: File size
+
+ '''
+ return FSPath(path).getsize(follow_symlinks)
+
+
+[docs]def fs_expanduser(path: PathLike):
+ '''Expand ~ and ~user constructions. If user or $HOME is unknown,
+ do nothing.
+ '''
+ return FSPath(path).expanduser()
+
+
+[docs]def fs_isdir(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if a path is directory
+
+ .. note::
+
+ The difference between this function and ``os.path.isdir`` is that this function regard symlink as file
+
+ :param path: Given path
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path is a directory, else False
+
+ '''
+ return FSPath(path).is_dir(followlinks)
+
+
+[docs]def fs_isfile(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if a path is file
+
+ .. note::
+
+ The difference between this function and ``os.path.isfile`` is that this function regard symlink as file
+
+ :param path: Given path
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path is a file, else False
+
+ '''
+ return FSPath(path).is_file(followlinks)
+
+
+[docs]def fs_listdir(path: PathLike) -> List[str]:
+ '''
+ Get all contents of given fs path. The result is in acsending alphabetical order.
+
+ :param path: Given path
+ :returns: All contents have in the path in acsending alphabetical order
+ '''
+ return FSPath(path).listdir()
+
+
+[docs]def fs_load_from(path: PathLike) -> BinaryIO:
+ '''Read all content on specified path and write into memory
+
+ User should close the BinaryIO manually
+
+ :param path: Given path
+ :returns: Binary stream
+ '''
+ return FSPath(path).load()
+
+
+[docs]def fs_realpath(path: PathLike) -> str:
+ '''Return the real path of given path
+
+ :param path: Given path
+ :returns: Real path of given path
+ '''
+ return FSPath(path).realpath()
+
+
+[docs]def fs_relpath(path: PathLike, start: Optional[str] = None) -> str:
+ '''Return the relative path of given path
+
+ :param path: Given path
+ :param start: Given start directory
+ :returns: Relative path from start
+ '''
+ return FSPath(path).relpath(start)
+
+
+[docs]def fs_remove(path: PathLike, missing_ok: bool = False) -> None:
+ '''
+ Remove the file or directory on fs
+
+ :param path: Given path
+ :param missing_ok: if False and target file/directory not exists, raise FileNotFoundError
+ '''
+ return FSPath(path).remove(missing_ok)
+
+
+[docs]def fs_scan(path: PathLike, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[str]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a path string.
+
+ If path is a file path, yields the file only
+ If path is a non-existent path, return an empty generator
+ If path is a bucket path, return all file paths in the bucket
+
+ :param path: Given path
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :returns: A file path generator
+ '''
+ return FSPath(path).scan(missing_ok, followlinks)
+
+
+[docs]def fs_scan_stat(
+ path: PathLike, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a tuple of path string and file stat
+
+ :param path: Given path
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :returns: A file path generator
+ '''
+ return FSPath(path).scan_stat(missing_ok, followlinks)
+
+
+[docs]def fs_scandir(path: PathLike) -> Iterator[FileEntry]:
+ '''
+ Get all content of given file path.
+
+ :param path: Given path
+ :returns: An iterator contains all contents have prefix path
+ '''
+ return FSPath(path).scandir()
+
+
+[docs]def fs_stat(path: PathLike, follow_symlinks=True) -> StatResult:
+ '''
+ Get StatResult of file on fs, including file size and mtime, referring to fs_getsize and fs_getmtime
+
+ :param path: Given path
+ :returns: StatResult
+ '''
+ return FSPath(path).stat(follow_symlinks)
+
+
+[docs]def fs_unlink(path: PathLike, missing_ok: bool = False) -> None:
+ '''
+ Remove the file on fs
+
+ :param path: Given path
+ :param missing_ok: if False and target file not exists, raise FileNotFoundError
+ '''
+ return FSPath(path).unlink(missing_ok)
+
+
+[docs]def fs_walk(path: PathLike, followlinks: bool = False
+ ) -> Iterator[Tuple[str, List[str], List[str]]]:
+ '''
+ Generate the file names in a directory tree by walking the tree top-down.
+ For each directory in the tree rooted at directory path (including path itself),
+ it yields a 3-tuple (root, dirs, files).
+
+ root: a string of current path
+ dirs: name list of subdirectories (excluding '.' and '..' if they exist) in 'root'. The list is sorted by ascending alphabetical order
+ files: name list of non-directory files (link is regarded as file) in 'root'. The list is sorted by ascending alphabetical order
+
+ If path not exists, or path is a file (link is regarded as file), return an empty generator
+
+ .. note::
+
+ Be aware that setting ``followlinks`` to True can lead to infinite recursion if a link points to a parent directory of itself. fs_walk() does not keep track of the directories it visited already.
+
+ :param path: Given path
+ :param followlinks: False if regard symlink as file, else True
+ :returns: A 3-tuple generator
+ '''
+ return FSPath(path).walk(followlinks)
+
+
+[docs]def fs_getmd5(
+ path: PathLike, recalculate: bool = False, followlinks: bool = True):
+ '''
+ Calculate the md5 value of the file
+
+ :param path: Given path
+ :param recalculate: Ignore this parameter, just for compatibility
+ :param followlinks: Ignore this parameter, just for compatibility
+ returns: md5 of file
+ '''
+ return FSPath(path).md5(recalculate, followlinks)
+
+
+[docs]def fs_copy(
+ src_path: PathLike,
+ dst_path: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ followlinks: bool = False,
+ overwrite: bool = True):
+ ''' File copy on file system
+ Copy content (excluding meta date) of file on `src_path` to `dst_path`. `dst_path` must be a complete file name
+
+ .. note ::
+
+ The differences between this function and shutil.copyfile are:
+
+ 1. If parent directory of dst_path doesn't exist, create it
+
+ 2. Allow callback function, None by default. callback: Optional[Callable[[int], None]],
+
+ the int data is means the size (in bytes) of the written data that is passed periodically
+
+ 3. This function is thread-unsafe
+
+ :param src_path: Given path
+ :param dst_path: Target file path
+ :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+ :param followlinks: False if regard symlink as file, else True
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ return FSPath(src_path).copy(dst_path, callback, followlinks, overwrite)
+
+
+[docs]def fs_sync(
+ src_path: PathLike,
+ dst_path: PathLike,
+ followlinks: bool = False,
+ force: bool = False,
+ overwrite: bool = True) -> None:
+ '''Force write of everything to disk.
+
+ :param src_path: Given path
+ :param dst_path: Target file path
+ :param followlinks: False if regard symlink as file, else True
+ :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ return FSPath(src_path).sync(dst_path, followlinks, force, overwrite)
+
+
+[docs]def fs_symlink(src_path: PathLike, dst_path: PathLike) -> None:
+ '''
+ Create a symbolic link pointing to src_path named dst_path.
+
+ :param src_path: Given path
+ :param dst_path: Desination path
+ '''
+ return FSPath(src_path).symlink(dst_path)
+
+
+[docs]def fs_islink(path: PathLike) -> bool:
+ '''Test whether a path is a symbolic link
+
+ :param path: Given path
+ :return: If path is a symbolic link return True, else False
+ :rtype: bool
+ '''
+ return FSPath(path).is_symlink()
+
+
+[docs]def fs_ismount(path: PathLike) -> bool:
+ '''Test whether a path is a mount point
+
+ :param path: Given path
+ :returns: True if a path is a mount point, else False
+ '''
+ return FSPath(path).is_mount()
+
+
+[docs]def fs_save_as(file_object: BinaryIO, path: PathLike):
+ '''Write the opened binary stream to path
+ If parent directory of path doesn't exist, it will be created.
+
+ :param path: Given path
+ :param file_object: stream to be read
+ '''
+ return FSPath(path).save(file_object)
+
+import hashlib
+import io
+import os
+import pathlib
+import shutil
+from stat import S_ISDIR as stat_isdir
+from stat import S_ISLNK as stat_islnk
+from typing import IO, AnyStr, BinaryIO, Callable, Iterator, List, Optional, Tuple, Union
+
+from megfile.errors import _create_missing_ok_generator
+from megfile.interfaces import Access, ContextIterator, FileEntry, PathLike, StatResult
+from megfile.lib.compare import is_same_file
+from megfile.lib.glob import iglob
+from megfile.lib.url import get_url_scheme
+from megfile.utils import cachedproperty, calculate_md5
+
+from .interfaces import PathLike, URIPath
+from .lib.compat import fspath
+from .lib.joinpath import path_join
+from .smart_path import SmartPath
+
+__all__ = [
+ 'FSPath',
+ 'is_fs',
+ 'StatResult',
+ 'fs_path_join',
+ '_make_stat',
+ 'fs_readlink',
+ 'fs_cwd',
+ 'fs_home',
+ 'fs_iglob',
+ 'fs_glob',
+ 'fs_glob_stat',
+ 'fs_rename',
+ 'fs_resolve',
+ 'fs_move',
+ 'fs_makedirs',
+ 'fs_lstat',
+]
+
+
+def _make_stat(stat: os.stat_result) -> StatResult:
+ return StatResult(
+ size=stat.st_size,
+ ctime=stat.st_ctime,
+ mtime=stat.st_mtime,
+ isdir=stat_isdir(stat.st_mode),
+ islnk=stat_islnk(stat.st_mode),
+ extra=stat,
+ )
+
+
+[docs]def is_fs(path: Union["PathLike", int]) -> bool:
+ '''Test if a path is fs path
+
+ :param path: Path to be tested
+ :returns: True of a path is fs path, else False
+ '''
+ if isinstance(path, int):
+ return True
+ path = fspath(path)
+ scheme = get_url_scheme(path)
+ return scheme == '' or scheme == 'file'
+
+
+[docs]def fs_path_join(path: PathLike, *other_paths: PathLike) -> str:
+ return path_join(fspath(path), *map(fspath, other_paths))
+
+
+[docs]def fs_readlink(path) -> str:
+ '''
+ Return a string representing the path to which the symbolic link points.
+ :returns: Return a string representing the path to which the symbolic link points.
+ '''
+ return os.readlink(path)
+
+
+[docs]def fs_cwd() -> str:
+ '''Return current working directory
+
+ returns: Current working directory
+ '''
+ return os.getcwd()
+
+
+[docs]def fs_home():
+ '''Return the home directory
+
+ returns: Home directory path
+ '''
+ return os.path.expanduser('~')
+
+
+[docs]def fs_iglob(path: PathLike, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator[str]:
+ '''Return path iterator in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: An iterator contains paths match `pathname`
+ '''
+ for path in _create_missing_ok_generator(
+ iglob(fspath(path), recursive=recursive), missing_ok,
+ FileNotFoundError('No match any file: %r' % path)):
+ yield path
+
+
+[docs]def fs_glob(path: PathLike, recursive: bool = True,
+ missing_ok: bool = True) -> List[str]:
+ '''Return path list in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: A list contains paths match `pathname`
+ '''
+ return list(fs_iglob(path=path, recursive=recursive, missing_ok=missing_ok))
+
+
+[docs]def fs_glob_stat(
+ path: PathLike, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator[FileEntry]:
+ '''Return a list contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: A list contains tuples of path and file stat, in which paths match `pathname`
+ '''
+ for path in fs_iglob(path=path, recursive=recursive, missing_ok=missing_ok):
+ yield FileEntry(
+ os.path.basename(path), path, _make_stat(os.lstat(path)))
+
+
+def _fs_rename_file(
+ src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
+ '''
+ rename file on fs
+
+ :param src_path: Given path
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ src_path, dst_path = fspath(src_path), fspath(dst_path)
+
+ if not overwrite and os.path.exists(dst_path):
+ return
+
+ dst_dir = os.path.dirname(dst_path)
+ if dst_dir and dst_dir != ".":
+ os.makedirs(dst_dir, exist_ok=True)
+ shutil.move(src_path, dst_path)
+
+
+[docs]def fs_rename(
+ src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
+ '''
+ rename file on fs
+
+ :param src_path: Given path
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ src_path, dst_path = fspath(src_path), fspath(dst_path)
+ if os.path.isfile(src_path):
+ return _fs_rename_file(src_path, dst_path, overwrite)
+ else:
+ os.makedirs(dst_path, exist_ok=True)
+
+ with os.scandir(src_path) as entries:
+ for file_entry in entries:
+ src_file_path = file_entry.path
+ dst_file_path = dst_path
+ relative_path = os.path.relpath(src_file_path, start=src_path)
+ if relative_path and relative_path != '.':
+ dst_file_path = os.path.join(dst_file_path, relative_path)
+ if os.path.exists(dst_file_path) and file_entry.is_dir():
+ fs_rename(src_file_path, dst_file_path, overwrite)
+ else:
+ _fs_rename_file(src_file_path, dst_file_path, overwrite)
+
+ if os.path.isdir(src_path):
+ shutil.rmtree(src_path)
+ else:
+ os.remove(src_path)
+
+
+[docs]def fs_move(
+ src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
+ '''
+ rename file on fs
+
+ :param src_path: Given path
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ return fs_rename(src_path, dst_path, overwrite)
+
+
+[docs]def fs_resolve(path: PathLike) -> str:
+ '''Equal to fs_realpath, return the real path of given path
+
+ :param path: Given path
+ :returns: Real path of given path
+ '''
+ return FSPath(path).realpath()
+
+
+[docs]def fs_makedirs(path: PathLike, exist_ok: bool = False):
+ '''
+ make a directory on fs, including parent directory
+
+ If there exists a file on the path, raise FileExistsError
+
+ :param path: Given path
+ :param exist_ok: If False and target directory exists, raise FileExistsError
+ :raises: FileExistsError
+ '''
+ return FSPath(path).mkdir(parents=True, exist_ok=exist_ok)
+
+
+[docs]def fs_lstat(path: PathLike) -> StatResult:
+ '''
+ Like Path.stat() but, if the path points to a symbolic link, return the symbolic link’s information rather than its target’s.
+
+ :param path: Given path
+ :returns: StatResult
+ '''
+ return FSPath(path).lstat()
+
+
+[docs]@SmartPath.register
+class FSPath(URIPath):
+ """file protocol
+ e.g. file:///data/test/ or /data/test
+ """
+
+ protocol = "file"
+
+ def __init__(self, path: Union["PathLike", int], *other_paths: "PathLike"):
+ if not isinstance(path, int):
+ if len(other_paths) > 0:
+ path = self.from_path(path).joinpath(*other_paths)
+ path = str(path)
+ self.path = path
+
+ def __fspath__(self) -> str:
+ return os.path.normpath(self.path_without_protocol)
+
+[docs] @cachedproperty
+ def root(self) -> str:
+ return pathlib.Path(self.path_without_protocol).root
+
+[docs] @cachedproperty
+ def anchor(self) -> str:
+ return pathlib.Path(self.path_without_protocol).anchor
+
+[docs] @cachedproperty
+ def drive(self) -> str:
+ return pathlib.Path(self.path_without_protocol).drive
+
+
+
+ @property
+ def path_with_protocol(self) -> Union[str, int]:
+ if isinstance(self.path, int):
+ return self.path
+ protocol_prefix = self.protocol + "://"
+ if self.path.startswith(protocol_prefix):
+ return self.path
+ return protocol_prefix + self.path
+
+[docs] def is_absolute(self) -> bool:
+ '''Test whether a path is absolute
+
+ :returns: True if a path is absolute, else False
+ '''
+ return os.path.isabs(self.path_without_protocol)
+
+[docs] def abspath(self) -> str:
+ '''Return the absolute path of given path
+
+ :returns: Absolute path of given path
+ '''
+ return fspath(os.path.abspath(self.path_without_protocol))
+
+[docs] def access(self, mode: Access = Access.READ) -> bool:
+ '''
+ Test if path has access permission described by mode
+ Using ``os.access``
+
+ :param mode: access mode
+ :returns: Access: Enum, the read/write access that path has.
+ '''
+ if not isinstance(mode, Access):
+ raise TypeError(
+ 'Unsupported mode: {} -- Mode should use one of the enums belonging to: {}'
+ .format(mode, ', '.join([str(a) for a in Access])))
+ if mode == Access.READ:
+ return os.access(self.path_without_protocol, os.R_OK)
+ if mode == Access.WRITE:
+ return os.access(self.path_without_protocol, os.W_OK)
+
+[docs] def exists(self, followlinks: bool = False) -> bool:
+ '''
+ Test if the path exists
+
+ .. note::
+
+ The difference between this function and ``os.path.exists`` is that this function regard symlink as file.
+ In other words, this function is equal to ``os.path.lexists``
+
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path exists, else False
+
+ '''
+ if followlinks:
+ return os.path.exists(self.path_without_protocol)
+ return os.path.lexists(self.path_without_protocol)
+
+[docs] def getmtime(self, follow_symlinks: bool = False) -> float:
+ '''
+ Get last-modified time of the file on the given path (in Unix timestamp format).
+ If the path is an existent directory, return the latest modified time of all file in it.
+
+ :returns: last-modified time
+ '''
+ return self.stat(follow_symlinks=follow_symlinks).mtime
+
+[docs] def getsize(self, follow_symlinks: bool = False) -> int:
+ '''
+ Get file size on the given file path (in bytes).
+ If the path in a directory, return the sum of all file size in it, including file in subdirectories (if exist).
+ The result excludes the size of directory itself. In other words, return 0 Byte on an empty directory path.
+
+ :returns: File size
+
+ '''
+ return self.stat(follow_symlinks=follow_symlinks).size
+
+[docs] def glob(self, pattern, recursive: bool = True,
+ missing_ok: bool = True) -> List['FSPath']:
+ '''Return path list in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: A list contains paths match `pathname`
+ '''
+ return list(
+ self.iglob(
+ pattern=pattern, recursive=recursive, missing_ok=missing_ok))
+
+[docs] def glob_stat(
+ self, pattern, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator[FileEntry]:
+ '''Return a list contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: A list contains tuples of path and file stat, in which paths match `pathname`
+ '''
+ for path_obj in self.iglob(pattern=pattern, recursive=recursive,
+ missing_ok=missing_ok):
+ yield FileEntry(
+ path_obj.name, path_obj.path,
+ _make_stat(os.lstat(path_obj.path)))
+
+[docs] def expanduser(self):
+ '''Expand ~ and ~user constructions. If user or $HOME is unknown,
+ do nothing.
+ '''
+ return os.path.expanduser(self.path_without_protocol)
+
+[docs] def iglob(self, pattern, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator['FSPath']:
+ '''Return path iterator in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: An iterator contains paths match `pathname`
+ '''
+ glob_path = self.path_without_protocol
+ if pattern:
+ glob_path = self.joinpath(pattern).path_without_protocol
+ for path in fs_iglob(glob_path, recursive=recursive,
+ missing_ok=missing_ok):
+ yield self.from_path(path)
+
+[docs] def is_dir(self, followlinks: bool = False) -> bool:
+ '''
+ Test if a path is directory
+
+ .. note::
+
+ The difference between this function and ``os.path.isdir`` is that this function regard symlink as file
+
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path is a directory, else False
+
+ '''
+ if os.path.islink(self.path_without_protocol) and not followlinks:
+ return False
+ return os.path.isdir(self.path_without_protocol)
+
+[docs] def is_file(self, followlinks: bool = False) -> bool:
+ '''
+ Test if a path is file
+
+ .. note::
+
+ The difference between this function and ``os.path.isfile`` is that this function regard symlink as file
+
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path is a file, else False
+
+ '''
+ if os.path.islink(self.path_without_protocol) and not followlinks:
+ return True
+ return os.path.isfile(self.path_without_protocol)
+
+[docs] def listdir(self) -> List[str]:
+ '''
+ Get all contents of given fs path. The result is in acsending alphabetical order.
+
+ :returns: All contents have in the path in acsending alphabetical order
+ '''
+ return sorted(os.listdir(self.path_without_protocol))
+
+[docs] def iterdir(self) -> Iterator['FSPath']:
+ '''
+ Get all contents of given fs path. The result is in acsending alphabetical order.
+
+ :returns: All contents have in the path in acsending alphabetical order
+ '''
+ for path in self.listdir():
+ yield self.joinpath(path) # type: ignore
+
+[docs] def load(self) -> BinaryIO:
+ '''Read all content on specified path and write into memory
+
+ User should close the BinaryIO manually
+
+ :returns: Binary stream
+ '''
+ with open(self.path_without_protocol, 'rb') as f:
+ data = f.read()
+ return io.BytesIO(data)
+
+[docs] def mkdir(self, mode=0o777, parents: bool = False, exist_ok: bool = False):
+ '''
+ make a directory on fs, including parent directory
+
+ If there exists a file on the path, raise FileExistsError
+
+ :param mode: If mode is given, it is combined with the process’ umask value to determine the file mode and access flags.
+ :param parents: If parents is true, any missing parents of this path are created as needed;
+ If parents is false (the default), a missing parent raises FileNotFoundError.
+ :param exist_ok: If False and target directory exists, raise FileExistsError
+ :raises: FileExistsError
+ '''
+ if exist_ok and self.path_without_protocol == '':
+ return
+ return pathlib.Path(self.path_without_protocol).mkdir(
+ mode=mode, parents=parents, exist_ok=exist_ok)
+
+[docs] def realpath(self) -> str:
+ '''Return the real path of given path
+
+ :returns: Real path of given path
+ '''
+ return fspath(os.path.realpath(self.path_without_protocol))
+
+[docs] def relpath(self, start: Optional[str] = None) -> str:
+ '''Return the relative path of given path
+
+ :param start: Given start directory
+ :returns: Relative path from start
+ '''
+ return fspath(os.path.relpath(self.path_without_protocol, start=start))
+
+[docs] def rename(self, dst_path: PathLike, overwrite: bool = True) -> 'FSPath':
+ '''
+ rename file on fs
+
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ fs_rename(self.path_without_protocol, dst_path, overwrite)
+ return self.from_path(dst_path)
+
+[docs] def replace(self, dst_path: PathLike, overwrite: bool = True) -> 'FSPath':
+ '''
+ move file on fs
+
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ return self.rename(dst_path=dst_path, overwrite=overwrite)
+
+[docs] def remove(self, missing_ok: bool = False) -> None:
+ '''
+ Remove the file or directory on fs
+
+ :param missing_ok: if False and target file/directory not exists, raise FileNotFoundError
+ '''
+ if missing_ok and not self.exists():
+ return
+ if self.is_dir():
+ shutil.rmtree(self.path_without_protocol)
+ else:
+ os.remove(self.path_without_protocol)
+
+ def _scan(self, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[str]:
+ if self.is_file(followlinks=followlinks):
+ path = fspath(self.path_without_protocol)
+ yield path
+
+ for root, _, files in self.walk(followlinks=followlinks):
+ for filename in files:
+ yield os.path.join(root, filename)
+
+[docs] def scan(self, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[str]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a path string.
+
+ If path is a file path, yields the file only
+ If path is a non-existent path, return an empty generator
+ If path is a bucket path, return all file paths in the bucket
+
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :returns: A file path generator
+ '''
+ return _create_missing_ok_generator(
+ self._scan(followlinks=followlinks), missing_ok,
+ FileNotFoundError('No match any file in: %r' % self.path))
+
+[docs] def scan_stat(self, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a tuple of path string and file stat
+
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :returns: A file path generator
+ '''
+ file_not_found = True
+ for path in self._scan(followlinks=followlinks):
+ yield FileEntry(
+ os.path.basename(path), path, _make_stat(os.lstat(path)))
+ file_not_found = False
+ if file_not_found:
+ if missing_ok:
+ return
+ raise FileNotFoundError(
+ 'No match any file in: %r' % self.path_without_protocol)
+
+[docs] def scandir(self) -> Iterator[FileEntry]:
+ '''
+ Get all content of given file path.
+
+ :returns: An iterator contains all contents have prefix path
+ '''
+
+ def create_generator():
+ with os.scandir(self.path_without_protocol) as entries:
+ for entry in entries:
+ yield FileEntry(
+ entry.name, entry.path,
+ _make_stat(entry.stat(follow_symlinks=False)))
+
+ return ContextIterator(create_generator())
+
+[docs] def stat(self, follow_symlinks=True) -> StatResult:
+ '''
+ Get StatResult of file on fs, including file size and mtime, referring to fs_getsize and fs_getmtime
+
+ :returns: StatResult
+ '''
+ if follow_symlinks:
+ result = _make_stat(os.stat(self.path_without_protocol))
+ else:
+ result = _make_stat(os.lstat(self.path_without_protocol))
+
+ if result.islnk or not result.isdir:
+ return result
+
+ size = 0
+ ctime = result.ctime
+ mtime = result.mtime
+ for root, _, files in os.walk(self.path_without_protocol):
+ for filename in files:
+ canonical_path = os.path.join(root, filename)
+ stat = os.lstat(canonical_path)
+ size += stat.st_size
+ if ctime > stat.st_ctime:
+ ctime = stat.st_ctime
+ if mtime < stat.st_mtime:
+ mtime = stat.st_mtime
+ return result._replace(size=size, ctime=ctime, mtime=mtime)
+
+[docs] def unlink(self, missing_ok: bool = False) -> None:
+ '''
+ Remove the file on fs
+
+ :param missing_ok: if False and target file not exists, raise FileNotFoundError
+ '''
+ if missing_ok and not self.exists():
+ return
+ os.unlink(self.path_without_protocol)
+
+[docs] def walk(self, followlinks: bool = False
+ ) -> Iterator[Tuple[str, List[str], List[str]]]:
+ '''
+ Generate the file names in a directory tree by walking the tree top-down.
+ For each directory in the tree rooted at directory path (including path itself),
+ it yields a 3-tuple (root, dirs, files).
+
+ root: a string of current path
+ dirs: name list of subdirectories (excluding '.' and '..' if they exist) in 'root'. The list is sorted by ascending alphabetical order
+ files: name list of non-directory files (link is regarded as file) in 'root'. The list is sorted by ascending alphabetical order
+
+ If path not exists, or path is a file (link is regarded as file), return an empty generator
+
+ .. note::
+
+ Be aware that setting ``followlinks`` to True can lead to infinite recursion if a link points to a parent directory of itself. fs_walk() does not keep track of the directories it visited already.
+
+ :param followlinks: False if regard symlink as file, else True
+ :returns: A 3-tuple generator
+ '''
+ if not self.exists(followlinks=followlinks):
+ return
+
+ if self.is_file(followlinks=followlinks):
+ return
+
+ path = fspath(self.path_without_protocol)
+ path = os.path.normpath(self.path_without_protocol)
+
+ stack = [path]
+ while stack:
+ root = stack.pop()
+ dirs, files = [], []
+ for entry in os.scandir(root):
+ name = fspath(entry.name)
+ path = entry.path
+ if FSPath(path).is_file(followlinks=followlinks):
+ files.append(name)
+ elif FSPath(path).is_dir(followlinks=followlinks):
+ dirs.append(name)
+
+ dirs = sorted(dirs)
+ files = sorted(files)
+
+ yield root, dirs, files
+
+ stack.extend(
+ (os.path.join(root, directory) for directory in reversed(dirs)))
+
+[docs] def resolve(self, strict=False) -> 'FSPath':
+ '''Equal to fs_realpath
+
+ :return: Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.
+ :rtype: FSPath
+ '''
+ return self.from_path(
+ fspath(
+ pathlib.Path(
+ self.path_without_protocol).resolve(strict=strict)))
+
+[docs] def md5(self, recalculate: bool = False, followlinks: bool = True):
+ '''
+ Calculate the md5 value of the file
+
+ :param recalculate: Ignore this parameter, just for compatibility
+ :param followlinks: Ignore this parameter, just for compatibility
+ returns: md5 of file
+ '''
+ if os.path.isdir(self.path_without_protocol):
+ hash_md5 = hashlib.md5() # nosec
+ for file_name in self.listdir():
+ chunk = FSPath(self.path_without_protocol, file_name).md5(
+ recalculate=recalculate, followlinks=followlinks).encode()
+ hash_md5.update(chunk)
+ return hash_md5.hexdigest()
+ with open(self.path_without_protocol, 'rb') as src: # type: ignore
+ md5 = calculate_md5(src)
+ return md5
+
+ def _copyfile(
+ self,
+ dst_path: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ followlinks: bool = False):
+
+ shutil.copy2(
+ self.path_without_protocol, dst_path, follow_symlinks=followlinks)
+
+ # After python3.8, patch `shutil.copyfile` is not a good way, because `shutil.copy2` will not call it in some cases.
+ if callback:
+ callback(self.stat(follow_symlinks=followlinks).size)
+
+[docs] def copy(
+ self,
+ dst_path: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ followlinks: bool = False,
+ overwrite: bool = True):
+ ''' File copy on file system
+ Copy content (excluding meta date) of file on `src_path` to `dst_path`. `dst_path` must be a complete file name
+
+ .. note ::
+
+ The differences between this function and shutil.copyfile are:
+
+ 1. If parent directory of dst_path doesn't exist, create it
+
+ 2. Allow callback function, None by default. callback: Optional[Callable[[int], None]],
+
+ the int data is means the size (in bytes) of the written data that is passed periodically
+
+ 3. This function is thread-unsafe
+
+ :param dst_path: Target file path
+ :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+ :param followlinks: False if regard symlink as file, else True
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ if not overwrite and os.path.exists((dst_path)):
+ return
+
+ try:
+ self._copyfile(dst_path, callback=callback, followlinks=followlinks)
+ except FileNotFoundError as error:
+ # Prevent the dst_path directory from being created when src_path does not exist
+ if dst_path == error.filename:
+ FSPath(os.path.dirname(dst_path)).mkdir(
+ parents=True, exist_ok=True)
+ self._copyfile(
+ dst_path, callback=callback, followlinks=followlinks)
+ else:
+ raise
+
+[docs] def sync(
+ self,
+ dst_path: PathLike,
+ followlinks: bool = False,
+ force: bool = False,
+ overwrite: bool = True) -> None:
+ '''Force write of everything to disk.
+
+ :param dst_path: Target file path
+ :param followlinks: False if regard symlink as file, else True
+ :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ if self.is_dir(followlinks=followlinks):
+
+ def ignore_same_file(src: str, names: List[str]) -> List[str]:
+ ignore_files = []
+ for name in names:
+ dst_obj = self.from_path(dst_path).joinpath(name)
+ if force:
+ pass
+ elif not overwrite and dst_obj.exists():
+ ignore_files.append(name)
+ elif dst_obj.exists() and is_same_file(
+ self.joinpath(name).stat(), dst_obj.stat(), 'copy'):
+ ignore_files.append(name)
+ return ignore_files
+
+ shutil.copytree(
+ self.path_without_protocol,
+ dst_path,
+ ignore=ignore_same_file,
+ dirs_exist_ok=True)
+ else:
+ self.copy(dst_path, followlinks=followlinks, overwrite=overwrite)
+
+[docs] def symlink(self, dst_path: PathLike) -> None:
+ '''
+ Create a symbolic link pointing to src_path named dst_path.
+
+ :param dst_path: Desination path
+ '''
+ return os.symlink(self.path_without_protocol, dst_path)
+
+[docs] def readlink(self) -> 'FSPath':
+ '''
+ Return a FSPath instance representing the path to which the symbolic link points.
+ :returns: Return a FSPath instance representing the path to which the symbolic link points.
+ '''
+ return self.from_path(fs_readlink(self.path_without_protocol))
+
+[docs] def is_symlink(self) -> bool:
+ '''Test whether a path is a symbolic link
+
+ :return: If path is a symbolic link return True, else False
+ :rtype: bool
+ '''
+ return os.path.islink(self.path_without_protocol)
+
+[docs] def is_mount(self) -> bool:
+ '''Test whether a path is a mount point
+
+ :returns: True if a path is a mount point, else False
+ '''
+ return os.path.ismount(self.path_without_protocol)
+
+[docs] def cwd(self) -> 'FSPath':
+ '''Return current working directory
+
+ returns: Current working directory
+ '''
+ return self.from_path(fs_cwd())
+
+[docs] def home(self):
+ '''Return the home directory
+
+ returns: Home directory path
+ '''
+ return self.from_path(fs_home())
+
+[docs] def joinpath(self, *other_paths: PathLike) -> "FSPath":
+ path = fspath(self)
+ if path == '.':
+ path = ''
+ return self.from_path(path_join(path, *map(fspath, other_paths)))
+
+[docs] def save(self, file_object: BinaryIO):
+ '''Write the opened binary stream to path
+ If parent directory of path doesn't exist, it will be created.
+
+ :param file_object: stream to be read
+ '''
+ FSPath(os.path.dirname(self.path_without_protocol)).mkdir(
+ parents=True, exist_ok=True)
+ with open(self.path_without_protocol, 'wb') as output:
+ output.write(file_object.read())
+
+[docs] def open(
+ self,
+ mode: str = 'r',
+ buffering=-1,
+ encoding=None,
+ errors=None,
+ newline=None,
+ closefd=True,
+ **kwargs) -> IO[AnyStr]: # pytype: disable=signature-mismatch
+ if not isinstance(self.path_without_protocol, int) and ('w' in mode or
+ 'x' in mode or
+ 'a' in mode):
+ FSPath(os.path.dirname(self.path_without_protocol)).mkdir(
+ parents=True, exist_ok=True)
+ return io.open(
+ self.path_without_protocol,
+ mode,
+ buffering=buffering,
+ encoding=encoding,
+ errors=errors,
+ newline=newline,
+ closefd=closefd)
+
+[docs] @cachedproperty
+ def parts(self) -> Tuple[str]:
+ '''
+ A tuple giving access to the path’s various components
+ '''
+ return pathlib.Path(self.path_without_protocol).parts
+
+[docs] def chmod(self, mode: int, *, follow_symlinks: bool = True):
+ '''
+ Change the file mode and permissions, like os.chmod().
+
+ This method normally follows symlinks.
+ Some Unix flavours support changing permissions on the symlink itself;
+ on these platforms you may add the argument follow_symlinks=False, or use lchmod().
+ '''
+ return os.chmod(
+ path=self.path_without_protocol,
+ mode=mode,
+ follow_symlinks=follow_symlinks)
+
+[docs] def group(self) -> str:
+ '''
+ Return the name of the group owning the file. KeyError is raised if the file’s gid isn’t found in the system database.
+ '''
+ return pathlib.Path(self.path_without_protocol).group()
+
+[docs] def is_socket(self) -> bool:
+ '''
+ Return True if the path points to a Unix socket (or a symbolic link pointing to a Unix socket), False if it points to another kind of file.
+
+ False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
+ '''
+ return pathlib.Path(self.path_without_protocol).is_socket()
+
+[docs] def is_fifo(self) -> bool:
+ '''
+ Return True if the path points to a FIFO (or a symbolic link pointing to a FIFO), False if it points to another kind of file.
+
+ False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
+ '''
+ return pathlib.Path(self.path_without_protocol).is_fifo()
+
+[docs] def is_block_device(self) -> bool:
+ '''
+ Return True if the path points to a block device (or a symbolic link pointing to a block device), False if it points to another kind of file.
+
+ False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
+ '''
+ return pathlib.Path(self.path_without_protocol).is_block_device()
+
+[docs] def is_char_device(self) -> bool:
+ '''
+ Return True if the path points to a character device (or a symbolic link pointing to a character device), False if it points to another kind of file.
+
+ False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
+ '''
+ return pathlib.Path(self.path_without_protocol).is_char_device()
+
+[docs] def owner(self) -> str:
+ '''
+ Return the name of the user owning the file. KeyError is raised if the file’s uid isn’t found in the system database.
+ '''
+ return pathlib.Path(self.path_without_protocol).owner()
+
+[docs] def absolute(self) -> 'FSPath':
+ '''
+ Make the path absolute, without normalization or resolving symlinks. Returns a new path object
+ '''
+ return self.from_path(os.path.abspath(self.path_without_protocol))
+
+[docs] def rmdir(self):
+ '''
+ Remove this directory. The directory must be empty.
+ '''
+ return os.rmdir(self.path_without_protocol)
+
+[docs] def hardlink_to(self, target):
+ '''
+ Make this path a hard link to the same file as target.
+ '''
+ return os.link(target, self.path)
+
+[docs] def utime(self, atime: Union[float, int], mtime: Union[float, int]):
+ """
+ Set the access and modified times of the file specified by path.
+
+ :param atime: a float or int representing the access time to be set. If it is set to None, the
+ access time is set to the current time.
+ :param mtime: a float or int representing the modified time to be set. If it is set to None, the
+ modified time is set to the current time.
+ :return: None
+ """
+ return os.utime(self.path_without_protocol, times=(atime, mtime))
+
+from typing import IO, AnyStr, BinaryIO, Iterator, List, Optional, Tuple
+
+from megfile.hdfs_path import HdfsPath, hdfs_glob, hdfs_glob_stat, hdfs_iglob, hdfs_makedirs, is_hdfs
+from megfile.interfaces import FileEntry, PathLike, StatResult
+
+__all__ = [
+ 'is_hdfs',
+ 'hdfs_glob',
+ 'hdfs_glob_stat',
+ 'hdfs_iglob',
+ 'hdfs_makedirs',
+ 'hdfs_exists',
+ 'hdfs_stat',
+ 'hdfs_getmtime',
+ 'hdfs_getsize',
+ 'hdfs_isdir',
+ 'hdfs_isfile',
+ 'hdfs_listdir',
+ 'hdfs_load_from',
+ 'hdfs_move',
+ 'hdfs_remove',
+ 'hdfs_scan',
+ 'hdfs_scan_stat',
+ 'hdfs_scandir',
+ 'hdfs_unlink',
+ 'hdfs_walk',
+ 'hdfs_getmd5',
+ 'hdfs_save_as',
+ 'hdfs_open',
+]
+
+
+[docs]def hdfs_exists(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if path exists
+
+ If the bucket of path are not permitted to read, return False
+
+ :param path: Given path
+ :returns: True if path eixsts, else False
+ '''
+ return HdfsPath(path).exists(followlinks)
+
+
+[docs]def hdfs_stat(path: PathLike, follow_symlinks=True) -> StatResult:
+ '''
+ Get StatResult of path file, including file size and mtime, referring to hdfs_getsize and hdfs_getmtime
+
+ If path is not an existent path, which means hdfs_exist(path) returns False, then raise FileNotFoundError
+ If attempt to get StatResult of complete hdfs, such as hdfs_dir_url == 'hdfs://', raise BucketNotFoundError
+
+ :param path: Given path
+ :returns: StatResult
+ :raises: FileNotFoundError
+ '''
+ return HdfsPath(path).stat(follow_symlinks)
+
+
+[docs]def hdfs_getmtime(path: PathLike, follow_symlinks: bool = False) -> float:
+ '''
+ Get last-modified time of the file on the given path path (in Unix timestamp format).
+ If the path is an existent directory, return the latest modified time of all file in it. The mtime of empty directory is 1970-01-01 00:00:00
+
+ If path is not an existent path, which means hdfs_exist(path) returns False, then raise FileNotFoundError
+
+ :param path: Given path
+ :returns: Last-modified time
+ :raises: FileNotFoundError
+ '''
+ return HdfsPath(path).getmtime(follow_symlinks)
+
+
+[docs]def hdfs_getsize(path: PathLike, follow_symlinks: bool = False) -> int:
+ '''
+ Get file size on the given path path (in bytes).
+ If the path in a directory, return the sum of all file size in it, including file in subdirectories (if exist).
+ The result excludes the size of directory itself. In other words, return 0 Byte on an empty directory path.
+
+ If path is not an existent path, which means hdfs_exist(path) returns False, then raise FileNotFoundError
+
+ :param path: Given path
+ :returns: File size
+ :raises: FileNotFoundError
+ '''
+ return HdfsPath(path).getsize(follow_symlinks)
+
+
+[docs]def hdfs_isdir(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if an hdfs url is directory
+ Specific procedures are as follows:
+ If there exists a suffix, of which ``os.path.join(path, suffix)`` is a file
+ If the url is empty bucket or hdfs://
+
+ :param path: Given path
+ :param followlinks: whether followlinks is True or False, result is the same. Because hdfs symlink not support dir.
+ :returns: True if path is hdfs directory, else False
+ '''
+ return HdfsPath(path).is_dir(followlinks)
+
+
+[docs]def hdfs_isfile(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if an path is file
+
+ :param path: Given path
+ :returns: True if path is hdfs file, else False
+ '''
+ return HdfsPath(path).is_file(followlinks)
+
+
+[docs]def hdfs_listdir(path: PathLike, followlinks: bool = False) -> List[str]:
+ '''
+ Get all contents of given path.
+
+ :param path: Given path
+ :returns: All contents have prefix of path.
+ :raises: FileNotFoundError, NotADirectoryError
+ '''
+ return HdfsPath(path).listdir(followlinks)
+
+
+[docs]def hdfs_load_from(path: PathLike, followlinks: bool = False) -> BinaryIO:
+ '''Read all content in binary on specified path and write into memory
+
+ User should close the BinaryIO manually
+
+ :param path: Given path
+ :returns: BinaryIO
+ '''
+ return HdfsPath(path).load(followlinks)
+
+
+[docs]def hdfs_move(
+ src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
+ '''
+ Move file/directory path from src_path to dst_path
+
+ :param src_path: Given path
+ :param dst_path: Given destination path
+ '''
+ return HdfsPath(src_path).move(dst_path, overwrite)
+
+
+[docs]def hdfs_remove(path: PathLike, missing_ok: bool = False) -> None:
+ '''
+ Remove the file or directory on hdfs, `hdfs://` and `hdfs://bucket` are not permitted to remove
+
+ :param path: Given path
+ :param missing_ok: if False and target file/directory not exists, raise FileNotFoundError
+ :raises: FileNotFoundError, UnsupportedError
+ '''
+ return HdfsPath(path).remove(missing_ok)
+
+
+[docs]def hdfs_scan(
+ path: PathLike, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[str]:
+ '''
+ Iteratively traverse only files in given hdfs directory.
+ Every iteration on generator yields a path string.
+
+ If path is a file path, yields the file only
+ If path is a non-existent path, return an empty generator
+ If path is a bucket path, return all file paths in the bucket
+ If path is an empty bucket, return an empty generator
+ If path doesn't contain any bucket, which is path == 'hdfs://', raise UnsupportedError. walk() on complete hdfs is not supported in megfile
+
+ :param path: Given path
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :raises: UnsupportedError
+ :returns: A file path generator
+ '''
+ return HdfsPath(path).scan(missing_ok, followlinks)
+
+
+[docs]def hdfs_scan_stat(
+ path: PathLike, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Iteratively traverse only files in given directory.
+ Every iteration on generator yields a tuple of path string and file stat
+
+ :param path: Given path
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :raises: UnsupportedError
+ :returns: A file path generator
+ '''
+ return HdfsPath(path).scan_stat(missing_ok, followlinks)
+
+
+[docs]def hdfs_scandir(path: PathLike,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Get all contents of given path, the order of result is not guaranteed.
+
+ :param path: Given path
+ :returns: All contents have prefix of path
+ :raises: FileNotFoundError, NotADirectoryError
+ '''
+ return HdfsPath(path).scandir(followlinks)
+
+
+[docs]def hdfs_unlink(path: PathLike, missing_ok: bool = False) -> None:
+ '''
+ Remove the file on hdfs
+
+ :param path: Given path
+ :param missing_ok: if False and target file not exists, raise FileNotFoundError
+ :raises: FileNotFoundError, IsADirectoryError
+ '''
+ return HdfsPath(path).unlink(missing_ok)
+
+
+[docs]def hdfs_walk(path: PathLike, followlinks: bool = False
+ ) -> Iterator[Tuple[str, List[str], List[str]]]:
+ '''
+ Iteratively traverse the given hdfs directory, in top-bottom order. In other words, firstly traverse parent directory, if subdirectories exist, traverse the subdirectories.
+ Every iteration on generator yields a 3-tuple: (root, dirs, files)
+
+ - root: Current hdfs path;
+ - dirs: Name list of subdirectories in current directory.
+ - files: Name list of files in current directory.
+
+ If path is a file path, return an empty generator
+ If path is a non-existent path, return an empty generator
+ If path is a bucket path, bucket will be the top directory, and will be returned at first iteration of generator
+ If path is an empty bucket, only yield one 3-tuple (notes: hdfs doesn't have empty directory)
+ If path doesn't contain any bucket, which is path == 'hdfs://', raise UnsupportedError. walk() on complete hdfs is not supported in megfile
+
+ :param path: Given path
+ :param followlinks: whether followlinks is True or False, result is the same. Because hdfs not support symlink.
+ :returns: A 3-tuple generator
+ '''
+ return HdfsPath(path).walk(followlinks)
+
+
+[docs]def hdfs_getmd5(
+ path: PathLike, recalculate: bool = False,
+ followlinks: bool = False) -> str:
+ '''
+ Get checksum of the file or dir.
+
+ :param path: Given path
+ :param recalculate: Ignore this parameter, just for compatibility
+ :param followlinks: Ignore this parameter, just for compatibility
+ :returns: checksum
+ '''
+ return HdfsPath(path).md5(recalculate, followlinks)
+
+
+[docs]def hdfs_save_as(file_object: BinaryIO, path: PathLike):
+ '''Write the opened binary stream to specified path, but the stream won't be closed
+
+ :param path: Given path
+ :param file_object: Stream to be read
+ '''
+ return HdfsPath(path).save(file_object)
+
+
+[docs]def hdfs_open(
+ path: PathLike,
+ mode: str = 'r',
+ *,
+ buffering: Optional[int] = None,
+ encoding: Optional[str] = None,
+ errors: Optional[str] = None,
+ **kwargs) -> IO[AnyStr]: # pytype: disable=signature-mismatch
+ return HdfsPath(path).open(
+ mode, buffering=buffering, encoding=encoding, errors=errors)
+
+import hashlib
+import io
+import os
+import sys
+from functools import lru_cache
+from typing import IO, AnyStr, BinaryIO, Iterator, List, Optional, Tuple
+
+from megfile.errors import _create_missing_ok_generator, raise_hdfs_error
+from megfile.interfaces import FileEntry, PathLike, StatResult, URIPath
+from megfile.lib.compat import fspath
+from megfile.lib.glob import FSFunc, iglob
+from megfile.lib.hdfs_prefetch_reader import HdfsPrefetchReader
+from megfile.lib.hdfs_tools import hdfs_api
+from megfile.lib.url import get_url_scheme
+from megfile.pathlike import PathLike, URIPath
+from megfile.smart_path import SmartPath
+from megfile.utils import _is_pickle, cachedproperty
+
+__all__ = [
+ 'HdfsPath',
+ 'is_hdfs',
+ 'hdfs_glob',
+ 'hdfs_glob_stat',
+ 'hdfs_iglob',
+ 'hdfs_makedirs',
+]
+
+HDFS_USER = "HDFS_USER"
+HDFS_URL = "HDFS_URL"
+HDFS_ROOT = "HDFS_ROOT"
+HDFS_TIMEOUT = "HDFS_TIMEOUT"
+HDFS_TOKEN = "HDFS_TOKEN"
+HDFS_CONFIG_PATH = "HDFS_CONFIG_PATH"
+MAX_RETRIES = 10
+DEFAULT_HDFS_TIMEOUT = 10
+
+
+[docs]def is_hdfs(path: PathLike) -> bool: # pytype: disable=invalid-annotation
+ '''Test if a path is sftp path
+
+ :param path: Path to be tested
+ :returns: True of a path is sftp path, else False
+ '''
+ return fspath(path).startswith("hdfs://")
+
+
+def get_hdfs_config(profile_name: Optional[str] = None):
+ env_profile = f"{profile_name.upper()}__" if profile_name else ""
+ config = {
+ 'user': os.getenv(f"{env_profile}{HDFS_USER}"),
+ 'url': os.getenv(f"{env_profile}{HDFS_URL}"),
+ 'root': os.getenv(f"{env_profile}{HDFS_ROOT}"),
+ 'timeout': DEFAULT_HDFS_TIMEOUT,
+ 'token': os.getenv(f"{env_profile}{HDFS_TOKEN}"),
+ }
+ timeout_env = f"{env_profile}{HDFS_TIMEOUT}"
+ if os.getenv(timeout_env):
+ config['timeout'] = int(os.getenv(timeout_env))
+
+ config_path = os.getenv(HDFS_CONFIG_PATH) or os.path.expanduser(
+ '~/.hdfscli.cfg')
+ if os.path.exists(config_path):
+ all_config = hdfs_api.config.Config(path=config_path)
+ if not profile_name:
+ if (all_config.has_section(all_config.global_section) and
+ all_config.has_option(all_config.global_section,
+ 'default.alias')):
+ profile_name = all_config.get(
+ all_config.global_section, 'default.alias')
+ for suffix in ('.alias', '_alias'):
+ section = '{}{}'.format(profile_name, suffix)
+ if all_config.has_section(section):
+ options = dict(all_config.items(section))
+ for key, value in config.items():
+ if not value and options.get(key):
+ config[key] = options[key]
+ break
+
+ if config['url']:
+ return config
+
+ raise hdfs_api.HdfsError(
+ 'Config error, please set environments or use "megfile config hdfs ..."'
+ )
+
+
+@lru_cache()
+def get_hdfs_client(profile_name: Optional[str] = None):
+ if not hdfs_api: # pragma: no cover
+ raise ImportError(
+ "hdfs not found, please `pip install 'megfile[hdfs]'`")
+
+ config = get_hdfs_config(profile_name)
+ if config['token']:
+ config.pop('user', None)
+ return hdfs_api.TokenClient(**config)
+ config.pop('token', None)
+ return hdfs_api.InsecureClient(**config)
+
+
+[docs]def hdfs_glob(
+ path: PathLike,
+ recursive: bool = True,
+ missing_ok: bool = True,
+) -> List[str]:
+ '''Return hdfs path list in ascending alphabetical order, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: A list contains paths match `path`
+ '''
+ return list(hdfs_iglob(path, recursive=recursive, missing_ok=missing_ok))
+
+
+[docs]def hdfs_glob_stat(
+ path: PathLike, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator[FileEntry]:
+ '''Return a generator contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: A generator contains tuples of path and file stat, in which paths match `path`
+ '''
+ return HdfsPath(path).glob_stat(
+ pattern="", recursive=recursive, missing_ok=missing_ok)
+
+
+[docs]def hdfs_iglob(
+ path: PathLike,
+ recursive: bool = True,
+ missing_ok: bool = True,
+) -> Iterator[str]:
+ '''Return hdfs path iterator in ascending alphabetical order, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: An iterator contains paths match `path`
+ '''
+ for path_obj in HdfsPath(path).iglob(pattern="", recursive=recursive,
+ missing_ok=missing_ok):
+ yield path_obj.path_with_protocol
+
+
+[docs]def hdfs_makedirs(path: PathLike, exist_ok: bool = False):
+ '''
+ Create an hdfs directory.
+ Purely creating directory is invalid because it's unavailable on OSS.
+ This function is to test the target bucket have WRITE access.
+
+ :param path: Given path
+ :param exist_ok: If False and target directory exists, raise S3FileExistsError
+ :raises: FileExistsError
+ '''
+ return HdfsPath(path).mkdir(parents=True, exist_ok=exist_ok)
+
+
+[docs]@SmartPath.register
+class HdfsPath(URIPath):
+ protocol = "hdfs"
+
+ def __init__(self, path: PathLike, *other_paths: PathLike):
+ super().__init__(path, *other_paths)
+ protocol = get_url_scheme(self.path)
+ self._protocol_with_profile = self.protocol
+ self._profile_name = None
+ if protocol.startswith('hdfs+'):
+ self._protocol_with_profile = protocol
+ self._profile_name = protocol[5:]
+
+ @property
+ def _client(self):
+ return get_hdfs_client(profile_name=self._profile_name)
+
+[docs] @cachedproperty
+ def path_with_protocol(self) -> str:
+ '''Return path with protocol, like hdfs://path'''
+ path = self.path
+ protocol_prefix = self._protocol_with_profile + "://"
+ if path.startswith(protocol_prefix):
+ return path
+ return protocol_prefix + path.lstrip('/')
+
+[docs] @cachedproperty
+ def path_without_protocol(self) -> str:
+ '''Return path without protocol, example: if path is hdfs://path, return path'''
+ path = self.path
+ protocol_prefix = self._protocol_with_profile + "://"
+ if path.startswith(protocol_prefix):
+ path = path[len(protocol_prefix):]
+ return path
+
+[docs] @cachedproperty
+ def parts(self) -> Tuple[str]:
+ '''A tuple giving access to the path’s various components'''
+ parts = [f"{self._protocol_with_profile}://"]
+ path = self.path_without_protocol
+ path = path.lstrip('/')
+ if path != '':
+ parts.extend(path.split('/'))
+ return tuple(parts)
+
+[docs] def exists(self, followlinks: bool = False) -> bool:
+ '''
+ Test if path exists
+
+ If the bucket of path are not permitted to read, return False
+
+ :returns: True if path eixsts, else False
+ '''
+ return bool(
+ self._client.status(self.path_without_protocol, strict=False))
+
+[docs] def stat(self, follow_symlinks=True) -> StatResult:
+ '''
+ Get StatResult of path file, including file size and mtime, referring to hdfs_getsize and hdfs_getmtime
+
+ If path is not an existent path, which means hdfs_exist(path) returns False, then raise FileNotFoundError
+ If attempt to get StatResult of complete hdfs, such as hdfs_dir_url == 'hdfs://', raise BucketNotFoundError
+
+ :returns: StatResult
+ :raises: FileNotFoundError
+ '''
+ with raise_hdfs_error(self.path_with_protocol):
+ stat_data = self._client.status(self.path_without_protocol)
+ return StatResult(
+ size=stat_data['length'],
+ mtime=stat_data['modificationTime'] / 1000,
+ isdir=stat_data['type'] == 'DIRECTORY',
+ islnk=False,
+ extra=stat_data,
+ )
+
+[docs] def getmtime(self, follow_symlinks: bool = False) -> float:
+ '''
+ Get last-modified time of the file on the given path path (in Unix timestamp format).
+ If the path is an existent directory, return the latest modified time of all file in it. The mtime of empty directory is 1970-01-01 00:00:00
+
+ If path is not an existent path, which means hdfs_exist(path) returns False, then raise FileNotFoundError
+
+ :returns: Last-modified time
+ :raises: FileNotFoundError
+ '''
+ return self.stat(follow_symlinks=follow_symlinks).mtime
+
+[docs] def getsize(self, follow_symlinks: bool = False) -> int:
+ '''
+ Get file size on the given path path (in bytes).
+ If the path in a directory, return the sum of all file size in it, including file in subdirectories (if exist).
+ The result excludes the size of directory itself. In other words, return 0 Byte on an empty directory path.
+
+ If path is not an existent path, which means hdfs_exist(path) returns False, then raise FileNotFoundError
+
+ :returns: File size
+ :raises: FileNotFoundError
+ '''
+ return self.stat(follow_symlinks=follow_symlinks).size
+
+[docs] def glob(
+ self,
+ pattern,
+ recursive: bool = True,
+ missing_ok: bool = True,
+ ) -> List['HdfsPath']:
+ '''Return hdfs path list, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: A list contains paths match `hdfs_pathname`
+ '''
+ return list(
+ self.iglob(
+ pattern=pattern, recursive=recursive, missing_ok=missing_ok))
+
+[docs] def glob_stat(
+ self, pattern, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator[FileEntry]:
+ '''Return a generator contains tuples of path and file stat, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: A generator contains tuples of path and file stat, in which paths match `hdfs_pathname`
+ '''
+ for path_obj in self.iglob(pattern=pattern, recursive=recursive,
+ missing_ok=missing_ok):
+ yield FileEntry(path_obj.name, path_obj.path, path_obj.stat())
+
+[docs] def iglob(
+ self,
+ pattern,
+ recursive: bool = True,
+ missing_ok: bool = True,
+ ) -> Iterator['HdfsPath']:
+ '''Return hdfs path iterator, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: An iterator contains paths match `hdfs_pathname`
+ '''
+ glob_path = self.path_with_protocol
+ if pattern:
+ glob_path = self.joinpath(pattern).path_with_protocol
+
+ def _scandir(dirname: str) -> Iterator[Tuple[str, bool]]:
+ for entry in self.from_path(dirname).scandir():
+ yield entry.name, entry.is_dir()
+
+ def _exist(path: PathLike, followlinks: bool = False):
+ return self.from_path(path).exists(followlinks=followlinks)
+
+ def _is_dir(path: PathLike, followlinks: bool = False):
+ return self.from_path(path).is_dir(followlinks=followlinks)
+
+ fs_func = FSFunc(_exist, _is_dir, _scandir)
+ for real_path in _create_missing_ok_generator(
+ iglob(fspath(glob_path), recursive=recursive,
+ fs=fs_func), missing_ok,
+ FileNotFoundError('No match any file: %r' % glob_path)):
+ yield self.from_path(real_path)
+
+[docs] def is_dir(self, followlinks: bool = False) -> bool:
+ '''
+ Test if an hdfs url is directory
+ Specific procedures are as follows:
+ If there exists a suffix, of which ``os.path.join(path, suffix)`` is a file
+ If the url is empty bucket or hdfs://
+
+ :param followlinks: whether followlinks is True or False, result is the same. Because hdfs symlink not support dir.
+ :returns: True if path is hdfs directory, else False
+ '''
+ return self.stat().is_dir()
+
+[docs] def is_file(self, followlinks: bool = False) -> bool:
+ '''
+ Test if an path is file
+
+ :returns: True if path is hdfs file, else False
+ '''
+ return self.stat().is_file()
+
+[docs] def listdir(self, followlinks: bool = False) -> List[str]:
+ '''
+ Get all contents of given path.
+
+ :returns: All contents have prefix of path.
+ :raises: FileNotFoundError, NotADirectoryError
+ '''
+ if not self.is_dir():
+ raise NotADirectoryError('Not a directory: %r' % self.path)
+ with raise_hdfs_error(self.path_with_protocol):
+ return self._client.list(self.path_without_protocol)
+
+[docs] def iterdir(self, followlinks: bool = False) -> Iterator['HdfsPath']:
+ '''
+ Get all contents of given path.
+
+ :returns: All contents have prefix of path.
+ :raises: FileNotFoundError, NotADirectoryError
+ '''
+ for filename in self.listdir(followlinks=followlinks):
+ yield self.joinpath(filename) # pytype: disable=bad-return-type
+
+[docs] def load(self, followlinks: bool = False) -> BinaryIO:
+ '''Read all content in binary on specified path and write into memory
+
+ User should close the BinaryIO manually
+
+ :returns: BinaryIO
+ '''
+
+ buffer = io.BytesIO()
+ with self.open('rb') as f:
+ buffer.write(f.read())
+ buffer.seek(0)
+ return buffer
+
+[docs] def mkdir(self, mode=0o777, parents: bool = False, exist_ok: bool = False):
+ '''
+ Create an hdfs directory.
+ Purely creating directory is invalid because it's unavailable on OSS.
+ This function is to test the target bucket have WRITE access.
+
+ :param mode: Octal permission to set on the newly created directory.
+ These permissions will only be set on directories that do not already exist.
+ :param parents: parents is ignored, only be compatible with pathlib.Path
+ :param exist_ok: If False and target directory exists, raise FileExistsError
+ :raises: BucketNotFoundError, FileExistsError
+ '''
+ if not exist_ok and self.exists():
+ raise FileExistsError('File exists: %r' % self.path)
+ with raise_hdfs_error(self.path_with_protocol):
+ self._client.makedirs(self.path_without_protocol, permission=mode)
+
+[docs] def rename(self, dst_path: PathLike, overwrite: bool = True) -> 'HdfsPath':
+ '''
+ Move hdfs file path from src_path to dst_path
+
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ dst_path = self.from_path(dst_path)
+ if self.is_dir():
+ for filename in self.iterdir():
+ self.joinpath(filename).rename(dst_path.joinpath(filename)) # pytype: disable=attribute-error
+ else:
+ if overwrite:
+ dst_path.remove(missing_ok=True)
+ if overwrite or not dst_path.exists():
+ with raise_hdfs_error(self.path_with_protocol):
+ self._client.rename(
+ self.path_without_protocol,
+ dst_path.path_without_protocol)
+ self.remove(missing_ok=True)
+ return dst_path
+
+[docs] def move(self, dst_path: PathLike, overwrite: bool = True) -> None:
+ '''
+ Move file/directory path from src_path to dst_path
+
+ :param dst_path: Given destination path
+ '''
+ self.rename(dst_path=dst_path, overwrite=overwrite)
+
+[docs] def remove(self, missing_ok: bool = False) -> None:
+ '''
+ Remove the file or directory on hdfs, `hdfs://` and `hdfs://bucket` are not permitted to remove
+
+ :param missing_ok: if False and target file/directory not exists, raise FileNotFoundError
+ :raises: FileNotFoundError, UnsupportedError
+ '''
+ try:
+ with raise_hdfs_error(self.path_with_protocol):
+ self._client.delete(self.path_without_protocol, recursive=True)
+ except Exception as e:
+ if not missing_ok or not isinstance(e, FileNotFoundError):
+ raise
+
+[docs] def scan(self, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[str]:
+ '''
+ Iteratively traverse only files in given hdfs directory.
+ Every iteration on generator yields a path string.
+
+ If path is a file path, yields the file only
+ If path is a non-existent path, return an empty generator
+ If path is a bucket path, return all file paths in the bucket
+ If path is an empty bucket, return an empty generator
+ If path doesn't contain any bucket, which is path == 'hdfs://', raise UnsupportedError. walk() on complete hdfs is not supported in megfile
+
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :raises: UnsupportedError
+ :returns: A file path generator
+ '''
+ for file_entry in self.scan_stat(missing_ok=missing_ok,
+ followlinks=followlinks):
+ yield file_entry.path
+
+[docs] def scan_stat(self, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Iteratively traverse only files in given directory.
+ Every iteration on generator yields a tuple of path string and file stat
+
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :raises: UnsupportedError
+ :returns: A file path generator
+ '''
+ with raise_hdfs_error(self.path_with_protocol):
+ for (root,
+ _root_status), _dir_infos, file_infos in self._client.walk(
+ self.path_without_protocol, status=True,
+ ignore_missing=missing_ok):
+ for filename, stat_data in file_infos:
+ yield FileEntry(
+ name=filename,
+ path=self.from_path(
+ f"{self._protocol_with_profile}://{root.lstrip('/')}"
+ ).joinpath(filename).path_with_protocol,
+ stat=StatResult(
+ size=stat_data['length'],
+ mtime=stat_data['modificationTime'] / 1000,
+ isdir=False,
+ islnk=False,
+ extra=stat_data,
+ ))
+
+[docs] def scandir(self, followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Get all contents of given path, the order of result is not guaranteed.
+
+ :returns: All contents have prefix of path
+ :raises: FileNotFoundError, NotADirectoryError
+ '''
+ with raise_hdfs_error(self.path_with_protocol):
+ for filename, stat_data in self._client.list(
+ self.path_without_protocol, status=True):
+ yield FileEntry(
+ name=filename,
+ path=self.joinpath(filename).path_with_protocol,
+ stat=StatResult(
+ size=stat_data['length'],
+ mtime=stat_data['modificationTime'] / 1000,
+ isdir=stat_data['type'] == 'DIRECTORY',
+ islnk=False,
+ extra=stat_data,
+ ))
+
+[docs] def unlink(self, missing_ok: bool = False) -> None:
+ '''
+ Remove the file on hdfs
+
+ :param missing_ok: if False and target file not exists, raise FileNotFoundError
+ :raises: FileNotFoundError, IsADirectoryError
+ '''
+ if self.is_dir():
+ raise IsADirectoryError('Path is a directory: %r' % self.path)
+ self.remove(missing_ok=missing_ok)
+
+[docs] def walk(self, followlinks: bool = False
+ ) -> Iterator[Tuple[str, List[str], List[str]]]:
+ '''
+ Iteratively traverse the given hdfs directory, in top-bottom order. In other words, firstly traverse parent directory, if subdirectories exist, traverse the subdirectories.
+ Every iteration on generator yields a 3-tuple: (root, dirs, files)
+
+ - root: Current hdfs path;
+ - dirs: Name list of subdirectories in current directory.
+ - files: Name list of files in current directory.
+
+ If path is a file path, return an empty generator
+ If path is a non-existent path, return an empty generator
+ If path is a bucket path, bucket will be the top directory, and will be returned at first iteration of generator
+ If path is an empty bucket, only yield one 3-tuple (notes: hdfs doesn't have empty directory)
+ If path doesn't contain any bucket, which is path == 'hdfs://', raise UnsupportedError. walk() on complete hdfs is not supported in megfile
+
+ :param followlinks: whether followlinks is True or False, result is the same. Because hdfs not support symlink.
+ :returns: A 3-tuple generator
+ '''
+ with raise_hdfs_error(self.path_with_protocol):
+ for path, dirs, files in self._client.walk(
+ self.path_without_protocol, ignore_missing=True,
+ allow_dir_changes=True):
+ yield f"{self._protocol_with_profile}://{path.lstrip('/')}", dirs, files
+
+[docs] def md5(self, recalculate: bool = False, followlinks: bool = False) -> str:
+ '''
+ Get checksum of the file or dir.
+
+ :param recalculate: Ignore this parameter, just for compatibility
+ :param followlinks: Ignore this parameter, just for compatibility
+ :returns: checksum
+ '''
+ if self.is_dir(followlinks=followlinks):
+ hash_md5 = hashlib.md5() # nosec
+ for file_name in self.listdir():
+ chunk = self.joinpath(file_name).md5( # pytype: disable=attribute-error
+ recalculate=recalculate).encode()
+ hash_md5.update(chunk)
+ return hash_md5.hexdigest()
+ with raise_hdfs_error(self.path_with_protocol):
+ return self._client.checksum(self.path_without_protocol)['bytes']
+
+[docs] def save(self, file_object: BinaryIO):
+ '''Write the opened binary stream to specified path, but the stream won't be closed
+
+ :param file_object: Stream to be read
+ '''
+ with raise_hdfs_error(self.path_with_protocol):
+ self._client.write(
+ self.path_without_protocol, overwrite=True, data=file_object)
+
+[docs] def open(
+ self,
+ mode: str = 'r',
+ *,
+ buffering: Optional[int] = None,
+ encoding: Optional[str] = None,
+ errors: Optional[str] = None,
+ **kwargs) -> IO[AnyStr]: # pytype: disable=signature-mismatch
+ if '+' in mode:
+ raise ValueError('unacceptable mode: %r' % mode)
+
+ if 'b' in mode:
+ encoding = None
+ elif not encoding:
+ encoding = sys.getdefaultencoding()
+
+ with raise_hdfs_error(self.path_with_protocol):
+ if mode in ('r', 'rb'):
+ keys = [
+ 'block_size', 'block_capacity', 'block_forward',
+ 'max_retries', 'max_workers'
+ ]
+ input_kwargs = {}
+ for key in keys:
+ if key in kwargs:
+ input_kwargs[key] = kwargs[key]
+ file_obj = HdfsPrefetchReader(
+ hdfs_path=self.path_without_protocol,
+ client=self._client,
+ profile_name=self._profile_name,
+ **input_kwargs)
+ if _is_pickle(file_obj): # pytype: disable=wrong-arg-types
+ file_obj = io.BufferedReader(file_obj) # pytype: disable=wrong-arg-types
+ if 'b' not in mode:
+ file_obj = io.TextIOWrapper(
+ file_obj, encoding=encoding, errors=errors) # pytype: disable=wrong-arg-types
+ file_obj.mode = mode
+ return file_obj # pytype: disable=bad-return-type
+ elif mode in ('w', 'wb'):
+ return self._client.write( # pytype: disable=bad-return-type
+ self.path_without_protocol,
+ overwrite=True,
+ buffersize=buffering,
+ encoding=encoding)
+ elif mode in ('a', 'ab'):
+ return self._client.write( # pytype: disable=bad-return-type
+ self.path_without_protocol,
+ append=True,
+ buffersize=buffering,
+ encoding=encoding)
+ raise ValueError('unacceptable mode: %r' % mode)
+
+[docs] def absolute(self) -> 'HdfsPath':
+ '''
+ Make the path absolute, without normalization or resolving symlinks. Returns a new path object
+ '''
+ with raise_hdfs_error(self.path_with_protocol):
+ real_path = self._client.resolve(self.path_without_protocol)
+ return self.from_path(
+ f"{self._protocol_with_profile}:///{real_path.lstrip('/')}")
+
+from megfile.http_path import HttpPath, HttpsPath, get_http_session, http_open, is_http
+from megfile.interfaces import PathLike, StatResult
+
+__all__ = [
+ 'get_http_session',
+ 'is_http',
+ 'http_open',
+ 'http_stat',
+ 'http_getsize',
+ 'http_getmtime',
+ 'http_exists',
+]
+
+
+[docs]def http_stat(path: PathLike, follow_symlinks=True) -> StatResult:
+ '''
+ Get StatResult of http_url response, including size and mtime, referring to http_getsize and http_getmtime
+
+ :param path: Given path
+ :param follow_symlinks: Ignore this parameter, just for compatibility
+ :returns: StatResult
+ :raises: HttpPermissionError, HttpFileNotFoundError
+ '''
+ return HttpPath(path).stat(follow_symlinks)
+
+
+[docs]def http_getsize(path: PathLike, follow_symlinks: bool = False) -> int:
+ '''
+ Get file size on the given http_url path.
+
+ If http response header don't support Content-Length, will return None
+
+ :param path: Given path
+ :param follow_symlinks: Ignore this parameter, just for compatibility
+ :returns: File size (in bytes)
+ :raises: HttpPermissionError, HttpFileNotFoundError
+ '''
+ return HttpPath(path).getsize(follow_symlinks)
+
+
+[docs]def http_getmtime(path: PathLike, follow_symlinks: bool = False) -> float:
+ '''
+ Get Last-Modified time of the http request on the given http_url path.
+
+ If http response header don't support Last-Modified, will return None
+
+ :param path: Given path
+ :param follow_symlinks: Ignore this parameter, just for compatibility
+ :returns: Last-Modified time (in Unix timestamp format)
+ :raises: HttpPermissionError, HttpFileNotFoundError
+ '''
+ return HttpPath(path).getmtime(follow_symlinks)
+
+
+[docs]def http_exists(path: PathLike, followlinks: bool = False) -> bool:
+ """Test if http path exists
+
+ :param path: Given path
+ :param followlinks: ignore this parameter, just for compatibility
+ :type followlinks: bool, optional
+ :return: return True if exists
+ :rtype: bool
+ """
+ return HttpPath(path).exists(followlinks)
+
+import time
+from copy import deepcopy
+from functools import partial
+from io import BufferedReader, BytesIO
+from logging import getLogger as get_logger
+from threading import Lock
+from typing import Iterable, Iterator, Optional, Tuple, Union
+
+import requests
+from urllib3 import HTTPResponse
+
+from megfile.config import DEFAULT_BLOCK_SIZE, HTTP_MAX_RETRY_TIMES
+from megfile.errors import http_should_retry, patch_method, translate_http_error
+from megfile.interfaces import PathLike, Readable, StatResult, URIPath
+from megfile.lib.compat import fspath
+from megfile.lib.http_prefetch_reader import DEFAULT_TIMEOUT, HttpPrefetchReader
+from megfile.lib.s3_buffered_writer import DEFAULT_MAX_BUFFER_SIZE
+from megfile.lib.url import get_url_scheme
+from megfile.pathlike import PathLike
+from megfile.smart_path import SmartPath
+from megfile.utils import _is_pickle, binary_open
+
+__all__ = [
+ 'HttpPath',
+ 'HttpsPath',
+ 'get_http_session',
+ 'is_http',
+ 'http_open',
+]
+
+_logger = get_logger(__name__)
+max_retries = HTTP_MAX_RETRY_TIMES
+
+
+[docs]def get_http_session(
+ timeout: Optional[Union[int, Tuple[int, int]]] = DEFAULT_TIMEOUT,
+ status_forcelist: Iterable[int] = (500, 502, 503, 504)
+) -> requests.Session:
+ session = requests.Session()
+
+ def after_callback(response, *args, **kwargs):
+ if response.status_code in status_forcelist:
+ response.raise_for_status()
+ return response
+
+ def before_callback(method, url, **kwargs):
+ _logger.debug(
+ 'send http request: %s %r, with parameters: %s', method, url,
+ kwargs)
+
+ def retry_callback(
+ error,
+ method,
+ url,
+ params=None,
+ data=None,
+ headers=None,
+ cookies=None,
+ files=None,
+ auth=None,
+ timeout=None,
+ allow_redirects=True,
+ proxies=None,
+ hooks=None,
+ stream=None,
+ verify=None,
+ cert=None,
+ json=None,
+ **kwargs,
+ ):
+ if data and hasattr(data, 'seek'):
+ data.seek(0)
+ elif isinstance(data, Iterator):
+ _logger.warning(f'Can not retry http request with iterator data')
+ raise
+ if files:
+
+ def seek_or_reopen(file_object):
+ if isinstance(file_object, (str, bytes)):
+ return file_object
+ elif hasattr(file_object, 'seek'):
+ file_object.seek(0)
+ return file_object
+ elif hasattr(file_object, 'name'):
+ with SmartPath(file_object.name).open('rb') as f:
+ return BytesIO(f.read())
+ else:
+ _logger.warning(
+ f'Can not retry http request, because the file object is not seekable and unsupport "name"'
+ )
+ raise
+
+ for key, file_info in files.items():
+ if hasattr(file_info, 'seek'):
+ file_info.seek(0)
+ elif isinstance(file_info,
+ (tuple, list)) and len(file_info) >= 2:
+ file_info = list(file_info)
+ if isinstance(file_info[1],
+ (tuple, list)) and len(file_info[1]) >= 2:
+ file_info[1] = list(file_info[1])
+ file_info[1] = seek_or_reopen(file_info[1])
+ else:
+ file_info[1] = seek_or_reopen(file_info[1])
+ files[key] = file_info
+
+ session.request = patch_method(
+ partial(session.request, timeout=timeout),
+ max_retries=max_retries,
+ should_retry=http_should_retry,
+ before_callback=before_callback,
+ after_callback=after_callback,
+ retry_callback=retry_callback,
+ )
+ return session
+
+
+[docs]def is_http(path: PathLike) -> bool:
+ '''http scheme definition: http(s)://domain/path
+
+ :param path: Path to be tested
+ :returns: True if path is http url, else False
+ '''
+
+ path = fspath(path)
+ if not isinstance(path, str) or not (path.startswith('http://') or
+ path.startswith('https://')):
+ return False
+
+ scheme = get_url_scheme(path)
+ return scheme == 'http' or scheme == 'https'
+
+
+[docs]def http_open(
+ path: PathLike,
+ mode: str = 'rb',
+ *,
+ encoding: Optional[str] = None,
+ errors: Optional[str] = None,
+ max_concurrency: Optional[int] = None,
+ max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
+ forward_ratio: Optional[float] = None,
+ block_size: int = DEFAULT_BLOCK_SIZE,
+ **kwargs) -> Union[BufferedReader, HttpPrefetchReader]:
+ '''Open a BytesIO to read binary data of given http(s) url
+
+ .. note ::
+
+ Essentially, it reads data of http(s) url to memory by requests, and then return BytesIO to user.
+
+ :param path: Given path
+ :param mode: Only supports 'rb' mode now
+ :param encoding: encoding is the name of the encoding used to decode or encode the file. This should only be used in text mode.
+ :param errors: errors is an optional string that specifies how encoding and decoding errors are to be handled—this cannot be used in binary mode.
+ :param max_concurrency: Max download thread number, None by default
+ :param max_buffer_size: Max cached buffer size in memory, 128MB by default
+ :param block_size: Size of single block, 8MB by default. Each block will be uploaded or downloaded by single thread.
+ :return: BytesIO initialized with http(s) data
+ '''
+ return HttpPath(path).open(
+ mode,
+ encoding=encoding,
+ errors=errors,
+ max_concurrency=max_concurrency,
+ max_buffer_size=max_buffer_size,
+ forward_ratio=forward_ratio,
+ block_size=block_size)
+
+
+[docs]@SmartPath.register
+class HttpPath(URIPath):
+
+ protocol = "http"
+
+ def __init__(self, path: PathLike, *other_paths: PathLike):
+ super().__init__(path, *other_paths)
+
+ if fspath(path).startswith('https://'):
+ self.protocol = 'https'
+ self.request_kwargs = {}
+
+[docs] @binary_open
+ def open(
+ self,
+ mode: str = 'rb',
+ *,
+ max_concurrency: Optional[int] = None,
+ max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
+ forward_ratio: Optional[float] = None,
+ block_size: int = DEFAULT_BLOCK_SIZE,
+ **kwargs) -> Union[BufferedReader, HttpPrefetchReader]:
+ '''Open a BytesIO to read binary data of given http(s) url
+
+ .. note ::
+
+ Essentially, it reads data of http(s) url to memory by requests, and then return BytesIO to user.
+
+ :param mode: Only supports 'rb' mode now
+ :param encoding: encoding is the name of the encoding used to decode or encode the file. This should only be used in text mode.
+ :param errors: errors is an optional string that specifies how encoding and decoding errors are to be handled—this cannot be used in binary mode.
+ :param max_concurrency: Max download thread number, None by default
+ :param max_buffer_size: Max cached buffer size in memory, 128MB by default
+ :param block_size: Size of single block, 8MB by default. Each block will be uploaded or downloaded by single thread.
+ :return: BytesIO initialized with http(s) data
+ '''
+ if mode not in ('rb',):
+ raise ValueError('unacceptable mode: %r' % mode)
+
+ response = None
+ request_kwargs = deepcopy(self.request_kwargs)
+ timeout = request_kwargs.pop('timeout', DEFAULT_TIMEOUT)
+ stream = request_kwargs.pop('stream', True)
+ try:
+ response = get_http_session(
+ timeout=timeout,
+ status_forcelist=(),
+ ).get(
+ self.path_with_protocol, stream=stream, **request_kwargs)
+ response.raise_for_status()
+ except Exception as error:
+ if response:
+ response.close()
+ raise translate_http_error(error, self.path_with_protocol)
+
+ content_size = int(response.headers['Content-Length'])
+ if (response.headers.get('Accept-Ranges') == 'bytes' and
+ content_size >= block_size * 2 and
+ not response.headers.get('Content-Encoding')):
+ response.close()
+
+ block_capacity = max_buffer_size // block_size
+ if forward_ratio is None:
+ block_forward = None
+ else:
+ block_forward = max(int(block_capacity * forward_ratio), 1)
+
+ reader = HttpPrefetchReader(
+ self,
+ content_size=content_size,
+ max_retries=max_retries,
+ max_workers=max_concurrency,
+ block_capacity=block_capacity,
+ block_forward=block_forward,
+ block_size=block_size,
+ )
+ if _is_pickle(reader): # pytype: disable=wrong-arg-types
+ reader = BufferedReader(reader) # pytype: disable=wrong-arg-types
+ return reader
+
+ response.raw.name = self.path_with_protocol
+ # TODO: When python version must bigger than 3.10, use urllib3>=2.0.0 instead of 'Response'
+ # response.raw.auto_close = False
+ # response.raw.decode_content = True
+ # return BufferedReader(response.raw)
+ return BufferedReader(Response(response.raw)) # pytype: disable=wrong-arg-types
+
+[docs] def stat(self, follow_symlinks=True) -> StatResult:
+ '''
+ Get StatResult of http_url response, including size and mtime, referring to http_getsize and http_getmtime
+
+ :param follow_symlinks: Ignore this parameter, just for compatibility
+ :returns: StatResult
+ :raises: HttpPermissionError, HttpFileNotFoundError
+ '''
+
+ request_kwargs = deepcopy(self.request_kwargs)
+ timeout = request_kwargs.pop('timeout', DEFAULT_TIMEOUT)
+ stream = request_kwargs.pop('stream', True)
+
+ try:
+ with get_http_session(timeout=timeout, status_forcelist=()).get(
+ self.path_with_protocol, stream=stream,
+ **request_kwargs) as response:
+ response.raise_for_status()
+ headers = response.headers
+ except Exception as error:
+ raise translate_http_error(error, self.path_with_protocol)
+
+ size = headers.get('Content-Length')
+ if size:
+ size = int(size)
+
+ last_modified = headers.get('Last-Modified')
+ if last_modified:
+ last_modified = time.mktime(
+ time.strptime(last_modified, "%a, %d %b %Y %H:%M:%S %Z"))
+
+ return StatResult( # pyre-ignore[20]
+ size=size, mtime=last_modified, isdir=False,
+ islnk=False, extra=headers)
+
+[docs] def getsize(self, follow_symlinks: bool = False) -> int:
+ '''
+ Get file size on the given http_url path.
+
+ If http response header don't support Content-Length, will return None
+
+ :param follow_symlinks: Ignore this parameter, just for compatibility
+ :returns: File size (in bytes)
+ :raises: HttpPermissionError, HttpFileNotFoundError
+ '''
+ return self.stat().size
+
+[docs] def getmtime(self, follow_symlinks: bool = False) -> float:
+ '''
+ Get Last-Modified time of the http request on the given http_url path.
+
+ If http response header don't support Last-Modified, will return None
+
+ :param follow_symlinks: Ignore this parameter, just for compatibility
+ :returns: Last-Modified time (in Unix timestamp format)
+ :raises: HttpPermissionError, HttpFileNotFoundError
+ '''
+ return self.stat().mtime
+
+[docs] def exists(self, followlinks: bool = False) -> bool:
+ """Test if http path exists
+
+ :param followlinks: ignore this parameter, just for compatibility
+ :type followlinks: bool, optional
+ :return: return True if exists
+ :rtype: bool
+ """
+ request_kwargs = deepcopy(self.request_kwargs)
+ timeout = request_kwargs.pop('timeout', DEFAULT_TIMEOUT)
+ stream = request_kwargs.pop('stream', True)
+
+ try:
+ with get_http_session(timeout=timeout, status_forcelist=()).get(
+ self.path_with_protocol, stream=stream,
+ **request_kwargs) as response:
+ if response.status_code == 404:
+ return False
+ return True
+ except requests.exceptions.ConnectionError:
+ return False
+
+
+
+
+
+class Response(Readable):
+
+ def __init__(self, raw: HTTPResponse) -> None:
+ super().__init__()
+
+ raw.auto_close = False
+ self._block_size = 128 * 2**10 # 128KB
+ self._raw = raw
+ self._offset = 0
+ self._buffer = BytesIO()
+ self._lock = Lock()
+
+ @property
+ def name(self):
+ return self._raw.name
+
+ @property
+ def mode(self):
+ return 'rb'
+
+ def tell(self) -> int:
+ return self._offset
+
+ def _clear_buffer(self) -> None:
+ self._buffer.seek(0)
+ self._buffer.truncate()
+
+ def read(self, size: Optional[int] = None) -> bytes:
+ if size == 0:
+ return b''
+ if size is not None and size < 0:
+ size = None
+
+ with self._lock:
+ while not size or self._buffer.tell() < size:
+ data = self._raw.read(self._block_size, decode_content=True)
+ if not data:
+ break
+ self._buffer.write(data)
+ self._buffer.seek(0)
+ content = self._buffer.read(size)
+ residue = self._buffer.read()
+ self._clear_buffer()
+ if residue:
+ self._buffer.write(residue)
+ self._offset += len(content)
+ return content
+
+ def readline(self, size: Optional[int] = None) -> bytes:
+ if size == 0:
+ return b''
+ if size is not None and size < 0:
+ size = None
+
+ with self._lock:
+ self._buffer.seek(0)
+ buffer = self._buffer.read()
+ self._clear_buffer()
+ if b'\n' in buffer:
+ content = buffer[:buffer.index(b'\n') + 1]
+ if size:
+ content = content[:size]
+ self._buffer.write(buffer[len(content):])
+ elif size and len(buffer) >= size:
+ content = buffer[:size]
+ self._buffer.write(buffer[size:])
+ else:
+ content = None
+ self._buffer.write(buffer)
+ while True:
+ if size and self._buffer.tell() >= size:
+ break
+ data = self._raw.read(self._block_size, decode_content=True)
+ if not data:
+ break
+ elif b"\n" in data:
+ last_content, residue = data.split(b"\n", 1)
+ self._buffer.write(last_content)
+ self._buffer.write(b"\n")
+ self._buffer.seek(0)
+ content = self._buffer.read()
+ self._clear_buffer()
+ if size and len(content) > size:
+ self._buffer.write(content[size:])
+ content = content[:size]
+ if residue:
+ self._buffer.write(residue)
+ break
+ else:
+ self._buffer.write(data)
+
+ if content is None:
+ self._buffer.seek(0)
+ content = self._buffer.read(size)
+ residue = self._buffer.read()
+ self._clear_buffer()
+ if residue:
+ self._buffer.write(residue)
+ self._offset += len(content)
+ return content
+
+ def _close(self) -> None:
+ return self._raw.close()
+
+from collections import OrderedDict
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from io import BytesIO
+from logging import getLogger as get_logger
+from threading import Lock
+from typing import NamedTuple, Optional
+
+from megfile.config import BACKOFF_FACTOR, BACKOFF_INITIAL, DEFAULT_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE, DEFAULT_MAX_BUFFER_SIZE, GLOBAL_MAX_WORKERS
+from megfile.errors import raise_s3_error
+from megfile.interfaces import Writable
+from megfile.utils import get_human_size, process_local
+
+_logger = get_logger(__name__)
+'''
+class PartResult(NamedTuple):
+
+ etag: str
+ part_number: int
+ content_size: int
+
+in Python 3.6+
+'''
+
+_PartResult = NamedTuple(
+ 'PartResult', [('etag', str), ('part_number', int), ('content_size', int)])
+
+
+class PartResult(_PartResult):
+
+ def asdict(self):
+ return {
+ 'PartNumber': self.part_number,
+ 'ETag': self.etag,
+ }
+
+
+[docs]class S3BufferedWriter(Writable):
+
+ def __init__(
+ self,
+ bucket: str,
+ key: str,
+ *,
+ s3_client,
+ block_size: int = DEFAULT_BLOCK_SIZE,
+ max_block_size: int = DEFAULT_MAX_BLOCK_SIZE,
+ max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
+ max_workers: Optional[int] = None,
+ profile_name: Optional[str] = None):
+
+ self._bucket = bucket
+ self._key = key
+ self._client = s3_client
+ self._profile_name = profile_name
+
+ self._block_size = block_size
+ self._max_block_size = max_block_size
+ self._max_buffer_size = max_buffer_size
+ self._total_buffer_size = 0
+ self._offset = 0
+ self.__content_size = 0
+ self._backoff_size = BACKOFF_INITIAL
+ self._buffer = BytesIO()
+
+ self._futures = OrderedDict()
+ self._is_global_executor = False
+ if max_workers is None:
+ self._executor = process_local(
+ 'S3BufferedWriter.executor',
+ ThreadPoolExecutor,
+ max_workers=GLOBAL_MAX_WORKERS)
+ self._is_global_executor = True
+ else:
+ self._executor = ThreadPoolExecutor(max_workers=max_workers)
+
+ self._part_number = 0
+ self.__upload_id = None
+ self.__upload_id_lock = Lock()
+
+ _logger.debug('open file: %r, mode: %s' % (self.name, self.mode))
+
+ @property
+ def name(self) -> str:
+ return 's3%s://%s/%s' % (
+ f"+{self._profile_name}" if self._profile_name else "",
+ self._bucket, self._key)
+
+ @property
+ def mode(self) -> str:
+ return 'wb'
+
+
+
+ @property
+ def _content_size(self) -> int:
+ return self.__content_size
+
+ @_content_size.setter
+ def _content_size(self, value: int):
+ if value > self._backoff_size:
+ _logger.debug(
+ 'writing file: %r, current size: %s' %
+ (self.name, get_human_size(value)))
+ while value > self._backoff_size:
+ self._backoff_size *= BACKOFF_FACTOR
+ self.__content_size = value
+
+ @property
+ def _is_multipart(self) -> bool:
+ return len(self._futures) > 0
+
+ @property
+ def _upload_id(self) -> str:
+ with self.__upload_id_lock:
+ if self.__upload_id is None:
+ with raise_s3_error(self.name):
+ self.__upload_id = self._client.create_multipart_upload(
+ Bucket=self._bucket,
+ Key=self._key,
+ )['UploadId']
+ return self.__upload_id
+
+ @property
+ def _buffer_size(self):
+ return self._total_buffer_size - sum(
+ future.result().content_size
+ for future in self._futures.values()
+ if future.done())
+
+ @property
+ def _uploading_futures(self):
+ return [
+ future for future in self._futures.values() if not future.done()
+ ]
+
+ @property
+ def _multipart_upload(self):
+ return {
+ 'Parts': [
+ future.result().asdict()
+ for _, future in sorted(self._futures.items())
+ ],
+ }
+
+ def _upload_buffer(self, part_number, content):
+ with raise_s3_error(self.name):
+ return PartResult(
+ self._client.upload_part(
+ Bucket=self._bucket,
+ Key=self._key,
+ UploadId=self._upload_id,
+ PartNumber=part_number,
+ Body=content,
+ )['ETag'], part_number, len(content))
+
+ def _submit_upload_buffer(self, part_number, content):
+ self._futures[part_number] = self._executor.submit(
+ self._upload_buffer, part_number, content)
+ self._total_buffer_size += len(content)
+ while self._buffer_size > self._max_buffer_size:
+ wait(self._uploading_futures, return_when=FIRST_COMPLETED)
+
+ def _submit_upload_content(self, content: bytes):
+ # s3 part needs at least 5MB, so we need to divide content into equal-size parts, and give last part more size
+ # e.g. 257MB can be divided into 2 parts, 128MB and 129MB
+ offset = 0
+ while len(content) - offset - self._max_block_size > self._block_size:
+ self._part_number += 1
+ offset_stop = offset + self._max_block_size
+ self._submit_upload_buffer(
+ self._part_number, content[offset:offset_stop])
+ offset = offset_stop
+ self._part_number += 1
+ self._submit_upload_buffer(self._part_number, content[offset:])
+
+ def _submit_futures(self):
+ content = self._buffer.getvalue()
+ if len(content) == 0:
+ return
+ self._buffer = BytesIO()
+ self._submit_upload_content(content)
+
+[docs] def write(self, data: bytes) -> int:
+ if self.closed:
+ raise IOError('file already closed: %r' % self.name)
+
+ result = self._buffer.write(data)
+ if self._buffer.tell() >= self._block_size:
+ self._submit_futures()
+ self._offset += result
+ self._content_size = self._offset
+ return result
+
+ def _shutdown(self):
+ if not self._is_global_executor:
+ self._executor.shutdown()
+
+ def _close(self):
+ _logger.debug('close file: %r' % self.name)
+
+ if not self._is_multipart:
+ with raise_s3_error(self.name):
+ self._client.put_object(
+ Bucket=self._bucket,
+ Key=self._key,
+ Body=self._buffer.getvalue())
+ self._shutdown()
+ return
+
+ self._submit_futures()
+
+ with raise_s3_error(self.name):
+ self._client.complete_multipart_upload(
+ Bucket=self._bucket,
+ Key=self._key,
+ MultipartUpload=self._multipart_upload,
+ UploadId=self._upload_id,
+ )
+
+ self._shutdown()
+
+import os
+from io import BytesIO
+from logging import getLogger as get_logger
+from typing import Optional
+
+from megfile.config import DEFAULT_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE, DEFAULT_MAX_BUFFER_SIZE
+from megfile.errors import raise_s3_error
+from megfile.interfaces import Seekable
+from megfile.lib.s3_buffered_writer import S3BufferedWriter
+
+_logger = get_logger(__name__)
+
+
+[docs]class S3LimitedSeekableWriter(Seekable, S3BufferedWriter):
+ ''' For file format like msgpack and mp4, it's a pain that you need to write
+ header before writing the data. So it's kind of hard to make streaming write
+ to unseekable file system like s3. In this case, we will try to keep the first
+ and last parts of data in memory, so we can come back to head again and write
+ the header at the last second.
+ '''
+
+ def __init__(
+ self,
+ bucket: str,
+ key: str,
+ *,
+ s3_client,
+ block_size: int = DEFAULT_BLOCK_SIZE,
+ head_block_size: Optional[int] = None,
+ tail_block_size: Optional[int] = None,
+ max_block_size: int = DEFAULT_MAX_BLOCK_SIZE,
+ max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
+ max_workers: Optional[int] = None,
+ profile_name: Optional[str] = None):
+
+ super().__init__(
+ bucket,
+ key,
+ s3_client=s3_client,
+ block_size=block_size,
+ max_block_size=max_block_size,
+ max_buffer_size=max_buffer_size,
+ max_workers=max_workers,
+ profile_name=profile_name)
+
+ self._head_block_size = head_block_size or block_size
+ self._tail_block_size = tail_block_size or block_size
+ self._head_buffer = BytesIO()
+
+ @property
+ def _head_size(self) -> int:
+ return len(self._head_buffer.getvalue())
+
+ @property
+ def _tail_size(self) -> int:
+ return len(self._buffer.getvalue())
+
+ @property
+ def _tail_offset(self) -> int:
+ return self._content_size - self._tail_size
+
+[docs] def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
+ if self.closed:
+ raise IOError('file already closed: %r' % self.name)
+
+ if whence == os.SEEK_SET:
+ target_offset = offset
+ elif whence == os.SEEK_CUR:
+ target_offset = self._offset + offset
+ elif whence == os.SEEK_END:
+ target_offset = self._content_size + offset
+
+ if target_offset < self._head_block_size:
+ self._head_buffer.seek(target_offset)
+ elif target_offset >= self._tail_offset:
+ self._buffer.seek(target_offset - self._tail_offset)
+ else:
+ raise OSError(
+ 'Can only seek inside of head, or seek to tail, target offset: %d'
+ % target_offset)
+
+ self._offset = target_offset
+ return self._offset
+
+[docs] def write(self, data: bytes) -> int:
+ if self.closed:
+ raise IOError('file already closed: %r' % self.name)
+
+ if self._head_size != self._head_block_size: # no tail part yet
+ self._write_to_head(data)
+ elif self._offset < self._head_block_size: # tail part already created
+ self._write_to_head_after_tail_part_created(data)
+ elif self._offset >= self._tail_offset:
+ self._write_to_tail(data)
+ else:
+ raise OSError(
+ 'Can only write inside of head, or write to tail, current offset: %d'
+ % self._offset)
+ return len(data)
+
+ def _write_to_head(self, data: bytes):
+ if self._offset + len(data) <= self._head_block_size:
+ self._head_buffer.write(data)
+ self._content_size = self._offset = self._head_size
+ if self._content_size == self._head_block_size:
+ self._part_number += 1
+ else: # head part exceeded
+ offset = self._head_block_size - self._offset
+ self._head_buffer.write(data[:offset])
+ self._content_size = self._offset = self._head_size
+ self._part_number += 1
+ self._write_to_tail(data[offset:])
+
+ def _write_to_head_after_tail_part_created(self, data: bytes):
+ if self._offset + len(data) > self._head_block_size:
+ raise Exception(
+ 'Head part overflow, %d bytes left but try to write %d bytes' %
+ (self._head_block_size - self._offset, len(data)))
+ self._head_buffer.write(data)
+ self._offset += len(data)
+
+ def _write_to_tail(self, data: bytes):
+ self._buffer.write(data)
+ if self._buffer.tell() >= self._block_size + self._tail_block_size:
+ self._submit_futures()
+ self._offset += len(data)
+ if self._offset > self._content_size:
+ self._content_size = self._offset
+
+ def _submit_futures(self):
+ content = self._buffer.getvalue()
+ if len(content) == 0:
+ return
+ offset = len(content) - self._tail_block_size
+ self._buffer = BytesIO(content[offset:])
+ self._buffer.seek(0, os.SEEK_END)
+ self._submit_upload_content(content[:offset])
+
+ def _close(self):
+ _logger.debug('close file: %r' % self.name)
+
+ if not self._is_multipart:
+ with raise_s3_error(self.name):
+ self._client.put_object(
+ Bucket=self._bucket,
+ Key=self._key,
+ Body=self._head_buffer.getvalue() + self._buffer.getvalue())
+ self._shutdown()
+ return
+
+ self._submit_upload_buffer(1, self._head_buffer.getvalue())
+ self._head_buffer = BytesIO() # clean memory
+
+ content = self._buffer.getvalue()
+ if len(content) > 0:
+ self._submit_upload_content(content)
+ self._buffer = BytesIO() # clean memory
+
+ with raise_s3_error(self.name):
+ self._client.complete_multipart_upload(
+ Bucket=self._bucket,
+ Key=self._key,
+ MultipartUpload=self._multipart_upload,
+ UploadId=self._upload_id,
+ )
+
+ self._shutdown()
+
+import os
+from concurrent.futures import Future
+from io import BytesIO
+from typing import Optional
+
+from megfile.config import BACKOFF_FACTOR, BACKOFF_INITIAL, DEFAULT_BLOCK_CAPACITY, DEFAULT_BLOCK_SIZE, GLOBAL_MAX_WORKERS, NEWLINE, S3_MAX_RETRY_TIMES
+from megfile.errors import S3FileChangedError, S3InvalidRangeError, patch_method, raise_s3_error, s3_should_retry
+from megfile.lib.base_prefetch_reader import BasePrefetchReader, LRUCacheFutureManager
+
+__all__ = [
+ 'DEFAULT_BLOCK_CAPACITY',
+ 'DEFAULT_BLOCK_SIZE',
+ 'GLOBAL_MAX_WORKERS',
+ 'BACKOFF_INITIAL',
+ 'BACKOFF_FACTOR',
+ 'NEWLINE',
+ 'S3PrefetchReader',
+ 'LRUCacheFutureManager',
+]
+
+
+[docs]class S3PrefetchReader(BasePrefetchReader):
+ '''
+ Reader to fast read the s3 content. This will divide the file content into equal parts of block_size size, and will use LRU to cache at most block_capacity blocks in memory.
+ open(), seek() and read() will trigger prefetch read. The prefetch will cached block_forward blocks of data from offset position (the position after reading if the called function is read).
+ '''
+
+ def __init__(
+ self,
+ bucket: str,
+ key: str,
+ *,
+ s3_client,
+ block_size: int = DEFAULT_BLOCK_SIZE,
+ block_capacity: int = DEFAULT_BLOCK_CAPACITY,
+ block_forward: Optional[int] = None,
+ max_retries: int = S3_MAX_RETRY_TIMES,
+ max_workers: Optional[int] = None,
+ profile_name: Optional[str] = None):
+
+ self._bucket = bucket
+ self._key = key
+ self._client = s3_client
+ self._profile_name = profile_name
+
+ super().__init__(
+ block_size=block_size,
+ block_capacity=block_capacity,
+ block_forward=block_forward,
+ max_retries=max_retries,
+ max_workers=max_workers)
+
+ def _get_content_size(self):
+ try:
+ start, end = 0, self._block_size - 1
+ first_index_response = self._fetch_response(start=start, end=end)
+ content_size = int(
+ first_index_response['ContentRange'].split('/')[-1])
+ except S3InvalidRangeError:
+ # usually when read a empty file
+ # TODO: use minio test empty file: https://hub.docker.com/r/minio/minio
+ first_index_response = self._fetch_response()
+ content_size = int(first_index_response['ContentLength'])
+
+ first_future = Future()
+ first_future.set_result(first_index_response['Body'])
+ self._insert_futures(index=0, future=first_future)
+ self._content_etag = first_index_response['ETag']
+ self._content_info = first_index_response
+ return content_size
+
+ @property
+ def name(self) -> str:
+ return 's3%s://%s/%s' % (
+ f"+{self._profile_name}" if self._profile_name else "",
+ self._bucket, self._key)
+
+ def _fetch_response(
+ self, start: Optional[int] = None,
+ end: Optional[int] = None) -> dict:
+
+ def fetch_response() -> dict:
+ if start is None or end is None:
+ return self._client.get_object(
+ Bucket=self._bucket, Key=self._key)
+
+ range_str = f'bytes={start}-{end}'
+ response = self._client.get_object(
+ Bucket=self._bucket, Key=self._key, Range=range_str)
+ response['Body'] = BytesIO(response['Body'].read())
+ return response
+
+ fetch_response = patch_method(
+ fetch_response,
+ max_retries=self._max_retries,
+ should_retry=s3_should_retry)
+
+ with raise_s3_error(self.name):
+ return fetch_response()
+
+ def _fetch_buffer(self, index: int) -> BytesIO:
+ start, end = index * self._block_size, (
+ index + 1) * self._block_size - 1
+ response = self._fetch_response(start=start, end=end)
+ etag = response.get('ETag', None)
+ if etag is not None and etag != self._content_etag: # pytype: disable=attribute-error
+ raise S3FileChangedError(
+ 'File changed: %r, etag before: %s, after: %s' %
+ (self.name, self._content_info, response)) # pytype: disable=attribute-error
+
+ return response['Body']
+
+from collections import Counter
+from concurrent.futures import Future
+from logging import getLogger as get_logger
+from typing import Optional
+
+from megfile.config import DEFAULT_BLOCK_CAPACITY, DEFAULT_BLOCK_SIZE, S3_MAX_RETRY_TIMES
+from megfile.lib.s3_prefetch_reader import LRUCacheFutureManager, S3PrefetchReader
+from megfile.utils import thread_local
+
+_logger = get_logger(__name__)
+
+
+
+
+
+class ShareCacheFutureManager(LRUCacheFutureManager):
+
+ def __init__(self):
+ super().__init__()
+ self._references = Counter()
+
+ def register(self, key):
+ self._references[key] += 1
+
+ def unregister(self, key):
+ self._references[key] -= 1
+ if self._references[key] == 0:
+ self._references.pop(key)
+ for key_tuple in list(self):
+ if key_tuple[0] != key:
+ continue
+ future = self.pop(key_tuple)
+ if not future.done():
+ future.cancel() # pragma: no cover
+
+import os
+import stat
+from collections.abc import Sequence
+from enum import Enum
+from functools import wraps
+from typing import IO, Any, AnyStr, BinaryIO, Callable, Iterator, List, NamedTuple, Optional, Tuple, Union
+
+from megfile.lib.compat import PathLike as _PathLike
+from megfile.lib.compat import fspath
+from megfile.lib.fnmatch import _compile_pattern
+from megfile.lib.joinpath import uri_join
+from megfile.utils import cachedproperty, classproperty
+
+# Python 3.5+ compatible
+'''
+class StatResult(NamedTuple):
+
+ size: int = 0
+ ctime: float = 0.0
+ mtime: float = 0.0
+ isdir: bool = False
+ islnk: bool = False
+ extra: Any = None # raw stat info
+
+in Python 3.6+
+'''
+
+_StatResult = NamedTuple(
+ 'StatResult', [
+ ('size', int), ('ctime', float), ('mtime', float), ('isdir', bool),
+ ('islnk', bool), ('extra', Any)
+ ])
+_StatResult.__new__.__defaults__ = (0, 0.0, 0.0, False, False, None)
+
+
+class Access(Enum):
+ READ = 1
+ WRITE = 2
+
+
+[docs]class StatResult(_StatResult):
+
+
+
+
+
+
+
+ @property
+ def st_mode(self) -> int:
+ '''
+ File mode: file type and file mode bits (permissions).
+ Only support fs.
+ '''
+ if self.extra and hasattr(self.extra, 'st_mode'):
+ return self.extra.st_mode
+ if self.is_symlink():
+ return stat.S_IFLNK
+ elif self.is_dir():
+ return stat.S_IFDIR
+ return stat.S_IFREG
+
+ @property
+ def st_ino(self) -> int:
+ '''
+ Platform dependent, but if non-zero, uniquely identifies the file for a given value of st_dev. Typically:
+
+ the inode number on Unix,
+ the file index on Windows,
+ the decimal of etag on oss.
+ '''
+ if self.extra:
+ if hasattr(self.extra, 'st_ino'):
+ return self.extra.st_ino
+ elif isinstance(self.extra, dict) and self.extra.get('ETag'):
+ return int(self.extra['ETag'][1:-1], 16)
+ return 0
+
+ @property
+ def st_dev(self) -> int:
+ '''
+ Identifier of the device on which this file resides.
+ '''
+ if self.extra:
+ if hasattr(self.extra, 'st_dev'):
+ return self.extra.st_dev
+ return 0
+
+ @property
+ def st_nlink(self) -> int:
+ '''
+ Number of hard links.
+ Only support fs.
+ '''
+ if self.extra and hasattr(self.extra, 'st_nlink'):
+ return self.extra.st_nlink
+ return 0
+
+ @property
+ def st_uid(self) -> int:
+ '''
+ User identifier of the file owner.
+ Only support fs.
+ '''
+ if self.extra and hasattr(self.extra, 'st_uid'):
+ return self.extra.st_uid
+ return 0
+
+ @property
+ def st_gid(self) -> int:
+ '''
+ Group identifier of the file owner.
+ Only support fs.
+ '''
+ if self.extra and hasattr(self.extra, 'st_gid'):
+ return self.extra.st_gid
+ return 0
+
+ @property
+ def st_size(self) -> int:
+ '''
+ Size of the file in bytes.
+ '''
+ if self.extra and hasattr(self.extra, 'st_size'):
+ return self.extra.st_size
+ return self.size
+
+ @property
+ def st_atime(self) -> float:
+ '''
+ Time of most recent access expressed in seconds.
+ Only support fs.
+ '''
+ if self.extra and hasattr(self.extra, 'st_atime'):
+ return self.extra.st_atime
+ return 0.0
+
+ @property
+ def st_mtime(self) -> float:
+ '''
+ Time of most recent content modification expressed in seconds.
+ '''
+ if self.extra and hasattr(self.extra, 'st_mtime'):
+ return self.extra.st_mtime
+ return self.mtime
+
+ @property
+ def st_ctime(self) -> float:
+ '''
+ Platform dependent:
+
+ the time of most recent metadata change on Unix,
+ the time of creation on Windows, expressed in seconds,
+ the time of file created on oss; if is dir, return the latest ctime of the files in dir.
+ '''
+ if self.extra and hasattr(self.extra, 'st_ctime'):
+ return self.extra.st_ctime
+ return self.ctime
+
+ @property
+ def st_atime_ns(self) -> int:
+ '''
+ Time of most recent access expressed in nanoseconds as an integer.
+ Only support fs.
+ '''
+ if self.extra and hasattr(self.extra, 'st_atime_ns'):
+ return self.extra.st_atime_ns
+ return 0
+
+ @property
+ def st_mtime_ns(self) -> int:
+ '''
+ Time of most recent content modification expressed in nanoseconds as an integer.
+ Only support fs.
+ '''
+ if self.extra and hasattr(self.extra, 'st_mtime_ns'):
+ return self.extra.st_mtime_ns
+ return 0
+
+ @property
+ def st_ctime_ns(self) -> int:
+ '''
+ Platform dependent:
+
+ the time of most recent metadata change on Unix,
+ the time of creation on Windows, expressed in nanoseconds as an integer.
+
+ Only support fs.
+ '''
+ if self.extra and hasattr(self.extra, 'st_ctime_ns'):
+ return self.extra.st_ctime_ns
+ return 0
+
+
+'''
+class FileEntry(NamedTuple):
+
+ name: str
+ stat: StatResult
+
+in Python 3.6+
+'''
+
+_FileEntry = NamedTuple(
+ 'FileEntry', [('name', str), ('path', str), ('stat', StatResult)])
+
+
+class FileEntry(_FileEntry):
+
+ def inode(self) -> Optional[Union[int, str]]:
+ return self.stat.st_ino
+
+ def is_file(self) -> bool:
+ return self.stat.is_file()
+
+ def is_dir(self) -> bool:
+ return self.stat.is_dir()
+
+ def is_symlink(self) -> bool:
+ return self.stat.is_symlink()
+
+
+def method_not_implemented(func):
+
+ @wraps(func)
+ def wrapper(self, *args, **kwargs):
+ raise NotImplementedError(
+ 'method %r not implemented: %r' % (func.__name__, self))
+
+ return wrapper
+
+
+class BasePath:
+
+ def __init__(self, path: "PathLike"):
+ self.path = str(path)
+
+ def __str__(self) -> str:
+ return self.path
+
+ def __repr__(self) -> str:
+ return '%s(%r)' % (self.__class__.__name__, str(self))
+
+ def __bytes__(self) -> bytes:
+ return str(self).encode()
+
+ def __fspath__(self) -> str:
+ return self.path
+
+ def __hash__(self) -> int:
+ return hash(fspath(self))
+
+ def __eq__(self, other_path: "BasePath") -> bool:
+ return fspath(self) == fspath(other_path)
+
+ # pytype: disable=bad-return-type
+
+ @method_not_implemented
+ def is_dir(self, followlinks: bool = False) -> bool: # type: ignore
+ """Return True if the path points to a directory."""
+
+ @method_not_implemented
+ def is_file(self, followlinks: bool = False) -> bool: # type: ignore
+ """Return True if the path points to a regular file."""
+
+ def is_symlink(self) -> bool:
+ return False
+
+ @method_not_implemented
+ def access(self, mode: Access) -> bool: # type: ignore
+ """Return True if the path has access permission described by mode."""
+
+ @method_not_implemented
+ def exists(self, followlinks: bool = False) -> bool: # type: ignore
+ """Whether the path points to an existing file or directory."""
+
+ # listdir or iterdir?
+ @method_not_implemented
+ def listdir(self) -> List[str]: # type: ignore
+ """Return the names of the entries in the directory the path points to."""
+
+ @method_not_implemented
+ def scandir(self) -> Iterator[FileEntry]: # type: ignore
+ """Return an iterator of FileEntry objects corresponding to the entries in the directory."""
+
+ @method_not_implemented
+ def getsize(self, follow_symlinks: bool = True) -> int: # type: ignore
+ """Return the size, in bytes."""
+
+ @method_not_implemented
+ def getmtime(self, follow_symlinks: bool = True) -> float: # type: ignore
+ """Return the time of last modification."""
+
+ @method_not_implemented
+ def stat(self, follow_symlinks=True) -> StatResult: # type: ignore
+ """Get the status of the path."""
+
+ @method_not_implemented
+ def remove(self, missing_ok: bool = False) -> None:
+ """Remove (delete) the file."""
+
+ @method_not_implemented
+ def unlink(self, missing_ok: bool = False) -> None:
+ """Remove (delete) the file."""
+
+ @method_not_implemented
+ def mkdir(
+ self, mode=0o777, parents: bool = False,
+ exist_ok: bool = False) -> None:
+ """Create a directory."""
+
+ @method_not_implemented
+ def rmdir(self) -> None:
+ """Remove (delete) the directory."""
+
+ @method_not_implemented
+ def open(self, mode: str = 'r', **kwargs) -> IO[AnyStr]: # type: ignore
+ """Open the file with mode."""
+
+ @method_not_implemented
+ def walk(self, followlinks: bool = False
+ ) -> Iterator[Tuple[str, List[str], List[str]]]: # type: ignore
+ """Generate the file names in a directory tree by walking the tree."""
+
+ @method_not_implemented
+ def scan(self, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[str]: # type: ignore
+ """Iterate through the files in the directory."""
+
+ @method_not_implemented
+ def scan_stat(self, missing_ok: bool = True, followlinks: bool = False
+ ) -> Iterator[FileEntry]: # type: ignore
+ """Iterate through the files in the directory, with file stat."""
+
+ @method_not_implemented
+ def glob(self, pattern, recursive: bool = True,
+ missing_ok: bool = True) -> List['BasePath']: # type: ignore
+ """Return files whose paths match the glob pattern."""
+
+ @method_not_implemented
+ def iglob(self, pattern, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator['BasePath']: # type: ignore
+ """Return an iterator of files whose paths match the glob pattern."""
+
+ @method_not_implemented
+ def glob_stat(
+ self, pattern, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator[FileEntry]: # type: ignore
+ """Return an iterator of files with stat whose paths match the glob pattern."""
+
+ @method_not_implemented
+ def load(self) -> BinaryIO: # type: ignore
+ """Read all content in binary."""
+
+ @method_not_implemented
+ def save(self, file_object: BinaryIO):
+ """Write the opened binary stream to the path."""
+
+ @method_not_implemented
+ def joinpath(self, *other_paths: "PathLike") -> 'BasePath': # type: ignore
+ """Join or or more path."""
+
+ @method_not_implemented
+ def abspath(self): # type: ignore
+ """Return a normalized absolutized version of the path."""
+
+ @method_not_implemented
+ def realpath(self): # type: ignore
+ """Return the canonical path of the path."""
+
+ @method_not_implemented
+ def relpath(self, start=None): # type: ignore
+ """Return the relative path."""
+
+ @method_not_implemented
+ def is_absolute(self) -> bool: # type: ignore
+ """Return True if the path is an absolute pathname."""
+
+ @method_not_implemented
+ def is_mount(self) -> bool: # type: ignore
+ """Return True if the path is a mount point."""
+
+ @method_not_implemented
+ def resolve(self): # type: ignore
+ """Alias of realpath."""
+
+ def touch(self):
+ with self.open('w'):
+ pass
+
+ # will be deleted in next version
+ def is_link(self) -> bool:
+ return self.is_symlink()
+
+ def makedirs(self, exist_ok: bool = False) -> None:
+ '''
+ Recursive directory creation function. Like mkdir(), but makes all intermediate-level directories needed to contain the leaf directory.
+ '''
+ self.mkdir(parents=True, exist_ok=exist_ok)
+
+ # pytype: enable=bad-return-type
+
+
+PathLike = Union[str, BasePath, _PathLike]
+
+
+class BaseURIPath(BasePath):
+
+ # #####
+ # Backwards compatible API, will be removed in megfile 1.0
+ @classmethod
+ def get_protocol(self) -> Optional[str]:
+ pass # pragma: no cover
+
+ @classproperty
+ def protocol(cls) -> str:
+ return cls.get_protocol()
+
+ def make_uri(self) -> str:
+ return self.path_with_protocol
+
+ def as_uri(self) -> str:
+ return self.make_uri()
+
+ # #####
+
+ @cachedproperty
+ def path_with_protocol(self) -> str:
+ '''Return path with protocol, like file:///root, s3://bucket/key'''
+ path = self.path
+ protocol_prefix = self.protocol + "://"
+ if path.startswith(protocol_prefix):
+ return path
+ return protocol_prefix + path.lstrip('/')
+
+ @cachedproperty
+ def path_without_protocol(self) -> str:
+ '''Return path without protocol, example: if path is s3://bucket/key, return bucket/key'''
+ path = self.path
+ protocol_prefix = self.protocol + "://"
+ if path.startswith(protocol_prefix):
+ path = path[len(protocol_prefix):]
+ return path
+
+ def as_posix(self) -> str:
+ '''Return a string representation of the path with forward slashes (/)'''
+ return self.path_with_protocol
+
+ @classmethod
+ def from_path(cls, path) -> "BaseURIPath":
+ """Return new instance of this class
+
+ :param path: new path
+ :return: new instance of new path
+ :rtype: BaseURIPath
+ """
+ return cls(path)
+
+ @classmethod
+ def from_uri(cls, path: str) -> "BaseURIPath":
+ protocol_prefix = cls.protocol + "://"
+ if path[:len(protocol_prefix)] != protocol_prefix:
+ raise ValueError(
+ "protocol not match, expected: %r, got: %r" %
+ (cls.protocol, path))
+ return cls.from_path(path[len(protocol_prefix):])
+
+ def __fspath__(self) -> str:
+ return self.as_uri()
+
+ def __lt__(self, other_path: "BaseURIPath") -> bool:
+ if not isinstance(other_path, BaseURIPath):
+ raise TypeError("%r is not 'URIPath'" % other_path)
+ if self.protocol != other_path.protocol:
+ raise TypeError(
+ "'<' not supported between instances of %r and %r" %
+ (type(self), type(other_path)))
+ return fspath(self) < fspath(other_path)
+
+ def __le__(self, other_path: "BaseURIPath") -> bool:
+ if not isinstance(other_path, BaseURIPath):
+ raise TypeError("%r is not 'URIPath'" % other_path)
+ if self.protocol != other_path.protocol:
+ raise TypeError(
+ "'<=' not supported between instances of %r and %r" %
+ (type(self), type(other_path)))
+ return str(self) <= str(other_path)
+
+ def __gt__(self, other_path: "BaseURIPath") -> bool:
+ if not isinstance(other_path, BaseURIPath):
+ raise TypeError("%r is not 'URIPath'" % other_path)
+ if self.protocol != other_path.protocol:
+ raise TypeError(
+ "'>' not supported between instances of %r and %r" %
+ (type(self), type(other_path)))
+ return str(self) > str(other_path)
+
+ def __ge__(self, other_path: "BaseURIPath") -> bool:
+ if not isinstance(other_path, BaseURIPath):
+ raise TypeError("%r is not 'URIPath'" % other_path)
+ if self.protocol != other_path.protocol:
+ raise TypeError(
+ "'>=' not supported between instances of %r and %r" %
+ (type(self), type(other_path)))
+ return str(self) >= str(other_path)
+
+ @classproperty
+ def drive(self) -> str:
+ return ''
+
+ @classproperty
+ def root(self) -> str:
+ return self.protocol + '://'
+
+ @classproperty
+ def anchor(self) -> str:
+ return self.root
+
+
+class URIPath(BaseURIPath):
+
+ def __init__(self, path: "PathLike", *other_paths: "PathLike"):
+ if len(other_paths) > 0:
+ path = self.from_path(path).joinpath(*other_paths)
+ self.path = str(path)
+
+ def __truediv__(self, other_path: PathLike) -> "BaseURIPath":
+ if isinstance(other_path, BaseURIPath):
+ if self.protocol != other_path.protocol:
+ raise TypeError(
+ "'/' not supported between instances of %r and %r" %
+ (type(self), type(other_path)))
+ elif not isinstance(other_path, str):
+ raise TypeError("%r is not 'str' nor 'URIPath'" % other_path)
+ return self.joinpath(other_path)
+
+ def joinpath(self, *other_paths: PathLike) -> "BaseURIPath":
+ '''Calling this method is equivalent to combining the path with each of the other arguments in turn'''
+ return self.from_path(uri_join(str(self), *map(str, other_paths)))
+
+ @cachedproperty
+ def parts(self) -> Tuple[str]:
+ '''A tuple giving access to the path’s various components'''
+ parts = [self.root]
+ path = self.path_without_protocol
+ path = path.lstrip('/')
+ if path != '':
+ parts.extend(path.split('/'))
+ return tuple(parts)
+
+ @cachedproperty
+ def parents(self) -> "URIPathParents":
+ '''An immutable sequence providing access to the logical ancestors of the path'''
+ return URIPathParents(self)
+
+ @cachedproperty
+ def parent(self) -> "BaseURIPath":
+ '''The logical parent of the path'''
+ if self.path_without_protocol == "/":
+ return self
+ elif len(self.parents) > 0:
+ return self.parents[0]
+ return self.from_path("")
+
+ @cachedproperty
+ def name(self) -> str:
+ '''A string representing the final path component, excluding the drive and root'''
+ parts = self.parts
+ if len(parts) == 1 and parts[0] == self.protocol + "://":
+ return ''
+ return parts[-1]
+
+ @cachedproperty
+ def suffix(self) -> str:
+ '''The file extension of the final component'''
+ name = self.name
+ i = name.rfind('.')
+ if 0 < i < len(name) - 1:
+ return name[i:]
+ return ''
+
+ @cachedproperty
+ def suffixes(self) -> List[str]:
+ '''A list of the path’s file extensions'''
+ name = self.name
+ if name.endswith('.'):
+ return []
+ name = name.lstrip('.')
+ return ['.' + suffix for suffix in name.split('.')[1:]]
+
+ @cachedproperty
+ def stem(self) -> str:
+ '''The final path component, without its suffix'''
+ name = self.name
+ i = name.rfind('.')
+ if 0 < i < len(name) - 1:
+ return name[:i]
+ return name
+
+ def is_reserved(self) -> bool:
+ return False
+
+ def match(self, pattern) -> bool:
+ '''Match this path against the provided glob-style pattern. Return True if matching is successful, False otherwise'''
+ match = _compile_pattern(pattern)
+ for index in range(len(self.parts), 0, -1):
+ path = '/'.join(self.parts[index:])
+ if match(path) is not None:
+ return True
+ return match(self.path_with_protocol) is not None
+
+ def is_relative_to(self, *other) -> bool:
+
+ try:
+ self.relative_to(*other)
+ return True
+ except Exception:
+ return False
+
+ def relative_to(self, *other) -> "BaseURIPath":
+ '''
+ Compute a version of this path relative to the path represented by other.
+ If it’s impossible, ValueError is raised.
+ '''
+ if not other:
+ raise TypeError("need at least one argument")
+
+ other_path = self.from_path(other[0])
+ if len(other) > 0:
+ other_path = other_path.joinpath(*other[1:])
+ other_path = other_path.path_with_protocol
+ path = self.path_with_protocol
+
+ if path.startswith(other_path):
+ relative = path[len(other_path):]
+ relative = relative.lstrip('/')
+ return type(self)(relative)
+ else:
+ raise ValueError("%r does not start with %r" % (path, other))
+
+ def with_name(self, name) -> "BaseURIPath":
+ '''Return a new path with the name changed'''
+ path = str(self)
+ raw_name = self.name
+ return self.from_path(path[:len(path) - len(raw_name)] + name)
+
+ def with_stem(self, stem) -> "BaseURIPath":
+ '''Return a new path with the stem changed'''
+ return self.with_name("".join([stem, self.suffix]))
+
+ def with_suffix(self, suffix) -> "BaseURIPath":
+ '''Return a new path with the suffix changed'''
+ path = str(self)
+ raw_suffix = self.suffix
+ return self.from_path(path[:len(path) - len(raw_suffix)] + suffix)
+
+ def is_absolute(self) -> bool:
+ return True
+
+ def is_mount(self) -> bool:
+ '''Test whether a path is a mount point
+
+ :returns: True if a path is a mount point, else False
+ '''
+ return False
+
+ def is_socket(self) -> bool:
+ '''
+ Return True if the path points to a Unix socket (or a symbolic link pointing to a Unix socket), False if it points to another kind of file.
+
+ False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
+ '''
+ return False
+
+ def is_fifo(self) -> bool:
+ '''
+ Return True if the path points to a FIFO (or a symbolic link pointing to a FIFO), False if it points to another kind of file.
+
+ False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
+ '''
+ return False
+
+ def is_block_device(self) -> bool:
+ '''
+ Return True if the path points to a block device (or a symbolic link pointing to a block device), False if it points to another kind of file.
+
+ False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
+ '''
+ return False
+
+ def is_char_device(self) -> bool:
+ '''
+ Return True if the path points to a character device (or a symbolic link pointing to a character device), False if it points to another kind of file.
+
+ False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
+ '''
+ return False
+
+ def abspath(self) -> str:
+ """Return a normalized absolutized version of the path."""
+ return self.path_with_protocol
+
+ def realpath(self) -> str:
+ """Return the canonical path of the path."""
+ return self.path_with_protocol
+
+ def resolve(self):
+ """Alias of realpath."""
+ return self.path_with_protocol
+
+ def chmod(self, mode: int, *, follow_symlinks: bool = True):
+ raise NotImplementedError(f"'chmod' is unsupported on '{type(self)}'")
+
+ def lchmod(self, mode: int):
+ '''
+ Like chmod() but, if the path points to a symbolic link, the symbolic link’s mode is changed rather than its target’s.
+ '''
+ return self.chmod(mode=mode, follow_symlinks=False)
+
+ def read_bytes(self) -> bytes:
+ '''Return the binary contents of the pointed-to file as a bytes object'''
+ with self.open(mode='rb') as f:
+ return f.read()
+
+ def read_text(self) -> str:
+ '''Return the decoded contents of the pointed-to file as a string'''
+ with self.open(mode='r') as f:
+ return f.read()
+
+ def rename(self, dst_path: PathLike, overwrite: bool = True) -> 'URIPath':
+ '''
+ rename file
+
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ raise NotImplementedError(f"'rename' is unsupported on '{type(self)}'")
+
+ def replace(self, dst_path: PathLike, overwrite: bool = True) -> 'URIPath':
+ '''
+ move file
+
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ return self.rename(dst_path=dst_path, overwrite=overwrite)
+
+ def rglob(self, pattern) -> List['URIPath']:
+ '''
+ This is like calling Path.glob() with “**/” added in front of the given relative pattern
+ '''
+ if not pattern:
+ pattern = ""
+ pattern = '**/' + pattern.lstrip('/')
+ return self.glob(pattern=pattern)
+
+ def md5(self, recalculate: bool = False, followlinks: bool = False) -> str:
+ raise NotImplementedError(f"'md5' is unsupported on '{type(self)}'")
+
+ def samefile(self, other_path) -> bool:
+ '''
+ Return whether this path points to the same file
+ '''
+ if hasattr(other_path, 'protocol'):
+ if other_path.protocol != self.protocol:
+ return False
+
+ stat = self.stat()
+ if hasattr(other_path, 'stat'):
+ other_path_stat = other_path.stat()
+ else:
+ other_path_stat = self.from_path(other_path).stat()
+
+ return stat.st_ino == other_path_stat.st_ino and stat.st_dev == other_path_stat.st_dev
+
+ def symlink(self, dst_path: PathLike) -> None:
+ raise NotImplementedError(f"'symlink' is unsupported on '{type(self)}'")
+
+ def symlink_to(self, target, target_is_directory=False):
+ '''
+ Make this path a symbolic link to target.
+ symlink_to's arguments is the reverse of symlink's.
+ Target_is_directory’s value is ignored, only be compatible with pathlib.Path
+ '''
+ return self.from_path(
+ target).symlink( # type: ignore
+ dst_path=self.path)
+
+ def hardlink_to(self, target):
+ '''
+ Make this path a hard link to the same file as target.
+ '''
+ raise NotImplementedError(
+ f"'hardlink_to' is unsupported on '{type(self)}'")
+
+ def write_bytes(self, data: bytes):
+ '''Open the file pointed to in bytes mode, write data to it, and close the file'''
+ with self.open(mode='wb') as f:
+ return f.write(data)
+
+ def write_text(self, data: str, encoding=None, errors=None, newline=None):
+ '''
+ Open the file pointed to in text mode, write data to it, and close the file.
+ The optional parameters have the same meaning as in open().
+ '''
+ with self.open(mode='w', encoding=encoding, errors=errors,
+ newline=newline) as f:
+ return f.write(data)
+
+ def home(self):
+ '''Return the home directory
+
+ returns: Home directory path
+ '''
+ raise NotImplementedError(f"'home' is unsupported on '{type(self)}'")
+
+ def group(self):
+ """
+ Return the name of the group owning the file.
+ """
+ raise NotImplementedError(f"'group' is unsupported on '{type(self)}'")
+
+ def expanduser(self):
+ """
+ Return a new path with expanded ~ and ~user constructs, as returned by os.path.expanduser().
+ Only fs path support this method.
+ """
+ raise NotImplementedError(
+ f"'expanduser' is unsupported on '{type(self)}'")
+
+ def cwd(self) -> 'URIPath':
+ '''Return current working directory
+
+ returns: Current working directory
+ '''
+ raise NotImplementedError(f"'cwd' is unsupported on '{type(self)}'")
+
+ def iterdir(self) -> Iterator['URIPath']:
+ '''
+ Get all contents of given fs path. The result is in acsending alphabetical order.
+
+ :returns: All contents have in the path in acsending alphabetical order
+ '''
+ raise NotImplementedError(f"'iterdir' is unsupported on '{type(self)}'")
+
+ def owner(self) -> str:
+ '''
+ Return the name of the user owning the file.
+ '''
+ raise NotImplementedError(f"'owner' is unsupported on '{type(self)}'")
+
+ def absolute(self) -> 'URIPath':
+ '''
+ Make the path absolute, without normalization or resolving symlinks. Returns a new path object
+ '''
+ raise NotImplementedError(
+ f"'absolute' is unsupported on '{type(self)}'")
+
+ def utime(self, atime: Union[float, int], mtime: Union[float, int]):
+ """
+ Sets the access and modified times of the file specified by path to the specified values.
+
+ :param atime: The access time to be set.
+ :type atime: Union[float, int]
+ :param mtime: The modification time to be set.
+ :type mtime: Union[float, int]
+ :raises NotImplementedError: Always raised, since the functionality is unsupported.
+ """
+ raise NotImplementedError(f"'utime' is unsupported on '{type(self)}'")
+
+ def lstat(self) -> StatResult:
+ '''Like stat() but, if the path points to a symbolic link, return the symbolic link’s information rather than its target’s.'''
+ return self.stat(follow_symlinks=False)
+
+
+class URIPathParents(Sequence):
+
+ def __init__(self, path):
+ # We don't store the instance to avoid reference cycles
+ self.cls = type(path)
+ parts = path.parts
+ if len(parts) > 0 and parts[0] == path.protocol + "://":
+ self.prefix = parts[0]
+ self.parts = parts[1:]
+ else:
+ self.prefix = ''
+ self.parts = parts
+
+ def __len__(self):
+ return max(len(self.parts) - 1, 0)
+
+ def __getitem__(self, idx):
+ if idx < 0 or idx > len(self):
+ raise IndexError(idx)
+
+ if len(self.parts[:-idx - 1]) > 1:
+ other_path = os.path.join(*self.parts[:-idx - 1])
+ elif len(self.parts[:-idx - 1]) == 1:
+ other_path = self.parts[:-idx - 1][0]
+ else:
+ other_path = ""
+ return self.cls(self.prefix + other_path)
+
+from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple
+
+from megfile.interfaces import Access, FileEntry, PathLike, StatResult
+from megfile.s3_path import S3BufferedWriter, S3Cacher, S3LimitedSeekableWriter, S3Path, S3PrefetchReader, S3ShareCacheReader, get_endpoint_url, get_s3_client, get_s3_session, is_s3, parse_s3_url, s3_buffered_open, s3_cached_open, s3_concat, s3_download, s3_glob, s3_glob_stat, s3_iglob, s3_load_content, s3_lstat, s3_makedirs, s3_memory_open, s3_open, s3_path_join, s3_pipe_open, s3_prefetch_open, s3_readlink, s3_rename, s3_share_cache_open, s3_upload
+
+__all__ = [
+ 'parse_s3_url',
+ 'get_endpoint_url',
+ 'get_s3_session',
+ 'get_s3_client',
+ 's3_path_join',
+ 'is_s3',
+ 's3_buffered_open',
+ 's3_cached_open',
+ 's3_memory_open',
+ 's3_pipe_open',
+ 's3_prefetch_open',
+ 's3_share_cache_open',
+ 's3_open',
+ 'S3Cacher',
+ 'S3BufferedWriter',
+ 'S3LimitedSeekableWriter',
+ 'S3PrefetchReader',
+ 'S3ShareCacheReader',
+ 's3_upload',
+ 's3_download',
+ 's3_load_content',
+ 's3_readlink',
+ 's3_glob',
+ 's3_glob_stat',
+ 's3_iglob',
+ 's3_rename',
+ 's3_makedirs',
+ 's3_concat',
+ 's3_lstat',
+ 's3_access',
+ 's3_exists',
+ 's3_getmtime',
+ 's3_getsize',
+ 's3_isdir',
+ 's3_isfile',
+ 's3_listdir',
+ 's3_load_from',
+ 's3_hasbucket',
+ 's3_move',
+ 's3_remove',
+ 's3_scan',
+ 's3_scan_stat',
+ 's3_scandir',
+ 's3_stat',
+ 's3_unlink',
+ 's3_walk',
+ 's3_getmd5',
+ 's3_copy',
+ 's3_sync',
+ 's3_symlink',
+ 's3_islink',
+ 's3_save_as',
+]
+
+
+[docs]def s3_access(
+ path: PathLike, mode: Access = Access.READ,
+ followlinks: bool = False) -> bool:
+ '''
+ Test if path has access permission described by mode
+
+ :param path: Given path
+ :param mode: access mode
+ :returns: bool, if the bucket of s3_url has read/write access.
+ '''
+ return S3Path(path).access(mode, followlinks)
+
+
+[docs]def s3_exists(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if s3_url exists
+
+ If the bucket of s3_url are not permitted to read, return False
+
+ :param path: Given path
+ :returns: True if s3_url eixsts, else False
+ '''
+ return S3Path(path).exists(followlinks)
+
+
+[docs]def s3_getmtime(path: PathLike, follow_symlinks: bool = False) -> float:
+ '''
+ Get last-modified time of the file on the given s3_url path (in Unix timestamp format).
+ If the path is an existent directory, return the latest modified time of all file in it. The mtime of empty directory is 1970-01-01 00:00:00
+
+ If s3_url is not an existent path, which means s3_exist(s3_url) returns False, then raise S3FileNotFoundError
+
+ :param path: Given path
+ :returns: Last-modified time
+ :raises: S3FileNotFoundError, UnsupportedError
+ '''
+ return S3Path(path).getmtime(follow_symlinks)
+
+
+[docs]def s3_getsize(path: PathLike, follow_symlinks: bool = False) -> int:
+ '''
+ Get file size on the given s3_url path (in bytes).
+ If the path in a directory, return the sum of all file size in it, including file in subdirectories (if exist).
+ The result excludes the size of directory itself. In other words, return 0 Byte on an empty directory path.
+
+ If s3_url is not an existent path, which means s3_exist(s3_url) returns False, then raise S3FileNotFoundError
+
+ :param path: Given path
+ :returns: File size
+ :raises: S3FileNotFoundError, UnsupportedError
+ '''
+ return S3Path(path).getsize(follow_symlinks)
+
+
+[docs]def s3_isdir(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if an s3 url is directory
+ Specific procedures are as follows:
+ If there exists a suffix, of which ``os.path.join(s3_url, suffix)`` is a file
+ If the url is empty bucket or s3://
+
+ :param path: Given path
+ :param followlinks: whether followlinks is True or False, result is the same. Because s3 symlink not support dir.
+ :returns: True if path is s3 directory, else False
+ '''
+ return S3Path(path).is_dir(followlinks)
+
+
+[docs]def s3_isfile(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if an s3_url is file
+
+ :param path: Given path
+ :returns: True if path is s3 file, else False
+ '''
+ return S3Path(path).is_file(followlinks)
+
+
+[docs]def s3_listdir(path: PathLike, followlinks: bool = False) -> List[str]:
+ '''
+ Get all contents of given s3_url. The result is in acsending alphabetical order.
+
+ :param path: Given path
+ :returns: All contents have prefix of s3_url in acsending alphabetical order
+ :raises: S3FileNotFoundError, S3NotADirectoryError
+ '''
+ return S3Path(path).listdir(followlinks)
+
+
+[docs]def s3_load_from(path: PathLike, followlinks: bool = False) -> BinaryIO:
+ '''Read all content in binary on specified path and write into memory
+
+ User should close the BinaryIO manually
+
+ :param path: Given path
+ :returns: BinaryIO
+ '''
+ return S3Path(path).load(followlinks)
+
+
+[docs]def s3_hasbucket(path: PathLike) -> bool:
+ '''
+ Test if the bucket of s3_url exists
+
+ :param path: Given path
+ :returns: True if bucket of s3_url eixsts, else False
+ '''
+ return S3Path(path).hasbucket()
+
+
+[docs]def s3_move(
+ src_url: PathLike, dst_url: PathLike, overwrite: bool = True) -> None:
+ '''
+ Move file/directory path from src_url to dst_url
+
+ :param src_url: Given path
+ :param dst_url: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ return S3Path(src_url).move(dst_url, overwrite)
+
+
+[docs]def s3_remove(path: PathLike, missing_ok: bool = False) -> None:
+ '''
+ Remove the file or directory on s3, `s3://` and `s3://bucket` are not permitted to remove
+
+ :param path: Given path
+ :param missing_ok: if False and target file/directory not exists, raise S3FileNotFoundError
+ :raises: S3PermissionError, S3FileNotFoundError, UnsupportedError
+ '''
+ return S3Path(path).remove(missing_ok)
+
+
+[docs]def s3_scan(path: PathLike, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[str]:
+ '''
+ Iteratively traverse only files in given s3 directory, in alphabetical order.
+ Every iteration on generator yields a path string.
+
+ If s3_url is a file path, yields the file only
+ If s3_url is a non-existent path, return an empty generator
+ If s3_url is a bucket path, return all file paths in the bucket
+ If s3_url is an empty bucket, return an empty generator
+ If s3_url doesn't contain any bucket, which is s3_url == 's3://', raise UnsupportedError. walk() on complete s3 is not supported in megfile
+
+ :param path: Given path
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :raises: UnsupportedError
+ :returns: A file path generator
+ '''
+ return S3Path(path).scan(missing_ok, followlinks)
+
+
+[docs]def s3_scan_stat(
+ path: PathLike, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a tuple of path string and file stat
+
+ :param path: Given path
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :raises: UnsupportedError
+ :returns: A file path generator
+ '''
+ return S3Path(path).scan_stat(missing_ok, followlinks)
+
+
+[docs]def s3_scandir(path: PathLike,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Get all contents of given s3_url, the order of result is not guaranteed.
+
+ :param path: Given path
+ :returns: All contents have prefix of s3_url
+ :raises: S3FileNotFoundError, S3NotADirectoryError
+ '''
+ return S3Path(path).scandir(followlinks)
+
+
+[docs]def s3_stat(path: PathLike, follow_symlinks=True) -> StatResult:
+ '''
+ Get StatResult of s3_url file, including file size and mtime, referring to s3_getsize and s3_getmtime
+
+ If s3_url is not an existent path, which means s3_exist(s3_url) returns False, then raise S3FileNotFoundError
+ If attempt to get StatResult of complete s3, such as s3_dir_url == 's3://', raise S3BucketNotFoundError
+
+ :param path: Given path
+ :returns: StatResult
+ :raises: S3FileNotFoundError, S3BucketNotFoundError
+ '''
+ return S3Path(path).stat(follow_symlinks)
+
+
+[docs]def s3_unlink(path: PathLike, missing_ok: bool = False) -> None:
+ '''
+ Remove the file on s3
+
+ :param path: Given path
+ :param missing_ok: if False and target file not exists, raise S3FileNotFoundError
+ :raises: S3PermissionError, S3FileNotFoundError, S3IsADirectoryError
+ '''
+ return S3Path(path).unlink(missing_ok)
+
+
+[docs]def s3_walk(path: PathLike, followlinks: bool = False
+ ) -> Iterator[Tuple[str, List[str], List[str]]]:
+ '''
+ Iteratively traverse the given s3 directory, in top-bottom order. In other words, firstly traverse parent directory, if subdirectories exist, traverse the subdirectories in alphabetical order.
+ Every iteration on generator yields a 3-tuple: (root, dirs, files)
+
+ - root: Current s3 path;
+ - dirs: Name list of subdirectories in current directory. The list is sorted by name in ascending alphabetical order;
+ - files: Name list of files in current directory. The list is sorted by name in ascending alphabetical order;
+
+ If s3_url is a file path, return an empty generator
+ If s3_url is a non-existent path, return an empty generator
+ If s3_url is a bucket path, bucket will be the top directory, and will be returned at first iteration of generator
+ If s3_url is an empty bucket, only yield one 3-tuple (notes: s3 doesn't have empty directory)
+ If s3_url doesn't contain any bucket, which is s3_url == 's3://', raise UnsupportedError. walk() on complete s3 is not supported in megfile
+
+ :param path: Given path
+ :param followlinks: whether followlinks is True or False, result is the same. Because s3 symlink not support dir.
+ :raises: UnsupportedError
+ :returns: A 3-tuple generator
+ '''
+ return S3Path(path).walk(followlinks)
+
+
+[docs]def s3_getmd5(
+ path: PathLike, recalculate: bool = False,
+ followlinks: bool = False) -> str:
+ '''
+ Get md5 meta info in files that uploaded/copied via megfile
+
+ If meta info is lost or non-existent, return None
+
+ :param path: Given path
+ :param recalculate: calculate md5 in real-time or return s3 etag
+ :param followlinks: If is True, calculate md5 for real file
+ :returns: md5 meta info
+ '''
+ return S3Path(path).md5(recalculate, followlinks)
+
+
+[docs]def s3_copy(
+ src_url: PathLike,
+ dst_url: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ followlinks: bool = False,
+ overwrite: bool = True) -> None:
+ ''' File copy on S3
+ Copy content of file on `src_path` to `dst_path`.
+ It's caller's responsebility to ensure the s3_isfile(src_url) == True
+
+ :param src_url: Given path
+ :param dst_path: Target file path
+ :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+ :param followlinks: False if regard symlink as file, else True
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ return S3Path(src_url).copy(dst_url, callback, followlinks, overwrite)
+
+
+[docs]def s3_sync(
+ src_url: PathLike,
+ dst_url: PathLike,
+ followlinks: bool = False,
+ force: bool = False,
+ overwrite: bool = True) -> None:
+ '''
+ Copy file/directory on src_url to dst_url
+
+ :param src_url: Given path
+ :param dst_url: Given destination path
+ :param followlinks: False if regard symlink as file, else True
+ :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ return S3Path(src_url).sync(dst_url, followlinks, force, overwrite)
+
+
+[docs]def s3_symlink(src_path: PathLike, dst_path: PathLike) -> None:
+ '''
+ Create a symbolic link pointing to src_path named dst_path.
+
+ :param src_path: Given path
+ :param dst_path: Desination path
+ :raises: S3NameTooLongError, S3BucketNotFoundError, S3IsADirectoryError
+ '''
+ return S3Path(src_path).symlink(dst_path)
+
+
+[docs]def s3_islink(path: PathLike) -> bool:
+ '''
+ Test whether a path is link
+
+ :param path: Given path
+ :returns: True if a path is link, else False
+ :raises: S3NotALinkError
+ '''
+ return S3Path(path).is_symlink()
+
+
+[docs]def s3_save_as(file_object: BinaryIO, path: PathLike):
+ '''Write the opened binary stream to specified path, but the stream won't be closed
+
+ :param path: Given path
+ :param file_object: Stream to be read
+ '''
+ return S3Path(path).save(file_object)
+
+import hashlib
+import io
+import os
+import re
+import time
+from concurrent.futures import ThreadPoolExecutor
+from functools import lru_cache, wraps
+from logging import getLogger as get_logger
+from typing import IO, Any, AnyStr, BinaryIO, Callable, Dict, Iterator, List, Optional, Tuple, Union
+
+import boto3
+import botocore
+from botocore.awsrequest import AWSResponse
+
+from megfile.config import DEFAULT_BLOCK_SIZE, GLOBAL_MAX_WORKERS, S3_CLIENT_CACHE_MODE, S3_MAX_RETRY_TIMES
+from megfile.errors import S3BucketNotFoundError, S3ConfigError, S3FileExistsError, S3FileNotFoundError, S3IsADirectoryError, S3NameTooLongError, S3NotADirectoryError, S3NotALinkError, S3PermissionError, S3UnknownError, SameFileError, UnsupportedError, _create_missing_ok_generator
+from megfile.errors import _logger as error_logger
+from megfile.errors import patch_method, raise_s3_error, s3_error_code_should_retry, s3_should_retry, translate_fs_error, translate_s3_error
+from megfile.interfaces import Access, ContextIterator, FileCacher, FileEntry, PathLike, StatResult, URIPath
+from megfile.lib.compare import is_same_file
+from megfile.lib.compat import fspath
+from megfile.lib.fnmatch import translate
+from megfile.lib.glob import has_magic, has_magic_ignore_brace, ungloblize
+from megfile.lib.joinpath import uri_join
+from megfile.lib.s3_buffered_writer import DEFAULT_MAX_BUFFER_SIZE, GLOBAL_MAX_WORKERS, S3BufferedWriter
+from megfile.lib.s3_cached_handler import S3CachedHandler
+from megfile.lib.s3_limited_seekable_writer import S3LimitedSeekableWriter
+from megfile.lib.s3_memory_handler import S3MemoryHandler
+from megfile.lib.s3_pipe_handler import S3PipeHandler
+from megfile.lib.s3_prefetch_reader import S3PrefetchReader
+from megfile.lib.s3_share_cache_reader import S3ShareCacheReader
+from megfile.lib.url import get_url_scheme
+from megfile.smart_path import SmartPath
+from megfile.utils import _is_pickle, cachedproperty, calculate_md5, generate_cache_path, get_binary_mode, get_content_offset, is_readable, necessary_params, process_local, thread_local
+
+__all__ = [
+ 'S3Path',
+ 'parse_s3_url',
+ 'get_endpoint_url',
+ 'get_s3_session',
+ 'get_s3_client',
+ 's3_path_join',
+ 'is_s3',
+ 's3_buffered_open',
+ 's3_cached_open',
+ 's3_memory_open',
+ 's3_pipe_open',
+ 's3_prefetch_open',
+ 's3_share_cache_open',
+ 's3_open',
+ 'S3Cacher',
+ 'S3BufferedWriter',
+ 'S3LimitedSeekableWriter',
+ 'S3PrefetchReader',
+ 'S3ShareCacheReader',
+ 's3_upload',
+ 's3_download',
+ 's3_load_content',
+ 's3_readlink',
+ 's3_glob',
+ 's3_glob_stat',
+ 's3_iglob',
+ 's3_rename',
+ 's3_makedirs',
+ 's3_concat',
+ 's3_lstat',
+]
+_logger = get_logger(__name__)
+content_md5_header = 'megfile-content-md5'
+endpoint_url = 'https://s3.amazonaws.com'
+max_pool_connections = GLOBAL_MAX_WORKERS # for compatibility
+max_retries = S3_MAX_RETRY_TIMES
+max_keys = 1000
+
+
+def _patch_make_request(client: botocore.client.BaseClient):
+
+ def after_callback(result: Tuple[AWSResponse, dict], *args, **kwargs):
+ if not isinstance(result, tuple) or len(result) != 2 \
+ or not isinstance(result[0], AWSResponse) or not isinstance(result[1], dict):
+ return result
+ http, parsed_response = result
+ if http.status_code >= 500:
+ error_code = parsed_response.get("Error", {}).get("Code")
+ operation_model = kwargs.get('operation_model') or (
+ args[0] if args else None)
+ operation_name = operation_model.name if operation_model else 'ProxyMethod'
+ error_class = client.exceptions.from_code(error_code)
+ raise error_class(parsed_response, operation_name)
+ return result
+
+ def retry_callback(error, operation_model, request_dict, request_context):
+ if is_readable(request_dict['body']):
+ request_dict['body'].seek(0)
+
+ def before_callback(operation_model, request_dict, request_context):
+ _logger.debug(
+ 'send s3 request: %r, with parameters: %s', operation_model.name,
+ request_dict)
+
+ client._make_request = patch_method(
+ client._make_request,
+ max_retries=max_retries,
+ should_retry=s3_should_retry,
+ after_callback=after_callback,
+ before_callback=before_callback,
+ retry_callback=retry_callback)
+ return client
+
+
+[docs]def parse_s3_url(s3_url: PathLike) -> Tuple[str, str]:
+ s3_url = fspath(s3_url)
+ if not is_s3(s3_url):
+ raise ValueError('Not a s3 url: %r' % s3_url)
+ rightpart = s3_url.split('://', maxsplit=1)[1]
+ bucketmatch = re.match('(.*?)/', rightpart)
+ if bucketmatch is None:
+ bucket = rightpart
+ path = ''
+ else:
+ bucket = bucketmatch.group(1)
+ path = rightpart[len(bucket) + 1:]
+ return bucket, path
+
+
+def get_scoped_config(profile_name: Optional[str] = None) -> Dict:
+ return get_s3_session(
+ profile_name=profile_name)._session.get_scoped_config()
+
+
+@lru_cache()
+def warning_endpoint_url(key: str, endpoint_url: str):
+ _logger.info("using %s: %s" % (key, endpoint_url))
+
+
+[docs]def get_endpoint_url(profile_name: Optional[str] = None) -> str:
+ '''Get the endpoint url of S3
+
+ :returns: S3 endpoint url
+ '''
+ if profile_name:
+ environ_keys = (f'{profile_name}__OSS_ENDPOINT'.upper(),)
+ else:
+ environ_keys = (
+ 'OSS_ENDPOINT', 'AWS_ENDPOINT_URL_S3', 'AWS_ENDPOINT_URL')
+ for environ_key in environ_keys:
+ environ_endpoint_url = os.environ.get(environ_key)
+ if environ_endpoint_url:
+ warning_endpoint_url(environ_key, environ_endpoint_url)
+ return environ_endpoint_url
+ try:
+ config = get_scoped_config(profile_name=profile_name)
+ config_endpoint_url = config.get('s3', {}).get('endpoint_url')
+ config_endpoint_url = config_endpoint_url or config.get('endpoint_url')
+ if config_endpoint_url:
+ warning_endpoint_url('~/.aws/config', config_endpoint_url)
+ return config_endpoint_url
+ except botocore.exceptions.ProfileNotFound:
+ pass
+ return endpoint_url
+
+
+[docs]def get_s3_session(profile_name=None) -> boto3.Session:
+ '''Get S3 session
+
+ :returns: S3 session
+ '''
+ return thread_local(
+ f's3_session:{profile_name}', boto3.Session, profile_name=profile_name)
+
+
+def get_access_token(profile_name=None):
+ access_key_env_name = f"{profile_name}__AWS_ACCESS_KEY_ID".upper(
+ ) if profile_name else "AWS_ACCESS_KEY_ID"
+ secret_key_env_name = f"{profile_name}__AWS_SECRET_ACCESS_KEY".upper(
+ ) if profile_name else "AWS_SECRET_ACCESS_KEY"
+ access_key = os.getenv(access_key_env_name)
+ secret_key = os.getenv(secret_key_env_name)
+ if access_key and secret_key:
+ return access_key, secret_key
+
+ try:
+ credentials = get_s3_session(
+ profile_name=profile_name).get_credentials()
+ except botocore.exceptions.ProfileNotFound:
+ credentials = None
+ if credentials:
+ if not access_key:
+ access_key = credentials.access_key
+ if not secret_key:
+ secret_key = credentials.secret_key
+ return access_key, secret_key
+
+
+[docs]def get_s3_client(
+ config: Optional[botocore.config.Config] = None,
+ cache_key: Optional[str] = None,
+ profile_name: Optional[str] = None):
+ '''Get S3 client
+
+ :returns: S3 client
+ '''
+ if cache_key is not None:
+ local_storage = thread_local
+ if S3_CLIENT_CACHE_MODE == 'process_local':
+ local_storage = process_local
+ return local_storage(
+ f"{cache_key}:{profile_name}",
+ get_s3_client,
+ config=config,
+ profile_name=profile_name)
+
+ if config:
+ config = botocore.config.Config(
+ connect_timeout=5,
+ max_pool_connections=GLOBAL_MAX_WORKERS).merge(config)
+ else:
+ config = botocore.config.Config(
+ connect_timeout=5, max_pool_connections=GLOBAL_MAX_WORKERS)
+
+ addressing_style_env_key = 'AWS_S3_ADDRESSING_STYLE'
+ if profile_name:
+ addressing_style_env_key = f'{profile_name}__AWS_S3_ADDRESSING_STYLE'.upper(
+ )
+ addressing_style = os.environ.get(addressing_style_env_key)
+ if addressing_style:
+ config = config.merge(
+ botocore.config.Config(s3={'addressing_style': addressing_style}))
+
+ access_key, secret_key = get_access_token(profile_name)
+ try:
+ session = get_s3_session(profile_name=profile_name)
+ except botocore.exceptions.ProfileNotFound:
+ session = get_s3_session()
+ client = session.client(
+ 's3',
+ endpoint_url=get_endpoint_url(profile_name=profile_name),
+ config=config,
+ aws_access_key_id=access_key,
+ aws_secret_access_key=secret_key,
+ )
+ client = _patch_make_request(client)
+ return client
+
+
+def get_s3_client_with_cache(
+ config: Optional[botocore.config.Config] = None,
+ profile_name: Optional[str] = None):
+ return get_s3_client(
+ config=config,
+ cache_key='s3_filelike_client',
+ profile_name=profile_name)
+
+
+[docs]def s3_path_join(path: PathLike, *other_paths: PathLike) -> str:
+ '''
+ Concat 2 or more path to a complete path
+
+ :param path: Given path
+ :param other_paths: Paths to be concatenated
+ :returns: Concatenated complete path
+
+ .. note ::
+
+ The difference between this function and ``os.path.join`` is that this function ignores left side slash (which indicates absolute path) in ``other_paths`` and will directly concat.
+ e.g. os.path.join('/path', 'to', '/file') => '/file', but s3_path_join('/path', 'to', '/file') => '/path/to/file'
+ '''
+ return uri_join(fspath(path), *map(fspath, other_paths))
+
+
+def _list_all_buckets(profile_name: Optional[str] = None) -> List[str]:
+ client = get_s3_client_with_cache(profile_name=profile_name)
+ response = client.list_buckets()
+ return [content['Name'] for content in response['Buckets']]
+
+
+def _parse_s3_url_ignore_brace(s3_url: str) -> Tuple[str, str]:
+ s3_url = fspath(s3_url)
+ s3_scheme, rightpart = s3_url[:5], s3_url[5:]
+ if s3_scheme != 's3://':
+ raise ValueError('Not a s3 url: %r' % s3_url)
+ left_brace = False
+ for current_index, current_character in enumerate(rightpart):
+ if current_character == "/" and left_brace is False:
+ return rightpart[:current_index], rightpart[current_index + 1:]
+ elif current_character == "{":
+ left_brace = True
+ elif current_character == "}":
+ left_brace = False
+ return rightpart, ""
+
+
+def _group_s3path_by_bucket(
+ s3_pathname: str, profile_name: Optional[str] = None) -> List[str]:
+ bucket, key = _parse_s3_url_ignore_brace(s3_pathname)
+ if not bucket:
+ if not key:
+ raise UnsupportedError('Glob whole s3', s3_pathname)
+ raise S3BucketNotFoundError('Empty bucket name: %r' % s3_pathname)
+
+ grouped_path = []
+
+ def generate_s3_path(bucket: str, key: str) -> str:
+ if key:
+ return "s3://%s/%s" % (bucket, key)
+ return "s3://%s%s" % (bucket, "/" if s3_pathname.endswith("/") else "")
+
+ all_bucket = lru_cache(maxsize=1)(_list_all_buckets)
+ for bucketname in ungloblize(bucket):
+ if has_magic(bucketname):
+ split_bucketname = bucketname.split("/", 1)
+ path_part = None
+ if len(split_bucketname) == 2:
+ bucketname, path_part = split_bucketname
+ pattern = re.compile(translate(re.sub(r'\*{2,}', '*', bucketname)))
+
+ for bucket in all_bucket(profile_name):
+ if pattern.fullmatch(bucket) is not None:
+ if path_part is not None:
+ bucket = "%s/%s" % (bucket, path_part)
+ grouped_path.append(generate_s3_path(bucket, key))
+ else:
+ grouped_path.append(generate_s3_path(bucketname, key))
+
+ return grouped_path
+
+
+def _s3_split_magic_ignore_brace(s3_pathname: str) -> Tuple[str, str]:
+ if not s3_pathname:
+ raise ValueError("s3_pathname: %s", s3_pathname)
+
+ has_protocol = False
+ if s3_pathname.startswith("s3://"):
+ has_protocol = True
+ s3_pathname = s3_pathname[5:]
+
+ has_delimiter = False
+ if s3_pathname.endswith("/"):
+ has_delimiter = True
+ s3_pathname = s3_pathname[:-1]
+
+ normal_parts = []
+ magic_parts = []
+ left_brace = False
+ left_index = 0
+ for current_index, current_character in enumerate(s3_pathname):
+ if current_character == "/" and left_brace is False:
+ if has_magic_ignore_brace(s3_pathname[left_index:current_index]):
+ magic_parts.append(s3_pathname[left_index:current_index])
+ if s3_pathname[current_index + 1:]:
+ magic_parts.append(s3_pathname[current_index + 1:])
+ left_index = len(s3_pathname)
+ break
+ normal_parts.append(s3_pathname[left_index:current_index])
+ left_index = current_index + 1
+ elif current_character == "{":
+ left_brace = True
+ elif current_character == "}":
+ left_brace = False
+ if s3_pathname[left_index:]:
+ if has_magic_ignore_brace(s3_pathname[left_index:]):
+ magic_parts.append(s3_pathname[left_index:])
+ else:
+ normal_parts.append(s3_pathname[left_index:])
+
+ if has_protocol and normal_parts:
+ normal_parts.insert(0, "s3:/")
+ elif has_protocol:
+ magic_parts.insert(0, "s3:/")
+
+ if has_delimiter and magic_parts:
+ magic_parts.append("")
+ elif has_delimiter:
+ normal_parts.append("")
+
+ return "/".join(normal_parts), "/".join(magic_parts)
+
+
+def _group_s3path_by_prefix(s3_pathname: str) -> List[str]:
+
+ _, key = parse_s3_url(s3_pathname)
+ if not key:
+ return ungloblize(s3_pathname)
+
+ top_dir, magic_part = _s3_split_magic_ignore_brace(s3_pathname)
+ if not top_dir:
+ return [magic_part]
+ grouped_path = []
+ for pathname in ungloblize(top_dir):
+ if magic_part:
+ pathname = "/".join([pathname, magic_part])
+ grouped_path.append(pathname)
+ return grouped_path
+
+
+def _become_prefix(prefix: str) -> str:
+ if prefix != '' and not prefix.endswith('/'):
+ prefix += '/'
+ return prefix
+
+
+def _s3_split_magic(s3_pathname: str) -> Tuple[str, str]:
+ if not has_magic(s3_pathname):
+ return s3_pathname, ''
+ delimiter = '/'
+ normal_parts = []
+ magic_parts = []
+ all_parts = s3_pathname.split(delimiter)
+ for i, part in enumerate(all_parts):
+ if not has_magic(part):
+ normal_parts.append(part)
+ else:
+ magic_parts = all_parts[i:]
+ break
+ return delimiter.join(normal_parts), delimiter.join(magic_parts)
+
+
+def _list_objects_recursive(
+ s3_client, bucket: str, prefix: str, delimiter: str = ''):
+
+ resp = s3_client.list_objects_v2(
+ Bucket=bucket, Prefix=prefix, Delimiter=delimiter, MaxKeys=max_keys)
+
+ while True:
+ yield resp
+
+ if not resp['IsTruncated']:
+ break
+
+ resp = s3_client.list_objects_v2(
+ Bucket=bucket,
+ Prefix=prefix,
+ Delimiter=delimiter,
+ ContinuationToken=resp['NextContinuationToken'],
+ MaxKeys=max_keys)
+
+
+def _make_stat(content: Dict[str, Any]):
+ return StatResult(
+ islnk=content.get('islnk', False),
+ size=content['Size'],
+ mtime=content['LastModified'].timestamp(),
+ extra=content,
+ )
+
+
+def _s3_glob_stat_single_path(
+ s3_pathname: PathLike,
+ recursive: bool = True,
+ missing_ok: bool = True,
+ followlinks: bool = False,
+ profile_name: Optional[str] = None) -> Iterator[FileEntry]:
+ if not recursive:
+ # If not recursive, replace ** with *
+ s3_pathname = re.sub(r'\*{2,}', '*', s3_pathname)
+ top_dir, wildcard_part = _s3_split_magic(s3_pathname)
+ search_dir = wildcard_part.endswith('/')
+
+ def should_recursive(wildcard_part: str) -> bool:
+ if '**' in wildcard_part:
+ return True
+ for expanded_path in ungloblize(wildcard_part):
+ parts_length = len(expanded_path.split('/'))
+ if parts_length + search_dir >= 2:
+ return True
+ return False
+
+ def create_generator(_s3_pathname) -> Iterator[FileEntry]:
+ top_dir_with_profile = top_dir
+ if profile_name:
+ top_dir_with_profile = f's3+{profile_name}://{top_dir[5:]}'
+ if not S3Path(top_dir_with_profile).exists():
+ return
+ if not has_magic(_s3_pathname):
+ _s3_pathname_obj = S3Path(_s3_pathname)
+ if _s3_pathname_obj.is_file():
+ stat = S3Path(_s3_pathname).stat(follow_symlinks=followlinks)
+ yield FileEntry(
+ _s3_pathname_obj.name, _s3_pathname_obj.path, stat)
+ if _s3_pathname_obj.is_dir():
+ yield FileEntry(
+ _s3_pathname_obj.name, _s3_pathname_obj.path,
+ StatResult(isdir=True))
+ return
+
+ delimiter = ''
+ if not should_recursive(wildcard_part):
+ delimiter = '/'
+
+ dirnames = set()
+ pattern = re.compile(translate(_s3_pathname))
+ bucket, key = parse_s3_url(top_dir)
+ prefix = _become_prefix(key)
+ client = get_s3_client_with_cache(profile_name=profile_name)
+ with raise_s3_error(_s3_pathname):
+ for resp in _list_objects_recursive(client, bucket, prefix,
+ delimiter):
+ for content in resp.get('Contents', []):
+ path = s3_path_join('s3://', bucket, content['Key'])
+ if not search_dir and pattern.match(path):
+ yield FileEntry(
+ S3Path(path).name, path, _make_stat(content))
+ dirname = os.path.dirname(path)
+ while dirname not in dirnames and dirname != top_dir:
+ dirnames.add(dirname)
+ path = dirname + '/' if search_dir else dirname
+ if pattern.match(path):
+ yield FileEntry(
+ S3Path(path).name, path, StatResult(isdir=True))
+ dirname = os.path.dirname(dirname)
+ for common_prefix in resp.get('CommonPrefixes', []):
+ path = s3_path_join(
+ 's3://', bucket, common_prefix['Prefix'])
+ dirname = os.path.dirname(path)
+ if dirname not in dirnames and dirname != top_dir:
+ dirnames.add(dirname)
+ path = dirname + '/' if search_dir else dirname
+ if pattern.match(path):
+ yield FileEntry(
+ S3Path(path).name, path, StatResult(isdir=True))
+
+ return create_generator(s3_pathname)
+
+
+def _s3_scan_pairs(src_url: PathLike,
+ dst_url: PathLike) -> Iterator[Tuple[PathLike, PathLike]]:
+ for src_file_path in S3Path(src_url).scan():
+ content_path = src_file_path[len(src_url):]
+ if len(content_path) > 0:
+ dst_file_path = s3_path_join(dst_url, content_path)
+ else:
+ dst_file_path = dst_url
+ yield src_file_path, dst_file_path
+
+
+[docs]def is_s3(path: PathLike) -> bool:
+ '''
+ 1. According to `aws-cli <https://docs.aws.amazon.com/cli/latest/reference/s3/index.html>`_ , test if a path is s3 path.
+ 2. megfile also support the path like `s3[+profile_name]://bucket/key`
+
+ :param path: Path to be tested
+ :returns: True if path is s3 path, else False
+ '''
+ path = fspath(path)
+ if re.match(r'^s3(\+\w+)?:\/\/', path):
+ return True
+ return False
+
+
+def _s3_binary_mode(s3_open_func):
+
+ @wraps(s3_open_func)
+ def wrapper(
+ s3_url,
+ mode: str = 'rb',
+ encoding: Optional[str] = None,
+ errors: Optional[str] = None,
+ **kwargs):
+ bucket, key = parse_s3_url(s3_url)
+ if not bucket:
+ raise S3BucketNotFoundError('Empty bucket name: %r' % s3_url)
+
+ if not key or key.endswith('/'):
+ raise S3IsADirectoryError('Is a directory: %r' % s3_url)
+
+ if 'x' in mode:
+ if S3Path(s3_url).is_file():
+ raise S3FileExistsError('File exists: %r' % s3_url)
+ mode = mode.replace('x', 'w')
+
+ fileobj = s3_open_func(s3_url, get_binary_mode(mode), **kwargs)
+ if 'b' not in mode:
+ fileobj = io.TextIOWrapper(
+ fileobj, encoding=encoding, errors=errors) # pytype: disable=wrong-arg-types
+ fileobj.mode = mode
+ return fileobj
+
+ return wrapper
+
+
+[docs]@_s3_binary_mode
+def s3_prefetch_open(
+ s3_url: PathLike,
+ mode: str = 'rb',
+ followlinks: bool = False,
+ *,
+ max_concurrency: Optional[int] = None,
+ max_block_size: int = DEFAULT_BLOCK_SIZE) -> S3PrefetchReader:
+ '''Open a asynchronous prefetch reader, to support fast sequential read and random read
+
+ .. note ::
+
+ User should make sure that reader / writer are closed correctly
+
+ Supports context manager
+
+ Some parameter setting may perform well: max_concurrency=10 or 20, max_block_size=8 or 16 MB, default value None means using global thread pool
+
+ :param max_concurrency: Max download thread number, None by default
+ :param max_block_size: Max data size downloaded by each thread, in bytes, 8MB by default
+ :returns: An opened S3PrefetchReader object
+ :raises: S3FileNotFoundError
+ '''
+ if mode != 'rb':
+ raise ValueError('unacceptable mode: %r' % mode)
+ if not isinstance(s3_url, S3Path):
+ s3_url = S3Path(s3_url)
+ if followlinks:
+ try:
+ s3_url = s3_url.readlink()
+ except S3NotALinkError:
+ pass
+
+ bucket, key = parse_s3_url(s3_url.path_with_protocol)
+ config = botocore.config.Config(max_pool_connections=max_pool_connections)
+ client = get_s3_client_with_cache(
+ config=config, profile_name=s3_url._profile_name)
+ return S3PrefetchReader(
+ bucket,
+ key,
+ s3_client=client,
+ max_retries=max_retries,
+ max_workers=max_concurrency,
+ block_size=max_block_size,
+ profile_name=s3_url._profile_name)
+
+
+
+
+
+[docs]@_s3_binary_mode
+def s3_pipe_open(
+ s3_url: PathLike,
+ mode: str,
+ followlinks: bool = False,
+ *,
+ join_thread: bool = True) -> S3PipeHandler:
+ '''Open a asynchronous read-write reader / writer, to support fast sequential read / write
+
+ .. note ::
+
+ User should make sure that reader / writer are closed correctly
+
+ Supports context manager
+
+ When join_thread is False, while the file handle are closing, this function will not wait until the asynchronous writing finishes;
+ False doesn't affect read-handle, but this can speed up write-handle because file will be written asynchronously.
+ But asynchronous behaviour can guarantee the file are successfully written, and frequent execution may cause thread and file handle exhaustion
+
+ :param mode: Mode to open file, either "rb" or "wb"
+ :param join_thread: If wait after function execution until s3 finishes writing
+ :returns: An opened BufferedReader / BufferedWriter object
+ '''
+ if mode not in ('rb', 'wb'):
+ raise ValueError('unacceptable mode: %r' % mode)
+
+ if mode[0] == 'r' and not S3Path(s3_url).is_file():
+ raise S3FileNotFoundError('No such file: %r' % s3_url)
+
+ if not isinstance(s3_url, S3Path):
+ s3_url = S3Path(s3_url)
+ if followlinks:
+ try:
+ s3_url = s3_url.readlink()
+ except S3NotALinkError:
+ pass
+
+ bucket, key = parse_s3_url(s3_url.path_with_protocol)
+ config = botocore.config.Config(max_pool_connections=max_pool_connections)
+ client = get_s3_client_with_cache(
+ config=config, profile_name=s3_url._profile_name)
+ return S3PipeHandler(
+ bucket,
+ key,
+ mode,
+ s3_client=client,
+ join_thread=join_thread,
+ profile_name=s3_url._profile_name)
+
+
+[docs]@_s3_binary_mode
+def s3_cached_open(
+ s3_url: PathLike,
+ mode: str,
+ followlinks: bool = False,
+ *,
+ cache_path: Optional[str] = None) -> S3CachedHandler:
+ '''Open a local-cache file reader / writer, for frequent random read / write
+
+ .. note ::
+
+ User should make sure that reader / writer are closed correctly
+
+ Supports context manager
+
+ cache_path can specify the path of cache file. Performance could be better if cache file path is on ssd or tmpfs
+
+ :param mode: Mode to open file, could be one of "rb", "wb" or "ab"
+ :param cache_path: cache file path
+ :returns: An opened BufferedReader / BufferedWriter object
+ '''
+ if mode not in ('rb', 'wb', 'ab', 'rb+', 'wb+', 'ab+'):
+ raise ValueError('unacceptable mode: %r' % mode)
+ if not isinstance(s3_url, S3Path):
+ s3_url = S3Path(s3_url)
+ if followlinks:
+ try:
+ s3_url = s3_url.readlink()
+ except S3NotALinkError:
+ pass
+
+ bucket, key = parse_s3_url(s3_url.path_with_protocol)
+ config = botocore.config.Config(max_pool_connections=max_pool_connections)
+ client = get_s3_client_with_cache(
+ config=config, profile_name=s3_url._profile_name)
+ return S3CachedHandler(
+ bucket,
+ key,
+ mode,
+ s3_client=client,
+ cache_path=cache_path,
+ profile_name=s3_url._profile_name)
+
+
+[docs]@_s3_binary_mode
+def s3_buffered_open(
+ s3_url: PathLike,
+ mode: str,
+ followlinks: bool = False,
+ *,
+ max_concurrency: Optional[int] = None,
+ max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
+ forward_ratio: Optional[float] = None,
+ block_size: int = DEFAULT_BLOCK_SIZE,
+ limited_seekable: bool = False,
+ buffered: bool = False,
+ share_cache_key: Optional[str] = None,
+ cache_path: Optional[str] = None
+) -> Union[S3PrefetchReader, S3BufferedWriter, io.BufferedReader, io.
+ BufferedWriter, S3MemoryHandler]:
+ '''Open an asynchronous prefetch reader, to support fast sequential read
+
+ .. note ::
+
+ User should make sure that reader / writer are closed correctly
+
+ Supports context manager
+
+ Some parameter setting may perform well: max_concurrency=10 or 20, max_block_size=8 or 16 MB, default value None means using global thread pool
+
+ :param max_concurrency: Max download thread number, None by default
+ :param max_buffer_size: Max cached buffer size in memory, 128MB by default
+ :param block_size: Size of single block, 8MB by default. Each block will be uploaded or downloaded by single thread.
+ :param limited_seekable: If write-handle supports limited seek (both file head part and tail part can seek block_size). Notes: This parameter are valid only for write-handle. Read-handle support arbitrary seek
+ :returns: An opened S3PrefetchReader object
+ :raises: S3FileNotFoundError
+ '''
+ if mode not in ('rb', 'wb', 'ab', 'rb+', 'wb+', 'ab+'):
+ raise ValueError('unacceptable mode: %r' % mode)
+ if not isinstance(s3_url, S3Path):
+ s3_url = S3Path(s3_url)
+ if followlinks:
+ try:
+ s3_url = s3_url.readlink()
+ except S3NotALinkError:
+ pass
+
+ bucket, key = parse_s3_url(s3_url.path_with_protocol)
+ config = botocore.config.Config(max_pool_connections=max_pool_connections)
+ client = get_s3_client_with_cache(
+ config=config, profile_name=s3_url._profile_name)
+
+ if 'a' in mode or '+' in mode:
+ if cache_path is None:
+ return S3MemoryHandler(
+ bucket,
+ key,
+ mode,
+ s3_client=client,
+ profile_name=s3_url._profile_name)
+ return S3CachedHandler(
+ bucket,
+ key,
+ mode,
+ s3_client=client,
+ cache_path=cache_path,
+ profile_name=s3_url._profile_name)
+
+ if mode == 'rb':
+ # A rough conversion algorithm to align 2 types of Reader / Writer parameters
+ # TODO: Optimize the conversion algorithm
+ block_capacity = max_buffer_size // block_size
+ if forward_ratio is None:
+ block_forward = None
+ else:
+ block_forward = max(int(block_capacity * forward_ratio), 1)
+ if share_cache_key is not None:
+ reader = S3ShareCacheReader(
+ bucket,
+ key,
+ cache_key=share_cache_key,
+ s3_client=client,
+ max_retries=max_retries,
+ max_workers=max_concurrency,
+ block_size=block_size,
+ block_forward=block_forward,
+ profile_name=s3_url._profile_name)
+ else:
+ reader = S3PrefetchReader(
+ bucket,
+ key,
+ s3_client=client,
+ max_retries=max_retries,
+ max_workers=max_concurrency,
+ block_capacity=block_capacity,
+ block_forward=block_forward,
+ block_size=block_size,
+ profile_name=s3_url._profile_name)
+ if buffered or _is_pickle(reader): # pytype: disable=wrong-arg-types
+ reader = io.BufferedReader(reader) # pytype: disable=wrong-arg-types
+ return reader
+
+ if limited_seekable:
+ writer = S3LimitedSeekableWriter(
+ bucket,
+ key,
+ s3_client=client,
+ max_workers=max_concurrency,
+ max_buffer_size=max_buffer_size,
+ block_size=block_size,
+ profile_name=s3_url._profile_name)
+ else:
+ writer = S3BufferedWriter(
+ bucket,
+ key,
+ s3_client=client,
+ max_workers=max_concurrency,
+ max_buffer_size=max_buffer_size,
+ block_size=block_size,
+ profile_name=s3_url._profile_name)
+ if buffered or _is_pickle(writer): # pytype: disable=wrong-arg-types
+ writer = io.BufferedWriter(writer) # pytype: disable=wrong-arg-types
+ return writer
+
+
+[docs]@_s3_binary_mode
+def s3_memory_open(
+ s3_url: PathLike, mode: str,
+ followlinks: bool = False) -> S3MemoryHandler:
+ '''Open a memory-cache file reader / writer, for frequent random read / write
+
+ .. note ::
+
+ User should make sure that reader / writer are closed correctly
+
+ Supports context manager
+
+ :param mode: Mode to open file, could be one of "rb", "wb", "ab", "rb+", "wb+" or "ab+"
+ :returns: An opened BufferedReader / BufferedWriter object
+ '''
+ if mode not in ('rb', 'wb', 'ab', 'rb+', 'wb+', 'ab+'):
+ raise ValueError('unacceptable mode: %r' % mode)
+ if not isinstance(s3_url, S3Path):
+ s3_url = S3Path(s3_url)
+ if followlinks:
+ try:
+ s3_url = s3_url.readlink()
+ except S3NotALinkError:
+ pass
+
+ bucket, key = parse_s3_url(s3_url.path_with_protocol)
+ config = botocore.config.Config(max_pool_connections=max_pool_connections)
+ client = get_s3_client_with_cache(
+ config=config, profile_name=s3_url._profile_name)
+ return S3MemoryHandler(
+ bucket, key, mode, s3_client=client, profile_name=s3_url._profile_name)
+
+
+s3_open = s3_buffered_open
+
+
+[docs]def s3_download(
+ src_url: PathLike,
+ dst_url: PathLike,
+ followlinks: bool = False,
+ callback: Optional[Callable[[int], None]] = None,
+ overwrite: bool = True) -> None:
+ '''
+ Downloads a file from s3 to local filesystem.
+ :param src_url: source s3 path
+ :param dst_url: target fs path
+ :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+ :param followlinks: False if regard symlink as file, else True
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ from megfile.fs import is_fs
+ from megfile.fs_path import FSPath
+
+ dst_url = fspath(dst_url)
+ if not is_fs(dst_url):
+ raise OSError(f'dst_url is not fs path: {dst_url}')
+ if not dst_url or dst_url.endswith('/'):
+ raise S3IsADirectoryError('Is a directory: %r' % dst_url)
+
+ dst_path = FSPath(dst_url)
+ if not overwrite and dst_path.exists():
+ return
+
+ if not isinstance(src_url, S3Path):
+ src_url = S3Path(src_url)
+ if followlinks:
+ try:
+ src_url = src_url.readlink()
+ except S3NotALinkError:
+ pass
+ src_bucket, src_key = parse_s3_url(src_url.path_with_protocol)
+ if not src_bucket:
+ raise S3BucketNotFoundError(
+ 'Empty bucket name: %r' % src_url.path_with_protocol)
+
+ if not src_url.exists():
+ raise S3FileNotFoundError(
+ 'File not found: %r' % src_url.path_with_protocol)
+
+ if not src_url.is_file():
+ raise S3IsADirectoryError(
+ 'Is a directory: %r' % src_url.path_with_protocol)
+
+ dst_directory = os.path.dirname(dst_path.path_without_protocol)
+ if dst_directory != '':
+ os.makedirs(dst_directory, exist_ok=True)
+
+ client = get_s3_client_with_cache(profile_name=src_url._profile_name)
+ download_file = patch_method(
+ client.download_file,
+ max_retries=max_retries,
+ should_retry=s3_should_retry,
+ )
+ try:
+ download_file(
+ src_bucket,
+ src_key,
+ dst_path.path_without_protocol,
+ Callback=callback)
+ except Exception as error:
+ error = translate_fs_error(error, dst_url)
+ error = translate_s3_error(error, src_url.path_with_protocol)
+ raise error
+
+ src_stat = src_url.stat()
+ os.utime(
+ dst_path.path_without_protocol, (src_stat.st_mtime, src_stat.st_mtime))
+
+
+[docs]def s3_upload(
+ src_url: PathLike,
+ dst_url: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ overwrite: bool = True,
+ **kwargs) -> None:
+ '''
+ Uploads a file from local filesystem to s3.
+ :param src_url: source fs path
+ :param dst_url: target s3 path
+ :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ from megfile.fs import is_fs
+ from megfile.fs_path import FSPath
+
+ if not is_fs(src_url):
+ raise OSError(f'src_url is not fs path: {src_url}')
+ src_path = FSPath(src_url)
+
+ dst_bucket, dst_key = parse_s3_url(dst_url)
+ if not dst_bucket:
+ raise S3BucketNotFoundError('Empty bucket name: %r' % dst_url)
+ if not dst_key or dst_key.endswith('/'):
+ raise S3IsADirectoryError('Is a directory: %r' % dst_url)
+
+ if not overwrite and S3Path(dst_url).is_file():
+ return
+
+ client = get_s3_client_with_cache(
+ profile_name=S3Path(dst_url)._profile_name)
+ upload_fileobj = patch_method(
+ client.upload_fileobj,
+ max_retries=max_retries,
+ should_retry=s3_should_retry,
+ )
+
+ with open(src_path.path_without_protocol,
+ 'rb') as src, raise_s3_error(dst_url):
+ upload_fileobj(src, Bucket=dst_bucket, Key=dst_key, Callback=callback)
+
+
+[docs]def s3_load_content(
+ s3_url,
+ start: Optional[int] = None,
+ stop: Optional[int] = None,
+ followlinks: bool = False) -> bytes:
+ '''
+ Get specified file from [start, stop) in bytes
+
+ :param s3_url: Specified path
+ :param start: start index
+ :param stop: stop index
+ :returns: bytes content in range [start, stop)
+ '''
+
+ def _get_object(client, bucket, key, range_str):
+ return client.get_object(
+ Bucket=bucket, Key=key, Range=range_str)['Body'].read()
+
+ s3_url = S3Path(s3_url)
+ if followlinks:
+ try:
+ s3_url = s3_url.readlink()
+ except S3NotALinkError:
+ pass
+
+ bucket, key = parse_s3_url(s3_url.path_with_protocol)
+ if not bucket:
+ raise S3BucketNotFoundError('Empty bucket name: %r' % s3_url)
+ if not key or key.endswith('/'):
+ raise S3IsADirectoryError('Is a directory: %r' % s3_url)
+
+ start, stop = get_content_offset(
+ start, stop, s3_url.getsize(follow_symlinks=False))
+ if start == 0 and stop == 0:
+ return b''
+ range_str = 'bytes=%d-%d' % (start, stop - 1)
+
+ client = get_s3_client_with_cache(profile_name=s3_url._profile_name)
+ with raise_s3_error(s3_url.path):
+ return patch_method(
+ _get_object,
+ max_retries=max_retries,
+ should_retry=s3_should_retry,
+ )(client, bucket, key, range_str)
+
+
+[docs]def s3_readlink(path) -> str:
+ '''
+ Return a string representing the path to which the symbolic link points.
+
+ :returns: Return a string representing the path to which the symbolic link points.
+ :raises: S3NameTooLongError, S3BucketNotFoundError, S3IsADirectoryError, S3NotALinkError
+ '''
+ return S3Path(path).readlink().path_with_protocol
+
+
+[docs]def s3_rename(
+ src_url: PathLike, dst_url: PathLike, overwrite: bool = True) -> None:
+ '''
+ Move s3 file path from src_url to dst_url
+
+ :param dst_url: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ S3Path(src_url).rename(dst_url, overwrite)
+
+
+[docs]class S3Cacher(FileCacher):
+ cache_path = None
+
+ def __init__(
+ self, path: str, cache_path: Optional[str] = None, mode: str = 'r'):
+ if mode not in ('r', 'w', 'a'):
+ raise ValueError('unacceptable mode: %r' % mode)
+ if mode in ('r', 'a'):
+ if cache_path is None:
+ cache_path = generate_cache_path(path)
+ s3_download(path, cache_path)
+ self.name = path
+ self.mode = mode
+ self.cache_path = cache_path
+
+ def _close(self):
+ if self.cache_path is not None and \
+ os.path.exists(self.cache_path):
+ if self.mode in ('w', 'a'):
+ s3_upload(self.cache_path, self.name)
+ os.unlink(self.cache_path)
+
+
+[docs]def s3_glob(
+ path: PathLike,
+ recursive: bool = True,
+ missing_ok: bool = True,
+ followlinks: bool = False,
+) -> List[str]:
+ '''Return s3 path list in ascending alphabetical order, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: A list contains paths match `s3_pathname`
+ '''
+ return list(
+ s3_iglob(
+ path=path,
+ recursive=recursive,
+ missing_ok=missing_ok,
+ followlinks=followlinks))
+
+
+[docs]def s3_glob_stat(
+ path: PathLike,
+ recursive: bool = True,
+ missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''Return a generator contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: A generator contains tuples of path and file stat, in which paths match `s3_pathname`
+ '''
+ return S3Path(path).glob_stat(
+ pattern="",
+ recursive=recursive,
+ missing_ok=missing_ok,
+ followlinks=followlinks)
+
+
+[docs]def s3_iglob(
+ path: PathLike,
+ recursive: bool = True,
+ missing_ok: bool = True,
+ followlinks: bool = False,
+) -> Iterator[str]:
+ '''Return s3 path iterator in ascending alphabetical order, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: An iterator contains paths match `s3_pathname`
+ '''
+ for path_obj in S3Path(path).iglob(pattern="", recursive=recursive,
+ missing_ok=missing_ok,
+ followlinks=followlinks):
+ yield path_obj.path_with_protocol
+
+
+[docs]def s3_makedirs(path: PathLike, exist_ok: bool = False):
+ '''
+ Create an s3 directory.
+ Purely creating directory is invalid because it's unavailable on OSS.
+ This function is to test the target bucket have WRITE access.
+
+ :param path: Given path
+ :param exist_ok: If False and target directory exists, raise S3FileExistsError
+ :raises: S3BucketNotFoundError, S3FileExistsError
+ '''
+ return S3Path(path).mkdir(parents=True, exist_ok=exist_ok)
+
+
+def _group_src_paths_by_block(
+ src_paths: List[PathLike], block_size: int = DEFAULT_BLOCK_SIZE
+) -> List[List[Tuple[PathLike, Optional[str]]]]:
+ groups = []
+ current_group, current_group_size = [], 0
+ for src_path in src_paths:
+ current_file_size = S3Path(src_path).stat().size
+ if current_file_size == 0:
+ continue
+
+ if current_file_size >= block_size:
+ if len(groups) == 0:
+ if current_group_size + current_file_size > 2 * block_size:
+ group_lack_size = block_size - current_group_size
+ current_group.append(
+ (src_path, f'bytes=0-{group_lack_size-1}'))
+ groups.extend(
+ [
+ current_group,
+ [
+ (
+ src_path,
+ f'bytes={group_lack_size}-{current_file_size-1}'
+ )
+ ]
+ ])
+ else:
+ current_group.append((src_path, None))
+ groups.append(current_group)
+ else:
+ groups[-1].extend(current_group)
+ groups.append([(src_path, None)])
+ current_group, current_group_size = [], 0
+ else:
+ current_group.append((src_path, None))
+ current_group_size += current_file_size
+ if current_group_size >= block_size:
+ groups.append(current_group)
+ current_group, current_group_size = [], 0
+ if current_group:
+ groups.append(current_group)
+ return groups
+
+
+[docs]def s3_concat(
+ src_paths: List[PathLike],
+ dst_path: PathLike,
+ block_size: int = DEFAULT_BLOCK_SIZE,
+ max_workers: int = GLOBAL_MAX_WORKERS) -> None:
+ '''Concatenate s3 files to one file.
+
+ :param src_paths: Given source paths
+ :param dst_path: Given destination path
+ '''
+ client = S3Path(dst_path)._client
+ with raise_s3_error(dst_path):
+ if block_size == 0:
+ groups = [[(src_path, None)] for src_path in src_paths]
+ else:
+ groups = _group_src_paths_by_block(src_paths, block_size=block_size)
+
+ with MultiPartWriter(client, dst_path) as writer, ThreadPoolExecutor(
+ max_workers=max_workers) as executor:
+ for index, group in enumerate(groups, start=1):
+ if len(group) == 1:
+ executor.submit(
+ writer.upload_part_copy, index, group[0][0],
+ group[0][1])
+ else:
+ executor.submit(writer.upload_part_by_paths, index, group)
+
+
+[docs]def s3_lstat(path: PathLike) -> StatResult:
+ '''Like Path.stat() but, if the path points to a symbolic link, return the symbolic link’s information rather than its target’s.'''
+ return S3Path(path).lstat()
+
+
+[docs]@SmartPath.register
+class S3Path(URIPath):
+
+ protocol = "s3"
+
+ def __init__(self, path: "PathLike", *other_paths: "PathLike"):
+ super().__init__(path, *other_paths)
+ protocol = get_url_scheme(self.path)
+ self._protocol_with_profile = self.protocol
+ self._profile_name = None
+ if protocol.startswith('s3+'):
+ self._protocol_with_profile = protocol
+ self._profile_name = protocol[3:]
+ self._s3_path = f"s3://{self.path[len(protocol)+3:]}"
+ elif not protocol:
+ self._s3_path = f"s3://{self.path.lstrip('/')}"
+ else:
+ self._s3_path = self.path
+
+[docs] @cachedproperty
+ def path_with_protocol(self) -> str:
+ '''Return path with protocol, like file:///root, s3://bucket/key'''
+ path = self.path
+ protocol_prefix = self._protocol_with_profile + "://"
+ if path.startswith(protocol_prefix):
+ return path
+ return protocol_prefix + path.lstrip('/')
+
+[docs] @cachedproperty
+ def path_without_protocol(self) -> str:
+ '''Return path without protocol, example: if path is s3://bucket/key, return bucket/key'''
+ path = self.path
+ protocol_prefix = self._protocol_with_profile + "://"
+ if path.startswith(protocol_prefix):
+ path = path[len(protocol_prefix):]
+ return path
+
+[docs] @cachedproperty
+ def parts(self) -> Tuple[str]:
+ '''A tuple giving access to the path’s various components'''
+ parts = [f"{self._protocol_with_profile}://"]
+ path = self.path_without_protocol
+ path = path.lstrip('/')
+ if path != '':
+ parts.extend(path.split('/'))
+ return tuple(parts)
+
+ @cachedproperty
+ def _client(self):
+ return get_s3_client_with_cache(profile_name=self._profile_name)
+
+ def _s3_get_metadata(self) -> dict:
+ '''
+ Get object metadata
+
+ :param path: Object path
+ :returns: Object metadata
+ '''
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket:
+ return {}
+ if not key or key.endswith('/'):
+ return {}
+ try:
+ with raise_s3_error(self.path_with_protocol):
+ resp = self._client.head_object(Bucket=bucket, Key=key)
+ return dict(
+ (key.lower(), value) for key, value in resp['Metadata'].items())
+ except Exception as error:
+ if isinstance(error,
+ (S3UnknownError, S3ConfigError, S3PermissionError)):
+ raise error
+ return {}
+
+[docs] def access(
+ self, mode: Access = Access.READ,
+ followlinks: bool = False) -> bool:
+ '''
+ Test if path has access permission described by mode
+
+ :param mode: access mode
+ :returns: bool, if the bucket of s3_url has read/write access.
+ '''
+ s3_url = self.path_with_protocol
+ if followlinks:
+ try:
+ s3_url = self.readlink().path_with_protocol
+ except S3NotALinkError:
+ pass
+ bucket, key = parse_s3_url(s3_url) # only check bucket accessibility
+ if not bucket:
+ raise Exception("No available bucket")
+ if not isinstance(mode, Access):
+ raise TypeError(
+ 'Unsupported mode: {} -- Mode should use one of the enums belonging to: {}'
+ .format(mode, ', '.join([str(a) for a in Access])))
+ if mode not in (Access.READ, Access.WRITE):
+ raise TypeError('Unsupported mode: {}'.format(mode))
+
+ try:
+ if not self.exists():
+ return False
+ except Exception as error:
+ error = translate_s3_error(error, s3_url)
+ if isinstance(error, S3PermissionError):
+ return False
+ raise error
+
+ if mode == Access.READ:
+ return True
+ try:
+ if not key:
+ key = 'test'
+ elif key.endswith('/'):
+ key = key[:-1]
+ upload_id = self._client.create_multipart_upload(
+ Bucket=bucket,
+ Key=key,
+ )['UploadId']
+ self._client.abort_multipart_upload(
+ Bucket=bucket,
+ Key=key,
+ UploadId=upload_id,
+ )
+ return True
+ except Exception as error:
+ error = translate_s3_error(error, s3_url)
+ if isinstance(error, S3PermissionError):
+ return False
+ raise error
+
+[docs] def exists(self, followlinks: bool = False) -> bool:
+ '''
+ Test if s3_url exists
+
+ If the bucket of s3_url are not permitted to read, return False
+
+ :returns: True if s3_url eixsts, else False
+ '''
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket: # s3:// => True, s3:///key => False
+ return not key
+
+ return self.is_file(followlinks) or self.is_dir()
+
+[docs] def getmtime(self, follow_symlinks: bool = False) -> float:
+ '''
+ Get last-modified time of the file on the given s3_url path (in Unix timestamp format).
+ If the path is an existent directory, return the latest modified time of all file in it. The mtime of empty directory is 1970-01-01 00:00:00
+
+ If s3_url is not an existent path, which means s3_exist(s3_url) returns False, then raise S3FileNotFoundError
+
+ :returns: Last-modified time
+ :raises: S3FileNotFoundError, UnsupportedError
+ '''
+ return self.stat(follow_symlinks=follow_symlinks).mtime
+
+[docs] def getsize(self, follow_symlinks: bool = False) -> int:
+ '''
+ Get file size on the given s3_url path (in bytes).
+ If the path in a directory, return the sum of all file size in it, including file in subdirectories (if exist).
+ The result excludes the size of directory itself. In other words, return 0 Byte on an empty directory path.
+
+ If s3_url is not an existent path, which means s3_exist(s3_url) returns False, then raise S3FileNotFoundError
+
+ :returns: File size
+ :raises: S3FileNotFoundError, UnsupportedError
+ '''
+ return self.stat(follow_symlinks=follow_symlinks).size
+
+[docs] def glob(
+ self,
+ pattern,
+ recursive: bool = True,
+ missing_ok: bool = True,
+ followlinks: bool = False,
+ ) -> List['S3Path']:
+ '''Return s3 path list in ascending alphabetical order, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: A list contains paths match `s3_pathname`
+ '''
+ return list(
+ self.iglob(
+ pattern=pattern,
+ recursive=recursive,
+ missing_ok=missing_ok,
+ followlinks=followlinks))
+
+[docs] def glob_stat(
+ self,
+ pattern,
+ recursive: bool = True,
+ missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''Return a generator contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: A generator contains tuples of path and file stat, in which paths match `s3_pathname`
+ '''
+ glob_path = self._s3_path
+ if pattern:
+ glob_path = self.joinpath(pattern)._s3_path # pytype: disable=attribute-error
+ s3_pathname = fspath(glob_path)
+
+ def create_generator():
+ for group_s3_pathname_1 in _group_s3path_by_bucket(
+ s3_pathname, self._profile_name):
+ for group_s3_pathname_2 in _group_s3path_by_prefix(
+ group_s3_pathname_1):
+ for file_entry in _s3_glob_stat_single_path(
+ group_s3_pathname_2, recursive, missing_ok,
+ followlinks=followlinks,
+ profile_name=self._profile_name):
+ if self._profile_name:
+ file_entry = file_entry._replace(
+ path=
+ f"{self._protocol_with_profile}://{file_entry.path[5:]}"
+ )
+ yield file_entry
+
+ return _create_missing_ok_generator(
+ create_generator(), missing_ok,
+ S3FileNotFoundError('No match any file: %r' % s3_pathname))
+
+[docs] def iglob(
+ self,
+ pattern,
+ recursive: bool = True,
+ missing_ok: bool = True,
+ followlinks: bool = False,
+ ) -> Iterator['S3Path']:
+ '''Return s3 path iterator in ascending alphabetical order, in which path matches glob pattern
+ Notes: Only glob in bucket. If trying to match bucket with wildcard characters, raise UnsupportedError
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :raises: UnsupportedError, when bucket part contains wildcard characters
+ :returns: An iterator contains paths match `s3_pathname`
+ '''
+ for file_entry in self.glob_stat(pattern=pattern, recursive=recursive,
+ missing_ok=missing_ok,
+ followlinks=followlinks):
+ yield self.from_path(file_entry.path)
+
+[docs] def is_dir(self, followlinks: bool = False) -> bool:
+ '''
+ Test if an s3 url is directory
+ Specific procedures are as follows:
+ If there exists a suffix, of which ``os.path.join(s3_url, suffix)`` is a file
+ If the url is empty bucket or s3://
+
+ :param followlinks: whether followlinks is True or False, result is the same. Because s3 symlink not support dir.
+ :returns: True if path is s3 directory, else False
+ '''
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket: # s3:// => True, s3:///key => False
+ return not key
+ prefix = _become_prefix(key)
+ try:
+ resp = self._client.list_objects_v2(
+ Bucket=bucket, Prefix=prefix, Delimiter='/', MaxKeys=1)
+ except Exception as error:
+ error = translate_s3_error(error, self.path_with_protocol)
+ if isinstance(error,
+ (S3UnknownError, S3ConfigError, S3PermissionError)):
+ raise error
+ return False
+
+ if not key: # bucket is accessible
+ return True
+
+ if 'KeyCount' in resp:
+ return resp['KeyCount'] > 0
+
+ return len(resp.get('Contents', [])) > 0 or \
+ len(resp.get('CommonPrefixes', [])) > 0
+
+[docs] def is_file(self, followlinks: bool = False) -> bool:
+ '''
+ Test if an s3_url is file
+
+ :returns: True if path is s3 file, else False
+ '''
+ s3_url = self.path_with_protocol
+ if followlinks:
+ try:
+ s3_url = self.readlink().path_with_protocol
+ except S3NotALinkError:
+ pass
+ bucket, key = parse_s3_url(s3_url)
+ if not bucket or not key or key.endswith('/'):
+ # s3://, s3:///key, s3://bucket, s3://bucket/prefix/
+ return False
+ try:
+ self._client.head_object(Bucket=bucket, Key=key)
+ except Exception as error:
+ error = translate_s3_error(error, s3_url)
+ if isinstance(error,
+ (S3UnknownError, S3ConfigError, S3PermissionError)):
+ raise error
+ return False
+ return True
+
+[docs] def listdir(self, followlinks: bool = False) -> List[str]:
+ '''
+ Get all contents of given s3_url. The result is in acsending alphabetical order.
+
+ :returns: All contents have prefix of s3_url in acsending alphabetical order
+ :raises: S3FileNotFoundError, S3NotADirectoryError
+ '''
+ entries = list(self.scandir(followlinks=followlinks))
+ return sorted([entry.name for entry in entries])
+
+[docs] def iterdir(self, followlinks: bool = False) -> Iterator['S3Path']:
+ '''
+ Get all contents of given s3_url. The result is in acsending alphabetical order.
+
+ :returns: All contents have prefix of s3_url in acsending alphabetical order
+ :raises: S3FileNotFoundError, S3NotADirectoryError
+ '''
+ for path in self.listdir(followlinks=followlinks):
+ yield self.joinpath(path) # type: ignore
+
+[docs] def load(self, followlinks: bool = False) -> BinaryIO:
+ '''Read all content in binary on specified path and write into memory
+
+ User should close the BinaryIO manually
+
+ :returns: BinaryIO
+ '''
+ s3_url = self.path_with_protocol
+ if followlinks:
+ try:
+ s3_url = self.readlink().path_with_protocol
+ except S3NotALinkError:
+ pass
+ bucket, key = parse_s3_url(s3_url)
+ if not bucket:
+ raise S3BucketNotFoundError('Empty bucket name: %r' % s3_url)
+ if not key or key.endswith('/'):
+ raise S3IsADirectoryError('Is a directory: %r' % s3_url)
+
+ buffer = io.BytesIO()
+ with raise_s3_error(s3_url):
+ self._client.download_fileobj(bucket, key, buffer)
+ buffer.seek(0)
+ return buffer
+
+[docs] def hasbucket(self) -> bool:
+ '''
+ Test if the bucket of s3_url exists
+
+ :returns: True if bucket of s3_url eixsts, else False
+ '''
+ bucket, _ = parse_s3_url(self.path_with_protocol)
+ if not bucket:
+ return False
+
+ try:
+ self._client.head_bucket(Bucket=bucket)
+ except Exception as error:
+ error = translate_s3_error(error, self.path_with_protocol)
+ if isinstance(error, S3PermissionError):
+ # Aliyun OSS doesn't give bucket api permission when you only have read and write permission
+ try:
+ self._client.list_objects_v2(Bucket=bucket, MaxKeys=1)
+ return True
+ except Exception as error2:
+ error2 = translate_s3_error(error2, self.path_with_protocol)
+ if isinstance(
+ error2,
+ (S3UnknownError, S3ConfigError, S3PermissionError)):
+ raise error2
+ return False
+ elif isinstance(error, (S3UnknownError, S3ConfigError)):
+ raise error
+ elif isinstance(error, S3FileNotFoundError):
+ return False
+
+ return True
+
+[docs] def mkdir(self, mode=0o777, parents: bool = False, exist_ok: bool = False):
+ '''
+ Create an s3 directory.
+ Purely creating directory is invalid because it's unavailable on OSS.
+ This function is to test the target bucket have WRITE access.
+
+ :param mode: mode is ignored, only be compatible with pathlib.Path
+ :param parents: parents is ignored, only be compatible with pathlib.Path
+ :param exist_ok: If False and target directory exists, raise S3FileExistsError
+ :raises: S3BucketNotFoundError, S3FileExistsError
+ '''
+ bucket, _ = parse_s3_url(self.path_with_protocol)
+ if not bucket:
+ raise S3BucketNotFoundError(
+ 'Empty bucket name: %r' % self.path_with_protocol)
+ if not self.hasbucket():
+ raise S3BucketNotFoundError(
+ 'No such bucket: %r' % self.path_with_protocol)
+ if exist_ok:
+ if self.is_file():
+ raise S3FileExistsError(
+ 'File exists: %r' % self.path_with_protocol)
+ return
+ if self.exists():
+ raise S3FileExistsError('File exists: %r' % self.path_with_protocol)
+
+[docs] def move(self, dst_url: PathLike, overwrite: bool = True) -> None:
+ '''
+ Move file/directory path from src_url to dst_url
+
+ :param dst_url: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ for src_file_path, dst_file_path in _s3_scan_pairs(
+ self.path_with_protocol, dst_url):
+ S3Path(src_file_path).rename(dst_file_path, overwrite)
+
+[docs] def remove(self, missing_ok: bool = False) -> None:
+ '''
+ Remove the file or directory on s3, `s3://` and `s3://bucket` are not permitted to remove
+
+ :param missing_ok: if False and target file/directory not exists, raise S3FileNotFoundError
+ :raises: S3PermissionError, S3FileNotFoundError, UnsupportedError
+ '''
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket:
+ if not key:
+ raise UnsupportedError(
+ 'Remove whole s3', self.path_with_protocol)
+ raise S3BucketNotFoundError(
+ 'Empty bucket name: %r' % self.path_with_protocol)
+ if not key:
+ raise UnsupportedError('Remove bucket', self.path_with_protocol)
+ if not self.exists():
+ if missing_ok:
+ return
+ raise S3FileNotFoundError(
+ 'No such file or directory: %r' % self.path_with_protocol)
+
+ client = self._client
+ with raise_s3_error(self.path_with_protocol):
+ if self.is_file():
+ client.delete_object(Bucket=bucket, Key=key)
+ return
+ prefix = _become_prefix(key)
+ total_count, error_count = 0, 0
+ for resp in _list_objects_recursive(client, bucket, prefix):
+ if 'Contents' in resp:
+ keys = [
+ {
+ 'Key': content['Key']
+ } for content in resp['Contents']
+ ]
+ total_count += len(keys)
+ errors = []
+ retries = 2
+ retry_interval = min(0.1 * 2**retries, 30)
+ for i in range(retries):
+ # doc: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.delete_objects
+ if not keys:
+ break
+ response = client.delete_objects(
+ Bucket=bucket, Delete={'Objects': keys})
+ keys = []
+ for error_info in response.get('Errors', []):
+ if s3_error_code_should_retry(
+ error_info.get('Code')):
+ error_logger.warning(
+ "retry %s times, removing file: %s, with error %s: %s"
+ % (
+ i + 1, error_info['Key'],
+ error_info['Code'],
+ error_info['Message']))
+ keys.append({'Key': error_info['Key']})
+ else:
+ errors.append(error_info)
+ time.sleep(retry_interval)
+ for error_info in errors:
+ error_logger.error(
+ "failed remove file: %s, with error %s: %s" % (
+ error_info['Key'], error_info['Code'],
+ error_info['Message']))
+ error_count += len(errors)
+ if error_count > 0:
+ error_msg = "failed remove path: %s, total file count: %s, failed count: %s" % (
+ self.path_with_protocol, total_count, error_count)
+ raise S3UnknownError(
+ Exception(error_msg), self.path_with_protocol)
+
+[docs] def rename(self, dst_path: PathLike, overwrite: bool = True) -> 'S3Path':
+ '''
+ Move s3 file path from src_url to dst_url
+
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ if self.is_file():
+ self.copy(dst_path, overwrite=overwrite)
+ else:
+ self.sync(dst_path, overwrite=overwrite)
+ self.remove(missing_ok=True)
+ return self.from_path(dst_path)
+
+[docs] def scan(self, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[str]:
+ '''
+ Iteratively traverse only files in given s3 directory, in alphabetical order.
+ Every iteration on generator yields a path string.
+
+ If s3_url is a file path, yields the file only
+ If s3_url is a non-existent path, return an empty generator
+ If s3_url is a bucket path, return all file paths in the bucket
+ If s3_url is an empty bucket, return an empty generator
+ If s3_url doesn't contain any bucket, which is s3_url == 's3://', raise UnsupportedError. walk() on complete s3 is not supported in megfile
+
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :raises: UnsupportedError
+ :returns: A file path generator
+ '''
+ scan_stat_iter = self.scan_stat(
+ missing_ok=missing_ok, followlinks=followlinks)
+
+ def create_generator() -> Iterator[str]:
+ for file_entry in scan_stat_iter:
+ yield file_entry.path
+
+ return create_generator()
+
+[docs] def scan_stat(self, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a tuple of path string and file stat
+
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :raises: UnsupportedError
+ :returns: A file path generator
+ '''
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket:
+ raise UnsupportedError('Scan whole s3', self.path_with_protocol)
+
+ def create_generator() -> Iterator[FileEntry]:
+ if not self.is_dir():
+ if self.is_file():
+ # On s3, file and directory may be of same name and level, so need to test the path is file or directory
+ yield FileEntry(
+ self.name, fspath(self.path_with_protocol),
+ self.stat(follow_symlinks=followlinks))
+ return
+
+ if not key.endswith('/') and self.is_file():
+ yield FileEntry(
+ self.name, fspath(self.path_with_protocol),
+ self.stat(follow_symlinks=followlinks))
+
+ prefix = _become_prefix(key)
+ client = self._client
+ with raise_s3_error(self.path_with_protocol):
+ for resp in _list_objects_recursive(client, bucket, prefix):
+ for content in resp.get('Contents', []):
+ full_path = s3_path_join(
+ f'{self._protocol_with_profile}://', bucket,
+ content['Key'])
+
+ if followlinks:
+ try:
+ origin_path = self.from_path(
+ full_path).readlink()
+ yield FileEntry(
+ origin_path.name,
+ origin_path.path_with_protocol,
+ origin_path.lstat())
+ continue
+ except S3NotALinkError:
+ pass
+
+ yield FileEntry(
+ S3Path(full_path).name, full_path,
+ _make_stat(content))
+
+ return _create_missing_ok_generator(
+ create_generator(), missing_ok,
+ S3FileNotFoundError(
+ 'No match any file in: %r' % self.path_with_protocol))
+
+[docs] def scandir(self, followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Get all contents of given s3_url, the order of result is not guaranteed.
+
+ :returns: All contents have prefix of s3_url
+ :raises: S3FileNotFoundError, S3NotADirectoryError
+ '''
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket and key:
+ raise S3BucketNotFoundError(
+ 'Empty bucket name: %r' % self.path_with_protocol)
+
+ if self.is_file():
+ raise S3NotADirectoryError(
+ 'Not a directory: %r' % self.path_with_protocol)
+ elif not self.is_dir():
+ raise S3FileNotFoundError(
+ 'No such directory: %r' % self.path_with_protocol)
+ prefix = _become_prefix(key)
+ client = self._client
+
+ # In order to do check on creation,
+ # we need to wrap the iterator in another function
+ def create_generator() -> Iterator[FileEntry]:
+ with raise_s3_error(self.path_with_protocol):
+
+ def generate_s3_path(
+ protocol: str, bucket: str, key: str) -> str:
+ return "%s://%s/%s" % (protocol, bucket, key)
+
+ if not bucket and not key: # list buckets
+ response = client.list_buckets()
+ for content in response['Buckets']:
+ yield FileEntry(
+ content['Name'], f"s3://{content['Name']}",
+ StatResult(
+ ctime=content['CreationDate'].timestamp(),
+ isdir=True,
+ extra=content,
+ ))
+ return
+
+ for resp in _list_objects_recursive(client, bucket, prefix,
+ '/'):
+ for common_prefix in resp.get('CommonPrefixes', []):
+ yield FileEntry(
+ common_prefix['Prefix'][len(prefix):-1],
+ generate_s3_path(
+ self._protocol_with_profile, bucket,
+ common_prefix['Prefix']),
+ StatResult(isdir=True, extra=common_prefix))
+ for content in resp.get('Contents', []):
+ src_url = generate_s3_path(
+ self._protocol_with_profile, bucket, content['Key'])
+
+ if followlinks:
+ try:
+ origin_path = self.from_path(src_url).readlink()
+ yield FileEntry(
+ origin_path.name,
+ origin_path.path_with_protocol,
+ origin_path.lstat())
+ continue
+ except S3NotALinkError:
+ pass
+
+ yield FileEntry(
+ content['Key'][len(prefix):], src_url,
+ _make_stat(content))
+
+ return ContextIterator(create_generator())
+
+ def _getdirstat(self) -> StatResult:
+ '''
+ Return StatResult of given s3_url directory, including:
+
+ 1. Directory size: the sum of all file size in it, including file in subdirectories (if exist).
+ The result excludes the size of directory itself. In other words, return 0 Byte on an empty directory path
+ 2. Last-modified time of directory: return the latest modified time of all file in it. The mtime of empty directory is 1970-01-01 00:00:00
+
+ :returns: An int indicates size in Bytes
+ '''
+ if not self.is_dir():
+ raise S3FileNotFoundError(
+ 'No such file or directory: %r' % self.path_with_protocol)
+
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ prefix = _become_prefix(key)
+ client = self._client
+ size = 0
+ mtime = 0.0
+ with raise_s3_error(self.path_with_protocol):
+ for resp in _list_objects_recursive(client, bucket, prefix):
+ for content in resp.get('Contents', []):
+ size += content['Size']
+ last_modified = content['LastModified'].timestamp()
+ if mtime < last_modified:
+ mtime = last_modified
+
+ return StatResult(size=size, mtime=mtime, isdir=True)
+
+[docs] def stat(self, follow_symlinks=True) -> StatResult:
+ '''
+ Get StatResult of s3_url file, including file size and mtime, referring to s3_getsize and s3_getmtime
+
+ If s3_url is not an existent path, which means s3_exist(s3_url) returns False, then raise S3FileNotFoundError
+ If attempt to get StatResult of complete s3, such as s3_dir_url == 's3://', raise S3BucketNotFoundError
+
+ :returns: StatResult
+ :raises: S3FileNotFoundError, S3BucketNotFoundError
+ '''
+ islnk = False
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket:
+ raise S3BucketNotFoundError(
+ 'Empty bucket name: %r' % self.path_with_protocol)
+
+ if not self.is_file():
+ return self._getdirstat()
+
+ client = self._client
+ with raise_s3_error(self.path_with_protocol):
+ content = client.head_object(Bucket=bucket, Key=key)
+ if 'Metadata' in content:
+ metadata = dict(
+ (key.lower(), value)
+ for key, value in content['Metadata'].items())
+ if metadata and 'symlink_to' in metadata:
+ islnk = True
+ if islnk and follow_symlinks:
+ s3_url = metadata['symlink_to']
+ bucket, key = parse_s3_url(s3_url)
+ content = client.head_object(Bucket=bucket, Key=key)
+ stat_record = StatResult(
+ islnk=islnk,
+ size=content['ContentLength'],
+ mtime=content['LastModified'].timestamp(),
+ extra=content)
+ return stat_record
+
+[docs] def unlink(self, missing_ok: bool = False) -> None:
+ '''
+ Remove the file on s3
+
+ :param missing_ok: if False and target file not exists, raise S3FileNotFoundError
+ :raises: S3PermissionError, S3FileNotFoundError, S3IsADirectoryError
+ '''
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket or not key or key.endswith('/'):
+ raise S3IsADirectoryError(
+ 'Is a directory: %r' % self.path_with_protocol)
+ if not self.is_file():
+ if missing_ok:
+ return
+ raise S3FileNotFoundError(
+ 'No such file: %r' % self.path_with_protocol)
+
+ with raise_s3_error(self.path_with_protocol):
+ self._client.delete_object(Bucket=bucket, Key=key)
+
+[docs] def walk(self, followlinks: bool = False
+ ) -> Iterator[Tuple[str, List[str], List[str]]]:
+ '''
+ Iteratively traverse the given s3 directory, in top-bottom order. In other words, firstly traverse parent directory, if subdirectories exist, traverse the subdirectories in alphabetical order.
+ Every iteration on generator yields a 3-tuple: (root, dirs, files)
+
+ - root: Current s3 path;
+ - dirs: Name list of subdirectories in current directory. The list is sorted by name in ascending alphabetical order;
+ - files: Name list of files in current directory. The list is sorted by name in ascending alphabetical order;
+
+ If s3_url is a file path, return an empty generator
+ If s3_url is a non-existent path, return an empty generator
+ If s3_url is a bucket path, bucket will be the top directory, and will be returned at first iteration of generator
+ If s3_url is an empty bucket, only yield one 3-tuple (notes: s3 doesn't have empty directory)
+ If s3_url doesn't contain any bucket, which is s3_url == 's3://', raise UnsupportedError. walk() on complete s3 is not supported in megfile
+
+ :param followlinks: whether followlinks is True or False, result is the same. Because s3 symlink not support dir.
+ :raises: UnsupportedError
+ :returns: A 3-tuple generator
+ '''
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket:
+ raise UnsupportedError('Walk whole s3', self.path_with_protocol)
+
+ if not self.is_dir():
+ return
+
+ stack = [key]
+ client = self._client
+ while len(stack) > 0:
+ current = _become_prefix(stack.pop())
+ dirs, files = [], []
+ for resp in _list_objects_recursive(client, bucket, current, '/'):
+ for common_prefix in resp.get('CommonPrefixes', []):
+ dirs.append(common_prefix['Prefix'][:-1])
+ for content in resp.get('Contents', []):
+ files.append(content['Key'])
+
+ dirs = sorted(dirs)
+ stack.extend(reversed(dirs))
+
+ root = s3_path_join(
+ f'{self._protocol_with_profile}://', bucket, current)[:-1]
+ dirs = [path[len(current):] for path in dirs]
+ files = sorted(path[len(current):] for path in files)
+ yield root, dirs, files
+
+[docs] def md5(self, recalculate: bool = False, followlinks: bool = False) -> str:
+ '''
+ Get md5 meta info in files that uploaded/copied via megfile
+
+ If meta info is lost or non-existent, return None
+
+ :param recalculate: calculate md5 in real-time or return s3 etag
+ :param followlinks: If is True, calculate md5 for real file
+ :returns: md5 meta info
+ '''
+ bucket, _ = parse_s3_url(self.path_with_protocol)
+ if not bucket:
+ raise S3BucketNotFoundError(
+ 'Empty bucket name: %r' % self.path_with_protocol)
+ stat = self.stat(follow_symlinks=followlinks)
+ if stat.isdir:
+ hash_md5 = hashlib.md5() # nosec
+ for file_name in self.listdir():
+ chunk = S3Path(
+ s3_path_join(
+ self.path_with_protocol,
+ file_name)).md5(recalculate=recalculate).encode()
+ hash_md5.update(chunk)
+ return hash_md5.hexdigest()
+ if recalculate:
+ path_instance = self
+ if followlinks:
+ try:
+ path_instance = self.readlink()
+ except S3NotALinkError:
+ pass
+ with path_instance.open('rb') as f:
+ return calculate_md5(f)
+ return stat.extra.get('ETag', '')[1:-1]
+
+[docs] def copy(
+ self,
+ dst_url: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ followlinks: bool = False,
+ overwrite: bool = True) -> None:
+ ''' File copy on S3
+ Copy content of file on `src_path` to `dst_path`.
+ It's caller's responsebility to ensure the s3_isfile(src_url) == True
+
+ :param dst_path: Target file path
+ :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+ :param followlinks: False if regard symlink as file, else True
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ if not overwrite and self.from_path(dst_url).is_file():
+ return
+
+ src_url = self.path_with_protocol
+ src_bucket, src_key = parse_s3_url(src_url)
+ dst_bucket, dst_key = parse_s3_url(dst_url)
+ if dst_bucket == src_bucket and src_key.rstrip('/') == dst_key.rstrip(
+ '/'):
+ raise SameFileError(
+ f"'{src_url}' and '{dst_url}' are the same file")
+
+ if not src_bucket:
+ raise S3BucketNotFoundError('Empty bucket name: %r' % src_url)
+ if self.is_dir():
+ raise S3IsADirectoryError('Is a directory: %r' % src_url)
+
+ if not dst_bucket:
+ raise S3BucketNotFoundError('Empty bucket name: %r' % dst_url)
+ if not dst_key or dst_key.endswith('/'):
+ raise S3IsADirectoryError('Is a directory: %r' % dst_url)
+
+ if followlinks:
+ try:
+ s3_url = self.readlink().path
+ src_bucket, src_key = parse_s3_url(s3_url)
+ except S3NotALinkError:
+ pass
+
+ with raise_s3_error(f"'{src_url}' or '{dst_url}'"):
+ self._client.copy(
+ {
+ 'Bucket': src_bucket,
+ 'Key': src_key,
+ },
+ Bucket=dst_bucket,
+ Key=dst_key,
+ Callback=callback)
+
+[docs] def sync(
+ self,
+ dst_url: PathLike,
+ followlinks: bool = False,
+ force: bool = False,
+ overwrite: bool = True) -> None:
+ '''
+ Copy file/directory on src_url to dst_url
+
+ :param dst_url: Given destination path
+ :param followlinks: False if regard symlink as file, else True
+ :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ for src_file_path, dst_file_path in _s3_scan_pairs(
+ self.path_with_protocol, dst_url):
+ src_file_path = self.from_path(src_file_path)
+ dst_file_path = self.from_path(dst_file_path)
+
+ if force:
+ pass
+ elif not overwrite and dst_file_path.exists():
+ continue
+ elif dst_file_path.exists() and is_same_file(
+ src_file_path.stat(), dst_file_path.stat(), 'copy'):
+ continue
+
+ src_file_path.copy(dst_file_path, followlinks=followlinks)
+
+[docs] def symlink(self, dst_path: PathLike) -> None:
+ '''
+ Create a symbolic link pointing to src_path named dst_path.
+
+ :param dst_path: Desination path
+ :raises: S3NameTooLongError, S3BucketNotFoundError, S3IsADirectoryError
+ '''
+ if len(fspath(self._s3_path).encode()) > 1024:
+ raise S3NameTooLongError('File name too long: %r' % dst_path)
+ src_bucket, src_key = parse_s3_url(self.path_with_protocol)
+ dst_bucket, dst_key = parse_s3_url(dst_path)
+
+ if not src_bucket:
+ raise S3BucketNotFoundError('Empty bucket name: %r' % self.path)
+ if not dst_bucket:
+ raise S3BucketNotFoundError('Empty bucket name: %r' % dst_path)
+ if not dst_key or dst_key.endswith('/'):
+ raise S3IsADirectoryError('Is a directory: %r' % dst_path)
+
+ src_path = self._s3_path
+ try:
+ src_path = self.readlink()._s3_path
+ except S3NotALinkError:
+ pass
+ with raise_s3_error(dst_path):
+ self._client.put_object(
+ Bucket=dst_bucket,
+ Key=dst_key,
+ Metadata={"symlink_to": src_path})
+
+[docs] def readlink(self) -> 'S3Path':
+ '''
+ Return a S3Path instance representing the path to which the symbolic link points.
+
+ :returns: Return a S3Path instance representing the path to which the symbolic link points.
+ :raises: S3NameTooLongError, S3BucketNotFoundError, S3IsADirectoryError, S3NotALinkError
+ '''
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket:
+ raise S3BucketNotFoundError(
+ 'Empty bucket name: %r' % self.path_with_protocol)
+ if not key or key.endswith('/'):
+ raise S3IsADirectoryError(
+ 'Is a directory: %r' % self.path_with_protocol)
+ metadata = self._s3_get_metadata()
+
+ if not 'symlink_to' in metadata:
+ raise S3NotALinkError('Not a link: %r' % self.path_with_protocol)
+ else:
+ return self.from_path(metadata['symlink_to'])
+
+[docs] def is_symlink(self) -> bool:
+ '''
+ Test whether a path is link
+
+ :returns: True if a path is link, else False
+ :raises: S3NotALinkError
+ '''
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket:
+ return False
+ if not key or key.endswith('/'):
+ return False
+ metadata = self._s3_get_metadata()
+ return 'symlink_to' in metadata
+
+[docs] def save(self, file_object: BinaryIO):
+ '''Write the opened binary stream to specified path, but the stream won't be closed
+
+ :param file_object: Stream to be read
+ '''
+ bucket, key = parse_s3_url(self.path_with_protocol)
+ if not bucket:
+ raise S3BucketNotFoundError(
+ 'Empty bucket name: %r' % self.path_with_protocol)
+ if not key or key.endswith('/'):
+ raise S3IsADirectoryError(
+ 'Is a directory: %r' % self.path_with_protocol)
+
+ with raise_s3_error(self.path_with_protocol):
+ self._client.upload_fileobj(file_object, Bucket=bucket, Key=key)
+
+[docs] def open(
+ self,
+ mode: str = 'r',
+ *,
+ encoding: Optional[str] = None,
+ errors: Optional[str] = None,
+ s3_open_func: Callable[[str, str], BinaryIO] = s3_open,
+ **kwargs) -> IO[AnyStr]: # pytype: disable=signature-mismatch
+ return s3_open_func( # pytype: disable=wrong-keyword-args
+ self,
+ mode,
+ encoding=encoding,
+ errors=errors,
+ **necessary_params(s3_open_func, **kwargs))
+
+[docs] def absolute(self) -> 'S3Path':
+ '''
+ Make the path absolute, without normalization or resolving symlinks. Returns a new path object
+ '''
+ return self
+
+[docs] def cwd(self) -> 'S3Path':
+ '''Return current working directory
+
+ returns: Current working directory
+ '''
+ return self.from_path(self.path_with_protocol)
+
+
+class MultiPartWriter:
+
+ def __init__(self, client, path: PathLike) -> None:
+ self._client = client
+ self._multipart_upload_info = []
+
+ bucket, key = parse_s3_url(path)
+ self._bucket = bucket
+ self._key = key
+ self._upload_id = self._client.create_multipart_upload(
+ Bucket=self._bucket, Key=self._key)['UploadId']
+
+ def upload_part(self, part_num: int, file_obj: io.BytesIO) -> None:
+ response = self._client.upload_part(
+ Body=file_obj,
+ UploadId=self._upload_id,
+ PartNumber=part_num,
+ Bucket=self._bucket,
+ Key=self._key,
+ )
+ self._multipart_upload_info.append(
+ {
+ 'PartNumber': part_num,
+ 'ETag': response['ETag']
+ })
+
+ def upload_part_by_paths(
+ self, part_num: int, paths: List[Tuple[PathLike, str]]) -> None:
+ file_obj = io.BytesIO()
+
+ def get_object(
+ client, bucket, key, range_str: Optional[str] = None) -> bytes:
+ if range_str:
+ return client.get_object(
+ Bucket=bucket, Key=key, Range=range_str)['Body'].read()
+ else:
+ return client.get_object(Bucket=bucket, Key=key)['Body'].read()
+
+ get_object = patch_method(
+ get_object,
+ max_retries=max_retries,
+ should_retry=s3_should_retry,
+ )
+ for path, bytes_range in paths:
+ bucket, key = parse_s3_url(path)
+ if bytes_range:
+ file_obj.write(
+ get_object(self._client, bucket, key, bytes_range))
+ else:
+ file_obj.write(get_object(self._client, bucket, key))
+ file_obj.seek(0, os.SEEK_SET)
+ self.upload_part(part_num, file_obj)
+
+ def upload_part_copy(
+ self,
+ part_num: int,
+ path: PathLike,
+ copy_source_range: Optional[str] = None) -> None:
+ bucket, key = parse_s3_url(path)
+ params = dict(
+ UploadId=self._upload_id,
+ PartNumber=part_num,
+ CopySource={
+ 'Bucket': bucket,
+ 'Key': key
+ },
+ Bucket=self._bucket,
+ Key=self._key,
+ )
+ if copy_source_range:
+ params['CopySourceRange'] = copy_source_range
+ response = self._client.upload_part_copy(**params)
+ self._multipart_upload_info.append(
+ {
+ 'PartNumber': part_num,
+ 'ETag': response['CopyPartResult']['ETag']
+ })
+
+ def close(self):
+ self._multipart_upload_info.sort(key=lambda t: t['PartNumber'])
+ self._client.complete_multipart_upload(
+ UploadId=self._upload_id,
+ Bucket=self._bucket,
+ Key=self._key,
+ MultipartUpload={'Parts': self._multipart_upload_info})
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
+from typing import IO, AnyStr, BinaryIO, Callable, Iterator, List, Optional, Tuple
+
+from megfile.interfaces import FileEntry, PathLike, StatResult
+from megfile.sftp_path import SftpPath, is_sftp, sftp_concat, sftp_download, sftp_glob, sftp_glob_stat, sftp_iglob, sftp_lstat, sftp_path_join, sftp_readlink, sftp_resolve, sftp_upload
+
+__all__ = [
+ 'is_sftp',
+ 'sftp_readlink',
+ 'sftp_glob',
+ 'sftp_iglob',
+ 'sftp_glob_stat',
+ 'sftp_resolve',
+ 'sftp_download',
+ 'sftp_upload',
+ 'sftp_path_join',
+ 'sftp_concat',
+ 'sftp_lstat',
+ 'sftp_exists',
+ 'sftp_getmtime',
+ 'sftp_getsize',
+ 'sftp_isdir',
+ 'sftp_isfile',
+ 'sftp_listdir',
+ 'sftp_load_from',
+ 'sftp_makedirs',
+ 'sftp_realpath',
+ 'sftp_rename',
+ 'sftp_move',
+ 'sftp_remove',
+ 'sftp_scan',
+ 'sftp_scan_stat',
+ 'sftp_scandir',
+ 'sftp_stat',
+ 'sftp_unlink',
+ 'sftp_walk',
+ 'sftp_getmd5',
+ 'sftp_symlink',
+ 'sftp_islink',
+ 'sftp_save_as',
+ 'sftp_open',
+ 'sftp_chmod',
+ 'sftp_absolute',
+ 'sftp_rmdir',
+ 'sftp_copy',
+ 'sftp_sync',
+]
+
+
+[docs]def sftp_exists(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if the path exists
+
+ :param path: Given path
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path exists, else False
+
+ '''
+ return SftpPath(path).exists(followlinks)
+
+
+[docs]def sftp_getmtime(path: PathLike, follow_symlinks: bool = False) -> float:
+ '''
+ Get last-modified time of the file on the given path (in Unix timestamp format).
+ If the path is an existent directory, return the latest modified time of all file in it.
+
+ :param path: Given path
+ :returns: last-modified time
+ '''
+ return SftpPath(path).getmtime(follow_symlinks)
+
+
+[docs]def sftp_getsize(path: PathLike, follow_symlinks: bool = False) -> int:
+ '''
+ Get file size on the given file path (in bytes).
+ If the path in a directory, return the sum of all file size in it, including file in subdirectories (if exist).
+ The result excludes the size of directory itself. In other words, return 0 Byte on an empty directory path.
+
+ :param path: Given path
+ :returns: File size
+
+ '''
+ return SftpPath(path).getsize(follow_symlinks)
+
+
+[docs]def sftp_isdir(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if a path is directory
+
+ .. note::
+
+ The difference between this function and ``os.path.isdir`` is that this function regard symlink as file
+
+ :param path: Given path
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path is a directory, else False
+
+ '''
+ return SftpPath(path).is_dir(followlinks)
+
+
+[docs]def sftp_isfile(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if a path is file
+
+ .. note::
+
+ The difference between this function and ``os.path.isfile`` is that this function regard symlink as file
+
+ :param path: Given path
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path is a file, else False
+
+ '''
+ return SftpPath(path).is_file(followlinks)
+
+
+[docs]def sftp_listdir(path: PathLike) -> List[str]:
+ '''
+ Get all contents of given sftp path. The result is in acsending alphabetical order.
+
+ :param path: Given path
+ :returns: All contents have in the path in acsending alphabetical order
+ '''
+ return SftpPath(path).listdir()
+
+
+[docs]def sftp_load_from(path: PathLike) -> BinaryIO:
+ '''Read all content on specified path and write into memory
+
+ User should close the BinaryIO manually
+
+ :param path: Given path
+ :returns: Binary stream
+ '''
+ return SftpPath(path).load()
+
+
+[docs]def sftp_makedirs(
+ path: PathLike, mode=0o777, parents: bool = False,
+ exist_ok: bool = False):
+ '''
+ make a directory on sftp, including parent directory
+
+ If there exists a file on the path, raise FileExistsError
+
+ :param path: Given path
+ :param mode: If mode is given, it is combined with the process’ umask value to determine the file mode and access flags.
+ :param parents: If parents is true, any missing parents of this path are created as needed;
+ If parents is false (the default), a missing parent raises FileNotFoundError.
+ :param exist_ok: If False and target directory exists, raise FileExistsError
+ :raises: FileExistsError
+ '''
+ return SftpPath(path).mkdir(mode, parents, exist_ok)
+
+
+[docs]def sftp_realpath(path: PathLike) -> str:
+ '''Return the real path of given path
+
+ :param path: Given path
+ :returns: Real path of given path
+ '''
+ return SftpPath(path).realpath()
+
+
+[docs]def sftp_rename(
+ src_path: PathLike, dst_path: PathLike,
+ overwrite: bool = True) -> 'SftpPath':
+ '''
+ rename file on sftp
+
+ :param src_path: Given path
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ return SftpPath(src_path).rename(dst_path, overwrite)
+
+
+[docs]def sftp_move(
+ src_path: PathLike, dst_path: PathLike,
+ overwrite: bool = True) -> 'SftpPath':
+ '''
+ move file on sftp
+
+ :param src_path: Given path
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ return SftpPath(src_path).replace(dst_path, overwrite)
+
+
+[docs]def sftp_remove(path: PathLike, missing_ok: bool = False) -> None:
+ '''
+ Remove the file or directory on sftp
+
+ :param path: Given path
+ :param missing_ok: if False and target file/directory not exists, raise FileNotFoundError
+ '''
+ return SftpPath(path).remove(missing_ok)
+
+
+[docs]def sftp_scan(
+ path: PathLike, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[str]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a path string.
+
+ If path is a file path, yields the file only
+ If path is a non-existent path, return an empty generator
+ If path is a bucket path, return all file paths in the bucket
+
+ :param path: Given path
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :returns: A file path generator
+ '''
+ return SftpPath(path).scan(missing_ok, followlinks)
+
+
+[docs]def sftp_scan_stat(
+ path: PathLike, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a tuple of path string and file stat
+
+ :param path: Given path
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :returns: A file path generator
+ '''
+ return SftpPath(path).scan_stat(missing_ok, followlinks)
+
+
+[docs]def sftp_scandir(path: PathLike) -> Iterator[FileEntry]:
+ '''
+ Get all content of given file path.
+
+ :param path: Given path
+ :returns: An iterator contains all contents have prefix path
+ '''
+ return SftpPath(path).scandir()
+
+
+[docs]def sftp_stat(path: PathLike, follow_symlinks=True) -> StatResult:
+ '''
+ Get StatResult of file on sftp, including file size and mtime, referring to fs_getsize and fs_getmtime
+
+ :param path: Given path
+ :returns: StatResult
+ '''
+ return SftpPath(path).stat(follow_symlinks)
+
+
+[docs]def sftp_unlink(path: PathLike, missing_ok: bool = False) -> None:
+ '''
+ Remove the file on sftp
+
+ :param path: Given path
+ :param missing_ok: if False and target file not exists, raise FileNotFoundError
+ '''
+ return SftpPath(path).unlink(missing_ok)
+
+
+[docs]def sftp_walk(path: PathLike, followlinks: bool = False
+ ) -> Iterator[Tuple[str, List[str], List[str]]]:
+ '''
+ Generate the file names in a directory tree by walking the tree top-down.
+ For each directory in the tree rooted at directory path (including path itself),
+ it yields a 3-tuple (root, dirs, files).
+
+ root: a string of current path
+ dirs: name list of subdirectories (excluding '.' and '..' if they exist) in 'root'. The list is sorted by ascending alphabetical order
+ files: name list of non-directory files (link is regarded as file) in 'root'. The list is sorted by ascending alphabetical order
+
+ If path not exists, or path is a file (link is regarded as file), return an empty generator
+
+ .. note::
+
+ Be aware that setting ``followlinks`` to True can lead to infinite recursion if a link points to a parent directory of itself. fs_walk() does not keep track of the directories it visited already.
+
+ :param path: Given path
+ :param followlinks: False if regard symlink as file, else True
+ :returns: A 3-tuple generator
+ '''
+ return SftpPath(path).walk(followlinks)
+
+
+[docs]def sftp_getmd5(
+ path: PathLike, recalculate: bool = False, followlinks: bool = True):
+ '''
+ Calculate the md5 value of the file
+
+ :param path: Given path
+ :param recalculate: Ignore this parameter, just for compatibility
+ :param followlinks: Ignore this parameter, just for compatibility
+ returns: md5 of file
+ '''
+ return SftpPath(path).md5(recalculate, followlinks)
+
+
+[docs]def sftp_symlink(src_path: PathLike, dst_path: PathLike) -> None:
+ '''
+ Create a symbolic link pointing to src_path named dst_path.
+
+ :param src_path: Given path
+ :param dst_path: Desination path
+ '''
+ return SftpPath(src_path).symlink(dst_path)
+
+
+[docs]def sftp_islink(path: PathLike) -> bool:
+ '''Test whether a path is a symbolic link
+
+ :param path: Given path
+ :return: If path is a symbolic link return True, else False
+ :rtype: bool
+ '''
+ return SftpPath(path).is_symlink()
+
+
+[docs]def sftp_save_as(file_object: BinaryIO, path: PathLike):
+ '''Write the opened binary stream to path
+ If parent directory of path doesn't exist, it will be created.
+
+ :param path: Given path
+ :param file_object: stream to be read
+ '''
+ return SftpPath(path).save(file_object)
+
+
+[docs]def sftp_open(
+ path: PathLike,
+ mode: str = 'r',
+ buffering=-1,
+ encoding: Optional[str] = None,
+ errors: Optional[str] = None,
+ **kwargs) -> IO[AnyStr]: # pytype: disable=signature-mismatch
+ '''Open a file on the path.
+
+ :param path: Given path
+ :param mode: Mode to open file
+ :param buffering: buffering is an optional integer used to set the buffering policy.
+ :param encoding: encoding is the name of the encoding used to decode or encode the file. This should only be used in text mode.
+ :param errors: errors is an optional string that specifies how encoding and decoding errors are to be handled—this cannot be used in binary mode.
+ :returns: File-Like object
+ '''
+ return SftpPath(path).open(mode, buffering, encoding, errors)
+
+
+[docs]def sftp_chmod(path: PathLike, mode: int, follow_symlinks: bool = True):
+ '''
+ Change the file mode and permissions, like os.chmod().
+
+ :param path: Given path
+ :param mode: the file mode you want to change
+ :param followlinks: Ignore this parameter, just for compatibility
+ '''
+ return SftpPath(path).chmod(mode, follow_symlinks)
+
+
+[docs]def sftp_absolute(path: PathLike) -> 'SftpPath':
+ '''
+ Make the path absolute, without normalization or resolving symlinks. Returns a new path object
+ '''
+ return SftpPath(path).absolute()
+
+
+[docs]def sftp_rmdir(path: PathLike):
+ '''
+ Remove this directory. The directory must be empty.
+ '''
+ return SftpPath(path).rmdir()
+
+
+[docs]def sftp_copy(
+ src_path: PathLike,
+ dst_path: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ followlinks: bool = False,
+ overwrite: bool = True):
+ """
+ Copy the file to the given destination path.
+
+ :param src_path: Given path
+ :param dst_path: The destination path to copy the file to.
+ :param callback: An optional callback function that takes an integer parameter and is called
+ periodically during the copy operation to report the number of bytes copied.
+ :param followlinks: Whether to follow symbolic links when copying directories.
+ :raises IsADirectoryError: If the source is a directory.
+ :raises OSError: If there is an error copying the file.
+ """
+ return SftpPath(src_path).copy(dst_path, callback, followlinks, overwrite)
+
+
+[docs]def sftp_sync(
+ src_path: PathLike,
+ dst_path: PathLike,
+ followlinks: bool = False,
+ force: bool = False,
+ overwrite: bool = True):
+ '''Copy file/directory on src_url to dst_url
+
+ :param src_path: Given path
+ :param dst_url: Given destination path
+ :param followlinks: False if regard symlink as file, else True
+ :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ return SftpPath(src_path).sync(dst_path, followlinks, force, overwrite)
+
+import atexit
+import fcntl
+import hashlib
+import io
+import os
+import random
+import shlex
+import socket
+import subprocess
+from logging import getLogger as get_logger
+from stat import S_ISDIR, S_ISLNK, S_ISREG
+from typing import IO, AnyStr, BinaryIO, Callable, Iterator, List, Optional, Tuple, Union
+from urllib.parse import urlsplit, urlunsplit
+
+import paramiko
+
+from megfile.config import SFTP_MAX_RETRY_TIMES
+from megfile.errors import SameFileError, _create_missing_ok_generator, patch_method
+from megfile.interfaces import ContextIterator, FileEntry, PathLike, StatResult
+from megfile.lib.compare import is_same_file
+from megfile.lib.compat import fspath
+from megfile.lib.glob import FSFunc, iglob
+from megfile.lib.joinpath import uri_join
+from megfile.pathlike import PathLike, URIPath
+from megfile.smart_path import SmartPath
+from megfile.utils import cachedproperty, calculate_md5, thread_local
+
+_logger = get_logger(__name__)
+
+__all__ = [
+ 'SftpPath',
+ 'is_sftp',
+ 'sftp_readlink',
+ 'sftp_glob',
+ 'sftp_iglob',
+ 'sftp_glob_stat',
+ 'sftp_resolve',
+ 'sftp_download',
+ 'sftp_upload',
+ 'sftp_path_join',
+ 'sftp_concat',
+ 'sftp_lstat',
+]
+
+SFTP_USERNAME = "SFTP_USERNAME"
+SFTP_PASSWORD = "SFTP_PASSWORD"
+SFTP_PRIVATE_KEY_PATH = "SFTP_PRIVATE_KEY_PATH"
+SFTP_PRIVATE_KEY_TYPE = "SFTP_PRIVATE_KEY_TYPE"
+SFTP_PRIVATE_KEY_PASSWORD = "SFTP_PRIVATE_KEY_PASSWORD"
+SFTP_MAX_UNAUTH_CONN = "SFTP_MAX_UNAUTH_CONN"
+MAX_RETRIES = SFTP_MAX_RETRY_TIMES
+DEFAULT_SSH_CONNECT_TIMEOUT = 5
+DEFAULT_SSH_KEEPALIVE_INTERVAL = 15
+
+
+def _make_stat(stat: paramiko.SFTPAttributes) -> StatResult:
+ return StatResult(
+ size=stat.st_size,
+ mtime=stat.st_mtime,
+ isdir=S_ISDIR(stat.st_mode),
+ islnk=S_ISLNK(stat.st_mode),
+ extra=stat,
+ )
+
+
+def get_private_key():
+ key_with_types = {
+ 'DSA': paramiko.DSSKey,
+ 'RSA': paramiko.RSAKey,
+ 'ECDSA': paramiko.ECDSAKey,
+ 'ED25519': paramiko.Ed25519Key,
+ }
+ key_type = os.getenv(SFTP_PRIVATE_KEY_TYPE, 'RSA').upper()
+ if os.getenv(SFTP_PRIVATE_KEY_PATH):
+ private_key_path = os.getenv(SFTP_PRIVATE_KEY_PATH)
+ if not os.path.exists(private_key_path):
+ raise FileNotFoundError(
+ f"Private key file not exist: '{SFTP_PRIVATE_KEY_PATH}'")
+ return key_with_types[key_type].from_private_key_file(
+ private_key_path, password=os.getenv(SFTP_PRIVATE_KEY_PASSWORD))
+ return None
+
+
+def provide_connect_info(
+ hostname: str,
+ port: Optional[int] = None,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+):
+ if not port:
+ port = 22
+ if not username:
+ username = os.getenv(SFTP_USERNAME)
+ if not password:
+ password = os.getenv(SFTP_PASSWORD)
+ private_key = get_private_key()
+ return hostname, port, username, password, private_key
+
+
+def sftp_should_retry(error: Exception) -> bool:
+ if type(error) is EOFError:
+ return False
+ elif isinstance(error, (
+ paramiko.ssh_exception.SSHException,
+ ConnectionError,
+ socket.timeout,
+ )):
+ return True
+ elif isinstance(error, OSError):
+ for err_msg in [
+ 'Socket is closed',
+ 'Cannot assign requested address',
+ ]:
+ if err_msg in str(error):
+ return True
+ return False
+
+
+def _patch_sftp_client_request(
+ client: paramiko.SFTPClient,
+ hostname: str,
+ port: Optional[int] = None,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+):
+
+ def retry_callback(error, *args, **kwargs):
+ client.close()
+ ssh_client = get_ssh_client(hostname, port, username, password)
+ ssh_client.close()
+ atexit.unregister(ssh_client.close)
+ ssh_key = f'ssh_client:{hostname},{port},{username},{password}'
+ if thread_local.get(ssh_key):
+ del thread_local[ssh_key]
+ sftp_key = f'sftp_client:{hostname},{port},{username},{password}'
+ if thread_local.get(sftp_key):
+ del thread_local[sftp_key]
+
+ new_sftp_client = get_sftp_client(
+ hostname=hostname,
+ port=port,
+ username=username,
+ password=password,
+ )
+ client.sock = new_sftp_client.sock
+
+ client._request = patch_method(
+ client._request, # pytype: disable=attribute-error
+ max_retries=MAX_RETRIES,
+ should_retry=sftp_should_retry,
+ retry_callback=retry_callback)
+ return client
+
+
+def _get_sftp_client(
+ hostname: str,
+ port: Optional[int] = None,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+) -> paramiko.SFTPClient:
+ '''Get sftp client
+
+ :returns: sftp client
+ '''
+ session = get_ssh_session(
+ hostname=hostname,
+ port=port,
+ username=username,
+ password=password,
+ )
+ session.invoke_subsystem("sftp")
+ sftp_client = paramiko.SFTPClient(session)
+ _patch_sftp_client_request(sftp_client, hostname, port, username, password)
+ return sftp_client
+
+
+def get_sftp_client(
+ hostname: str,
+ port: Optional[int] = None,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+) -> paramiko.SFTPClient:
+ '''Get sftp client
+
+ :returns: sftp client
+ '''
+ return thread_local(
+ f'sftp_client:{hostname},{port},{username},{password}',
+ _get_sftp_client, hostname, port, username, password)
+
+
+def _get_ssh_client(
+ hostname: str,
+ port: Optional[int] = None,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+) -> paramiko.SSHClient:
+ hostname, port, username, password, private_key = provide_connect_info(
+ hostname=hostname,
+ port=port,
+ username=username,
+ password=password,
+ )
+
+ ssh_client = paramiko.SSHClient()
+ ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ max_unauth_connections = int(os.getenv(SFTP_MAX_UNAUTH_CONN, 10))
+ try:
+ fd = os.open(
+ os.path.join(
+ '/tmp',
+ f'megfile-sftp-{hostname}-{random.randint(1, max_unauth_connections)}'
+ ), os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
+ except Exception:
+ _logger.warning(
+ "Can't create file lock in '/tmp', please control the SFTP concurrency count by yourself."
+ )
+ fd = None
+ if fd:
+ fcntl.flock(fd, fcntl.LOCK_EX)
+ ssh_client.connect(
+ hostname=hostname,
+ port=port,
+ username=username,
+ password=password,
+ pkey=private_key,
+ timeout=DEFAULT_SSH_CONNECT_TIMEOUT,
+ auth_timeout=DEFAULT_SSH_CONNECT_TIMEOUT,
+ banner_timeout=DEFAULT_SSH_CONNECT_TIMEOUT,
+ )
+ if fd:
+ fcntl.flock(fd, fcntl.LOCK_UN)
+ os.close(fd)
+ atexit.register(ssh_client.close)
+ return ssh_client
+
+
+def get_ssh_client(
+ hostname: str,
+ port: Optional[int] = None,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+) -> paramiko.SSHClient:
+ return thread_local(
+ f'ssh_client:{hostname},{port},{username},{password}', _get_ssh_client,
+ hostname, port, username, password)
+
+
+def get_ssh_session(
+ hostname: str,
+ port: Optional[int] = None,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+) -> paramiko.Channel:
+
+ def retry_callback(error, *args, **kwargs):
+ ssh_client = get_ssh_client(hostname, port, username, password)
+ ssh_client.close()
+ atexit.unregister(ssh_client.close)
+ ssh_key = f'ssh_client:{hostname},{port},{username},{password}'
+ if thread_local.get(ssh_key):
+ del thread_local[ssh_key]
+ sftp_key = f'sftp_client:{hostname},{port},{username},{password}'
+ if thread_local.get(sftp_key):
+ del thread_local[sftp_key]
+
+ return patch_method(
+ _open_session, # pytype: disable=attribute-error
+ max_retries=MAX_RETRIES,
+ should_retry=sftp_should_retry,
+ retry_callback=retry_callback)(
+ hostname,
+ port,
+ username,
+ password,
+ )
+
+
+def _open_session(
+ hostname: str,
+ port: Optional[int] = None,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+) -> paramiko.Channel:
+ ssh_client = get_ssh_client(hostname, port, username, password)
+ transport = ssh_client.get_transport()
+ if not transport:
+ raise paramiko.SSHException('Get transport error')
+ transport.set_keepalive(DEFAULT_SSH_KEEPALIVE_INTERVAL)
+ session = transport.open_session(timeout=DEFAULT_SSH_CONNECT_TIMEOUT)
+ if not session:
+ raise paramiko.SSHException('Create session error')
+ session.settimeout(DEFAULT_SSH_CONNECT_TIMEOUT)
+ return session
+
+
+[docs]def is_sftp(path: PathLike) -> bool:
+ '''Test if a path is sftp path
+
+ :param path: Path to be tested
+ :returns: True of a path is sftp path, else False
+ '''
+ path = fspath(path)
+ parts = urlsplit(path)
+ return parts.scheme == 'sftp'
+
+
+[docs]def sftp_readlink(path: PathLike) -> 'str':
+ '''
+ Return a SftpPath instance representing the path to which the symbolic link points.
+ :param path: Given path
+ :returns: Return a SftpPath instance representing the path to which the symbolic link points.
+ '''
+ return SftpPath(path).readlink().path_with_protocol
+
+
+[docs]def sftp_glob(path: PathLike, recursive: bool = True,
+ missing_ok: bool = True) -> List[str]:
+ '''Return path list in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param path: Given path
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: A list contains paths match `pathname`
+ '''
+ return list(
+ sftp_iglob(path=path, recursive=recursive, missing_ok=missing_ok))
+
+
+[docs]def sftp_glob_stat(
+ path: PathLike, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator[FileEntry]:
+ '''Return a list contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. sftp_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param path: Given path
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: A list contains tuples of path and file stat, in which paths match `pathname`
+ '''
+ for path in sftp_iglob(path=path, recursive=recursive,
+ missing_ok=missing_ok):
+ path_object = SftpPath(path)
+ yield FileEntry(
+ path_object.name, path_object.path_with_protocol,
+ path_object.lstat())
+
+
+[docs]def sftp_iglob(path: PathLike, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator[str]:
+ '''Return path iterator in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param path: Given path
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: An iterator contains paths match `pathname`
+ '''
+
+ for path in SftpPath(path).iglob(pattern="", recursive=recursive,
+ missing_ok=missing_ok):
+ yield path.path_with_protocol
+
+
+[docs]def sftp_resolve(path: PathLike, strict=False) -> 'str':
+ '''Equal to fs_realpath
+
+ :param path: Given path
+ :param strict: Ignore this parameter, just for compatibility
+ :return: Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.
+ :rtype: SftpPath
+ '''
+ return SftpPath(path).resolve(strict).path_with_protocol
+
+
+def _sftp_scan_pairs(src_url: PathLike,
+ dst_url: PathLike) -> Iterator[Tuple[PathLike, PathLike]]:
+ for src_file_path in SftpPath(src_url).scan():
+ content_path = src_file_path[len(src_url):]
+ if len(content_path) > 0:
+ dst_file_path = SftpPath(dst_url).joinpath(
+ content_path).path_with_protocol
+ else:
+ dst_file_path = dst_url
+ yield src_file_path, dst_file_path
+
+
+[docs]def sftp_download(
+ src_url: PathLike,
+ dst_url: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ followlinks: bool = False,
+ overwrite: bool = True):
+ '''
+ Downloads a file from sftp to local filesystem.
+ :param src_url: source sftp path
+ :param dst_url: target fs path
+ :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+ :param followlinks: False if regard symlink as file, else True
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ from megfile.fs import is_fs
+ from megfile.fs_path import FSPath
+
+ if not is_fs(dst_url):
+ raise OSError(f'dst_url is not fs path: {dst_url}')
+ if not is_sftp(src_url) and not isinstance(src_url, SftpPath):
+ raise OSError(f'src_url is not sftp path: {src_url}')
+
+ dst_path = FSPath(dst_url)
+ if not overwrite and dst_path.exists():
+ return
+
+ if isinstance(src_url, SftpPath):
+ src_path = src_url
+ else:
+ src_path = SftpPath(src_url)
+
+ if followlinks and src_path.is_symlink():
+ src_path = src_path.readlink()
+ if src_path.is_dir():
+ raise IsADirectoryError('Is a directory: %r' % src_url)
+ if str(dst_url).endswith('/'):
+ raise IsADirectoryError('Is a directory: %r' % dst_url)
+
+ dst_path.parent.makedirs(exist_ok=True)
+
+ sftp_callback = None
+ if callback:
+ bytes_transferred_before = 0
+
+ def sftp_callback(bytes_transferred: int, _total_bytes: int):
+ nonlocal bytes_transferred_before
+ callback(bytes_transferred - bytes_transferred_before)
+ bytes_transferred_before = bytes_transferred
+
+ src_path._client.get(
+ src_path._real_path,
+ dst_path.path_without_protocol,
+ callback=sftp_callback)
+
+ src_stat = src_path.stat()
+ dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
+ dst_path.chmod(src_stat.st_mode)
+
+
+[docs]def sftp_upload(
+ src_url: PathLike,
+ dst_url: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ followlinks: bool = False,
+ overwrite: bool = True):
+ '''
+ Uploads a file from local filesystem to sftp server.
+ :param src_url: source fs path
+ :param dst_url: target sftp path
+ :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ from megfile.fs import is_fs
+ from megfile.fs_path import FSPath
+
+ if not is_fs(src_url):
+ raise OSError(f'src_url is not fs path: {src_url}')
+ if not is_sftp(dst_url) and not isinstance(dst_url, SftpPath):
+ raise OSError(f'dst_url is not sftp path: {dst_url}')
+
+ if followlinks and os.path.islink(src_url):
+ src_url = os.readlink(src_url)
+ if os.path.isdir(src_url):
+ raise IsADirectoryError('Is a directory: %r' % src_url)
+ if str(dst_url).endswith('/'):
+ raise IsADirectoryError('Is a directory: %r' % dst_url)
+
+ src_path = FSPath(src_url)
+ if isinstance(dst_url, SftpPath):
+ dst_path = dst_url
+ else:
+ dst_path = SftpPath(dst_url)
+ if not overwrite and dst_path.exists():
+ return
+
+ dst_path.parent.makedirs(exist_ok=True)
+
+ sftp_callback = None
+ if callback:
+ bytes_transferred_before = 0
+
+ def sftp_callback(bytes_transferred: int, _total_bytes: int):
+ nonlocal bytes_transferred_before
+ callback(bytes_transferred - bytes_transferred_before)
+ bytes_transferred_before = bytes_transferred
+
+ dst_path._client.put(
+ src_path.path_without_protocol,
+ dst_path._real_path,
+ callback=sftp_callback)
+
+ src_stat = src_path.stat()
+ dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
+ dst_path.chmod(src_stat.st_mode)
+
+
+[docs]def sftp_path_join(path: PathLike, *other_paths: PathLike) -> str:
+ '''
+ Concat 2 or more path to a complete path
+
+ :param path: Given path
+ :param other_paths: Paths to be concatenated
+ :returns: Concatenated complete path
+
+ .. note ::
+
+ The difference between this function and ``os.path.join`` is that this function ignores left side slash (which indicates absolute path) in ``other_paths`` and will directly concat.
+ e.g. os.path.join('/path', 'to', '/file') => '/file', but sftp_path_join('/path', 'to', '/file') => '/path/to/file'
+ '''
+ return uri_join(fspath(path), *map(fspath, other_paths))
+
+
+[docs]def sftp_concat(src_paths: List[PathLike], dst_path: PathLike) -> None:
+ '''Concatenate sftp files to one file.
+
+ :param src_paths: Given source paths
+ :param dst_path: Given destination path
+ '''
+ dst_path_obj = SftpPath(dst_path)
+
+ def get_real_path(path: PathLike) -> str:
+ return SftpPath(path)._real_path
+
+ command = [
+ 'cat', *map(get_real_path, src_paths), '>',
+ get_real_path(dst_path)
+ ]
+ exec_result = dst_path_obj._exec_command(command)
+ if exec_result.returncode != 0:
+ _logger.error(exec_result.stderr)
+ raise OSError(f'Failed to concat {src_paths} to {dst_path}')
+
+
+[docs]def sftp_lstat(path: PathLike) -> StatResult:
+ '''
+ Get StatResult of file on sftp, including file size and mtime, referring to fs_getsize and fs_getmtime
+
+ :param path: Given path
+ :returns: StatResult
+ '''
+ return SftpPath(path).lstat()
+
+
+[docs]@SmartPath.register
+class SftpPath(URIPath):
+ """sftp protocol
+
+ uri format:
+ - absolute path
+ - sftp://[username[:password]@]hostname[:port]//file_path
+ - relative path
+ - - sftp://[username[:password]@]hostname[:port]/file_path
+ """
+
+ protocol = "sftp"
+
+ def __init__(self, path: "PathLike", *other_paths: "PathLike"):
+ super().__init__(path, *other_paths)
+ parts = urlsplit(self.path)
+ self._urlsplit_parts = parts
+ self._real_path = parts.path
+ if parts.path.startswith('//'):
+ self._root_dir = '/'
+ else:
+ self._root_dir = self._client.normalize('.')
+ self._real_path = os.path.join(self._root_dir, parts.path.lstrip('/'))
+
+[docs] @cachedproperty
+ def parts(self) -> Tuple[str]:
+ '''A tuple giving access to the path’s various components'''
+ if self._urlsplit_parts.path.startswith('//'):
+ new_parts = self._urlsplit_parts._replace(path='//')
+ else:
+ new_parts = self._urlsplit_parts._replace(path='/')
+ parts = [urlunsplit(new_parts)]
+ path = self._urlsplit_parts.path.lstrip('/')
+ if path != '':
+ parts.extend(path.split('/'))
+ return tuple(parts)
+
+ @property
+ def _client(self):
+ return get_sftp_client(
+ hostname=self._urlsplit_parts.hostname,
+ port=self._urlsplit_parts.port,
+ username=self._urlsplit_parts.username,
+ password=self._urlsplit_parts.password)
+
+ def _generate_path_object(
+ self, sftp_local_path: str, resolve: bool = False):
+ if resolve or self._root_dir == '/':
+ sftp_local_path = f"//{sftp_local_path.lstrip('/')}"
+ else:
+ sftp_local_path = os.path.relpath(
+ sftp_local_path, start=self._root_dir)
+ if sftp_local_path == ".":
+ sftp_local_path = "/"
+ new_parts = self._urlsplit_parts._replace(path=sftp_local_path)
+ return self.from_path(urlunsplit(new_parts))
+
+[docs] def exists(self, followlinks: bool = False) -> bool:
+ '''
+ Test if the path exists
+
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path exists, else False
+
+ '''
+ try:
+ if followlinks:
+ self._client.stat(self._real_path)
+ else:
+ self._client.lstat(self._real_path)
+ return True
+ except FileNotFoundError:
+ return False
+
+[docs] def getmtime(self, follow_symlinks: bool = False) -> float:
+ '''
+ Get last-modified time of the file on the given path (in Unix timestamp format).
+ If the path is an existent directory, return the latest modified time of all file in it.
+
+ :returns: last-modified time
+ '''
+ return self.stat(follow_symlinks=follow_symlinks).mtime
+
+[docs] def getsize(self, follow_symlinks: bool = False) -> int:
+ '''
+ Get file size on the given file path (in bytes).
+ If the path in a directory, return the sum of all file size in it, including file in subdirectories (if exist).
+ The result excludes the size of directory itself. In other words, return 0 Byte on an empty directory path.
+
+ :returns: File size
+
+ '''
+ return self.stat(follow_symlinks=follow_symlinks).size
+
+[docs] def glob(self, pattern, recursive: bool = True,
+ missing_ok: bool = True) -> List['SftpPath']:
+ '''Return path list in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: A list contains paths match `pathname`
+ '''
+ return list(
+ self.iglob(
+ pattern=pattern, recursive=recursive, missing_ok=missing_ok))
+
+[docs] def glob_stat(
+ self, pattern, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator[FileEntry]:
+ '''Return a list contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. sftp_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: A list contains tuples of path and file stat, in which paths match `pathname`
+ '''
+ for path_obj in self.iglob(pattern=pattern, recursive=recursive,
+ missing_ok=missing_ok):
+ yield FileEntry(path_obj.name, path_obj.path, path_obj.lstat())
+
+[docs] def iglob(self, pattern, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator['SftpPath']:
+ '''Return path iterator in ascending alphabetical order, in which path matches glob pattern
+
+ 1. If doesn't match any path, return empty list
+ Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist. fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+ 2. No guarantee that each path in result is different, which means:
+ Assume there exists a path `/a/b/c/b/d.txt`
+ use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
+ 3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
+ 5. Hidden files (filename stars with '.') will not be found in the result
+
+ :param pattern: Glob the given relative pattern in the directory represented by this path
+ :param recursive: If False, `**` will not search directory recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ :returns: An iterator contains paths match `pathname`
+ '''
+ glob_path = self.path_with_protocol
+ if pattern:
+ glob_path = self.joinpath(pattern).path_with_protocol
+
+ def _scandir(dirname: str) -> Iterator[Tuple[str, bool]]:
+ for entry in self.from_path(dirname).scandir():
+ yield entry.name, entry.is_dir()
+
+ def _exist(path: PathLike, followlinks: bool = False):
+ return self.from_path(path).exists(followlinks=followlinks)
+
+ def _is_dir(path: PathLike, followlinks: bool = False):
+ return self.from_path(path).is_dir(followlinks=followlinks)
+
+ fs = FSFunc(_exist, _is_dir, _scandir)
+ for real_path in _create_missing_ok_generator(
+ iglob(fspath(glob_path), recursive=recursive,
+ fs=fs), missing_ok,
+ FileNotFoundError('No match any file: %r' % glob_path)):
+ yield self.from_path(real_path)
+
+[docs] def is_dir(self, followlinks: bool = False) -> bool:
+ '''
+ Test if a path is directory
+
+ .. note::
+
+ The difference between this function and ``os.path.isdir`` is that this function regard symlink as file
+
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path is a directory, else False
+
+ '''
+ try:
+ stat = self.stat(follow_symlinks=followlinks)
+ if S_ISDIR(stat.st_mode):
+ return True
+ except FileNotFoundError:
+ pass
+ return False
+
+[docs] def is_file(self, followlinks: bool = False) -> bool:
+ '''
+ Test if a path is file
+
+ .. note::
+
+ The difference between this function and ``os.path.isfile`` is that this function regard symlink as file
+
+ :param followlinks: False if regard symlink as file, else True
+ :returns: True if the path is a file, else False
+
+ '''
+ try:
+ stat = self.stat(follow_symlinks=followlinks)
+ if S_ISREG(stat.st_mode):
+ return True
+ except FileNotFoundError:
+ pass
+ return False
+
+[docs] def listdir(self) -> List[str]:
+ '''
+ Get all contents of given sftp path. The result is in acsending alphabetical order.
+
+ :returns: All contents have in the path in acsending alphabetical order
+ '''
+ if not self.is_dir():
+ raise NotADirectoryError(
+ f"Not a directory: '{self.path_with_protocol}'")
+ return sorted(self._client.listdir(self._real_path))
+
+[docs] def iterdir(self) -> Iterator['SftpPath']:
+ '''
+ Get all contents of given sftp path. The result is in acsending alphabetical order.
+
+ :returns: All contents have in the path in acsending alphabetical order
+ '''
+ if not self.is_dir():
+ raise NotADirectoryError(
+ f"Not a directory: '{self.path_with_protocol}'")
+ for path in self.listdir():
+ yield self.joinpath(path) # type: ignore
+
+[docs] def load(self) -> BinaryIO:
+ '''Read all content on specified path and write into memory
+
+ User should close the BinaryIO manually
+
+ :returns: Binary stream
+ '''
+ with self.open(mode='rb') as f:
+ data = f.read()
+ return io.BytesIO(data)
+
+[docs] def mkdir(self, mode=0o777, parents: bool = False, exist_ok: bool = False):
+ '''
+ make a directory on sftp, including parent directory
+
+ If there exists a file on the path, raise FileExistsError
+
+ :param mode: If mode is given, it is combined with the process’ umask value to determine the file mode and access flags.
+ :param parents: If parents is true, any missing parents of this path are created as needed;
+ If parents is false (the default), a missing parent raises FileNotFoundError.
+ :param exist_ok: If False and target directory exists, raise FileExistsError
+ :raises: FileExistsError
+ '''
+ if self.exists():
+ if not exist_ok:
+ raise FileExistsError(
+ f"File exists: '{self.path_with_protocol}'")
+ return
+
+ if parents:
+ parent_path_objects = []
+ for parent_path_object in self.parents:
+ if parent_path_object.exists():
+ break
+ else:
+ parent_path_objects.append(parent_path_object)
+ for parent_path_object in parent_path_objects[::-1]:
+ parent_path_object.mkdir(
+ mode=mode, parents=False, exist_ok=True)
+ try:
+ self._client.mkdir(path=self._real_path, mode=mode)
+ except OSError:
+ # catch OSError when mkdir concurrently
+ if not self.exists():
+ raise
+
+[docs] def realpath(self) -> str:
+ '''Return the real path of given path
+
+ :returns: Real path of given path
+ '''
+ return self.resolve().path_with_protocol
+
+ def _is_same_backend(self, other: 'SftpPath') -> bool:
+ return self._urlsplit_parts.hostname == other._urlsplit_parts.hostname and self._urlsplit_parts.username == other._urlsplit_parts.username and self._urlsplit_parts.password == other._urlsplit_parts.password and self._urlsplit_parts.port == other._urlsplit_parts.port
+
+ def _is_same_protocol(self, path):
+ return is_sftp(path)
+
+[docs] def rename(self, dst_path: PathLike, overwrite: bool = True) -> 'SftpPath':
+ '''
+ rename file on sftp
+
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ if not self._is_same_protocol(dst_path):
+ raise OSError('Not a %s path: %r' % (self.protocol, dst_path))
+
+ dst_path = self.from_path(str(dst_path).rstrip('/'))
+
+ src_stat = self.stat()
+
+ if self._is_same_backend(dst_path):
+ if overwrite:
+ dst_path.remove(missing_ok=True)
+ self._client.rename(self._real_path, dst_path._real_path)
+ else:
+ self.sync(dst_path, overwrite=overwrite)
+ self.remove(missing_ok=True)
+ else:
+ if self.is_dir():
+ for file_entry in self.scandir():
+ self.from_path(file_entry.path).rename(
+ dst_path.joinpath(file_entry.name))
+ self._client.rmdir(self._real_path)
+ else:
+ if overwrite or not dst_path.exists():
+ with self.open('rb') as fsrc:
+ with dst_path.open('wb') as fdst:
+ length = 16 * 1024
+ while True:
+ buf = fsrc.read(length)
+ if not buf:
+ break
+ fdst.write(buf)
+ self.unlink()
+
+ dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
+ dst_path.chmod(src_stat.st_mode)
+ return dst_path
+
+[docs] def replace(self, dst_path: PathLike, overwrite: bool = True) -> 'SftpPath':
+ '''
+ move file on sftp
+
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ return self.rename(dst_path=dst_path, overwrite=overwrite)
+
+[docs] def remove(self, missing_ok: bool = False) -> None:
+ '''
+ Remove the file or directory on sftp
+
+ :param missing_ok: if False and target file/directory not exists, raise FileNotFoundError
+ '''
+ if missing_ok and not self.exists():
+ return
+ if self.is_dir():
+ for file_entry in self.scandir():
+ self.from_path(file_entry.path).remove(missing_ok=missing_ok)
+ self._client.rmdir(self._real_path)
+ else:
+ self._client.unlink(self._real_path)
+
+[docs] def scan(self, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[str]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a path string.
+
+ If path is a file path, yields the file only
+ If path is a non-existent path, return an empty generator
+ If path is a bucket path, return all file paths in the bucket
+
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :returns: A file path generator
+ '''
+ scan_stat_iter = self.scan_stat(
+ missing_ok=missing_ok, followlinks=followlinks)
+
+ for file_entry in scan_stat_iter:
+ yield file_entry.path
+
+[docs] def scan_stat(self, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a tuple of path string and file stat
+
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :returns: A file path generator
+ '''
+
+ def create_generator() -> Iterator[FileEntry]:
+
+ try:
+ stat = self.stat(follow_symlinks=followlinks)
+ except FileNotFoundError:
+ return
+ if S_ISREG(stat.st_mode):
+ yield FileEntry(
+ self.name, self.path_with_protocol,
+ self.stat(follow_symlinks=followlinks))
+ return
+
+ for name in self.listdir():
+ current_path = self.joinpath(name)
+ if current_path.is_dir():
+ yield from current_path.scan_stat(
+ missing_ok=missing_ok,
+ followlinks=followlinks,
+ )
+ else:
+ yield FileEntry(
+ current_path.name, # type: ignore
+ current_path.path_with_protocol,
+ current_path.stat(follow_symlinks=followlinks))
+
+ return _create_missing_ok_generator(
+ create_generator(), missing_ok,
+ FileNotFoundError(
+ 'No match any file in: %r' % self.path_with_protocol))
+
+[docs] def scandir(self) -> Iterator[FileEntry]:
+ '''
+ Get all content of given file path.
+
+ :returns: An iterator contains all contents have prefix path
+ '''
+ if not self.exists():
+ raise FileNotFoundError(
+ 'No such directory: %r' % self.path_with_protocol)
+
+ if not self.is_dir():
+ raise NotADirectoryError(
+ 'Not a directory: %r' % self.path_with_protocol)
+
+ def create_generator():
+ for name in self.listdir():
+ current_path = self.joinpath(name)
+ yield FileEntry(
+ current_path.name, # type: ignore
+ current_path.path_with_protocol,
+ current_path.lstat()) # type: ignore
+
+ return ContextIterator(create_generator())
+
+[docs] def stat(self, follow_symlinks=True) -> StatResult:
+ '''
+ Get StatResult of file on sftp, including file size and mtime, referring to fs_getsize and fs_getmtime
+
+ :returns: StatResult
+ '''
+ if follow_symlinks:
+ result = _make_stat(self._client.stat(self._real_path))
+ else:
+ result = _make_stat(self._client.lstat(self._real_path))
+ return result
+
+[docs] def unlink(self, missing_ok: bool = False) -> None:
+ '''
+ Remove the file on sftp
+
+ :param missing_ok: if False and target file not exists, raise FileNotFoundError
+ '''
+ if missing_ok and not self.exists():
+ return
+ self._client.unlink(self._real_path)
+
+[docs] def walk(self, followlinks: bool = False
+ ) -> Iterator[Tuple[str, List[str], List[str]]]:
+ '''
+ Generate the file names in a directory tree by walking the tree top-down.
+ For each directory in the tree rooted at directory path (including path itself),
+ it yields a 3-tuple (root, dirs, files).
+
+ root: a string of current path
+ dirs: name list of subdirectories (excluding '.' and '..' if they exist) in 'root'. The list is sorted by ascending alphabetical order
+ files: name list of non-directory files (link is regarded as file) in 'root'. The list is sorted by ascending alphabetical order
+
+ If path not exists, or path is a file (link is regarded as file), return an empty generator
+
+ .. note::
+
+ Be aware that setting ``followlinks`` to True can lead to infinite recursion if a link points to a parent directory of itself. fs_walk() does not keep track of the directories it visited already.
+
+ :param followlinks: False if regard symlink as file, else True
+ :returns: A 3-tuple generator
+ '''
+ if not self.exists(followlinks=followlinks):
+ return
+
+ if self.is_file(followlinks=followlinks):
+ return
+
+ stack = [self._real_path]
+ while stack:
+ root = stack.pop()
+ dirs, files = [], []
+ filenames = self._client.listdir(root)
+ for name in filenames:
+ current_path = self._generate_path_object(root).joinpath(name)
+ if current_path.is_file(followlinks=followlinks):
+ files.append(name)
+ elif current_path.is_dir(followlinks=followlinks):
+ dirs.append(name)
+
+ dirs = sorted(dirs)
+ files = sorted(files)
+
+ yield self._generate_path_object(
+ root).path_with_protocol, dirs, files
+
+ stack.extend(
+ (os.path.join(root, directory) for directory in reversed(dirs)))
+
+[docs] def resolve(self, strict=False) -> 'SftpPath':
+ '''Equal to sftp_realpath
+
+ :param strict: Ignore this parameter, just for compatibility
+ :return: Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.
+ :rtype: SftpPath
+ '''
+ path = self._client.normalize(self._real_path)
+ return self._generate_path_object(path, resolve=True)
+
+[docs] def md5(self, recalculate: bool = False, followlinks: bool = True):
+ '''
+ Calculate the md5 value of the file
+
+ :param recalculate: Ignore this parameter, just for compatibility
+ :param followlinks: Ignore this parameter, just for compatibility
+ returns: md5 of file
+ '''
+ if self.is_dir():
+ hash_md5 = hashlib.md5() # nosec
+ for file_name in self.listdir():
+ chunk = self.joinpath(file_name).md5( # type: ignore
+ recalculate=recalculate, followlinks=followlinks).encode()
+ hash_md5.update(chunk)
+ return hash_md5.hexdigest()
+ with self.open('rb') as src: # type: ignore
+ md5 = calculate_md5(src)
+ return md5
+
+[docs] def symlink(self, dst_path: PathLike) -> None:
+ '''
+ Create a symbolic link pointing to src_path named dst_path.
+
+ :param dst_path: Desination path
+ '''
+ dst_path = self.from_path(dst_path)
+ if dst_path.exists(followlinks=False):
+ raise FileExistsError(
+ f"File exists: '{dst_path.path_with_protocol}'")
+ return self._client.symlink(self._real_path, dst_path._real_path)
+
+[docs] def readlink(self) -> 'SftpPath':
+ '''
+ Return a SftpPath instance representing the path to which the symbolic link points.
+ :returns: Return a SftpPath instance representing the path to which the symbolic link points.
+ '''
+ if not self.is_symlink():
+ raise OSError('Not a symlink: %s' % self.path_with_protocol)
+ path = self._client.readlink(self._real_path)
+ if not path:
+ raise OSError('Not a symlink: %s' % self.path_with_protocol)
+ if not path.startswith('/'):
+ return self.parent.joinpath(path)
+ return self._generate_path_object(path)
+
+[docs] def is_symlink(self) -> bool:
+ '''Test whether a path is a symbolic link
+
+ :return: If path is a symbolic link return True, else False
+ :rtype: bool
+ '''
+ return self.lstat().is_symlink()
+
+[docs] def cwd(self) -> 'SftpPath':
+ '''Return current working directory
+
+ returns: Current working directory
+ '''
+ return self._generate_path_object(self._client.normalize('.'))
+
+[docs] def save(self, file_object: BinaryIO):
+ '''Write the opened binary stream to path
+ If parent directory of path doesn't exist, it will be created.
+
+ :param file_object: stream to be read
+ '''
+ with self.open(mode='wb') as output:
+ output.write(file_object.read())
+
+[docs] def open(
+ self,
+ mode: str = 'r',
+ buffering=-1,
+ encoding: Optional[str] = None,
+ errors: Optional[str] = None,
+ **kwargs) -> IO[AnyStr]: # pytype: disable=signature-mismatch
+ '''Open a file on the path.
+
+ :param mode: Mode to open file
+ :param buffering: buffering is an optional integer used to set the buffering policy.
+ :param encoding: encoding is the name of the encoding used to decode or encode the file. This should only be used in text mode.
+ :param errors: errors is an optional string that specifies how encoding and decoding errors are to be handled—this cannot be used in binary mode.
+ :returns: File-Like object
+ '''
+ if 'w' in mode or 'x' in mode or 'a' in mode:
+ if self.is_dir():
+ raise IsADirectoryError(
+ 'Is a directory: %r' % self.path_with_protocol)
+ self.parent.mkdir(parents=True, exist_ok=True)
+ elif not self.exists():
+ raise FileNotFoundError(
+ 'No such file: %r' % self.path_with_protocol)
+ fileobj = self._client.open(self._real_path, mode, bufsize=buffering)
+ fileobj.name = self.path
+ if 'r' in mode and 'b' not in mode:
+ return io.TextIOWrapper(
+ fileobj, encoding=encoding, errors=errors) # type: ignore
+ return fileobj # type: ignore
+
+[docs] def chmod(self, mode: int, follow_symlinks: bool = True):
+ '''
+ Change the file mode and permissions, like os.chmod().
+
+ :param mode: the file mode you want to change
+ :param followlinks: Ignore this parameter, just for compatibility
+ '''
+ return self._client.chmod(path=self._real_path, mode=mode)
+
+[docs] def absolute(self) -> 'SftpPath':
+ '''
+ Make the path absolute, without normalization or resolving symlinks. Returns a new path object
+ '''
+ return self.resolve()
+
+[docs] def rmdir(self):
+ '''
+ Remove this directory. The directory must be empty.
+ '''
+ if len(self.listdir()) > 0:
+ raise OSError(f"Directory not empty: '{self.path_with_protocol}'")
+ return self._client.rmdir(self._real_path)
+
+ def _exec_command(
+ self,
+ command: List[str],
+ bufsize: int = -1,
+ timeout: Optional[int] = None,
+ environment: Optional[dict] = None,
+ ) -> subprocess.CompletedProcess:
+ with get_ssh_session(
+ hostname=self._urlsplit_parts.hostname,
+ port=self._urlsplit_parts.port,
+ username=self._urlsplit_parts.username,
+ password=self._urlsplit_parts.password,
+ ) as chan:
+ chan.settimeout(timeout)
+ if environment:
+ chan.update_environment(environment)
+ chan.exec_command(' '.join([shlex.quote(arg) for arg in command]))
+ stdout = chan.makefile(
+ "r", bufsize).read().decode(errors="backslashreplace")
+ stderr = chan.makefile_stderr(
+ "r", bufsize).read().decode(errors="backslashreplace")
+ returncode = chan.recv_exit_status()
+ return subprocess.CompletedProcess(
+ args=command, returncode=returncode, stdout=stdout, stderr=stderr)
+
+[docs] def copy(
+ self,
+ dst_path: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ followlinks: bool = False,
+ overwrite: bool = True):
+ """
+ Copy the file to the given destination path.
+
+ :param dst_path: The destination path to copy the file to.
+ :param callback: An optional callback function that takes an integer parameter and is called
+ periodically during the copy operation to report the number of bytes copied.
+ :param followlinks: Whether to follow symbolic links when copying directories.
+ :raises IsADirectoryError: If the source is a directory.
+ :raises OSError: If there is an error copying the file.
+ """
+ if followlinks and self.is_symlink():
+ return self.readlink().copy(dst_path=dst_path, callback=callback)
+
+ if not self._is_same_protocol(dst_path):
+ raise OSError('Not a %s path: %r' % (self.protocol, dst_path))
+ if str(dst_path).endswith('/'):
+ raise IsADirectoryError('Is a directory: %r' % dst_path)
+
+ if self.is_dir():
+ raise IsADirectoryError(
+ 'Is a directory: %r' % self.path_with_protocol)
+
+ if not overwrite and self.from_path(dst_path).exists():
+ return
+
+ self.from_path(os.path.dirname(dst_path)).makedirs(exist_ok=True)
+ dst_path = self.from_path(dst_path)
+ if self._is_same_backend(dst_path):
+ if self._real_path == dst_path._real_path:
+ raise SameFileError(
+ f"'{self.path}' and '{dst_path.path}' are the same file")
+ exec_result = self._exec_command(
+ ["cp", self._real_path, dst_path._real_path])
+ if exec_result.returncode != 0:
+ _logger.error(exec_result.stderr)
+ raise OSError(
+ f'Copy file error, returncode: {exec_result.returncode}')
+ if callback:
+ callback(self.stat(follow_symlinks=followlinks).size)
+ else:
+ with self.open('rb') as fsrc:
+ with dst_path.open('wb') as fdst:
+ length = 16 * 1024
+ while True:
+ buf = fsrc.read(length)
+ if not buf:
+ break
+ fdst.write(buf)
+ if callback:
+ callback(len(buf))
+
+ src_stat = self.stat()
+ dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
+ dst_path._client.chmod(dst_path._real_path, src_stat.st_mode)
+
+[docs] def sync(
+ self,
+ dst_path: PathLike,
+ followlinks: bool = False,
+ force: bool = False,
+ overwrite: bool = True):
+ '''Copy file/directory on src_url to dst_url
+
+ :param dst_url: Given destination path
+ :param followlinks: False if regard symlink as file, else True
+ :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ if not self._is_same_protocol(dst_path):
+ raise OSError('Not a %s path: %r' % (self.protocol, dst_path))
+
+ for src_file_path, dst_file_path in _sftp_scan_pairs(
+ self.path_with_protocol, dst_path):
+ dst_path = self.from_path(dst_file_path)
+ src_path = self.from_path(src_file_path)
+
+ if force:
+ pass
+ elif not overwrite and dst_path.exists():
+ continue
+ elif dst_path.exists() and is_same_file(src_path.stat(),
+ dst_path.stat(), 'copy'):
+ continue
+
+ self.from_path(src_file_path).copy(
+ dst_file_path, followlinks=followlinks)
+
+[docs] def utime(self, atime: Union[float, int], mtime: Union[float, int]) -> None:
+ """
+ Set the access and modified times of the file specified by path.
+
+ :param atime: The access time to be set.
+ :type atime: Union[float, int]
+ :param mtime: The modification time to be set.
+ :type mtime: Union[float, int]
+ :return: None
+ """
+ return self._client.utime(self._real_path, (atime, mtime))
+
+import os
+from collections import defaultdict
+from functools import partial
+from stat import S_ISDIR as stat_isdir
+from stat import S_ISLNK as stat_islnk
+from typing import IO, Any, AnyStr, BinaryIO, Callable, Iterable, Iterator, List, Optional, Tuple
+
+from tqdm import tqdm
+
+from megfile.errors import S3UnknownError
+from megfile.fs import fs_copy, is_fs
+from megfile.interfaces import Access, ContextIterator, FileCacher, FileEntry, NullCacher, PathLike, StatResult
+from megfile.lib.combine_reader import CombineReader
+from megfile.lib.compare import get_sync_type, is_same_file
+from megfile.lib.compat import fspath
+from megfile.lib.glob import globlize, ungloblize
+from megfile.s3 import is_s3, s3_concat, s3_copy, s3_download, s3_load_content, s3_open, s3_upload
+from megfile.sftp import sftp_concat, sftp_copy, sftp_download, sftp_upload
+from megfile.smart_path import SmartPath, get_traditional_path
+from megfile.utils import combine, generate_cache_path
+
+__all__ = [
+ 'smart_access',
+ 'smart_cache',
+ 'smart_combine_open',
+ 'smart_copy',
+ 'smart_exists',
+ 'smart_getmtime',
+ 'smart_getsize',
+ 'smart_glob_stat',
+ 'smart_glob',
+ 'smart_iglob',
+ 'smart_isdir',
+ 'smart_isfile',
+ 'smart_islink',
+ 'smart_listdir',
+ 'smart_load_content',
+ 'smart_save_content',
+ 'smart_load_from',
+ 'smart_load_text',
+ 'smart_save_text',
+ 'smart_makedirs',
+ 'smart_open',
+ 'smart_path_join',
+ 'smart_remove',
+ 'smart_move',
+ 'smart_rename',
+ 'smart_save_as',
+ 'smart_scan_stat',
+ 'smart_scan',
+ 'smart_scandir',
+ 'smart_stat',
+ 'smart_sync',
+ 'smart_sync_with_progress',
+ 'smart_touch',
+ 'smart_unlink',
+ 'smart_walk',
+ 'smart_getmd5',
+ 'smart_realpath',
+ 'smart_ismount',
+ 'smart_relpath',
+ 'smart_abspath',
+ 'smart_isabs',
+ 'smart_symlink',
+ 'smart_readlink',
+ 'register_copy_func',
+ 'smart_concat',
+ 'SmartCacher',
+]
+
+
+[docs]def smart_symlink(src_path: PathLike, dst_path: PathLike) -> None:
+ '''
+ Create a symbolic link pointing to src_path named path.
+
+ :param src_path: Source path
+ :param dst_path: Desination path
+ '''
+ return SmartPath(src_path).symlink(dst_path)
+
+
+[docs]def smart_readlink(path: PathLike) -> PathLike:
+ '''
+ Return a string representing the path to which the symbolic link points.
+ :param path: Path to be read
+ :returns: Return a string representing the path to which the symbolic link points.
+ '''
+ return SmartPath(path).readlink()
+
+
+[docs]def smart_isdir(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if a file path or an s3 url is directory
+
+ :param path: Path to be tested
+ :returns: True if path is directory, else False
+ '''
+ return SmartPath(path).is_dir(followlinks=followlinks)
+
+
+[docs]def smart_isfile(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if a file path or an s3 url is file
+
+ :param path: Path to be tested
+ :returns: True if path is file, else False
+ '''
+ return SmartPath(path).is_file(followlinks=followlinks)
+
+
+
+
+
+[docs]def smart_access(path: PathLike, mode: Access) -> bool:
+ '''
+ Test if path has access permission described by mode
+
+ :param path: Path to be tested
+ :param mode: Access mode(Access.READ, Access.WRITE, Access.BUCKETREAD, Access.BUCKETWRITE)
+ :returns: bool, if the path has read/write access.
+ '''
+ return SmartPath(path).access(mode)
+
+
+[docs]def smart_exists(path: PathLike, followlinks: bool = False) -> bool:
+ '''
+ Test if path or s3_url exists
+
+ :param path: Path to be tested
+ :returns: True if path eixsts, else False
+ '''
+ return SmartPath(path).exists(followlinks=followlinks)
+
+
+[docs]def smart_listdir(path: Optional[PathLike] = None) -> List[str]:
+ '''
+ Get all contents of given s3_url or file path. The result is in acsending alphabetical order.
+
+ :param path: Given path
+ :returns: All contents of given s3_url or file path in acsending alphabetical order.
+ :raises: FileNotFoundError, NotADirectoryError
+ '''
+ if path is None:
+ return sorted(os.listdir(path))
+ return SmartPath(path).listdir()
+
+
+[docs]def smart_scandir(path: Optional[PathLike] = None) -> Iterator[FileEntry]:
+ '''
+ Get all content of given s3_url or file path.
+
+ :param path: Given path
+ :returns: An iterator contains all contents have prefix path
+ :raises: FileNotFoundError, NotADirectoryError
+ '''
+ if path is None:
+
+ def create_generator():
+ with os.scandir(None) as entries:
+ for entry in entries:
+ stat = entry.stat()
+ yield FileEntry(
+ entry.name, entry.path,
+ StatResult(
+ size=stat.st_size,
+ ctime=stat.st_ctime,
+ mtime=stat.st_mtime,
+ isdir=stat_isdir(stat.st_mode),
+ islnk=stat_islnk(stat.st_mode),
+ extra=stat,
+ ))
+
+ return ContextIterator(create_generator())
+ return SmartPath(path).scandir()
+
+
+[docs]def smart_getsize(path: PathLike) -> int:
+ '''
+ Get file size on the given s3_url or file path (in bytes).
+ If the path in a directory, return the sum of all file size in it, including file in subdirectories (if exist).
+ The result excludes the size of directory itself. In other words, return 0 Byte on an empty directory path.
+
+ :param path: Given path
+ :returns: File size
+ :raises: FileNotFoundError
+ '''
+ return SmartPath(path).getsize()
+
+
+[docs]def smart_getmtime(path: PathLike) -> float:
+ '''
+ Get last-modified time of the file on the given s3_url or file path (in Unix timestamp format).
+ If the path is an existent directory, return the latest modified time of all file in it. The mtime of empty directory is 1970-01-01 00:00:00
+
+ :param path: Given path
+ :returns: Last-modified time
+ :raises: FileNotFoundError
+ '''
+ return SmartPath(path).getmtime()
+
+
+[docs]def smart_stat(path: PathLike, follow_symlinks=True) -> StatResult:
+ '''
+ Get StatResult of s3_url or file path
+
+ :param path: Given path
+ :returns: StatResult
+ :raises: FileNotFoundError
+ '''
+ return SmartPath(path).stat(follow_symlinks=follow_symlinks)
+
+
+def smart_lstat(path: PathLike) -> StatResult:
+ '''
+ Get StatResult of path but do not follow symbolic links
+
+ :param path: Given path
+ :returns: StatResult
+ :raises: FileNotFoundError
+ '''
+ return SmartPath(path).lstat()
+
+
+_copy_funcs = {
+ 's3': {
+ 's3': s3_copy,
+ 'file': s3_download,
+ },
+ 'file': {
+ 's3': s3_upload,
+ 'file': fs_copy,
+ 'sftp': sftp_upload,
+ },
+ 'sftp': {
+ 'file': sftp_download,
+ 'sftp': sftp_copy,
+ },
+}
+
+
+[docs]def register_copy_func(
+ src_protocol: str,
+ dst_protocol: str,
+ copy_func: Optional[
+ Callable[[str, str, Optional[Callable[[int], None]]], None]] = None,
+) -> None:
+ '''
+ Used to register copy func between protocols, and do not allow duplicate registration
+
+ :param src_protocol: protocol name of source file, e.g. 's3'
+ :param dst_protocol: protocol name of destination file, e.g. 's3'
+ :param copy_func: copy func, its type is: Callable[[str, str, Optional[Callable[[int], None]]], None]
+ '''
+ try:
+ _copy_funcs[src_protocol][dst_protocol]
+ except KeyError:
+ dst_dict = _copy_funcs.get(src_protocol, {})
+ dst_dict[dst_protocol] = copy_func
+ _copy_funcs[src_protocol] = dst_dict
+ except Exception as error:
+ raise error
+ else:
+ raise ValueError(
+ 'Copy Function has already existed: {}->{}'.format(
+ src_protocol, dst_protocol))
+
+
+def _default_copy_func(
+ src_path: PathLike,
+ dst_path: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ followlinks: bool = False,
+ overwrite: bool = True) -> None:
+ if not overwrite and smart_exists(dst_path):
+ return
+
+ with smart_open(src_path, 'rb', followlinks=followlinks) as fsrc:
+ with smart_open(dst_path, 'wb') as fdst:
+ # This magic number is copied from copyfileobj
+ length = 16 * 1024
+ while True:
+ buf = fsrc.read(length)
+ if not buf:
+ break
+ fdst.write(buf)
+ if callback is None:
+ continue
+ callback(len(buf))
+ try:
+ src_stat = smart_stat(src_path)
+ dst_path = SmartPath(dst_path)
+ dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
+ except (NotImplementedError, TypeError):
+ pass
+
+
+[docs]def smart_copy(
+ src_path: PathLike,
+ dst_path: PathLike,
+ callback: Optional[Callable[[int], None]] = None,
+ followlinks: bool = False,
+ overwrite: bool = True) -> None:
+ '''
+ Copy file from source path to destination path
+
+ Here are a few examples: ::
+
+ >>> from tqdm import tqdm
+ >>> from megfile import smart_copy, smart_stat
+ >>> class Bar:
+ ... def __init__(self, total=10):
+ ... self._bar = tqdm(total=10)
+ ...
+ ... def __call__(self, bytes_num):
+ ... self._bar.update(bytes_num)
+ ...
+ >>> src_path = 'test.png'
+ >>> dst_path = 'test1.png'
+ >>> smart_copy(src_path, dst_path, callback=Bar(total=smart_stat(src_path).size), followlinks=False)
+ 856960it [00:00, 260592384.24it/s]
+
+ :param src_path: Given source path
+ :param dst_path: Given destination path
+ :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+ :param followlinks: False if regard symlink as file, else True
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ # this function contains plenty of manual polymorphism
+ if smart_islink(src_path) and is_s3(dst_path) and not followlinks:
+ return
+
+ src_protocol, _ = SmartPath._extract_protocol(src_path)
+ dst_protocol, _ = SmartPath._extract_protocol(dst_path)
+
+ try:
+ copy_func = _copy_funcs[src_protocol][dst_protocol]
+ except KeyError:
+ copy_func = _default_copy_func
+ try:
+ copy_func(
+ src_path,
+ dst_path,
+ callback=callback,
+ followlinks=followlinks,
+ overwrite=overwrite) # type: ignore
+ except S3UnknownError as e:
+ if 'cannot schedule new futures after interpreter shutdown' in str(e):
+ _default_copy_func(
+ src_path,
+ dst_path,
+ callback=callback,
+ followlinks=followlinks,
+ overwrite=overwrite) # type: ignore
+ else:
+ raise
+
+
+def _smart_sync_single_file(items: dict):
+ src_root_path = items['src_root_path']
+ dst_root_path = items['dst_root_path']
+ src_file_path = items['src_file_path']
+ callback = items['callback']
+ followlinks = items['followlinks']
+ callback_after_copy_file = items['callback_after_copy_file']
+ force = items['force']
+ overwrite = items['overwrite']
+
+ content_path = os.path.relpath(src_file_path, start=src_root_path)
+ if len(content_path) and content_path != '.':
+ content_path = content_path.lstrip('/')
+ dst_abs_file_path = smart_path_join(dst_root_path, content_path)
+ else:
+ # if content_path is empty, which means smart_isfile(src_path) is True, this function is equal to smart_copy
+ dst_abs_file_path = dst_root_path
+
+ src_protocol, _ = SmartPath._extract_protocol(src_file_path)
+ dst_protocol, _ = SmartPath._extract_protocol(dst_abs_file_path)
+ should_sync = True
+ try:
+ if force:
+ pass
+ elif not overwrite and smart_exists(dst_abs_file_path):
+ should_sync = False
+ elif smart_exists(dst_abs_file_path) and is_same_file(
+ smart_stat(src_file_path, follow_symlinks=followlinks),
+ smart_stat(dst_abs_file_path, follow_symlinks=followlinks),
+ get_sync_type(src_protocol, dst_protocol)):
+ should_sync = False
+ except NotImplementedError:
+ pass
+
+ if should_sync:
+ copy_callback = partial(callback, src_file_path) if callback else None
+ smart_copy(
+ src_file_path,
+ dst_abs_file_path,
+ callback=copy_callback,
+ followlinks=followlinks)
+ if callback_after_copy_file:
+ callback_after_copy_file(src_file_path, dst_abs_file_path)
+ return should_sync
+
+
+[docs]def smart_sync(
+ src_path: PathLike,
+ dst_path: PathLike,
+ callback: Optional[Callable[[str, int], None]] = None,
+ followlinks: bool = False,
+ callback_after_copy_file: Optional[Callable[[str, str], None]] = None,
+ src_file_stats: Optional[Iterable[FileEntry]] = None,
+ map_func: Callable[[Callable, Iterable], Any] = map,
+ force: bool = False,
+ overwrite: bool = True) -> None:
+ '''
+ Sync file or directory
+
+ .. note ::
+
+ When the parameter is file, this function bahaves like ``smart_copy``.
+
+ If file and directory of same name and same level, sync consider it's file first.
+
+ Here are a few examples: ::
+
+ >>> from tqdm import tqdm
+ >>> from threading import Lock
+ >>> from megfile import smart_sync, smart_stat, smart_glob
+ >>> class Bar:
+ ... def __init__(self, total_file):
+ ... self._total_file = total_file
+ ... self._bar = None
+ ... self._now = None
+ ... self._file_index = 0
+ ... self._lock = Lock()
+ ... def __call__(self, path, num_bytes):
+ ... with self._lock:
+ ... if path != self._now:
+ ... self._file_index += 1
+ ... print("copy file {}/{}:".format(self._file_index, self._total_file))
+ ... if self._bar:
+ ... self._bar.close()
+ ... self._bar = tqdm(total=smart_stat(path).size)
+ ... self._now = path
+ ... self._bar.update(num_bytes)
+ >>> total_file = len(list(smart_glob('src_path')))
+ >>> smart_sync('src_path', 'dst_path', callback=Bar(total_file=total_file))
+
+ :param src_path: Given source path
+ :param dst_path: Given destination path
+ :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+ :param followlinks: False if regard symlink as file, else True
+ :param callback_after_copy_file: Called after copy success, and the input parameter is src file path and dst file path
+ :param src_file_stats: If this parameter is not None, only this parameter's files will be synced,
+ and src_path is the root_path of these files used to calculate the path of the target file.
+ This parameter is in order to reduce file traversal times.
+ :param map_func: A Callable func like `map`. You can use ThreadPoolExecutor.map, Pool.map and so on if you need concurrent capability.
+ default is standard library `map`.
+ :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ if not smart_exists(src_path):
+ raise FileNotFoundError(f'No match file: {src_path}')
+
+ src_path, dst_path = get_traditional_path(src_path), get_traditional_path(
+ dst_path)
+ if not src_file_stats:
+ src_file_stats = smart_scan_stat(src_path, followlinks=followlinks)
+
+ def create_generator():
+ for src_file_entry in src_file_stats:
+ if src_file_entry.name:
+ src_file_path = src_file_entry.path
+ yield dict(
+ src_root_path=src_path,
+ dst_root_path=dst_path,
+ src_file_path=src_file_path,
+ callback=callback,
+ followlinks=followlinks,
+ callback_after_copy_file=callback_after_copy_file,
+ force=force,
+ overwrite=overwrite,
+ )
+
+ for _ in map_func(_smart_sync_single_file, create_generator()):
+ pass
+
+
+[docs]def smart_sync_with_progress(
+ src_path,
+ dst_path,
+ callback: Optional[Callable[[str, int], None]] = None,
+ followlinks: bool = False,
+ map_func: Callable[[Callable, Iterable], Iterator] = map,
+ force: bool = False,
+ overwrite: bool = True):
+ '''
+ Sync file or directory with progress bar
+
+ :param src_path: Given source path
+ :param dst_path: Given destination path
+ :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+ :param followlinks: False if regard symlink as file, else True
+ :param callback_after_copy_file: Called after copy success, and the input parameter is src file path and dst file path
+ :param src_file_stats: If this parameter is not None, only this parameter's files will be synced,
+ and src_path is the root_path of these files used to calculate the path of the target file.
+ This parameter is in order to reduce file traversal times.
+ :param map_func: A Callable func like `map`. You can use ThreadPoolExecutor.map, Pool.map and so on if you need concurrent capability.
+ default is standard library `map`.
+ :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+ :param overwrite: whether or not overwrite file when exists, default is True
+ '''
+ if not smart_exists(src_path):
+ raise FileNotFoundError(f'No match file: {src_path}')
+
+ src_path, dst_path = get_traditional_path(src_path), get_traditional_path(
+ dst_path)
+ file_stats = list(smart_scan_stat(src_path, followlinks=followlinks))
+ tbar = tqdm(total=len(file_stats), ascii=True)
+ sbar = tqdm(unit='B', ascii=True, unit_scale=True, unit_divisor=1024)
+
+ def tqdm_callback(current_src_path, length: int):
+ sbar.update(length)
+ if callback:
+ callback(current_src_path, length)
+
+ def callback_after_copy_file(src_file_path, dst_file_path):
+ tbar.update(1)
+
+ smart_sync(
+ src_path,
+ dst_path,
+ callback=tqdm_callback,
+ followlinks=followlinks,
+ callback_after_copy_file=callback_after_copy_file,
+ src_file_stats=file_stats,
+ map_func=map_func,
+ force=force,
+ overwrite=overwrite,
+ )
+ tbar.close()
+ sbar.close()
+
+
+[docs]def smart_remove(path: PathLike, missing_ok: bool = False) -> None:
+ '''
+ Remove the file or directory on s3 or fs, `s3://` and `s3://bucket` are not permitted to remove
+
+ :param path: Given path
+ :param missing_ok: if False and target file/directory not exists, raise FileNotFoundError
+ :raises: PermissionError, FileNotFoundError
+ '''
+ SmartPath(path).remove(missing_ok=missing_ok)
+
+
+[docs]def smart_rename(
+ src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
+ '''
+ Move file on s3 or fs. `s3://` or `s3://bucket` is not allowed to move
+
+ :param src_path: Given source path
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ if smart_isdir(src_path):
+ raise IsADirectoryError('%r is a directory' % src_path)
+ src_protocol, _ = SmartPath._extract_protocol(src_path)
+ dst_protocol, _ = SmartPath._extract_protocol(dst_path)
+ if src_protocol == dst_protocol:
+ SmartPath(src_path).rename(dst_path, overwrite=overwrite)
+ return
+ smart_copy(src_path, dst_path, overwrite=overwrite)
+ smart_unlink(src_path)
+
+
+[docs]def smart_move(
+ src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
+ '''
+ Move file/directory on s3 or fs. `s3://` or `s3://bucket` is not allowed to move
+
+ :param src_path: Given source path
+ :param dst_path: Given destination path
+ :param overwrite: whether or not overwrite file when exists
+ '''
+ src_protocol, _ = SmartPath._extract_protocol(src_path)
+ dst_protocol, _ = SmartPath._extract_protocol(dst_path)
+ if src_protocol == dst_protocol:
+ SmartPath(src_path).rename(dst_path, overwrite=overwrite)
+ return
+ smart_sync(src_path, dst_path, followlinks=True, overwrite=overwrite)
+ smart_remove(src_path)
+
+
+[docs]def smart_unlink(path: PathLike, missing_ok: bool = False) -> None:
+ '''
+ Remove the file on s3 or fs
+
+ :param path: Given path
+ :param missing_ok: if False and target file not exists, raise FileNotFoundError
+ :raises: PermissionError, FileNotFoundError, IsADirectoryError
+ '''
+ SmartPath(path).unlink(missing_ok=missing_ok)
+
+
+[docs]def smart_makedirs(path: PathLike, exist_ok: bool = False) -> None:
+ '''
+ Create a directory if is on fs.
+ If on s3, it actually check if target exists, and check if bucket has WRITE access
+
+ :param path: Given path
+ :param missing_ok: if False and target directory not exists, raise FileNotFoundError
+ :raises: PermissionError, FileExistsError
+ '''
+ SmartPath(path).makedirs(exist_ok)
+
+
+[docs]def smart_open(
+ path: PathLike,
+ mode: str = 'r',
+ s3_open_func: Callable[[str, str], BinaryIO] = s3_open,
+ encoding: Optional[str] = None,
+ errors: Optional[str] = None,
+ **options) -> IO[AnyStr]: # pytype: disable=signature-mismatch
+ '''
+ Open a file on the path
+
+ .. note ::
+
+ On fs, the difference between this function and ``io.open`` is that this function create directories automatically, instead of raising FileNotFoundError
+
+ Currently, supported protocols are:
+
+ 1. s3: "s3://<bucket>/<key>"
+
+ 2. http(s): http(s) url
+
+ 3. stdio: "stdio://-"
+
+ 4. FS file: Besides above mentioned protocols, other path are considered fs path
+
+ Here are a few examples: ::
+
+ >>> import cv2
+ >>> import numpy as np
+ >>> raw = smart_open('https://ss2.bdstatic.com/70cFvnSh_Q1YnxGkpoWK1HF6hhy/it/u=2275743969,3715493841&fm=26&gp=0.jpg').read()
+ >>> img = cv2.imdecode(np.frombuffer(raw, np.uint8), cv2.IMREAD_ANYDEPTH | cv2.IMREAD_COLOR)
+
+ :param path: Given path
+ :param mode: Mode to open file, supports r'[rwa][tb]?\+?'
+ :param s3_open_func: Function used to open s3_url. Require the function includes 2 necessary parameters, file path and mode
+ :param encoding: encoding is the name of the encoding used to decode or encode the file. This should only be used in text mode.
+ :param errors: errors is an optional string that specifies how encoding and decoding errors are to be handled—this cannot be used in binary mode.
+ :returns: File-Like object
+ :raises: FileNotFoundError, IsADirectoryError, ValueError
+ '''
+ options = {
+ 's3_open_func': s3_open_func,
+ 'encoding': encoding,
+ 'errors': errors,
+ **options,
+ }
+ return SmartPath(path).open(mode, **options)
+
+
+[docs]def smart_path_join(path: PathLike, *other_paths: PathLike) -> str:
+ '''
+ Concat 2 or more path to a complete path
+
+ :param path: Given path
+ :param other_paths: Paths to be concatenated
+ :returns: Concatenated complete path
+
+ .. note ::
+
+ For URI, the difference between this function and ``os.path.join`` is that this function ignores left side slash (which indicates absolute path) in ``other_paths`` and will directly concat.
+ e.g. os.path.join('s3://path', 'to', '/file') => '/file', and smart_path_join('s3://path', 'to', '/file') => '/path/to/file'
+ But for fs path, this function behaves exactly like ``os.path.join``
+ e.g. smart_path_join('/path', 'to', '/file') => '/file'
+ '''
+ return fspath(SmartPath(path).joinpath(*other_paths))
+
+
+[docs]def smart_walk(path: PathLike, followlinks: bool = False
+ ) -> Iterator[Tuple[str, List[str], List[str]]]:
+ '''
+ Generate the file names in a directory tree by walking the tree top-down.
+ For each directory in the tree rooted at directory path (including path itself),
+ it yields a 3-tuple (root, dirs, files).
+
+ root: a string of current path
+ dirs: name list of subdirectories (excluding '.' and '..' if they exist) in 'root'. The list is sorted by ascending alphabetical order
+ files: name list of non-directory files (link is regarded as file) in 'root'. The list is sorted by ascending alphabetical order
+
+ If path not exists, return an empty generator
+ If path is a file, return an empty generator
+ If try to apply walk() on unsupported path, raise UnsupportedError
+
+ :param path: Given path
+ :raises: UnsupportedError
+ :returns: A 3-tuple generator
+ '''
+ return SmartPath(path).walk(followlinks=followlinks)
+
+
+[docs]def smart_scan(
+ path: PathLike, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[str]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a path string.
+
+ If path is a file path, yields the file only
+ If path is a non-existent path, return an empty generator
+ If path is a bucket path, return all file paths in the bucket
+
+ :param path: Given path
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :raises: UnsupportedError
+ :returns: A file path generator
+ '''
+ return SmartPath(path).scan(missing_ok=missing_ok, followlinks=followlinks)
+
+
+[docs]def smart_scan_stat(
+ path: PathLike, missing_ok: bool = True,
+ followlinks: bool = False) -> Iterator[FileEntry]:
+ '''
+ Iteratively traverse only files in given directory, in alphabetical order.
+ Every iteration on generator yields a tuple of path string and file stat
+
+ :param path: Given path
+ :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+ :raises: UnsupportedError
+ :returns: A file path generator
+ '''
+ return SmartPath(path).scan_stat(
+ missing_ok=missing_ok, followlinks=followlinks)
+
+
+def _group_glob(globstr: str) -> List[str]:
+ '''
+ Split pathname, and group them by protocol, return the glob list of same group.
+
+ :param globstr: A glob string
+ :returns: A glob list after being grouped by protocol
+ '''
+ glob_dict = defaultdict(list)
+ expanded_glob = ungloblize(globstr)
+
+ for single_glob in expanded_glob:
+ protocol, _ = SmartPath._extract_protocol(single_glob)
+ glob_dict[protocol].append(single_glob)
+
+ group_glob_list = []
+
+ for protocol, glob_list in glob_dict.items():
+ group_glob_list.append(globlize(glob_list))
+ return group_glob_list
+
+
+[docs]def smart_glob(
+ pathname: PathLike, recursive: bool = True,
+ missing_ok: bool = True) -> List[str]:
+ '''
+ Given pathname may contain shell wildcard characters, return path list in ascending alphabetical order, in which path matches glob pattern
+
+ :param pathname: A path pattern may contain shell wildcard characters
+ :param recursive: If False, this function will not glob recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ '''
+ # Split pathname, group by protocol, call glob respectively
+ # SmartPath(pathname).glob(recursive, missing_ok)
+ result = []
+ group_glob_list = _group_glob(pathname)
+ for glob_path in group_glob_list:
+ for path_obj in SmartPath(glob_path).glob(pattern='',
+ recursive=recursive,
+ missing_ok=missing_ok):
+ result.append(path_obj.path)
+ return result
+
+
+[docs]def smart_iglob(
+ pathname: PathLike, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator[str]:
+ '''
+ Given pathname may contain shell wildcard characters, return path iterator in ascending alphabetical order, in which path matches glob pattern
+
+ :param pathname: A path pattern may contain shell wildcard characters
+ :param recursive: If False, this function will not glob recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ '''
+ # Split pathname, group by protocol, call glob respectively
+ # SmartPath(pathname).glob(recursive, missing_ok)
+ group_glob_list = _group_glob(pathname)
+ for glob_path in group_glob_list:
+ for path_obj in SmartPath(glob_path).iglob(pattern='',
+ recursive=recursive,
+ missing_ok=missing_ok):
+ yield path_obj.path
+
+
+[docs]def smart_glob_stat(
+ pathname: PathLike, recursive: bool = True,
+ missing_ok: bool = True) -> Iterator[FileEntry]:
+ '''
+ Given pathname may contain shell wildcard characters, return a list contains tuples of path and file stat in ascending alphabetical order, in which path matches glob pattern
+
+ :param pathname: A path pattern may contain shell wildcard characters
+ :param recursive: If False, this function will not glob recursively
+ :param missing_ok: If False and target path doesn't match any file, raise FileNotFoundError
+ '''
+ # Split pathname, group by protocol, call glob respectively
+ # SmartPath(pathname).glob(recursive, missing_ok)
+ group_glob_list = _group_glob(pathname)
+ for glob_path in group_glob_list:
+ yield from SmartPath(glob_path).glob_stat(
+ pattern='', recursive=recursive, missing_ok=missing_ok)
+
+
+[docs]def smart_save_as(file_object: BinaryIO, path: PathLike) -> None:
+ '''Write the opened binary stream to specified path, but the stream won't be closed
+
+ :param file_object: Stream to be read
+ :param path: Specified target path
+ '''
+ SmartPath(path).save(file_object)
+
+
+[docs]def smart_load_from(path: PathLike) -> BinaryIO:
+ '''Read all content in binary on specified path and write into memory
+
+ User should close the BinaryIO manually
+
+ :param path: Specified path
+ :returns: BinaryIO
+ '''
+ return SmartPath(path).load()
+
+
+[docs]def smart_combine_open(
+ path_glob: str, mode: str = 'rb',
+ open_func=smart_open) -> CombineReader:
+ '''Open a unified reader that supports multi file reading.
+
+ :param path_glob: A path may contain shell wildcard characters
+ :param mode: Mode to open file, supports 'rb'
+ :returns: A ```CombineReader```
+ '''
+ file_objects = list(
+ open_func(path, mode) for path in sorted(smart_glob(path_glob)))
+ return combine(file_objects, path_glob)
+
+
+[docs]def smart_abspath(path: PathLike):
+ '''Return the absolute path of given path
+
+ :param path: Given path
+ :returns: Absolute path of given path
+ '''
+ return SmartPath(path).abspath()
+
+
+[docs]def smart_realpath(path: PathLike):
+ '''Return the real path of given path
+
+ :param path: Given path
+ :returns: Real path of given path
+ '''
+ return SmartPath(path).realpath()
+
+
+[docs]def smart_relpath(path: PathLike, start=None):
+ '''Return the relative path of given path
+
+ :param path: Given path
+ :param start: Given start directory
+ :returns: Relative path from start
+ '''
+ return SmartPath(path).relpath(start)
+
+
+[docs]def smart_isabs(path: PathLike) -> bool:
+ '''Test whether a path is absolute
+
+ :param path: Given path
+ :returns: True if a path is absolute, else False
+ '''
+ return SmartPath(path).is_absolute()
+
+
+[docs]def smart_ismount(path: PathLike) -> bool:
+ '''Test whether a path is a mount point
+
+ :param path: Given path
+ :returns: True if a path is a mount point, else False
+ '''
+ return SmartPath(path).is_mount()
+
+
+[docs]def smart_load_content(
+ path: PathLike, start: Optional[int] = None,
+ stop: Optional[int] = None) -> bytes:
+ '''
+ Get specified file from [start, stop) in bytes
+
+ :param path: Specified path
+ :param start: start index
+ :param stop: stop index
+ :returns: bytes content in range [start, stop)
+ '''
+ if is_s3(path):
+ return s3_load_content(path, start, stop)
+
+ with smart_open(path, 'rb') as fd:
+ if start:
+ fd.seek(start)
+ offset = -1
+ if start and stop:
+ offset = stop - start
+ return fd.read(offset)
+
+
+[docs]def smart_save_content(path: PathLike, content: bytes) -> None:
+ '''Save bytes content to specified path
+
+ param path: Path to save content
+ '''
+ with smart_open(path, 'wb') as fd:
+ fd.write(content)
+
+
+[docs]def smart_load_text(path: PathLike) -> str:
+ '''
+ Read content from path
+
+ param path: Path to be read
+ '''
+ with smart_open(path) as fd:
+ return fd.read()
+
+
+[docs]def smart_save_text(path: PathLike, text: str) -> None:
+ '''Save text to specified path
+
+ param path: Path to save text
+ '''
+ with smart_open(path, 'w') as fd:
+ fd.write(text)
+
+
+[docs]class SmartCacher(FileCacher):
+ cache_path = None
+
+ def __init__(
+ self, path: str, cache_path: Optional[str] = None, mode: str = 'r'):
+ if mode not in ('r', 'w', 'a'):
+ raise ValueError('unacceptable mode: %r' % mode)
+ if mode in ('r', 'a'):
+ if cache_path is None:
+ cache_path = generate_cache_path(path)
+ smart_copy(path, cache_path)
+ self.name = path
+ self.mode = mode
+ self.cache_path = cache_path
+
+ def _close(self):
+ if self.cache_path is not None and \
+ os.path.exists(self.cache_path):
+ if self.mode in ('w', 'a'):
+ smart_copy(self.cache_path, self.name)
+ os.unlink(self.cache_path)
+
+
+[docs]def smart_cache(path, cacher=SmartCacher, **options):
+ '''Return a path to Posixpath Interface
+
+ param path: Path to cache
+ param s3_cacher: Cacher for s3 path
+ param options: Optional arguments for s3_cacher
+ '''
+ if not is_fs(path):
+ return cacher(path, **options)
+ return NullCacher(path)
+
+
+[docs]def smart_touch(path: PathLike):
+ '''Create a new file on path
+
+ param path: Path to create file
+ '''
+ with smart_open(path, 'w'):
+ pass
+
+
+[docs]def smart_getmd5(path: PathLike, recalculate: bool = False):
+ '''Get md5 value of file
+
+ param path: File path
+ param recalculate: calculate md5 in real-time or not return s3 etag when path is s3
+ '''
+ return SmartPath(path).md5(recalculate=recalculate)
+
+
+_concat_funcs = {
+ 's3': s3_concat,
+ 'sftp': sftp_concat,
+}
+
+
+def _default_concat_func(src_paths: List[PathLike], dst_path: PathLike) -> None:
+ length = 16 * 1024
+ with smart_open(dst_path, 'wb') as dst_fd:
+ for src_path in src_paths:
+ with smart_open(src_path, 'rb') as src_fd:
+ while True:
+ buf = src_fd.read(length)
+ if not buf:
+ break
+ dst_fd.write(buf)
+
+
+[docs]def smart_concat(src_paths: List[PathLike], dst_path: PathLike) -> None:
+ '''
+ Concatenate src_paths to dst_path
+
+ :param src_paths: List of source paths
+ :param dst_path: Destination path
+ '''
+ if not src_paths:
+ return
+
+ dst_protocol, _ = SmartPath._extract_protocol(dst_path)
+ for src_path in src_paths:
+ src_protocol, _ = SmartPath._extract_protocol(src_path)
+ if src_protocol != dst_protocol:
+ concat_func = _default_concat_func
+ break
+ else:
+ concat_func = _concat_funcs.get(dst_protocol, _default_concat_func)
+ concat_func(src_paths, dst_path)
+
+from pathlib import PurePath
+from typing import Tuple, Union
+
+from megfile.lib.compat import fspath
+from megfile.lib.url import get_url_scheme
+
+from .errors import ProtocolExistsError, ProtocolNotFoundError
+from .interfaces import BasePath, BaseURIPath, PathLike
+
+
+def _bind_function(name):
+
+ def smart_method(self, *args, **kwargs):
+ return getattr(self.pathlike, name)(*args, **kwargs)
+
+ smart_method.__name__ = name
+
+ return smart_method
+
+
+def _bind_property(name):
+
+ @property
+ def smart_property(self):
+ return getattr(self.pathlike, name)
+
+ return smart_property
+
+
+[docs]class SmartPath(BasePath):
+ _registered_protocols = dict()
+
+ def __init__(self, path: Union[PathLike, int], *other_paths: PathLike):
+ self.path = str(path) if not isinstance(path, int) else path
+ pathlike = path
+ if not isinstance(pathlike, BaseURIPath):
+ pathlike = self._create_pathlike(path)
+ if len(other_paths) > 0:
+ pathlike = pathlike.joinpath(*other_paths)
+ self.path = str(pathlike)
+ self.pathlike = pathlike
+
+ @staticmethod
+ def _extract_protocol(path: Union[PathLike, int]
+ ) -> Tuple[str, Union[str, int]]:
+ if isinstance(path, int):
+ protocol = "file"
+ path_without_protocol = path
+ elif isinstance(path, str):
+ protocol = get_url_scheme(path)
+ if protocol == "":
+ protocol = "file"
+ path_without_protocol = path
+ else:
+ path_without_protocol = path[len(protocol) + 3:]
+ elif isinstance(path, (BaseURIPath, SmartPath)):
+ protocol = path.protocol
+ path_without_protocol = str(path)
+ elif isinstance(path, (PurePath, BasePath)):
+ protocol, path_without_protocol = SmartPath._extract_protocol(
+ fspath(path))
+ else:
+ raise ProtocolNotFoundError('protocol not found: %r' % path)
+ return protocol, path_without_protocol
+
+ @classmethod
+ def _create_pathlike(cls, path: Union[PathLike, int]) -> BasePath:
+ protocol, _ = cls._extract_protocol(path)
+ if protocol.startswith('s3+'):
+ protocol = 's3'
+ if protocol not in cls._registered_protocols:
+ raise ProtocolNotFoundError(
+ 'protocol %r not found: %r' % (protocol, path))
+ path_class = cls._registered_protocols[protocol]
+ return path_class(path)
+
+[docs] @classmethod
+ def register(cls, path_class, override_ok: bool = False):
+ protocol = path_class.protocol
+ if protocol in cls._registered_protocols and not override_ok:
+ raise ProtocolExistsError('protocol already exists: %r' % protocol)
+ cls._registered_protocols[protocol] = path_class
+ return path_class
+
+ symlink = _bind_function('symlink')
+ symlink_to = _bind_function('symlink_to')
+ hardlink_to = _bind_function('hardlink_to')
+ readlink = _bind_function('readlink')
+ is_dir = _bind_function('is_dir')
+ is_file = _bind_function('is_file')
+ is_symlink = _bind_function('is_symlink')
+ access = _bind_function('access')
+ exists = _bind_function('exists')
+ listdir = _bind_function('listdir')
+ scandir = _bind_function('scandir')
+ getsize = _bind_function('getsize')
+ getmtime = _bind_function('getmtime')
+ stat = _bind_function('stat')
+ lstat = _bind_function('lstat')
+ remove = _bind_function('remove')
+ rename = _bind_function('rename')
+ replace = _bind_function('replace')
+ unlink = _bind_function('unlink')
+ mkdir = _bind_function('mkdir')
+ open = _bind_function('open')
+ touch = _bind_function('touch')
+ walk = _bind_function('walk')
+ scan = _bind_function('scan')
+ scan_stat = _bind_function('scan_stat')
+ glob = _bind_function('glob')
+ iglob = _bind_function('iglob')
+ glob_stat = _bind_function('glob_stat')
+ load = _bind_function('load')
+ save = _bind_function('save')
+ joinpath = _bind_function('joinpath')
+ abspath = _bind_function('abspath')
+ realpath = _bind_function('realpath')
+ relpath = _bind_function('relpath')
+ is_absolute = _bind_function('is_absolute')
+ is_mount = _bind_function('is_mount')
+ md5 = _bind_function('md5')
+
+ @property
+ def protocol(self) -> str:
+ return self.pathlike.protocol # pytype: disable=attribute-error
+
+
+
+ as_uri = _bind_function('as_uri')
+ as_posix = _bind_function('as_posix')
+ __lt__ = _bind_function('__lt__')
+ __le__ = _bind_function('__le__')
+ __gt__ = _bind_function('__gt__')
+ __ge__ = _bind_function('__ge__')
+ __fspath__ = _bind_function('__fspath__')
+ __truediv__ = _bind_function('__truediv__')
+
+ joinpath = _bind_function('joinpath')
+ is_reserved = _bind_function('is_reserved')
+ match = _bind_function('match')
+ relative_to = _bind_function('relative_to')
+ with_name = _bind_function('with_name')
+ with_suffix = _bind_function('with_suffix')
+ with_stem = _bind_function('with_stem')
+ is_absolute = _bind_function('is_absolute')
+ is_mount = _bind_function('is_mount')
+ abspath = _bind_function('abspath')
+ realpath = _bind_function('realpath')
+ relpath = _bind_function('relpath')
+ iterdir = _bind_function('iterdir')
+ cwd = _bind_function('cwd')
+ home = _bind_function('home')
+ expanduser = _bind_function('expanduser')
+ resolve = _bind_function('resolve')
+ chmod = _bind_function('chmod')
+ lchmod = _bind_function('lchmod')
+ group = _bind_function('group')
+ is_socket = _bind_function('is_socket')
+ is_fifo = _bind_function('is_fifo')
+ is_block_device = _bind_function('is_block_device')
+ is_char_device = _bind_function('is_char_device')
+ owner = _bind_function('owner')
+ absolute = _bind_function('absolute')
+ rmdir = _bind_function('rmdir')
+ is_relative_to = _bind_function('is_relative_to')
+ read_bytes = _bind_function('read_bytes')
+ read_text = _bind_function('read_text')
+ rglob = _bind_function('rglob')
+ samefile = _bind_function('samefile')
+ write_bytes = _bind_function('write_bytes')
+ write_text = _bind_function('write_text')
+ utime = _bind_function('utime')
+
+ drive = _bind_property('drive')
+ root = _bind_property('root')
+ anchor = _bind_property('anchor')
+ parts = _bind_property('parts')
+ parents = _bind_property('parents')
+ parent = _bind_property('parent')
+ name = _bind_property('name')
+ suffix = _bind_property('suffix')
+ suffixes = _bind_property('suffixes')
+ stem = _bind_property('stem')
+
+
+
+
+from typing import IO, AnyStr, Optional
+
+from megfile.interfaces import PathLike
+from megfile.stdio_path import StdioPath, is_stdio
+
+__all__ = [
+ 'is_stdio',
+ 'stdio_open',
+]
+
+
+[docs]def stdio_open(
+ path: PathLike,
+ mode: str = 'rb',
+ encoding: Optional[str] = None,
+ errors: Optional[str] = None,
+ **kwargs) -> IO[AnyStr]: # pytype: disable=signature-mismatch
+ '''Used to read or write stdio
+
+ .. note ::
+
+ Essentially invoke sys.stdin.buffer | sys.stdout.buffer to read or write
+
+ :param path: Given path
+ :param mode: Only supports 'rb' and 'wb' now
+ :return: STDReader, STDWriter
+ '''
+ return StdioPath(path).open(mode, encoding, errors)
+
+import io
+from typing import IO, AnyStr, Optional, Union
+
+from megfile.interfaces import BaseURIPath, PathLike
+from megfile.lib.compat import fspath
+from megfile.lib.stdio_handler import STDReader, STDWriter
+from megfile.lib.url import get_url_scheme
+from megfile.smart_path import SmartPath
+from megfile.utils import get_binary_mode
+
+__all__ = [
+ "StdioPath",
+ "is_stdio",
+]
+
+
+[docs]def is_stdio(path: PathLike) -> bool:
+ '''stdio scheme definition: stdio://-
+
+ .. note ::
+
+ Only tests protocol
+
+ :param path: Path to be tested
+ :returns: True of a path is stdio url, else False
+ '''
+
+ path = fspath(path)
+ if not isinstance(path, str) or not path.startswith('stdio://'):
+ return False
+
+ scheme = get_url_scheme(path)
+ return scheme == 'stdio'
+
+
+[docs]@SmartPath.register
+class StdioPath(BaseURIPath):
+
+ protocol = "stdio"
+
+ def _open(self, mode: str = 'rb') -> Union[STDReader, STDWriter]:
+ '''Used to read or write stdio
+
+ .. note ::
+
+ Essentially invoke sys.stdin.buffer | sys.stdout.buffer to read or write
+
+ :param path: stdio path, stdio://- or stdio://0 stdio://1 stdio://2
+ :param mode: Only supports 'rb' and 'wb' now
+ :return: STDReader, STDWriter
+ '''
+
+ if mode not in ('rb', 'wb', 'rt', 'wt', 'r', 'w'):
+ raise ValueError('unacceptable mode: %r' % mode)
+
+ mode = get_binary_mode(mode)
+
+ if self.path_with_protocol not in ('stdio://-', 'stdio://0',
+ 'stdio://1', 'stdio://2'):
+ raise ValueError('unacceptable path: %r' % self.path_with_protocol)
+
+ if self.path_with_protocol in ('stdio://1',
+ 'stdio://2') and 'r' in mode:
+ raise ValueError(
+ 'cannot open for reading: %r' % self.path_with_protocol)
+
+ if self.path_with_protocol == 'stdio://0' and 'w' in mode:
+ raise ValueError(
+ 'cannot open for writing: %r' % self.path_with_protocol)
+
+ if 'r' in mode:
+ return STDReader(mode)
+ return STDWriter(self.path_with_protocol, mode)
+
+[docs] def open(
+ self,
+ mode: str = 'rb',
+ encoding: Optional[str] = None,
+ errors: Optional[str] = None,
+ **kwargs) -> IO[AnyStr]: # pytype: disable=signature-mismatch
+ '''Used to read or write stdio
+
+ .. note ::
+
+ Essentially invoke sys.stdin.buffer | sys.stdout.buffer to read or write
+
+ :param mode: Only supports 'rb' and 'wb' now
+ :return: STDReader, STDWriter
+ '''
+ fileobj = self._open(mode)
+
+ if 'b' not in mode:
+ fileobj = io.TextIOWrapper(
+ fileobj, encoding=encoding, errors=errors) # type: ignore
+ fileobj.mode = mode
+
+ return fileobj # type: ignore
+' + + '' + + Documentation.gettext("Hide Search Matches") + + "
" + ) + ); + }, + + /** + * helper function to hide the search marks again + */ + hideSearchWords: () => { + document + .querySelectorAll("#searchbox .highlight-link") + .forEach((el) => el.remove()); + document + .querySelectorAll("span.highlighted") + .forEach((el) => el.classList.remove("highlighted")); + const url = new URL(window.location); + url.searchParams.delete("highlight"); + window.history.replaceState({}, "", url); + }, + /** * helper function to focus on search bar */ @@ -110,11 +210,15 @@ const Documentation = { ) return; + const blacklistedElements = new Set([ + "TEXTAREA", + "INPUT", + "SELECT", + "BUTTON", + ]); document.addEventListener("keydown", (event) => { - // bail for input elements - if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return; - // bail with special keys - if (event.altKey || event.ctrlKey || event.metaKey) return; + if (blacklistedElements.has(document.activeElement.tagName)) return; // bail for input elements + if (event.altKey || event.ctrlKey || event.metaKey) return; // bail with special keys if (!event.shiftKey) { switch (event.key) { @@ -136,6 +240,10 @@ const Documentation = { event.preventDefault(); } break; + case "Escape": + if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) break; + Documentation.hideSearchWords(); + event.preventDefault(); } } diff --git a/_static/documentation_options.js b/_static/documentation_options.js index 7e4c114f..a750e4d5 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,4 +1,5 @@ -const DOCUMENTATION_OPTIONS = { +var DOCUMENTATION_OPTIONS = { + URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), VERSION: '', LANGUAGE: 'en', COLLAPSE_INDEX: false, @@ -9,5 +10,5 @@ const DOCUMENTATION_OPTIONS = { SOURCELINK_SUFFIX: '.txt', NAVIGATION_WITH_KEYS: false, SHOW_SEARCH_SUMMARY: true, - ENABLE_SEARCH_SHORTCUTS: true, + ENABLE_SEARCH_SHORTCUTS: false, }; \ No newline at end of file diff --git a/_static/jquery-3.6.0.js b/_static/jquery-3.6.0.js new file mode 100644 index 00000000..fc6c299b --- /dev/null +++ b/_static/jquery-3.6.0.js @@ -0,0 +1,10881 @@ +/*! + * jQuery JavaScript Library v3.6.0 + * https://jquery.com/ + * + * Includes Sizzle.js + * https://sizzlejs.com/ + * + * Copyright OpenJS Foundation and other contributors + * Released under the MIT license + * https://jquery.org/license + * + * Date: 2021-03-02T17:08Z + */ +( function( global, factory ) { + + "use strict"; + + if ( typeof module === "object" && typeof module.exports === "object" ) { + + // For CommonJS and CommonJS-like environments where a proper `window` + // is present, execute the factory and get jQuery. + // For environments that do not have a `window` with a `document` + // (such as Node.js), expose a factory as module.exports. + // This accentuates the need for the creation of a real `window`. + // e.g. var jQuery = require("jquery")(window); + // See ticket #14549 for more info. + module.exports = global.document ? + factory( global, true ) : + function( w ) { + if ( !w.document ) { + throw new Error( "jQuery requires a window with a document" ); + } + return factory( w ); + }; + } else { + factory( global ); + } + +// Pass this if window is not defined yet +} )( typeof window !== "undefined" ? window : this, function( window, noGlobal ) { + +// Edge <= 12 - 13+, Firefox <=18 - 45+, IE 10 - 11, Safari 5.1 - 9+, iOS 6 - 9.1 +// throw exceptions when non-strict code (e.g., ASP.NET 4.5) accesses strict mode +// arguments.callee.caller (trac-13335). But as of jQuery 3.0 (2016), strict mode should be common +// enough that all such attempts are guarded in a try block. +"use strict"; + +var arr = []; + +var getProto = Object.getPrototypeOf; + +var slice = arr.slice; + +var flat = arr.flat ? function( array ) { + return arr.flat.call( array ); +} : function( array ) { + return arr.concat.apply( [], array ); +}; + + +var push = arr.push; + +var indexOf = arr.indexOf; + +var class2type = {}; + +var toString = class2type.toString; + +var hasOwn = class2type.hasOwnProperty; + +var fnToString = hasOwn.toString; + +var ObjectFunctionString = fnToString.call( Object ); + +var support = {}; + +var isFunction = function isFunction( obj ) { + + // Support: Chrome <=57, Firefox <=52 + // In some browsers, typeof returns "function" for HTML