From 1137ef312eef1fb0555522121b95604160eba2e4 Mon Sep 17 00:00:00 2001 From: lewzylu <327874225@qq.com> Date: Tue, 3 Apr 2018 10:28:33 +0800 Subject: [PATCH] Fix copy -r (#118) * 1.fix copy 2.add copy folder interface * fix spare print * fix with pep8 rules * fix copy * fix pep8 --- coscmd/cos_client.py | 57 ++++++++++++++++++++++++++++++++++++++++---- coscmd/cos_cmd.py | 27 +++++++++++++-------- 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/coscmd/cos_client.py b/coscmd/cos_client.py index aef70ef..71ce473 100644 --- a/coscmd/cos_client.py +++ b/coscmd/cos_client.py @@ -189,8 +189,9 @@ def list_part(self, cos_path): cos_path = to_printable_str(cos_path) try: while IsTruncated == "true": - url = self._conf.uri(path=cos_path+ - '?uploadId={UploadId}&upload&max-parts=1000&part-number-marker={nextmarker}'.format(UploadId=self._upload_id, nextmarker=NextMarker)) + url = self._conf.uri(path=cos_path + '?uploadId={UploadId}&upload&max-parts=1000&part-number-marker={nextmarker}'.format( + UploadId=self._upload_id, + nextmarker=NextMarker)) rt = self._session.get(url=url, auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key)) if rt.status_code == 200: root = minidom.parseString(rt.content).documentElement @@ -282,7 +283,7 @@ def single_upload(): def init_multiupload(): url = self._conf.uri(path=cos_path) self._md5 = {} - self._have_finished = 0 + self.c = 0 self._have_uploaded = [] self._upload_id = None self._type = _type @@ -458,6 +459,53 @@ def complete_multiupload(): logger.warn("complete multipart upload failed") return False + def copy_folder(self, source_path, cos_path, _type='Standard'): + self._type = _type + source_schema = source_path.split('/')[0] + '/' + source_path = source_path[len(source_schema):] + NextMarker = "" + IsTruncated = "true" + self._file_num = 0 + self._success_num = 0 + self._fail_num = 0 + while IsTruncated == "true": + tmp_url = '?prefix={prefix}&marker={nextmarker}'.format( + prefix=urllib.quote(to_printable_str(source_path)), + nextmarker=urllib.quote(to_printable_str(NextMarker))) + url = self._conf._schema + "://" + source_schema + tmp_url + rt = self._session.get(url=url, auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key)) + if rt.status_code == 200: + root = minidom.parseString(rt.content).documentElement + IsTruncated = root.getElementsByTagName("IsTruncated")[0].childNodes[0].data + if IsTruncated == 'true': + NextMarker = root.getElementsByTagName("NextMarker")[0].childNodes[0].data + fileset = root.getElementsByTagName("Contents") + for _file in fileset: + _tmp = _file.getElementsByTagName("Key")[0].childNodes[0].data + _source_path = source_schema + _tmp + _cos_path = cos_path + _tmp[len(cos_path):] + _cos_path = to_unicode(_cos_path) + _source_path = to_unicode(_source_path) + if _cos_path.endswith('/'): + continue + self._file_num += 1 + if self.copy_file(_source_path, _cos_path, _type): + self._success_num += 1 + else: + self._fail_num += 1 + else: + logger.warn(response_info(rt)) + return False + if self._file_num == 0: + logger.info("The directory does not exist") + return False + logger.info("copy {success_files} files successful, {fail_files} files failed" + .format(success_files=self._success_num, fail_files=self._fail_num)) + if self._file_num == self._success_num: + return True + else: + return False + def copy_file(self, source_path, cos_path, _type='Standard'): def single_copy(): @@ -556,7 +604,7 @@ def copy_parts_data(local_path, offset, length, parts_size, idx): offset = 0 logger.debug("file size: " + str(file_size)) - chunk_size = 1024 * 1024 * 5 + chunk_size = 1024 * 1024 * self._conf._part_size while file_size / chunk_size > 8000: chunk_size = chunk_size * 10 parts_num = file_size / chunk_size @@ -582,6 +630,7 @@ def copy_parts_data(local_path, offset, length, parts_size, idx): else: pool.add_task(copy_parts_data, source_path, offset, chunk_size, parts_num, i+1) offset += chunk_size + pool.wait_completion() result = pool.get_result() self._pbar.close() diff --git a/coscmd/cos_cmd.py b/coscmd/cos_cmd.py index e906acc..ba0a207 100644 --- a/coscmd/cos_cmd.py +++ b/coscmd/cos_cmd.py @@ -125,7 +125,7 @@ def load_conf(): bucket=bucket, part_size=part_size, max_thread=max_thread, - schema = schema + schema=schema ) return conf @@ -188,12 +188,6 @@ def download(args): if not isinstance(args. cos_path, unicode): args.cos_path = args.cos_path.decode(fs_coding) - -# if args.cos_path.endswith('/') is False: -# args.cos_path += '/' -# if args.local_path.endswith('/') is True: -# args.local_path += args.cos_path.split('/')[-2] -# args.cos_path = args.cos_path[:-1] args.cos_path, args.local_path = concat_path(args.cos_path, args.local_path) if args.recursive: rt = Interface.download_folder(args.cos_path, args.local_path, args.force) @@ -252,10 +246,22 @@ def copy(args): args.source_path = args.source_path.decode(fs_coding) if not isinstance(args.cos_path, unicode): args.cos_path = args.cos_path.decode(fs_coding) - if Interface.copy_file(args.source_path, args.cos_path, args.type) is True: - return 0 + if args.recursive: + _, args.cos_path = concat_path(args.source_path, args.cos_path) + if args.cos_path.endswith('/') is False: + args.cos_path += '/' + if args.cos_path == '/': + args.cos_path = '' + + if Interface.copy_folder(args.source_path, args.cos_path, args.type) is True: + return 0 + else: + return 1 else: - return -1 + if Interface.copy_file(args.source_path, args.cos_path, args.type) is True: + return 0 + else: + return -1 @staticmethod def list(args): @@ -479,6 +485,7 @@ def command_thread(): parser_copy = sub_parser.add_parser("copy", help="copy file from COS to COS.") parser_copy.add_argument('source_path', help="source file path as 'bucket-appid.cos.ap-guangzhou.myqcloud.com/a.txt'", type=str) parser_copy.add_argument("cos_path", help="cos_path as a/b.txt", type=str) + parser_copy.add_argument('-r', '--recursive', help="copy files recursively", action="store_true", default=False) parser_copy.add_argument('-t', '--type', help='specify x-cos-storage-class of files to upload', type=str, choices=['STANDARD', 'STANDARD_IA', 'NEARLINE'], default='STANDARD') parser_copy.set_defaults(func=Op.copy)