Fix copy -r (#118)

* 1.fix copy
2.add copy folder interface

* fix spare print

* fix with pep8 rules

* fix copy

* fix pep8

coscmd/cos_client.py

@@ -189,8 +189,9 @@ def list_part(self, cos_path):
         cos_path = to_printable_str(cos_path)
         try:
             while IsTruncated == "true":
-                url = self._conf.uri(path=cos_path+
-                    '?uploadId={UploadId}&upload&max-parts=1000&part-number-marker={nextmarker}'.format(UploadId=self._upload_id, nextmarker=NextMarker))
+                url = self._conf.uri(path=cos_path + '?uploadId={UploadId}&upload&max-parts=1000&part-number-marker={nextmarker}'.format(
+                                                    UploadId=self._upload_id,
+                                                    nextmarker=NextMarker))
                 rt = self._session.get(url=url, auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key))
                 if rt.status_code == 200:
                     root = minidom.parseString(rt.content).documentElement
@@ -282,7 +283,7 @@ def single_upload():
         def init_multiupload():
             url = self._conf.uri(path=cos_path)
             self._md5 = {}
-            self._have_finished = 0
+            self.c = 0
             self._have_uploaded = []
             self._upload_id = None
             self._type = _type
@@ -458,6 +459,53 @@ def complete_multiupload():
             logger.warn("complete multipart upload failed")
             return False
 
+    def copy_folder(self, source_path, cos_path, _type='Standard'):
+        self._type = _type
+        source_schema = source_path.split('/')[0] + '/'
+        source_path = source_path[len(source_schema):]
+        NextMarker = ""
+        IsTruncated = "true"
+        self._file_num = 0
+        self._success_num = 0
+        self._fail_num = 0
+        while IsTruncated == "true":
+            tmp_url = '?prefix={prefix}&marker={nextmarker}'.format(
+                    prefix=urllib.quote(to_printable_str(source_path)),
+                    nextmarker=urllib.quote(to_printable_str(NextMarker)))
+            url = self._conf._schema + "://" + source_schema + tmp_url
+            rt = self._session.get(url=url, auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key))
+            if rt.status_code == 200:
+                root = minidom.parseString(rt.content).documentElement
+                IsTruncated = root.getElementsByTagName("IsTruncated")[0].childNodes[0].data
+                if IsTruncated == 'true':
+                    NextMarker = root.getElementsByTagName("NextMarker")[0].childNodes[0].data
+                fileset = root.getElementsByTagName("Contents")
+                for _file in fileset:
+                    _tmp = _file.getElementsByTagName("Key")[0].childNodes[0].data
+                    _source_path = source_schema + _tmp
+                    _cos_path = cos_path + _tmp[len(cos_path):]
+                    _cos_path = to_unicode(_cos_path)
+                    _source_path = to_unicode(_source_path)
+                    if _cos_path.endswith('/'):
+                        continue
+                    self._file_num += 1
+                    if self.copy_file(_source_path, _cos_path, _type):
+                        self._success_num += 1
+                    else:
+                        self._fail_num += 1
+            else:
+                logger.warn(response_info(rt))
+                return False
+        if self._file_num == 0:
+            logger.info("The directory does not exist")
+            return False
+        logger.info("copy {success_files} files successful, {fail_files} files failed"
+                    .format(success_files=self._success_num, fail_files=self._fail_num))
+        if self._file_num == self._success_num:
+            return True
+        else:
+            return False
+
     def copy_file(self, source_path, cos_path, _type='Standard'):
 
         def single_copy():
@@ -556,7 +604,7 @@ def copy_parts_data(local_path, offset, length, parts_size, idx):
 
             offset = 0
             logger.debug("file size: " + str(file_size))
-            chunk_size = 1024 * 1024 * 5
+            chunk_size = 1024 * 1024 * self._conf._part_size
             while file_size / chunk_size > 8000:
                 chunk_size = chunk_size * 10
             parts_num = file_size / chunk_size
@@ -582,6 +630,7 @@ def copy_parts_data(local_path, offset, length, parts_size, idx):
                     else:
                         pool.add_task(copy_parts_data, source_path, offset, chunk_size, parts_num, i+1)
                         offset += chunk_size
+
             pool.wait_completion()
             result = pool.get_result()
             self._pbar.close()

coscmd/cos_cmd.py

@@ -125,7 +125,7 @@ def load_conf():
             bucket=bucket,
             part_size=part_size,
             max_thread=max_thread,
-            schema = schema
+            schema=schema
         )
         return conf
 
@@ -188,12 +188,6 @@ def download(args):
 
         if not isinstance(args. cos_path, unicode):
             args.cos_path = args.cos_path.decode(fs_coding)
-
-#         if args.cos_path.endswith('/') is False:
-#             args.cos_path += '/'
-#         if args.local_path.endswith('/') is True:
-#             args.local_path += args.cos_path.split('/')[-2]
-#         args.cos_path = args.cos_path[:-1]
         args.cos_path, args.local_path = concat_path(args.cos_path, args.local_path)
         if args.recursive:
             rt = Interface.download_folder(args.cos_path, args.local_path, args.force)
@@ -252,10 +246,22 @@ def copy(args):
             args.source_path = args.source_path.decode(fs_coding)
         if not isinstance(args.cos_path, unicode):
             args.cos_path = args.cos_path.decode(fs_coding)
-        if Interface.copy_file(args.source_path, args.cos_path, args.type) is True:
-            return 0
+        if args.recursive:
+            _, args.cos_path = concat_path(args.source_path, args.cos_path)
+            if args.cos_path.endswith('/') is False:
+                args.cos_path += '/'
+            if args.cos_path == '/':
+                args.cos_path = ''
+
+            if Interface.copy_folder(args.source_path, args.cos_path, args.type) is True:
+                return 0
+            else:
+                return 1
         else:
-            return -1
+            if Interface.copy_file(args.source_path, args.cos_path, args.type) is True:
+                return 0
+            else:
+                return -1
 
     @staticmethod
     def list(args):
@@ -479,6 +485,7 @@ def command_thread():
     parser_copy = sub_parser.add_parser("copy", help="copy file from COS to COS.")
     parser_copy.add_argument('source_path', help="source file path as 'bucket-appid.cos.ap-guangzhou.myqcloud.com/a.txt'", type=str)
     parser_copy.add_argument("cos_path", help="cos_path as a/b.txt", type=str)
+    parser_copy.add_argument('-r', '--recursive', help="copy files recursively", action="store_true", default=False)
     parser_copy.add_argument('-t', '--type', help='specify x-cos-storage-class of files to upload', type=str, choices=['STANDARD', 'STANDARD_IA', 'NEARLINE'], default='STANDARD')
     parser_copy.set_defaults(func=Op.copy)