Add GitLab issue tracker to YouTrack migration script

youtrackutils/csv2youtrack.py

@@ -457,6 +457,8 @@ def to_unix_date(date):
                 date[:-6],
                 csvClient.DATE_FORMAT_STRING[:-2].rstrip())
         else:
+            p = re.compile('(\.\d\d\d)+', re.VERBOSE)
+            date = p.sub(r'', date)
             dt = datetime.datetime.strptime(
                 date, csvClient.DATE_FORMAT_STRING)
         return str(calendar.timegm(dt.timetuple()) * 1000)

youtrackutils/gitlab2youtrack.py

@@ -0,0 +1,295 @@
+#! /usr/bin/env python
+
+# Contributed by yavin5 <[email protected]>
+# Adapted from github2youtrack.py, tested against Gitlab 11.0.x with API v4
+# Currently issue comment attachments are not migrated, but everything else is.
+# 
+# Example command to invoke it:
+#
+#      $ gitlab2youtrack -t perm:avre4asv834.A0sd9f4aSSDFa343t.3sd23ER48cl2FecSAd45Sd7vAE2Z05 -u yavin5 -p pw4yavin5 https://mycompany.myjetbrains.com/youtrack https://git.mycompany.com/api/v4 yavin5 b4iEIjf6RcOaQP26doRx my-group/my-project 2
+import getopt
+import sys
+
+if sys.version_info >= (3, 0):
+    print("\nThe script doesn't support python 3. Please use python 2.7+\n")
+    sys.exit(1)
+
+import os
+import re
+import requests
+import logging
+import csv
+import youtrack
+import youtrackutils.csvClient
+import csv2youtrack
+from youtrack.importHelper import utf8encode
+
+import httplib as http_client
+http_client.HTTPConnection.debuglevel = 1
+logging.basicConfig()
+logging.getLogger().setLevel(logging.DEBUG)
+requests_log = logging.getLogger("requests.packages.urllib3")
+requests_log.setLevel(logging.DEBUG)
+requests_log.propagate = True
+
+youtrackutils.csvClient.FIELD_NAMES = {
+    "Project Name" : "project_name",
+    "Project Id"   : "project_id",
+    "Summary"      : "summary",
+    "State"        : "State",
+    "Id"           : "numberInProject",
+    "Created"      : "created",
+    "Updated"      : "updated",
+    "Resolved"     : "resolved",
+    "Assignee"     : "Assignee",
+    "Description"  : "description",
+    "Labels"       : "Labels",
+    "Author"       : "reporterName",
+    "Milestone"    : "Fix versions",
+    "Weight"       : "Estimation",
+    "Time Spent"   : "Spent Time"
+}
+
+youtrackutils.csvClient.FIELD_TYPES = {
+    "State"        : "state[1]",
+    "Assignee"     : "user[1]",
+    "Labels"       : "enum[*]",
+    "Fix versions" : "version[*]",
+    "Type"         : "enum[1]"
+}
+
+youtrackutils.csvClient.DATE_FORMAT_STRING = "%Y-%m-%dT%H:%M:%SZ"
+youtrackutils.csvClient.VALUE_DELIMITER = "|"
+youtrackutils.csvClient.USE_MARKDOWN = True
+
+CSV_FILE = "gitlab2youtrack-{repo}-{data}.csv"
+
+help_url = "\
+https://www.jetbrains.com/help/youtrack/standalone/import-from-gitlab.html"
+
+GITLAB_API_ENDPOINT = "https://www.gitlab.com/api/v4"
+gitlab_api_endpoint = GITLAB_API_ENDPOINT
+
+def usage():
+    basename = os.path.basename(sys.argv[0])
+
+    print("""
+Usage:
+    %s [OPTIONS] yt_url gl_api_endpoint gl_login gl_token gl_project_name gl_project_id
+
+    yt_url          YouTrack base URL
+
+    gl_api_endpoint The REST API endpoint to the instance of GitLab
+
+    gl_login        The username to log in to GitLab
+
+    gl_token        The private token to log in to GitLab
+
+    gl_project_name The name of the GitLab project to import issues from
+
+    gl_project_id   The id of the GitLab project to import issues from
+
+    For instructions, see:
+    %s
+
+Options:
+    -h,  Show this help and exit
+    -T TOKEN_FILE,
+         Path to file with permanent token
+    -t TOKEN,
+         Value for permanent token as text
+    -u LOGIN,
+         YouTrack user login to perform import on behalf of
+    -p PASSWORD,
+         YouTrack user password
+
+Examples:
+
+    $ %s -T token https://youtrack.company.com https://api.gitlab.com/api/v4 gl-user gl-token gl_project_name gl_project_id
+
+
+""" % (basename, help_url, basename))
+
+
+def main():
+    try:
+        params = {}
+        opts, args = getopt.getopt(sys.argv[1:], 'hu:p:t:T:')
+        for opt, val in opts:
+            if opt == '-h':
+                usage()
+                sys.exit(0)
+            elif opt == '-u':
+                params['login'] = val
+            elif opt == '-p':
+                params['password'] = val
+            elif opt == '-t':
+                params['token'] = val
+            elif opt == '-T':
+                check_file_and_save(val, params, 'token_file')
+    except getopt.GetoptError as e:
+        print(e)
+        usage()
+        sys.exit(1)
+
+    try:
+        params['target_url'], gitlab_api_endpoint, gitlab_user, gitlab_token, gitlab_project_name, gitlab_project_id = args
+    except (ValueError, KeyError, IndexError):
+        print("Bad arguments")
+        usage()
+        sys.exit(1)
+
+    if gitlab_project_name.find('/') > -1:
+        gitlab_project_owner, gitlab_project_name = gitlab_project_name.split('/')
+    else:
+        gitlab_project_owner = gitlab_user
+
+    params['issues_file'] = CSV_FILE.format(repo=gitlab_project_name, data='issues')
+    params['comments_file'] = CSV_FILE.format(repo=gitlab_project_name, data='comments')
+
+    gitlab2csv(params['issues_file'],
+               params['comments_file'],
+               gitlab_api_endpoint,
+               gitlab_user,
+               gitlab_token,
+               gitlab_project_name,
+               gitlab_project_owner,
+               gitlab_project_id)
+
+    try:
+        csv2youtrack.csv2youtrack(params)
+    except youtrack.YouTrackException as e:
+        print e
+        pass
+
+def check_file_and_save(filename, params, key):
+    try:
+        params[key] = os.path.abspath(filename)
+    except (OSError, IOError) as e:
+        print("Data file is not accessible: " + str(e))
+        print(filename)
+        sys.exit(1)
+
+
+def get_last_part_of_url(url_string):
+    return url_string.split('/').pop()
+
+
+# based on https://gist.gitlab.com/unbracketed/3380407
+def write_issues(r, issues_csvout, comments_csvout, gitlab_api_endpoint, gitlab_project_owner, gitlab_project_name, gitlab_project_id, headers):
+    """output a list of issues to csv"""
+    if not r.status_code == 200:
+        raise Exception(r.status_code)
+    for issue in r.json():
+        labels = []
+        labels_lowercase = []
+        for label in issue['labels']:
+            label_name = label
+            if not label_name:
+                continue
+            labels.append(label_name)
+            labels_lowercase.append(label_name)
+
+        # TODO: Join writerow
+        #labels = csvClient.VALUE_DELIMITER.join([str(x) for x in labels])
+
+        assignee = issue['assignee']
+        if assignee:
+            assignee = assignee.get('username')
+        else:
+            assignee = issue['assignees']
+            if assignee:
+                assignee = assignee.get('username')
+
+        created = issue['created_at']
+        updated = issue.get('updated_at', '')
+        resolved = issue.get('closed_at', '')
+
+        author = issue['author'].get('username')
+        if not author:
+            # I'm not sure if the line below applies to Gitlab or just Github.
+            author = get_last_part_of_url(issue['user'].get('url'))
+
+        project = re.sub(r'[^\w]', '_', get_last_part_of_url(gitlab_project_name))
+
+        milestone = issue.get('milestone')
+        if milestone:
+            milestone = milestone['title']
+        else:
+            milestone = ''
+
+        state = issue['state'].lower()
+        if state == 'closed':
+            if 'wontfix' in labels_lowercase or 'invalid' in labels_lowercase:
+                state = "Won't fix"
+            else:
+                state = "Fixed"
+
+        issue_type = 'Task'
+        if 'bug' in labels_lowercase:
+            issue_type = 'Bug'
+
+        issue_row = [project, project, issue['iid'], state, issue['title'],
+                     issue['description'], created, updated, resolved, author or 'guest',
+                     assignee, youtrackutils.csvClient.VALUE_DELIMITER.join(labels),
+                     issue_type, milestone, str(issue['weight']) + "d",
+                     issue['time_stats']['human_total_time_spent']]
+        issues_csvout.writerow([utf8encode(e) for e in issue_row])
+
+        # Handle migrating issue comments from GitLab to YouTrack.
+        if int(issue.get('user_notes_count', 0)) > 0:
+            gitlab_comments_url = "%s/projects/%s/issues/%s/notes?sort=asc&order_by=updated_at" % (gitlab_api_endpoint, gitlab_project_id, issue['iid']) 
+            rc = requests.get(gitlab_comments_url, headers=headers)
+            if not rc.status_code == 200:
+                raise Exception(r.status_code)
+            for comment in rc.json():
+                author = comment['author'].get('username')
+                if not author:
+                    author = "guest"
+                reg1 = re.compile('api/v4', re.VERBOSE)
+                gitlab_base = reg1.sub(r'', gitlab_api_endpoint)
+                comment_body = ''
+                try:
+                    reg2 = re.compile("mentioned in commit (\w+)", re.MULTILINE)
+                    comment_body = reg2.sub(r"mentioned in commit [\1](" + gitlab_base + gitlab_project_owner + "/" + gitlab_project_name + r"/commit/\1) in GitLab.", str(comment['body']))
+                    reg3 = re.compile("mentioned in merge request \!(\d+)", re.MULTILINE)
+                    comment_body = reg3.sub(r"mentioned in merge request [!\1](" + gitlab_base + gitlab_project_owner + "/" + gitlab_project_name + r"/merge_requests/\1) in GitLab.", comment_body)
+                    #print str(comment['body'])
+                    #print comment_body
+                except (UnicodeEncodeError, RuntimeError, TypeError, NameError):
+                    pass
+                comment_row = [project, issue['iid'], author or 'guest',
+                               comment['created_at'], comment_body]
+                comments_csvout.writerow([utf8encode(e) for e in comment_row])
+
+
+def gitlab2csv(issues_csv_file, comments_csv_file, gitlab_api_endpoint, gitlab_user, gitlab_token, gitlab_project_name, gitlab_project_owner, gitlab_project_id):
+    issues_url = '%s/projects/%s/issues?id=18&order_by=created_at&page=1&per_page=100&sort=desc&state=all' % (gitlab_api_endpoint, gitlab_project_id)
+    HEADERS = {'PRIVATE-TOKEN': gitlab_token}
+
+    r = requests.get(issues_url, headers=HEADERS)
+    issues_csvout = csv.writer(open(issues_csv_file, 'wb'))
+    issues_csvout.writerow(
+        ('Project Name', 'Project Id', 'Id', 'State', 'Summary', 'Description',
+         'Created', 'Updated', 'Resolved', 'Author', 'Assignee', 'Labels',
+         'Type', 'Milestone', 'Weight', 'Time Spent'))
+    comments_csvout = csv.writer(open(comments_csv_file, 'wb'))
+    write_issues(r, issues_csvout, comments_csvout, gitlab_api_endpoint, gitlab_project_owner, gitlab_project_name, gitlab_project_id, HEADERS)
+
+    #more pages? examine the 'link' header returned
+    if 'link' in r.headers:
+        pages = dict(
+            [(rel[6:-1], url[url.index('<')+1:-1]) for url, rel in
+                [link.split(';') for link in
+                    r.headers['link'].split(',')]])
+        while 'last' in pages and 'next' in pages:
+            r = requests.get(pages['next'], headers=HEADERS)
+            write_issues(r, issues_csvout, comments_csvout, gitlab_api_endpoint, gitlab_project_owner, gitlab_project_name, gitlab_project_id, HEADERS)
+            pages = dict(
+                [(rel[6:-1], url[url.index('<') + 1:-1]) for url, rel in
+                 [link.split(';') for link in
+                  r.headers['link'].split(',')]])
+
+
+if __name__ == "__main__":
+    main()