process_ml_events.py

#!/usr/bin/env python3
'''Reads emails generated by the filter script and submits patches/make comments'''
import os
import re
import time
import datetime
from collections import namedtuple

import git
import gitlab

import cfg
import db_helper
import mail_helper

# Do some initialization early so we can abort in case of failure
LOCAL_REPO = git.Repo(cfg.LOCAL_REPO_PATH)
assert not LOCAL_REPO.bare
assert not LOCAL_REPO.is_dirty()
assert LOCAL_REPO.head.ref == LOCAL_REPO.heads.master
LOCAL_REPO_GIT = LOCAL_REPO.git
# Ensure we are up to date
LOCAL_REPO_GIT.fetch('upstream')
LOCAL_REPO_GIT.merge('upstream/master')
GITLAB = gitlab.Gitlab.from_config(cfg.BOT_LOGIN_CFG_NAME, [])
assert GITLAB is not None
FORK_REPO = GITLAB.projects.get(cfg.FORK_REPO_ID)
assert FORK_REPO is not None
MAIN_REPO = GITLAB.projects.get(cfg.MAIN_REPO_ID)
assert MAIN_REPO is not None
GITLAB_ADMIN = gitlab.Gitlab.from_config(cfg.ADMIN_LOGIN_CFG_NAME, [])
assert GITLAB_ADMIN is not None
ADMIN_PROJECT_VIEW = GITLAB_ADMIN.projects.get(cfg.MAIN_REPO_ID)
assert ADMIN_PROJECT_VIEW is not None

# Utility functions
def is_full(array):
  for element in array:
    if element is None:
      return False
  return True

# End Utility Functions

Patch = namedtuple('Patch', 'path msgid subject')

def create_or_update_merge_request(mr, title, author, description, patches, prologue_msg_id):
  # create the local git branch
  branch_name = None
  if mr is None:
    branch_name = 'ml-patchset-{0}'.format(time.time())
    LOCAL_REPO_GIT.checkout('HEAD', b=branch_name)
  else:
    branch_name = mr.source_branch
    LOCAL_REPO_GIT.checkout(branch_name)
    LOCAL_REPO_GIT.reset('master', hard=True)

  # apply the patches
  try:
    for patch in patches:
      LOCAL_REPO_GIT.am(str(patch.path))
  except:
    print('Failed to apply patches, discarding patchset')
    # TODO: make more robust, and send email back if it didn't apply
    if mr is not None:
      LOCAL_REPO_GIT.reset('origin/'+branch_name, hard=True)
    LOCAL_REPO_GIT.checkout('master')
    if mr is None:
      LOCAL_REPO_GIT.branch(D=branch_name)
    return
  finally:
    for patch in patches:
      patch.path.unlink()

  LOCAL_REPO_GIT.checkout('master')

  # push work to origin
  LOCAL_REPO_GIT.push('origin', branch_name, force=True)

  # create merge request
  if mr is None:
    mr = FORK_REPO.mergerequests.create({'source_branch': branch_name,
                                         'target_project_id': cfg.MAIN_REPO_ID,
                                         'target_branch': 'master',
                                         'title': title if title is not None else 'Multi-Patch Patchset from Mailing List',
                                         'description': description})
    if not cfg.BIDIRECTIONAL_COMM:
      admin_mr = ADMIN_PROJECT_VIEW.mergerequests.get(mr.id)
      admin_mr.discussion_locked = True
      admin_mr.save()

    # send email to mailing list as a place to put MR comments
    if prologue_msg_id is None:
      if len(patches) == 1:
        db_helper.link_discussion_to_mail(db_helper.Discussion(mr.id, 0), patches[0].msgid)
        return
      # send meta email if prologue wasn't sent
      mail_helper.send_mail('Gitlab discussion thread for recent patchset by' + author,
                            'Merge-Request Link: ' + mr.web_url)
    else:
      db_helper.link_discussion_to_mail(db_helper.Discussion(mr.id, 0), prologue_msg_id)
  elif prologue_msg_id and len(patches) != 1:
    extra_discussion = mr.discussions.create({'body': 'Discussion on updated commits'})
    db_helper.link_discussion_to_mail(db_helper.Discussion(mr.id, extra_discussion.id), prologue_msg_id)

  # create a discussion for each patch
  for patch in patches:
    patch_discussion = mr.discussions.create({'body': 'Discussion for {0}'.format(patch.subject)})
    # link mail thread to discussion
    db_helper.link_discussion_to_mail(db_helper.Discussion(mr.id, patch_discussion.id), patch.msgid)

Mail = namedtuple('Mail', 'msg_id reply_to sender body')

def format_email_body(raw_body):
  # for now, just put the entire email in a code block to prevent markdown formatting on patches
  # TODO: detect patches and put them in codeblocks, with the right language set
  return '```\n' + raw_body + '\n```'

# if it's not a patch, see if it's a comment on a MR thread
def process_standard_mail(mail):
  root_msg = db_helper.get_root_msg_id(mail.reply_to)
  discussion_entry = db_helper.lookup_discussion(root_msg)
  if discussion_entry is None:
    print(mail.reply_to, root_msg)
    return
  db_helper.add_child(root_msg, mail.msg_id)
  # TODO: Handle fancy emails
  comment_body = 'Mail from {0} on mailing list:\n\n{1}'.format(mail.sender, format_email_body(mail.body))
  print(comment_body)
  mr = MAIN_REPO.mergerequests.get(discussion_entry.mr_id)
  # get the discussion id, if present
  discussion = mr.discussions.get(discussion_entry.disc_id) if discussion_entry.disc_id != 0 else None

  if (not cfg.BIDIRECTIONAL_COMM and mr.author['id'] == cfg.BOT_GITLAB_ID):
    return

  admin_mr = ADMIN_PROJECT_VIEW.mergerequests.get(mr.id)
  relock = False
  if admin_mr.discussion_locked:
    admin_mr.discussion_locked = False
    admin_mr.save()
    relock = True

  if discussion is None:
    mr.notes.create({'body': comment_body})
  else:
    discussion.notes.create({'body': comment_body})

  if relock:
    admin_mr.discussion_locked = True
    admin_mr.save()

def find_root_mr(author_email, title):
  for mr in MAIN_REPO.mergerequests.list(all=True):
    if mr.commits().next().author_email == author_email and mr.title == title:
      return mr
  return None

def main():
  out_of_patches = False
  processed_patch_files = []
  while not out_of_patches:
    patches = None
    prologue_msg_id = None
    # Set to subject of either PATCH[0/n], or PATCH[1/1]
    patchset_title = None
     # If PATCH 0 exists, set the patchset description to the content of the email.
    patchset_description = ''
    current_author = None
    mr = None
    out_of_patches = True
    for file_path in cfg.PATCHES_PATH.iterdir():
      # discard if we've reached timeout
      create_time = datetime.datetime.fromtimestamp(os.path.getctime(file_path))
      if create_time < cfg.PATCH_PROCESS_TIMEOUT:
        file_path.unlink()
        continue

      if file_path.name in processed_patch_files:
        continue
      out_of_patches = False

      with file_path.open() as file:
        mail_contents = file.read()

      author = None
      email = None
      subject = None
      msg_id = None
      reply_to = None
      try:
        author  = re.search(r'(?m)^From: (.*)$', mail_contents).group(1)
        subject = re.search(r'(?m)^Subject: (.*)$', mail_contents).group(1)
        msg_id  = re.search(r'(?m)^Message-Id: (.*)$', mail_contents).group(1)
      except AttributeError:
        print('Invalid Message')
        file_path.unlink()
        continue
      search = re.search(r'(?m)^In-Reply-To: (.*)$', mail_contents)
      reply_to = search.group(1) if search is not None else None

      patch_prefix = re.search(r'^\[PATCH(?: v(?P<version>\d+))?(?: (?P<patch_idx>\d+)/(?P<patch_total>\d+))?\]', subject)
      author_search = re.search(r'^\"?(?P<name>[^\"]*)\"? <(?P<email>[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)>$', author)
      email = author_search.group('email') if author_search is not None else None

      if email is None or email == cfg.BOT_MAIL_ADDRESS:
        file_path.unlink()
        continue

      if patch_prefix is None:
        process_standard_mail(Mail(msg_id, reply_to, author, mail_contents[mail_contents.find('\n\n'):]))
        file_path.unlink()
        continue

      if 'resend' in patch_prefix.group(0):
        file_path.unlink()
        continue

      if current_author is not None and author != current_author:
        continue

      version = patch_prefix.group('version')
      patch_idx = patch_prefix.group('patch_idx')
      patch_total = patch_prefix.group('patch_total')

      if patch_total is None:
        patch_total = 1
        patch_idx = 1
        patchset_title = subject[patch_prefix.end() + 1:]

      patch_idx = int(patch_idx)
      patch_total = int(patch_total)

      if version is not None and version != 1 and patch_idx == 1:
        # Right now we only use patch 1 data to find the MR
        mr = find_root_mr(email, subject[patch_prefix.end() + 1:])
        if mr is None:
          print('unable to find MR for versioned patch')
          file_path.unlink()
          continue

      if patch_total < patch_idx:
        file_path.unlink()
        continue

      if patches is None:
        patches = [None] * patch_total
      elif len(patches) != patch_total:
        continue

      current_author = author
      processed_patch_files.append(file_path.name)

      if patch_idx == 0:
        patchset_title = subject[patch_prefix.end() + 1:]
        prologue_msg_id = msg_id
        continue

      patches[patch_idx - 1] = Patch(file_path, msg_id, subject)

      if is_full(patches):
        create_or_update_merge_request(mr, patchset_title, current_author, patchset_description, patches, prologue_msg_id)
        break

main()