Begin breaking apart gam.py into logical pieces

Start with one of the deepest parts of the stack, Google API request execution calls and associated errors. Critical information printing functions and application control logic are also broken out into their own components.

This change also adds unit tests for migrated content and makes code more PEP8 compliant.

This commit starts work on  GAM-team#147

src/controlflow.py

@@ -0,0 +1,66 @@
+"""Methods related to the central control flow of an application."""
+import random
+import sys
+import time
+
+import display  # TODO: Change to relative import when gam is setup as a package
+from var import MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS
+from var import MESSAGE_INVALID_JSON
+
+
+def system_error_exit(return_code, message):
+  """Raises a system exit with the given return code and message.
+
+  Args:
+    return_code: Int, the return code to yield when the system exits.
+    message: An error message to print before the system exits.
+  """
+  if message:
+    display.print_error(message)
+  sys.exit(return_code)
+
+
+def csv_field_error_exit(field_name, field_names):
+  """Raises a system exit when a CSV field is malformed.
+
+  Args:
+    field_name: The CSV field name for which a header does not exist in the
+      existing CSV headers.
+    field_names: The known list of CSV headers.
+  """
+  system_error_exit(
+      2,
+      MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(field_name,
+                                                     ','.join(field_names)))
+
+
+def invalid_json_exit(file_name):
+  """Raises a sysyem exit when invalid JSON content is encountered."""
+  system_error_exit(17, MESSAGE_INVALID_JSON.format(file_name))
+
+
+def wait_on_failure(current_attempt_num,
+                    total_num_retries,
+                    error_message,
+                    error_print_threshold=3):
+  """Executes an exponential backoff-style system sleep.
+
+  Args:
+    current_attempt_num: Int, the current number of retries.
+    total_num_retries: Int, the total number of times the current action will be
+      retried.
+    error_message: String, a message to be displayed that will give more context
+      around why the action is being retried.
+    error_print_threshold: Int, the number of attempts which will have their
+      error messages suppressed. Any current_attempt_num greater than
+      error_print_threshold will print the prescribed error.
+  """
+  wait_on_fail = min(2**current_attempt_num,
+                     60) + float(random.randint(1, 1000)) / 1000
+  if current_attempt_num > error_print_threshold:
+    sys.stderr.write(
+        'Temporary error: {0}, Backing off: {1} seconds, Retry: {2}/{3}\n'
+        .format(error_message, int(wait_on_fail), current_attempt_num,
+                total_num_retries))
+    sys.stderr.flush()
+  time.sleep(wait_on_fail)

src/controlflow_test.py

@@ -0,0 +1,108 @@
+"""Tests for controlflow."""
+
+import unittest
+from unittest.mock import patch
+
+import controlflow
+
+
+class ControlFlowTest(unittest.TestCase):
+
+  def test_system_error_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.system_error_exit(1, 'exit message')
+
+  def test_system_error_exit_raises_systemexit_with_return_code(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.system_error_exit(100, 'exit message')
+    self.assertEqual(context_manager.exception.code, 100)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_system_error_exit_prints_error_before_exiting(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.system_error_exit(100, 'exit message')
+    self.assertIn('exit message', mock_print_err.call_args[0][0])
+
+  def test_csv_field_error_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+
+  def test_csv_field_error_exit_exits_code_2(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+    self.assertEqual(context_manager.exception.code, 2)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_csv_field_error_exit_prints_error_details(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.csv_field_error_exit('aField',
+                                       ['unusedField1', 'unusedField2'])
+    printed_message = mock_print_err.call_args[0][0]
+    self.assertIn('aField', printed_message)
+    self.assertIn('unusedField1', printed_message)
+    self.assertIn('unusedField2', printed_message)
+
+  def test_invalid_json_exit_raises_systemexit_error(self):
+    with self.assertRaises(SystemExit):
+      controlflow.invalid_json_exit('filename')
+
+  def test_invalid_json_exit_exit_exits_code_17(self):
+    with self.assertRaises(SystemExit) as context_manager:
+      controlflow.invalid_json_exit('filename')
+    self.assertEqual(context_manager.exception.code, 17)
+
+  @patch.object(controlflow.display, 'print_error')
+  def test_invalid_json_exit_prints_error_details(self, mock_print_err):
+    with self.assertRaises(SystemExit):
+      controlflow.invalid_json_exit('filename')
+    printed_message = mock_print_err.call_args[0][0]
+    self.assertIn('filename', printed_message)
+
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_waits_exponentially(self, mock_sleep):
+    controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
+    controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
+    controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')
+
+    sleep_calls = mock_sleep.call_args_list
+    self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
+    self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
+    self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)
+
+  @patch.object(controlflow.time, 'sleep')
+  def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
+    total_attempts = 20
+    for attempt in range(1, total_attempts + 1):
+      controlflow.wait_on_failure(
+          attempt,
+          total_attempts,
+          'Attempt #%s' % attempt,
+          # Suppress messages while we make a lot of attempts.
+          error_print_threshold=total_attempts + 1)
+      # Wait time may be between 60 and 61 secs, due to rand addition.
+      self.assertLess(mock_sleep.call_args[0][0], 61)
+
+  # Prevent the system from actually sleeping and thus slowing down the test.
+  @patch.object(controlflow.time, 'sleep')
+  @patch.object(controlflow.sys.stderr, 'write')
+  def test_wait_on_failure_prints_errors(self, mock_stderr_write,
+                                         unused_mock_sleep):
+    message = 'An error message to display'
+    controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
+    self.assertIn(message, mock_stderr_write.call_args[0][0])
+
+  @patch.object(controlflow.time, 'sleep')
+  @patch.object(controlflow.sys.stderr, 'write')
+  def test_wait_on_failure_only_prints_after_threshold(self, mock_stderr_write,
+                                                       unused_mock_sleep):
+    total_attempts = 5
+    threshold = 3
+    for attempt in range(1, total_attempts + 1):
+      controlflow.wait_on_failure(
+          attempt,
+          total_attempts,
+          'Attempt #%s' % attempt,
+          error_print_threshold=threshold)
+    self.assertEqual(total_attempts - threshold, mock_stderr_write.call_count)

src/display.py

@@ -0,0 +1,18 @@
+"""Methods related to display of information to the user."""
+
+import sys
+import utils
+from var import ERROR_PREFIX
+from var import WARNING_PREFIX
+
+
+def print_error(message):
+  """Prints a one-line error message to stderr in a standard format."""
+  sys.stderr.write(
+      utils.convertUTF8('\n{0}{1}\n'.format(ERROR_PREFIX, message)))
+
+
+def print_warning(message):
+  """Prints a one-line warning message to stderr in a standard format."""
+  sys.stderr.write(
+      utils.convertUTF8('\n{0}{1}\n'.format(WARNING_PREFIX, message)))

src/display_test.py

@@ -0,0 +1,59 @@
+"""Tests for display."""
+
+import unittest
+from unittest.mock import patch
+
+import display
+from var import ERROR_PREFIX
+from var import WARNING_PREFIX
+
+
+class DisplayTest(unittest.TestCase):
+
+  @patch.object(display.sys.stderr, 'write')
+  def test_print_error_prints_to_stderr(self, mock_write):
+    message = 'test error'
+    display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertIn(message, printed_message)
+
+  @patch.object(display.sys.stderr, 'write')
+  def test_print_error_prints_error_prefix(self, mock_write):
+    message = 'test error'
+    display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertLess(
+        printed_message.find(ERROR_PREFIX), printed_message.find(message),
+        'The error prefix does not appear before the error message')
+
+  @patch.object(display.sys.stderr, 'write')
+  def test_print_error_ends_message_with_newline(self, mock_write):
+    message = 'test error'
+    display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertRegex(printed_message, '\n$',
+                     'The error message does not end in a newline.')
+
+  @patch.object(display.sys.stderr, 'write')
+  def test_print_warning_prints_to_stderr(self, mock_write):
+    message = 'test warning'
+    display.print_warning(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertIn(message, printed_message)
+
+  @patch.object(display.sys.stderr, 'write')
+  def test_print_warning_prints_error_prefix(self, mock_write):
+    message = 'test warning'
+    display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertLess(
+        printed_message.find(WARNING_PREFIX), printed_message.find(message),
+        'The warning prefix does not appear before the error message')
+
+  @patch.object(display.sys.stderr, 'write')
+  def test_print_warning_ends_message_with_newline(self, mock_write):
+    message = 'test warning'
+    display.print_error(message)
+    printed_message = mock_write.call_args[0][0]
+    self.assertRegex(printed_message, '\n$',
+                     'The warning message does not end in a newline.')