Enable GCP Support

cartography/cli.py

@@ -235,6 +235,22 @@ def _build_parser(self):
                 'AWS config file for which cartography needs to be run'
             ),
         )
+        parser.add_argument(
+            '--borneo-gcp-creds',
+            type=str,
+            default=None,
+            help=(
+                'Path to GCP Creds file'
+            ),
+        )
+        parser.add_argument(
+            '--borneo-gcp-project',
+            type=str,
+            default=None,
+            help=(
+                'GCP Project Name'
+            ),
+        )
         parser.add_argument(
             '--crxcavator-api-base-uri',
             type=str,
@@ -446,9 +462,16 @@ def main(self, argv: str) -> int:
         """
         # TODO support parameter lookup in environment variables if not present on command line
         config: argparse.Namespace = self.parser.parse_args(argv)
-
-        os.environ['AWS_PROFILE'] = config.borneo_aws_profile
-        os.environ['AWS_CONFIG_FILE'] = config.borneo_aws_config
+
+        if config.borneo_aws_profile is not None:
+            os.environ['AWS_PROFILE'] = config.borneo_aws_profile
+        if config.borneo_aws_config is not None:
+            os.environ['AWS_CONFIG_FILE'] = config.borneo_aws_config
+        if config.borneo_gcp_creds is not None:
+            os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = config.borneo_gcp_creds
+        if config.borneo_gcp_project is not None:
+            os.environ["GOOGLE_CLOUD_PROJECT"] = config.borneo_gcp_project
+
         # Logging config
         if config.verbose:
             logging.getLogger('cartography').setLevel(logging.DEBUG)
@@ -595,9 +618,13 @@ def main(argv=None, sync_flag=None):
     logging.getLogger('neo4j').setLevel(logging.WARNING)
     argv = argv if argv is not None else sys.argv[1:]
     requested_sync = sync_flag if sync_flag is not None else 'default'
+    logger.info("Running cartography sync with requested sync: %s", requested_sync)
     if(requested_sync == 'rule_check'):
         sync = cartography.sync.build_rule_check_sync()
         result = CLI(sync, prog='cartography').main(argv)
+    if(requested_sync.startswith("gcp")):
+        sync = cartography.sync.build_borneo_gcp_sync("skip_index" in requested_sync)
+        result = CLI(sync, prog='cartography').main(argv)
     else:
         if(requested_sync != "default"):
             logger.warning("The requested sync doesn't exist, running the default sync")

cartography/intel/gcp/__init__.py

@@ -8,8 +8,9 @@
 import googleapiclient.discovery
 import neo4j
 from googleapiclient.discovery import Resource
-from oauth2client.client import ApplicationDefaultCredentialsError
-from oauth2client.client import GoogleCredentials
+from google.auth import default as google_default_auth
+from google.auth.credentials import Credentials
+from google.auth.exceptions import DefaultCredentialsError
 
 from cartography.config import Config
 from cartography.intel.gcp import compute
@@ -34,87 +35,87 @@
 )
 
 
-def _get_crm_resource_v1(credentials: GoogleCredentials) -> Resource:
+def _get_crm_resource_v1(credentials: Credentials) -> Resource:
     """
     Instantiates a Google Compute Resource Manager v1 resource object to call the Resource Manager API.
     See https://cloud.google.com/resource-manager/reference/rest/.
-    :param credentials: The GoogleCredentials object
+    :param credentials: The Credentials object
     :return: A CRM v1 resource object
     """
     # cache_discovery=False to suppress extra warnings.
     # See https://github.com/googleapis/google-api-python-client/issues/299#issuecomment-268915510 and related issues
     return googleapiclient.discovery.build('cloudresourcemanager', 'v1', credentials=credentials, cache_discovery=False)
 
 
-def _get_crm_resource_v2(credentials: GoogleCredentials) -> Resource:
+def _get_crm_resource_v2(credentials: Credentials) -> Resource:
     """
     Instantiates a Google Compute Resource Manager v2 resource object to call the Resource Manager API.
     We need a v2 resource object to query for GCP folders.
-    :param credentials: The GoogleCredentials object
+    :param credentials: The Credentials object
     :return: A CRM v2 resource object
     """
     return googleapiclient.discovery.build('cloudresourcemanager', 'v2', credentials=credentials, cache_discovery=False)
 
 
-def _get_compute_resource(credentials: GoogleCredentials) -> Resource:
+def _get_compute_resource(credentials: Credentials) -> Resource:
     """
     Instantiates a Google Compute resource object to call the Compute API. This is used to pull zone, instance, and
     networking data. See https://cloud.google.com/compute/docs/reference/rest/v1/.
-    :param credentials: The GoogleCredentials object
+    :param credentials: The Credentials object
     :return: A Compute resource object
     """
     return googleapiclient.discovery.build('compute', 'v1', credentials=credentials, cache_discovery=False)
 
 
-def _get_storage_resource(credentials: GoogleCredentials) -> Resource:
+def _get_storage_resource(credentials: Credentials) -> Resource:
     """
     Instantiates a Google Cloud Storage resource object to call the Storage API.
     This is used to pull bucket metadata and IAM Policies
     as well as list buckets in a specified project.
     See https://cloud.google.com/storage/docs/json_api/.
-    :param credentials: The GoogleCredentials object
+    :param credentials: The Credentials object
     :return: A Storage resource object
     """
     return googleapiclient.discovery.build('storage', 'v1', credentials=credentials, cache_discovery=False)
 
 
-def _get_container_resource(credentials: GoogleCredentials) -> Resource:
+def _get_container_resource(credentials: Credentials) -> Resource:
     """
     Instantiates a Google Cloud Container resource object to call the
     Container API. See: https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/.
 
-    :param credentials: The GoogleCredentials object
+    :param credentials: The Credentials object
     :return: A Container resource object
     """
     return googleapiclient.discovery.build('container', 'v1', credentials=credentials, cache_discovery=False)
 
 
-def _get_dns_resource(credentials: GoogleCredentials) -> Resource:
+def _get_dns_resource(credentials: Credentials) -> Resource:
     """
     Instantiates a Google Cloud DNS resource object to call the
     Container API. See: https://cloud.google.com/dns/docs/reference/v1/.
 
-    :param credentials: The GoogleCredentials object
+    :param credentials: The Credentials object
     :return: A DNS resource object
     """
     return googleapiclient.discovery.build('dns', 'v1', credentials=credentials, cache_discovery=False)
 
 
-def _get_serviceusage_resource(credentials: GoogleCredentials) -> Resource:
+def _get_serviceusage_resource(credentials: Credentials) -> Resource:
     """
     Instantiates a serviceusage resource object.
     See: https://cloud.google.com/service-usage/docs/reference/rest/v1/operations/list.
 
-    :param credentials: The GoogleCredentials object
+    :param credentials: The Credentials object
     :return: A serviceusage resource object
     """
     return googleapiclient.discovery.build('serviceusage', 'v1', credentials=credentials, cache_discovery=False)
 
 
-def _initialize_resources(credentials: GoogleCredentials) -> Resource:
+def _initialize_resources(credentials: Credentials) -> Resource:
     """
     Create namedtuple of all resource objects necessary for GCP data gathering.
-    :param credentials: The GoogleCredentials object
+    :param credentials: The Credentials object
     :return: namedtuple of all resource objects
     """
     return Resources(
@@ -222,12 +223,10 @@ def start_gcp_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
         "UPDATE_TAG": config.update_tag,
     }
     try:
-        # Explicitly use Application Default Credentials.
-        # See https://oauth2client.readthedocs.io/en/latest/source/
-        #             oauth2client.client.html#oauth2client.client.OAuth2Credentials
-        credentials = GoogleCredentials.get_application_default()
-    except ApplicationDefaultCredentialsError as e:
-        logger.debug("Error occurred calling GoogleCredentials.get_application_default().", exc_info=True)
+        credentials, project = google_default_auth()
+        logger.info("Successfully initialized Google Compute Platform creds for project %s.", project)
+    except DefaultCredentialsError as e:
+        logger.debug("Error occurred calling Credentials.get_application_default().", exc_info=True)
         logger.error(
             (
                 "Unable to initialize Google Compute Platform creds. If you don't have GCP data or don't want to load "

cartography/sync.py

@@ -199,6 +199,16 @@ def build_default_borneo_sync(skipIndex: bool) -> Sync:
     ])
     return sync
 
+def build_borneo_gcp_sync(skipIndex: bool) -> Sync:
+    sync = Sync()
+    if skipIndex != True:
+        sync.add_stages([('create-indexes', cartography.intel.create_indexes.run)])
+    sync.add_stages([
+        ('gcp', cartography.intel.gcp.start_gcp_ingestion),
+        ('analysis', cartography.intel.analysis.run)
+    ])
+    return sync
+
 def build_rule_check_sync() -> Sync:
     sync = Sync()
     sync.add_stages([