diff --git a/materializationengine/blueprints/materialize/api.py b/materializationengine/blueprints/materialize/api.py index c22e5d99..bd5c739d 100644 --- a/materializationengine/blueprints/materialize/api.py +++ b/materializationengine/blueprints/materialize/api.py @@ -466,57 +466,57 @@ def post(self, datastack_name: str, version: int, table_name: str): "message": f"failed to activate service account using {activate_command}. Error: {stderr.decode()} stdout: {stdout.decode()}" }, 500 - # run a gcloud command to select * from table and write it to disk as a csv - export_command = [ + header_file = f"{datastack_name}/v{version}/{table_name}_header.csv" + header_cloudpath = os.path.join(bucket, header_file) + + header_command = [ "gcloud", "sql", "export", "csv", sql_instance_name, - cloudpath, + header_cloudpath, "--database", mat_db_name, - "--async", "--query", - f"SELECT * from {table_name}", + f"SELECT column_name, data_type from INFORMATION_SCHEMA.COLUMNS where TABLE_NAME = '{table_name}'", ] - process = subprocess.Popen( - export_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + header_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout, stderr = process.communicate() # run this command and capture the stdout and return code return_code = process.returncode if return_code != 0: return { - "message": f"file failed to create using: {export_command}. Error: {stderr.decode()} stdout: {stdout.decode()}" + "message": f"header file failed to create using:\ + {header_command}. Error: {stderr.decode()} stdout: {stdout.decode()}" }, 500 - header_file = f"{datastack_name}/v{version}/{table_name}_header.csv" - header_cloudpath = os.path.join(bucket, header_file) - header_command = [ + # run a gcloud command to select * from table and write it to disk as a csv + export_command = [ "gcloud", "sql", "export", "csv", sql_instance_name, - header_cloudpath, + cloudpath, "--database", mat_db_name, "--async", "--query", - f"SELECT column_name, data_type from INFORMATION_SCHEMA.COLUMNS where TABLE_NAME = '{table_name}'", + f"SELECT * from {table_name}", ] + process = subprocess.Popen( - header_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + export_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout, stderr = process.communicate() # run this command and capture the stdout and return code return_code = process.returncode if return_code != 0: return { - "message": f"header file failed to create using:\ - {header_command}. Error: {stderr.decode()} stdout: {stdout.decode()}" + "message": f"file failed to create using: {export_command}. Error: {stderr.decode()} stdout: {stdout.decode()}" }, 500 else: