Merge pull request #77 from moj-analytical-services/data_deletion_fix

Fix issue with deleting data from extra tables when creating a temp t…

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v5.3.1 - 2022-07-22
+
+- Fix issue when creating temporary tables where tables which start with another table's name could have their data deleted.
+
 ## v5.3.0 - 2022-07-13
 
 - No longer supporting Python 3.7 due to numpy security issue.

pydbtools/__init__.py

@@ -28,4 +28,4 @@
     render_sql_template,
 )
 
-__version__ = "5.3.0"
+__version__ = "5.3.1"

pydbtools/_wrangler.py

@@ -277,8 +277,12 @@ def create_temp_table(
 
     _ = create_temp_database(temp_db_name, boto3_session)
 
-    # Clear out table every time
-    wr.s3.delete_objects(table_path, boto3_session=boto3_session)
+    # Clear out table every time, making sure other tables aren't being
+    # cleared out
+    wr.s3.delete_objects(
+        table_path if table_path.endswith("/") else table_path + "/",
+        boto3_session=boto3_session,
+    )
 
     drop_table_query = f"DROP TABLE IF EXISTS {temp_db_name}.{table_name}"
 
@@ -550,5 +554,5 @@ def dataframe_to_temp_table(df: pd.DataFrame, table: str, boto3_session=None) ->
         database=db,
         table=table,
         boto3_session=boto3_session,
-        mode="overwrite"
+        mode="overwrite",
     )

pyproject.toml

@@ -1,7 +1,7 @@
 [tool]
 [tool.poetry]
 name = "pydbtools"
-version = "5.3.0"
+version = "5.3.1"
 description = "A python package to query data via amazon athena and bring it into a pandas df using aws-wrangler."
 license = "MIT"
 authors = ["Karik Isichei <[email protected]>"]