add management task for cleaning up stale S3 files

.gitignore

@@ -27,6 +27,7 @@ coverage.xml
 local_settings.py
 staticfiles/
 db.sqlite3
+dump.rdb
 
 # dotenv (environment variables, used for secret/local stuff)
 .env

project_admin/management/commands/cleanup_files.py

@@ -0,0 +1,23 @@
+from django.core.management.base import BaseCommand
+import datetime
+from django.conf import settings
+from project_admin.models import S3Upload
+from datetime import timedelta
+import boto3
+
+
+class Command(BaseCommand):
+    help = 'Delete stale zip files'
+
+    def handle(self, *args, **options):
+        s3_client = boto3.client('s3', aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
+                                 aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY)
+        old_files = S3Upload.objects.filter(
+            created_at__date__lt=datetime.datetime.now() - timedelta(hours=24))
+        for of in old_files:
+            delete_response = s3_client.delete_object(
+                    Bucket=settings.AWS_STORAGE_BUCKET_NAME,
+                    Key=of.key)
+            if delete_response.get("ResponseMetadata").get('HTTPStatusCode') != 204:
+                print("Couldn't delete file {}".format(of.key))
+        old_files.delete()

project_admin/migrations/0013_s3upload.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.0.7 on 2018-08-20 23:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('project_admin', '0012_auto_20180615_0240'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='S3Upload',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('key', models.CharField(max_length=60)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+            ],
+        ),
+    ]

project_admin/models.py

@@ -70,3 +70,8 @@ class Note(models.Model):
     modified_at = models.DateTimeField(auto_now=True)
     project = models.ForeignKey(Project, on_delete=models.CASCADE)
     member = models.ForeignKey(ProjectMember, on_delete=models.CASCADE)
+
+
+class S3Upload(models.Model):
+    key = models.CharField(max_length=60)
+    created_at = models.DateTimeField(auto_now_add=True)

project_admin/tasks.py

@@ -7,7 +7,7 @@
 from celery.decorators import task
 from celery.utils.log import get_task_logger
 from django.conf import settings
-from .models import Project
+from .models import Project, S3Upload
 from .utility import send_email
 from .helpers import get_all_members, filter_members_group_id
 logger = get_task_logger(__name__)
@@ -58,6 +58,8 @@ def download_zip_files(user, group_id=None):
             zipfile_url = s3_client.generate_presigned_url('get_object',
                                                            Params={'Bucket': settings.AWS_STORAGE_BUCKET_NAME,
                                                                    'Key': zip_filename}, ExpiresIn=86400)
+            s3uploaded_file = S3Upload(key=zip_filename)
+            s3uploaded_file.save()
         send_email(download_success, zipfile_url, project.contact_email, project.name)
     except Exception as e:
         logger.error('Downloading zip file crashed', e)