🧱(backend) serve peertube videos via Scaleway Edge Services

Peertube-transcoded videos are stored in Scaleway S3.
To migrate away from Cloudfront, let's update Marsha so the videos are served
using Scaleway Edge Services.
Note that Cloudfront is still used to served other media files (document,
deposited files, classroom document etc.) and static files.

.circleci/config.yml

@@ -267,6 +267,7 @@ jobs:
           DJANGO_AWS_ACCESS_KEY_ID: aws-access-key-id
           DJANGO_AWS_SECRET_ACCESS_KEY: aws-secret-access-key
           DJANGO_CLOUDFRONT_DOMAIN: abc.cloudfront.net
+          DJANGO_SCW_EDGE_SERVICE_DOMAIN: abc.svc.edge.scw.cloud
           DJANGO_UPDATE_STATE_SHARED_SECRETS: dummy,secret
           DJANGO_AWS_MEDIALIVE_ROLE_ARN: aws:medialive:arn:role
           DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN: aws:mediapackage:arn:role
@@ -349,6 +350,7 @@ jobs:
           DJANGO_AWS_ACCESS_KEY_ID: aws-access-key-id
           DJANGO_AWS_SECRET_ACCESS_KEY: aws-secret-access-key
           DJANGO_CLOUDFRONT_DOMAIN: abc.cloudfront.net
+          DJANGO_SCW_EDGE_SERVICE_DOMAIN: abc.svc.edge.scw.cloud
           DJANGO_UPDATE_STATE_SHARED_SECRETS: dummy,secret
           DJANGO_AWS_MEDIALIVE_ROLE_ARN: aws:medialive:arn:role
           DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN: aws:mediapackage:arn:role
@@ -589,6 +591,7 @@ jobs:
           DJANGO_AWS_ACCESS_KEY_ID: aws-access-key-id
           DJANGO_AWS_SECRET_ACCESS_KEY: aws-secret-access-key
           DJANGO_CLOUDFRONT_DOMAIN: abc.cloudfront.net
+          DJANGO_SCW_EDGE_SERVICE_DOMAIN: abc.svc.edge.scw.cloud
           DJANGO_UPDATE_STATE_SHARED_SECRETS: dummy,secret
           DJANGO_AWS_MEDIALIVE_ROLE_ARN: aws:medialive:arn:role
           DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN: aws:mediapackage:arn:role

CHANGELOG.md

@@ -8,6 +8,10 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Added
+
+- Add SCW_EDGE_SERVICE_DOMAIN setting for serving videos with SCW Edge Services
+
 ### Changed
 
 - Increase connection timeout on Nginx for peertube runner success request

docs/env.md

@@ -291,6 +291,15 @@ that will be used to distribute processed files to end users.
 - Required: Yes
 - Default: None
 
+#### DJANGO_SCW_EDGE_SERVICE_DOMAIN
+
+The domain for the Scaleway Edge Service.
+This is the domain that will be used to distribute videos from Scaleway S3 to end users.
+
+- Type: string
+- Required: Yes
+- Default: None
+
 ### XMPP settings
 
 #### DJANGO_LIVE_CHAT_ENABLED

env.d/test

@@ -15,6 +15,7 @@ DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN=aws:mediapackage:arn:role
 
 # AWS
 DJANGO_CLOUDFRONT_DOMAIN=abc.cloudfront.net
+DJANGO_SCW_EDGE_SERVICE_DOMAIN=abc.svc.edge.scw.cloud
 DJANGO_UPDATE_STATE_SHARED_SECRETS=dummy,secret
 
 # BBB

src/backend/marsha/core/serializers/video.py

@@ -207,13 +207,7 @@ def get_vod_urls(self, obj):
 
         urls = {"mp4": {}, "thumbnails": {}, "manifests": {}}
 
-        base = f"{settings.AWS_S3_URL_PROTOCOL}://{settings.CLOUDFRONT_DOMAIN}/{obj.pk}"
         stamp = time_utils.to_timestamp(obj.uploaded_on)
-        if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
-            params = get_video_cloudfront_url_params(obj.pk)
-
-        filename = f"{slugify(obj.playlist.title)}_{stamp}.mp4"
-        content_disposition = quote_plus(f"attachment; filename={filename}")
 
         # Trying to recover the transcoding pipeline
         if obj.transcode_pipeline is None:
@@ -228,6 +222,13 @@ def get_vod_urls(self, obj):
             )
 
         if obj.transcode_pipeline == AWS_PIPELINE:
+            base = f"{settings.AWS_S3_URL_PROTOCOL}://{settings.CLOUDFRONT_DOMAIN}/{obj.pk}"
+            if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
+                params = get_video_cloudfront_url_params(obj.pk)
+
+            filename = f"{slugify(obj.playlist.title)}_{stamp}.mp4"
+            content_disposition = quote_plus(f"attachment; filename={filename}")
+
             for resolution in obj.resolutions:
                 # MP4
                 mp4_url = (
@@ -255,6 +256,7 @@ def get_vod_urls(self, obj):
 
                 # Previews
                 urls["previews"] = f"{base}/previews/{stamp}_100.jpg"
+
         elif obj.transcode_pipeline == PEERTUBE_PIPELINE:
             base = obj.get_videos_storage_prefix(stamp=stamp)
             for resolution in obj.resolutions:

src/backend/marsha/core/storage/s3.py

@@ -32,7 +32,7 @@ class S3VideoStorage(S3Storage):
 
     object_parameters = settings.VIDEOS_STORAGE_S3_OBJECT_PARAMETERS
 
-    custom_domain = settings.CLOUDFRONT_DOMAIN
+    custom_domain = settings.SCW_EDGE_SERVICE_DOMAIN
     url_protocol = "https:"
 
     if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:

src/backend/marsha/core/tests/serializers/test_video.py

@@ -45,7 +45,7 @@ def test_video_serializer_urls_with_aws_pipeline(self):
         )
 
     @override_settings(
-        MEDIA_URL="https://abc.cloudfront.net/",
+        MEDIA_URL="https://abc.svc.edge.scw.cloud/",
     )
     def test_video_serializer_urls_with_peertube_pipeline(self):
         """The VideoBaseSerializer should return the right URLs."""
@@ -63,20 +63,20 @@ def test_video_serializer_urls_with_peertube_pipeline(self):
             serializer = VideoBaseSerializer(video)
 
             self.assertEqual(
-                f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg",
+                f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg",
                 serializer.data["urls"]["thumbnails"][1080],
             )
             self.assertEqual(
-                f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/master.m3u8?"
+                f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/master.m3u8?"
                 "v=25301066da0654a2967176ea47c26f6735b2cfcbd206f49fe1379f5103bb7b1f",
                 serializer.data["urls"]["manifests"]["hls"],
             )
             self.assertEqual(
-                f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg",
+                f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg",
                 serializer.data["urls"]["previews"],
             )
             self.assertTrue(
-                f"https://abc.cloudfront.net/vod/{video.pk}/"
+                f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/"
                 "video/1640995200/1640995200-1080-fragmented.mp4"
                 in serializer.data["urls"]["mp4"][1080]
             )
@@ -117,7 +117,7 @@ def test_video_serializer_urls_with_no_pipeline(self):
             )
 
     @override_settings(
-        MEDIA_URL="https://abc.cloudfront.net/",
+        MEDIA_URL="https://abc.svc.edge.scw.cloud/",
     )
     def test_video_serializer_urls_with_no_pipeline_recovered_to_peertube(self):
         """The VideoBaseSerializer should return the right URLs with Peertube pipeline."""
@@ -129,29 +129,32 @@ def test_video_serializer_urls_with_no_pipeline_recovered_to_peertube(self):
             uploaded_on=date,
         )
 
-        with mock.patch(
-            "marsha.core.serializers.video.capture_message"
-        ) as sentry_capture_message, mock.patch(
-            "marsha.core.serializers.video.video_storage"
-        ) as mock_video_storage:
+        with (
+            mock.patch(
+                "marsha.core.serializers.video.capture_message"
+            ) as sentry_capture_message,
+            mock.patch(
+                "marsha.core.serializers.video.video_storage"
+            ) as mock_video_storage,
+        ):
             mock_video_storage.url = storage_class.video_storage.url
             mock_video_storage.exists.return_value = True
 
             serializer = VideoBaseSerializer(video)
             self.assertEqual(
-                f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg",
+                f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg",
                 serializer.data["urls"]["thumbnails"][1080],
             )
             self.assertEqual(
-                f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/master.m3u8",
+                f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/master.m3u8",
                 serializer.data["urls"]["manifests"]["hls"],
             )
             self.assertEqual(
-                f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg",
+                f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg",
                 serializer.data["urls"]["previews"],
             )
             self.assertTrue(
-                f"https://abc.cloudfront.net/vod/{video.pk}/"
+                f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/"
                 "video/1640995200/1640995200-1080-fragmented.mp4"
                 in serializer.data["urls"]["mp4"][1080]
             )

src/backend/marsha/settings.py

@@ -385,6 +385,8 @@ class Base(Configuration):
         {"ContentDisposition": "attachment"}
     )
 
+    SCW_EDGE_SERVICE_DOMAIN = values.Value(None)
+
     # LTI Config
     LTI_CONFIG_TITLE = values.Value("Marsha")
     LTI_CONFIG_TITLES = values.DictValue(

src/tray/templates/services/app/secret.yml.j2

@@ -14,6 +14,7 @@ data:
   DJANGO_UPDATE_STATE_SHARED_SECRETS: "{{ MARSHA_VAULT.DJANGO_UPDATE_STATE_SHARED_SECRETS | default('secret') | b64encode }}"
   DJANGO_CLOUDFRONT_ACCESS_KEY_ID: "{{ MARSHA_VAULT.DJANGO_CLOUDFRONT_ACCESS_KEY_ID | default('secret') | b64encode }}"
   DJANGO_CLOUDFRONT_DOMAIN: "{{ MARSHA_VAULT.DJANGO_CLOUDFRONT_DOMAIN | default('foo.com') | b64encode }}"
+  DJANGO_SCW_EDGE_SERVICE_DOMAIN: "{{ MARSHA_VAULT.DJANGO_SCW_EDGE_SERVICE_DOMAIN | default('foo.com') | b64encode }}"
   DJANGO_JWT_SIGNING_KEY: "{{ MARSHA_VAULT.DJANGO_JWT_SIGNING_KEY | default('secret') | b64encode }}"
   DJANGO_SECRET_KEY: "{{ MARSHA_VAULT.DJANGO_SECRET_KEY | default('supersecret') | b64encode }}"
   DJANGO_AWS_MEDIALIVE_ROLE_ARN: "{{ MARSHA_VAULT.DJANGO_AWS_MEDIALIVE_ROLE_ARN | default('secret') | b64encode }}"