From c800fe30053f23183e3f503bf5a5d93d9db7e8ba Mon Sep 17 00:00:00 2001 From: Wilfried BARADAT Date: Tue, 18 Feb 2025 15:57:47 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B1(backend)=20serve=20peertube=20vide?= =?UTF-8?q?os=20via=20Scaleway=20Edge=20Services?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Peertube-transcoded videos are stored in Scaleway S3. To migrate away from Cloudfront, let's update Marsha so the videos are served using Scaleway Edge Services. Note that Cloudfront is still used to served other media files (document, deposited files, classroom document etc.) and static files. --- .circleci/config.yml | 3 ++ CHANGELOG.md | 4 +++ docs/env.md | 9 +++++ env.d/test | 1 + src/backend/marsha/core/serializers/video.py | 14 ++++---- src/backend/marsha/core/storage/s3.py | 2 +- .../core/tests/serializers/test_video.py | 33 ++++++++++--------- src/backend/marsha/settings.py | 2 ++ src/tray/templates/services/app/secret.yml.j2 | 1 + 9 files changed, 47 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c97becd362..c6eca60705 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -267,6 +267,7 @@ jobs: DJANGO_AWS_ACCESS_KEY_ID: aws-access-key-id DJANGO_AWS_SECRET_ACCESS_KEY: aws-secret-access-key DJANGO_CLOUDFRONT_DOMAIN: abc.cloudfront.net + DJANGO_SCW_EDGE_SERVICE_DOMAIN: abc.svc.edge.scw.cloud DJANGO_UPDATE_STATE_SHARED_SECRETS: dummy,secret DJANGO_AWS_MEDIALIVE_ROLE_ARN: aws:medialive:arn:role DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN: aws:mediapackage:arn:role @@ -349,6 +350,7 @@ jobs: DJANGO_AWS_ACCESS_KEY_ID: aws-access-key-id DJANGO_AWS_SECRET_ACCESS_KEY: aws-secret-access-key DJANGO_CLOUDFRONT_DOMAIN: abc.cloudfront.net + DJANGO_SCW_EDGE_SERVICE_DOMAIN: abc.svc.edge.scw.cloud DJANGO_UPDATE_STATE_SHARED_SECRETS: dummy,secret DJANGO_AWS_MEDIALIVE_ROLE_ARN: aws:medialive:arn:role DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN: aws:mediapackage:arn:role @@ -589,6 +591,7 @@ jobs: DJANGO_AWS_ACCESS_KEY_ID: aws-access-key-id DJANGO_AWS_SECRET_ACCESS_KEY: aws-secret-access-key DJANGO_CLOUDFRONT_DOMAIN: abc.cloudfront.net + DJANGO_SCW_EDGE_SERVICE_DOMAIN: abc.svc.edge.scw.cloud DJANGO_UPDATE_STATE_SHARED_SECRETS: dummy,secret DJANGO_AWS_MEDIALIVE_ROLE_ARN: aws:medialive:arn:role DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN: aws:mediapackage:arn:role diff --git a/CHANGELOG.md b/CHANGELOG.md index b134aff862..21dd617e6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- Add SCW_EDGE_SERVICE_DOMAIN setting for serving videos with SCW Edge Services + ### Changed - Increase connection timeout on Nginx for peertube runner success request diff --git a/docs/env.md b/docs/env.md index a174115f79..8a68aec915 100644 --- a/docs/env.md +++ b/docs/env.md @@ -291,6 +291,15 @@ that will be used to distribute processed files to end users. - Required: Yes - Default: None +#### DJANGO_SCW_EDGE_SERVICE_DOMAIN + +The domain for the Scaleway Edge Service. +This is the domain that will be used to distribute videos from Scaleway S3 to end users. + +- Type: string +- Required: Yes +- Default: None + ### XMPP settings #### DJANGO_LIVE_CHAT_ENABLED diff --git a/env.d/test b/env.d/test index d7143c7219..689f563454 100644 --- a/env.d/test +++ b/env.d/test @@ -15,6 +15,7 @@ DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN=aws:mediapackage:arn:role # AWS DJANGO_CLOUDFRONT_DOMAIN=abc.cloudfront.net +DJANGO_SCW_EDGE_SERVICE_DOMAIN=abc.svc.edge.scw.cloud DJANGO_UPDATE_STATE_SHARED_SECRETS=dummy,secret # BBB diff --git a/src/backend/marsha/core/serializers/video.py b/src/backend/marsha/core/serializers/video.py index 62b5085268..326af61c15 100644 --- a/src/backend/marsha/core/serializers/video.py +++ b/src/backend/marsha/core/serializers/video.py @@ -207,13 +207,7 @@ def get_vod_urls(self, obj): urls = {"mp4": {}, "thumbnails": {}, "manifests": {}} - base = f"{settings.AWS_S3_URL_PROTOCOL}://{settings.CLOUDFRONT_DOMAIN}/{obj.pk}" stamp = time_utils.to_timestamp(obj.uploaded_on) - if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE: - params = get_video_cloudfront_url_params(obj.pk) - - filename = f"{slugify(obj.playlist.title)}_{stamp}.mp4" - content_disposition = quote_plus(f"attachment; filename={filename}") # Trying to recover the transcoding pipeline if obj.transcode_pipeline is None: @@ -228,6 +222,13 @@ def get_vod_urls(self, obj): ) if obj.transcode_pipeline == AWS_PIPELINE: + base = f"{settings.AWS_S3_URL_PROTOCOL}://{settings.CLOUDFRONT_DOMAIN}/{obj.pk}" + if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE: + params = get_video_cloudfront_url_params(obj.pk) + + filename = f"{slugify(obj.playlist.title)}_{stamp}.mp4" + content_disposition = quote_plus(f"attachment; filename={filename}") + for resolution in obj.resolutions: # MP4 mp4_url = ( @@ -255,6 +256,7 @@ def get_vod_urls(self, obj): # Previews urls["previews"] = f"{base}/previews/{stamp}_100.jpg" + elif obj.transcode_pipeline == PEERTUBE_PIPELINE: base = obj.get_videos_storage_prefix(stamp=stamp) for resolution in obj.resolutions: diff --git a/src/backend/marsha/core/storage/s3.py b/src/backend/marsha/core/storage/s3.py index 40e7a4ff4c..1f812c7b84 100644 --- a/src/backend/marsha/core/storage/s3.py +++ b/src/backend/marsha/core/storage/s3.py @@ -32,7 +32,7 @@ class S3VideoStorage(S3Storage): object_parameters = settings.VIDEOS_STORAGE_S3_OBJECT_PARAMETERS - custom_domain = settings.CLOUDFRONT_DOMAIN + custom_domain = settings.SCW_EDGE_SERVICE_DOMAIN url_protocol = "https:" if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE: diff --git a/src/backend/marsha/core/tests/serializers/test_video.py b/src/backend/marsha/core/tests/serializers/test_video.py index 5ea56e1ce0..f80c2448b1 100644 --- a/src/backend/marsha/core/tests/serializers/test_video.py +++ b/src/backend/marsha/core/tests/serializers/test_video.py @@ -45,7 +45,7 @@ def test_video_serializer_urls_with_aws_pipeline(self): ) @override_settings( - MEDIA_URL="https://abc.cloudfront.net/", + MEDIA_URL="https://abc.svc.edge.scw.cloud/", ) def test_video_serializer_urls_with_peertube_pipeline(self): """The VideoBaseSerializer should return the right URLs.""" @@ -63,20 +63,20 @@ def test_video_serializer_urls_with_peertube_pipeline(self): serializer = VideoBaseSerializer(video) self.assertEqual( - f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg", + f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg", serializer.data["urls"]["thumbnails"][1080], ) self.assertEqual( - f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/master.m3u8?" + f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/master.m3u8?" "v=25301066da0654a2967176ea47c26f6735b2cfcbd206f49fe1379f5103bb7b1f", serializer.data["urls"]["manifests"]["hls"], ) self.assertEqual( - f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg", + f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg", serializer.data["urls"]["previews"], ) self.assertTrue( - f"https://abc.cloudfront.net/vod/{video.pk}/" + f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/" "video/1640995200/1640995200-1080-fragmented.mp4" in serializer.data["urls"]["mp4"][1080] ) @@ -117,7 +117,7 @@ def test_video_serializer_urls_with_no_pipeline(self): ) @override_settings( - MEDIA_URL="https://abc.cloudfront.net/", + MEDIA_URL="https://abc.svc.edge.scw.cloud/", ) def test_video_serializer_urls_with_no_pipeline_recovered_to_peertube(self): """The VideoBaseSerializer should return the right URLs with Peertube pipeline.""" @@ -129,29 +129,32 @@ def test_video_serializer_urls_with_no_pipeline_recovered_to_peertube(self): uploaded_on=date, ) - with mock.patch( - "marsha.core.serializers.video.capture_message" - ) as sentry_capture_message, mock.patch( - "marsha.core.serializers.video.video_storage" - ) as mock_video_storage: + with ( + mock.patch( + "marsha.core.serializers.video.capture_message" + ) as sentry_capture_message, + mock.patch( + "marsha.core.serializers.video.video_storage" + ) as mock_video_storage, + ): mock_video_storage.url = storage_class.video_storage.url mock_video_storage.exists.return_value = True serializer = VideoBaseSerializer(video) self.assertEqual( - f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg", + f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg", serializer.data["urls"]["thumbnails"][1080], ) self.assertEqual( - f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/master.m3u8", + f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/master.m3u8", serializer.data["urls"]["manifests"]["hls"], ) self.assertEqual( - f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg", + f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg", serializer.data["urls"]["previews"], ) self.assertTrue( - f"https://abc.cloudfront.net/vod/{video.pk}/" + f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/" "video/1640995200/1640995200-1080-fragmented.mp4" in serializer.data["urls"]["mp4"][1080] ) diff --git a/src/backend/marsha/settings.py b/src/backend/marsha/settings.py index 1f2b0b70a4..37ab18fdd5 100644 --- a/src/backend/marsha/settings.py +++ b/src/backend/marsha/settings.py @@ -385,6 +385,8 @@ class Base(Configuration): {"ContentDisposition": "attachment"} ) + SCW_EDGE_SERVICE_DOMAIN = values.Value(None) + # LTI Config LTI_CONFIG_TITLE = values.Value("Marsha") LTI_CONFIG_TITLES = values.DictValue( diff --git a/src/tray/templates/services/app/secret.yml.j2 b/src/tray/templates/services/app/secret.yml.j2 index 0ab471a78d..004027b19c 100644 --- a/src/tray/templates/services/app/secret.yml.j2 +++ b/src/tray/templates/services/app/secret.yml.j2 @@ -14,6 +14,7 @@ data: DJANGO_UPDATE_STATE_SHARED_SECRETS: "{{ MARSHA_VAULT.DJANGO_UPDATE_STATE_SHARED_SECRETS | default('secret') | b64encode }}" DJANGO_CLOUDFRONT_ACCESS_KEY_ID: "{{ MARSHA_VAULT.DJANGO_CLOUDFRONT_ACCESS_KEY_ID | default('secret') | b64encode }}" DJANGO_CLOUDFRONT_DOMAIN: "{{ MARSHA_VAULT.DJANGO_CLOUDFRONT_DOMAIN | default('foo.com') | b64encode }}" + DJANGO_SCW_EDGE_SERVICE_DOMAIN: "{{ MARSHA_VAULT.DJANGO_SCW_EDGE_SERVICE_DOMAIN | default('foo.com') | b64encode }}" DJANGO_JWT_SIGNING_KEY: "{{ MARSHA_VAULT.DJANGO_JWT_SIGNING_KEY | default('secret') | b64encode }}" DJANGO_SECRET_KEY: "{{ MARSHA_VAULT.DJANGO_SECRET_KEY | default('supersecret') | b64encode }}" DJANGO_AWS_MEDIALIVE_ROLE_ARN: "{{ MARSHA_VAULT.DJANGO_AWS_MEDIALIVE_ROLE_ARN | default('secret') | b64encode }}"