From bf73c5d4dfbad34dfc6b5b325beb5c8022ef723e Mon Sep 17 00:00:00 2001 From: Rory McNicholl Date: Thu, 13 Jul 2023 21:16:15 +0100 Subject: [PATCH 1/6] config for be-s3 --- config/browse_everything_providers.yml | 20 ++++++++++---------- ops/staging-deploy.tmpl.yaml | 8 ++++++++ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/config/browse_everything_providers.yml b/config/browse_everything_providers.yml index 9fefd0a8c..c00f90677 100644 --- a/config/browse_everything_providers.yml +++ b/config/browse_everything_providers.yml @@ -2,8 +2,8 @@ # To make browse-everything aware of a provider, uncomment the info for that provider and add your API key information. # The file_system provider can be a path to any directory on the server where your application is running. # -file_system: - home: /app/samvera/hyrax-webapp/data +#file_system: +# home: /app/samvera/hyrax-webapp/data # dropbox: # client_id: YOUR_DROPBOX_APP_KEY # client_secret: YOUR_DROPBOX_APP_SECRET @@ -14,11 +14,11 @@ file_system: # google_drive: # client_id: YOUR_GOOGLE_API_CLIENT_ID # client_secret: YOUR_GOOGLE_API_CLIENT_SECRET -# s3: -# bucket: YOUR_AWS_S3_BUCKET -# response_type: signed_url # set to :public_url for public urls or :s3_uri for an s3://BUCKET/KEY uri -# expires_in: 14400 # for signed_url response_type, number of seconds url will be valid for. -# app_key: YOUR_AWS_S3_KEY # :app_key, :app_secret, and :region can be specified -# app_secret: YOUR_AWS_S3_SECRET # explicitly here, or left out to use system-configured -# region: YOUR_AWS_S3_REGION # defaults. -# See https://aws.amazon.com/blogs/security/a-new-and-standardized-way-to-manage-credentials-in-the-aws-sdks/ +s3: + bucket: <%= ENV['AWS_S3_BUCKET'] %> # .s3.amazonaws.com #arn:aws:s3:::temp-bl-bucket-for-browse-everything + response_type: signed_url # set to :public_url for public urls or :s3_uri for an s3://BUCKET/KEY uri + expires_in: 14400 # for signed_url response_type, number of seconds url will be valid for. + app_key: <%= ENV['AWS_ACCESS_KEY_ID'] %> # :app_key, :app_secret, and :region can be specified + app_secret: <%= ENV['AWS_SECRET_ACCESS_KEY'] %> # explicitly here, or left out to use system-configured + region: <%= ENV['AWS_S3_REGION'] %> # defaults. + # See https://aws.amazon.com/blogs/security/a-new-and-standardized-way-to-manage-credentials-in-the-aws-sdks/ diff --git a/ops/staging-deploy.tmpl.yaml b/ops/staging-deploy.tmpl.yaml index 588dcd992..73f1dd32e 100644 --- a/ops/staging-deploy.tmpl.yaml +++ b/ops/staging-deploy.tmpl.yaml @@ -167,6 +167,14 @@ extraEnvVars: &envVars value: "1" - name: EXTERNAL_IIIF_URL value: https://d2ltm8fy0v0agc.cloudfront.net/iiif/2 + - name: AWS_S3_BUCKET + value: $AWS_S3_BUCKET + - name: AWS_ACCESS_KEY_ID + value: $AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + value: $AWS_SECRET_ACCESS_KEY + - name: AWS_S3_REGION + value: $AWS_S3_REGION worker: replicaCount: 1 From 27085f0835c74b0d1dc13d501f64fb84ccf65449 Mon Sep 17 00:00:00 2001 From: Rory McNicholl Date: Mon, 17 Jul 2023 09:16:48 +0100 Subject: [PATCH 2/6] set bucket name and region direct in deploy yaml --- ops/staging-deploy.tmpl.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ops/staging-deploy.tmpl.yaml b/ops/staging-deploy.tmpl.yaml index 73f1dd32e..236cde5f7 100644 --- a/ops/staging-deploy.tmpl.yaml +++ b/ops/staging-deploy.tmpl.yaml @@ -168,13 +168,13 @@ extraEnvVars: &envVars - name: EXTERNAL_IIIF_URL value: https://d2ltm8fy0v0agc.cloudfront.net/iiif/2 - name: AWS_S3_BUCKET - value: $AWS_S3_BUCKET + value: temp-bl-bucket-for-browse-everything - name: AWS_ACCESS_KEY_ID value: $AWS_ACCESS_KEY_ID - name: AWS_SECRET_ACCESS_KEY value: $AWS_SECRET_ACCESS_KEY - name: AWS_S3_REGION - value: $AWS_S3_REGION + value: eu-west-1 worker: replicaCount: 1 From e43fc65bd98e4d0c0dbdc10656b981514dc3bf1e Mon Sep 17 00:00:00 2001 From: Rory McNicholl Date: Wed, 20 Mar 2024 10:41:01 +0000 Subject: [PATCH 3/6] use existing iam user so no need for these here --- ops/staging-deploy.tmpl.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ops/staging-deploy.tmpl.yaml b/ops/staging-deploy.tmpl.yaml index 5bf9c96da..5f57bcf7d 100644 --- a/ops/staging-deploy.tmpl.yaml +++ b/ops/staging-deploy.tmpl.yaml @@ -181,14 +181,6 @@ extraEnvVars: &envVars value: "1" - name: EXTERNAL_IIIF_URL value: https://d2ltm8fy0v0agc.cloudfront.net/iiif/2 - - name: AWS_S3_BUCKET - value: temp-bl-bucket-for-browse-everything - - name: AWS_ACCESS_KEY_ID - value: $AWS_ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - value: $AWS_SECRET_ACCESS_KEY - - name: AWS_S3_REGION - value: eu-west-1 worker: replicaCount: 1 From ad5354df3622df26d6cd89164126687bc7ee8495 Mon Sep 17 00:00:00 2001 From: Rory McNicholl Date: Wed, 20 Mar 2024 10:44:05 +0000 Subject: [PATCH 4/6] use existing defaults for s3 access --- config/browse_everything_providers.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/browse_everything_providers.yml b/config/browse_everything_providers.yml index c00f90677..9cb730ba6 100644 --- a/config/browse_everything_providers.yml +++ b/config/browse_everything_providers.yml @@ -2,8 +2,8 @@ # To make browse-everything aware of a provider, uncomment the info for that provider and add your API key information. # The file_system provider can be a path to any directory on the server where your application is running. # -#file_system: -# home: /app/samvera/hyrax-webapp/data +file_system: + home: /app/samvera/hyrax-webapp/data # dropbox: # client_id: YOUR_DROPBOX_APP_KEY # client_secret: YOUR_DROPBOX_APP_SECRET @@ -15,10 +15,10 @@ # client_id: YOUR_GOOGLE_API_CLIENT_ID # client_secret: YOUR_GOOGLE_API_CLIENT_SECRET s3: - bucket: <%= ENV['AWS_S3_BUCKET'] %> # .s3.amazonaws.com #arn:aws:s3:::temp-bl-bucket-for-browse-everything + bucket: temp-bl-bucket-for-browse-everything # .s3.amazonaws.com #arn:aws:s3:::temp-bl-bucket-for-browse-everything response_type: signed_url # set to :public_url for public urls or :s3_uri for an s3://BUCKET/KEY uri expires_in: 14400 # for signed_url response_type, number of seconds url will be valid for. app_key: <%= ENV['AWS_ACCESS_KEY_ID'] %> # :app_key, :app_secret, and :region can be specified app_secret: <%= ENV['AWS_SECRET_ACCESS_KEY'] %> # explicitly here, or left out to use system-configured - region: <%= ENV['AWS_S3_REGION'] %> # defaults. + region: eu-west-1 # defaults. # See https://aws.amazon.com/blogs/security/a-new-and-standardized-way-to-manage-credentials-in-the-aws-sdks/ From d888ca9f1ba279875ede1bac842f13d9be20f3aa Mon Sep 17 00:00:00 2001 From: Kirk Wang Date: Mon, 25 Mar 2024 13:13:57 -0700 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=90=9B=20Override=20BrowseEverything?= =?UTF-8?q?=20for=20S3=20file=20size?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit will bring in an override for the BrowseEverything gem to retrieve the file size from an S3 object and pass it into the params so that the Bulkrax importer can use it. --- config/application.rb | 4 +++ lib/browse_everything/driver/s3_decorator.rb | 31 ++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 lib/browse_everything/driver/s3_decorator.rb diff --git a/config/application.rb b/config/application.rb index 81e96553a..aec03e850 100644 --- a/config/application.rb +++ b/config/application.rb @@ -36,6 +36,10 @@ class Application < Rails::Application Dir.glob(File.join(File.dirname(__FILE__), "../app/**/*_decorator*.rb")).sort.each do |c| Rails.configuration.cache_classes ? require(c) : load(c) end + + Dir.glob(File.join(File.dirname(__FILE__), "../lib/**/*_decorator*.rb")).sort.each do |c| + Rails.configuration.cache_classes ? require(c) : load(c) + end end # resolve reloading issue in dev mode diff --git a/lib/browse_everything/driver/s3_decorator.rb b/lib/browse_everything/driver/s3_decorator.rb new file mode 100644 index 000000000..9f73b348b --- /dev/null +++ b/lib/browse_everything/driver/s3_decorator.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +# OVERRIDE BrowseEverything v1.1.2 to add file_size for S3 files + +module BrowseEverything + module Driver + module S3Decorator + def link_for(path) + obj = bucket.object(full_path(path)) + obj_head = obj.head + file_size = obj_head.content_length + + extras = { + file_name: File.basename(path), + file_size: file_size, + expires: (config[:expires_in] if config[:response_type] == :signed_url) + }.compact + + url = case config[:response_type].to_sym + when :signed_url then obj.presigned_url(:get, expires_in: config[:expires_in]) + when :public_url then obj.public_url + when :s3_uri then "s3://#{obj.bucket_name}/#{obj.key}" + end + + [url, extras] + end + end + end +end + +BrowseEverything::Driver::S3.prepend(BrowseEverything::Driver::S3Decorator) From 6c2aa934d5554832dcff50f5d9b0ad1f29bea2fa Mon Sep 17 00:00:00 2001 From: Rory McNicholl Date: Wed, 17 Apr 2024 11:33:03 +0100 Subject: [PATCH 6/6] remove file-system option from BE providers --- config/browse_everything_providers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/browse_everything_providers.yml b/config/browse_everything_providers.yml index 9cb730ba6..f6e818cd3 100644 --- a/config/browse_everything_providers.yml +++ b/config/browse_everything_providers.yml @@ -2,8 +2,8 @@ # To make browse-everything aware of a provider, uncomment the info for that provider and add your API key information. # The file_system provider can be a path to any directory on the server where your application is running. # -file_system: - home: /app/samvera/hyrax-webapp/data +#file_system: +# home: /app/samvera/hyrax-webapp/data # dropbox: # client_id: YOUR_DROPBOX_APP_KEY # client_secret: YOUR_DROPBOX_APP_SECRET