diff --git a/README.md b/README.md index 1ca7fd5..9663e77 100644 --- a/README.md +++ b/README.md @@ -42,3 +42,13 @@ TripleEyeEffable.configure do |config| end ``` + +## Resource Transfer + +The `triple-eye-effable` gem comes packages with a rake task to assist with managing data. Let's say you want to pull a backup of your application's staging or production environment locally to test. You can easily restore the database, however all of the IIIF resources are still on the staging or production IIIF Cloud instance. + +The following rake task will allow for "pulling" the resources and uploading them to another IIIF Cloud instance (either hosted locally, or somewhere else): + +```shell +bundle exec rake triple_eye_effable:transfer_resources -- --api-key --api-url --project-id +``` diff --git a/app/services/triple_eye_effable/cloud.rb b/app/services/triple_eye_effable/cloud.rb index 0c03b25..8ecbc9d 100644 --- a/app/services/triple_eye_effable/cloud.rb +++ b/app/services/triple_eye_effable/cloud.rb @@ -18,24 +18,30 @@ class Cloud uuid ) - def initialize - @api_key = TripleEyeEffable.config.api_key - @api_url = TripleEyeEffable.config.url - @project_id = TripleEyeEffable.config.project_id + def self.filename(name) + name&.force_encoding(Encoding::ASCII_8BIT) + end + + def initialize(api_key: nil, api_url: nil, project_id: nil, read_only: false) + @api_key = api_key || TripleEyeEffable.config.api_key + @api_url = api_url || TripleEyeEffable.config.url + @project_id = project_id || TripleEyeEffable.config.project_id + @read_only = read_only end def create_resource(resourceable) - response = self.class.post(base_url, body: request_body(resourceable), headers: headers) - add_error(resourceable, response) and return unless response.success? + raise I18n.t('errors.read_only') if @read_only + response = upload_resource(resourceable) resource_id, data = parse_response(response) - resource_description = ResourceDescription.new(resource_id: resource_id) - populate_description resource_description, data - resourceable.resource_description = resource_description + resourceable.resource_description = ResourceDescription.new(resource_id: resource_id) + populate_description resourceable.resource_description, data end def delete_resource(resourceable) + raise I18n.t('errors.read_only') if @read_only + return if resourceable.resource_description.nil? id = resourceable.resource_description.resource_id @@ -47,6 +53,26 @@ def load_resource(resourceable) return if resourceable.resource_description.nil? resource_description = resourceable.resource_description + response = self.class.get("#{base_url}/#{resource_description.resource_id}", headers: headers) + + resource_id, data = parse_response(response) + populate_description resource_description, data unless data.nil? + end + + def update_resource(resourceable) + raise I18n.t('errors.read_only') if @read_only + + resource_description = resourceable.resource_description + id = resource_description.resource_id + + response = self.class.put("#{base_url}/#{id}", body: request_body(resourceable), headers: headers) + resource_id, data = parse_response(response) + populate_description(resource_description, data) + end + + def upload_resource(resourceable) + raise I18n.t('errors.read_only') if @read_only + response = self.class.get("#{base_url}/#{resource_description.resource_id}", headers: headers) add_error(resourceable, response) and return unless response.success? @@ -83,18 +109,18 @@ def parse_response(response) end def populate_description(resource_description, data) - data&.keys&.each do |key| + RESPONSE_KEYS.each do |key| next unless resource_description.respond_to?("#{key.to_s}=") resource_description.send("#{key.to_s}=", data[key]) end end def request_body(resourceable) - name = resourceable.name.force_encoding(Encoding::ASCII_8BIT) if resourceable.respond_to?(:name) + name = self.class.filename(resourceable.name) if resourceable.respond_to?(:name) content = resourceable.content if resourceable.respond_to?(:content) metadata = resourceable.metadata if resourceable.respond_to?(:metadata) - body = { + body = { resource: { project_id: @project_id, name: name, diff --git a/config/locales/en.yml b/config/locales/en.yml new file mode 100644 index 0000000..7a8383c --- /dev/null +++ b/config/locales/en.yml @@ -0,0 +1,3 @@ +en: + errors: + read_only: 'This method is not allowed in read-only mode.' diff --git a/lib/tasks/triple_eye_effable_tasks.rake b/lib/tasks/triple_eye_effable_tasks.rake index d6a750d..81255ea 100644 --- a/lib/tasks/triple_eye_effable_tasks.rake +++ b/lib/tasks/triple_eye_effable_tasks.rake @@ -1,4 +1,122 @@ -# desc "Explaining what the task does" -# task :triple_eye_effable do -# # Task goes here -# end +require 'httparty' +require 'optparse' + +namespace :triple_eye_effable do + + desc 'Transfer resources from one IIIF Cloud instance to another.' + task :transfer_resources => :environment do + # Parse the arguments + options = {} + + opt_parser = OptionParser.new do |opts| + opts.banner = 'Usage: rake triple_eye_effable:transfer_resources [options]' + + opts.on('--api-key api_key', 'IIIF Cloud API Key') do |api_key| + options[:api_key] = api_key + end + + opts.on('--api-url api_url', 'IIIF Cloud API URL') do |api_url| + options[:api_url] = api_url + end + + opts.on('--project-id project_id', 'IIIF Cloud Project ID') do |project_id| + options[:project_id] = project_id + end + end + + args = opt_parser.order!(ARGV) {} + opt_parser.parse!(args) + + if options[:api_key].blank? + puts 'Please specify an API key...' + exit 0 + end + + if options[:api_url].blank? + puts 'Please specify an API URL...' + exit 0 + end + + if options[:project_id].blank? + puts 'Please specify a project ID...' + exit 0 + end + + # Build the list of classes that include the Resourcable concern + classes = [] + + ActiveRecord::Base.connection.tables.each do |table_name| + begin + klass = table_name.classify.constantize + next unless klass.ancestors.include?(TripleEyeEffable::Resourceable) + + classes << klass + rescue + # Skip the record, there's a chance a table exists with no model + end + end + + source_service = TripleEyeEffable::Cloud.new( + api_key: options[:api_key], + api_url: options[:api_url], + project_id: options[:project_id], + read_only: true + ) + + destination_service = TripleEyeEffable::Cloud.new + + classes.each do |klass| + count = klass.count + + klass.find_each.with_index do |resourceable, index| + # Load the resource from the source application + source_service.load_resource(resourceable) + + # Download the binary content + resourceable.content = download_content(resourceable) + + # Upload the resource to the destination application + response = destination_service.upload_resource(resourceable) + + # Set the new resource ID on the resource_description + resource_id = response['resource']['uuid'] + resourceable.resource_description.update(resource_id: resource_id) + + # Log the progress + log_progress(klass, index, count) + end + + log_progress(klass, count, count, true) + end + end + + # Downloads the content for the passed resourceable record + def download_content(resourceable) + begin + response = HTTParty.get(resourceable.content_url) + + file = Tempfile.new + file.binmode + file.write(response.body) + file.rewind + + content = ActionDispatch::Http::UploadedFile.new( + tempfile: file, + type: resourceable.content_type, + filename: TripleEyeEffable::Cloud.filename(resourceable.name) + ) + rescue + puts "Error downloading file #{resourceable.id}" + content = nil + end + + content + end + + # Logs the progress for the passed class + def log_progress(klass, index, count, force = false) + return unless force || index + 1 % count != 10 + + puts "#{klass.to_s}: Uploaded #{index + 1} out of #{count} records..." + end +end diff --git a/lib/triple_eye_effable/engine.rb b/lib/triple_eye_effable/engine.rb index 0dc590a..3c3b600 100644 --- a/lib/triple_eye_effable/engine.rb +++ b/lib/triple_eye_effable/engine.rb @@ -1,5 +1,9 @@ module TripleEyeEffable class Engine < ::Rails::Engine isolate_namespace TripleEyeEffable + + config.before_initialize do + config.i18n.load_path += Dir["#{config.root}/config/locales/**/*.yml"] + end end end