Merge pull request #8 from performant-software/feature/udcsl175_data_…

…transfer

UDCSL #175 - Data transfer

README.md

@@ -42,3 +42,13 @@ TripleEyeEffable.configure do |config|
 end
 
 ```
+
+## Resource Transfer
+
+The `triple-eye-effable` gem comes packages with a rake task to assist with managing data. Let's say you want to pull a backup of your application's staging or production environment locally to test. You can easily restore the database, however all of the IIIF resources are still on the staging or production IIIF Cloud instance.
+
+The following rake task will allow for "pulling" the resources and uploading them to another IIIF Cloud instance (either hosted locally, or somewhere else):
+
+```shell
+bundle exec rake triple_eye_effable:transfer_resources -- --api-key <YOUR_API_KEY> --api-url <SOURCE_IIIF_CLOUD_URL> --project-id <YOUR_PROJECT_ID>
+```

app/services/triple_eye_effable/cloud.rb

@@ -18,24 +18,30 @@ class Cloud
       uuid
     )
 
-    def initialize
-      @api_key = TripleEyeEffable.config.api_key
-      @api_url = TripleEyeEffable.config.url
-      @project_id = TripleEyeEffable.config.project_id
+    def self.filename(name)
+      name&.force_encoding(Encoding::ASCII_8BIT)
+    end
+
+    def initialize(api_key: nil, api_url: nil, project_id: nil, read_only: false)
+      @api_key = api_key || TripleEyeEffable.config.api_key
+      @api_url = api_url || TripleEyeEffable.config.url
+      @project_id = project_id || TripleEyeEffable.config.project_id
+      @read_only = read_only
     end
 
     def create_resource(resourceable)
-      response = self.class.post(base_url, body: request_body(resourceable), headers: headers)
-      add_error(resourceable, response) and return unless response.success?
+      raise I18n.t('errors.read_only') if @read_only
 
+      response = upload_resource(resourceable)
       resource_id, data = parse_response(response)
-      resource_description = ResourceDescription.new(resource_id: resource_id)
 
-      populate_description resource_description, data
-      resourceable.resource_description = resource_description
+      resourceable.resource_description = ResourceDescription.new(resource_id: resource_id)
+      populate_description resourceable.resource_description, data
     end
 
     def delete_resource(resourceable)
+      raise I18n.t('errors.read_only') if @read_only
+
       return if resourceable.resource_description.nil?
 
       id = resourceable.resource_description.resource_id
@@ -47,6 +53,26 @@ def load_resource(resourceable)
       return if resourceable.resource_description.nil?
 
       resource_description = resourceable.resource_description
+      response = self.class.get("#{base_url}/#{resource_description.resource_id}", headers: headers)
+
+      resource_id, data = parse_response(response)
+      populate_description resource_description, data unless data.nil?
+    end
+
+    def update_resource(resourceable)
+      raise I18n.t('errors.read_only') if @read_only
+
+      resource_description = resourceable.resource_description
+      id = resource_description.resource_id
+
+      response = self.class.put("#{base_url}/#{id}", body: request_body(resourceable), headers: headers)
+      resource_id, data = parse_response(response)
+      populate_description(resource_description, data)
+    end
+
+    def upload_resource(resourceable)
+      raise I18n.t('errors.read_only') if @read_only
+
       response = self.class.get("#{base_url}/#{resource_description.resource_id}", headers: headers)
       add_error(resourceable, response) and return unless response.success?
 
@@ -83,18 +109,18 @@ def parse_response(response)
     end
 
     def populate_description(resource_description, data)
-      data&.keys&.each do |key|
+      RESPONSE_KEYS.each do |key|
         next unless resource_description.respond_to?("#{key.to_s}=")
         resource_description.send("#{key.to_s}=", data[key])
       end
     end
 
     def request_body(resourceable)
-      name = resourceable.name.force_encoding(Encoding::ASCII_8BIT) if resourceable.respond_to?(:name)
+      name = self.class.filename(resourceable.name) if resourceable.respond_to?(:name)
       content = resourceable.content if resourceable.respond_to?(:content)
       metadata = resourceable.metadata if resourceable.respond_to?(:metadata)
 
-      body =       {
+      body = {
         resource: {
           project_id: @project_id,
           name: name,

config/locales/en.yml

@@ -0,0 +1,3 @@
+en:
+  errors:
+    read_only: 'This method is not allowed in read-only mode.'

lib/tasks/triple_eye_effable_tasks.rake

@@ -1,4 +1,122 @@
-# desc "Explaining what the task does"
-# task :triple_eye_effable do
-#   # Task goes here
-# end
+require 'httparty'
+require 'optparse'
+
+namespace :triple_eye_effable do
+
+  desc 'Transfer resources from one IIIF Cloud instance to another.'
+  task :transfer_resources => :environment do
+    # Parse the arguments
+    options = {}
+
+    opt_parser = OptionParser.new do |opts|
+      opts.banner = 'Usage: rake triple_eye_effable:transfer_resources [options]'
+
+      opts.on('--api-key api_key', 'IIIF Cloud API Key') do |api_key|
+        options[:api_key] = api_key
+      end
+
+      opts.on('--api-url api_url', 'IIIF Cloud API URL') do |api_url|
+        options[:api_url] = api_url
+      end
+
+      opts.on('--project-id project_id', 'IIIF Cloud Project ID') do |project_id|
+        options[:project_id] = project_id
+      end
+    end
+
+    args = opt_parser.order!(ARGV) {}
+    opt_parser.parse!(args)
+
+    if options[:api_key].blank?
+      puts 'Please specify an API key...'
+      exit 0
+    end
+
+    if options[:api_url].blank?
+      puts 'Please specify an API URL...'
+      exit 0
+    end
+
+    if options[:project_id].blank?
+      puts 'Please specify a project ID...'
+      exit 0
+    end
+
+    # Build the list of classes that include the Resourcable concern
+    classes = []
+
+    ActiveRecord::Base.connection.tables.each do |table_name|
+      begin
+        klass = table_name.classify.constantize
+        next unless klass.ancestors.include?(TripleEyeEffable::Resourceable)
+
+        classes << klass
+      rescue
+        # Skip the record, there's a chance a table exists with no model
+      end
+    end
+
+    source_service = TripleEyeEffable::Cloud.new(
+      api_key: options[:api_key],
+      api_url: options[:api_url],
+      project_id: options[:project_id],
+      read_only: true
+    )
+
+    destination_service = TripleEyeEffable::Cloud.new
+
+    classes.each do |klass|
+      count = klass.count
+
+      klass.find_each.with_index do |resourceable, index|
+        # Load the resource from the source application
+        source_service.load_resource(resourceable)
+
+        # Download the binary content
+        resourceable.content = download_content(resourceable)
+
+        # Upload the resource to the destination application
+        response = destination_service.upload_resource(resourceable)
+
+        # Set the new resource ID on the resource_description
+        resource_id = response['resource']['uuid']
+        resourceable.resource_description.update(resource_id: resource_id)
+
+        # Log the progress
+        log_progress(klass, index, count)
+      end
+
+      log_progress(klass, count, count, true)
+    end
+  end
+
+  # Downloads the content for the passed resourceable record
+  def download_content(resourceable)
+    begin
+      response = HTTParty.get(resourceable.content_url)
+
+      file = Tempfile.new
+      file.binmode
+      file.write(response.body)
+      file.rewind
+
+      content = ActionDispatch::Http::UploadedFile.new(
+        tempfile: file,
+        type: resourceable.content_type,
+        filename: TripleEyeEffable::Cloud.filename(resourceable.name)
+      )
+    rescue
+      puts "Error downloading file #{resourceable.id}"
+      content = nil
+    end
+
+    content
+  end
+
+  # Logs the progress for the passed class
+  def log_progress(klass, index, count, force = false)
+    return unless force || index + 1 % count != 10
+
+    puts "#{klass.to_s}: Uploaded #{index + 1} out of #{count} records..."
+  end
+end

lib/triple_eye_effable/engine.rb

@@ -1,5 +1,9 @@
 module TripleEyeEffable
   class Engine < ::Rails::Engine
     isolate_namespace TripleEyeEffable
+
+    config.before_initialize do
+      config.i18n.load_path += Dir["#{config.root}/config/locales/**/*.yml"]
+    end
   end
 end