From ea3ae588152cccdabae65b8d7effc12502953c74 Mon Sep 17 00:00:00 2001 From: David Kinzer Date: Wed, 26 Jun 2019 17:03:04 -0400 Subject: [PATCH] Initial commit. --- .coveralls.yml | 1 + .gitignore | 11 +++ .rspec | 3 + .rubocop.yml | 139 +++++++++++++++++++++++++++ .travis.yml | 11 +++ CODE_OF_CONDUCT.md | 74 +++++++++++++++ Gemfile | 8 ++ Gemfile.lock | 147 +++++++++++++++++++++++++++++ LICENSE.txt | 21 +++++ README.md | 53 +++++++++++ Rakefile | 8 ++ bin/console | 14 +++ bin/ingest-libguides.sh | 5 + bin/libguide_cache.rb | 25 +++++ bin/setup | 8 ++ cob_az_index.gemspec | 51 ++++++++++ exe/cob_az_index | 52 ++++++++++ lib/cob_az_index.rb | 32 +++++++ lib/cob_az_index/indexer_config.rb | 93 ++++++++++++++++++ lib/cob_az_index/macros.rb | 10 ++ lib/cob_az_index/version.rb | 5 + spec/cob_az_index_spec.rb | 7 ++ spec/spec_helper.rb | 19 ++++ 23 files changed, 797 insertions(+) create mode 100644 .coveralls.yml create mode 100644 .gitignore create mode 100644 .rspec create mode 100644 .rubocop.yml create mode 100644 .travis.yml create mode 100644 CODE_OF_CONDUCT.md create mode 100644 Gemfile create mode 100644 Gemfile.lock create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 Rakefile create mode 100755 bin/console create mode 100755 bin/ingest-libguides.sh create mode 100755 bin/libguide_cache.rb create mode 100755 bin/setup create mode 100644 cob_az_index.gemspec create mode 100755 exe/cob_az_index create mode 100644 lib/cob_az_index.rb create mode 100644 lib/cob_az_index/indexer_config.rb create mode 100644 lib/cob_az_index/macros.rb create mode 100644 lib/cob_az_index/version.rb create mode 100644 spec/cob_az_index_spec.rb create mode 100644 spec/spec_helper.rb diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 0000000..9160059 --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1 @@ +service_name: travis-ci diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b04a8c8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ + +# rspec failure tracking +.rspec_status diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..34c5164 --- /dev/null +++ b/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..1dfc303 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,139 @@ +AllCops: + TargetRubyVersion: 2.4 + # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop + # to ignore them, so only the ones explicitly set in this file are enabled. + DisabledByDefault: true + Exclude: + - 'node_modules/**/*' + - '**/templates/**/*' + - '**/vendor/**/*' + - 'actionpack/lib/action_dispatch/journey/parser.rb' + - 'bin/*' + +# Prefer &&/|| over and/or. +Style/AndOr: + Enabled: true + +# Do not use braces for hash literals when they are the last argument of a +# method call. +Style/BracesAroundHashParameters: + Enabled: true + EnforcedStyle: context_dependent + +# Align `when` with `case`. +Layout/CaseIndentation: + Enabled: true + +# Align comments with method definitions. +Layout/CommentIndentation: + Enabled: true + +Layout/EmptyLineAfterMagicComment: + Enabled: true + +# In a regular class definition, no empty lines around the body. +Layout/EmptyLinesAroundClassBody: + Enabled: true + +# In a regular method definition, no empty lines around the body. +Layout/EmptyLinesAroundMethodBody: + Enabled: true + +# In a regular module definition, no empty lines around the body. +Layout/EmptyLinesAroundModuleBody: + Enabled: true + +Layout/IndentFirstArgument: + Enabled: true + +# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }. +Style/HashSyntax: + Enabled: true + +# Method definitions after `private` or `protected` isolated calls need one +# extra level of indentation. +Layout/IndentationConsistency: + Enabled: true + EnforcedStyle: rails + +# Two spaces, no tabs (for indentation). +Layout/IndentationWidth: + Enabled: true + +Layout/SpaceAfterColon: + Enabled: true + +Layout/SpaceAfterComma: + Enabled: true + +Layout/SpaceAroundEqualsInParameterDefault: + Enabled: true + +Layout/SpaceAroundKeyword: + Enabled: true + +Layout/SpaceAroundOperators: + Enabled: true + +Layout/SpaceBeforeFirstArg: + Enabled: true + +# Defining a method with parameters needs parentheses. +Style/MethodDefParentheses: + Enabled: true + +Style/FrozenStringLiteralComment: + Enabled: true + EnforcedStyle: always + Exclude: + - 'app/models/traject_indexer.rb' + - 'db/schema.rb' + +# Use `foo {}` not `foo{}`. +Layout/SpaceBeforeBlockBraces: + Enabled: true + +# Use `foo { bar }` not `foo {bar}`. +Layout/SpaceInsideBlockBraces: + Enabled: true + +# Use `{ a: 1 }` not `{a:1}`. +Layout/SpaceInsideHashLiteralBraces: + Enabled: true + +Layout/SpaceInsideParens: + Enabled: true + +# Check quotes usage according to lint rule below. +Style/StringLiterals: + Enabled: true + EnforcedStyle: double_quotes + +# Detect hard tabs, no hard tabs. +Layout/Tab: + Enabled: true + +# Blank lines should not have any spaces. +Layout/TrailingBlankLines: + Enabled: true + +# No trailing whitespace. +Layout/TrailingWhitespace: + Enabled: true + +# Use quotes for string literals when they are enough. +Style/UnneededPercentQ: + Enabled: true + +# Align `end` with the matching keyword or starting expression except for +# assignments, where it should be aligned with the LHS. +Layout/EndAlignment: + Enabled: true + EnforcedStyleAlignWith: variable + +# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg. +Lint/RequireParentheses: + Enabled: true + +Rails: + Enabled: true diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c2d9497 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +--- +sudo: false +language: ruby +cache: bundler +rvm: + - 2.4.1 +before_install: gem install bundler -v 1.16.6 + +script: + - bundle exec rake + - bundle exec rubocop diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..42ecd05 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at dtkinzer@gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..4b7bcf0 --- /dev/null +++ b/Gemfile @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } + +# Specify your gem's dependencies in cob_az_index.gemspec +gemspec diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..e09eff4 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,147 @@ +PATH + remote: . + specs: + cob_az_index (0.1.0) + gli (= 2.18.0) + httparty + traject (~> 3.1) + traject_plus (~> 1.1) + +GEM + remote: https://rubygems.org/ + specs: + activesupport (5.2.3) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + addressable (2.6.0) + public_suffix (>= 2.0.2, < 4.0) + ast (2.4.0) + concurrent-ruby (1.1.5) + coveralls (0.8.23) + json (>= 1.8, < 3) + simplecov (~> 0.16.1) + term-ansicolor (~> 1.3) + thor (>= 0.19.4, < 2.0) + tins (~> 1.6) + deprecation (1.0.0) + activesupport + diff-lcs (1.3) + docile (1.3.1) + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + dot-properties (0.1.3) + gli (2.18.0) + hashie (3.6.0) + http (3.3.0) + addressable (~> 2.3) + http-cookie (~> 1.0) + http-form_data (~> 2.0) + http_parser.rb (~> 0.6.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http-form_data (2.1.1) + http_parser.rb (0.6.0) + httparty (0.17.0) + mime-types (~> 3.0) + multi_xml (>= 0.5.2) + httpclient (2.8.3) + i18n (1.6.0) + concurrent-ruby (~> 1.0) + jaro_winkler (1.5.2) + json (2.2.0) + jsonpath (1.0.4) + multi_json + to_regexp (~> 0.2.1) + marc (1.0.3) + scrub_rb (>= 1.0.1, < 2) + unf + marc-fastxmlwriter (1.0.0) + marc (~> 1.0) + mime-types (3.2.2) + mime-types-data (~> 3.2015) + mime-types-data (3.2019.0331) + mini_portile2 (2.4.0) + minitest (5.11.3) + multi_json (1.13.1) + multi_xml (0.6.0) + nokogiri (1.10.3) + mini_portile2 (~> 2.4.0) + parallel (1.17.0) + parser (2.6.3.0) + ast (~> 2.4.0) + public_suffix (3.1.1) + rainbow (3.0.0) + rake (10.5.0) + rspec (3.8.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-core (3.8.1) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.4) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-mocks (3.8.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-support (3.8.2) + rubocop (0.71.0) + jaro_winkler (~> 1.5.1) + parallel (~> 1.10) + parser (>= 2.6) + rainbow (>= 2.2.2, < 4.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 1.4.0, < 1.7) + ruby-progressbar (1.10.1) + scrub_rb (1.0.1) + simplecov (0.16.1) + docile (~> 1.1) + json (>= 1.8, < 3) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.2) + slop (3.6.0) + term-ansicolor (1.7.1) + tins (~> 1.0) + thor (0.20.3) + thread_safe (0.3.6) + tins (1.20.2) + to_regexp (0.2.1) + traject (3.1.0) + concurrent-ruby (>= 0.8.0) + dot-properties (>= 0.1.1) + hashie (~> 3.1) + http (~> 3.0) + httpclient (~> 2.5) + marc (~> 1.0) + marc-fastxmlwriter (~> 1.0) + nokogiri (~> 1.9) + slop (>= 3.4.5, < 4.0) + yell + traject_plus (1.1.1) + activesupport + deprecation + jsonpath + traject (~> 3.0) + tzinfo (1.2.5) + thread_safe (~> 0.1) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.6) + unicode-display_width (1.6.0) + yell (2.2.0) + +PLATFORMS + ruby + +DEPENDENCIES + bundler (~> 1.16) + cob_az_index! + coveralls + rake (~> 10.0) + rspec (~> 3.0) + rubocop (~> 0.52) + +BUNDLED WITH + 1.16.6 diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..4cb6750 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2019 David Kinzer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f62b7a0 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# CobAzIndex +[![Build Status](https://travis-ci.org/tulibraries/cob_az_index.svg?branch=master)](https://travis-ci.org/tulibraries/cob_az_index) +[![Coverage Status](https://coveralls.io/repos/github/tulibraries/cob_az_index/badge.svg?branch=master)](https://coveralls.io/github/tulibraries/cob_az_index?branch=master) + +Cob Index is a repository to hold the traject configuration files and scripts +associated with indexing of the tul_cob books catalog. + +## Installation + +Add this line to your application's Gemfile: + +```ruby +gem 'cob_az_index' +``` + +And then execute: + + $ bundle + +Or install it yourself as: + + $ gem install cob_az_index + +## Usage + +`cob_az_index` is an executable. You can use it to ingest files into SOLR_URL with + +``` +cob_az_index ingest $path_to_file +``` + +`$path_to_file` can also be a URL. + +Note also that `$path_to_file` is a an optional arugment. If not provided then by default ingest will attempt to download a copy of the az db and import that file. + + +## Development + +After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. + +To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). + +## Contributing + +Bug reports and pull requests are welcome on GitHub at https://github.com/tulibraries/cob_az_index. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct. + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). + +## Code of Conduct + +Everyone interacting in the CobAzIndex project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/tulibraries/cob_az_index/blob/master/CODE_OF_CONDUCT.md). diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..b6ae734 --- /dev/null +++ b/Rakefile @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require "bundler/gem_tasks" +require "rspec/core/rake_task" + +RSpec::Core::RakeTask.new(:spec) + +task default: :spec diff --git a/bin/console b/bin/console new file mode 100755 index 0000000..bc0e47b --- /dev/null +++ b/bin/console @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby + +require "bundler/setup" +require "cob_az_index" + +# You can add fixtures and/or initialization code here to make experimenting +# with your gem easier. You can also use a different console, if you like. + +# (If you use this, don't forget to add pry to your Gemfile!) +# require "pry" +# Pry.start + +require "irb" +IRB.start(__FILE__) diff --git a/bin/ingest-libguides.sh b/bin/ingest-libguides.sh new file mode 100755 index 0000000..c9bec3a --- /dev/null +++ b/bin/ingest-libguides.sh @@ -0,0 +1,5 @@ +#! /usr/bin/env bash + +SOLR_URL=$SOLR_AZ_URL SOLR_DISABLE_UPDATE_DATE_CHECK=yes traject -c lib/traject/databases_az_indexer_config.rb tmp/cache/databases.json + +curl $SOLR_AZ_URL/update?commit=true diff --git a/bin/libguide_cache.rb b/bin/libguide_cache.rb new file mode 100755 index 0000000..90bf163 --- /dev/null +++ b/bin/libguide_cache.rb @@ -0,0 +1,25 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# +# A script for retrieving a local repository of libguide databases. +# + +require "httparty" +require "fileutils" + +client_id = ENV["AZ_CLIENT_ID"] +client_secret = ENV["AZ_CLIENT_SECRET"] +endpoint = "https://lgapi-us.libapps.com/1.2/oauth/token" + +response = HTTParty.post(endpoint, body: { client_id: client_id, client_secret: client_secret, grant_type: "client_credentials" }) +cred = JSON.parse(response) + +endpoint = "https://lgapi-us.libapps.com/1.2/az" +token = cred["access_token"] +response = HTTParty.get(endpoint, headers: { Authorization: "Bearer #{token}" }, query: { expand: "subjects,icons,friendly_url,az_types,az_props,permitted_uses" }) + +FileUtils.mkdir_p "tmp/cache" +File.open "tmp/cache/databases.json", "w+" do |file| + file.write(response.body) +end diff --git a/bin/setup b/bin/setup new file mode 100755 index 0000000..dce67d8 --- /dev/null +++ b/bin/setup @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +set -vx + +bundle install + +# Do any other automated setup that you need to do here diff --git a/cob_az_index.gemspec b/cob_az_index.gemspec new file mode 100644 index 0000000..2ef612e --- /dev/null +++ b/cob_az_index.gemspec @@ -0,0 +1,51 @@ +# coding: utf-8 +# frozen_string_literal: true + +lib = File.expand_path("../lib", __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require "cob_az_index/version" + +Gem::Specification.new do |spec| + spec.name = "cob_az_index" + spec.version = CobAzIndex::VERSION + spec.authors = ["David Kinzer"] + spec.email = ["dtkinzer@gmail.com"] + + spec.summary = "Traject configuraiton and scripts for tul_cob az database" + spec.description = "Separates az indexing out of tulibraries/tul_cob repo" + spec.homepage = "https://github.com/tulibraries/cob_az_index" + spec.license = "MIT" + + # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host' + # to allow pushing to a single host or delete this section to allow pushing to any host. + if spec.respond_to?(:metadata) + spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'" + + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = spec.homepage + else + raise "RubyGems 2.0 or newer is required to protect against " \ + "public gem pushes." + end + + # Specify which files should be added to the gem when it is released. + # The `git ls-files -z` loads the files in the RubyGem that have been added into git. + spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do + `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + end + spec.bindir = "exe" + spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } + spec.executables << "cob_az_index" + spec.require_paths = ["lib"] + + spec.add_dependency "traject", "~> 3.1" + spec.add_dependency "traject_plus", "~> 1.1" + spec.add_dependency "httparty" + spec.add_runtime_dependency("gli", "2.18.0") + + spec.add_development_dependency "bundler", "~> 1.16" + spec.add_development_dependency "rake", "~> 10.0" + spec.add_development_dependency "rspec", "~> 3.0" + spec.add_development_dependency "rubocop", "~> 0.52" + spec.add_development_dependency "coveralls" +end diff --git a/exe/cob_az_index b/exe/cob_az_index new file mode 100755 index 0000000..a8686cc --- /dev/null +++ b/exe/cob_az_index @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "gli" +require "cob_az_index" + +class App + extend GLI::App + + program_desc "Wrapper cli app for indexing tul_cob app" + + version CobAzIndex::VERSION + + subcommand_option_handling :normal + arguments :strict + + desc "Ingest files into solr endpoint using the az db traject config" + arg_name "ingest_path", :optional + command :ingest do |c| + c.desc "Ingest files into SOLR_URL using the az db traject config" + c.action do |global_options, options, args| + + options = args.empty? ? { ingest_path: args[0] } : + { ingest_string: CobAzIndex::CLI.pull } + + CobAzIndex::CLI.ingest(options) + end + end + + pre do |global, command, options, args| + # Pre logic here + # Return true to proceed; false to abort and not call the + # chosen command + # Use skips_pre before a command to skip this block + # on that command only + true + end + + post do |global, command, options, args| + # Post logic here + # Use skips_post before a command to skip this + # block on that command only + end + + on_error do |exception| + # Error logic here + # return false to skip default error handling + true + end +end + +exit App.run(ARGV) diff --git a/lib/cob_az_index.rb b/lib/cob_az_index.rb new file mode 100644 index 0000000..17193fd --- /dev/null +++ b/lib/cob_az_index.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require "cob_az_index/version" +require "traject" +require "httparty" + +module CobAzIndex + module CLI + def self.ingest(ingest_path: nil, ingest_string: "") + indexer = Traject::Indexer::MarcIndexer.new("solr_writer.commit_on_close": true) + indexer.load_config_file("#{File.dirname(__FILE__)}/cob_az_index/indexer_config.rb") + + if ingest_path + ingest_string = open(ingest_path).read + end + + indexer.process(StringIO.new(ingest_string)) + end + + def self.pull(client_id: ENV["AZ_CLIENT_ID"], client_secret: ENV["AZ_CLIENT_SECRET"]) + endpoint = "https://lgapi-us.libapps.com/1.2/oauth/token" + + response = HTTParty.post(endpoint, body: { client_id: client_id, client_secret: client_secret, grant_type: "client_credentials" }) + cred = JSON.parse(response) + + endpoint = "https://lgapi-us.libapps.com/1.2/az" + token = cred["access_token"] + response = HTTParty.get(endpoint, headers: { Authorization: "Bearer #{token}" }, query: { expand: "subjects,icons,friendly_url,az_types,az_props,permitted_uses" }) + response.body + end + end +end diff --git a/lib/cob_az_index/indexer_config.rb b/lib/cob_az_index/indexer_config.rb new file mode 100644 index 0000000..771a25c --- /dev/null +++ b/lib/cob_az_index/indexer_config.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require "traject_plus" +require "traject_plus/json_reader.rb" +require "traject_plus/macros" +require "traject_plus/macros/json" +require "cob_az_index/macros" + +extend TrajectPlus::Macros +extend TrajectPlus::Macros::JSON +extend CobAzIndex::Macros + +solr_config = YAML.load_file("config/blacklight.yml")[(ENV["RAILS_ENV"] || "development")] + +solr_url = ERB.new(solr_config["url"]).result + +settings do + provide "reader_class_name", "TrajectPlus::JsonReader" + provide "solr_writer.commit_timeout", (15 * 60) + provide "solr.url", solr_url + provide "solr_writer.commit_on_close", "false" + + # set this to be non-negative if threshold should be enforced + provide "solr_writer.max_skipped", -1 +end + +each_record do |record, context| + if record["enable_hidden"].to_s == "1" + context.skip!("Skipping hidden database.") + end +end + +to_field "id", extract_json("$.id") + +to_field "format", ->(rec, acc) { + types = rec.fetch("az_types", []) + types.each { |type| acc << type["name"] unless type["name"] == "Database" } +} +to_field "format_t", ->(rec, acc) { + types = rec.fetch("az_types", []) + types.each { |type| acc << type["name"] unless type["name"] == "Database" } +} +to_field "database_display", ->(rec, acc) { + types = rec.fetch("az_types", []) + types = types << { "name" => "Database" } + types.each { |type| acc << type["name"] if type["name"] == "Database" } +} + +to_field "title_t", extract_json("$.name") +to_field "title_sort", extract_json("$.name") +to_field "alt_names_t", extract_json("$.alt_names") +to_field "title_statement_display", extract_json("$.name") +to_field "title_truncated_display", extract_json("$.name"), &truncate(300) +to_field "az_vendor_id_display", extract_json("$.az_vendor_id") +to_field "az_vendor_name_display", extract_json("$.az_vendor_name") + +to_field "note_display", extract_json("$.description") +to_field "note_t", extract_json("$.description") +to_field "availability_facet", -> (rec, acc) { + if rec["enable_trial"].to_s == "1" + acc << "Trial" + end +} + +to_field "electronic_resource_display", -> (rec, acc) { + url = + if rec.dig("meta", "enable_proxy") == 1 + "http://libproxy.temple.edu/login?url=#{rec['url']}" + else + rec["url"] + end + acc << { title: rec["name"], url: url }.to_json +} + +to_field "record_update_date", extract_json("$.updated") + +to_field "subject_display", -> (rec, acc) { + rec["subjects"]&.each { |subject| acc << subject["name"] } +} + +to_field "subject_facet", -> (rec, acc) { + rec["subjects"]&.each { |subject| acc << subject["name"] } +} + +to_field "subject_t", -> (rec, acc) { + rec["subjects"]&.each { |subject| acc << subject["name"] } +} + +each_record do |record, context| + if ENV["SOLR_DISABLE_UPDATE_DATE_CHECK"] == "yes" + context.output_hash["record_update_date"] = [ Time.now.to_s ] + end +end diff --git a/lib/cob_az_index/macros.rb b/lib/cob_az_index/macros.rb new file mode 100644 index 0000000..de57f97 --- /dev/null +++ b/lib/cob_az_index/macros.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Custom traject macros for cob_az_index +module CobAzIndex::Macros + def truncate(max = 300) + Proc.new do |rec, acc| + acc.map! { |s| s.length > max ? s[0...max] + " ..." : s unless s.nil? } + end + end +end diff --git a/lib/cob_az_index/version.rb b/lib/cob_az_index/version.rb new file mode 100644 index 0000000..19efa41 --- /dev/null +++ b/lib/cob_az_index/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module CobAzIndex + VERSION = "0.1.0" +end diff --git a/spec/cob_az_index_spec.rb b/spec/cob_az_index_spec.rb new file mode 100644 index 0000000..3a26cfc --- /dev/null +++ b/spec/cob_az_index_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +RSpec.describe CobAzIndex do + it "has a version number" do + expect(CobAzIndex::VERSION).not_to be nil + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..941478e --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require "coveralls" +Coveralls.wear! + +require "bundler/setup" +require "cob_az_index" + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end