Solr support in AtoM

docker/docker-compose.dev.yml

@@ -1,12 +1,112 @@
 ---
+networks:
+  solr:
+
 volumes:
   elasticsearch_data:
   percona_data:
   composer_deps:
   npm_deps:
+  solr-data:
 
 services:
 
+  zoo1:
+    image: zookeeper
+    container_name: zoo1
+    restart: always
+    hostname: zoo1
+    ports:
+      - 2181:2181
+      - 7001:7000
+    environment:
+      ZOO_MY_ID: 1
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+      ZOO_4LW_COMMANDS_WHITELIST: mntr, conf, ruok
+      ZOO_CFG_EXTRA: "metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000 metricsProvider.exportJvmInfo=true"
+    networks:
+      - solr
+
+  zoo2:
+    image: zookeeper
+    container_name: zoo2
+    restart: always
+    hostname: zoo2
+    ports:
+      - 2182:2181
+      - 7002:7000
+    environment:
+      ZOO_MY_ID: 2
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+      ZOO_4LW_COMMANDS_WHITELIST: mntr, conf, ruok
+      ZOO_CFG_EXTRA: "metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000 metricsProvider.exportJvmInfo=true"
+    networks:
+      - solr
+
+  zoo3:
+    image: zookeeper
+    container_name: zoo3
+    restart: always
+    hostname: zoo3
+    ports:
+      - 2183:2181
+      - 7003:7000
+    environment:
+      ZOO_MY_ID: 3
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+      ZOO_4LW_COMMANDS_WHITELIST: mntr, conf, ruok
+      ZOO_CFG_EXTRA: "metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000 metricsProvider.exportJvmInfo=true"
+    networks:
+      - solr
+
+  solr1:
+    image: solr
+    container_name: solr1
+    ports:
+     - "8981:8983"
+    environment:
+      - ZK_HOST=zoo1:2181,zoo2:2181,zoo3:2181
+    networks:
+      - solr
+    depends_on:
+      - zoo1
+      - zoo2
+      - zoo3
+    volumes:
+      - solr-data:/var/solr
+
+  solr2:
+    image: solr
+    container_name: solr2
+    ports:
+     - "8982:8983"
+    environment:
+      - ZK_HOST=zoo1:2181,zoo2:2181,zoo3:2181
+    networks:
+      - solr
+    depends_on:
+      - zoo1
+      - zoo2
+      - zoo3
+    volumes:
+      - solr-data:/var/solr
+
+  solr3:
+    image: solr
+    container_name: solr3
+    ports:
+     - "8983:8983"
+    environment:
+      - ZK_HOST=zoo1:2181,zoo2:2181,zoo3:2181
+    networks:
+      - solr
+    depends_on:
+      - zoo1
+      - zoo2
+      - zoo3
+    volumes:
+      - solr-data:/var/solr
+
   atom:
     build: ..
     env_file: etc/environment
@@ -16,6 +116,8 @@ services:
       - composer_deps:/atom/src/vendor/composer
       - npm_deps:/atom/src/node_modules
       - ..:/atom/src:rw
+    networks:
+      - solr
 
   atom_worker:
     build: ..
@@ -31,6 +133,8 @@ services:
       - composer_deps:/atom/src/vendor/composer
       - npm_deps:/atom/src/node_modules
       - ..:/atom/src:rw
+    networks:
+      - solr
 
   nginx:
     image: nginx:latest
@@ -41,6 +145,8 @@ services:
       - ./etc/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
     ports:
       - "63001:80"
+    networks:
+      - solr
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:5.6.16
@@ -53,6 +159,8 @@ services:
       - elasticsearch_data:/usr/share/elasticsearch/data
     ports:
       - "127.0.0.1:63002:9200"
+    networks:
+      - solr
 
   percona:
     image: percona:8.0
@@ -62,14 +170,20 @@ services:
       - ./etc/mysql/mysqld.cnf:/etc/my.cnf.d/mysqld.cnf:ro
     ports:
       - "127.0.0.1:63003:3306"
+    networks:
+      - solr
 
   memcached:
     image: memcached
     command: -p 11211 -m 128 -u memcache
     ports:
       - "127.0.0.1:63004:11211"
+    networks:
+      - solr
 
   gearmand:
     image: artefactual/gearmand
     ports:
       - "127.0.0.1:63005:4730"
+    networks:
+      - solr

lib/search/QubitSearch.class.php

@@ -23,16 +23,51 @@
 class QubitSearch
 {
     protected static $instance;
+    protected static $solrInstance;
 
     // protected function __construct() { }
     // protected function __clone() { }
 
+    public static function getSolrInstance(array $options = [])
+    {
+        $configuration = ProjectConfiguration::getActive();
+        if (!$configuration->isPluginEnabled('arSolrPlugin')) {
+            return false;
+        }
+
+        if (!isset(self::$solrInstance)) {
+            self::$solrInstance = new arSolrPlugin($options);
+        }
+
+        return self::$solrInstance;
+    }
+
+    public static function disableSolr()
+    {
+        if (!isset(self::$solrInstance)) {
+            self::$solrInstance = self::getSolrInstance(['initialize' => false]);
+        }
+
+        self::$solrInstance->disable();
+    }
+
+    public static function enableSolr()
+    {
+        self::$solrInstance = self::getSolrInstance();
+
+        self::$solrInstance->enable();
+    }
+
     public static function getInstance(array $options = [])
     {
         if (!isset(self::$instance)) {
             // Using arElasticSearchPlugin but other classes could be
             // implemented, for example: arSphinxSearchPlugin
             self::$instance = new arElasticSearchPlugin($options);
+            //$configuration = ProjectConfiguration::getActive();
+            //if ($configuration->isPluginEnabled('arSolrPlugin')) {
+              //self::$solr = new arSolrPlugin($options);
+            //}
         }
 
         return self::$instance;

lib/search/QubitSolrSearchPager.class.php

@@ -0,0 +1,63 @@
+<?php
+
+/*
+ * This file is part of the Access to Memory (AtoM) software.
+ *
+ * Access to Memory (AtoM) is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Access to Memory (AtoM) is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Access to Memory (AtoM).  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+class QubitSolrSearchPager extends sfPager
+{
+    protected $nbResults;
+    protected $resultSet;
+
+    public function __construct(arSolrResultSet $resultSet)
+    {
+        $this->resultSet = $resultSet;
+    }
+
+    /**
+     * @see sfPager
+     */
+    public function init()
+    {
+        $this->setNbResults($this->resultSet->getTotalHits());
+
+        if (0 == $this->getPage() || 0 == $this->getMaxPerPage()) {
+            $this->setLastPage(0);
+        } else {
+            $this->setLastPage(ceil($this->getNbResults() / $this->getMaxPerPage()));
+        }
+    }
+
+    /**
+     * @see sfPager
+     */
+    public function getResults()
+    {
+        // Note: to get results here beyond page 1, you'll need to call $resultSet->setFrom()
+        // prior to this pager's creation.
+        return $this->resultSet->getResults();
+    }
+
+    /**
+     * @see sfPager
+     *
+     * @param mixed $offset
+     */
+    public function retrieveObject($offset)
+    {
+        return array_slice($this->getResults, $offset, 1);
+    }
+}

lib/task/search/arSolrPopulateTask.class.php

@@ -0,0 +1,108 @@
+<?php
+
+/*
+ * This file is part of the Access to Memory (AtoM) software.
+ *
+ * Access to Memory (AtoM) is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Access to Memory (AtoM) is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Access to Memory (AtoM).  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Populate search index.
+ */
+class arSolrPopulateTask extends sfBaseTask
+{
+    public function execute($arguments = [], $options = [])
+    {
+        sfContext::createInstance($this->configuration);
+        sfConfig::add(QubitSetting::getSettingsArray());
+
+        // If show-types flag set, show types available to index
+        //if (!empty($options['show-types'])) {
+        //$this->log(sprintf('Available document types that can be excluded: %s', implode(', ', $this->availableDocumentTypes())));
+        //$this->ask('Press the Enter key to continue indexing or CTRL-C to abort...');
+        //}
+
+        new sfDatabaseManager($this->configuration);
+
+        $solr = QubitSearch::getSolrInstance();
+
+        // Index by slug, if specified, or all indexable resources except those with an excluded type
+        //if ($options['slug']) {
+        //$logMessage = (false !== $this->attemptIndexBySlug($options)) ? 'Slug indexed.' : 'Slug not found.';
+        //$this->log($logMessage);
+        //} else {
+        //$populateOptions = [];
+        //$populateOptions['excludeTypes'] = (!empty($options['exclude-types'])) ? explode(',', strtolower($options['exclude-types'])) : null;
+        //$populateOptions['update'] = $options['update'];
+
+        //QubitSearch::getInstance()->populate($populateOptions);
+        //}
+        $populateOptions = [];
+        $populateOptions['excludeTypes'] = (!empty($options['exclude-types'])) ? explode(',', strtolower($options['exclude-types'])) : null;
+        $populateOptions['update'] = $options['update'];
+        $solr->populate($populateOptions);
+    }
+
+    protected function configure()
+    {
+        $this->addOptions([
+            new sfCommandOption('application', null, sfCommandOption::PARAMETER_OPTIONAL, 'The application name', 'qubit'),
+            new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED, 'The environment', 'cli'),
+            //new sfCommandOption('slug', null, sfCommandOption::PARAMETER_OPTIONAL, 'Slug of resource to index (ignoring exclude-types option).'),
+            //new sfCommandOption('ignore-descendants', null, sfCommandOption::PARAMETER_NONE, "Don't index resource's descendants (applies to --slug option only)."),
+            new sfCommandOption('exclude-types', null, sfCommandOption::PARAMETER_OPTIONAL, 'Exclude document type(s) (command-separated) from indexing'),
+            //new sfCommandOption('show-types', null, sfCommandOption::PARAMETER_NONE, 'Show available document type(s), that can be excluded, before indexing'),
+            new sfCommandOption('update', null, sfCommandOption::PARAMETER_NONE, "Don't delete existing records before indexing."),
+        ]);
+
+        $this->namespace = 'solr';
+        $this->name = 'populate';
+
+        $this->briefDescription = 'Populates the search index';
+        $this->detailedDescription = <<<'EOF'
+The [solr:populate|INFO] task empties, populates, and optimizes the index
+in the current project. It may take quite a while to run.
+
+To exclude a document type, use the --exclude-types option. For example:
+
+  php symfony solr:populate --exclude-types="term,actor"
+
+To see a list of available document types that can be excluded use the --show-types option.
+EOF;
+    }
+
+    //private function availableDocumentTypes()
+    //{
+        //$types = array_keys(QubitSearch::getInstance()->loadMappings()->asArray());
+        //sort($types);
+
+        //return $types;
+    //}
+
+    //private function attemptIndexBySlug($options)
+    //{
+        //// Abort if resource doesn't exist for the provided slug
+        //if (null == $resource = QubitObject::getBySlug($options['slug'])) {
+            //return false;
+        //}
+
+        //// For information objects, allow optional skipping of descendants
+        //if ($resource instanceof QubitInformationObject) {
+            //$options = ['updateDescendants' => !$options['ignore-descendants']];
+            //QubitSearch::getInstance()->update($resource, $options);
+        //} else {
+            //QubitSearch::getInstance()->update($resource);
+        //}
+    //}
+}