diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 92ed06742c..ab6b26bccf 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -1,12 +1,112 @@ --- +networks: + solr: + volumes: elasticsearch_data: percona_data: composer_deps: npm_deps: + solr-data: services: + zoo1: + image: zookeeper + container_name: zoo1 + restart: always + hostname: zoo1 + ports: + - 2181:2181 + - 7001:7000 + environment: + ZOO_MY_ID: 1 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 + ZOO_4LW_COMMANDS_WHITELIST: mntr, conf, ruok + ZOO_CFG_EXTRA: "metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000 metricsProvider.exportJvmInfo=true" + networks: + - solr + + zoo2: + image: zookeeper + container_name: zoo2 + restart: always + hostname: zoo2 + ports: + - 2182:2181 + - 7002:7000 + environment: + ZOO_MY_ID: 2 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 + ZOO_4LW_COMMANDS_WHITELIST: mntr, conf, ruok + ZOO_CFG_EXTRA: "metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000 metricsProvider.exportJvmInfo=true" + networks: + - solr + + zoo3: + image: zookeeper + container_name: zoo3 + restart: always + hostname: zoo3 + ports: + - 2183:2181 + - 7003:7000 + environment: + ZOO_MY_ID: 3 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 + ZOO_4LW_COMMANDS_WHITELIST: mntr, conf, ruok + ZOO_CFG_EXTRA: "metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000 metricsProvider.exportJvmInfo=true" + networks: + - solr + + solr1: + image: solr + container_name: solr1 + ports: + - "8981:8983" + environment: + - ZK_HOST=zoo1:2181,zoo2:2181,zoo3:2181 + networks: + - solr + depends_on: + - zoo1 + - zoo2 + - zoo3 + volumes: + - solr-data:/var/solr + + solr2: + image: solr + container_name: solr2 + ports: + - "8982:8983" + environment: + - ZK_HOST=zoo1:2181,zoo2:2181,zoo3:2181 + networks: + - solr + depends_on: + - zoo1 + - zoo2 + - zoo3 + volumes: + - solr-data:/var/solr + + solr3: + image: solr + container_name: solr3 + ports: + - "8983:8983" + environment: + - ZK_HOST=zoo1:2181,zoo2:2181,zoo3:2181 + networks: + - solr + depends_on: + - zoo1 + - zoo2 + - zoo3 + volumes: + - solr-data:/var/solr + atom: build: .. env_file: etc/environment @@ -16,6 +116,8 @@ services: - composer_deps:/atom/src/vendor/composer - npm_deps:/atom/src/node_modules - ..:/atom/src:rw + networks: + - solr atom_worker: build: .. @@ -31,6 +133,8 @@ services: - composer_deps:/atom/src/vendor/composer - npm_deps:/atom/src/node_modules - ..:/atom/src:rw + networks: + - solr nginx: image: nginx:latest @@ -41,6 +145,8 @@ services: - ./etc/nginx/nginx.conf:/etc/nginx/nginx.conf:ro ports: - "63001:80" + networks: + - solr elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:5.6.16 @@ -53,6 +159,8 @@ services: - elasticsearch_data:/usr/share/elasticsearch/data ports: - "127.0.0.1:63002:9200" + networks: + - solr percona: image: percona:8.0 @@ -62,14 +170,20 @@ services: - ./etc/mysql/mysqld.cnf:/etc/my.cnf.d/mysqld.cnf:ro ports: - "127.0.0.1:63003:3306" + networks: + - solr memcached: image: memcached command: -p 11211 -m 128 -u memcache ports: - "127.0.0.1:63004:11211" + networks: + - solr gearmand: image: artefactual/gearmand ports: - "127.0.0.1:63005:4730" + networks: + - solr diff --git a/lib/search/QubitSearch.class.php b/lib/search/QubitSearch.class.php index 6c2e4c508a..3feaede32d 100644 --- a/lib/search/QubitSearch.class.php +++ b/lib/search/QubitSearch.class.php @@ -23,16 +23,51 @@ class QubitSearch { protected static $instance; + protected static $solrInstance; // protected function __construct() { } // protected function __clone() { } + public static function getSolrInstance(array $options = []) + { + $configuration = ProjectConfiguration::getActive(); + if (!$configuration->isPluginEnabled('arSolrPlugin')) { + return false; + } + + if (!isset(self::$solrInstance)) { + self::$solrInstance = new arSolrPlugin($options); + } + + return self::$solrInstance; + } + + public static function disableSolr() + { + if (!isset(self::$solrInstance)) { + self::$solrInstance = self::getSolrInstance(['initialize' => false]); + } + + self::$solrInstance->disable(); + } + + public static function enableSolr() + { + self::$solrInstance = self::getSolrInstance(); + + self::$solrInstance->enable(); + } + public static function getInstance(array $options = []) { if (!isset(self::$instance)) { // Using arElasticSearchPlugin but other classes could be // implemented, for example: arSphinxSearchPlugin self::$instance = new arElasticSearchPlugin($options); + //$configuration = ProjectConfiguration::getActive(); + //if ($configuration->isPluginEnabled('arSolrPlugin')) { + //self::$solr = new arSolrPlugin($options); + //} } return self::$instance; diff --git a/lib/search/QubitSolrSearchPager.class.php b/lib/search/QubitSolrSearchPager.class.php new file mode 100644 index 0000000000..27558cdc19 --- /dev/null +++ b/lib/search/QubitSolrSearchPager.class.php @@ -0,0 +1,63 @@ +. + */ + +class QubitSolrSearchPager extends sfPager +{ + protected $nbResults; + protected $resultSet; + + public function __construct(arSolrResultSet $resultSet) + { + $this->resultSet = $resultSet; + } + + /** + * @see sfPager + */ + public function init() + { + $this->setNbResults($this->resultSet->getTotalHits()); + + if (0 == $this->getPage() || 0 == $this->getMaxPerPage()) { + $this->setLastPage(0); + } else { + $this->setLastPage(ceil($this->getNbResults() / $this->getMaxPerPage())); + } + } + + /** + * @see sfPager + */ + public function getResults() + { + // Note: to get results here beyond page 1, you'll need to call $resultSet->setFrom() + // prior to this pager's creation. + return $this->resultSet->getResults(); + } + + /** + * @see sfPager + * + * @param mixed $offset + */ + public function retrieveObject($offset) + { + return array_slice($this->getResults, $offset, 1); + } +} diff --git a/lib/task/search/arSolrPopulateTask.class.php b/lib/task/search/arSolrPopulateTask.class.php new file mode 100644 index 0000000000..1c275ca8cb --- /dev/null +++ b/lib/task/search/arSolrPopulateTask.class.php @@ -0,0 +1,108 @@ +. + */ + +/** + * Populate search index. + */ +class arSolrPopulateTask extends sfBaseTask +{ + public function execute($arguments = [], $options = []) + { + sfContext::createInstance($this->configuration); + sfConfig::add(QubitSetting::getSettingsArray()); + + // If show-types flag set, show types available to index + //if (!empty($options['show-types'])) { + //$this->log(sprintf('Available document types that can be excluded: %s', implode(', ', $this->availableDocumentTypes()))); + //$this->ask('Press the Enter key to continue indexing or CTRL-C to abort...'); + //} + + new sfDatabaseManager($this->configuration); + + $solr = QubitSearch::getSolrInstance(); + + // Index by slug, if specified, or all indexable resources except those with an excluded type + //if ($options['slug']) { + //$logMessage = (false !== $this->attemptIndexBySlug($options)) ? 'Slug indexed.' : 'Slug not found.'; + //$this->log($logMessage); + //} else { + //$populateOptions = []; + //$populateOptions['excludeTypes'] = (!empty($options['exclude-types'])) ? explode(',', strtolower($options['exclude-types'])) : null; + //$populateOptions['update'] = $options['update']; + + //QubitSearch::getInstance()->populate($populateOptions); + //} + $populateOptions = []; + $populateOptions['excludeTypes'] = (!empty($options['exclude-types'])) ? explode(',', strtolower($options['exclude-types'])) : null; + $populateOptions['update'] = $options['update']; + $solr->populate($populateOptions); + } + + protected function configure() + { + $this->addOptions([ + new sfCommandOption('application', null, sfCommandOption::PARAMETER_OPTIONAL, 'The application name', 'qubit'), + new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED, 'The environment', 'cli'), + //new sfCommandOption('slug', null, sfCommandOption::PARAMETER_OPTIONAL, 'Slug of resource to index (ignoring exclude-types option).'), + //new sfCommandOption('ignore-descendants', null, sfCommandOption::PARAMETER_NONE, "Don't index resource's descendants (applies to --slug option only)."), + new sfCommandOption('exclude-types', null, sfCommandOption::PARAMETER_OPTIONAL, 'Exclude document type(s) (command-separated) from indexing'), + //new sfCommandOption('show-types', null, sfCommandOption::PARAMETER_NONE, 'Show available document type(s), that can be excluded, before indexing'), + new sfCommandOption('update', null, sfCommandOption::PARAMETER_NONE, "Don't delete existing records before indexing."), + ]); + + $this->namespace = 'solr'; + $this->name = 'populate'; + + $this->briefDescription = 'Populates the search index'; + $this->detailedDescription = <<<'EOF' +The [solr:populate|INFO] task empties, populates, and optimizes the index +in the current project. It may take quite a while to run. + +To exclude a document type, use the --exclude-types option. For example: + + php symfony solr:populate --exclude-types="term,actor" + +To see a list of available document types that can be excluded use the --show-types option. +EOF; + } + + //private function availableDocumentTypes() + //{ + //$types = array_keys(QubitSearch::getInstance()->loadMappings()->asArray()); + //sort($types); + + //return $types; + //} + + //private function attemptIndexBySlug($options) + //{ + //// Abort if resource doesn't exist for the provided slug + //if (null == $resource = QubitObject::getBySlug($options['slug'])) { + //return false; + //} + + //// For information objects, allow optional skipping of descendants + //if ($resource instanceof QubitInformationObject) { + //$options = ['updateDescendants' => !$options['ignore-descendants']]; + //QubitSearch::getInstance()->update($resource, $options); + //} else { + //QubitSearch::getInstance()->update($resource); + //} + //} +} diff --git a/lib/task/search/arSolrSearchTask.class.php b/lib/task/search/arSolrSearchTask.class.php new file mode 100644 index 0000000000..46c8298b88 --- /dev/null +++ b/lib/task/search/arSolrSearchTask.class.php @@ -0,0 +1,140 @@ +. + */ + +/** + * Populate search index. + */ +class arSolrSearchTask extends sfBaseTask +{ + public function execute($arguments = [], $options = []) + { + sfContext::createInstance($this->configuration); + sfConfig::add(QubitSetting::getSettingsArray()); + + new sfDatabaseManager($this->configuration); + + $solr = new arSolrPlugin($options); + + $this->runSolrQuery($solr, $arguments['query'], (int) $options['rows'], (int) $options['start'], $options['fields'], $options['type']); + } + + protected function configure() + { + $this->addArguments([ + new sfCommandArgument('query', sfCommandArgument::OPTIONAL, 'Search query.'), + ]); + + $this->addOptions([ + new sfCommandOption('application', null, sfCommandOption::PARAMETER_OPTIONAL, 'The application name', 'qubit'), + new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED, 'The environment', 'cli'), + new sfCommandOption('start', null, sfCommandOption::PARAMETER_OPTIONAL, 'Offset for the result set output. 0 by default', 0), + new sfCommandOption('rows', null, sfCommandOption::PARAMETER_OPTIONAL, 'Number of rows to return in the results', 5), + new sfCommandOption('fields', null, sfCommandOption::PARAMETER_OPTIONAL, 'Fields to query("comma seperated")', null), + new sfCommandOption('type', null, sfCommandOption::PARAMETER_OPTIONAL, 'Query type', null), + ]); + + $this->namespace = 'solr'; + $this->name = 'search'; + + $this->briefDescription = 'Search the search index for a result'; + $this->detailedDescription = <<<'EOF' +The [solr:search] task runs a search query on solr. Usage: + php symfony solr:search + +To get paginated results, use rows and start. For example: + php symfony solr:search fonds --rows=5 --start=10 + +This wll get 5 search results starting from the 10th result + +To search specific fields use the --fields option. For example: + php symfony solr:search fonds --fields=QubitInformationObject.i18n.%s.title^10,QubitInformationObject.identifier^5 + +This will search only i18n.(language code).title and identifier fields and +boost them by 10 and 5 respectively + +EOF; + } + + private function createQuery($queryText, $fields) + { + $query = new arSolrStringQuery(arSolrPluginUtil::escapeTerm($queryText)); + if ($fields) { + $fieldsArr = explode(',', $fields); + $newFields = []; + foreach ($fieldsArr as $field) { + $newField = explode('^', $field); + $fieldName = $newField[0]; + $fieldBoost = $newField[1]; + if (!$fieldBoost) { + $fieldBoost = 1; + } + $newFields[$fieldName] = (int) $fieldBoost; + } + $query->setFields(arSolrPluginUtil::getBoostedSearchFields($newFields)); + } else { + $fields = arSolrPluginUtil::getAllFields('informationObject'); + $query->setFields(arSolrPluginUtil::getBoostedSearchFields($fields)); + } + + return $query; + } + + private function runSolrQuery($solrInstance, $queryText, $rows, $start, $fields, $type) + { + $modelType = 'QubitInformationObject'; + if ('matchall' === $type) { + $query = new arSolrMatchAllQuery(); + $modelType = null; + } elseif ('term' === $type) { + $term = explode(',', $queryText); + $query = new arSolrTermQuery([$term[0] => $term[1]]); + $query->setType($modelType); + } elseif ('match' === $type) { + $queryField = explode(',', $queryText); + $query = new arSolrMatchQuery([$queryField[0] => $queryField[1]]); + $query->setType($modelType); + } elseif ('bool' === $type) { + $query = new arSolrBoolQuery(); + $mustClause = $this->createQuery($queryText, $fields); + $mustClause->setType($modelType); + $query->addMust($mustClause); + } else { + $query = $this->createQuery($queryText, $fields); + $query->setType($modelType); + } + $query->setSize($rows); + $query->setOffset($start); + + $resultSet = $solrInstance->search($query, $modelType); + if ($resultSet->count() > 0) { + $docs = $resultSet->getDocuments(); + foreach ($docs as $resp) { + $this->log(sprintf('%s - %s', $resp['id'][0], $resp['i18n']['en']['title'])); + + // print entire object if no title is present + if (!$resp['i18n']['en']['title']) { + $this->log(var_export($resp, true)); + } + } + } else { + $this->log('No results found'); + $this->log(print_r($resultSet->getResults(), true)); + } + } +} diff --git a/plugins/arDominionB5Plugin/templates/_header.php b/plugins/arDominionB5Plugin/templates/_header.php index 88fe6d5e4b..994cfb6c02 100644 --- a/plugins/arDominionB5Plugin/templates/_header.php +++ b/plugins/arDominionB5Plugin/templates/_header.php @@ -1,9 +1,13 @@ +getConfiguration()->isPluginEnabled('arSolrPlugin')) { ?> + + +
- + @@ -27,8 +31,8 @@