Merge pull request #372 from pelias/elasticsearch-5-schema

Update schema for Elasticsearch 5 and 6

bin/integration

@@ -4,4 +4,4 @@
 # see https://github.com/pelias/pelias/issues/744
 set -o pipefail
 
-node --max-http-header-size=16384 integration/run.js | npx tap-spec
+node integration/run.js | npx tap-spec

integration/dynamic_templates.js

@@ -129,20 +129,13 @@ function addendumAssertion( type, namespace, value, common ){
         var addendumProperties = properties.addendum.properties;
 
         t.true([
-          'string', // elasticsearch 2.4
           'keyword' // elasticsearch 5.6
         ].includes( addendumProperties[namespace].type ));
 
         t.true([
-          'no', // elasticsearch 2.4
           false // elasticsearch 5.6
         ].includes( addendumProperties[namespace].index ));
 
-        // elasticsearch 2.4
-        if( addendumProperties[namespace].fielddata ){
-          t.equal( addendumProperties[namespace].fielddata.format, 'disabled' );
-        }
-
         // elasticsearch 5.6
         if( addendumProperties[namespace].doc_values ){
           t.equal( addendumProperties[namespace].doc_values, false );
@@ -167,4 +160,4 @@ function addendumAssertion( type, namespace, value, common ){
 
     suite.run( t.end );
   };
-}
+}

mappings/document.js

@@ -2,16 +2,16 @@ const admin = require('./partial/admin');
 const postalcode = require('./partial/postalcode');
 const hash = require('./partial/hash');
 const multiplier = require('./partial/multiplier');
-const literal = require('./partial/literal');
-const literal_with_doc_values = require('./partial/literal_with_doc_values');
+const keyword = require('./partial/keyword');
+const keyword_with_doc_values = require('./partial/keyword_with_doc_values');
 const config = require('pelias-config').generate();
 
 var schema = {
   properties: {
 
     // data partitioning
-    source: literal_with_doc_values,
-    layer: literal_with_doc_values,
+    source: keyword_with_doc_values,
+    layer: keyword_with_doc_values,
 
     // place name (ngram analysis)
     name: hash,
@@ -25,27 +25,27 @@ var schema = {
       dynamic: 'strict',
       properties: {
         name: {
-          type: 'string',
+          type: 'text',
           analyzer: 'keyword',
         },
         unit: {
-          type: 'string',
+          type: 'text',
           analyzer: 'peliasUnit',
         },
         number: {
-          type: 'string',
+          type: 'text',
           analyzer: 'peliasHousenumber',
         },
         street: {
-          type: 'string',
+          type: 'text',
           analyzer: 'peliasStreet',
         },
         cross_street: {
-          type: 'string',
+          type: 'text',
           analyzer: 'peliasStreet',
         },
         zip: {
-          type: 'string',
+          type: 'text',
           analyzer: 'peliasZip',
         },
       }
@@ -59,77 +59,77 @@ var schema = {
         // https://github.com/whosonfirst/whosonfirst-placetypes#continent
         continent: admin,
         continent_a: admin,
-        continent_id: literal,
+        continent_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#ocean
         ocean: admin,
         ocean_a: admin,
-        ocean_id: literal,
+        ocean_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#empire
         empire: admin,
         empire_a: admin,
-        empire_id: literal,
+        empire_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#country
         country: admin,
         country_a: admin,
-        country_id: literal,
+        country_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#dependency
         dependency: admin,
         dependency_a: admin,
-        dependency_id: literal,
+        dependency_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#marinearea
         marinearea: admin,
         marinearea_a: admin,
-        marinearea_id: literal,
+        marinearea_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#macroregion
         macroregion: admin,
         macroregion_a: admin,
-        macroregion_id: literal,
+        macroregion_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#region
         region: admin,
         region_a: admin,
-        region_id: literal,
+        region_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#macrocounty
         macrocounty: admin,
         macrocounty_a: admin,
-        macrocounty_id: literal,
+        macrocounty_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#county
         county: admin,
         county_a: admin,
-        county_id: literal,
+        county_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#locality
         locality: admin,
         locality_a: admin,
-        locality_id: literal,
+        locality_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#borough
         borough: admin,
         borough_a: admin,
-        borough_id: literal,
+        borough_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#localadmin
         localadmin: admin,
         localadmin_a: admin,
-        localadmin_id: literal,
+        localadmin_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#neighbourhood
         neighbourhood: admin,
         neighbourhood_a: admin,
-        neighbourhood_id: literal,
+        neighbourhood_id: keyword,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#postalcode
         postalcode: postalcode,
         postalcode_a: postalcode,
-        postalcode_id: literal
+        postalcode_id: keyword
       }
     },
 
@@ -139,8 +139,8 @@ var schema = {
     bounding_box: require('./partial/boundingbox'),
 
     // meta info
-    source_id: literal,
-    category: literal,
+    source_id: keyword,
+    category: keyword,
     population: multiplier,
     popularity: multiplier,
 
@@ -152,36 +152,27 @@ var schema = {
       path_match: 'name.*',
       match_mapping_type: 'string',
       mapping: {
-        type: 'string',
-        analyzer: 'peliasIndexOneEdgeGram',
-        fielddata : {
-          format: 'disabled'
-        }
+        type: 'text',
+        analyzer: 'peliasIndexOneEdgeGram'
       }
     },
   },{
     phrase: {
       path_match: 'phrase.*',
       match_mapping_type: 'string',
       mapping: {
-        type: 'string',
-        analyzer: 'peliasPhrase',
-        fielddata : {
-          format: 'disabled'
-        }
+        type: 'text',
+        analyzer: 'peliasPhrase'
       }
     }
   },{
     addendum: {
       path_match: 'addendum.*',
       match_mapping_type: 'string',
       mapping: {
-        type: 'string',
-        index: 'no',
-        doc_values: false,
-        fielddata : {
-          format: 'disabled'
-        }
+        type: 'keyword',
+        index: false,
+        doc_values: false
       }
     }
   }],

mappings/partial/admin.json

@@ -1,14 +1,11 @@
 {
-  "type": "string",
+  "type": "text",
   "analyzer": "peliasAdmin",
   "fields": {
     "ngram": {
-      "type": "string",
+      "type": "text",
       "analyzer": "peliasIndexOneEdgeGram",
-      "doc_values": false,
-      "fielddata": {
-        "format": "disabled"
-      }
+      "doc_values": false
     }
   }
-}
+}

mappings/partial/boundingbox.json

@@ -1,4 +1,4 @@
 {
-  "type": "string",
-  "index": "no"
+  "type": "keyword",
+  "index": false
 }

mappings/partial/keyword.json

@@ -0,0 +1,4 @@
+{
+  "type": "keyword",
+  "doc_values": false
+}

mappings/partial/keyword_with_doc_values.json

@@ -0,0 +1,3 @@
+{
+  "type": "keyword"
+}

mappings/partial/postalcode.json

@@ -1,14 +1,10 @@
 {
-  "type": "string",
+  "type": "text",
   "analyzer": "peliasZip",
   "fields": {
     "ngram": {
-      "type": "string",
-      "analyzer": "peliasIndexOneEdgeGram",
-      "doc_values": false,
-      "fielddata": {
-        "format": "disabled"
-      }
+      "type": "text",
+      "analyzer": "peliasIndexOneEdgeGram"
     }
   }
 }

scripts/create_index.js

@@ -18,20 +18,6 @@ try {
   process.exit(1);
 }
 
-if (http.maxHeaderSize === undefined) {
-  logger.warn('You are using a version of Node.js that does not support the --max-http-header-size option.' +
-    'You may experience issues when using Elasticsearch 5.' +
-    'See https://github.com/pelias/schema#compatibility for more details.');
-}
-
-if (http.maxHeaderSize < 16384) {
-  logger.error('Max header size is below 16384 bytes. ' +
-    'Be sure to use the provided wrapper script in \'./bin\' rather than calling this script directly.' +
-    'Otherwise, you may experience issues when using Elasticsearch 5.' +
-    'See https://github.com/pelias/schema#compatibility for more details.');
-  process.exit(1);
-}
-
 cli.header("create index");
 const indexName = config.schema.indexName;
 

test/compile.js

@@ -40,11 +40,8 @@ module.exports.tests.dynamic_templates = function(test, common) {
     t.equal(template.path_match, 'name.*');
     t.equal(template.match_mapping_type, 'string');
     t.deepEqual(template.mapping, {
-      type: 'string',
-      analyzer: 'peliasIndexOneEdgeGram',
-      fielddata : {
-        format: "disabled"
-      }
+      type: 'text',
+      analyzer: 'peliasIndexOneEdgeGram'
     });
     t.end();
   });