Rework the way Nominatim responses are handled. Closes opening-hours#167

.

CHANGELOG.rst

@@ -112,6 +112,9 @@ Changed
 ~~~~~~~
 
 * Migrated to use `ES2015 modules`_ and rollup_ for module bundling [simon04_]
+* Update to holiday definition format 2.2.0. Holidays are now maintained in
+  YAML files, one for each country. [ypid_]
+* Rework the way Nominatim responses which are used for testing are handled. [ypid_]
 
 * Input tolerance: For a value such as ``Mo-Fr 08:00-12:00 by_appointment`` the
   tool did previously suggest to use ``Mo-Fr 08:00-12:00 "on appointment"`` but
@@ -122,6 +125,11 @@ Changed
 .. _ES2015 modules: http://exploringjs.com/es6/ch_modules.html
 .. _rollup: http://rollupjs.org/
 
+Fixed
+~~~~~
+
+* Fix Russian public holiday definitions. Regions where not in local language and thus not matched properly. [ypid_]
+
 
 v3.5.0_ - 2017-02-17
 --------------------

PH_SH_batch_exporter.sh

@@ -30,7 +30,7 @@ for state in bw by be bb hb hh he mv ni nw rp sn st sl sh th
 do
     filepath="feiertage_${state}.conf"
     echo "Generating $filepath …"
-    ./PH_SH_exporter.js /tmp/PH_SH_export.list --from 2013 --until 2042 --public-holidays --region $state --omit-date-hyphens
+    ./PH_SH_exporter.js /tmp/PH_SH_export.list --from 2013 --to 2042 --public-holidays --state $state --omit-date-hyphens
     (
         print_header
         cat /tmp/PH_SH_export.list
@@ -42,7 +42,7 @@ for state in bw by be bb hb hh he mv ni nw rp sn st
 do
     filepath="ferien_${state}.conf"
     echo "Generating $filepath …"
-    ./PH_SH_exporter.js /tmp/PH_SH_export.list --from 2013 --until 2016 --school-holidays --region $state --omit-date-hyphens
+    ./PH_SH_exporter.js /tmp/PH_SH_export.list --from 2013 --to 2016 --school-holidays --state $state --omit-date-hyphens
     (
         print_header
         cat /tmp/PH_SH_export.list

PH_SH_exporter.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env nodejs
 /* Info, license and author {{{
  * @license AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>
- * @author Copyright (C) 2015 Robin Schneider <[email protected]>
+ * @author Copyright (C) 2015-2017 Robin Schneider <[email protected]>
  *
  * Written for: https://github.com/anschuetz/linuxmuster/issues/1#issuecomment-110888829
  *
@@ -22,10 +22,8 @@
 /* Required modules {{{ */
 var opening_hours = require('./opening_hours.js');
 var fs            = require('fs');
-/* }}} */
-
-/* Constants {{{ */
-var nominatim_object = require('./js/nominatim_definitions.js').for_loc;
+var glob          = require('glob');
+var yaml          = require('js-yaml');
 /* }}} */
 
 /* Parameter handling {{{ */
@@ -44,8 +42,7 @@ var optimist = require('optimist')
     .describe('o', 'Omit hyphen in ISO 8061 dates.')
     .default('o', false)
     .default('c', 'de')
-    .describe('r', 'Region (for which the holidays apply). Defaults to Baden-Württemberg.')
-    .default('r', 'bw')
+    .describe('r', 'Region (for which the holidays apply). If not given, the country wide definition is used.')
     .boolean(['p', 's', ])
     .alias('h', 'help')
     .alias('v', 'verbose')
@@ -54,7 +51,7 @@ var optimist = require('optimist')
     .alias('p', ['public-holidays', 'ph'])
     .alias('s', ['school-holidays', 'sh'])
     .alias('c', 'country')
-    .alias('r', 'region')
+    .alias('r', 'state')
     .alias('o', 'omit-date-hyphens');
 
 var argv = optimist.argv;
@@ -67,11 +64,22 @@ if (argv.help || argv._.length === 0) {
 /* Error handling {{{ */
 if (argv['public-holidays'] && argv['school-holidays']) {
     console.error("--school-holidays and --public-holidays can not be used together.");
-    process.exit(0);
+    process.exit(1);
 }
-if (typeof nominatim_object[argv.country] !== 'object' || typeof nominatim_object[argv.country][argv.region] !== 'object') {
-    console.error(argv.country + ", " + argv.region + " is currently not supported.");
-    process.exit(0);
+if (!(argv['public-holidays'] || argv['school-holidays'])) {
+    console.error("Either --school-holidays or --public-holidays has to be specified.");
+    process.exit(1);
+}
+let nominatim_by_loc = {};
+for (let nominatim_file of glob.sync("holidays/nominatim_cache/*.yaml")) {
+    let country_state = nominatim_file.match(/^.*\/([^/]*)\.yaml$/)[1];
+    nominatim_by_loc[country_state] = yaml.safeLoad(fs.readFileSync(nominatim_file));
+}
+const nominatim_data = nominatim_by_loc[argv.country + '_' + argv.state] || nominatim_by_loc[argv.country];
+
+if (typeof nominatim_data !== 'object') {
+    console.error(argv.country + (", " + argv.state ? typeof nominatim_data !== 'object' : '') + " is currently not supported.");
+    process.exit(1);
 }
 
 /* }}} */
@@ -81,11 +89,11 @@ var filepath = argv._[0];
 
 var oh_value = argv['public-holidays'] ? 'PH' : 'SH';
 
-write_config_file(filepath, oh_value, nominatim_object[argv.country][argv.region], new Date(argv.from, 0, 1), new Date(argv.to + 1, 0, 1));
+write_config_file(filepath, oh_value, nominatim_data, new Date(argv.from, 0, 1), new Date(argv.to + 1, 0, 1));
 
-function write_config_file(filepath, oh_value, nominatim_object, from_date, to_date) {
+function write_config_file(filepath, oh_value, nominatim_data, from_date, to_date) {
     try {
-        oh = new opening_hours(oh_value, nominatim_object);
+        oh = new opening_hours(oh_value, nominatim_data);
     } catch (err) {
         console.error('Something went wrong. Please file a issue at https://github.com/opening-hours/opening_hours.js/issues');
         process.exit(0);

README.md

@@ -652,7 +652,7 @@ YoHours currently only checks with this lib if the opening_hours value can be ev
 
 ## Bindings and ports
 
-* Python: https://github.com/ypid/pyopening_hours (using the JS implementation through Python subprocess and JSON passing to a Node.JS process executing the JS implementation, access to the [simple API](https://github.com/ypid/opening_hours.js#simple-api))
+* Python: https://github.com/ypid/pyopening_hours (using the JS implementation through Python subprocess and JSON passing to a Node.JS process executing the JS implementation, access to the [simple API](https://github.com/opening-hours/opening_hours.js#simple-api))
 * Java/Nashorn: https://josm.openstreetmap.de/ticket/11755 (using the JS implementation through [Nashorn](http://openjdk.java.net/projects/nashorn/), Status: Nashorn provides access to all features of the JS implementation)
 - Java/Android: https://github.com/ypid/ComplexAlarm (using the JS implementation through [js-evaluator-for-android](https://github.com/evgenyneu/js-evaluator-for-android), Status: Library runs on Android, Return code/Result passing from JS to Java not yet clear/tested)
 
@@ -715,7 +715,7 @@ Please refer to the [holiday documentation][ohlib.docs.holiday] for more details
 Please consider adding a test (with a time range of one year for example) to see if everything works as expected and to ensure that it will stay that way.
 See under [testing][ohlib.testing].
 
-In case your holiday definition does only change the `holiday_definitions` variable (and not core code) it is also ok to test the definition using the `PH_SH_exporter.js` script. In that case writing a test is not required but still appreciated. Example: `./PH_SH_exporter.js --verbose --from=2016 --to=2016 --public-holidays --country dk --region dk /tmp/dk_holidays.txt`
+In case your holiday definition does only change the `holiday_definitions` variable (and not core code) it is also ok to test the definition using the `PH_SH_exporter.js` script. In that case writing a test is not required but still appreciated. Example: `./PH_SH_exporter.js --verbose --from=2016 --to=2016 --public-holidays --country dk --state dk /tmp/dk_holidays.txt`
 
 ### Core code
 

holidays/.gitignore

@@ -0,0 +1 @@
+requests_cache.sqlite

holidays/README.md

@@ -6,7 +6,7 @@
 
 Because each country/culture has it‘s own important dates/events, holidays are defined on a country level. The country wide definition can be overwritten as needed by more specific definitions. Because this library works with the OSM ecosystem, those boundaries are based on OSM.
 
-More specifically, the dataset on which a decision is made which holidays apply for a given location is based on the information returned from [Nominatim](https://wiki.openstreetmap.org/wiki/Nominatim).
+More specifically, the dataset on which a decision is made which holidays apply for a given location is based on [Nominatim](https://wiki.openstreetmap.org/wiki/Nominatim).
 
 Consider this [Nominatim reverse geocoding][] query: https://nominatim.openstreetmap.org/reverse?format=json&lat=49.5487&lon=9.8160&zoom=18&addressdetails=1&accept-language=de,en
 
@@ -45,23 +45,23 @@ as of 2016-06-29.
 For now it has been enough to make a decision based on the fields `address.country_code` and `address.state` and thus only those two levels are supported right now in the data format. But the other information is there when needed, just extend the data format and source code to make use of it.
 
 Note that you will need to use exactly the same values that Nominatim returns in the holiday definition data format which is described next.
-Also note that the data format is based on Nominatim results in local language so you will likely need to adjust the `accept-language` URL get parameter from the example.
+Also note that the definition is based on Nominatim results in local language so you will likely need to adjust the `accept-language` URL get parameter from the example.
 Refer to [Nominatim/Country Codes](https://wiki.openstreetmap.org/wiki/Nominatim/Country_Codes) for the country codes to language code mapping used for this specification.
 
-You can use https://www.openstreetmap.org to get the coordinates for the regions you are defining holidays for. Just search the region and position the map view on that region. Copy the latitude and longitude from the address bar. Consider this example of a permalink:
+You can use https://www.openstreetmap.org to get the coordinates for the states you are defining holidays for. Just search the state and position the map view on that state. Copy the latitude and longitude from the address bar. Consider this example of a permalink:
 
 https://www.openstreetmap.org/#map=15/49.5487/9.8160
 
-You can now use the `&lat=49.5487&lon=9.8160` parameters and use them instead of the once in the example Nominatim query shown above. Note that you should include this Nominatim URL for each defined region using the `_nominatim_url` key (see below).
+You can now use the `&lat=49.5487&lon=9.8160` parameters and use them instead of the once in the example Nominatim query shown above. Note that you should include this Nominatim URL for each defined state using the `_nominatim_url` key (see below).
 The `_nominatim_url` is intended to make testing of the holiday definitions easier.
 
 You could also do a [Nominatim search][] directly using: https://nominatim.openstreetmap.org/search?format=json&country=Deutschland&state=Berlin&&zoom=18&addressdetails=1&limit=1&accept-language=de,en
 and then do the [Nominatim reverse geocoding][] using the returned coordinates. But note that the `_nominatim_url` needs to be
 a [Nominatim reverse geocoding][] query because of different attributes being returned.
 
-## Holiday definition data format
+## Holiday definition format
 
-Data format version `2.2.0`. The data format will probably need to be adopted to support more holiday definitions in the future.
+Data format version `2.2.0`. The data format will probably need to be adapted to support more holiday definitions in the future.
 The data format versioning complies with [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 Each country has it’s own [YAML] file below [./holidays/][ohlib.holidays] with
@@ -78,8 +78,13 @@ _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=49.
 PH: {}  # https://de.wikipedia.org/wiki/Feiertage_in_Deutschland
 
 Baden-Württemberg:  # Does only apply in Baden-Württemberg
+  _state_code: bw
+  # Short string which can be used to refer to this entry in the test framework.
+  # Needs to be unique for the country wide definition.
+  # Should be specified when a commonly known code exists for the country/state.
+
   _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=49.5487&lon=9.8160&zoom=18&addressdetails=1&accept-language=de,en'
-  # Somewhere in this state/region.
+  # Somewhere in this state/state.
 
   # PH: {}
   # This more specific rule set overwrites the country wide one (they are just ignored).
@@ -94,7 +99,7 @@ The dictionary of the country either consists of `PH`, `SH` or the value of `add
 
 Note that the data format versions below 2.2.0 used JSON as data serialization language. The data structure remains the same as 2.1.0 however.
 
-### Holiday definition data format: PH
+### Holiday definition format: PH
 
 Now lets look at the public holiday (`PH`) definition in more detail. Each `PH` definition consists of a dictionary with holiday name as key and date definition as values (specifying one day).
 Holiday names should be in the local language. A date definition either consists of two integers representing month and day or the name of a movable event and the offset to that event. The movable events and the formulas for calculating them for a given year are defined in the `getMovableEventsForYear` function.
@@ -122,7 +127,7 @@ PH:  # https://de.wikipedia.org/wiki/Feiertage_in_Deutschland
 
 Note that the array of date definitions can hold an optional array of `address.state` strings. When this list is present, the holiday is only applied when `address.state` is found in that array of strings.
 
-### Holiday definition data format: SH
+### Holiday definition format: SH
 
 School holiday (`SH`) definitions look a little bit different. This is because school holidays usually spans multiple days and because `SH` are different for each year/can not be mathematically calculated (at least the countries that @ypid has seen so far).
 This is not very nice, but as we are hackers, we can just grab the data and convert it into the data format documented here. This is what @ypid has been doing for all states in Germany using the [hc] tool which was developed for use with this library and which fellows might find useful to convert holidays for other countries as well.

holidays/br.yaml

@@ -1,5 +1,7 @@
 ---
 
+_nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=-10&lon=-52&zoom=18&addressdetails=1&accept-language=pt,en'
+
 PH:  # https://pt.wikipedia.org/wiki/Feriados_no_Brasil
   'Ano Novo': [1, 1]
   'Carnaval': [easter, -47]

holidays/cz.yaml

@@ -1,5 +1,7 @@
 ---
 
+_nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=50.0874401&lon=14.4212556&zoom=18&addressdetails=1&accept-language=cs,en'
+
 # http://www.mpsv.cz/cs/74 or english: https://en.wikipedia.org/wiki/Public_holidays_in_the_Czech_Republic
 PH:
   'Den obnovy samostatného českého státu': [1, 1]

holidays/de.yaml

@@ -23,7 +23,8 @@ PH:  # https://de.wikipedia.org/wiki/Feiertage_in_Deutschland
 
 # Everything below is generated and kept up-to-date by hc.
 Baden-Württemberg:
-  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=49.5487&lon=9.8160&zoom=18&addressdetails=1&accept-language=de,en'
+  _state_code: bw
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=49.5487429714954&lon=9.81602098644987&zoom=18&addressdetails=1'
   SH:
     - name: Osterferien
       '2012': [4, 2, 4, 13]
@@ -80,6 +81,8 @@ Baden-Württemberg:
       '2019': [12, 23, 1, 4]
 
 Bayern:
+  _state_code: by
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=48.9467562&lon=11.4038717&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2012': [2, 20, 2, 24]
@@ -166,6 +169,8 @@ Bayern:
       '2023': [12, 23, 1, 5]
 
 Berlin:
+  _state_code: be
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=52.5170365&lon=13.3888599&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2012': [1, 30, 2, 4]
@@ -252,6 +257,8 @@ Berlin:
       '2023': [12, 23, 1, 5]
 
 Brandenburg:
+  _state_code: bb
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=52.8455492&lon=13.2461296&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2012': [1, 30, 2, 4]
@@ -338,6 +345,8 @@ Brandenburg:
       '2023': [12, 23, 1, 5]
 
 Bremen:
+  _state_code: hb
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=53.0758196&lon=8.8071646&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2012': [1, 30, 1, 31]
@@ -424,6 +433,8 @@ Bremen:
       '2023': [12, 23, 1, 5]
 
 Hamburg:
+  _state_code: hh
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=53.5437641&lon=10.0099133&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2012': [1, 30, 1, 30]
@@ -510,6 +521,8 @@ Hamburg:
       '2023': [12, 22, 1, 5]
 
 Hessen:
+  _state_code: he
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=50.6118537&lon=9.1909725&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Osterferien
       '2012': [4, 2, 4, 14]
@@ -568,6 +581,8 @@ Hessen:
       '2023': [12, 27, 1, 13]
 
 Mecklenburg-Vorpommern:
+  _state_code: mv
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=53.7735234&lon=12.5755746&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2012': [2, 6, 2, 17]
@@ -654,6 +669,8 @@ Mecklenburg-Vorpommern:
       '2023': [12, 21, 1, 3]
 
 Niedersachsen:
+  _state_code: ni
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=52.6368397&lon=9.8456838&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2012': [1, 30, 1, 31]
@@ -740,6 +757,8 @@ Niedersachsen:
       '2023': [12, 27, 1, 5]
 
 Nordrhein-Westfalen:
+  _state_code: nw
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=51.4785568&lon=7.5533645&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Osterferien
       '2012': [4, 2, 4, 14]
@@ -811,6 +830,8 @@ Nordrhein-Westfalen:
       '2023': [12, 21, 1, 5]
 
 Rheinland-Pfalz:
+  _state_code: rp
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=49.7497346&lon=7.4396553&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2019': [2, 25, 3, 1]
@@ -877,6 +898,8 @@ Rheinland-Pfalz:
       '2023': [12, 27, 1, 5]
 
 Saarland:
+  _state_code: sl
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=49.4173988&lon=6.9805789&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2012': [2, 20, 2, 25]
@@ -954,6 +977,8 @@ Saarland:
       '2023': [12, 21, 1, 2]
 
 Sachsen:
+  _state_code: sn
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=50.9295798&lon=13.4585052&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2012': [2, 13, 2, 25]
@@ -1040,6 +1065,8 @@ Sachsen:
       '2023': [12, 23, 1, 2]
 
 Sachsen-Anhalt:
+  _state_code: st
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=51.908526&lon=11.4939134&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2012': [2, 4, 2, 11]
@@ -1106,6 +1133,8 @@ Sachsen-Anhalt:
       '2019': [12, 23, 1, 4]
 
 Schleswig-Holstein:
+  _state_code: sh
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=54.1853998&lon=9.8220089&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Osterferien
       '2012': [3, 30, 4, 13]
@@ -1178,6 +1207,8 @@ Schleswig-Holstein:
       '2023': [12, 27, 1, 6]
 
 Thüringen:
+  _state_code: th
+  _nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=50.7333163&lon=11.0747905&zoom=18&addressdetails=1&accept-language=de,en'
   SH:
     - name: Winterferien
       '2012': [2, 6, 2, 11]

holidays/dk.yaml

@@ -1,5 +1,7 @@
 ---
 
+_nominatim_url: 'https://nominatim.openstreetmap.org/reverse?format=json&lat=55.670249&lon=10.3333283&zoom=18&addressdetails=1&accept-language=da,en'
+
 PH:  # https://en.wikipedia.org/wiki/Public_holidays_in_Denmark
   'Nytårsdag': [1, 1]
   'Skærtorsdag': [easter, -3]