Merge pull request #2047 from tilezen/nvkelso/water-lake-labels-fix

Restore early lake labels, add kind_detail for lakes and etc

docs/layers.md

@@ -2508,12 +2508,24 @@ Tilezen calculates the composite exterior edge for overlapping water polygons an
 * `swimming_pool` - polygon
 * `water` - polygon
 
+#### Water `kind_detail` values:
+
+When `water` polygons are sourced from OpenStreetMap, we add `kind_detail` values in an allowlist.
+
+* `basin`
+* `canal`
+* `ditch`
+* `drain`
+* `lake` - remap of lake, lagoon, oxbow, pond, reservoir, and wastewater source values
+* `river`
+* `stream`
+
 Additionally, a `reservoir: true` or `alkaline: true` value can be present on the appropriate `kind=lake` features. Intermittent water features that sometimes run dry or disappear seasonally are marked `intermittent: true`.
 
 The kinds `bay`, `strait` and `fjord` are ranked by size and given a `kind_tile_rank` property that starts from 1 and counts up as the feature gets smaller. Note that the ranking is done on a "metatile", which means that each tile (of size 256px, 512px or other) won't necessarily contain the full range from 1 to N of `kind_tile_rank`s.
 
 **Gotchas:**
 
-* `lake` features with `alkaline: true` and `playa` features are sourced solely from Natural Earth. Zooming in, your feature may disappear (there is no equivalent in OpenStreetMap). Beware the desert around Great Salt Lake in Utah!
-* `lake` features from Natural Earth sometimes change to `water` features on zoom into OpenStreetMap data. _See planned bug fix in [#984](https://github.com/tilezen/vector-datasource/issues/984)._
+* `lake` features with `alkaline: true` and `playa` features are sourced from Natural Earth at low zooms and are sparesly populated at high zooms from OpenStreetMap. Zooming in, your feature may disappear (when there is no equivalent in OpenStreetMap), or the feature may still exist as a water or lake polygon but without the alkaline indicator. Beware the desert around Great Salt Lake in Utah!
+* `lake` features from Natural Earth sometimes change to `water` features on zoom into OpenStreetMap data. _See planned bug fix in [#984](https://github.com/tilezen/vector-datasource/issues/984). However, kind_detail value is available._
 * Some of the minor kinds (like `bay`, `strait`, and `fjord`) are used for label_placement points only, as their area would duplicate water polygons already present from osmdata.openstreetmap.de.

integration-test/148-sea-ocean-labels-water-layer.py

@@ -1,6 +1,7 @@
 # -*- encoding: utf-8 -*-
 import dsl
 from shapely.wkt import loads as wkt_loads
+from tilequeue.tile import deg2num
 
 from . import FixtureTest
 
@@ -10,27 +11,32 @@
 class SeaOceanLabelsWaterLayer(FixtureTest):
 
     def test_gulf_of_california(self):
+        z = 16
+        x, y = deg2num(27.5578100671435, -111.704650024786, z)
+
         # Gulf of California: http://www.openstreetmap.org/node/305639734
         self.generate_fixtures(dsl.way(305639734, wkt_loads('POINT (-111.704650024786 27.5578100671435)'), {u'name:ia': u'Golfo de California', u'alt_name:eo': u'Maro de Korteso', u'name:ms': u'Teluk California', u'name:ko': u'\uce7c\ub9ac\ud3ec\ub974\ub2c8\uc544 \ub9cc', u'gns:dsg': u'GULF', u'name:cs': u'Kalifornsk\xfd z\xe1liv', u'alt_name:pl': u'Morze Cort\xe9za', u'wikidata': u'Q132811', u'name:it': u'Golfo di California', u'name:vi': u'V\u1ecbnh Ca Li', u'name:ru': u'\u041a\u0430\u043b\u0438\u0444\u043e\u0440\u043d\u0438\u0439\u0441\u043a\u0438\u0439 \u0437\u0430\u043b\u0438\u0432', u'name:pl': u'Zatoka Kalifornijska', u'name:ta': u'\u0b95\u0bb2\u0bbf\u0baa\u0bcb\u0bb0\u0bcd\u0ba9\u0bbf\u0baf\u0bbe \u0bb5\u0bb3\u0bc8\u0b95\u0bc1\u0b9f\u0bbe', u'wikipedia': u'en:Gulf of California', u'name:de': u'Golf von Kalifornien', u'source': u'openstreetmap.org',
                                u'name:fr': u'Golfe de Californie', u'name:zh': u'\u52a0\u5229\u798f\u5c3c\u4e9a\u6e7e', u'name:sl': u'Kalifornijski zaliv', u'name:lt': u'Kalifornijos \u012flanka', u'gns:uni': u'-2327700', u'name:he': u'\u05de\u05e4\u05e8\u05e5 \u05e7\u05dc\u05d9\u05e4\u05d5\u05e8\u05e0\u05d9\u05d4', u'sqkm': u'160000', u'name:uk': u'\u041a\u0430\u043b\u0456\u0444\u043e\u0440\u043d\u0456\u0439\u0441\u044c\u043a\u0430 \u0437\u0430\u0442\u043e\u043a\u0430', u'name:hu': u'Kaliforniai-\xf6b\xf6l', u'name:el': u'\u039a\u03cc\u03bb\u03c0\u03bf\u03c2 \u03c4\u03b7\u03c2 \u039a\u03b1\u03bb\u03b9\u03c6\u03cc\u03c1\u03bd\u03b9\u03b1\u03c2', u'name:eo': u'Kalifornia Golfo', u'name:en': u'Gulf of California', u'natural': u'sea', u'name': u'Gulf of California', u'place': u'sea', u'alt_name:cs': u'Cort\xe9zovo mo\u0159e', u'name:es': u'Golfo de California'}))
         self.assert_has_feature(
-            9, 97, 215, 'water',
+            z, x, y, 'water',
             {'kind': 'sea', 'name': 'Gulf of California',
              'label_placement': True})
         self.assert_no_matching_feature(
             9, 97, 215, 'places',
             {'kind': 'sea', 'name': 'Gulf of California'})
 
     def test_greenland_sea(self):
+        z = 16
+        x, y = deg2num(74.9999999953685, -10.0000000185705, z)
         # Greenland Sea: http://www.openstreetmap.org/node/305639396
         self.generate_fixtures(dsl.way(305639396, wkt_loads('POINT (-10.0000000185705 74.9999999953685)'), {u'name:pt': u'Mar da Gronel\xe2ndia', u'name:ms': u'Laut Greenland', u'name:ko': u'\uadf8\ub9b0\ub780\ub4dc \ud574', u'gns:dsg': u'SEA', u'name:ar': u'\u0628\u062d\u0631 \u063a\u0631\u064a\u0646\u0644\u0627\u0646\u062f', u'name:cs': u'Gr\xf3nsk\xe9 mo\u0159e', u'wikidata': u'Q132868', u'name:it': u'Mare di Groenlandia', u'name:vi': u'Bi\u1ec3n Greenland', u'name:pl': u'Morze Grenlandzkie', u'name:fi': u'Gr\xf6nlanninmeri', u'name:da': u'Gr\xf8nlandshavet', u'wikipedia': u'en:Greenland Sea',
                                u'name:de': u'Gr\xf6nlandsee', u'source': u'openstreetmap.org', u'name:fr': u'Mer du Groenland', u'name:zh': u'\u683c\u9675\u5170\u6d77', u'name:sk': u'Gr\xf3nske more', u'name:lt': u'Grenlandijos j\u016bra', u'gns:uni': u'-2149780', u'name:uk': u'\u0413\u0440\u0435\u043d\u043b\u0430\u043d\u0434\u0441\u044c\u043a\u0435 \u043c\u043e\u0440\u0435', u'name:sv': u'Gr\xf6nlandshavet', u'name:hu': u'Gr\xf6nlandi-tenger', u'name:hr': u'Grenlandsko more', u'name:eo': u'Gronlanda Maro', u'name:en': u'Greenland Sea', u'name': u'Greenland Sea', u'place': u'sea'}))
         self.assert_has_feature(
-            9, 241, 90, 'water',
+            z, x, y, 'water',
             {'kind': 'sea', 'name': 'Greenland Sea',
              'label_placement': True})
         self.assert_no_matching_feature(
-            9, 241, 90, 'places',
+            z, x, y, 'places',
             {'kind': 'sea', 'name': 'Greenland Sea'})
 
 # NOTE: No ocean points in the North America extract :-(

integration-test/1730-further-water-layer-name-dropping.py

@@ -216,6 +216,6 @@ def test_label_lake_athabasca_z5(self):
             z, x, y, 'water', {
                 'kind': 'lake',
                 'min_zoom': 2,  # min_zoom changed at https://github.com/tilezen/vector-datasource/pull/2010/
-                'name': str,
-                'name:de': str,
+                'name': type(None),
+                'name:de': type(None),
             })

integration-test/1838-too-many-bays.py

@@ -7,7 +7,7 @@ class BayTest(FixtureTest):
     def test_bays(self):
         import dsl
 
-        z, x, y = 8, 0, 0
+        z, x, y = 10, 0, 0
 
         def _bay(osm_id, area, name):
             return dsl.way(osm_id, dsl.box_area(z, x, y, area), {

integration-test/2047-restore-early-lake-labels.py

@@ -0,0 +1,128 @@
+# -*- encoding: utf-8 -*-
+import dsl
+from shapely.wkt import loads as wkt_loads
+from tilequeue.tile import deg2num
+
+from . import FixtureTest
+
+
+class WaterKinds(FixtureTest):
+    def test_water_reservoir(self):
+        self.generate_fixtures(dsl.way(1, wkt_loads(
+            'POLYGON ((-122.421611012467 37.80395991667648, '
+            '-122.420227786592 37.80413210674329, -122.420159873957 '
+            '37.80408277780808, -122.420117832801 37.80383748120349, '
+            '-122.42154480663 37.80365748294528, -122.421611012467 '
+            '37.80395991667648))'),
+            {
+            u'way_area': u'6936.45',
+            u'natural': u'water',
+            u'source': u'openstreetmap.org',
+            u'water': u'reservoir',
+            u'name': u'Francisco Reservoir',
+            u'covered': u'no'}))
+
+        self.assert_has_feature(
+            16, 10481, 25324, 'water',
+            {
+                'kind': 'water',
+                'reservoir': True,
+                'kind_detail': 'lake'
+            })
+
+    def test_water_lagoon(self):
+        self.generate_fixtures(dsl.way(1, wkt_loads(
+            'POLYGON ((-122.421611012467 37.80395991667648, '
+            '-122.420227786592 37.80413210674329, -122.420159873957 '
+            '37.80408277780808, -122.420117832801 37.80383748120349, '
+            '-122.42154480663 37.80365748294528, -122.421611012467 '
+            '37.80395991667648))'),
+            {
+            u'natural': u'water',
+            u'way_area': u'6936.45',
+            u'source': u'openstreetmap.org',
+            u'water': u'lagoon',
+            u'name': u'Francisco Reservoir',
+            u'covered': u'no'}))
+
+        self.assert_has_feature(
+            16, 10481, 25324, 'water',
+            {
+                'kind': 'water',
+                'alkaline': True,
+                'kind_detail': 'lake'
+            })
+
+    def test_water_ditch(self):
+        self.generate_fixtures(dsl.way(1, wkt_loads(
+            'POLYGON ((-122.421611012467 37.80395991667648, -122.420227786592 37.80413210674329, -122.420159873957 37.80408277780808, -122.420117832801 37.80383748120349, -122.42154480663 37.80365748294528, -122.421611012467 37.80395991667648))'),
+            {
+            u'natural': u'water',
+            u'way_area': u'6936.45',
+            u'source': u'openstreetmap.org',
+            u'water': u'ditch',
+            u'name': u'Francisco Reservoir',
+            u'covered': u'no'}))
+
+        self.assert_has_feature(
+            16, 10481, 25324, 'water',
+            {
+                'kind': 'water',
+                'kind_detail': 'ditch'
+            })
+
+    def test_prop_drop(self):
+        import dsl
+
+        lon, lat = (-122.417169, 37.769196)
+
+        for z in range(0, 16):
+            x, y = deg2num(lat, lon, z)
+            area = 265548000
+            self.generate_fixtures(
+                dsl.way(1, dsl.box_area(z, x, y, area), {
+                    'name': u'Clear Lake',
+                    'natural': u'water',
+                    'water': u'lake',
+                    'wikidata': u'Q1099503',
+                    'source': u'openstreetmap.org',
+                }),
+            )
+
+            if z < 6:
+                self.assert_no_matching_feature(z, x, y, 'water',
+                                                {
+                                                    'kind': 'water',
+                                                })
+            if 6 <= z <= 14:
+                with self.features_in_tile_layer(z, x, y, 'water') as features:
+                    for f in features:
+                        if f['geometry']['type'] == 'Polygon':
+                            assert f['id'] is None
+                            assert f['properties']['min_zoom'] == 6
+                            assert 'name' not in f['properties']
+                            assert 'old_name' not in f['properties']
+            if z == 15:
+                with self.features_in_tile_layer(z, x, y, 'water') as features:
+                    for f in features:
+                        if f['geometry']['type'] == 'Polygon':
+                            assert f['id'] is None
+                            assert f['properties']['name'] == 'Clear Lake'
+                            assert f['properties']['kind'] == 'water'
+                            assert f['properties']['kind_detail'] == 'lake'
+
+            if z < 9:
+                self.assert_no_matching_feature(z, x, y, 'water',
+                                                {
+                                                    'kind': 'water',
+                                                    'label_placement': True,
+                                                })
+
+            if z >= 9:
+                self.assert_has_feature(z, x, y, 'water', {
+                    'kind': 'water',
+                    'name': 'Clear Lake',
+                    'kind_detail': 'lake',
+                    'label_placement': True,
+                    'min_zoom': 9.0,
+                })

queries.yaml

@@ -813,24 +813,32 @@ post_process:
       properties:
         - name
         - old_name
+      # TODO the properties.get('is_tunnel') is True might not work, need to fix it; refer to a similar fix in https://github.com/tilezen/vector-datasource/pull/2047
       where: >-
         (kind == 'sea' and zoom < 4) or
         (kind == 'bay' and zoom < 5) or
         (kind == 'fjord' and zoom < 5) or
         (kind == 'strait' and zoom < 5) or
         (kind == 'lake' and zoom < 5) or
+        (kind_detail == 'lake' and zoom < 5) or
         (kind == 'playa' and zoom < 6) or
         (kind == 'reef' and zoom < 6) or
         (kind == 'river' and zoom <  12) or
+        (kind_detail == 'river' and zoom <  12) or
         (kind == 'riverbank' and zoom <  12) or
-        (kind == 'water' and zoom <  12) or
+        (kind == 'water' and kind_detail is None and zoom <  12) or
         (kind == 'canal' and zoom <  13) or
+        (kind_detail == 'canal' and zoom <  13) or
         (kind == 'basin' and zoom <  13) or
+        (kind_detail == 'basin' and zoom <  13) or
         (kind == 'dock' and zoom <  13) or
         (kind == 'dam' and zoom <  14) or
         (kind == 'stream' and zoom <  14) or
+        (kind_detail == 'stream' and zoom <  14) or
         (kind == 'ditch' and zoom <  15) or
+        (kind_detail == 'ditch' and zoom <  15) or
         (kind == 'drain' and zoom <  15) or
+        (kind_detail == 'drain' and zoom <  15) or
         (kind == 'swimming_pool' and zoom <  15) or
         (kind == 'fountain' and zoom <  15) or
         (properties is not None and properties.get('is_tunnel') is True)
@@ -880,8 +888,10 @@ post_process:
             (15,        10000),
             (16,         None),
         ] if area >= area_threshold][0]
+      # note the following line cannot be written as
+      # properties is not None and properties.get('label_placement') is True
       where: >-
-        (properties is not None and properties.get('label_placement') is True)
+        properties is not None and label_placement
 
   # now that we have the label points
   # drop most water properties at lower zooms

queries/planet_osm_point.jinja2

@@ -52,7 +52,6 @@ SELECT
   END AS __water_properties__
 
 FROM (
-
   SELECT
     osm_id,
     way,

vectordatasource/transform.py

@@ -463,7 +463,6 @@ def pois_direction_int(shape, props, fid, zoom):
     direction = props.get('direction')
     if not direction:
         return shape, props, fid
-
     props['direction'] = _to_int_degrees(direction)
     return shape, props, fid
 
@@ -9535,6 +9534,9 @@ def update_min_zoom(ctx):
         local = defaultdict(lambda: None)
         local.update(props)
         local['zoom'] = zoom
+        # this is to make the name `properties` visible in the queries.yaml's
+        # where clause
+        local['properties'] = props
 
         if where and eval(where, {}, local):
             new_min_zoom = eval(min_zoom, {}, local)

yaml/water.yaml

@@ -80,6 +80,25 @@ filters:
     output:
       <<: *water_standard_properties_osm
       kind: water
+      # OSM buries the detailed type of water so let's surface that, see Issue #984
+      # in v2.x series we'd promote these to kind values, but we add them as kind_detail
+      # here as a non-breaking change for v1.x series as a bridge
+      kind_detail:
+        case:
+          # existing Tilezen kinds
+          - when: { water: [basin, canal, ditch, drain, lake, river, stream] }
+            then: { col: water }
+          # other common OSM water values that we'd expect to map to Tilezen lake value
+          - when: { water: [lagoon, oxbow, pond, reservoir, wastewater] }
+            then: lake
+      reservoir:
+        case:
+          - when: { water: [reservoir] }
+            then: true
+      alkaline:
+        case:
+          - when: { water: [lagoon, salt, salt_pool] }
+            then: true
     table: osm
   - filter: {natural: bay}
     min_zoom: *water_standard_min_zoom