Merge branch 'master' into json-join

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 1.26.7
+
+* Add an option to quiet the progress indicator but not warnings
+* Enable more compiler warnings and fix related problems
+
 ## 1.26.6
 
 * Be more careful about checking for overflow when parsing numbers

Makefile

@@ -9,7 +9,7 @@ CXX := $(CXX)
 CFLAGS := $(CFLAGS)
 CXXFLAGS := $(CXXFLAGS) -std=c++11
 LDFLAGS := $(LDFLAGS)
-WARNING_FLAGS := -Wall -Wshadow -Wsign-compare
+WARNING_FLAGS := -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow
 RELEASE_FLAGS := -O3 -DNDEBUG
 DEBUG_FLAGS := -O0 -DDEBUG -fno-inline-functions -fno-omit-frame-pointer
 

README.md

@@ -96,13 +96,15 @@ delete the file that already exists with that name.
 If you aren't sure what the right maxzoom is for your data, `-zg` will guess one for you
 based on the density of features.
 
-If you are mapping point features, you will often want to use `-Bg` to automatically choose
-a base zoom level for dot dropping. If that doesn't work out for you, try
-`-r1 --drop-fraction-as-needed` to turn off the normal dot dropping and instead
-only drop features if the tiles get too big.
+Tippecanoe will normally drop a fraction of point features at zooms below the maxzoom,
+to keep the low-zoom tiles from getting too big. If you have a smaller data set where
+all the points would fit without dropping any of them, use `-r1` to keep them all.
+If you do want point dropping, but you still want the tiles to be denser than `-zg`
+thinks they should be, use `-B` to set a basezoom lower than the maxzoom.
 
-If you are mapping points or polygons, you will often want to use `--drop-densest-as-needed`
-to drop some of them if necessary to make the low zoom levels work.
+If some of your tiles are coming out too big in spite of the settings above, you will
+often want to use `--drop-densest-as-needed` to drop whatever fraction of the features
+is necessary at each zoom level to make that zoom level's tiles work.
 
 If your features have a lot of attributes, use `-y` to keep only the ones you really need.
 
@@ -262,7 +264,8 @@ tippecanoe -z5 -o filtered.mbtiles -j '{ "ne_10m_admin_0_countries": [ "all", [
 
 ### Progress indicator
 
- * `-q` or `--quiet`: Work quietly instead of reporting progress
+ * `-q` or `--quiet`: Work quietly instead of reporting progress or warning messages
+ * `-Q` or `--no-progress-indicator`: Don't report progress, but still give warnings
  * `-v` or `--version`: Report Tippecanoe's version number
 
 ### Filters

decode.cpp

@@ -120,7 +120,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st
 		}
 
 		// X and Y are unsigned, so no need to check <0
-		if (x > (1 << z) || y > (1 << z)) {
+		if (x > (1ULL << z) || y > (1ULL << z)) {
 			fprintf(stderr, "Impossible tile %d/%u/%u\n", z, x, y);
 			exit(EXIT_FAILURE);
 		}

geobuf.cpp

@@ -22,13 +22,13 @@
 #define MULTIPOLYGON 5
 
 struct queued_feature {
-	protozero::pbf_reader pbf;
-	size_t dim;
-	double e;
-	std::vector<std::string> *keys;
-	struct serialization_state *sst;
-	int layer;
-	std::string layername;
+	protozero::pbf_reader pbf{};
+	size_t dim = 0;
+	double e = 0;
+	std::vector<std::string> *keys = NULL;
+	std::vector<struct serialization_state> *sst = NULL;
+	int layer = 0;
+	std::string layername = "";
 };
 
 static std::vector<queued_feature> feature_queue;
@@ -40,7 +40,7 @@ void ensureDim(size_t dim) {
 	}
 }
 
-serial_val readValue(protozero::pbf_reader &pbf, std::vector<std::string> &keys) {
+serial_val readValue(protozero::pbf_reader &pbf) {
 	serial_val sv;
 	sv.type = mvt_null;
 	sv.s = "null";
@@ -94,7 +94,7 @@ serial_val readValue(protozero::pbf_reader &pbf, std::vector<std::string> &keys)
 	return sv;
 }
 
-drawvec readPoint(std::vector<long long> &coords, std::vector<int> &lengths, size_t dim, double e) {
+drawvec readPoint(std::vector<long long> &coords, size_t dim, double e) {
 	ensureDim(dim);
 
 	long long x, y;
@@ -104,7 +104,7 @@ drawvec readPoint(std::vector<long long> &coords, std::vector<int> &lengths, siz
 	return dv;
 }
 
-drawvec readLinePart(std::vector<long long> &coords, std::vector<int> &lengths, size_t dim, double e, size_t start, size_t end, bool closed) {
+drawvec readLinePart(std::vector<long long> &coords, size_t dim, double e, size_t start, size_t end, bool closed) {
 	ensureDim(dim);
 
 	drawvec dv;
@@ -141,19 +141,19 @@ drawvec readLinePart(std::vector<long long> &coords, std::vector<int> &lengths,
 	return dv;
 }
 
-drawvec readLine(std::vector<long long> &coords, std::vector<int> &lengths, size_t dim, double e, bool closed) {
-	return readLinePart(coords, lengths, dim, e, 0, coords.size(), closed);
+drawvec readLine(std::vector<long long> &coords, size_t dim, double e, bool closed) {
+	return readLinePart(coords, dim, e, 0, coords.size(), closed);
 }
 
 drawvec readMultiLine(std::vector<long long> &coords, std::vector<int> &lengths, size_t dim, double e, bool closed) {
 	if (lengths.size() == 0) {
-		return readLinePart(coords, lengths, dim, e, 0, coords.size(), closed);
+		return readLinePart(coords, dim, e, 0, coords.size(), closed);
 	}
 
 	drawvec dv;
 	size_t here = 0;
 	for (size_t i = 0; i < lengths.size(); i++) {
-		drawvec dv2 = readLinePart(coords, lengths, dim, e, here, here + lengths[i] * dim, closed);
+		drawvec dv2 = readLinePart(coords, dim, e, here, here + lengths[i] * dim, closed);
 		here += lengths[i] * dim;
 
 		for (size_t j = 0; j < dv2.size(); j++) {
@@ -168,7 +168,7 @@ drawvec readMultiPolygon(std::vector<long long> &coords, std::vector<int> &lengt
 	ensureDim(dim);
 
 	if (lengths.size() == 0) {
-		return readLinePart(coords, lengths, dim, e, 0, coords.size(), true);
+		return readLinePart(coords, dim, e, 0, coords.size(), true);
 	}
 
 	size_t polys = lengths[0];
@@ -180,7 +180,7 @@ drawvec readMultiPolygon(std::vector<long long> &coords, std::vector<int> &lengt
 		size_t rings = lengths[n++];
 
 		for (size_t j = 0; j < rings; j++) {
-			drawvec dv2 = readLinePart(coords, lengths, dim, e, here, here + lengths[n] * dim, true);
+			drawvec dv2 = readLinePart(coords, dim, e, here, here + lengths[n] * dim, true);
 			here += lengths[n] * dim;
 			n++;
 
@@ -196,8 +196,8 @@ drawvec readMultiPolygon(std::vector<long long> &coords, std::vector<int> &lengt
 }
 
 struct drawvec_type {
-	drawvec dv;
-	int type;
+	drawvec dv{};
+	int type = 0;
 };
 
 std::vector<drawvec_type> readGeometry(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys) {
@@ -246,11 +246,11 @@ std::vector<drawvec_type> readGeometry(protozero::pbf_reader &pbf, size_t dim, d
 
 	drawvec_type dv;
 	if (type == POINT) {
-		dv.dv = readPoint(coords, lengths, dim, e);
+		dv.dv = readPoint(coords, dim, e);
 	} else if (type == MULTIPOINT) {
-		dv.dv = readLine(coords, lengths, dim, e, false);
+		dv.dv = readLine(coords, dim, e, false);
 	} else if (type == LINESTRING) {
-		dv.dv = readLine(coords, lengths, dim, e, false);
+		dv.dv = readLine(coords, dim, e, false);
 	} else if (type == POLYGON) {
 		dv.dv = readMultiLine(coords, lengths, dim, e, true);
 	} else if (type == MULTIPOLYGON) {
@@ -311,7 +311,7 @@ void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<s
 
 		case 13: {
 			protozero::pbf_reader value_reader(pbf.get_message());
-			values.push_back(readValue(value_reader, keys));
+			values.push_back(readValue(value_reader));
 			break;
 		}
 
@@ -425,14 +425,18 @@ struct queue_run_arg {
 	size_t start;
 	size_t end;
 	size_t segment;
+
+	queue_run_arg(size_t start1, size_t end1, size_t segment1)
+	    : start(start1), end(end1), segment(segment1) {
+	}
 };
 
 void *run_parse_feature(void *v) {
 	struct queue_run_arg *qra = (struct queue_run_arg *) v;
 
 	for (size_t i = qra->start; i < qra->end; i++) {
 		struct queued_feature &qf = feature_queue[i];
-		readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &qf.sst[qra->segment], qf.layer, qf.layername);
+		readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &(*qf.sst)[qra->segment], qf.layer, qf.layername);
 	}
 
 	return NULL;
@@ -443,16 +447,21 @@ void runQueue() {
 		return;
 	}
 
-	struct queue_run_arg qra[CPUS];
-	pthread_t pthreads[CPUS];
+	std::vector<struct queue_run_arg> qra;
+
+	std::vector<pthread_t> pthreads;
+	pthreads.resize(CPUS);
 
 	for (size_t i = 0; i < CPUS; i++) {
-		*(feature_queue[0].sst[i].layer_seq) = *(feature_queue[0].sst[0].layer_seq) + feature_queue.size() * i / CPUS;
+		*((*(feature_queue[0].sst))[i].layer_seq) = *((*(feature_queue[0].sst))[0].layer_seq) + feature_queue.size() * i / CPUS;
 
-		qra[i].start = feature_queue.size() * i / CPUS;
-		qra[i].end = feature_queue.size() * (i + 1) / CPUS;
-		qra[i].segment = i;
+		qra.push_back(queue_run_arg(
+			feature_queue.size() * i / CPUS,
+			feature_queue.size() * (i + 1) / CPUS,
+			i));
+	}
 
+	for (size_t i = 0; i < CPUS; i++) {
 		if (pthread_create(&pthreads[i], NULL, run_parse_feature, &qra[i]) != 0) {
 			perror("pthread_create");
 			exit(EXIT_FAILURE);
@@ -467,11 +476,11 @@ void runQueue() {
 		}
 	}
 
-	*(feature_queue[0].sst[0].layer_seq) = *(feature_queue[0].sst[CPUS - 1].layer_seq);
+	*((*(feature_queue[0].sst))[0].layer_seq) = *((*(feature_queue[0].sst))[CPUS - 1].layer_seq);
 	feature_queue.clear();
 }
 
-void queueFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername) {
+void queueFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, std::vector<struct serialization_state> *sst, int layer, std::string layername) {
 	struct queued_feature qf;
 	qf.pbf = pbf;
 	qf.dim = dim;
@@ -488,7 +497,7 @@ void queueFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<
 	}
 }
 
-void outBareGeometry(drawvec const &dv, int type, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername) {
+void outBareGeometry(drawvec const &dv, int type, struct serialization_state *sst, int layer, std::string layername) {
 	serial_feature sf;
 
 	sf.layer = layer;
@@ -506,7 +515,7 @@ void outBareGeometry(drawvec const &dv, int type, size_t dim, double e, std::vec
 	serialize_feature(sst, sf);
 }
 
-void readFeatureCollection(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername) {
+void readFeatureCollection(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, std::vector<struct serialization_state> *sst, int layer, std::string layername) {
 	while (pbf.next()) {
 		switch (pbf.tag()) {
 		case 1: {
@@ -521,7 +530,7 @@ void readFeatureCollection(protozero::pbf_reader &pbf, size_t dim, double e, std
 	}
 }
 
-void parse_geobuf(struct serialization_state *sst, const char *src, size_t len, int layer, std::string layername) {
+void parse_geobuf(std::vector<struct serialization_state> *sst, const char *src, size_t len, int layer, std::string layername) {
 	protozero::pbf_reader pbf(src, len);
 
 	size_t dim = 2;
@@ -558,7 +567,8 @@ void parse_geobuf(struct serialization_state *sst, const char *src, size_t len,
 			protozero::pbf_reader geometry_reader(pbf.get_message());
 			std::vector<drawvec_type> dv = readGeometry(geometry_reader, dim, e, keys);
 			for (size_t i = 0; i < dv.size(); i++) {
-				outBareGeometry(dv[i].dv, dv[i].type, dim, e, keys, sst, layer, layername);
+				// Always on thread 0
+				outBareGeometry(dv[i].dv, dv[i].type, &(*sst)[0], layer, layername);
 			}
 			break;
 		}

geobuf.hpp

@@ -8,6 +8,6 @@
 #include "mbtiles.hpp"
 #include "serial.hpp"
 
-void parse_geobuf(struct serialization_state *sst, const char *s, size_t len, int layer, std::string layername);
+void parse_geobuf(std::vector<struct serialization_state> *sst, const char *s, size_t len, int layer, std::string layername);
 
 #endif

geojson.cpp

@@ -146,10 +146,15 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
 		nprop = properties->length;
 	}
 
-	char *metakey[nprop];
+	std::vector<char *> metakey;
+	metakey.resize(nprop);
+
 	std::vector<std::string> metaval;
 	metaval.resize(nprop);
-	int metatype[nprop];
+
+	std::vector<int> metatype;
+	metatype.resize(nprop);
+
 	size_t m = 0;
 
 	for (size_t i = 0; i < nprop; i++) {
@@ -158,7 +163,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
 
 			int type = -1;
 			std::string val;
-			stringify_value(properties->values[i], type, val, sst->fname, sst->line, feature, properties->keys[i]->string);
+			stringify_value(properties->values[i], type, val, sst->fname, sst->line, feature);
 
 			if (type >= 0) {
 				metakey[m] = properties->keys[i]->string;

geojson.hpp

@@ -13,10 +13,12 @@ struct parse_json_args {
 	json_pull *jp;
 	int layer;
 	std::string *layername;
-	std::map<std::string, int> const *attribute_types;
-	bool want_dist;
 
 	struct serialization_state *sst;
+
+	parse_json_args(json_pull *jp1, int layer1, std::string *layername1, struct serialization_state *sst1)
+	    : jp(jp1), layer(layer1), layername(layername1), sst(sst1) {
+	}
 };
 
 struct json_pull *json_begin_map(char *map, long long len);

geometry.cpp

@@ -219,7 +219,7 @@ static void decode_clipped(mapbox::geometry::multi_polygon<long long> &t, drawve
 	}
 }
 
-drawvec clean_or_clip_poly(drawvec &geom, int z, int detail, int buffer, bool clip) {
+drawvec clean_or_clip_poly(drawvec &geom, int z, int buffer, bool clip) {
 	mapbox::geometry::wagyu::wagyu<long long> wagyu;
 
 	geom = remove_noop(geom, VT_POLYGON, 0);
@@ -345,7 +345,7 @@ static int pnpoly(drawvec &vert, size_t start, size_t nvert, long long testx, lo
 	return c;
 }
 
-void check_polygon(drawvec &geom, drawvec &before) {
+void check_polygon(drawvec &geom) {
 	geom = remove_noop(geom, VT_POLYGON, 0);
 
 	mapbox::geometry::multi_polygon<long long> mp;
@@ -635,7 +635,7 @@ int quick_check(long long *bbox, int z, long long buffer) {
 	return 2;
 }
 
-bool point_within_tile(long long x, long long y, int z, long long buffer) {
+bool point_within_tile(long long x, long long y, int z) {
 	// No adjustment for buffer, because the point must be
 	// strictly within the tile to appear exactly once