From a0e9681f518e18c88e9eeb1e9f372034a794da3f Mon Sep 17 00:00:00 2001
From: Hummeltech <6109326+hummeltech@users.noreply.github.com>
Date: Thu, 22 Feb 2024 21:29:10 -0700
Subject: [PATCH] Additional tests (#394)
* Add tests for `ModTileCacheExtendedHostName` and expired tileOld/TileVeryOld
* Add tests for `ModTileEnableTileThrottling`/`ModTileEnableTileThrottlingXForward`
---
includes/store_file_utils.h | 12 ++++-----
src/store_file_utils.c | 48 ++++++++++++++++-----------------
tests/CMakeLists.txt | 54 ++++++++++++++++++++++++++++++++-----
tests/httpd.conf.in | 28 ++++++++++---------
4 files changed, 93 insertions(+), 49 deletions(-)
diff --git a/includes/store_file_utils.h b/includes/store_file_utils.h
index 6f55addb..4d92e4dc 100644
--- a/includes/store_file_utils.h
+++ b/includes/store_file_utils.h
@@ -32,18 +32,18 @@ extern "C" {
*/
int mkdirp(const char *path);
-/* File path hashing. Used by both mod_tile and render daemon
- * The two must both agree on the file layout for meta-tiling
- * to work
- */
-void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
-
int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px, int *py, int *pz);
#ifdef METATILE
/* New meta-tile storage functions */
/* Returns the path to the meta-tile and the offset within the meta-tile */
int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
+#else
+/* File path hashing. Used by both mod_tile and render daemon
+ * The two must both agree on the file layout for meta-tiling
+ * to work
+ */
+void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
#endif
#ifdef __cplusplus
diff --git a/src/store_file_utils.c b/src/store_file_utils.c
index 9dd83fca..6c46efa3 100644
--- a/src/store_file_utils.c
+++ b/src/store_file_utils.c
@@ -117,27 +117,6 @@ static int check_xyz(int x, int y, int z)
return oob;
}
-void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
-{
-#ifdef DIRECTORY_HASH
- // We attempt to cluster the tiles so that a 16x16 square of tiles will be in a single directory
- // Hash stores our 40 bit result of mixing the 20 bits of the x & y co-ordinates
- // 4 bits of x & y are used per byte of output
- unsigned char i, hash[5];
-
- for (i = 0; i < 5; i++) {
- hash[i] = ((x & 0x0f) << 4) | (y & 0x0f);
- x >>= 4;
- y >>= 4;
- }
-
- snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.png", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
-#else
- snprintf(path, len, TILE_PATH "/%s/%d/%d/%d.png", xmlconfig, z, x, y);
-#endif
- return;
-}
-
int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px, int *py, int *pz)
{
#ifdef DIRECTORY_HASH
@@ -224,7 +203,7 @@ int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlco
snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.meta", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
}
-#else
+#else // DIRECTORY_HASH
if (strlen(options)) {
snprintf(path, len, "%s/%s/%d/%u/%u.%s.meta", tile_dir, xmlconfig, z, x, y, options);
@@ -232,7 +211,28 @@ int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlco
snprintf(path, len, "%s/%s/%d/%u/%u.meta", tile_dir, xmlconfig, z, x, y);
}
-#endif
+#endif // DIRECTORY_HASH
return offset;
}
-#endif
+#else // METATILE
+void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
+{
+#ifdef DIRECTORY_HASH
+ // We attempt to cluster the tiles so that a 16x16 square of tiles will be in a single directory
+ // Hash stores our 40 bit result of mixing the 20 bits of the x & y co-ordinates
+ // 4 bits of x & y are used per byte of output
+ unsigned char i, hash[5];
+
+ for (i = 0; i < 5; i++) {
+ hash[i] = ((x & 0x0f) << 4) | (y & 0x0f);
+ x >>= 4;
+ y >>= 4;
+ }
+
+ snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.png", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
+#else // DIRECTORY_HASH
+ snprintf(path, len, TILE_PATH "/%s/%d/%d/%d.png", xmlconfig, z, x, y);
+#endif // DIRECTORY_HASH
+ return;
+}
+#endif // METATILE
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 8bf48d05..2710ac48 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -438,7 +438,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
)
add_test(NAME add_tile_config_${SOCKET_TYPE}_${STORAGE_BACKEND}
COMMAND ${BASH} -c "
- CONFIG_NAME=\"add_tile_config_${SOCKET_TYPE}\"
+ CONFIG_NAME=\"bad_tile_config_${SOCKET_TYPE}\"
SEARCH_LINE=\$(${GREP_EXECUTABLE} -m1 \"Loading tile config \${CONFIG_NAME}\" ${HTTPD_LOG})
SEARCH_STRS=(
\" at /\${CONFIG_NAME}/ \"
@@ -461,7 +461,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
add_test(NAME add_tile_mime_config_${SOCKET_TYPE}_${STORAGE_BACKEND}
COMMAND ${BASH} -c "
for SEARCH_CONFIG in js png; do
- CONFIG_NAME=\"add_tile_mime_config_\${SEARCH_CONFIG}_${SOCKET_TYPE}\"
+ CONFIG_NAME=\"bad_tile_mime_config_\${SEARCH_CONFIG}_${SOCKET_TYPE}\"
MIME_TYPE=image/png
if [ \"\${SEARCH_CONFIG}\" = \"js\" ]; then
MIME_TYPE=text/javascript
@@ -492,12 +492,12 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
set_tests_properties(add_tile_mime_config_${SOCKET_TYPE}_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
)
- foreach(BAD_ADD_CONFIG add_tile_config add_tile_mime_config_js add_tile_mime_config_png)
- if(BAD_ADD_CONFIG STREQUAL add_tile_config)
+ foreach(BAD_ADD_CONFIG bad_tile_config bad_tile_mime_config_js bad_tile_mime_config_png)
+ if(BAD_ADD_CONFIG STREQUAL bad_tile_config)
set(EXTENSION jpg)
- elseif(BAD_ADD_CONFIG STREQUAL add_tile_mime_config_js)
+ elseif(BAD_ADD_CONFIG STREQUAL bad_tile_mime_config_js)
set(EXTENSION js)
- elseif(BAD_ADD_CONFIG STREQUAL add_tile_mime_config_png)
+ elseif(BAD_ADD_CONFIG STREQUAL bad_tile_mime_config_png)
set(EXTENSION png)
endif()
# Generate URL path for tiles
@@ -529,6 +529,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
# Generate tile URLs
set(HTTPD0_URL "http://${HTTPD0_HOST}:${HTTPD0_PORT}${TILE_URL_PATH}")
set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
+ set(HTTPD2_URL "http://${HTTPD2_HOST}:${HTTPD2_PORT}${TILE_URL_PATH}")
# Generate tile dirty and status URLs
set(DIRTY_OFF_URL "${HTTPD1_URL}/dirty")
set(DIRTY_ON_URL "${HTTPD0_URL}/dirty")
@@ -624,11 +625,47 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
set_tests_properties(stats_urls_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED "services_started_${STORAGE_BACKEND};tiles_downloaded_${STORAGE_BACKEND}"
)
+ add_test(NAME tile_expired_${MAP_NAME}_${STORAGE_BACKEND}
+ COMMAND ${BASH} -c "
+ ${TOUCH_EXECUTABLE} -d '-1 month' ${TEST_TILES_DIR}/planet-import-complete
+ if ! ${CURL_CMD} --output /dev/null ${HTTPD1_URL}; then
+ echo \"Past import: ${HTTPD1_URL}\";
+ fi
+ if ! ${CURL_CMD} --output /dev/null ${HTTPD2_URL}; then
+ echo \"Past import: ${HTTPD2_URL}\";
+ fi
+ ${TOUCH_EXECUTABLE} -d '+1 month' ${TEST_TILES_DIR}/planet-import-complete
+ if ! ${CURL_CMD} --output /dev/null ${HTTPD1_URL}; then
+ echo \"Future import: ${HTTPD1_URL}\";
+ fi
+ if ! ${CURL_CMD} --output /dev/null ${HTTPD2_URL}; then
+ echo \"Future import: ${HTTPD2_URL}\";
+ fi
+ "
+ WORKING_DIRECTORY tests
+ )
+ set_tests_properties(tile_expired_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
+ FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
+ )
+ add_test(NAME throttling_xforward_${MAP_NAME}_${STORAGE_BACKEND}
+ COMMAND ${BASH} -c "
+ if ! ${CURL_CMD} --header \"X-Forwarded-For: ${CTEST_HOST}, ${CTEST_HOST}\" --output /dev/null ${HTTPD1_URL}; then
+ echo \"${HTTPD1_URL}\";
+ fi
+ if ! ${CURL_CMD} --header \"X-Forwarded-For: ${CTEST_HOST}, ${CTEST_HOST}\" --output /dev/null ${HTTPD2_URL}; then
+ echo \"${HTTPD2_URL}\";
+ fi
+ "
+ WORKING_DIRECTORY tests
+ )
+ set_tests_properties(throttling_xforward_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
+ FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
+ )
endforeach()
# Generate file and URL paths for tiles
set(TILE_FILE_NAME "tile.add_tile_config.${STORAGE_BACKEND}")
- set(TILE_URL_PATH "/download_add_tile_config/${TILE_ZXY}.png")
+ set(TILE_URL_PATH "/good_add_tile_config/${TILE_ZXY}.png")
# Generate tile URLs
set(HTTPD0_URL "http://${HTTPD0_HOST}:${HTTPD0_PORT}${TILE_URL_PATH}")
set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
@@ -768,6 +805,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
exit 1;
fi
"
+ WORKING_DIRECTORY tests
)
set_tests_properties(cors_all_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
@@ -802,6 +840,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
exit 1;
fi
"
+ WORKING_DIRECTORY tests
)
set_tests_properties(cors_localhost_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
@@ -820,6 +859,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
exit 1;
fi
"
+ WORKING_DIRECTORY tests
)
set_tests_properties(max_load_missing_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
diff --git a/tests/httpd.conf.in b/tests/httpd.conf.in
index 1b9ee013..9406ef1f 100644
--- a/tests/httpd.conf.in
+++ b/tests/httpd.conf.in
@@ -15,10 +15,10 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
- AddTileConfig /add_tile_config_sock/ add_tile_config_sock extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
- AddTileConfig /download_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
- AddTileMimeConfig /add_tile_mime_config_js_sock/ add_tile_mime_config_js_sock js
- AddTileMimeConfig /add_tile_mime_config_png_sock/ add_tile_mime_config_png_sock png
+ AddTileConfig /bad_tile_config_sock/ bad_tile_config_sock extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
+ AddTileConfig /good_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
+ AddTileMimeConfig /bad_tile_mime_config_js_sock/ bad_tile_mime_config_js_sock js
+ AddTileMimeConfig /bad_tile_mime_config_png_sock/ bad_tile_mime_config_png_sock png
LoadTileConfigFile @RENDERD_CONF@
ModTileBulkMode Off
ModTileCacheDurationDirty 900
@@ -41,10 +41,10 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
- AddTileConfig /add_tile_config_tcp/ add_tile_config_tcp extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
- AddTileConfig /download_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
- AddTileMimeConfig /add_tile_mime_config_js_tcp/ add_tile_mime_config_js_tcp js
- AddTileMimeConfig /add_tile_mime_config_png_tcp/ add_tile_mime_config_png_tcp png
+ AddTileConfig /bad_tile_config_tcp/ bad_tile_config_tcp extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
+ AddTileConfig /good_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
+ AddTileMimeConfig /bad_tile_mime_config_js_tcp/ bad_tile_mime_config_js_tcp js
+ AddTileMimeConfig /bad_tile_mime_config_png_tcp/ bad_tile_mime_config_png_tcp png
LoadTileConfigFile @RENDERD_CONF@
ModTileBulkMode Off
ModTileCacheDurationDirty 900
@@ -52,18 +52,21 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
ModTileCacheDurationMax 604800
ModTileCacheDurationMediumZoom 13 86400
ModTileCacheDurationMinimum 10800
+ ModTileCacheExtendedDuration 2592000
+ ModTileCacheExtendedHostName @HTTPD1_HOST@
ModTileCacheLastModifiedFactor 0.20
ModTileEnableDirtyURL Off
ModTileEnableStats Off
ModTileEnableStatusURL Off
- ModTileEnableTileThrottling Off
- ModTileEnableTileThrottlingXForward 0
+ ModTileEnableTileThrottling On
+ ModTileEnableTileThrottlingXForward 1
ModTileMissingRequestTimeout 3
ModTileRenderdSocketAddr @RENDERD1_HOST@ @RENDERD1_PORT@
ModTileRequestTimeout 3
ModTileThrottlingRenders 128 0.2
ModTileThrottlingTiles 10000 1
ModTileTileDir @TILE_DIR@
+ ModTileVeryOldThreshold 2678400
@@ -78,8 +81,8 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
ModTileEnableDirtyURL Off
ModTileEnableStats Off
ModTileEnableStatusURL Off
- ModTileEnableTileThrottling Off
- ModTileEnableTileThrottlingXForward 0
+ ModTileEnableTileThrottling On
+ ModTileEnableTileThrottlingXForward 2
ModTileMaxLoadMissing 0
ModTileMaxLoadOld 0
ModTileMissingRequestTimeout 3
@@ -88,6 +91,7 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
ModTileThrottlingRenders 128 0.2
ModTileThrottlingTiles 10000 1
ModTileTileDir @TILE_DIR@
+ ModTileVeryOldThreshold -10000000
CustomLog @HTTPD_LOG_ACCESS@ "%A:%p %t \"%r\" %>s %b"