Skip to content

Commit

Permalink
Additional tests (#394)
Browse files Browse the repository at this point in the history 
* Add tests for `ModTileCacheExtendedHostName` and expired tileOld/TileVeryOld
* Add tests for `ModTileEnableTileThrottling`/`ModTileEnableTileThrottlingXForward`
  • Loading branch information
@hummeltech
hummeltech authored Feb 23, 2024
1 parent c31d526 commit a0e9681
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 49 deletions.
12 changes: 6 additions & 6 deletions includes/store_file_utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,18 @@ extern "C" {
*/
int mkdirp(const char *path);

/* File path hashing. Used by both mod_tile and render daemon
* The two must both agree on the file layout for meta-tiling
* to work
*/
void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);

int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px, int *py, int *pz);

#ifdef METATILE
/* New meta-tile storage functions */
/* Returns the path to the meta-tile and the offset within the meta-tile */
int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
#else
/* File path hashing. Used by both mod_tile and render daemon
* The two must both agree on the file layout for meta-tiling
* to work
*/
void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
#endif

#ifdef __cplusplus
Expand Down
48 changes: 24 additions & 24 deletions src/store_file_utils.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,27 +117,6 @@ static int check_xyz(int x, int y, int z)
return oob;
}

void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
{
#ifdef DIRECTORY_HASH
// We attempt to cluster the tiles so that a 16x16 square of tiles will be in a single directory
// Hash stores our 40 bit result of mixing the 20 bits of the x & y co-ordinates
// 4 bits of x & y are used per byte of output
unsigned char i, hash[5];

for (i = 0; i < 5; i++) {
hash[i] = ((x & 0x0f) << 4) | (y & 0x0f);
x >>= 4;
y >>= 4;
}

snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.png", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
#else
snprintf(path, len, TILE_PATH "/%s/%d/%d/%d.png", xmlconfig, z, x, y);
#endif
return;
}

int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px, int *py, int *pz)
{
#ifdef DIRECTORY_HASH
Expand Down Expand Up @@ -224,15 +203,36 @@ int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlco
snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.meta", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
}

#else
#else // DIRECTORY_HASH

if (strlen(options)) {
snprintf(path, len, "%s/%s/%d/%u/%u.%s.meta", tile_dir, xmlconfig, z, x, y, options);
} else {
snprintf(path, len, "%s/%s/%d/%u/%u.meta", tile_dir, xmlconfig, z, x, y);
}

#endif
#endif // DIRECTORY_HASH
return offset;
}
#endif
#else // METATILE
void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
{
#ifdef DIRECTORY_HASH
// We attempt to cluster the tiles so that a 16x16 square of tiles will be in a single directory
// Hash stores our 40 bit result of mixing the 20 bits of the x & y co-ordinates
// 4 bits of x & y are used per byte of output
unsigned char i, hash[5];

for (i = 0; i < 5; i++) {
hash[i] = ((x & 0x0f) << 4) | (y & 0x0f);
x >>= 4;
y >>= 4;
}

snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.png", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
#else // DIRECTORY_HASH
snprintf(path, len, TILE_PATH "/%s/%d/%d/%d.png", xmlconfig, z, x, y);
#endif // DIRECTORY_HASH
return;
}
#endif // METATILE
54 changes: 47 additions & 7 deletions tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
)
add_test(NAME add_tile_config_${SOCKET_TYPE}_${STORAGE_BACKEND}
COMMAND ${BASH} -c "
CONFIG_NAME=\"add_tile_config_${SOCKET_TYPE}\"
CONFIG_NAME=\"bad_tile_config_${SOCKET_TYPE}\"
SEARCH_LINE=\$(${GREP_EXECUTABLE} -m1 \"Loading tile config \${CONFIG_NAME}\" ${HTTPD_LOG})
SEARCH_STRS=(
\" at /\${CONFIG_NAME}/ \"
Expand All @@ -461,7 +461,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
add_test(NAME add_tile_mime_config_${SOCKET_TYPE}_${STORAGE_BACKEND}
COMMAND ${BASH} -c "
for SEARCH_CONFIG in js png; do
CONFIG_NAME=\"add_tile_mime_config_\${SEARCH_CONFIG}_${SOCKET_TYPE}\"
CONFIG_NAME=\"bad_tile_mime_config_\${SEARCH_CONFIG}_${SOCKET_TYPE}\"
MIME_TYPE=image/png
if [ \"\${SEARCH_CONFIG}\" = \"js\" ]; then
MIME_TYPE=text/javascript
Expand Down Expand Up @@ -492,12 +492,12 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
set_tests_properties(add_tile_mime_config_${SOCKET_TYPE}_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
)
foreach(BAD_ADD_CONFIG add_tile_config add_tile_mime_config_js add_tile_mime_config_png)
if(BAD_ADD_CONFIG STREQUAL add_tile_config)
foreach(BAD_ADD_CONFIG bad_tile_config bad_tile_mime_config_js bad_tile_mime_config_png)
if(BAD_ADD_CONFIG STREQUAL bad_tile_config)
set(EXTENSION jpg)
elseif(BAD_ADD_CONFIG STREQUAL add_tile_mime_config_js)
elseif(BAD_ADD_CONFIG STREQUAL bad_tile_mime_config_js)
set(EXTENSION js)
elseif(BAD_ADD_CONFIG STREQUAL add_tile_mime_config_png)
elseif(BAD_ADD_CONFIG STREQUAL bad_tile_mime_config_png)
set(EXTENSION png)
endif()
# Generate URL path for tiles
Expand Down Expand Up @@ -529,6 +529,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
# Generate tile URLs
set(HTTPD0_URL "http://${HTTPD0_HOST}:${HTTPD0_PORT}${TILE_URL_PATH}")
set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
set(HTTPD2_URL "http://${HTTPD2_HOST}:${HTTPD2_PORT}${TILE_URL_PATH}")
# Generate tile dirty and status URLs
set(DIRTY_OFF_URL "${HTTPD1_URL}/dirty")
set(DIRTY_ON_URL "${HTTPD0_URL}/dirty")
Expand Down Expand Up @@ -624,11 +625,47 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
set_tests_properties(stats_urls_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED "services_started_${STORAGE_BACKEND};tiles_downloaded_${STORAGE_BACKEND}"
)
add_test(NAME tile_expired_${MAP_NAME}_${STORAGE_BACKEND}
COMMAND ${BASH} -c "
${TOUCH_EXECUTABLE} -d '-1 month' ${TEST_TILES_DIR}/planet-import-complete
if ! ${CURL_CMD} --output /dev/null ${HTTPD1_URL}; then
echo \"Past import: ${HTTPD1_URL}\";
fi
if ! ${CURL_CMD} --output /dev/null ${HTTPD2_URL}; then
echo \"Past import: ${HTTPD2_URL}\";
fi
${TOUCH_EXECUTABLE} -d '+1 month' ${TEST_TILES_DIR}/planet-import-complete
if ! ${CURL_CMD} --output /dev/null ${HTTPD1_URL}; then
echo \"Future import: ${HTTPD1_URL}\";
fi
if ! ${CURL_CMD} --output /dev/null ${HTTPD2_URL}; then
echo \"Future import: ${HTTPD2_URL}\";
fi
"
WORKING_DIRECTORY tests
)
set_tests_properties(tile_expired_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
)
add_test(NAME throttling_xforward_${MAP_NAME}_${STORAGE_BACKEND}
COMMAND ${BASH} -c "
if ! ${CURL_CMD} --header \"X-Forwarded-For: ${CTEST_HOST}, ${CTEST_HOST}\" --output /dev/null ${HTTPD1_URL}; then
echo \"${HTTPD1_URL}\";
fi
if ! ${CURL_CMD} --header \"X-Forwarded-For: ${CTEST_HOST}, ${CTEST_HOST}\" --output /dev/null ${HTTPD2_URL}; then
echo \"${HTTPD2_URL}\";
fi
"
WORKING_DIRECTORY tests
)
set_tests_properties(throttling_xforward_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
)
endforeach()

# Generate file and URL paths for tiles
set(TILE_FILE_NAME "tile.add_tile_config.${STORAGE_BACKEND}")
set(TILE_URL_PATH "/download_add_tile_config/${TILE_ZXY}.png")
set(TILE_URL_PATH "/good_add_tile_config/${TILE_ZXY}.png")
# Generate tile URLs
set(HTTPD0_URL "http://${HTTPD0_HOST}:${HTTPD0_PORT}${TILE_URL_PATH}")
set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
Expand Down Expand Up @@ -768,6 +805,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
exit 1;
fi
"
WORKING_DIRECTORY tests
)
set_tests_properties(cors_all_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
Expand Down Expand Up @@ -802,6 +840,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
exit 1;
fi
"
WORKING_DIRECTORY tests
)
set_tests_properties(cors_localhost_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
Expand All @@ -820,6 +859,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
exit 1;
fi
"
WORKING_DIRECTORY tests
)
set_tests_properties(max_load_missing_${STORAGE_BACKEND} PROPERTIES
FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
Expand Down
28 changes: 16 additions & 12 deletions tests/httpd.conf.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
</IfModule>

<VirtualHost @HTTPD0_HOST@:@HTTPD0_PORT@>
AddTileConfig /add_tile_config_sock/ add_tile_config_sock extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
AddTileConfig /download_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
AddTileMimeConfig /add_tile_mime_config_js_sock/ add_tile_mime_config_js_sock js
AddTileMimeConfig /add_tile_mime_config_png_sock/ add_tile_mime_config_png_sock png
AddTileConfig /bad_tile_config_sock/ bad_tile_config_sock extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
AddTileConfig /good_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
AddTileMimeConfig /bad_tile_mime_config_js_sock/ bad_tile_mime_config_js_sock js
AddTileMimeConfig /bad_tile_mime_config_png_sock/ bad_tile_mime_config_png_sock png
LoadTileConfigFile @RENDERD_CONF@
ModTileBulkMode Off
ModTileCacheDurationDirty 900
Expand All @@ -41,29 +41,32 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
</VirtualHost>

<VirtualHost @HTTPD1_HOST@:@HTTPD1_PORT@>
AddTileConfig /add_tile_config_tcp/ add_tile_config_tcp extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
AddTileConfig /download_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
AddTileMimeConfig /add_tile_mime_config_js_tcp/ add_tile_mime_config_js_tcp js
AddTileMimeConfig /add_tile_mime_config_png_tcp/ add_tile_mime_config_png_tcp png
AddTileConfig /bad_tile_config_tcp/ bad_tile_config_tcp extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
AddTileConfig /good_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
AddTileMimeConfig /bad_tile_mime_config_js_tcp/ bad_tile_mime_config_js_tcp js
AddTileMimeConfig /bad_tile_mime_config_png_tcp/ bad_tile_mime_config_png_tcp png
LoadTileConfigFile @RENDERD_CONF@
ModTileBulkMode Off
ModTileCacheDurationDirty 900
ModTileCacheDurationLowZoom 9 518400
ModTileCacheDurationMax 604800
ModTileCacheDurationMediumZoom 13 86400
ModTileCacheDurationMinimum 10800
ModTileCacheExtendedDuration 2592000
ModTileCacheExtendedHostName @HTTPD1_HOST@
ModTileCacheLastModifiedFactor 0.20
ModTileEnableDirtyURL Off
ModTileEnableStats Off
ModTileEnableStatusURL Off
ModTileEnableTileThrottling Off
ModTileEnableTileThrottlingXForward 0
ModTileEnableTileThrottling On
ModTileEnableTileThrottlingXForward 1
ModTileMissingRequestTimeout 3
ModTileRenderdSocketAddr @RENDERD1_HOST@ @RENDERD1_PORT@
ModTileRequestTimeout 3
ModTileThrottlingRenders 128 0.2
ModTileThrottlingTiles 10000 1
ModTileTileDir @TILE_DIR@
ModTileVeryOldThreshold 2678400
</VirtualHost>

<VirtualHost @HTTPD2_HOST@:@HTTPD2_PORT@>
Expand All @@ -78,8 +81,8 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
ModTileEnableDirtyURL Off
ModTileEnableStats Off
ModTileEnableStatusURL Off
ModTileEnableTileThrottling Off
ModTileEnableTileThrottlingXForward 0
ModTileEnableTileThrottling On
ModTileEnableTileThrottlingXForward 2
ModTileMaxLoadMissing 0
ModTileMaxLoadOld 0
ModTileMissingRequestTimeout 3
Expand All @@ -88,6 +91,7 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
ModTileThrottlingRenders 128 0.2
ModTileThrottlingTiles 10000 1
ModTileTileDir @TILE_DIR@
ModTileVeryOldThreshold -10000000
</VirtualHost>

CustomLog @HTTPD_LOG_ACCESS@ "%A:%p %t \"%r\" %>s %b"
Expand Down

0 comments on commit a0e9681

Please sign in to comment.