From 7c1c6e6aa166b85a27fd112175aa7052869eea2e Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Wed, 25 Sep 2024 09:35:06 -0400 Subject: [PATCH 1/4] update ckanext-datajson for max_resource_count --- .env | 3 +++ .profile | 3 +++ ckan/requirements.txt | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.env b/.env index ed272696..060783e3 100644 --- a/.env +++ b/.env @@ -170,6 +170,9 @@ CKANEXT__DATAGOVCATALOG__ADD_PACKAGES_TRACKING_INFO=false # Render recent view using AJAX call to boost page loading speed CKANEXT__DATAGOVTHEME__JS_RECENT_VIEW=true +# Max number of resources to be allowed in a dataset to be harvested +CKANEXT__DATAJSON__MAX_RESOURCE_COUNT=1500 + # Remove all translated pages, for less crawling CKAN__LOCALES_FILTERED_OUT=am ar bg bs ca cs_CZ da_DK de el en_AU en_GB es es_AR eu fa_IR fi fr gl he hr hu id is it ja km ko_KR lt lv mk mn_MN my_MM nb_NO ne nl no pl pt_BR pt_PT ro ru sk sl sq sr sr_Latn sv th tl tr uk uk_UA vi zh_Hans_CN zh_Hant_TW diff --git a/.profile b/.profile index f982528b..aad05301 100755 --- a/.profile +++ b/.profile @@ -141,6 +141,9 @@ export CKANEXT__DATAGOVCATALOG__ADD_PACKAGES_TRACKING_INFO=false # Render recent view using AJAX call to boost page loading speed export CKANEXT__DATAGOVTHEME__JS_RECENT_VIEW=true +# Max number of resources to be allowed in a dataset to be harvested +export CKANEXT__DATAJSON__MAX_RESOURCE_COUNT=1500 + # Set up the collection in Solr echo Setting up Solr collection export SOLR_COLLECTION=ckan diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 07d601d0..1dfdc72d 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -14,7 +14,7 @@ ckan @ git+https://github.com/GSA/ckan.git@8c4a517efeac80db098cc6ba144cb742bbeca -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver ckanext-datagovcatalog==0.1.0 ckanext-datagovtheme==0.2.34 -ckanext-datajson==0.1.25 +ckanext-datajson==0.1.26 ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@b8ebf24004cd3f3edb7f9d01c87c20259c102093 ckanext-envvars==0.0.3 ckanext-geodatagov==0.2.9 From a6cd3c93b976ad5ec8b09f7bf311ca16e994b7f6 Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Wed, 25 Sep 2024 10:28:53 -0400 Subject: [PATCH 2/4] bump again --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 1dfdc72d..d077300d 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -14,7 +14,7 @@ ckan @ git+https://github.com/GSA/ckan.git@8c4a517efeac80db098cc6ba144cb742bbeca -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver ckanext-datagovcatalog==0.1.0 ckanext-datagovtheme==0.2.34 -ckanext-datajson==0.1.26 +ckanext-datajson==0.1.27 ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@b8ebf24004cd3f3edb7f9d01c87c20259c102093 ckanext-envvars==0.0.3 ckanext-geodatagov==0.2.9 From d3ed2aabcd67b9d063dbde9aacf361217d9d5ac2 Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Thu, 26 Sep 2024 12:18:18 -0400 Subject: [PATCH 3/4] move some ckan config from .profile to ckan.ini --- .profile | 9 --------- ckan/setup/ckan.ini | 9 +++++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.profile b/.profile index aad05301..232044ac 100755 --- a/.profile +++ b/.profile @@ -135,15 +135,6 @@ export CKANEXT__S3SITEMAP__AWS_BUCKET_NAME=$(vcap_get_service s3 .credentials.bu export CKANEXT__S3SITEMAP__AWS_STORAGE_PATH=catalog/sitemap export CKANEXT__S3SITEMAP__ENDPOINT_URL=https://$(vcap_get_service s3 .credentials.endpoint) -# Disable this in favor of CKANEXT__DATAGOVTHEME__JS_RECENT_VIEW -export CKANEXT__DATAGOVCATALOG__ADD_PACKAGES_TRACKING_INFO=false - -# Render recent view using AJAX call to boost page loading speed -export CKANEXT__DATAGOVTHEME__JS_RECENT_VIEW=true - -# Max number of resources to be allowed in a dataset to be harvested -export CKANEXT__DATAJSON__MAX_RESOURCE_COUNT=1500 - # Set up the collection in Solr echo Setting up Solr collection export SOLR_COLLECTION=ckan diff --git a/ckan/setup/ckan.ini b/ckan/setup/ckan.ini index f4d2d94b..631d9806 100644 --- a/ckan/setup/ckan.ini +++ b/ckan/setup/ckan.ini @@ -229,6 +229,15 @@ ckan.jobs.timeout = 180 ckan.tracking_enabled = True +# Disable this in favor of CKANEXT__DATAGOVTHEME__JS_RECENT_VIEW +ckanext.datagovcatalog.add_packages_tracking_info = false + +# Render recent view using AJAX call to boost page loading speed +ckanext.datagovtheme.js_recent_view = true + +# Max number of resources to be allowed in a dataset to be harvested +ckanext.datajson.max_resource_count = 800 + ## Spatial settings ckanext.spatial.search_backend = solr-bbox # Customize map widget From ef38349bab23fb806aaa2ce06ba470c97fdcc9cf Mon Sep 17 00:00:00 2001 From: Fuhu Xia <1392461+FuhuXia@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:22:09 -0400 Subject: [PATCH 4/4] Update ckan.ini temporary testing number used for prod number. --- ckan/setup/ckan.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/setup/ckan.ini b/ckan/setup/ckan.ini index 631d9806..b8cbd096 100644 --- a/ckan/setup/ckan.ini +++ b/ckan/setup/ckan.ini @@ -236,7 +236,7 @@ ckanext.datagovcatalog.add_packages_tracking_info = false ckanext.datagovtheme.js_recent_view = true # Max number of resources to be allowed in a dataset to be harvested -ckanext.datajson.max_resource_count = 800 +ckanext.datajson.max_resource_count = 1500 ## Spatial settings ckanext.spatial.search_backend = solr-bbox