Merge branch 'main' into feature/datagov-deploy-refactor

.env

@@ -170,6 +170,9 @@ CKANEXT__DATAGOVCATALOG__ADD_PACKAGES_TRACKING_INFO=false
 # Render recent view using AJAX call to boost page loading speed
 CKANEXT__DATAGOVTHEME__JS_RECENT_VIEW=true
 
+# Max number of resources to be allowed in a dataset to be harvested
+CKANEXT__DATAJSON__MAX_RESOURCE_COUNT=1500
+
 # Remove all translated pages, for less crawling
 CKAN__LOCALES_FILTERED_OUT=am ar bg bs ca cs_CZ da_DK de el en_AU en_GB es es_AR eu fa_IR fi fr gl he hr hu id is it ja km ko_KR lt lv mk mn_MN my_MM nb_NO ne nl no pl pt_BR pt_PT ro ru sk sl sq sr sr_Latn sv th tl tr uk uk_UA vi zh_Hans_CN zh_Hant_TW
 

.github/dependabot.yml

@@ -0,0 +1,15 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    ignore:
+      # Ignore minor and patch updates
+      - dependency-name: "*"
+        update-types: ["version-update:semver-minor", "version-update:semver-patch"]
+    groups:
+      # Group all updates together
+      github-actions:
+        patterns:
+          - "*"

.github/workflows/add_to_project.yml

@@ -10,7 +10,7 @@ jobs:
     name: Add issue to project
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/add-to-project@v0.3.0
+      - uses: actions/add-to-project@v1.0.2
         with:
           project-url: 'https://github.com/orgs/GSA/projects/11/views/1'
           github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}

.github/workflows/commit.yml

@@ -24,7 +24,7 @@ jobs:
       - name: test
         run: make test
       - name: cypress-artifacs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: cypress-artifacts

.github/workflows/maintenance.yml

@@ -28,6 +28,7 @@ on:
           - Normal
           - Scheduled_Maintenance
           - Unscheduled_Downtime
+          - Federal_Shutdown
       notification:
         description: 'Notification to Slack?'
         required: true
@@ -41,7 +42,7 @@ jobs:
     environment: ${{inputs.environ}}
     steps:
       - name: checkout datagov
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: './catalog'
       - name: run task
@@ -54,7 +55,7 @@ jobs:
           cf_password: ${{secrets.CF_SERVICE_AUTH}}
       - name: Send notification to Slack
         if: ${{ inputs.notification }}
-        uses: slackapi/slack-github-action@v1.25.0
+        uses: slackapi/slack-github-action@v2
         with:
           payload: |
             {

.github/workflows/publish.yml

@@ -28,13 +28,13 @@ jobs:
       - name: checkout
         uses: actions/checkout@v4
       - name: login to ghcr
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: publish ${{ matrix.name }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: ${{ matrix.image }}
           push: true

.github/workflows/scale_web.yml

@@ -12,5 +12,6 @@ jobs:
     uses: gsa/data.gov/.github/workflows/scale-web-template.yml@main
     with:
       environ: prod
-      app_names: "{\"include\":[{\"app\":\"catalog-web\"},]}"
+      app_url: https://catalog.data.gov
+      app_names: "{\"include\":[{\"app\":\"catalog-web\",\"smoketest\":true},]}"
     secrets: inherit

.github/workflows/snyk.yml

@@ -61,7 +61,7 @@ jobs:
       - name: Create Pull Request
         if: ${{ failure() && github.event_name == 'schedule' }}
         id: scpr
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.ADD_TO_PROJECT_PAT }}
           commit-message: Update Pip Requirements

.profile

@@ -135,12 +135,6 @@ export CKANEXT__S3SITEMAP__AWS_BUCKET_NAME=$(vcap_get_service s3 .credentials.bu
 export CKANEXT__S3SITEMAP__AWS_STORAGE_PATH=catalog/sitemap
 export CKANEXT__S3SITEMAP__ENDPOINT_URL=https://$(vcap_get_service s3 .credentials.endpoint)
 
-# Disable this in favor of CKANEXT__DATAGOVTHEME__JS_RECENT_VIEW
-export CKANEXT__DATAGOVCATALOG__ADD_PACKAGES_TRACKING_INFO=false
-
-# Render recent view using AJAX call to boost page loading speed
-export CKANEXT__DATAGOVTHEME__JS_RECENT_VIEW=true
-
 # Set up the collection in Solr
 echo Setting up Solr collection
 export SOLR_COLLECTION=ckan

README.md

@@ -245,4 +245,4 @@ Continuous Deployment via [GitHub Actions](https://github.com/GSA/catalog.data.g
 
 ## Put site into maintenance mode
 
-To block access to the catalog apps (`catalog-web`, `catalog-admin`), set the environment variables (`CATALOG_WEB_MODE`, `CATALOG_ADMIN_MODE`) in the `catalog-proxy` app. Use 'MAINTENANCE' for scheduled downtime, 'DOWN' for unscheduled downtime. Any other value will resume normal operation.
+To block access to the catalog apps (`catalog-web`, `catalog-admin`), set the environment variables (`CATALOG_WEB_MODE`, `CATALOG_ADMIN_MODE`) in the `catalog-proxy` app. Use 'MAINTENANCE' for scheduled downtime, 'DOWN' for unscheduled downtime,  'FEDERAL-SHUTDOWN' for the special occasion. Any other value will resume normal operation. Any change on `CATALOG_WEB_MODE` need to be followed by a CloudFront cache clear.

ckan/.snyk

@@ -7,59 +7,70 @@ ignore:
         reason: >-
           No remediation available yet; Not affecting us since the storage is
           not accessible to any other client
-        expires: 2024-11-30T19:29:54.032Z
+        expires: 2025-02-28T19:29:54.032Z
         created: 2022-12-08T16:20:58.023Z
   SNYK-PYTHON-WERKZEUG-6035177:
     - '*':
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-11-30T19:29:54.032Z
+        expires: 2025-02-28T19:29:54.032Z
         created: 2023-10-30T16:50:58.023Z
   SNYK-PYTHON-WERKZEUG-3319936:
     - '*':
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-11-30T19:29:54.032Z
+        expires: 2025-02-28T19:29:54.032Z
         created: 2023-02-15T16:20:58.023Z
   SNYK-PYTHON-WERKZEUG-3319935:
     - '*':
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-11-30T19:29:54.032Z
+        expires: 2025-02-28T19:29:54.032Z
         created: 2023-02-15T16:20:58.023Z
   SNYK-PYTHON-FLASK-5490129:
     - '*':
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4303
-        expires: 2024-11-30T19:29:54.032Z
+        expires: 2025-02-28T19:29:54.032Z
         created: 2023-05-08T16:20:58.023Z
   SNYK-PYTHON-PYOPENSSL-6149520:
     - '*':
         reason: >-
           No remediation available yet; Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4532
-        expires: 2024-11-30T19:29:54.032Z
+        expires: 2025-02-28T19:29:54.032Z
         created: 2024-01-08T00:00:00.000Z
   SNYK-PYTHON-PYOPENSSL-6157250:
     - '*':
         reason: >-
           No remediation available yet; Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4591
-        expires: 2024-11-30T19:29:54.032Z
+        expires: 2025-02-28T19:29:54.032Z
         created: 2024-01-14T00:00:00.000Z
   SNYK-PYTHON-PYOPENSSL-6592766:
     - '*':
         reason: >-
           No remediation available yet; Low severity.
-        expires: 2024-11-30T17:24:47.251Z
+        expires: 2025-02-28T19:29:54.032Z
         created: 2024-04-24T17:24:47.257Z
   SNYK-PYTHON-WERKZEUG-6808933:
     - '*':
         reason: >-
            Not affecting us since no debugger is enabled in cloud.gov apps
-        expires: 2024-11-30T16:20:58.017Z
+        expires: 2025-02-28T16:20:58.017Z
+  SNYK-PYTHON-WERKZEUG-8309091:
+    - '*':
+        reason: >-
+           Not affecting us since it only affects Windows systems
+        expires: 2025-01-28T16:20:58.017Z
+  SNYK-PYTHON-WERKZEUG-8309092:
+    - '*':
+        reason: >-
+           Issue created and triaged. GitHub issue:
+           https://github.com/GSA/data.gov/issues/4951
+        expires: 2025-01-28T16:20:58.017Z
 patch: {}

ckan/requirements.in

@@ -75,7 +75,7 @@ Flask-Babel==1.0.0
 Flask-Login==0.6.1
 Flask-WTF==1.0.1
 flask-multistatic==1.0
-greenlet==2.0.2
+# greenlet==2.0.2
 #Jinja2==3.1.2
 Markdown==3.4.1
 packaging==24.1
@@ -118,10 +118,12 @@ requests~=2.32.3
 
 # avoid ImportError error https://github.com/GSA/data.gov/issues/4396
 importlib-resources<6.0
-gevent>=23.9.0
-jinja2>=3.1.4
+jinja2>=3.1.5
 cryptography>=42.0.4
 
+# fix for https://security.snyk.io/vuln/SNYK-PYTHON-GEVENT-8320934
+gevent>=24.10.1
+
 # lxml beyond 5.1.0 show error module 'lxml.etree' has no attribute '_ElementStringResult'
 # as in https://github.com/GSA/data.gov/issues/4681
 lxml==5.1.0
@@ -136,3 +138,9 @@ certifi>=2024.7.4
 
 # snyk finding
 setuptools~=71.0.3
+
+# Pin MarkupSafe to avoid button issue for logged in user
+MarkupSafe==2.*
+
+# avoid conflic dependencies issue
+greenlet>=3.1.1

ckan/requirements.txt

@@ -1,34 +1,34 @@
 alembic==1.8.1
-async-timeout==4.0.3
+async-timeout==5.0.1
 Babel==2.10.3
 Beaker==1.11.0
 bleach==5.0.1
 blinker==1.5
-boto3==1.35.12
-botocore==1.35.12
-certifi==2024.8.30
-cffi==1.17.0
+boto3==1.35.90
+botocore==1.35.90
+certifi==2024.12.14
+cffi==1.17.1
 chardet==5.2.0
-charset-normalizer==3.3.2
+charset-normalizer==3.4.1
 ckan @ git+https://github.com/GSA/ckan.git@8c4a517efeac80db098cc6ba144cb742bbeca194
 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver
-ckanext-datagovcatalog==0.1.0
-ckanext-datagovtheme==0.2.34
-ckanext-datajson==0.1.25
+ckanext-datagovcatalog==0.1.1
+ckanext-datagovtheme==0.2.41
+ckanext-datajson==0.1.27
 ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@b8ebf24004cd3f3edb7f9d01c87c20259c102093
-ckanext-envvars==0.0.3
+ckanext-envvars==0.0.6
 ckanext-geodatagov==0.2.9
 -e git+https://github.com/GSA/ckanext-harvest.git@9039e7a5d563a40177d62487758b366ab77434b6#egg=ckanext_harvest
-ckanext-metrics-dashboard==0.1.6
+ckanext-metrics-dashboard==0.1.7
 -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report
 ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@387cfc1c6a7619f670bf387384f2634516de5844
--e git+https://github.com/GSA/ckanext-spatial.git@3d0a375fe98edc70a0d12efd2f4ac54f0e05b597#egg=ckanext_spatial
+-e git+https://github.com/GSA/ckanext-spatial.git@6d83a53efa1e9ff225daf4e7a5751b98367ee7f2#egg=ckanext_spatial
 ckantoolkit==0.0.7
 click==8.1.3
-cryptography==43.0.1
+cryptography==44.0.0
 defusedxml==0.7.1
 dominate==2.7.0
-elementpath==4.4.0
+elementpath==4.7.0
 feedgen==0.9.0
 Flask==2.0.3
 Flask-Babel==1.0.0
@@ -39,79 +39,79 @@ future==1.0.0
 GeoAlchemy2==0.5.0
 geojson==3.0.1
 geomet==1.1.0
-gevent==24.2.1
-greenlet==2.0.2
+gevent==24.11.1
+greenlet==3.1.1
 gunicorn==23.0.0
 html5lib==1.1
-idna==3.8
+idna==3.10
 importlib-resources==5.13.0
-isodate==0.6.1
+isodate==0.7.2
 itsdangerous==2.2.0
-Jinja2==3.1.4
+Jinja2==3.1.5
 jmespath==1.0.1
 json-table-schema==0.2.1
 jsonschema==2.4.0
 lxml==5.1.0
-Mako==1.3.5
+Mako==1.3.8
 Markdown==3.4.1
 MarkupSafe==2.1.5
 messytables==0.15.2
 mypy==1.10.1
 mypy-extensions==1.0.0
-newrelic==9.13.0
+newrelic==10.4.0
 nose==1.3.7
 numpy==1.26.4
-OWSLib==0.31.0
+OWSLib==0.32.0
 packaging==24.1
 passlib==1.7.4
 pika==1.2.1
-pip==24.1
+pip==24.3.1
 ply==3.11
 polib==1.1.1
 progressbar==2.5
 progressbar2==3.53.3
 psycopg2==2.9.3
 pycparser==2.22
 PyJWT==2.4.0
-pyOpenSSL==24.2.1
-pyparsing==3.1.4
+pyOpenSSL==24.3.0
+pyparsing==3.2.1
 pyproj==3.4.1
 pysaml2==7.0.1
 pysolr==3.9.0
 python-dateutil==2.8.2
 python-magic==0.4.27
-python-utils==3.8.2
-pytz==2024.1
+python-utils==3.9.1
+pytz==2024.2
 pytz-deprecation-shim==0.1.0.post0
 PyUtilib==6.0.0
 PyYAML==6.0.1
 PyZ3950 @ git+https://github.com/danizen/PyZ3950@6d44a4ab85c8bda3a7542c2c9efdfad46c830219
 rdflib==6.1.1
-redis==5.0.8
+redis==5.2.1
 requests==2.32.3
 rfc3987==1.3.8
 rq==1.11.0
-s3transfer==0.10.2
+s3transfer==0.10.4
 sansjson==0.3.0
 setuptools==71.0.4
 shapely==2.0.1
 simplejson==3.18.0
-six==1.16.0
+six==1.17.0
 SQLAlchemy==1.4.41
 sqlalchemy2-stubs==0.0.2a38
 sqlparse==0.5.0
-tomli==2.0.1
+tomli==2.2.1
 typing_extensions==4.3.0
-tzdata==2024.1
+tzdata==2024.2
 tzlocal==4.2
-urllib3==2.2.2
-watchdog==5.0.2
+urllib3==2.3.0
+watchdog==6.0.0
 webassets==2.0
 webencodings==0.5.1
 Werkzeug==2.0.3
 wheel==0.42.0
-WTForms==3.1.2
+WTForms==3.2.1
 xlrd==2.0.1
-xmlschema==3.3.2
+xmlschema==3.4.3
 zope.event==5.0
 zope.interface==5.4.0