Merge pull request #552 from MITLibraries/maintenance-week-updates

Maintenance week updates

.github/pull-request-template.md

@@ -1,38 +1,29 @@
-#### What does this PR do?
-
-Describe the overall purpose of the PR changes. Doesn't need to be as specific as the
-individual commits.
-
-#### Helpful background context
-
-Describe any additional context beyond what the PR accomplishes if it is likely
-to be useful to a reviewer.
-
-Delete this section if it isn't applicable to the PR.
-
-#### How can a reviewer manually see the effects of these changes?
+### Purpose and background context
+Describe the overall purpose of the PR changes and any useful background context.
 
+### How can a reviewer manually see the effects of these changes?
 Explain how to see the proposed changes in the application if possible.
 
 Delete this section if it isn't applicable to the PR.
 
-#### Includes new or updated dependencies?
-
+### Includes new or updated dependencies?
 YES | NO
 
-#### What are the relevant tickets?
-
-Include links to Jira Software and/or Jira Service Management tickets here.
+### Changes expectations for external applications?
+YES | NO
 
-#### Developer checklist
+### What are the relevant tickets?
+- Include links to Jira Software and/or Jira Service Management tickets here.
 
+### Developer
 - [ ] All new ENV is documented in README
+- [ ] All new ENV has been added to staging and production environments
+- [ ] All related Jira tickets are linked in commit message(s)
 - [ ] Stakeholder approval has been confirmed (or is not needed)
 
-#### Code reviewer checklist
-
-- [ ] The commit message is clear and follows our guidelines (not just this pull request message)
+### Code Reviewer(s)
+- [ ] The commit message is clear and follows our guidelines (not just this PR message)
 - [ ] There are appropriate tests covering any new functionality
-- [ ] The documentation has been updated or is unnecessary
-- [ ] The changes have been verified
+- [ ] The provided documentation is sufficient for understanding any new functionality introduced
+- [ ] Any manual tests have been performed or provided examples verified
 - [ ] New dependencies are appropriate or there were no changes

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+default_language_version:
+  python: python3.11 # set for project python version
+repos:
+  - repo: local
+    hooks:
+      - id: black-apply
+        name: black-apply
+        entry: pipenv run black
+        language: system
+        pass_filenames: true
+        types: ["python"]
+      - id: mypy
+        name: mypy
+        entry: pipenv run mypy
+        language: system
+        pass_filenames: true
+        types: ["python"]
+        exclude: "tests/"
+      - id: ruff-apply
+        name: ruff-apply
+        entry: pipenv run ruff check --fix
+        language: system
+        pass_filenames: true
+        types: ["python"]
+      - id: safety
+        name: safety
+        entry: pipenv check
+        language: system
+        pass_filenames: false

Makefile

@@ -1,70 +1,86 @@
 .PHONY: install test coveralls lint bandit black flake8 isort mypy dist-dev update publish-dev
 SHELL=/bin/bash
 DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ)
+
 ### This is the Terraform-generated header for oai-pmh-harvester-dev ###
 ECR_NAME_DEV:=oai-pmh-harvester-dev
 ECR_URL_DEV:=222053980223.dkr.ecr.us-east-1.amazonaws.com/oai-pmh-harvester-dev
 ### End of Terraform-generated header ###
 
-help: ## Print this message
-	@awk 'BEGIN { FS = ":.*##"; print "Usage:  make <target>\n\nTargets:" } \
-/^[-_[:alpha:]]+:.?*##/ { printf "  %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST)
+help: # preview Makefile commands
+	@awk 'BEGIN { FS = ":.*#"; print "Usage:  make <target>\n\nTargets:" } \
+/^[-_[:alpha:]]+:.?*#/ { printf "  %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST)
 
-install: ## Install script and dependencies
+## Dependency commands
+install: # install Python dependencies and pre-commit hook
 	pipenv install --dev
+	pipenv run pre-commit install
+
+update: install # update Python dependencies
+	pipenv clean
+	pipenv update --dev
 
-test: ## Run tests and print a coverage report
+## Unit test commands
+test: # run tests and print a coverage report
 	pipenv run coverage run --source=harvester -m pytest -vv
 	pipenv run coverage report -m
 
-coveralls: test
+coveralls: test # write coverage data to an LCOV report
 	pipenv run coverage lcov -o ./coverage/lcov.info
 
-### Linting commands ###
-lint: bandit black flake8 isort mypy ## Lint the repo
+## Code quality and safety commands
 
-bandit:
-	pipenv run bandit -r harvester
+lint: black mypy ruff safety # run linters
 
-black:
+black: # run 'black' linter and print a preview of suggested changes
 	pipenv run black --check --diff .
 
-flake8:
-	pipenv run flake8 .
+mypy: # run 'mypy' linter
+	pipenv run mypy .
 
-isort:
-	pipenv run isort . --diff
+ruff: # run 'ruff' linter and print a preview of errors
+	pipenv run ruff check .
 
-mypy:
-	pipenv run mypy harvester
+safety: # check for security vulnerabilities and verify Pipfile.lock is up-to-date
+	pipenv check
+	pipenv verify
 
-update: install ## Update all Python dependencies
-	pipenv clean
-	pipenv update --dev
+lint-apply: # apply changes with 'black' and resolve fixable errors with 'ruff'
+	black-apply ruff-apply
+
+black-apply: # apply changes with 'black'
+	pipenv run black .
+
+ruff-apply: # resolve fixable errors with 'ruff'
+	pipenv run ruff check --fix .
 
-### Terraform-generated Developer Deploy Commands for Dev environment ###
-dist-dev: ## Build docker container (intended for developer-based manual build)
+## Terraform-generated commands for container build and deployment in dev
+dist-dev: # build docker container (intended for developer-based manual build)
 	docker build --platform linux/amd64 \
 	    -t $(ECR_URL_DEV):latest \
 		-t $(ECR_URL_DEV):`git describe --always` \
 		-t $(ECR_NAME_DEV):latest .
 
-publish-dev: dist-dev ## Build, tag and push (intended for developer-based manual publish)
+publish-dev: dist-dev # build, tag and push (intended for developer-based manual publish)
 	docker login -u AWS -p $$(aws ecr get-login-password --region us-east-1) $(ECR_URL_DEV)
 	docker push $(ECR_URL_DEV):latest
 	docker push $(ECR_URL_DEV):`git describe --always`
 
-### Terraform-generated manual shortcuts for deploying to Stage ###
-### This requires that ECR_NAME_STAGE & ECR_URL_STAGE environment variables are set locally
-### by the developer and that the developer has authenticated to the correct AWS Account.
-### The values for the environment variables can be found in the stage_build.yml caller workflow.
-dist-stage: ## Only use in an emergency
+## Terraform-generated commands for container build and deployment in stage \
+This requires that ECR_NAME_STAGE and ECR_URL_STAGE environment variables \
+are set locally by the developer and that the developer has \
+authenticated to the correct AWS Account. The values for the environment \
+variables can be found in the stage_build.yml caller workflow. \
+While Stage should generally only be used in an emergency for most repos, \
+it is necessary for any testing requiring access to the Data Warehouse \
+because Cloud Connector is not enabled on Dev1.
+dist-stage:
 	docker build --platform linux/amd64 \
 	    -t $(ECR_URL_STAGE):latest \
 		-t $(ECR_URL_STAGE):`git describe --always` \
 		-t $(ECR_NAME_STAGE):latest .
 
-publish-stage: ## Only use in an emergency
+publish-stage:
 	docker login -u AWS -p $$(aws ecr get-login-password --region us-east-1) $(ECR_URL_STAGE)
 	docker push $(ECR_URL_STAGE):latest
 	docker push $(ECR_URL_STAGE):`git describe --always`

Pipfile

@@ -13,11 +13,12 @@ urllib3 = "==1.26.18"
 [dev-packages]
 bandit = "*"
 black = "*"
-coverage = "*"
-flake8 = "*"
-isort = "*"
+coveralls = "*"
 mypy = "*"
+pre-commit = "*"
 pytest = "*"
+ruff = "*"
+types-requests = "*"
 vcrpy = "==4.2.1"
 
 [requires]