Skip to content
CI - Pull Request

Fixed CSV file upload does not correctly handle 'UTF-8 with BOM' encoding #12158

Sign in to view logs

Fixed CSV file upload does not correctly handle 'UTF-8 with BOM' encoding

Fixed CSV file upload does not correctly handle 'UTF-8 with BOM' encoding #12158

Workflow file for this run

.github/workflows/ci_pullrequest.yml at 5832eca
---
# A lighter-weight version of CI specifically for pull requests
name: "CI - Pull Request"
concurrency: # Cancel any existing runs of this workflow for this same PR
group: "${{ github.workflow }}-${{ github.ref }}"
cancel-in-progress: true
on: # yamllint disable
pull_request:
workflow_call:
jobs:
ruff:
runs-on: "ubuntu-20.04"
env:
INVOKE_NAUTOBOT_LOCAL: "True"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
- name: "Linting: ruff"
run: "poetry run invoke ruff --output-format github"
yamllint:
runs-on: "ubuntu-20.04"
env:
INVOKE_NAUTOBOT_LOCAL: "True"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
- name: "Linting: yamllint"
run: "poetry run invoke yamllint"
markdownlint:
runs-on: "ubuntu-20.04"
env:
INVOKE_NAUTOBOT_LOCAL: "True"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
- name: "Install markdownlint"
run: "npm install [email protected]"
- name: "Linting: markdownlint"
run: "poetry run invoke markdownlint"
hadolint:
runs-on: "ubuntu-20.04"
env:
INVOKE_NAUTOBOT_LOCAL: "True"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Run Hadolint"
uses: "hadolint/[email protected]"
with:
dockerfile: "docker/Dockerfile"
check-migrations:
name: "Run 'nautobot-server makemigrations --dry-run --check'"
runs-on: "ubuntu-20.04"
env:
INVOKE_NAUTOBOT_LOCAL: "True"
NAUTOBOT_DB_USER: "nautobot"
NAUTOBOT_DB_PASSWORD: "decinablesprewad"
NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.postgresql"
services:
postgres:
image: "postgres:13-alpine"
env:
POSTGRES_USER: "nautobot"
POSTGRES_PASSWORD: "decinablesprewad"
POSTGRES_DB: "nautobot"
ports:
- "5432:5432"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
- name: "Run check migrations"
run: "poetry run invoke check-migrations"
check-schema:
name: "Run REST API schema checks"
runs-on: "ubuntu-20.04"
env:
INVOKE_NAUTOBOT_LOCAL: "True"
NAUTOBOT_DB_USER: "nautobot"
NAUTOBOT_DB_PASSWORD: "decinablesprewad"
NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.postgresql"
NAUTOBOT_CONFIG: "nautobot/core/tests/nautobot_config.py"
services:
postgres:
image: "postgres:13-alpine"
env:
POSTGRES_USER: "nautobot"
POSTGRES_PASSWORD: "decinablesprewad"
POSTGRES_DB: "nautobot"
ports:
- "5432:5432"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
- name: "Run check-schema"
# REST API schema generation currently includes GraphQL schema generation, which is dependent on the DB :-(
run: "poetry run invoke migrate check-schema"
pylint:
runs-on: "ubuntu-20.04"
env:
INVOKE_NAUTOBOT_LOCAL: "True"
NAUTOBOT_DB_USER: "nautobot"
NAUTOBOT_DB_PASSWORD: "decinablesprewad"
NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.postgresql"
NAUTOBOT_CONFIG: "nautobot/core/tests/nautobot_config.py"
services:
postgres:
image: "postgres:13-alpine"
env:
POSTGRES_USER: "nautobot"
POSTGRES_PASSWORD: "decinablesprewad"
POSTGRES_DB: "nautobot"
ports:
- "5432:5432"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
- name: "Run pylint"
run: "poetry run invoke pylint"
tests-postgres:
name: "Unit tests (PostgreSQL and Python 3.8)"
runs-on: "ubuntu-20.04"
env:
INVOKE_NAUTOBOT_LOCAL: "True"
NAUTOBOT_DB_USER: "nautobot"
NAUTOBOT_DB_PASSWORD: "decinablesprewad"
NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.postgresql"
services:
postgres:
image: "postgres:13-alpine"
env:
POSTGRES_USER: "nautobot"
POSTGRES_PASSWORD: "decinablesprewad"
POSTGRES_DB: "nautobot"
ports:
- "5432:5432"
redis:
image: "redis:6-alpine"
ports:
- "6379:6379"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
with:
python-version: "3.8"
- name: "Remove Installed Nautobots (Poetry Workaround)" # Workaround for https://github.com/python-poetry/poetry/issues/2918
run: "for i in $(ls -d1 ~/.cache/pypoetry/virtualenvs/*/lib/*/site-packages/nautobot-*.dist-info); do poetry run pip uninstall -y nautobot; done"
- name: "Reinstall Nautobot Local"
run: "poetry install"
- name: "Run unittest"
run: "poetry run invoke unittest --failfast --keepdb --parallel"
needs:
- "check-migrations"
- "check-schema"
- "hadolint"
- "markdownlint"
- "ruff"
- "yamllint"
tests-mysql:
name: "Unit tests (MySQL and Python 3.11)"
runs-on: "ubuntu-20.04"
env:
INVOKE_NAUTOBOT_LOCAL: "True"
NAUTOBOT_DB_USER: "root"
NAUTOBOT_DB_PASSWORD: "decinablesprewad"
NAUTOBOT_DB_HOST: "127.0.0.1"
NAUTOBOT_DB_PORT: "3306"
NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.mysql"
services:
mysql:
image: "mysql:8"
env:
MYSQL_ROOT_PASSWORD: "decinablesprewad"
MYSQL_PASSWORD: "decinablesprewad"
MYSQL_DATABASE: "nautobot"
ports:
- "3306:3306"
redis:
image: "redis:6-alpine"
ports:
- "6379:6379"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
with:
python-version: "3.11"
- name: "Remove Installed Nautobots (Poetry Workaround)" # Workaround for https://github.com/python-poetry/poetry/issues/2918
run: "for i in $(ls -d1 ~/.cache/pypoetry/virtualenvs/*/lib/*/site-packages/nautobot-*.dist-info); do poetry run pip uninstall -y nautobot; done"
- name: "Reinstall Nautobot Local"
run: "poetry install"
- name: "Install Required Dependencies for MySql Client"
run: "sudo apt-get update && sudo apt-get install --no-install-recommends -y default-libmysqlclient-dev gcc"
- name: "Install MySql Dependencies"
run: "poetry install --no-interaction --no-root --extras mysql"
- name: "Run unittest"
run: "poetry run invoke unittest --failfast --keepdb --parallel"
needs:
- "check-migrations"
- "check-schema"
- "hadolint"
- "markdownlint"
- "ruff"
- "yamllint"
migration-tests-mysql:
name: "Run migrations against test dataset (MySQL and Python 3.8)"
runs-on: "ubuntu-20.04"
env:
MIGRATION_TEST_DATASET: "development/datasets/nautobot_1.5.16_mysql.tar.gz"
INVOKE_NAUTOBOT_LOCAL: "True"
NAUTOBOT_CONFIG: "nautobot/core/tests/nautobot_config.py"
NAUTOBOT_DB_USER: "root"
NAUTOBOT_DB_PASSWORD: "decinablesprewad"
NAUTOBOT_DB_HOST: "127.0.0.1"
NAUTOBOT_DB_PORT: "3306"
NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.mysql"
services:
mysql:
image: "mysql:8"
env:
MYSQL_ROOT_PASSWORD: "decinablesprewad"
MYSQL_PASSWORD: "decinablesprewad"
MYSQL_DATABASE: "nautobot"
ports:
- "3306:3306"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
with:
python-version: "3.8"
- name: "Remove Installed Nautobots (Poetry Workaround)" # Workaround for https://github.com/python-poetry/poetry/issues/2918
run: "for i in $(ls -d1 ~/.cache/pypoetry/virtualenvs/*/lib/*/site-packages/nautobot-*.dist-info); do poetry run pip uninstall -y nautobot; done"
- name: "Reinstall Nautobot Local"
run: "poetry install"
- name: "Install Required Dependencies for MySql Client"
run: "sudo apt-get update && sudo apt-get install --no-install-recommends -y default-libmysqlclient-dev gcc"
- name: "Install MySql Dependencies"
run: "poetry install --no-interaction --no-root --extras mysql"
- name: "Run migration test"
run: "poetry run invoke migration-test --db-engine mysql --dataset $MIGRATION_TEST_DATASET"
needs:
- "check-migrations"
- "check-schema"
- "hadolint"
- "markdownlint"
- "ruff"
- "yamllint"
migration-tests-postgres:
name: "Run migrations against test dataset (PostgreSQL and Python 3.11)"
runs-on: "ubuntu-20.04"
env:
MIGRATION_TEST_DATASET: "development/datasets/nautobot_1.5.16_postgres.tar.gz"
INVOKE_NAUTOBOT_LOCAL: "True"
NAUTOBOT_CONFIG: "nautobot/core/tests/nautobot_config.py"
NAUTOBOT_DB_USER: "nautobot"
NAUTOBOT_DB_PASSWORD: "decinablesprewad"
NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.postgresql"
PGPASSWORD: "decinablesprewad"
services:
postgres:
image: "postgres:13-alpine"
env:
POSTGRES_USER: "nautobot"
POSTGRES_PASSWORD: "decinablesprewad"
POSTGRES_DB: "nautobot"
ports:
- "5432:5432"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
with:
python-version: "3.11"
- name: "Remove Installed Nautobots (Poetry Workaround)" # Workaround for https://github.com/python-poetry/poetry/issues/2918
run: "for i in $(ls -d1 ~/.cache/pypoetry/virtualenvs/*/lib/*/site-packages/nautobot-*.dist-info); do poetry run pip uninstall -y nautobot; done"
- name: "Reinstall Nautobot Local"
run: "poetry install"
- name: "Run migration test"
run: "poetry run invoke migration-test --db-engine postgres --dataset $MIGRATION_TEST_DATASET"
needs:
- "check-migrations"
- "check-schema"
- "hadolint"
- "markdownlint"
- "ruff"
- "yamllint"
integration-test:
runs-on: "ubuntu-20.04"
env:
INVOKE_NAUTOBOT_LOCAL: "True"
NAUTOBOT_DB_USER: "nautobot"
NAUTOBOT_DB_PASSWORD: "decinablesprewad"
NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.postgresql"
NAUTOBOT_SELENIUM_URL: "http://localhost:4444/wd/hub"
services:
postgres:
image: "postgres:13-alpine"
env:
POSTGRES_USER: "nautobot"
POSTGRES_PASSWORD: "decinablesprewad"
POSTGRES_DB: "nautobot"
ports:
- "5432:5432"
redis:
image: "redis:6-alpine"
ports:
- "6379:6379"
selenium:
image: "selenium/standalone-firefox:4.19"
ports:
- "4444:4444"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
- name: "Remove Installed Nautobots (Poetry Workaround)" # Workaround for https://github.com/python-poetry/poetry/issues/2918
run: "for i in $(ls -d1 ~/.cache/pypoetry/virtualenvs/*/lib/*/site-packages/nautobot-*.dist-info); do poetry run pip uninstall -y nautobot; done"
- name: "Reinstall Nautobot Local"
run: "poetry install"
- name: "Run Integration Tests"
# If NAUTOBOT_SELENIUM_HOST is set to 'localhost' or '127.0.0.1' the connection does not work
run: "NAUTOBOT_SELENIUM_HOST=`hostname -f` poetry run invoke integration-test --keepdb"
needs:
- "check-migrations"
- "check-schema"
- "hadolint"
- "markdownlint"
- "ruff"
- "yamllint"
changelog:
if: ${{ (contains(fromJson('["develop","next"]'), github.base_ref)) || (startsWith(github.base_ref, 'ltm')) }}
runs-on: "ubuntu-20.04"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
with:
fetch-depth: "0"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v2"
- name: "Check for changelog entry"
run: |
git fetch --no-tags origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
poetry run towncrier check --compare-with origin/${{ github.base_ref }}
container-build-test:
name: "Test Container Build (amd64 only on Python 3.11)"
runs-on: "ubuntu-22.04"
needs:
- "check-migrations"
- "check-schema"
- "hadolint"
- "markdownlint"
- "ruff"
- "yamllint"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v3"
- name: "Set up QEMU"
uses: "docker/setup-qemu-action@v2"
- name: "Set up Docker Buildx"
uses: "docker/setup-buildx-action@v2"
- name: "Login to GitHub Container Registry"
uses: "docker/login-action@v2"
with:
registry: "ghcr.io"
username: "${{ github.actor }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- name: "Build `final`"
uses: "./.github/actions/build-nautobot-image"
with:
branch: "${{ github.head_ref }}"
image: "ghcr.io/nautobot/nautobot"
platforms: "linux/amd64"
push: "false"
python-version: "3.11"
tag-latest: "false"
tag-latest-for-branch: "false"
tag-latest-for-py: "false"
target: "final"
all-tests-passed:
runs-on: "ubuntu-20.04"
steps:
- run: "echo 'All tests succeeded'"
needs:
- "container-build-test"
- "integration-test"
- "migration-tests-mysql"
- "migration-tests-postgres"
- "pylint"
- "tests-mysql"
- "tests-postgres"