Test and CI modernization

.codespell_ignore_list

@@ -0,0 +1,5 @@
+nce
+fpr
+shepard
+dum
+theis

.github/environment-lint.yml

@@ -0,0 +1,21 @@
+name: lint
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  # required
+  - pip
+  - bandit
+  - codespell
+  - flake8
+  - pytest
+  - pydocstyle
+
+  # Dependencies for velin
+  - numpydoc>=1.1.0
+  - sphinx>=5.1.0
+  - pygments
+  - black
+
+  - pip:
+    - velin

.github/environment-minimal.yml

@@ -0,0 +1,12 @@
+name: test
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - pip
+  - numpy ==1.7.0
+  - scipy ==1.0.0
+  - matplotlib==2.1.0
+  - pytest
+  - pytest-cov
+  - pytest-mpl

.github/environment.yml

@@ -4,9 +4,9 @@ channels:
   - defaults
 dependencies:
   - pip
-  - numpy
-  - scipy
-  - matplotlib
+  - numpy >=1.7.0
+  - scipy >=1.0.0
+  - matplotlib>=2.1.0
   - pytest
   - pytest-cov
   - pytest-mpl

.github/workflows/lint_python.yml

@@ -0,0 +1,71 @@
+name: lint_python
+on: [pull_request, push]
+jobs:
+  lint_python:
+    name: "Lint and code analysis"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - os: ubuntu-latest
+            python-version: "3.11"
+            channel-priority: "flexible"
+            envfile: ".github/environment-lint.yml"
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Cache conda
+      uses: actions/cache@v4
+      env:
+        CACHE_NUMBER: 0
+      with:
+        path: ~/conda_pkgs_dir
+        key: ${{ runner.os }}-${{ matrix.python-version }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles( matrix.envfile ) }}
+    - name: Install conda environmnent
+      uses: conda-incubator/setup-miniconda@v3
+      with: 
+        auto-update-conda: false
+        python-version: ${{ matrix.python-version }}
+        add-pip-as-python-dependency: true
+        auto-activate-base: false
+        activate-environment: lint
+        # mamba-version: "*"
+        channel-priority: ${{ matrix.channel-priority }}
+        environment-file: ${{ matrix.envfile }}
+        use-only-tar-bz2: false 
+
+    - name: Conda info
+      shell: bash -l {0}
+      run: |
+        conda info -a
+        conda list
+
+    - name: Spell check package
+      shell: bash -l {0}
+      run: codespell --ignore-words .codespell_ignore_list mir_eval
+
+    - name: Security check
+      shell: bash -l {0}
+      run: bandit --recursive --skip B101,B110 .
+
+    - name: Style check package
+      shell: bash -l {0}
+      run: python -m flake8 mir_eval
+
+    - name: Format check package
+      shell: bash -l {0}
+      run: python -m black --check mir_eval
+
+    - name: Format check tests
+      shell: bash -l {0}
+      run: python -m black --check tests
+
+    - name: Docstring check
+      shell: bash -l {0}
+      run: python -m velin --check mir_eval
+
+    - name: Docstring style check
+      shell: bash -l {0}
+      run: python -m pydocstyle mir_eval

.github/workflows/test.yml

@@ -3,30 +3,38 @@ name: Test Python code
 on:
   pull_request:
     branches:
-      - master
+      - main
   push:
     branches:
-      - master
+      - main
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: True
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: True
 
 jobs:
   test:
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.10", "3.11"]
         channel-priority: [strict]
+        envfile: [".github/environment.yml"]
         include:
-          - python-version: "3.10"
+          - python-version: "3.12"
             os: macos-latest
-          - python-version: "3.10"
+          - python-version: "3.12"
             os: windows-latest
-          - python-version: "3.10"
+          - python-version: "3.12"
             os: ubuntu-latest
             channel-priority: flexible
+          - os: ubuntu-latest
+            python-version: "3.7"
+            envfile: ".github/environment-minimal.yml"
+            channel-priority: "flexible"
+            name: "Minimal dependencies"
+
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
@@ -35,16 +43,18 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('.github/environment.yml') }}
+          key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles( matrix.envfile ) }}
 
       - name: Create conda environment
         uses: conda-incubator/setup-miniconda@v2
         with:
           python-version: ${{ matrix.python-version }}
           auto-activate-base: false
           channel-priority: ${{ matrix.channel-priority }}
-          environment-file: .github/environment.yml
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+          environment-file: ${{ matrix.envfile }}
+          # Disabling bz2 to get more recent dependencies.
+          # NOTE: this breaks cache support, so CI will be slower.
+          use-only-tar-bz2: False # IMPORTANT: This needs to be set for caching to work properly!
 
       - name: Install package in development mode
         shell: bash -l {0}
@@ -60,7 +70,22 @@ jobs:
         shell: bash -l {0}
         run: python -c "import numpy; numpy.show_config()"
 
+      - name: Show libraries in the system on which SciPy was built
+        shell: bash -l {0}
+        run: python -c "import scipy; scipy.show_config()"
+
       - name: Run unit tests
         shell: bash -l {0}
-        run: pytest --cov=mir_eval
+        run: pytest
         working-directory: tests
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          flags: unittests
+          env_vars: OS,PYTHON
+          name: codecov-umbrella
+          fail_ci_if_error: true
+          verbose: true

.gitignore

@@ -44,5 +44,9 @@ Thumbs.db
 # docs
 docs/_build/*
 
-# matplotlib tsets
+# matplotlib tests
 tests/result_images/*
+
+# coverage
+coverage.xml
+

mir_eval/__init__.py

@@ -20,4 +20,4 @@
 from . import transcription_velocity
 from . import key
 
-__version__ = '0.7'
+__version__ = "0.7"

mir_eval/alignment.py

@@ -56,10 +56,8 @@
 from mir_eval.util import filter_kwargs
 
 
-def validate(
-    reference_timestamps: np.ndarray, estimated_timestamps: np.ndarray
-):
-    """Checks that the input annotations to a metric look like valid onset time
+def validate(reference_timestamps: np.ndarray, estimated_timestamps: np.ndarray):
+    """Check that the input annotations to a metric look like valid onset time
     arrays, and throws helpful errors if not.
 
     Parameters
@@ -103,13 +101,9 @@ def validate(
 
     # Check monotonicity
     if not np.all(reference_timestamps[1:] - reference_timestamps[:-1] >= 0):
-        raise ValueError(
-            "Reference timestamps are not monotonically increasing!"
-        )
+        raise ValueError("Reference timestamps are not monotonically increasing!")
     if not np.all(estimated_timestamps[1:] - estimated_timestamps[:-1] >= 0):
-        raise ValueError(
-            "Estimated timestamps are not monotonically increasing!"
-        )
+        raise ValueError("Estimated timestamps are not monotonically increasing!")
 
     # Check positivity (need for correct PCS metric calculation)
     if not np.all(reference_timestamps >= 0):
@@ -181,7 +175,7 @@ def percentage_correct(reference_timestamps, estimated_timestamps, window=0.3):
 def percentage_correct_segments(
     reference_timestamps, estimated_timestamps, duration: Optional[float] = None
 ):
-    """Calculates the percentage of correct segments (PCS) metric.
+    """Calculate the percentage of correct segments (PCS) metric.
 
     It constructs segments out of predicted and estimated timestamps separately
     out of each given timestamp vector and calculates the percentage of overlap between correct
@@ -317,6 +311,7 @@ def karaoke_perceptual_metric(reference_timestamps, estimated_timestamps):
 
 def evaluate(reference_timestamps, estimated_timestamps, **kwargs):
     """Compute all metrics for the given reference and estimated annotations.
+
     Examples
     --------
     >>> reference_timestamps = mir_eval.io.load_events('reference.txt')
@@ -330,7 +325,7 @@ def evaluate(reference_timestamps, estimated_timestamps, **kwargs):
         reference timestamp locations, in seconds
     estimated_timestamps : np.ndarray
         estimated timestamp locations, in seconds
-    kwargs
+    **kwargs
         Additional keyword arguments which will be passed to the
         appropriate metric or preprocessing functions.