Merge branch 'main' into 1626-configurable-batch-size-and-max-wait-li…

…mit-for-targets

.github/ISSUE_TEMPLATE/bug.yml

@@ -15,7 +15,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.32.0"
+      placeholder: "0.33.0"
     validations:
       required: true
   - type: checkboxes

.github/workflows/codspeed.yml

@@ -0,0 +1,44 @@
+name: codspeed
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out the repository
+      uses: actions/[email protected]
+
+    - name: Setup Python 3.11
+      uses: actions/[email protected]
+      with:
+        python-version: 3.11
+        architecture: x64
+
+    - name: Install poetry
+      run: |
+        curl -fsS https://install.python-poetry.org | python - -y
+
+    - name: Configure poetry
+      run: poetry config virtualenvs.create false
+
+    - name: Install project
+      run: >
+        poetry install
+        -vvv
+        --with dev
+        --with benchmark
+        --all-extras
+
+    - name: Run benchmarks
+      uses: CodSpeedHQ/action@v1
+      with:
+        token: ${{ secrets.CODSPEED_TOKEN }}
+        run: pytest tests/ --codspeed

.github/workflows/constraints.txt

@@ -1,5 +1,5 @@
-pip==23.2.1
+pip==23.3
 poetry==1.6.1
-pre-commit==3.4.0
+pre-commit==3.5.0
 nox==2023.4.22
 nox-poetry==1.0.3

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ ci:
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.5.0
   hooks:
   - id: check-json
   - id: check-toml

CHANGELOG.md

@@ -5,22 +5,28 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## v0.32.0 (2023-09-22)
+## v0.33.0 (2023-10-12)
 
-## v0.32.0b3 (2023-09-22)
+### ✨ New
 
-### 🐛 Fixes
+- [#1999](https://github.com/meltano/sdk/issues/1999) Log JSONPath match count at the INFO level
+- [#1779](https://github.com/meltano/sdk/issues/1779) Cache SQL columns and schemas
+- [#2003](https://github.com/meltano/sdk/issues/2003) Add ability to do list comprehensions in stream map expressions -- _**Thanks @haleemur!**_
+- [#2018](https://github.com/meltano/sdk/issues/2018) Drop Python 3.7 support in cookiecutter templates -- _**Thanks @visch!**_
 
-- [#1977](https://github.com/meltano/sdk/issues/1977) Fix hanging downstream tests in tap-postgres
-- [#1970](https://github.com/meltano/sdk/issues/1970) Warn instead of crashing when schema helpers cannot append `null` to types
+### 🐛 Fixes
 
-## v0.32.0b2 (2023-09-15)
+- [#2006](https://github.com/meltano/sdk/issues/2006) Parse record `time_extracted` into `datetime.datetime` instance
+- [#1996](https://github.com/meltano/sdk/issues/1996) Respect nullability of leaf properties when flattening schema
+- [#1844](https://github.com/meltano/sdk/issues/1844) Safely skip parsing record field as date-time if it is missing in schema
+- [#1885](https://github.com/meltano/sdk/issues/1885) Map `record` field to a JSON `object` type
+- [#2015](https://github.com/meltano/sdk/issues/2015) Ensure `default` property is passed to SCHEMA messages -- _**Thanks @prakharcode!**_
 
-### ⚡ Performance Improvements
+### 📚 Documentation Improvements
 
-- [#1962](https://github.com/meltano/sdk/issues/1962) Ensure `raw_schema` in stream mapper is immutable
+- [#2017](https://github.com/meltano/sdk/issues/2017) Document support for comprehensions in stream maps
 
-## v0.32.0b1 (2023-09-13)
+## v0.32.0 (2023-09-22)
 
 ### ✨ New
 
@@ -34,11 +40,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#1949](https://github.com/meltano/sdk/issues/1949) Retry SQLAlchemy engine creation for adapters without JSON SerDe support
 - [#1939](https://github.com/meltano/sdk/issues/1939) Handle `decimal.Decimal` instances in flattening
 - [#1927](https://github.com/meltano/sdk/issues/1927) Handle replication key not found in stream schema -- _**Thanks @mjsqu!**_
+- [#1977](https://github.com/meltano/sdk/issues/1977) Fix hanging downstream tests in tap-postgres
+- [#1970](https://github.com/meltano/sdk/issues/1970) Warn instead of crashing when schema helpers cannot append `null` to types
 
-### 📚 Documentation Improvements
+### ⚡ Performance Improvements
 
 - [#1925](https://github.com/meltano/sdk/issues/1925) Add viztracer command for testing targets -- _**Thanks @mjsqu!**_
 
+- [#1962](https://github.com/meltano/sdk/issues/1962) Ensure `raw_schema` in stream mapper is immutable
+
 ## v0.31.1 (2023-08-17)
 
 ### ✨ New

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml

@@ -21,13 +21,13 @@ packages = [
 {%- endif %}
 
 [tool.poetry.dependencies]
-python = ">=3.7.1,<4"
-singer-sdk = { version="~=0.32.0" }
+python = ">=3.8,<4"
+singer-sdk = { version="~=0.33.0" }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.0"
-singer-sdk = { version="~=0.32.0", extras = ["testing"] }
+singer-sdk = { version="~=0.33.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml

@@ -20,8 +20,8 @@ packages = [
 {%- endif %}
 
 [tool.poetry.dependencies]
-python = ">=3.7.1,<4"
-singer-sdk = { version="~=0.32.0" }
+python = ">=3.8,<4"
+singer-sdk = { version="~=0.33.0" }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
 requests = "~=2.31.0"
@@ -32,7 +32,7 @@ cached-property = "~=1" # Remove after Python 3.7 support is dropped
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.0"
-singer-sdk = { version="~=0.32.0", extras = ["testing"] }
+singer-sdk = { version="~=0.33.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml

@@ -20,16 +20,16 @@ packages = [
 {%- endif %}
 
 [tool.poetry.dependencies]
-python = ">=3.7.1,<4"
-singer-sdk = { version="~=0.32.0" }
+python = ">=3.8,<4"
+singer-sdk = { version="~=0.33.0" }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.serialization_method != "SQL" %}
 requests = "~=2.31.0"
 {%- endif %}
 
 [tool.poetry.dev-dependencies]
 pytest = ">=7.4.0"
-singer-sdk = { version="~=0.32.0", extras = ["testing"] }
+singer-sdk = { version="~=0.33.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

docs/conf.py

@@ -26,7 +26,7 @@
 author = "Meltano Core Team and Contributors"
 
 # The full version, including alpha/beta/rc tags
-release = "0.32.0"
+release = "0.33.0"
 
 
 # -- General configuration ---------------------------------------------------

docs/stream_maps.md

@@ -155,6 +155,32 @@ Expressions are defined and parsed using the
 accepts most native python expressions and is extended by custom functions which have been declared
 within the SDK.
 
+#### Compound Expressions
+
+Starting in version 0.33.0, the SDK supports the use of simple comprehensions, e.g. `[x + 1 for x in [1,2,3]]`. This is a powerful feature which allows you to perform complex transformations on lists of values. For example, you can use comprehensions to filter out values in an array:
+
+````{tab} meltano.yml
+```yaml
+stream_maps:
+  users:
+    id: id
+    fields: "[f for f in fields if f['key'] != 'age']"
+```
+````
+
+````{tab} JSON
+```json
+{
+  "stream_maps": {
+    "users": {
+      "id": "id",
+      "fields": "[f for f in fields if f['key'] != 'age']"
+    }
+  }
+}
+```
+````
+
 ### Accessing Stream Properties within Mapping Expressions
 
 By default, all stream properties are made available via the property's given name. For

noxfile.py

@@ -34,6 +34,7 @@
 nox.options.sessions = (
     "mypy",
     "tests",
+    "benches",
     "doctest",
     "test_cookiecutter",
 )
@@ -42,8 +43,9 @@
     "duckdb",
     "duckdb-engine",
     "pytest",
-    "pytest-snapshot",
+    "pytest-benchmark",
     "pytest-durations",
+    "pytest-snapshot",
     "pyarrow",
     "requests-mock",
     "time-machine",
@@ -104,13 +106,34 @@ def tests(session: Session) -> None:
             "pytest",
             "-v",
             "--durations=10",
+            "--benchmark-skip",
             *session.posargs,
         )
     finally:
         if session.interactive:
             session.notify("coverage", posargs=[])
 
 
+@session(python=main_python_version)
+def benches(session: Session) -> None:
+    """Run benchmarks."""
+    session.install(".[s3]")
+    session.install(*test_dependencies)
+    sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION")
+    if sqlalchemy_version:
+        # Bypass nox-poetry use of --constraint so we can install a version of
+        # SQLAlchemy that doesn't match what's in poetry.lock.
+        session.poetry.session.install(  # type: ignore[attr-defined]
+            f"sqlalchemy=={sqlalchemy_version}",
+        )
+    session.run(
+        "pytest",
+        "--benchmark-only",
+        "--benchmark-json=output.json",
+        *session.posargs,
+    )
+
+
 @session(python=main_python_version)
 def update_snapshots(session: Session) -> None:
     """Update pytest snapshots."""
@@ -249,3 +272,25 @@ def test_cookiecutter(session: Session, replay_file_path) -> None:
     session.run("git", "init", external=True)
     session.run("git", "add", ".", external=True)
     session.run("pre-commit", "run", "--all-files", external=True)
+
+
+@session(name="version-bump")
+def version_bump(session: Session) -> None:
+    """Run commitizen."""
+    session.install(
+        "commitizen",
+        "commitizen-version-bump @ git+https://github.com/meltano/commitizen-version-bump.git@main",
+    )
+    default_args = [
+        "--changelog",
+        "--files-only",
+        "--check-consistency",
+        "--changelog-to-stdout",
+    ]
+    args = session.posargs or default_args
+
+    session.run(
+        "cz",
+        "bump",
+        *args,
+    )