From 6f769eb63f39b4dedfe061542824c7619fa8b203 Mon Sep 17 00:00:00 2001 From: z3z1ma Date: Mon, 1 Aug 2022 14:10:08 -0700 Subject: [PATCH] add support for launch darkly and clean up some stuff --- .gitignore | 3 +- README.md | 83 +++++++++++++++++++++---------- dbt_feature_flags/base.py | 5 +- dbt_feature_flags/harness.py | 12 ++++- dbt_feature_flags/launchdarkly.py | 72 +++++++++++++++++++++++++++ dbt_feature_flags/patch.py | 4 +- poetry.lock | 59 +++++++++++++++++++++- pyproject.toml | 3 +- 8 files changed, 207 insertions(+), 34 deletions(-) create mode 100644 dbt_feature_flags/launchdarkly.py diff --git a/.gitignore b/.gitignore index 7773828..1be5bdb 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -dist/ \ No newline at end of file +__pycache__/ +dist/ diff --git a/README.md b/README.md index 919252c..0c0bc05 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # dbt-feature-flags -## Basics +## Why Feature Flags? > At a foundational level, feature flags enable code to be committed and deployed to production in a dormant state and then activated later. This gives teams more control over the user experience of the end product. Development teams can choose when and to which users new code is delivered. - Atlassian (Ian Buchannan) @@ -11,16 +11,30 @@ This ELT pattern heavily encourages experimentation. dbt-feature-flags allow dbt ## Usage -This integration uses Harness Feature Flags. Sign up [here](https://harness.io/products/feature-flags). It's free to use and provides the interface for controlling your feature flags. +This integration uses Harness Feature Flags by default. Sign up [here](https://harness.io/products/feature-flags). It's free to use and provides the interface for controlling your feature flags. Interface ![flow](https://files.helpdocs.io/kw8ldg1itf/articles/1j7pdkqh7j/1657792368788/screenshot-2022-07-14-at-10-52-03.png) +Alternatively we also support [LaunchDarkly](https://launchdarkly.com/) and the package is architected in such a way that adding a new client is fairly straightforward. + ### Set Up +Supported clients + +| clients | supported | +|--------------|-----------| +| harness | ✅ | +| launchdarkly | ✅ | +| unleashed | ⛔️ | + +The below options are applicable to all clients unless specifically noted otherwise. + Required env vars: -`DBT_FF_API_KEY` - your feature flags key. Instructions [here](https://docs.harness.io/article/1j7pdkqh7j-create-a-feature-flag#step_3_create_an_sdk_key) to set it up +`FF_PROVIDER` - Must be one of above supported providers exactly as shown. Defaults to harness if unset out of convenience. So to override: FF_PROVIDER=launchdarkly + +`DBT_FF_API_KEY` - your feature flags key. Instructions [here](https://docs.harness.io/article/1j7pdkqh7j-create-a-feature-flag#step_3_create_an_sdk_key) to set up a harness key. Because of the server-side use case with no client SDKs in play, the Harness free tier can sustain **any size** dbt deployment. Optional: @@ -28,13 +42,28 @@ Optional: `DBT_FF_DISABLE` - disable the patch, note that feature_flag expressions will cause your dbt models not to compile until removed or replaced. If you have the package as a dependency and aren't using it, you can save a second of initialization -`DBT_FF_DELAY` - delay before evaluating feature flags, you shouldn't need this but feature flags have a cache that is seeded asynchronously on initialization so a small delay is required to evaluate properly. Our default delay is 1s +`DBT_FF_DELAY` - delay before evaluating feature flags, you shouldn't need this but feature flags have a cache that is seeded asynchronously on initialization so a small delay is required to evaluate properly. Our default delay is 1s (HARNESS CLIENT ONLY) + +### Jinja Functions + +These are available *anywhere* dbt jinja is evaluated. That includes profiles.yml, dbt_project.yml, models, macros, etc. + +`feature_flag(flag: str)`: Looks for boolean variation flag. By default returns False. Most flags are boolean. Will throw RuntimeError if different return type is detected. + +`feature_flag_str(flag: str)`: Looks for string variation flag. By default returns "". Will throw RuntimeError if different return type is detected. + +`feature_flag_num(flag: str)`: Looks for number variation flag. By default returns 0. Will throw RuntimeError if different return type is detected. + +`feature_flag_json(flag: str)`: Looks for json variation flag. By default returns an empty dict {}. Will throw RuntimeError if different return type is detected. ## Examples A contrived example: ```sql +-- Use a feature_flag call as a bool value +{{ config(enabled=feature_flag("custom_date_model")) }} + select * {%- if feature_flag("new_relative_date_columns") %}, @@ -55,42 +84,44 @@ from {{ ref('dim_dates__base') }} ``` -BQ ML Model example (this could be ran in a run-operation, feature flags are valid anywhere dbt evaluates jinja) +BQ ML model example (this could be ran in a `run-operation`, feature flags are valid anywhere dbt evaluates jinja) ```sql -CREATE OR REPLACE MODEL `bqml_tutorial.penguins_model` -OPTIONS - (model_type='linear_reg', - input_label_cols=['body_mass_g']) AS -SELECT +create or replace model `bqml_tutorial.penguins_model` +options ( + model_type='linear_reg', + input_label_cols=['body_mass_g'] ) as +select * -FROM - `bigquery-public-data.ml_datasets.penguins` -WHERE +from + {{ source('ml_datasets', 'penguins') }} +where {% if feature_flag("penguins_model_min_weight_filter") %} body_mass_g > 100 {% else %} - body_mass_g IS NOT NULL + body_mass_g is not null {% endif %} ``` +Another BQ ML example + ```sql -SELECT +select * -FROM - ML.EVALUATE( +from + ml.evaluate( {% if feature_flag("use_v2_ml_model") %} - MODEL `bqml_tutorial.penguins_model_v2`, + model `bqml_tutorial.penguins_model_v2`, {% else %} - MODEL `bqml_tutorial.penguins_model`, - {% endif %} - ( - SELECT + model `bqml_tutorial.penguins_model`, + {% endif %} ( + select * - FROM - `bigquery-public-data.ml_datasets.penguins` - WHERE - body_mass_g IS NOT NULL)) + from + {{ source('ml_datasets', 'penguins') }} + where + body_mass_g is not null +)) ``` A dbt yaml example diff --git a/dbt_feature_flags/base.py b/dbt_feature_flags/base.py index d6ce87c..c0b5e98 100644 --- a/dbt_feature_flags/base.py +++ b/dbt_feature_flags/base.py @@ -1,4 +1,5 @@ import abc +from typing import Union class BaseFeatureFlagsClient(abc.ABC): @@ -19,13 +20,13 @@ def string_variation(self, flag: str) -> str: ) @abc.abstractmethod - def number_variation(self, flag: str) -> float | int: + def number_variation(self, flag: str) -> Union[float, int]: raise NotImplementedError( "Number feature flags are not implemented for this driver" ) @abc.abstractmethod - def json_variation(self, flag: str) -> dict | list: + def json_variation(self, flag: str) -> Union[dict, list]: raise NotImplementedError( "JSON feature flags are not implemented for this driver" ) diff --git a/dbt_feature_flags/harness.py b/dbt_feature_flags/harness.py index e3fa9c5..e5a0b7e 100644 --- a/dbt_feature_flags/harness.py +++ b/dbt_feature_flags/harness.py @@ -1,9 +1,12 @@ +from typing import Union + from dbt_feature_flags.base import BaseFeatureFlagsClient class HarnessFeatureFlagsClient(BaseFeatureFlagsClient): def __init__(self): # Lazy imports + import atexit import logging import os import time @@ -31,14 +34,19 @@ def __init__(self): self.client = CfClient(FF_KEY) time.sleep(float(os.getenv("DBT_FF_DELAY", 1.0))) + def exit_handler(c: CfClient): + c.close() + + atexit.register(exit_handler, self.client) + def bool_variation(self, flag: str) -> bool: return self.client.bool_variation(flag, target=self.target, default=False) def string_variation(self, flag: str) -> str: return self.client.string_variation(flag, target=self.target, default="") - def number_variation(self, flag: str) -> float | int: + def number_variation(self, flag: str) -> Union[float, int]: return self.client.number_variation(flag, target=self.target, default=0) - def json_variation(self, flag: str) -> dict | list: + def json_variation(self, flag: str) -> Union[dict, list]: return self.client.json_variation(flag, target=self.target, default={}) diff --git a/dbt_feature_flags/launchdarkly.py b/dbt_feature_flags/launchdarkly.py new file mode 100644 index 0000000..4b3fc8f --- /dev/null +++ b/dbt_feature_flags/launchdarkly.py @@ -0,0 +1,72 @@ +from typing import Union + +from dbt_feature_flags.base import BaseFeatureFlagsClient + + +class LaunchDarklyFeatureFlagsClient(BaseFeatureFlagsClient): + def __init__(self): + # Lazy imports + import atexit + import os + + import ldclient + from ldclient.config import Config + + # Set up target + self.target = { + "key": "dbt-" + os.getenv("DBT_TARGET", "default"), + "name": os.getenv("DBT_TARGET", "default").title(), + } + + # Get key + FF_KEY = os.getenv("DBT_FF_API_KEY") + if FF_KEY is None: + raise RuntimeError( + "dbt-feature-flags injected in environment, this patch requires the env var DBT_FF_API_KEY" + ) + + # Init client + ldclient.set_config(Config(FF_KEY)) + self.client = ldclient.get() + if not self.client.is_initialized(): + raise RuntimeError( + "LaunchDarkly SDK failed to initialize, ensure (DBT_FF_API_KEY=%s) is correct.", + FF_KEY, + ) + + def exit_handler(c: ldclient.LDClient): + c.close() + + atexit.register(exit_handler, self.client) + + def bool_variation(self, flag: str) -> bool: + v = self.client.variation(flag, user=self.target, default=False) + if not isinstance(v, bool): + raise ValueError( + "Non boolean return type found for feature_flag call, use appropriate feature_flag_* call" + ) + return v + + def string_variation(self, flag: str) -> str: + v = self.client.variation(flag, user=self.target, default="") + if not isinstance(v, str): + raise ValueError( + "Non string return type found for feature_flag_str call, use appropriate feature_flag_* call" + ) + return v + + def number_variation(self, flag: str) -> Union[float, int]: + v = self.client.variation(flag, user=self.target, default=0) + if not isinstance(v, (float, int)): + raise ValueError( + "Non number return type found for feature_flag_num call, use appropriate feature_flag_* call" + ) + return v + + def json_variation(self, flag: str) -> Union[dict, list]: + v = self.client.variation(flag, user=self.target, default={}) + if not isinstance(v, (dict, list)): + raise ValueError( + "Non JSON return type found for feature_flag_json call, use appropriate feature_flag_* call" + ) + return v diff --git a/dbt_feature_flags/patch.py b/dbt_feature_flags/patch.py index 213cecc..3e854c6 100644 --- a/dbt_feature_flags/patch.py +++ b/dbt_feature_flags/patch.py @@ -3,7 +3,7 @@ """ import os -from dbt_feature_flags import base, harness +from dbt_feature_flags import base, harness, launchdarkly def _get_client() -> base.BaseFeatureFlagsClient: @@ -13,6 +13,8 @@ def _get_client() -> base.BaseFeatureFlagsClient: ff_client = None if ff_provider == "harness": ff_client = harness.HarnessFeatureFlagsClient() + elif ff_provider == "launchdarkly": + ff_client = launchdarkly.LaunchDarklyFeatureFlagsClient() if not isinstance(ff_client, base.BaseFeatureFlagsClient): raise RuntimeError( "Invalid feature flag client specified by (FF_PROVIDER=%s)", diff --git a/poetry.lock b/poetry.lock index 1440d96..11f951c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -135,6 +135,17 @@ category = "main" optional = false python-versions = ">=3.6.1" +[[package]] +name = "expiringdict" +version = "1.2.2" +description = "Dictionary with auto-expiring values for caching purposes" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +tests = ["nose", "mock", "coveralls", "coverage", "dill"] + [[package]] name = "future" version = "0.18.2" @@ -261,6 +272,26 @@ six = ">=1.11.0" format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] format_nongpl = ["idna", "jsonpointer (>1.13)", "webcolors", "rfc3986-validator (>0.1.0)", "rfc3339-validator"] +[[package]] +name = "launchdarkly-server-sdk" +version = "7.5.0" +description = "LaunchDarkly SDK for Python" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +certifi = ">=2018.4.16" +expiringdict = ">=1.1.4" +pyRFC3339 = ">=1.0" +semver = ">=2.10.2,<3.0.0" +urllib3 = ">=1.22.0" + +[package.extras] +consul = ["python-consul (>=1.0.1)"] +dynamodb = ["boto3 (>=1.9.71)"] +redis = ["redis (>=2.10.5)"] + [[package]] name = "leather" version = "0.3.4" @@ -437,6 +468,17 @@ python-versions = ">=3.6.8" [package.extras] diagrams = ["railroad-diagrams", "jinja2"] +[[package]] +name = "pyrfc3339" +version = "1.1" +description = "Generate and parse RFC 3339 timestamps" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pytz = "*" + [[package]] name = "pyrsistent" version = "0.18.1" @@ -548,6 +590,14 @@ idna = {version = "*", optional = true, markers = "extra == \"idna2008\""} [package.extras] idna2008 = ["idna"] +[[package]] +name = "semver" +version = "2.13.0" +description = "Python helper for Semantic Versioning (http://semver.org/)" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [[package]] name = "six" version = "1.16.0" @@ -623,7 +673,7 @@ watchdog = ["watchdog"] [metadata] lock-version = "1.1" python-versions = ">=3.8,<4" -content-hash = "bd27bcc9b5741122d5d9767e31ae4b59c9f8632ea3e684e96637a45d6840e062" +content-hash = "4290b3da8f549c70d2673772be57bdfa06cc65c4b1aab8a316cc7c41b2651d86" [metadata.files] agate = [ @@ -740,6 +790,7 @@ dbt-extractor = [ {file = "dbt_extractor-0.4.1-cp36-abi3-win_amd64.whl", hash = "sha256:35265a0ae0a250623b0c2e3308b2738dc8212e40e0aa88407849e9ea090bb312"}, {file = "dbt_extractor-0.4.1.tar.gz", hash = "sha256:75b1c665699ec0f1ffce1ba3d776f7dfce802156f22e70a7b9c8f0b4d7e80f42"}, ] +expiringdict = [] future = [ {file = "future-0.18.2.tar.gz", hash = "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"}, ] @@ -767,6 +818,7 @@ jsonschema = [ {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, ] +launchdarkly-server-sdk = [] leather = [ {file = "leather-0.3.4-py2.py3-none-any.whl", hash = "sha256:5e741daee96e9f1e9e06081b8c8a10c4ac199301a0564cdd99b09df15b4603d2"}, {file = "leather-0.3.4.tar.gz", hash = "sha256:b43e21c8fa46b2679de8449f4d953c06418666dc058ce41055ee8a8d3bb40918"}, @@ -945,6 +997,7 @@ pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] +pyrfc3339 = [] pyrsistent = [ {file = "pyrsistent-0.18.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1"}, {file = "pyrsistent-0.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26"}, @@ -1031,6 +1084,10 @@ rfc3986 = [ {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, ] +semver = [ + {file = "semver-2.13.0-py2.py3-none-any.whl", hash = "sha256:ced8b23dceb22134307c1b8abfa523da14198793d9787ac838e70e29e77458d4"}, + {file = "semver-2.13.0.tar.gz", hash = "sha256:fa0fe2722ee1c3f57eac478820c3a5ae2f624af8264cbdf9000c980ff7f75e3f"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, diff --git a/pyproject.toml b/pyproject.toml index 806d4aa..ad1e4ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dbt-feature-flags" -version = "0.2.0" +version = "0.3.0" description = "Use feature flags in dbt models" authors = ["z3z1ma "] include = ["zzz_dbt_feature_flags.pth"] @@ -12,6 +12,7 @@ homepage = "https://github.com/z3z1ma/dbt-feature-flags" python = ">=3.8,<4" harness-featureflags = "^1.1.0" dbt-core = ">=1.1.0,<1.3.0" +launchdarkly-server-sdk = "^7.5.0" [tool.poetry.dev-dependencies] pytest = "^5.2"