userdev.meltano.yml

environments:
- name: userdev
  config:
    plugins:
      extractors:
      - name: tap-cloudwatch
        config:
          start_date: '2023-01-16'
      - name: tap-slack-public
        config:
          start_date: '2022-09-01'
      - name: tap-google-analytics
        config:
          start_date: '2022-02-20'
      - name: tap-gitlab
        config:
          start_date: '2020-01-01T00:00:00Z'
      - name: tap-github
        config:
          start_date: '2021-05-01'
      - name: tap-snowflake
        config:
          database: USERDEV_PROD
          user: ${USER_PREFIX}
          role: ${USER_PREFIX}
          warehouse: CORE
      # https://github.com/meltano/meltano/issues/3171 is blocking templating from working so I had to hardcode the user prefix
      - name: tap-snowflake-metrics
        config:
          tables:
          - PNADOLNY_MELTANO_HUB.FACT_VARIANT_HUB_METRICS
      - name: tap-snowflake-audit
        config:
          tables:
          - PNADOLNY_MELTANO_HUB.HUB_METRICS_AUDIT
      - name: tap-snowflake-singer-activity
        config:
          tables:
          - PNADOLNY_SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
      - name: tap-snowflake-meltano-activity
        config:
          tables:
          - PNADOLNY_SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
      - name: tap-snowflake-hubspot-companies
        config:
          tables:
          - PNADOLNY_HUBSPOT.ORG_ACTIVITY
          stream_maps:
            pnadolny_hubspot-org_activity:
              org_first_active_date: str(record.get('org_first_active_date')[:10])
                if record.get('org_first_active_date') else None
              org_last_active_date: str(record.get('org_last_active_date')[:10]) if
                record.get('org_last_active_date') else None
      loaders:
      - name: target-snowflake
        config:
          database: USERDEV_RAW
          user: ${USER_PREFIX}
          role: ${USER_PREFIX}
          warehouse: CORE
          default_target_schema: ${USER_PREFIX}_${MELTANO_EXTRACT__LOAD_SCHEMA}
      utilities:
      - name: dbt-snowflake
        config:
          user: ${USER_PREFIX}
          role: ${USER_PREFIX}
          warehouse: CORE
          skip_pre_invoke: true
          database: USERDEV_PROD
          database_prep: USERDEV_PREP
          database_raw: RAW
          # Uncomment for reading a custom RAW schema
          # database_raw: USERDEV_RAW
          # source_schema_prefix: ${USER_PREFIX}_
          target_schema_prefix: ${USER_PREFIX}_
      - name: sqlfluff
        config:
          user: ${USER_PREFIX}
          role: ${USER_PREFIX}
      - name: great_expectations
        config:
          prod_database: USERDEV_PROD
          raw_database: USERDEV_RAW
          username: ${USER_PREFIX}
          role: ${USER_PREFIX}
          warehouse: CORE
      - name: snowflake-cloner
        config:
          clone_from_db_prep: PREP
          clone_from_db_prod: PROD
          clone_to_db_prep: USERDEV_PREP
          clone_to_db_prod: USERDEV_PROD
          account: epa06486
          user: ${USER_PREFIX}
          password: ${SNOWFLAKE_PASSWORD}
          role: ${USER_PREFIX}
          warehouse: CORE
          threads: 6
          schema_prefix: ${USER_PREFIX}_

  env:
    USER_PREFIX: PNADOLNY
    SUPERSET_API_URL: http://localhost:8088
    SUPERSET_USER: admin
    SUPERSET_PASS: admin
    # https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html
    AIRFLOW__CORE__PLUGINS_FOLDER: $MELTANO_PROJECT_ROOT/orchestrate/plugins_local
    AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: '30'
    AIRFLOW_VAR_MELTANO_ENVIRONMENT: userdev
    AIRFLOW_VAR_OPERATOR_TYPE: bash
    # Secrets via KMS
    KMS_PUBLIC_KEY_PATH: utilities/kms/Publickey.pem
    KMS_DOTENV_PATH: .env
    KMS_SECTRETS_PATH: secrets.yml