Skip to content

Commit

Permalink
Fix the api_key, simplify, docs
Browse files Browse the repository at this point in the history
  • Loading branch information
@VioletM
VioletM committed Dec 5, 2024
1 parent 310c42c commit 17254a4
Show file tree
Hide file tree
Showing 4 changed files with 121 additions and 138 deletions.
37 changes: 23 additions & 14 deletions sources/hubspot/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,29 @@

HubSpot is a customer relationship management (CRM) software and inbound marketing platform that helps businesses attract visitors, engage customers, and close leads.

The `dlt` HubSpot verified source allows you to automatically load data from HubSpot into a [destination](https://dlthub.com/docs/dlt-ecosystem/destinations/) of your choice. It loads data from the following endpoints:

| API | Data |
| --- |-----------------------------------------------------------------|
| Contacts | visitors, potential customers, leads |
| Companies | information about organizations |
| Deals | deal records, deal tracking |
| Products | goods, services |
| Tickets | requests for help from customers or users |
| Quotes | pricing information of a product |
| Web analytics | events |
| Owners | information about account managers or users |
| Pipelines | stages and progress tracking, separate resource for each object |
| Properties | custom labels for properties with multiple choice |
The `dlt` HubSpot verified source allows you to automatically load data from HubSpot into a [destination](https://dlthub.com/docs/dlt-ecosystem/destinations/) of your choice. It loads data to the following resources:

| resource | data |
|----------------------------|--------------------------------------------------------------------------|
| contacts | visitors, potential customers, leads |
| contacts_property_history | information about historical changes in contacts properties |
| companies | information about organizations |
| companies_property_history | information about historical changes in companies properties |
| deals | deal records, deal tracking |
| deals_property_history | information about historical changes in deals properties |
| products | goods, services |
| products_property_history | information about historical changes in products properties |
| tickets | requests for help from customers or users |
| tickets_property_history | information about historical changes in tickets properties |
| quotes | pricing information of a product |
| quotes_property_history | information about historical changes in quotes properties |
| Web analytics | events |
| owners | information about account managers or users |
| pipelines_deals | stages and progress tracking for deals |
| stages_timing_deals | history of entering and exiting different stages for the deals pipelines |
| pipelines_tickets | stages and progress tracking for tickets |
| stages_timing_tickets | history of entering and exiting different stages for the tickets pipelines |
| properties | custom labels for properties with multiple choice |

## Initialize the pipeline with Hubspot verified source
```bash
Expand Down
110 changes: 39 additions & 71 deletions sources/hubspot/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ def fetch_data_for_properties(

def crm_objects(
object_type: str,
api_key: str = dlt.secrets.value,
api_key: str,
props: Optional[Sequence[str]] = None,
include_custom_props: bool = True,
archived: bool = False,
Expand All @@ -128,13 +128,15 @@ def crm_objects(
Yields:
Iterator[TDataItems]: Data items retrieved from the API.
"""
props = fetch_props(object_type, api_key, props, include_custom_props)
yield from fetch_data_for_properties(props, api_key, object_type, archived)
props_entry: Sequence[str] = props or ENTITY_PROPERTIES.get(object_type, [])
props_fetched = fetch_props(object_type, api_key, props_entry, include_custom_props)
yield from fetch_data_for_properties(props_fetched, api_key, object_type, archived)


def crm_object_history(
object_type: THubspotObjectType,
api_key: str = dlt.secrets.value,
api_key: str,
props: Optional[Sequence[str]] = None,
include_custom_props: bool = True,
) -> Iterator[TDataItems]:
"""
Expand All @@ -143,77 +145,28 @@ def crm_object_history(
Args:
object_type (THubspotObjectType): Type of HubSpot object (e.g., 'company', 'contact').
api_key (str, optional): API key for HubSpot authentication.
props (Optional[Sequence[str]], optional): List of properties to retrieve. Defaults to None.
include_custom_props (bool, optional): Include custom properties in the result. Defaults to True.
Yields:
Iterator[TDataItems]: Historical property data.
"""

# Fetch the properties from ENTITY_PROPERTIES or default to "All"
props_entry: Union[List[str], str] = ENTITY_PROPERTIES.get(object_type, "All")
props_entry: Union[Sequence[str], str] = props or ENTITY_PROPERTIES.get(
object_type, ALL
)

# Fetch the properties with the option to include custom properties
props: str = fetch_props(object_type, api_key, props_entry, include_custom_props)
props_fetched: str = fetch_props(
object_type, api_key, props_entry, include_custom_props
)

# Yield the property history
yield from fetch_property_history(
CRM_OBJECT_ENDPOINTS[object_type],
api_key,
props,
)


def resource_template(
entity: THubspotObjectType,
api_key: str = dlt.config.value,
include_custom_props: bool = False,
soft_delete: bool = False,
) -> Iterator[TDataItems]:
"""
Template function to yield CRM resources for a specific HubSpot entity.
Args:
entity (THubspotObjectType): Type of HubSpot object (e.g., 'company', 'contact').
api_key (str, optional): HubSpot API key for authentication.
props (Optional[Sequence[str]], optional): List of properties to retrieve. Defaults to None.
include_custom_props (bool, optional): Include custom properties in the result. Defaults to False.
soft_delete (bool, optional): Fetch soft-deleted (archived) records. Defaults to False.
Yields:
Iterator[TDataItems]: CRM object data retrieved from the API.
"""

# Use provided props or fetch from ENTITY_PROPERTIES if not provided
properties: List[str] = ENTITY_PROPERTIES.get(entity, [])

# Use these properties to yield the crm_objects
yield from crm_objects(
entity,
api_key,
props=properties, # Pass the properties to the crm_objects function
include_custom_props=include_custom_props,
archived=soft_delete,
)


def resource_history_template(
entity: THubspotObjectType,
api_key: str = dlt.config.value,
include_custom_props: bool = False,
) -> Iterator[TDataItems]:
"""
Template function to yield historical CRM resource data for a specific HubSpot entity.
Args:
entity (THubspotObjectType): Type of HubSpot object (e.g., 'company', 'contact').
api_key (str, optional): HubSpot API key for authentication.
include_custom_props (bool, optional): Include custom properties in the result. Defaults to False.
Yields:
Iterator[TDataItems]: Historical data for the CRM object.
"""
yield from crm_object_history(
entity, api_key, include_custom_props=include_custom_props
props_fetched,
)


Expand Down Expand Up @@ -251,7 +204,7 @@ def pivot_stages_properties(

def stages_timing(
object_type: str,
api_key: str = dlt.config.value,
api_key: str,
soft_delete: bool = False,
) -> Iterator[TDataItems]:
"""
Expand Down Expand Up @@ -294,14 +247,15 @@ def hubspot(
include_history: bool = False,
soft_delete: bool = False,
include_custom_props: bool = True,
properties: Optional[Dict[str, Any]] = None,
) -> Iterator[DltResource]:
"""
A dlt source that retrieves data from the HubSpot API using the
specified API key.
This function retrieves data for several HubSpot API endpoints,
including companies, contacts, deals, tickets, products and web
analytics events. It returns a tuple of Dlt resources, one for
analytics events. It returns a tuple of dlt resources, one for
each endpoint.
Args:
Expand All @@ -315,6 +269,10 @@ def hubspot(
Whether to fetch deleted properties and mark them as `is_deleted`.
include_custom_props (bool):
Whether to include custom properties.
properties (Optional(dict)):
A dictionary containing lists of properties for all the resources.
Will override the default properties ENTITY_PROPERTIES from settings
For ex., {"contact": ["createdate", "email", "firstname", "hs_object_id", "lastmodifieddate", "lastname",]}
Returns:
Sequence[DltResource]: Dlt resources, one for each HubSpot API endpoint.
Expand Down Expand Up @@ -424,30 +382,40 @@ def get_pipelines(object_type: THubspotObjectType) -> Iterator[TDataItems]:
name=name,
write_disposition="merge",
primary_key=["id", "stage"],
)(OBJECT_TYPE_SINGULAR[obj_type], soft_delete=soft_delete)
)(
OBJECT_TYPE_SINGULAR[obj_type],
api_key=api_key,
soft_delete=soft_delete,
)

# resources for all objects
for obj in ALL_OBJECTS:
yield dlt.resource(
resource_template,
crm_objects,
name=OBJECT_TYPE_PLURAL[obj],
write_disposition="merge",
primary_key="id",
)(
entity=obj,
object_type=obj,
api_key=api_key,
props=properties.get(obj) if properties else None,
include_custom_props=include_custom_props,
soft_delete=soft_delete,
archived=soft_delete,
)

# corresponding history resources
if include_history:
for obj in ALL_OBJECTS:
yield dlt.resource(
resource_history_template,
crm_object_history,
name=f"{OBJECT_TYPE_PLURAL[obj]}_property_history",
write_disposition="merge",
primary_key="object_id",
)(entity=obj, include_custom_props=include_custom_props)
write_disposition="append",
)(
object_type=obj,
api_key=api_key,
props=properties.get(obj) if properties else None,
include_custom_props=include_custom_props,
)

# owners resource
yield owners
Expand Down
4 changes: 2 additions & 2 deletions sources/hubspot/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,9 @@


# 'ALL' represents a list of all available properties for all types
ALL = [{"properties": "All"}]
ALL = "All"

PIPELINES_OBJECTS = ["deals"]
PIPELINES_OBJECTS = ["deals", "tickets"]
SOFT_DELETE_KEY = "is_deleted"
ARCHIVED_PARAM = {"archived": True}
PREPROCESSING = {"split": ["hs_merged_object_ids"]}
Expand Down
Loading

0 comments on commit 17254a4

Please sign in to comment.