Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add issues and issue_comments streams #9

Merged
merged 4 commits into from
Sep 11, 2021
Merged

Add issues and issue_comments streams #9

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
59bdbb9
Update schema for issues stream
laurentS Sep 10, 2021
71b07b7
Update issue comments stream to be a child of repositories and update…
laurentS Sep 10, 2021
da12574
Add issue number to issue_comments stream
laurentS Sep 10, 2021
c9fcc33
Fix black style
laurentS Sep 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 128 additions & 6 deletions tap_github/streams.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,28 +130,131 @@ def http_headers(self) -> dict:

schema = th.PropertiesList(
th.Property("id", th.IntegerType),
th.Property("node_id", th.StringType),
th.Property("url", th.StringType),
th.Property("html_url", th.StringType),
th.Property("repo", th.StringType),
th.Property("org", th.StringType),
th.Property("issue_number", th.IntegerType),
th.Property("number", th.IntegerType),
th.Property("updated_at", th.DateTimeType),
th.Property("created_at", th.DateTimeType),
# th.Property("closed_at", th.DateTimeType), # Nulls causing parse error
th.Property("closed_at", th.DateTimeType),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for restoring this. I think the null behavior problem came up very early in testing and I forgot to retest and restore the field.

th.Property("state", th.StringType),
th.Property("title", th.StringType),
th.Property("comments", th.IntegerType),
th.Property("author_association", th.StringType),
th.Property("body", th.StringType),
th.Property(
"user",
th.ObjectType(
th.Property("login", th.StringType),
th.Property("id", th.IntegerType),
th.Property("node_id", th.StringType),
th.Property("avatar_url", th.StringType),
th.Property("gravatar_id", th.StringType),
th.Property("html_url", th.StringType),
th.Property("type", th.StringType),
th.Property("site_admin", th.BooleanType),
),
),
th.Property(
"labels",
th.ArrayType(
th.ObjectType(
th.Property("id", th.IntegerType),
th.Property("node_id", th.StringType),
th.Property("url", th.StringType),
th.Property("name", th.StringType),
th.Property("description", th.StringType),
th.Property("color", th.StringType),
th.Property("default", th.BooleanType),
),
),
),
th.Property(
"assignee",
th.ObjectType(
th.Property("login", th.StringType),
th.Property("id", th.IntegerType),
th.Property("node_id", th.StringType),
th.Property("avatar_url", th.StringType),
th.Property("gravatar_id", th.StringType),
th.Property("html_url", th.StringType),
th.Property("type", th.StringType),
th.Property("site_admin", th.BooleanType),
),
),
th.Property(
"assignees",
th.ArrayType(
th.ObjectType(
th.Property("login", th.StringType),
th.Property("id", th.IntegerType),
th.Property("node_id", th.StringType),
th.Property("avatar_url", th.StringType),
th.Property("gravatar_id", th.StringType),
th.Property("html_url", th.StringType),
th.Property("type", th.StringType),
th.Property("site_admin", th.BooleanType),
),
),
),
th.Property(
"milestone",
th.ObjectType(
th.Property("html_url", th.StringType),
th.Property("node_id", th.StringType),
th.Property("id", th.IntegerType),
th.Property("number", th.IntegerType),
th.Property("state", th.StringType),
th.Property("title", th.StringType),
th.Property("description", th.StringType),
th.Property(
"creator",
th.ObjectType(
th.Property("login", th.StringType),
th.Property("id", th.IntegerType),
th.Property("node_id", th.StringType),
th.Property("avatar_url", th.StringType),
th.Property("gravatar_id", th.StringType),
th.Property("html_url", th.StringType),
th.Property("type", th.StringType),
th.Property("site_admin", th.BooleanType),
),
),
th.Property("open_issues", th.IntegerType),
th.Property("closed_issues", th.IntegerType),
th.Property("created_at", th.DateTimeType),
th.Property("updated_at", th.DateTimeType),
th.Property("closed_at", th.DateTimeType),
th.Property("due_on", th.DateTimeType),
),
),
th.Property("locked", th.BooleanType),
th.Property(
"pull_request",
th.ArrayType(
th.ObjectType(
th.Property("html_url", th.StringType),
th.Property("url", th.StringType),
),
),
),
).to_dict()


class IssueCommentsStream(GitHubStream):
"""Defines 'Issues' stream."""
"""
Defines 'Issues' stream.
Issue comments are fetched from the repository level (as opposed to per issue)
to optimize for API quota usage.
"""

name = "issue_comments"
path = "/repos/{org}/{repo}/issues/{issue_number}/comments"
path = "/repos/{org}/{repo}/issues/comments"
primary_keys = ["id"]
replication_key = "updated_at"
parent_stream_type = IssuesStream
parent_stream_type = RepositoryStream
state_partitioning_keys = ["repo", "org"]
ignore_parent_replication_key = False

Expand All @@ -177,13 +280,32 @@ def get_url_params(
params["since"] = since
return params

def post_process(self, row: dict, context: Optional[dict] = None) -> dict:
row["issue_number"] = int(row["issue_url"].split("/")[-1])
return row

schema = th.PropertiesList(
th.Property("id", th.IntegerType),
th.Property("node_id", th.StringType),
th.Property("issue_number", th.IntegerType),
th.Property("repo", th.StringType),
th.Property("org", th.StringType),
th.Property("issue_number", th.IntegerType),
th.Property("issue_url", th.IntegerType),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the issue_number property was previously coming from parent context. What do you think of adding back via post_process() method?

Something like "/".split(record["issue_url"])[-1] might do the trick.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like that! I'll add it in. It does seem handy to have the issue number available.

th.Property("updated_at", th.DateTimeType),
th.Property("created_at", th.DateTimeType),
th.Property("author_association", th.StringType),
th.Property("body", th.StringType),
th.Property(
"user",
th.ObjectType(
th.Property("login", th.StringType),
th.Property("id", th.IntegerType),
th.Property("node_id", th.StringType),
th.Property("avatar_url", th.StringType),
th.Property("gravatar_id", th.StringType),
th.Property("html_url", th.StringType),
th.Property("type", th.StringType),
th.Property("site_admin", th.BooleanType),
),
),
).to_dict()