-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Add X profiles scraping functionality and unstar. + Project str…
…ucture.
- Loading branch information
Showing
13 changed files
with
280 additions
and
31 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,167 @@ | ||
import json | ||
import time | ||
import requests | ||
from src.utils.logger import logger | ||
|
||
class XScraper: | ||
def __init__(self, config, max_accounts): | ||
self.github_token = config.get_api_key() | ||
self.headers = { | ||
'Authorization': f'token {self.github_token}', | ||
'Accept': 'application/vnd.github.v3+json' | ||
} | ||
self.jsonl_file = "X_profiles.jsonl" | ||
self.max_accounts = max_accounts | ||
|
||
def get_github_followers(self, username, max_accounts): | ||
url = f'https://api.github.com/users/{username}/followers' | ||
self.max_accounts = max_accounts | ||
followers = self._get_paginated_data(url) | ||
X_profiles = self._extract_X_profiles(followers) | ||
self._update_jsonl_file(X_profiles) | ||
|
||
def _get_paginated_data(self, url): | ||
page = 1 | ||
data = [] | ||
while True: | ||
paginated_url = f'{url}?page={page}' | ||
response = self._make_request('GET', paginated_url) | ||
if response.status_code != 200: | ||
raise Exception(f"Error fetching data from {paginated_url}. Status code: {response.status_code}") | ||
|
||
page_data = response.json() | ||
if not page_data: | ||
break | ||
|
||
data.extend(page_data) | ||
if len(data) >= self.max_accounts: | ||
data = data[:self.max_accounts] | ||
break | ||
|
||
page += 1 | ||
|
||
return data | ||
|
||
def _update_jsonl_file(self, X_profiles): | ||
valid_profiles = [profile for profile in X_profiles if self._is_valid_X(profile['X_url'])] | ||
with open(self.jsonl_file, 'w') as file: | ||
for profile in valid_profiles: | ||
file.write(json.dumps(profile) + '\n') | ||
|
||
def _is_valid_X(self, X_url): | ||
# Placeholder for actual X URL validation logic | ||
return True | ||
|
||
def _make_request(self, method, url): | ||
for _ in range(3): | ||
try: | ||
response = requests.request(method, url, headers=self.headers) | ||
if response.status_code in [500, 429]: | ||
time.sleep(10) | ||
continue | ||
return response | ||
except requests.RequestException as e: | ||
logger.error(f"Request to {url} failed: {e}") | ||
time.sleep(10) | ||
return None | ||
|
||
def _get_paginated_data(self, url): | ||
page = 1 | ||
data = [] | ||
while True: | ||
paginated_url = f'{url}?page={page}' | ||
response = self._make_request('GET', paginated_url) | ||
if response.status_code != 200: | ||
raise Exception(f"Error fetching data from {paginated_url}. Status code: {response.status_code}") | ||
|
||
page_data = response.json() | ||
if not page_data: | ||
break | ||
|
||
data.extend(page_data) | ||
if len(data) >= self.max_accounts: | ||
data = data[:self.max_accounts] | ||
break | ||
|
||
page += 1 | ||
|
||
return data | ||
|
||
def _extract_X_profiles(self, followers): | ||
X_profiles = [] | ||
for follower in followers: | ||
social_accounts_url = f"https://api.github.com/users/{follower['login']}/social_accounts" | ||
response = self._make_request('GET', social_accounts_url) | ||
if response.status_code == 200: | ||
social_accounts = response.json() | ||
X_url = next((account['url'] for account in social_accounts if 'x.com' in account['url']), None) | ||
if X_url: | ||
X_profiles.append({'github_username': follower['login'], 'X_url': X_url}) | ||
return X_profiles | ||
|
||
def _update_jsonl_file(self, X_profiles): | ||
valid_profiles = [profile for profile in X_profiles if self._is_valid_X(profile['X_url'])] | ||
with open(self.jsonl_file, 'w') as file: | ||
for profile in valid_profiles: | ||
file.write(json.dumps(profile) + '\n') | ||
|
||
def _is_valid_X(self, X_url): | ||
# Placeholder for actual X URL validation logic | ||
return True | ||
|
||
def _make_request(self, method, url): | ||
for _ in range(3): | ||
try: | ||
response = requests.request(method, url, headers=self.headers) | ||
if response.status_code in [500, 429]: | ||
time.sleep(10) | ||
continue | ||
return response | ||
except requests.RequestException as e: | ||
logger.error(f"Request to {url} failed: {e}") | ||
time.sleep(10) | ||
return None | ||
|
||
def scrape_X_profiles(self, followers): | ||
if followers is None: | ||
logger.error("Followers list is None, cannot scrape X profiles.") | ||
return | ||
|
||
X_profiles = [] | ||
existing_profiles = self.load_existing_profiles() | ||
|
||
for follower in followers: | ||
username = follower['login'] | ||
X_url = f"https://x.com/{username}" | ||
|
||
if X_url in existing_profiles: | ||
logger.info(f"X profile for {username} already exists, skipping.") | ||
continue | ||
|
||
X_profiles.append({ | ||
"github_username": username, | ||
"X_url": X_url | ||
}) | ||
|
||
if len(X_profiles) >= self.max_accounts: | ||
break | ||
|
||
self.save_profiles_to_jsonl(X_profiles) | ||
|
||
def load_existing_profiles(self): | ||
existing_profiles = set() | ||
try: | ||
with open(self.jsonl_file, 'r') as file: | ||
for line in file: | ||
data = json.loads(line.strip()) | ||
existing_profiles.add(data['X_url']) | ||
except FileNotFoundError: | ||
pass | ||
return existing_profiles | ||
|
||
def save_profiles_to_jsonl(self, profiles): | ||
with open(self.jsonl_file, 'a') as file: | ||
for profile in profiles: | ||
json.dump(profile, file) | ||
file.write('\n') | ||
logger.info(f"Saved {len(profiles)} new X profiles to {self.jsonl_file}.") |
Empty file.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
import json | ||
import time | ||
from src.utils.logger import logger | ||
import requests | ||
|
||
class GitHubClientUnstar: | ||
def __init__(self, config, username): | ||
self.token = config.get_api_key() | ||
self.headers = { | ||
'Authorization': f'token {self.token}', | ||
'Accept': 'application/vnd.github.v3+json' | ||
} | ||
self.username = username | ||
|
||
def get_starred_repos(self): | ||
return self._get_paginated_data(f'https://api.github.com/users/{self.username}/starred') | ||
|
||
def get_followers(self): | ||
return self._get_paginated_data(f'https://api.github.com/users/{self.username}/followers') | ||
|
||
def _get_paginated_data(self, url): | ||
page = 1 | ||
data = [] | ||
while True: | ||
paginated_url = f'{url}?page={page}' | ||
response = self._make_request('GET', paginated_url) | ||
|
||
if response.status_code != 200: | ||
raise Exception(f"Error fetching data from {paginated_url}. Status code: {response.status_code}") | ||
|
||
page_data = response.json() | ||
if not page_data: | ||
break # No more data to fetch | ||
|
||
data.extend(page_data) | ||
page += 1 | ||
|
||
return data | ||
|
||
def _make_request(self, method, url): | ||
max_retries = 3 | ||
for attempt in range(max_retries): | ||
try: | ||
if method == 'GET': | ||
response = requests.get(url, headers=self.headers) | ||
elif method == 'DELETE': | ||
response = requests.delete(url, headers=self.headers) | ||
else: | ||
raise ValueError("Unsupported HTTP method") | ||
return response | ||
except requests.RequestException as e: | ||
logger.error(f"Request failed: {e}") | ||
time.sleep(2 ** attempt) | ||
raise Exception(f"Failed to {method} {url} after {max_retries} attempts") | ||
|
||
def unstar_repo(self, owner, repo): | ||
url = f'https://api.github.com/user/starred/{owner}/{repo}' | ||
response = self._make_request('DELETE', url) | ||
if response.status_code == 204: | ||
logger.info(f"Successfully unstarred {owner}/{repo}") | ||
else: | ||
logger.error(f"Failed to unstar {owner}/{repo}. Status code: {response.status_code}") | ||
|
||
def unstar_non_followers_repos(self): | ||
followers = {follower['login'] for follower in self.get_followers()} | ||
starred_repos = self.get_starred_repos() | ||
|
||
for repo in starred_repos: | ||
owner = repo['owner']['login'] | ||
if owner == "Errahum": | ||
logger.info(f"Skipping {owner}'s repo as it should never be unstarred") | ||
continue | ||
if owner not in followers: | ||
self.unstar_repo(owner, repo['name']) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters