diff --git a/lib/scrape.js b/lib/scrape.js index 33214b5..56a91a2 100644 --- a/lib/scrape.js +++ b/lib/scrape.js @@ -17,6 +17,17 @@ const CHAT_IMAGES = { OTHER: 'static/images/chat.png', } +const GITHUB_OPTIONS = { + headers: process.env.GITHUB_TOKEN + ? { Authorization: `token ${process.env.GITHUB_TOKEN}` } + : {}, +} + +async function fetchProgram() { + const res = await fetch('https://codein.withgoogle.com/api/program/2017/') + return await res.json() +} + async function fetchOrgs() { const res = await fetch( 'https://codein.withgoogle.com/api/program/2017/organization/?status=2' @@ -34,17 +45,40 @@ async function fetchLeaders(id) { } async function searchGitHubOrgs(query) { - const token = process.env.GITHUB_TOKEN const res = await fetch( `${GITHUB_API_BASE}/search/users?q=${query}%20type:org`, - { - headers: token ? { Authorization: `token ${token}` } : {}, - } + GITHUB_OPTIONS ) const { items } = await res.json() return items || [] } +async function getGitHubUserCommits(user, from, to) { + const commitPattern = /([a-zA-Z1-9/-]+)<\/a>/g + const res = await fetch( + `https://github.com/users/${user}/created_commits?from=${from}&to=${to}` + ) + const body = await res.text() + + const commits = [] + let match = commitPattern.exec(body) + while (match) { + commits.push(match[1]) + match = commitPattern.exec(body) + } + + return commits +} + +async function getGitHubUser(user) { + const res = await fetch(`${GITHUB_API_BASE}/users/${user}`, GITHUB_OPTIONS) + let response = await res.json() + if (response && response.message) { + response = undefined + } + return response +} + async function findOrganization({ name, description, @@ -92,6 +126,37 @@ async function findOrganization({ return null } +async function findGitHubUser({ display_name }, org) { + if (!org) return + + display_name = display_name.replace(/ /g, '') + + const displayNamePattern = /^[a-zA-Z0-9-]{1,39}$/ + + const displayNameMatches = displayNamePattern.exec(display_name) + if (!displayNameMatches) return + + const user = await getGitHubUser(display_name) + if (!user) return + + const login = user.login + + const { competition_open_starts } = await fetchProgram() + + const updatedTime = new Date(user.updated_at) + const openTime = new Date(competition_open_starts) + + if (updatedTime.getTime() - openTime.getTime() < 0) return + + const nov = await getGitHubUserCommits(login, '2017-11-28', '2017-11-30') + const dec = await getGitHubUserCommits(login, '2017-12-01', '2017-12-31') + const jan = await getGitHubUserCommits(login, '2018-01-01', '2018-01-17') + const orgs = [...nov, ...dec, ...jan].map(repo => repo.split('/')[0]) + if (orgs.includes(org)) { + return user.login + } +} + async function fetchOrgsWithData() { const orgs = await fetchOrgs() const fetchingLeaders = orgs.map(org => fetchLeaders(org.id)) @@ -101,9 +166,20 @@ async function fetchOrgsWithData() { const orgGitHub = await Promise.all(fetchingGitHub) const orgChats = await Promise.all(fetchingChat) - return orgs.map((org, index) => - Object.assign(org, { - leaders: orgLeaders[index], + const fetchingAll = orgs.map(async (org, index) => { + const fetchingUsers = orgLeaders[index].map(user => + findGitHubUser(user, orgGitHub[index]) + ) + const orgUsers = await Promise.all(fetchingUsers) + + const leaders = orgLeaders[index].map((user, index) => + Object.assign(user, { + github_account: orgUsers[index], + }) + ) + + return Object.assign(org, { + leaders: leaders, github: orgGitHub[index], chat: { url: orgChats[index].url, @@ -111,7 +187,9 @@ async function fetchOrgsWithData() { image: CHAT_IMAGES[chattie.CHAT[orgChats[index].type]], }, }) - ) + }) + + return await Promise.all(fetchingAll) } async function fetchDates() { diff --git a/templates/main.html b/templates/main.html index 7f5c229..2f3eeb5 100644 --- a/templates/main.html +++ b/templates/main.html @@ -28,7 +28,8 @@

Google Code-in 2017 Current Leaders


- The leading participants for each organization are listed randomly. + The leading participants for each organization (and their GitHub accounts, + if applicable) are listed randomly.
{{#withLeader}} @@ -67,7 +68,15 @@