From 8c79f1c53687475926b255295ac2819f80493e55 Mon Sep 17 00:00:00 2001 From: Dan Allen Date: Thu, 30 May 2019 02:07:35 -0600 Subject: [PATCH] resolves #510 add support for projects hosted on GitLab --- package.json | 1 + scripts/fetch-archive.js | 50 ++++++++++++++++++++++++++-------------- site.yaml | 2 +- src/Home/DeployButton.js | 2 +- src/Home/Project.js | 6 ++--- src/Project/Project.js | 4 ++-- static.config.js | 19 ++++++++++++--- 7 files changed, 57 insertions(+), 27 deletions(-) diff --git a/package.json b/package.json index 44fa53682..bcf1de6f9 100755 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "@octokit/rest": "^14.0.9", "date-fns": "^1.29.0", "dotenv": "^5.0.0", + "gitlab": "^5.0.1", "gray-matter": "^3.1.1", "js-yaml": "^3.11.0", "marked": "^0.3.14", diff --git a/scripts/fetch-archive.js b/scripts/fetch-archive.js index 126cb0398..c9ee57135 100644 --- a/scripts/fetch-archive.js +++ b/scripts/fetch-archive.js @@ -1,14 +1,16 @@ import path from 'path' import fs from 'fs-extra' -import { map, find, fromPairs, mapValues } from 'lodash' +import { compact, map, find, fromPairs, mapValues } from 'lodash' import fetch from 'node-fetch' import { differenceInMinutes } from 'date-fns' import Octokit from '@octokit/rest' +import { Gitlab } from 'gitlab' import twitterFollowersCount from 'twitter-followers-count' require('dotenv').config() const GITHUB_TOKEN = process.env.STATICGEN_GITHUB_TOKEN +const GITLAB_TOKEN = process.env.STATICGEN_GITLAB_TOKEN const TWITTER_CONSUMER_KEY = process.env.STATICGEN_TWITTER_CONSUMER_KEY const TWITTER_CONSUMER_SECRET = process.env.STATICGEN_TWITTER_CONSUMER_SECRET const TWITTER_ACCESS_TOKEN_KEY = process.env.STATICGEN_TWITTER_ACCESS_TOKEN_KEY @@ -18,11 +20,13 @@ const LOCAL_ARCHIVE_PATH = `tmp/${ARCHIVE_FILENAME}` const GIST_ARCHIVE_DESCRIPTION = 'STATICGEN.COM DATA ARCHIVE' let octokit +let gitlab let getTwitterFollowers function authenticate () { octokit = Octokit() octokit.authenticate({ type: 'token', token: GITHUB_TOKEN }) + gitlab = new Gitlab({ personaltoken: GITLAB_TOKEN }) getTwitterFollowers = twitterFollowersCount({ consumer_key: TWITTER_CONSUMER_KEY, consumer_secret: TWITTER_CONSUMER_SECRET, @@ -34,22 +38,35 @@ function authenticate () { async function getProjectGitHubData (repo) { const [owner, repoName] = repo.split('/') const { data } = await octokit.repos.get({ owner, repo: repoName }) - const { stargazers_count, forks_count, open_issues_count } = data - return { stars: stargazers_count, forks: forks_count, issues: open_issues_count } + const { stargazers_count, forks_count, open_issues_count, html_url } = data + return { stars: stargazers_count, forks: forks_count, issues: open_issues_count, repo: html_url } } -async function getAllProjectGitHubData (repos) { +async function getProjectGitLabData (repo) { + const { id, star_count, forks_count, web_url } = await gitlab.Projects.show(repo) + const open_issues_count = (await gitlab.Issues.all({ projectId: id, state: 'opened' })).length + return { stars: star_count, forks: forks_count, issues: open_issues_count, repo: web_url } +} + +async function getAllProjectRepoData (repos) { const data = [] // eslint-disable-next-line no-restricted-syntax - for (const repo of repos) { + for (const repoWithHost of repos) { + const [repoHost, repo] = repoWithHost.split(':') // eslint-disable-next-line no-await-in-loop await new Promise(res => setTimeout(res, 100)) try { - // eslint-disable-next-line no-await-in-loop - const repoData = await getProjectGitHubData(repo) - data.push([repo, repoData]) + if (repoHost === 'gitlab') { + // eslint-disable-next-line no-await-in-loop + const repoData = await getProjectGitLabData(repo) + data.push([repoWithHost, repoData]) + } else { + // eslint-disable-next-line no-await-in-loop + const repoData = await getProjectGitHubData(repo) + data.push([repoWithHost, repoData]) + } } catch (err) { - console.error(`Could not load repository "${repo}". Please make sure it still exists.`) + console.error(`Could not load repository "${repo}" from ${repoHost}. Please make sure it still exists.`) } } return fromPairs(data) @@ -57,15 +74,14 @@ async function getAllProjectGitHubData (repos) { async function getAllProjectData (projects) { const timestamp = Date.now() - const twitterScreenNames = map(projects, 'twitter').filter(val => val) - const twitterFollowers = twitterScreenNames.length && - await getTwitterFollowers(twitterScreenNames) - const gitHubRepos = map(projects, 'repo').filter(val => val) - const gitHubReposData = await getAllProjectGitHubData(gitHubRepos) - const data = projects.reduce((obj, { key, repo, twitter }) => { + const twitterScreenNames = compact(map(projects, 'twitter')) + const twitterFollowers = twitterScreenNames.length && await getTwitterFollowers(twitterScreenNames) + const repos = compact(map(projects, val => val.repo ? [val.repohost || 'github', val.repo].join(':') : undefined)) + const reposData = await getAllProjectRepoData(repos) + const data = projects.reduce((obj, { key, repo, repohost, twitter }) => { const twitterData = twitter ? { followers: twitterFollowers[twitter] } : {} - const gitHubData = repo ? { ...(gitHubReposData[repo]) } : {} - return { ...obj, [key]: [{ timestamp, ...twitterData, ...gitHubData }] } + const repoData = repo ? { ...(reposData[[repohost || 'github', repo].join(':')]) } : {} + return { ...obj, [key]: [{ timestamp, ...twitterData, ...repoData }] } }, {}) return { timestamp, data } } diff --git a/site.yaml b/site.yaml index 682dda3e1..79b9390f0 100644 --- a/site.yaml +++ b/site.yaml @@ -16,7 +16,7 @@ navLinks: - { url: https://jamstack.org, text: About JAMstack } - { url: https://headlesscms.org, text: Need a Static CMS? } sorts: - - { field: "stars", label: "GitHub stars", reverse: true } + - { field: "stars", label: "Repo stars", reverse: true } - { field: "followers", label: "Twitter followers", reverse: true } - { field: "title", label: "Title" } filters: diff --git a/src/Home/DeployButton.js b/src/Home/DeployButton.js index 7337d805e..a2bb5551a 100644 --- a/src/Home/DeployButton.js +++ b/src/Home/DeployButton.js @@ -5,7 +5,7 @@ import netlifyLogo from 'Images/netlify-logo.svg' const DeployButton = styled(({ repo, className }) => ( Deploy to Netlify diff --git a/src/Home/Project.js b/src/Home/Project.js index 057a8e642..e7022ae7c 100644 --- a/src/Home/Project.js +++ b/src/Home/Project.js @@ -40,7 +40,7 @@ const Project = ({ } - label="GitHub stars" + label="Repo stars" value={stars} change={stars - starsPrevious} indicateColor @@ -49,7 +49,7 @@ const Project = ({ } - label="GitHub open issues" + label="Open issues" value={issues} change={issues - issuesPrevious} dataAgeInDays={dataAgeInDays} @@ -57,7 +57,7 @@ const Project = ({ } - label="GitHub forks" + label="Repo forks" value={forks} change={forks - forksPrevious} indicateColor diff --git a/src/Project/Project.js b/src/Project/Project.js index 3d318b43c..af3369903 100644 --- a/src/Project/Project.js +++ b/src/Project/Project.js @@ -97,8 +97,8 @@ class Project extends React.Component { } {repo && - - https://github.com/{repo} ({stars}) + + {repo} ({stars}) } diff --git a/static.config.js b/static.config.js index 9ca52ca16..0d5f6ca43 100644 --- a/static.config.js +++ b/static.config.js @@ -31,20 +31,31 @@ function processMarkdown (markdown, key) { function mapProjectFrontMatter ({ title, repo, + repohost, homepage, description, startertemplaterepo, content, twitter, }) { + const repoHost = repohost || 'github' + let starterTemplateRepo = startertemplaterepo + if (starterTemplateRepo && !starterTemplateRepo.includes(':')) { + if (repoHost === 'github') { + starterTemplateRepo = `https://github.com/${starterTemplateRepo}` + } else if (repoHost === 'gitlab') { + starterTemplateRepo = `https://gitlab.com/${starterTemplateRepo}` + } + } return { title, slug: toSlug(title), repo, + repoHost, homepage, twitter, description, - starterTemplateRepo: startertemplaterepo, + starterTemplateRepo, content, } } @@ -56,7 +67,7 @@ function extractRelevantProjectData (data) { const oldestTimestamp = dateFns.min(...timestamps).getTime() const dataAgeInDays = dateFns.differenceInDays(Date.now(), oldestTimestamp) const { - followers, forks, stars, issues, + followers, forks, stars, issues, repo } = find(project, { timestamp: newestTimestamp }) || {} const { forks: forksPrevious, @@ -64,7 +75,7 @@ function extractRelevantProjectData (data) { issues: issuesPrevious, followers: followersPrevious, } = find(project, { timestamp: oldestTimestamp }) || {} - return { + const relevantData = { followers, forks, stars, @@ -75,6 +86,8 @@ function extractRelevantProjectData (data) { followersPrevious, dataAgeInDays, } + if (repo) relevantData.repo = repo + return relevantData }) }