From 05be846c6fe55a9157827d1b7fdc978548e56d9d Mon Sep 17 00:00:00 2001 From: jorsmatthys <67681021+jorsmatthys@users.noreply.github.com> Date: Tue, 12 Apr 2022 13:58:35 +0200 Subject: [PATCH] Allow unattended execution (#62) * tested unattended version * change user var * run prettier:write --- README.md | 64 ++++++++++++++++++++--- src/utils/clients/index.ts | 3 +- src/utils/clients/rest.ts | 1 - src/utils/commands.ts | 101 +++++++++++++++++++++++++++++-------- src/utils/commitFile.ts | 59 +++++++++++++++++----- src/utils/globals.ts | 15 +++++- src/utils/worker.ts | 5 +- 7 files changed, 202 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index abe7a8c..7ae94e6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ There are two main actions this tool does: Goes and collects repositories that will have Code Scanning(CodeQL)/Secret Scanning/Dependabot Alerts/Dependabot Security Updates enabled. There are three main ways these repositories are collected. - Collect the repositories where the primary language matches a specific value. For example, if you provide JavaScript, all repositories will be collected where the primary language is, Javascript. -- Collect the repositories to which a user has administrative access, or a GitHub App has access. +- Collect the repositories to which a user (PAT) has administrative access, or a GitHub App has access. - Manually create `repos.json`. If you select option 1, the script will return all repositories in the language you specify (which you have access to). The repositories collected from this script are then stored within a `repos.json` file. If you specify option 2, the script will return all repositories you are an administrator over. The third option is to define the `repos.json` manually. We don't recommend this, but it's possible. If you want to go down this path, first run one of the above options for collecting repository information automatically, look at the structure, and build your fine of the laid out format. @@ -49,11 +49,11 @@ If you pick Dependabot Security Updates: - [Node v16](https://nodejs.org/en/download/) or higher installed. - [Yarn](https://yarnpkg.com/)\* - [TypeScript](https://www.typescriptlang.org/download) -- [Git](https://git-scm.com/downloads) installed on the user's machine running this tool. -- Someone who has at least admin access over the repositories they want to enable Code Scanning on. Or, access to GitHub App credentails which has access to the repositories you want to enable Code Scanning on +- [Git](https://git-scm.com/downloads) installed on the (user's) machine running this tool. +- A Personal Access Token (PAT) that has at least admin access over the repositories they want to enable Code Scanning on or GitHub App credentials which have access to the repositories you want to enable Code Scanning on. - Some basic software development skills, e.g., can navigate their way around a terminal or command prompt. -* You can use `npm` but for the sake of this `README.md`; we are going to standardise the commands on yarn. These are easily replacable though with `npm` commands. +* You can use `npm` but for the sake of this `README.md`; we are going to standardise the commands on yarn. These are easily replaceable though with `npm` commands. ## Set up Instructions @@ -175,17 +175,67 @@ There are some key considerations which you will need to put into place if you a } ``` -The reason you need this within your `.devcontainer/devcontainer.json` file is the `GITHUB_TOKEN` tied to the Codepsace will need to access other repositories within your organisation which this script may interact with. You will need to create a new Codespace **after** you have added the above and pushed it to your repository. +The reason you need this within your `.devcontainer/devcontainer.json` file is the `GITHUB_TOKEN` tied to the Codespace will need to access other repositories within your organisation which this script may interact with. You will need to create a new Codespace **after** you have added the above and pushed it to your repository. You do not need to do the above if you are not running it from a Codespace. +## Running as a (scheduled) GitHub workflow + +Since this tool uses a PAT or GitHub App Authentication wherever authentication is required, it can be run unattended. You can see in the example +below how you could run the tool in a scheduled GitHub workflow. Instead of using the `.env` +file you can configure all the variables from the `.env.sample` directly as environment variables. This will allow you to +(easily) make use of GitHub action secrets for the PAT or GitHub App credentials. + +```yaml +on: + schedule: + - cron: "5 16 * * 1" + +env: + APP_ID: ${{ secrets.GHAS_ENABLEMENT_APP_ID }} + APP_CLIENT_ID: ${{ secrets.GHAS_ENABLEMENT_APP_CLIENT_ID }} + APP_CLIENT_SECRET: ${{ secrets.GHAS_ENABLEMENT_APP_CLIENT_SECRET }} + APP_PRIVATE_KEY: ${{ secrets.GHAS_ENABLEMENT_APP_PRIVATE_KEY }} + ENABLE_ON: "codescanning,secretscanning,dependabot,dependabotupdates" + DEBUG: "ghas:*" + CREATE_ISSUE: "false" + GHES: "false" + # Organization specific variables + APP_INSTALLATION_ID: "12345678" + GITHUB_ORG: "my-target-org" + +jobs: + enable-security-javascript: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + repository: NickLiffen/ghas-enablement + - name: Get dependencies and configure + run: | + yarn + git config --global user.name "ghas-enablement" + git config --global user.email "ghas.enablement@example.com" + - name: Enable security on organization (javascript) + run: | + npm run getRepos + npm run start + env: + LANGUAGE_TO_CHECK: "javascript" +``` + +You can duplicate the last step for the other languages commonly used within your enterprise/organisation. +If you didn't configure the tool as a GitHub App, you can remove all the `APP_*` and set `GITHUB_API_TOKEN` instead. +Above we rely on the sample codeql file for javascript included in this repository. Alternatively you could add this workflow to a repository +containing your customized codeql files and use those to overwrite the samples. + ## Found an Issue? Create an issue within the repository and make it to `@nickliffen`. Key things to mention within your issue: -- Windows or Mac +- Windows, Linux, Codespaces or Mac - What version of NodeJS you are running. -- Print any logs that appear on the terminal or command prompt +- Add any logs that appeared when you ran into the issue. ## Want to Contribute? diff --git a/src/utils/clients/index.ts b/src/utils/clients/index.ts index 6ff3db9..d5dc1db 100644 --- a/src/utils/clients/index.ts +++ b/src/utils/clients/index.ts @@ -1,4 +1,5 @@ import { graphQLClient } from "./graphql"; import { restClient } from "./rest"; +import { auth } from "./auth"; -export { graphQLClient, restClient }; +export { graphQLClient, restClient, auth }; diff --git a/src/utils/clients/rest.ts b/src/utils/clients/rest.ts index 5273fdd..9e6b083 100644 --- a/src/utils/clients/rest.ts +++ b/src/utils/clients/rest.ts @@ -13,7 +13,6 @@ let MyOctokit = Octokit.plugin(paginateRest, retry, throttling); export const restClient = async (testPlugin?: any): Promise => { try { const auth = (await generateAuth()) as string; - if (testPlugin) { MyOctokit = Octokit.plugin(testPlugin, retry, throttling); } diff --git a/src/utils/commands.ts b/src/utils/commands.ts index 66a05aa..8a28c44 100644 --- a/src/utils/commands.ts +++ b/src/utils/commands.ts @@ -1,19 +1,13 @@ import { commands } from "../../types/common"; -import { - destDir, - user, - winUser, - windestDir, - tempDIR, - baseURL, -} from "./globals"; +import { destDir, user, windestDir, tempDIR } from "./globals"; export const codespacesCommands = ( owner: string, repo: string, branch: string, - fileName: string + fileName: string, + baseURL: string ): commands => { const commands = [ { @@ -85,7 +79,8 @@ export const macCommands = ( owner: string, repo: string, branch: string, - fileName: string + fileName: string, + baseURL: string ): commands => { const commands = [ { @@ -154,18 +149,19 @@ export const windowsCommands = ( owner: string, repo: string, branch: string, - fileName: string + fileName: string, + baseURL: string ): commands => { const commands = [ { command: "mkdir", args: ["-p", `${tempDIR}`], - cwd: `/Users/${winUser}/${windestDir}`, + cwd: `/Users/${user}/${windestDir}`, }, { command: "git", args: ["clone", `${baseURL}/${owner}/${repo}.git`], - cwd: `/Users/${winUser}/${windestDir}/${tempDIR}`, + cwd: `/Users/${user}/${windestDir}/${tempDIR}`, }, { command: "git", @@ -175,45 +171,108 @@ export const windowsCommands = ( { command: "mkdir", args: ["-p", ".github/workflows"], - cwd: `/Users/${winUser}/${windestDir}/${tempDIR}/${repo}`, + cwd: `/Users/${user}/${windestDir}/${tempDIR}/${repo}`, }, { command: "cp", args: [ `./bin/workflows/${fileName}`, - `c:\\Users\\${winUser}\\${windestDir}\\${tempDIR}/${repo}\\.github\\workflows\\`, + `c:\\Users\\${user}\\${windestDir}\\${tempDIR}/${repo}\\.github\\workflows\\`, ], cwd: process.cwd(), }, { command: "rm", args: ["-rf", '"./-p/"'], - cwd: `/Users/${winUser}/${windestDir}/${tempDIR}/${repo}`, + cwd: `/Users/${user}/${windestDir}/${tempDIR}/${repo}`, }, { command: "git", args: ["add", `.github/workflows/${fileName}`], - cwd: `/Users/${winUser}/${windestDir}/${tempDIR}/${repo}`, + cwd: `/Users/${user}/${windestDir}/${tempDIR}/${repo}`, }, { command: "git", args: ["commit", "-m", '"Commit CodeQL File"'], - cwd: `/Users/${winUser}/${windestDir}/${tempDIR}/${repo}`, + cwd: `/Users/${user}/${windestDir}/${tempDIR}/${repo}`, }, { command: "git", args: ["push", "origin", `${branch}`], - cwd: `/Users/${winUser}/${windestDir}/${tempDIR}/${repo}`, + cwd: `/Users/${user}/${windestDir}/${tempDIR}/${repo}`, }, { command: "rm", args: ["-rf", `"./${tempDIR}/"`], - cwd: `/Users/${winUser}/${windestDir}/`, + cwd: `/Users/${user}/${windestDir}/`, }, { command: "rm", args: ["-rf", '"./-p/"'], - cwd: `/Users/${winUser}/${windestDir}`, + cwd: `/Users/${user}/${windestDir}`, + }, + ] as commands; + return commands; +}; + +export const wslLinuxCommands = ( + owner: string, + repo: string, + branch: string, + fileName: string, + baseURL: string +): commands => { + const commands = [ + { + command: "mkdir", + args: ["-p", `${tempDIR}`], + cwd: `/home/${user}`, + }, + { + command: "git", + args: ["clone", `${baseURL}/${owner}/${repo}.git`], + cwd: `/home/${user}/${tempDIR}`, + }, + { + command: "git", + args: ["checkout", "-b", `${branch}`], + cwd: `/home/${user}/${tempDIR}/${repo}`, + }, + { + command: "mkdir", + args: ["-p", ".github/workflows"], + cwd: `/home/${user}/${tempDIR}/${repo}`, + }, + { + command: "cp", + args: [ + `./bin/workflows/${fileName}`, + `/home/${user}/${tempDIR}/${repo}/.github/workflows/`, + ], + cwd: process.cwd(), + }, + { + command: "mv", + args: [ + `./.github/workflows/${fileName}`, + `./.github/workflows/codeql-analysis.yml`, + ], + cwd: `/home/${user}/${tempDIR}/${repo}`, + }, + { + command: "git", + args: ["add", ".github/workflows/codeql-analysis.yml"], + cwd: `/home/${user}/${tempDIR}/${repo}`, + }, + { + command: "git", + args: ["commit", "-m", '"Commit CodeQL File"'], + cwd: `/home/${user}/${tempDIR}/${repo}`, + }, + { + command: "git", + args: ["push", "--set-upstream", "origin", `${branch}`], + cwd: `/home/${user}/${tempDIR}/${repo}`, }, ] as commands; return commands; diff --git a/src/utils/commitFile.ts b/src/utils/commitFile.ts index 549428c..44b42ec 100644 --- a/src/utils/commitFile.ts +++ b/src/utils/commitFile.ts @@ -5,11 +5,21 @@ import delay from "delay"; import { existsSync } from "fs"; -import os from "os"; - -import { inform, error } from "./globals"; - -import { macCommands, windowsCommands, codespacesCommands } from "./commands"; +import { + inform, + error, + isWindows, + isLinux, + baseURL, + platform, +} from "./globals"; + +import { + macCommands, + windowsCommands, + codespacesCommands, + wslLinuxCommands, +} from "./commands"; import { execFile as ImportedExec } from "child_process"; @@ -17,9 +27,6 @@ import { response, commands } from "../../types/common"; const execFile = util.promisify(ImportedExec); -const platform = os.platform(); - -const isWindows = platform === "win32"; if (platform !== "win32" && platform !== "darwin" && platform !== "linux") { error("You can only use either windows or mac machine!"); throw new Error( @@ -30,12 +37,17 @@ if (platform !== "win32" && platform !== "darwin" && platform !== "linux") { export const commitFileMac = async ( owner: string, repo: string, - refs: string + refs: string, + authToken: string ): Promise => { let gitCommands: commands; let index: number; let isCodespace = false as boolean; + const authBaseURL = baseURL!.replace( + "https://", + `https://x-access-token:${authToken}@` + ) as string; const regExpExecArray = /[^/]*$/.exec(refs); const branch = regExpExecArray ? regExpExecArray[0] : ""; @@ -53,12 +65,33 @@ export const commitFileMac = async ( : "codeql-analysis-standard.yml"; try { + /* Codespaces is also a linux environment, so this check has to happen first */ gitCommands = isWindows === true - ? (windowsCommands(owner, repo, branch, fileName) as commands) - : isWindows === false && isCodespace === false - ? (macCommands(owner, repo, branch, fileName) as commands) - : (codespacesCommands(owner, repo, branch, fileName) as commands); + ? (windowsCommands( + owner, + repo, + branch, + fileName, + authBaseURL + ) as commands) + : isCodespace === true + ? (codespacesCommands( + owner, + repo, + branch, + fileName, + authBaseURL + ) as commands) + : isLinux === true + ? (wslLinuxCommands( + owner, + repo, + branch, + fileName, + authBaseURL + ) as commands) + : (macCommands(owner, repo, branch, fileName, authBaseURL) as commands); inform(gitCommands); } catch (err) { error(err); diff --git a/src/utils/globals.ts b/src/utils/globals.ts index 444531c..6b52cea 100644 --- a/src/utils/globals.ts +++ b/src/utils/globals.ts @@ -1,12 +1,18 @@ import randomstring from "randomstring"; import Debug from "debug"; +import os from "os"; + +export const platform = os.platform(); const rs = randomstring.generate({ length: 5, charset: "alphabetic", }) as string; +export const isWindows = platform === "win32"; +export const isLinux = platform === "linux"; + export const baseRestApiURL = process.env.GHES == "true" ? `${process.env.GHES_SERVER_BASE_URL}/api/v3` @@ -19,6 +25,7 @@ export const baseURL = process.env.GHES == "true" ? process.env.GHES_SERVER_BASE_URL : "https://github.com"; + export const ref = `refs/heads/ghas-${rs}` as string; export const message = "Created CodeQL Analysis File"; export const title = "GitHub Advanced Security - Code Scanning" as string; @@ -28,7 +35,11 @@ export const inform = Debug("ghas:inform") as Debug.Debugger; export const error = Debug("ghas:error") as Debug.Debugger; export const destDir = "Desktop" as string; export const windestDir = "Documents" as string; -export const user = process.cwd().split("/")[2] as string; -export const winUser = process.cwd().split("\\")[2] as string; +export const user = + isLinux === true + ? process.env.USER + : isWindows === true + ? (process.cwd().split("\\")[2] as string) + : (process.cwd().split("/")[2] as string); export const reposFileLocation = "./bin/repos.json" as string; export const orgsFileLocation = "./bin/organizations.json" as string; diff --git a/src/utils/worker.ts b/src/utils/worker.ts index 8d0ffdb..5707e7b 100644 --- a/src/utils/worker.ts +++ b/src/utils/worker.ts @@ -12,6 +12,8 @@ import { enableGHAS } from "./enableGHAS.js"; import { enableDependabotAlerts } from "./enableDependabotAlerts"; import { enableDependabotFixes } from "./enableDependabotUpdates"; import { enableIssueCreation } from "./enableIssueCreation"; +import { auth as generateAuth } from "./clients"; + import repos from "../../bin/repos.json"; import { Octokit } from "./octokitTypes"; @@ -75,7 +77,8 @@ export const worker = async (): Promise => { client ); const ref = await createBranch(defaultBranchSHA, owner, repo, client); - await commitFileMac(owner, repo, ref); + const authToken = (await generateAuth()) as string; + await commitFileMac(owner, repo, ref, authToken); const pullRequestURL = await createPullRequest( defaultBranch, ref,