From 2346f6c6776ec5cc5d88c53046aa0b496145aea7 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Thu, 28 Nov 2024 13:16:37 +0100 Subject: [PATCH] Add tools --- tools/create-index.mjs | 122 +++++++++++++++++++++++++++++++++ tools/find-broken-packages.mjs | 51 ++++++++++++++ tools/package-lock.json | 34 +++++++++ tools/package.json | 14 ++++ 4 files changed, 221 insertions(+) create mode 100644 tools/create-index.mjs create mode 100644 tools/find-broken-packages.mjs create mode 100644 tools/package-lock.json create mode 100644 tools/package.json diff --git a/tools/create-index.mjs b/tools/create-index.mjs new file mode 100644 index 0000000..61c2165 --- /dev/null +++ b/tools/create-index.mjs @@ -0,0 +1,122 @@ +import fs from 'fs'; +import path from 'path'; +import { execSync } from 'child_process'; +import yaml from 'js-yaml'; + +const getGitHubUrl = (rootPath) => { + try { + // Get the GitHub repository URL + const remoteUrl = execSync('git config --get remote.origin.url', { cwd: rootPath, encoding: 'utf-8' }).trim(); + + // Convert SSH format to HTTPS format + const httpsUrl = remoteUrl.replace(/^git@github\.com:/, 'https://github.com/').replace(/\.git$/, ''); + + return httpsUrl; + } catch (error) { + console.error(`Error getting GitHub repository URL: ${error.message}`); + return null; + } +}; + +const getCurrentBranch = (rootPath) => { + try { + // Get the current branch name + const branchName = execSync('git branch --show-current', { cwd: rootPath, encoding: 'utf-8' }).trim(); + return branchName; + } catch (error) { + console.error(`Error getting current branch: ${error.message}`); + return null; + } +}; + +const getRepositoryRoot = (rootPath) => { + try { + // Get the root folder of the repository + const repositoryRoot = execSync('git rev-parse --show-toplevel', { cwd: rootPath, encoding: 'utf-8' }).trim(); + return repositoryRoot; + } catch (error) { + console.error(`Error getting repository root: ${error.message}`); + return null; + } +}; + +const searchPackages = (directory, outputFilename, indexUrl) => { + const result = { packages: [] }; + + const repositoryRoot = getRepositoryRoot(directory); + const gitHubUrl = getGitHubUrl(repositoryRoot); + const currentBranch = getCurrentBranch(repositoryRoot); + + if (!repositoryRoot || !gitHubUrl || !currentBranch) { + return; + } + + const search = (dir, rootPath) => { + const files = fs.readdirSync(dir); + + for (const file of files) { + const filePath = path.join(dir, file); + const isDirectory = fs.statSync(filePath).isDirectory(); + + if (isDirectory) { + search(filePath, rootPath); + } else { + const isPackageJson = file === 'package.json'; + const isManifestPy = file === 'manifest.py'; + + if (isPackageJson || isManifestPy) { + const packageInfo = { + name: path.basename(dir), + docs: constructGitHubUrl(gitHubUrl, currentBranch, repositoryRoot, dir), + index: indexUrl, + }; + + if (isManifestPy) { + try { + const content = fs.readFileSync(filePath, 'utf8'); + const descriptionMatch = /description="(.*?)"/.exec(content); + + if (descriptionMatch && descriptionMatch[1]) { + packageInfo.description = descriptionMatch[1]; + } + } catch (error) { + console.error(`Error reading ${file}: ${error.message}`); + } + } + + result.packages.push(packageInfo); + } + } + } + }; + + const constructGitHubUrl = (baseUrl, branch, repositoryRoot, dirPath) => { + const relativePath = path.relative(repositoryRoot, dirPath); + const normalizedPath = relativePath.replace(/\\/g, '/'); // Normalize path separators for Windows + + return `${baseUrl}/tree/${branch}/${normalizedPath}`; + }; + + search(directory, repositoryRoot); + + try { + const yamlData = yaml.dump(result); + fs.writeFileSync(outputFilename, `---\n${yamlData}`); + console.log(`YAML file saved to ${outputFilename}`); + } catch (error) { + console.error(`Error writing YAML file: ${error.message}`); + } +}; + +// Check if command line arguments are provided +if (process.argv.length < 5) { + // Note: Official MicroPython lib index is: https://micropython.org/pi/v2 + // Example usage: node create-index.mjs ../micropython-lib/micropython micropython-lib.yml https://micropython.org/pi/v2 + console.error('Usage: node create-index.mjs '); +} else { + const directory = process.argv[2]; + const outputFilename = process.argv[3]; + const indexUrl = process.argv[4]; + + searchPackages(directory, outputFilename, indexUrl); +} diff --git a/tools/find-broken-packages.mjs b/tools/find-broken-packages.mjs new file mode 100644 index 0000000..9a40d24 --- /dev/null +++ b/tools/find-broken-packages.mjs @@ -0,0 +1,51 @@ +// This script helps to find packages in the Arduino package index that are missing the package.json file. +// Without the package.json file, the package cannot be installed using mpremote or the Arduino Package Installer. + +import yaml from 'js-yaml'; + +function convertToRawURL(url, suffix = null) { + url = url.replace('https://github.com/', ''); + const parts = url.split('/'); + const owner = parts[0]; + const repoName = parts[1]; + const branch = 'HEAD'; + let path = parts[2] ? "/" + parts[2] : ''; + if (suffix) { + path += "/" + suffix; + } + return `https://raw.githubusercontent.com/${owner}/${repoName}/${branch}${path}`; +} + +const indexURL = "https://raw.githubusercontent.com/arduino/package-index-py/refs/heads/main/package-list.yaml"; +const data = await fetch(indexURL); +const doc = yaml.load(await data.text()); + +// Filter packages that have the package_descriptor field +// which overrides the package.json file +doc.packages = doc.packages.filter(pkg => pkg.package_descriptor == undefined); +doc.packages.map(pkg => { + // Convert the URLs to https://raw.githubusercontent.com/... format + pkg.rawURL = convertToRawURL(pkg.url, 'package.json'); + return pkg; +}); + +let incompletePackages = []; + +// Check the existence of the package.json file for each package +for (const aPackage of doc.packages) { + const response = await fetch(aPackage.rawURL); + if (!response.ok) { + incompletePackages.push(aPackage.url); + console.log(`āŒ Package file ${aPackage.rawURL} not found.`); + } else { + console.log(`āœ… Package file ${aPackage.rawURL} found.`); + } +} + +console.log("\nšŸ‘€ Packages with missing package.json:"); +console.log(incompletePackages); +const message = ` +Consider making pull requests to add a package.json file to these repositories +or add the package_descriptor field to the package in the package-list.yaml file +`; +console.log(message); \ No newline at end of file diff --git a/tools/package-lock.json b/tools/package-lock.json new file mode 100644 index 0000000..348337a --- /dev/null +++ b/tools/package-lock.json @@ -0,0 +1,34 @@ +{ + "name": "tools", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tools", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "js-yaml": "^4.1.0" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + } + } +} diff --git a/tools/package.json b/tools/package.json new file mode 100644 index 0000000..5496f24 --- /dev/null +++ b/tools/package.json @@ -0,0 +1,14 @@ +{ + "name": "tools", + "version": "1.0.0", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "description": "", + "dependencies": { + "js-yaml": "^4.1.0" + } +}