fix: ENOBUF bug fix attempt (#426)

fix: ENOBUF bug Co-authored-by: Parth Suthar <[email protected]>
DevCycleHQ · Aug 27, 2024 · f4540b8 · f4540b8
1 parent 1083392
commit f4540b8
Show file tree

Hide file tree

Showing 5 changed files with 89 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -108,7 +108,7 @@ $ npm install -g @devcycle/cli
 $ dvc COMMAND
 running command...
 $ dvc (--version)
-@devcycle/cli/5.16.0 linux-x64 node-v20.10.0
+@devcycle/cli/5.16.0 darwin-arm64 node-v22.2.0
 $ dvc --help [COMMAND]
 USAGE
   $ dvc COMMAND

diff --git a/docs/usages.md b/docs/usages.md
@@ -14,9 +14,10 @@ USAGE
   $ dvc usages [--config-path <value>] [--auth-path <value>] [--repo-config-path <value>] [--client-id
     <value>] [--client-secret <value>] [--project <value>] [--no-api] [--headless] [--include <value>] [--exclude
     <value>] [--client-name <value>] [--match-pattern <value>] [--var-alias <value>] [--format console|json]
-    [--show-regex] [--only-unused]
+    [--show-regex] [--only-unused] [-o <value>]
 
 FLAGS
+  -o, --output=<value>        Output file path for JSON format. If not specified, output will be written to stdout.
   --client-name=<value>...    Name(s) of the DevCycle client variable to match on. Accepts multiple values.
   --exclude=<value>...        Files to exclude when scanning for usages. By default all files are included. Accepts
                               multiple glob patterns.

diff --git a/oclif.manifest.json b/oclif.manifest.json
@@ -4977,6 +4977,13 @@
           "type": "boolean",
           "description": "Show usages of variables that are not defined in your DevCycle config.",
           "allowNo": false
+        },
+        "output": {
+          "name": "output",
+          "type": "option",
+          "char": "o",
+          "description": "Output file path for JSON format. If not specified, output will be written to stdout.",
+          "multiple": false
         }
       },
       "args": {}

diff --git a/src/commands/base.ts b/src/commands/base.ts
@@ -275,7 +275,6 @@ export default abstract class Base extends Command {
         await this.authorizeApi()
         setDVCReferrer(this.id, this.config.version, this.caller)
     }
-
     isAuthExpired() {
         const tokenExpiry = getTokenExpiry(this.authToken)
         return !tokenExpiry || tokenExpiry < Date.now()
@@ -324,7 +323,6 @@ export default abstract class Base extends Command {
 
         return findProjectByKey(this.projectKey)
     }
-
     hasToken(): boolean {
         return this.authToken !== ''
     }
@@ -373,7 +371,6 @@ export default abstract class Base extends Command {
         const parse = schema.parse(input, { errorMap })
         return parse
     }
-
     public async populateParametersWithFlags(
         prompts: Prompt[],
         flags: Record<string, unknown>,

diff --git a/src/commands/usages/index.ts b/src/commands/usages/index.ts
@@ -11,7 +11,8 @@ import { VariableMatch, VariableUsageMatch } from '../../utils/parsers/types'
 import { fetchAllVariables } from '../../api/variables'
 import { Variable } from '../../api/schemas'
 import { FileFilters } from '../../utils/FileFilters'
-
+import { Writable } from 'stream'
+import { createWriteStream } from 'fs'
 export default class Usages extends Base {
     static hidden = false
 
@@ -50,6 +51,11 @@ export default class Usages extends Base {
             description:
                 'Show usages of variables that are not defined in your DevCycle config.',
         }),
+        output: Flags.string({
+            char: 'o',
+            description:
+                'Output file path for JSON format. If not specified, output will be written to stdout.',
+        }),
     }
 
     useMarkdown = false
@@ -61,25 +67,55 @@ export default class Usages extends Base {
         this.useMarkdown = flags.format === 'markdown'
 
         const processFile = (filepath: string): File => {
-            let lines: LineItem[] = []
+            const chunkSize = 1024 * 1024 // 1MB chunks
+            const fd = fs.openSync(filepath, 'r')
+            const buffer = Buffer.alloc(chunkSize)
+            const lines: LineItem[] = []
+            let lineNumber = 1
+            let remainingData = ''
+
             try {
-                lines = fs
-                    .readFileSync(filepath, 'utf8')
-                    .split('\n')
-                    .map((content, idx) => ({ content, ln: idx + 1 }))
+                let bytesRead
+                while (
+                    (bytesRead = fs.readSync(fd, buffer, 0, chunkSize, null)) >
+                    0
+                ) {
+                    const chunk =
+                        remainingData + buffer.toString('utf8', 0, bytesRead)
+                    const chunkLines = chunk.split('\n')
+                    remainingData = chunkLines.pop() || ''
+
+                    chunkLines.forEach((content) => {
+                        lines.push({ content, ln: lineNumber++ })
+                    })
+                }
+
+                if (remainingData) {
+                    lines.push({ content: remainingData, ln: lineNumber })
+                }
             } catch (err) {
                 this.warn(`Error parsing file ${filepath}`)
                 this.debug(err)
+            } finally {
+                fs.closeSync(fd)
             }
+
             return {
                 name: filepath,
                 lines,
             }
         }
 
         const fileFilters = new FileFilters(flags, codeInsightsConfig)
-        const files = fileFilters.getFiles().map(processFile)
+        const filePaths = fileFilters.getFiles()
+        const batchSize = 100 // Process 100 files at a time
+        const files: File[] = []
 
+        for (let i = 0; i < filePaths.length; i += batchSize) {
+            const batch = filePaths.slice(i, i + batchSize)
+            const batchFiles = batch.map(processFile)
+            files.push(...batchFiles)
+        }
         if (!files.length) {
             this.warn('No files found to process.')
             return
@@ -110,11 +146,46 @@ export default class Usages extends Base {
 
         if (flags['format'] === 'json') {
             const matchesByVariableJSON = this.formatMatchesToJSON(usages)
-            this.log(JSON.stringify(matchesByVariableJSON, null, 2))
+            await this.writeJSONOutput(matchesByVariableJSON, flags.output)
         } else {
             this.formatConsoleOutput(usages)
         }
     }
+    private async writeJSONOutput(
+        data: JSONMatch[],
+        outputPath: string | undefined,
+    ): Promise<void> {
+        const outputStream: Writable = outputPath
+            ? createWriteStream(outputPath)
+            : process.stdout
+
+        return new Promise((resolve, reject) => {
+            const jsonStream = JSON.stringify(data, null, 2)
+            const writeStream = new Writable({
+                write(chunk, encoding, callback) {
+                    if (!outputStream.write(chunk, encoding)) {
+                        outputStream.once('drain', callback)
+                    } else {
+                        process.nextTick(callback)
+                    }
+                },
+            })
+
+            writeStream.on('finish', () => {
+                if (outputPath) {
+                    outputStream.end(() => resolve())
+                } else {
+                    resolve()
+                }
+            })
+
+            writeStream.on('error', reject)
+            outputStream.on('error', reject)
+
+            writeStream.write(jsonStream)
+            writeStream.end()
+        })
+    }
 
     private getMatchesByVariable(
         matchesBySdk: Record<string, VariableUsageMatch[]>,