Fix arg escaping for external python prompts on Windows (promptfoo#179)

* readPrompts test * Use double quotes to fix windows issue * remove moduleNameMapper * Update prompts.test.ts * resolve path
SamuraiBarbi · Sep 23, 2023 · 3b442b7 · 3b442b7
1 parent d465f2c
commit 3b442b7
Show file tree

Hide file tree

Showing 7 changed files with 30 additions and 10 deletions.
diff --git a/jest.config.ts b/jest.config.ts
@@ -10,9 +10,11 @@ const config: Config = {
       useESM: true,
     },
   },
+  /*
   moduleNameMapper: {
     '(.+)\\.js': '$1',
   },
+  */
   extensionsToTreatAsEsm: ['.ts'],
   setupFiles: ['<rootDir>/.jest/setEnvVars.js'],
   testPathIgnorePatterns: ['<rootDir>/examples', '<rootDir>/node_modules', '<rootDir>/dist'],

diff --git a/src/prompts.ts b/src/prompts.ts
@@ -139,8 +139,8 @@ export function readPrompts(
         const fileContent = fs.readFileSync(promptPath, 'utf-8');
         const promptFunction = (context: { vars: Record<string, string | object> }) => {
           const { execSync } = require('child_process');
-          const contextString = JSON.stringify(context).replace(/'/g, "\\'");
-          const output = execSync(`python ${promptPath} '${contextString}'`);
+          const contextString = JSON.stringify(context).replace(/"/g, '\\"').replace(/\n/g, '\\n');
+          const output = execSync(`python "${promptPath}" "${contextString}"`);
           return output.toString();
         };
         promptContents.push({

diff --git a/test/cache.test.ts b/test/cache.test.ts
@@ -1,4 +1,4 @@
-import { fetchWithCache, disableCache, enableCache } from '../src/cache.js';
+import { fetchWithCache, disableCache, enableCache } from '../src/cache';
 import fetch, { Response } from 'node-fetch';
 
 jest.mock('node-fetch');

diff --git a/test/evaluator.test.ts b/test/evaluator.test.ts
@@ -1,10 +1,10 @@
-import { evaluate, renderPrompt } from '../src/evaluator.js';
+import { evaluate, renderPrompt } from '../src/evaluator';
 
-import type { ApiProvider, TestSuite, Prompt } from '../src/types.js';
+import type { ApiProvider, TestSuite, Prompt } from '../src/types';
 
 jest.mock('node-fetch', () => jest.fn());
 
-jest.mock('../src/esm.js');
+jest.mock('../src/esm');
 
 const mockApiProvider: ApiProvider = {
   id: jest.fn().mockReturnValue('test-provider'),

diff --git a/test/prompts.test.ts b/test/prompts.test.ts
@@ -119,4 +119,22 @@ describe('prompts', () => {
     expect(fs.readFileSync).toHaveBeenCalledTimes(1);
     expect(result).toEqual([toPrompt(JSON.stringify(data[0])), toPrompt(JSON.stringify(data[1]))]);
   });
+
+  test('readPrompts with .py file', () => {
+    const code = `print('dummy prompt')`;
+    (fs.readFileSync as jest.Mock).mockReturnValue(code);
+    const result = readPrompts('prompt.py');
+    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+    expect(result[0].raw).toEqual(code);
+    expect(result[0].display).toEqual(code);
+    expect(result[0].function).toBeDefined();
+  });
+
+  test('readPrompts with .js file', () => {
+    jest.doMock(path.resolve('prompt.js'), () => {
+      return jest.fn(() => console.log('dummy prompt'));
+    }, { virtual: true });
+    const result = readPrompts('prompt.js');
+    expect(result[0].function).toBeDefined();
+  });
 });
diff --git a/test/providers.test.ts b/test/providers.test.ts
@@ -4,8 +4,8 @@ import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from '../src/p
 import { AnthropicCompletionProvider } from '../src/providers/anthropic';
 import { LlamaProvider } from '../src/providers/llama';
 
-import { disableCache, enableCache } from '../src/cache.js';
-import { loadApiProvider, loadApiProviders } from '../src/providers.js';
+import { disableCache, enableCache } from '../src/cache';
+import { loadApiProvider, loadApiProviders } from '../src/providers';
 import {
   AzureOpenAiChatCompletionProvider,
   AzureOpenAiCompletionProvider,
@@ -17,7 +17,7 @@ import type { ProviderOptionsMap, ProviderFunction } from '../src/types';
 
 jest.mock('node-fetch', () => jest.fn());
 
-jest.mock('../src/esm.js');
+jest.mock('../src/esm');
 
 describe('providers', () => {
   afterEach(() => {

diff --git a/test/util.test.ts b/test/util.test.ts
@@ -15,7 +15,7 @@ jest.mock('fs', () => ({
   mkdirSync: jest.fn(),
 }));
 
-jest.mock('../src/esm.js');
+jest.mock('../src/esm');
 
 beforeEach(() => {
   jest.clearAllMocks();