feat: add 'skip GPT processing' Globs

actors/extended-gpt-scraper/.actor/input_schema.json

@@ -98,6 +98,14 @@
             "default": 10,
             "unit": "pages"
         },
+        "skipGptGlobs": {
+            "title": "Skip GPT processing for Globs",
+            "type": "array",
+            "description": "This setting allows you to specify certain page URLs to skip GPT instructions for. Pages matching these glob patterns will only be crawled for links, excluding them from GPT processing. Useful for intermediary pages used for navigation or undesired content.",
+            "editor": "globs",
+            "default": [],
+            "prefill": []
+        },
         "useStructureOutput": {
             "sectionCaption": "Formatted output",
             "sectionDescription": "By default, the scraper outputs text answers for each page. If you want to get data in a structured format, you can define a JSON schema. The scraper uses [function](https://platform.openai.com/docs/api-reference/chat/create#chat/create-functions), which is called for each page. The function receives the page content and returns the answer in the defined JSON format.",

actors/gpt-scraper/src/main.ts

@@ -29,14 +29,16 @@ if (process.env.ACTOR_MAX_PAID_DATASET_ITEMS) {
 await updateDeprecatedInput(input);
 
 if (process.env.OPENAI_API_KEY) {
-    const crawler = await createCrawler({
-        input: {
-            ...input,
-            maxPagesPerCrawl: maxRequestsPerCrawl,
-            model: DEFAULT_PEY_PER_RESULT_OPENAI_MODEL,
-            openaiApiKey: process.env.OPENAI_API_KEY,
-        },
-    });
+    /** Input for the PPR Actor. We explicitly override some of these values, so that users don't change them. */
+    const adjustedPayPerResultInput = {
+        ...input,
+        maxPagesPerCrawl: maxRequestsPerCrawl,
+        skipGptGlobs: [],
+        model: DEFAULT_PEY_PER_RESULT_OPENAI_MODEL,
+        openaiApiKey: process.env.OPENAI_API_KEY,
+    };
+
+    const crawler = await createCrawler({ input: adjustedPayPerResultInput });
 
     // We explicitly remove it so we are sure the key is only passed through params to remove double source of truth
     delete process.env.OPENAI_API_KEY;

packages/gpt-scraper-core/package.json

@@ -27,6 +27,7 @@
         "gpt-3-encoder": "^1.1.4",
         "joplin-turndown-plugin-gfm": "^1.0.12",
         "langchain": "^0.0.209",
+        "minimatch": "^9.0.3",
         "openai": "^3.3.0",
         "playwright": "*",
         "turndown": "^7.1.2"

packages/gpt-scraper-core/src/crawler.ts

@@ -7,10 +7,10 @@
 import { getModelByName } from './models/models.js';
 import { getNumberOfTextTokens, htmlToMarkdown, maybeShortsTextByTokenLength, shrinkHtml } from './processors.js';
 import { Input, PAGE_FORMAT } from './types/input.js';
-import { parseInput } from './input.js';
-import { NonRetryableOpenaiAPIError } from './errors.js';
 import { parseInput, validateInput, validateInputCssSelectors } from './input.js';
+import { NonRetryableOpenaiAPIError } from './errors.js';
 import { OpenAIModelSettings } from './types/models.js';
+import { doesUrlMatchGlobs } from './utils.js';
 
 interface State {
     pageOutputted: number;
@@ -30,7 +30,7 @@
         addFormats(validator);
         validator.compile(schema);
         return schema;
     } catch (e: any) {
         log.error(`Schema is not valid: ${e.message}`, { error: e });
         await Actor.fail('Schema is not valid. Go to Actor run log, '
                     + 'where you can find error details or disable "Use JSON schema to format answer" option.');
@@ -118,6 +118,11 @@
                 );
             }
 
+            const skipGptProcessing = input.skipGptGlobs && doesUrlMatchGlobs(url, input.skipGptGlobs);
+            if (skipGptProcessing) {
+                log.info(`Skipping page from GPT processing because it matched 'skipGptGlobs', crawling only.`, { url });
+            }
+
             // A function to be evaluated by Playwright within the browser context.
             let originContentHtml;
             if (input.targetSelector) {
@@ -183,7 +188,7 @@
                 answer = answerResult.answer;
                 jsonAnswer = answerResult.jsonAnswer;
                 model.updateApiCallUsage(answerResult.usage);
             } catch (error: any) {
                 if (error instanceof NonRetryableOpenaiAPIError) {
                     throw await Actor.fail(error.message);
                 }

packages/gpt-scraper-core/src/types/index.ts

@@ -1,2 +1,2 @@
-export { HTML_TAGS_TO_IGNORE, type Input, PAGE_FORMAT } from './input.js';
+export { type Input, PAGE_FORMAT } from './input.js';
 export { type OpenAIModelSettings } from './models.js';

packages/gpt-scraper-core/src/types/input.ts

@@ -21,6 +21,7 @@ export interface Input extends OpenAIModelSettings {
     useStructureOutput?: boolean;
     pageFormatInRequest?: PAGE_FORMAT;
     saveSnapshots?: boolean;
+    skipGptGlobs?: GlobInput[];
     initialCookies?: Cookie[];
     removeElementsCssSelector?: string;
 }

packages/gpt-scraper-core/src/utils.ts

@@ -0,0 +1,12 @@
+import { GlobInput } from 'crawlee';
+import { minimatch } from 'minimatch';
+
+export const doesUrlMatchGlobs = (url: string, globs: GlobInput[]): boolean => {
+    return globs.some((glob) => doesUrlMatchGlob(url, glob));
+};
+
+const doesUrlMatchGlob = (url: string, glob: GlobInput): boolean => {
+    const globString = typeof glob === 'string' ? glob : glob.glob;
+
+    return minimatch(url, globString, { nocase: true });
+};