fix(core): use try/catch in .evaluate() to avoid errors

packages/gpt-scraper-core/src/crawler.ts

@@ -33,7 +33,7 @@
         addFormats(validator);
         validator.compile(schema);
         return schema;
     } catch (e: any) {
         log.error(`Schema is not valid: ${e.message}`, { error: e });
         await Actor.fail('Schema is not valid. Go to Actor run log, '
             + 'where you can find error details or disable "Use JSON schema to format answer" option.');
@@ -92,7 +92,16 @@
                 }
             },
         ],
-
+        postNavigationHooks: [
+            async ({ page }) => {
+                // see https://github.com/apify/crawlee/issues/2314
+                // will solve client-side redirects through meta tags
+                await page.waitForSelector('body', {
+                    state: 'attached',
+                    timeout: 60_000,
+                });
+            },
+        ],
         async requestHandler({ request, page, enqueueLinks, closeCookieModals }) {
             const { depth = 0 } = request.userData;
             const state = await crawler.useState<State>(DEFAULT_STATE);
@@ -217,7 +226,7 @@
                 answer = answerResult.answer;
                 jsonAnswer = answerResult.jsonAnswer;
                 model.updateApiCallUsage(answerResult.usage);
             } catch (error: any) {
                 if (error instanceof OpenaiAPIErrorToExitActor) {
                     throw await Actor.fail(error.message);
                 }

packages/gpt-scraper-core/src/models/openai.ts

@@ -3,8 +3,7 @@
 import { ChatOpenAI } from 'langchain/chat_models/openai';
 import { OpenAI } from 'langchain/llms/openai';
 import { LLMResult } from 'langchain/schema';
-import { REPETITIVE_PROMPT_ERROR_MESSAGE } from '../errors.js';
-import { NonRetryableOpenaiAPIError, OpenaiAPIError, OpenaiAPIErrorToExitActor, RateLimitedError } from '../errors.js';
+import { NonRetryableOpenaiAPIError, OpenaiAPIError, OpenaiAPIErrorToExitActor, RateLimitedError, REPETITIVE_PROMPT_ERROR_MESSAGE } from '../errors.js';
 import { tryToParseJsonFromString } from '../processors.js';
 import { ProcessInstructionsOptions } from '../types/model.js';
 import { OpenAIModelSettings } from '../types/models.js';
@@ -19,7 +18,7 @@
  *
  * see OpenAI errors documentation: https://platform.openai.com/docs/guides/error-codes/api-errors
  */
 const wrapInOpenaiError = (error: any): OpenaiAPIError => {
     const errorMessage = error.error?.message || error.code || error.message;
 
     // The error structure is completely different for insufficient quota errors. We need to handle it separately.
@@ -79,7 +78,7 @@
         for (let retry = 1; retry < MAX_GPT_RETRIES + 1; retry++) {
             try {
                 return await this.processInstructions(options);
             } catch (error: any) {
                 const wrappedError = wrapInOpenaiError(error);
 
                 if (wrappedError instanceof NonRetryableOpenaiAPIError) throw wrappedError;
@@ -121,7 +120,7 @@
      * Parses the function arguments from the OpenAI LLM's output.
      */
     private parseFunctionArguments = (functionOutput: LLMResult): string | null => {
         const generations = functionOutput.generations as any;
         const firstFunction = generations?.[0]?.[0]?.message;
         const functionArguments = firstFunction?.lc_kwargs?.additional_kwargs?.function_call?.arguments;
 

packages/gpt-scraper-core/src/processors.ts

@@ -15,7 +15,13 @@ export const shrinkHtml = async (html: string, page: Page, removeElementsCssSele
             if (removeSelector) {
                 const elements = doc.querySelectorAll(removeSelector);
                 for (const element of elements) {
-                    element.remove();
+                    // there have been some cases when the page's own scripts cause errors and running this line
+                    // causes them to reemerge, so what in try/cartch
+                    try {
+                        element.remove();
+                    } catch (err) {
+                        /* ignore */
+                    }
                 }
             }
             return doc.documentElement.outerHTML;
@@ -34,7 +40,7 @@ export const htmlToMarkdown = (html: string) => {
     return htmlToMarkdownProcessor.turndown(html);
 };
 
-const chunkText = (text:string, maxLength: number) => {
+const chunkText = (text: string, maxLength: number) => {
     const numChunks = Math.ceil(text.length / maxLength);
     const chunks = new Array(numChunks);
 

shared/CHANGELOG.md

@@ -1,5 +1,9 @@
 This changelog tracks updates to both GTP Scraper and Extended GPT Scraper actors.
 
+# 2023-01-31
+*Fixes*
+- Eliminated the bug, when on some sites that contain erronous javascript the scraper would fail
+
 ### 2023-01-26
 *Fixes*
 - Fixed "max pages per run" not working correctly on specific websites.
@@ -31,4 +35,4 @@ This changelog tracks updates to both GTP Scraper and Extended GPT Scraper actor
 
 *Changes*
 - Use LangChain to connect to GPT models. This means some error messages are different.
-- The default model `temperature` is now set to `0` instead of `1`. This should improve the reliability of scraping. While this is technically a breaking change, it should mostly behave as an improvement so we don't consider need to release a separate version.
+- The default model `temperature` is now set to `0` instead of `1`. This should improve the reliability of scraping. While this is technically a breaking change, it should mostly behave as an improvement so we don't consider need to release a separate version.