Toggle waitUntil option for robots and sitemap scans

GSA · May 8, 2024 · 7cd89fb · 7cd89fb
1 parent f3ec2cb
commit 7cd89fb
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/libs/core-scanner/src/pages/robots-txt.ts b/libs/core-scanner/src/pages/robots-txt.ts
@@ -14,7 +14,9 @@ export const createRobotsTxtScanner = (logger: Logger, input: CoreInputDto) => {
     // go to the robots page from the target url
     const robotsUrl = new URL(url);
     robotsUrl.pathname = 'robots.txt';
-    const robotsResponse = await robotsPage.goto(robotsUrl.toString());
+    const robotsResponse = await robotsPage.goto(robotsUrl.toString(), {
+      waitUntil: 'networkidle0',
+    });
     // extract the html page source
     const robotsText = await robotsResponse.text();
 

diff --git a/libs/core-scanner/src/pages/sitemap-xml.ts b/libs/core-scanner/src/pages/sitemap-xml.ts
@@ -17,7 +17,9 @@ export const createSitemapXmlScanner = (
     const sitemapUrl = new URL(url);
     sitemapUrl.pathname = 'sitemap.xml';
     logger.info('Going to sitemap.xml...');
-    const sitemapResponse = await sitemapPage.goto(sitemapUrl.toString());
+    const sitemapResponse = await sitemapPage.goto(sitemapUrl.toString(), {
+      waitUntil: 'networkidle0',
+    });
     logger.info('Got sitemap.xml!');
     // extract the html page source
     const sitemapText = await sitemapResponse.text();