diff --git a/entities/core-result.entity.ts b/entities/core-result.entity.ts index 272162c1..e3007a71 100644 --- a/entities/core-result.entity.ts +++ b/entities/core-result.entity.ts @@ -420,212 +420,127 @@ export class CoreResult { @Expose({ name: 'cumulative_layout_shift' }) cumulativeLayoutShift?: number; - /** - * #852 - * The following fields are experimental and were added in March 2024 for - * prototyping purposes. They are not yet used in the application and may be - * removed in the future. - */ - @Column({ nullable: true }) - @Expose({ name: 'meta_keywords_content' }) - @Exclude() - metaKeywordsContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'meta_robots_content' }) - @Exclude() - metaRobotsContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'meta_article_section_content' }) - @Exclude() - metaArticleSectionContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'meta_article_tag_content' }) - @Exclude() - metaArticleTagContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'og_image_final_url' }) - @Exclude() - ogImageFinalUrl?: string; - @Column({ nullable: true }) - @Expose({ name: 'dcterms_keywords_content' }) + @Expose({ name: 'keywords' }) @Exclude() - dctermsKeywordsContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'dc_subject_content' }) - @Exclude() - dcSubjectContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'dcterms_subject_content' }) - @Exclude() - dctermsSubjectContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'dcterms_audience_content' }) - @Exclude() - dctermsAudienceContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'dc_type_content' }) - @Exclude() - dcTypeContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'dcterms_type_content' }) - @Exclude() - dctermsTypeContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'dc_date_content' }) - @Exclude() - dcDateContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'dc_date_created_content' }) - @Exclude() - dcDateCreatedContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'dcterms_created_content' }) - @Exclude() - dctermsCreatedContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'og_locale_content' }) - @Exclude() - ogLocaleContent?: string; + metaKeywordsContent?: string; @Column({ nullable: true }) - @Expose({ name: 'og_site_name_content' }) + @Expose({ name: 'og_image' }) @Exclude() - ogSiteNameContent?: string; + ogImageContent?: string; @Column({ nullable: true }) - @Expose({ name: 'og_type_content' }) + @Expose({ name: 'og_type' }) @Exclude() ogTypeContent?: string; @Column({ nullable: true }) - @Expose({ name: 'og_url_content' }) + @Expose({ name: 'og_url' }) @Exclude() ogUrlContent?: string; @Column({ nullable: true }) - @Expose({ name: 'og_image_alt_content' }) - @Exclude() - ogImageAltContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'revised_content' }) - @Exclude() - revisedContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'last_modified_content' }) - @Exclude() - lastModifiedContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'language_content' }) - @Exclude() - languageContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'date_content' }) - @Exclude() - dateContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'subject_content' }) - @Exclude() - subjectContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'owner_content' }) - @Exclude() - ownerContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'pagename_content' }) - @Exclude() - pagenameContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'dc_title_content' }) - @Exclude() - dcTitleContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'og_site_name' }) - @Exclude() - ogSiteName?: string; - - @Column({ nullable: true }) - @Expose({ name: 'item_type_content' }) - @Exclude() - itemTypeContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'item_scope_content' }) - @Exclude() - itemScopeContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'item_prop_content' }) - @Exclude() - itemPropContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'vocab_content' }) - @Exclude() - vocabContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'type_of_content' }) - @Exclude() - typeOfContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'propertyContent' }) - @Exclude() - propertyContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'context_content' }) - @Exclude() - contextContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'type_content' }) - @Exclude() - typeContent?: string; - - @Column({ nullable: true }) - @Expose({ name: 'html_lang_content' }) + @Expose({ name: 'language' }) @Exclude() htmlLangContent?: string; @Column({ nullable: true }) - @Expose({ name: 'hreflang_content' }) + @Expose({ name: 'language_link' }) @Exclude() hrefLangContent?: string; - @Column({ nullable: true }) - @Expose({ name: 'me_content' }) - @Exclude() - meContent?: string; - /** - * End experimental fields. - */ - static getColumnNames(): string[] { // return class-transformer version of column names return Object.keys(classToPlain(new CoreResult())); } + + static snapshotColumnOrder = [ + 'target_url', + 'target_url_domain', + 'target_url_top_level_domain', + 'target_url_redirects', + 'final_url', + 'final_url_domain', + 'final_url_top_level_domain', + 'final_url_website', + 'final_url_live', + 'final_url_status_code', + 'final_url_media_type', + 'final_url_same_domain', + 'final_url_same_website', + 'target_url_agency_owner', + 'target_url_bureau_owner', + 'target_url_branch', + 'target_url_404_test', + 'source_list', + 'public', + 'scan_date', + 'primary_scan_status', + 'accessibility_scan_status', + 'dns_scan_status', + 'not_found_scan_status', + 'performance_scan_status', + 'robots_txt_scan_status', + 'sitemap_xml_scan_status', + 'ipv6', + 'hostname', + 'cms', + 'login_provider', + 'login', + 'site_search', + 'search_dot_gov', + 'dap', + 'dap_parameters', + 'third_party_service_domains', + 'third_party_service_count', + 'cookie_domains', + 'viewport_meta_tag', + 'cumulative_layout_shift', + 'largest_contentful_paint', + 'required_links_url', + 'required_links_text', + 'title', + 'description', + 'keywords', + 'og_title', + 'og_description', + 'og_article_published', + 'og_article_modified', + 'og_image', + 'og_type', + 'og_url', + 'canonical_link', + 'language', + 'language_link', + 'main_element_present', + 'robots_txt_detected', + 'robots_txt_target_url_redirects', + 'robots_txt_final_url', + 'robots_txt_final_url_live', + 'robots_txt_final_url_status_code', + 'robots_txt_final_url_media_type', + 'robots_txt_final_url_filesize', + 'robots_txt_crawl_delay', + 'robots_txt_sitemap_locations', + 'sitemap_xml_detected', + 'sitemap_xml_target_url_redirects', + 'sitemap_xml_final_url', + 'sitemap_xml_final_url_live', + 'sitemap_xml_final_url_status_code', + 'sitemap_xml_final_url_media_type', + 'sitemap_xml_final_url_filesize', + 'sitemap_xml_count', + 'sitemap_xml_pdf_count', + 'uswds_favicon', + 'uswds_favicon_in_css', + 'uswds_publicsans_font', + 'uswds_inpage_css', + 'uswds_usa_classes', + 'uswds_string', + 'uswds_string_in_css', + 'uswds_semantic_version', + 'uswds_version', + 'uswds_count', + ]; } diff --git a/entities/scan-data.entity.ts b/entities/scan-data.entity.ts index fb53f4c9..e8eb7462 100644 --- a/entities/scan-data.entity.ts +++ b/entities/scan-data.entity.ts @@ -30,47 +30,12 @@ export type SeoScan = { pageTitle: string; metaDescriptionContent: string; hreflangCodes: string; - // #852 Begin March 2024 experimental fields metaKeywordsContent: string; - metaRobotsContent: string; - metaArticleSectionContent: string; - metaArticleTagContent: string; - ogImageFinalUrl: string; - dctermsKeywordsContent: string; - dcSubjectContent: string; - dctermsSubjectContent: string; - dctermsAudienceContent: string; - dcTypeContent: string; - dctermsTypeContent: string; - dcDateContent: string; - dcDateCreatedContent: string; - dctermsCreatedContent: string; - ogLocaleContent: string; - ogSiteNameContent: string; + ogImageContent: string; ogTypeContent: string; ogUrlContent: string; - ogImageAltContent: string; - revisedContent: string; - lastModifiedContent: string; - languageContent: string; - dateContent: string; - subjectContent: string; - ownerContent: string; - pagenameContent: string; - dcTitleContent: string; - ogSiteName: string; - itemTypeContent: string; - itemScopeContent: string; - itemPropContent: string; - vocabContent: string; - typeOfContent: string; - propertyContent: string; - contextContent: string; - typeContent: string; htmlLangContent: string; hrefLangContent: string; - meContent: string; - // End March 2024 experimental fields }; export type ThirdPartyScan = { diff --git a/libs/core-scanner/src/pages/primary.spec.ts b/libs/core-scanner/src/pages/primary.spec.ts index ade91f0e..6ca2e937 100644 --- a/libs/core-scanner/src/pages/primary.spec.ts +++ b/libs/core-scanner/src/pages/primary.spec.ts @@ -50,48 +50,12 @@ describe('primary scanner', () => { metaDescriptionContent: '18F builds effective, user-centric digital services focused on the interaction between government and the people and businesses it serves.', hreflangCodes: '', - // #852 Begin March 2024 experimental fields - contextContent: null, - dateContent: null, - dcDateContent: null, - dcDateCreatedContent: null, - dcSubjectContent: null, - dcTitleContent: null, - dcTypeContent: null, - dctermsAudienceContent: null, - dctermsCreatedContent: null, - dctermsKeywordsContent: null, - dctermsSubjectContent: null, - dctermsTypeContent: null, hrefLangContent: '', htmlLangContent: 'en-US', - itemPropContent: null, - itemScopeContent: null, - itemTypeContent: null, - languageContent: null, - lastModifiedContent: null, - meContent: null, - metaArticleSectionContent: null, - metaArticleTagContent: null, metaKeywordsContent: null, - metaRobotsContent: null, - ogImageAltContent: null, - ogImageFinalUrl: - 'https://18f.gsa.gov/assets/img/logos/18F-Logo-M.png', - ogLocaleContent: null, - ogSiteName: null, - ogSiteNameContent: null, + ogImageContent: 'https://18f.gsa.gov/assets/img/logos/18F-Logo-M.png', ogTypeContent: null, ogUrlContent: null, - ownerContent: null, - pagenameContent: null, - propertyContent: 'og:title', - revisedContent: null, - subjectContent: null, - typeContent: 'image/png', - typeOfContent: null, - vocabContent: null, - // End March 2024 experimental fields }, thirdPartyScan: { thirdPartyServiceDomains: diff --git a/libs/core-scanner/src/scans/seo.spec.ts b/libs/core-scanner/src/scans/seo.spec.ts index 0a76e203..b87b4e86 100644 --- a/libs/core-scanner/src/scans/seo.spec.ts +++ b/libs/core-scanner/src/scans/seo.spec.ts @@ -26,50 +26,13 @@ describe('seo scan', () => { metaDescriptionContent: '', hreflangCodes: '', - // #852 Begin March 2024 experimental fields - contextContent: null, - dateContent: null, - dcDateContent: null, - dcDateCreatedContent: null, - dcSubjectContent: null, - dcTitleContent: null, - dcTypeContent: null, - dctermsAudienceContent: null, - dctermsCreatedContent: null, - dctermsKeywordsContent: null, - dctermsSubjectContent: null, - dctermsTypeContent: null, hrefLangContent: '', htmlLangContent: 'en-US', - itemPropContent: null, - itemScopeContent: null, - itemTypeContent: null, - languageContent: null, - lastModifiedContent: null, - meContent: null, - metaArticleSectionContent: null, - metaArticleTagContent: null, metaKeywordsContent: null, - metaRobotsContent: - 'index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1', - ogImageAltContent: null, - ogImageFinalUrl: + ogImageContent: 'https://www.arc.gov/wp-content/uploads/2020/08/48982308566_5ce274f0ab_o-washington-scaled.jpg', - ogLocaleContent: 'en_US', - ogSiteName: null, - ogSiteNameContent: 'Appalachian Regional Commission', ogTypeContent: 'website', ogUrlContent: 'https://www.arc.gov/', - ownerContent: null, - pagenameContent: null, - propertyContent: 'og:locale', - revisedContent: null, - subjectContent: null, - typeContent: - 'text/css,application/rss+xml,application/json,application/rsd+xml,application/wlwmanifest+xml,application/json+oembed,text/xml+oembed', - typeOfContent: null, - vocabContent: null, - // End March 2024 experimental fields }); }, 'arc_gov_dump.mht'); }); diff --git a/libs/core-scanner/src/scans/seo.ts b/libs/core-scanner/src/scans/seo.ts index b3ca602b..cc4df348 100644 --- a/libs/core-scanner/src/scans/seo.ts +++ b/libs/core-scanner/src/scans/seo.ts @@ -31,88 +31,12 @@ export const buildSeoResult = async ( (await findHrefLangCodes(page)) ?? (await extractHrefLangValues(response)) ?? null; - - /** - * #852 - * The following fields are experimental and were added in March 2024 for - * prototyping purposes. They are not yet used in the application and may be - * removed in the future. - */ const metaKeywordsContent = await findMetaContent(page, 'name', 'keywords'); - const metaRobotsContent = await findMetaContent(page, 'name', 'robots'); - const metaArticleSectionContent = await findMetaContent( - page, - 'name', - 'article:section', - ); - const metaArticleTagContent = await findMetaContent( - page, - 'name', - 'article:tag', - ); - const ogImageFinalUrl = await findOpenGraphTag(page, 'og:image'); - const dctermsKeywordsContent = await findMetaContent( - page, - 'name', - 'dcterms.keywords', - ); - const dcSubjectContent = await findMetaContent(page, 'name', 'dc.subject'); - const dctermsSubjectContent = await findMetaContent( - page, - 'name', - 'dcterms.subject', - ); - const dctermsAudienceContent = await findMetaContent( - page, - 'name', - 'dcterms.audience', - ); - const dcTypeContent = await findMetaContent(page, 'name', 'dc.type'); - const dctermsTypeContent = await findMetaContent( - page, - 'name', - 'dcterms.type', - ); - const dcDateContent = await findMetaContent(page, 'name', 'dc.date'); - const dcDateCreatedContent = await findMetaContent( - page, - 'name', - 'dc.date.created', - ); - const dctermsCreatedContent = await findMetaContent( - page, - 'name', - 'dcterms.created', - ); - const ogLocaleContent = await findOpenGraphTag(page, 'og:locale'); - const ogSiteNameContent = await findOpenGraphTag(page, 'og:site_name'); + const ogImageContent = await findOpenGraphTag(page, 'og:image'); const ogTypeContent = await findOpenGraphTag(page, 'og:type'); const ogUrlContent = await findOpenGraphTag(page, 'og:url'); - const ogImageAltContent = await findOpenGraphTag(page, 'og:image:alt'); - const revisedContent = await findMetaContent(page, 'name', 'revised'); - const lastModifiedContent = await findMetaContent( - page, - 'http-equiv', - 'last-modified', - ); - const languageContent = await findMetaContent(page, 'name', 'language'); - const dateContent = await findMetaContent(page, 'name', 'date'); - const subjectContent = await findMetaContent(page, 'name', 'subject'); - const ownerContent = await findMetaContent(page, 'name', 'owner'); - const pagenameContent = await findMetaContent(page, 'name', 'pagename'); - const dcTitleContent = await findMetaContent(page, 'name', 'DC.title'); - const ogSiteName = await findMetaContent(page, 'name', 'og:site_name'); - const itemTypeContent = await findAttributeContent(page, 'itemtype'); - const itemScopeContent = await findAttributeContent(page, 'itemscope'); - const itemPropContent = await findAttributeContent(page, 'itemprop'); - const vocabContent = await findAttributeContent(page, 'vocab'); - const typeOfContent = await findAttributeContent(page, 'typeof'); - const propertyContent = await findAttributeContent(page, 'property'); - const contextContent = await findAttributeContent(page, 'context'); - const typeContent = await findTypeContent(page); const htmlLangContent = await findAttributeContent(page, 'lang'); const hrefLangContent = await getHreflangContent(page, 'hreflang'); - const meContent = await getMeContent(page); return { ogTitleFinalUrl, @@ -124,47 +48,12 @@ export const buildSeoResult = async ( pageTitle, metaDescriptionContent, hreflangCodes, - // Beging March 2024 experimental fields metaKeywordsContent, - metaRobotsContent, - metaArticleSectionContent, - metaArticleTagContent, - ogImageFinalUrl, - dctermsKeywordsContent, - dcSubjectContent, - dctermsSubjectContent, - dctermsAudienceContent, - dcTypeContent, - dctermsTypeContent, - dcDateContent, - dcDateCreatedContent, - dctermsCreatedContent, - ogLocaleContent, - ogSiteNameContent, + ogImageContent, ogTypeContent, ogUrlContent, - ogImageAltContent, - revisedContent, - lastModifiedContent, - languageContent, - dateContent, - subjectContent, - ownerContent, - pagenameContent, - dcTitleContent, - ogSiteName, - itemTypeContent, - itemScopeContent, - itemPropContent, - vocabContent, - typeOfContent, - propertyContent, - contextContent, - typeContent, htmlLangContent, hrefLangContent, - meContent, - // End March 2024 experimental fields }; }; @@ -358,35 +247,3 @@ const getHreflangContent = async ( return content; }; - -const findTypeContent = async (page: Page): Promise => { - const content = await page.evaluate(() => { - const typeElements = document.querySelectorAll('link[type]'); - - const typeValues = Array.from( - new Set( - Array.from(typeElements).map((el) => { - return el.getAttribute('type').trim().toLowerCase(); - }), - ), - ); - - return typeValues.join(','); - }); - - return content; -}; - -const getMeContent = async (page: Page): Promise => { - const content = await page.evaluate(() => { - const meLink = document.querySelector('link[rel="me"]'); - - if (meLink && meLink.hasAttribute('value')) { - return meLink.getAttribute('value').trim(); - } - - return null; - }); - - return content; -}; diff --git a/libs/database/src/core-results/core-result.service.spec.ts b/libs/database/src/core-results/core-result.service.spec.ts index c9038ad2..282be45e 100644 --- a/libs/database/src/core-results/core-result.service.spec.ts +++ b/libs/database/src/core-results/core-result.service.spec.ts @@ -114,54 +114,12 @@ describe('CoreResultService', () => { pageTitle: null, metaDescriptionContent: null, hreflangCodes: null, - /** - * #852 - * The following fields are experimental and were added in March 2024 for - * prototyping purposes. They are not yet used in the application and may be - * removed in the future. - */ metaKeywordsContent: null, - metaRobotsContent: null, - metaArticleSectionContent: null, - metaArticleTagContent: null, - ogImageFinalUrl: null, - dctermsKeywordsContent: null, - dcSubjectContent: null, - dctermsSubjectContent: null, - dctermsAudienceContent: null, - dcTypeContent: null, - dctermsTypeContent: null, - dcDateContent: null, - dcDateCreatedContent: null, - dctermsCreatedContent: null, - ogLocaleContent: null, - ogSiteNameContent: null, + ogImageContent: null, ogTypeContent: null, ogUrlContent: null, - ogImageAltContent: null, - revisedContent: null, - lastModifiedContent: null, - languageContent: null, - dateContent: null, - subjectContent: null, - ownerContent: null, - pagenameContent: null, - dcTitleContent: null, - ogSiteName: null, - itemTypeContent: null, - itemScopeContent: null, - itemPropContent: null, - vocabContent: null, - typeOfContent: null, - propertyContent: null, - contextContent: null, - typeContent: null, htmlLangContent: null, hrefLangContent: null, - meContent: null, - /** - * End experimental fields. - */ }, thirdPartyScan: { thirdPartyServiceDomains: null, diff --git a/libs/database/src/core-results/core-result.service.ts b/libs/database/src/core-results/core-result.service.ts index 51474209..3ffc5dce 100644 --- a/libs/database/src/core-results/core-result.service.ts +++ b/libs/database/src/core-results/core-result.service.ts @@ -86,54 +86,12 @@ export class CoreResultService { coreResult.pageTitle = result.seoScan.pageTitle; coreResult.metaDescriptionContent = result.seoScan.metaDescriptionContent; coreResult.hreflangCodes = result.seoScan.hreflangCodes; - /** - * The following fields are experimental and were added in March 2024 for - * prototyping purposes. They are not yet used in the application and may be - * removed in the future. - */ coreResult.metaKeywordsContent = result.seoScan.metaKeywordsContent; - coreResult.metaRobotsContent = result.seoScan.metaRobotsContent; - coreResult.metaArticleSectionContent = - result.seoScan.metaArticleSectionContent; - coreResult.metaArticleTagContent = result.seoScan.metaArticleTagContent; - coreResult.ogImageFinalUrl = result.seoScan.ogImageFinalUrl; - coreResult.dctermsKeywordsContent = result.seoScan.dctermsKeywordsContent; - coreResult.dcSubjectContent = result.seoScan.dcSubjectContent; - coreResult.dctermsSubjectContent = result.seoScan.dctermsSubjectContent; - coreResult.dctermsAudienceContent = result.seoScan.dctermsAudienceContent; - coreResult.dcTypeContent = result.seoScan.dcTypeContent; - coreResult.dctermsTypeContent = result.seoScan.dctermsTypeContent; - coreResult.dcDateContent = result.seoScan.dcDateContent; - coreResult.dcDateCreatedContent = result.seoScan.dcDateCreatedContent; - coreResult.dctermsCreatedContent = result.seoScan.dctermsCreatedContent; - coreResult.ogLocaleContent = result.seoScan.ogLocaleContent; - coreResult.ogSiteNameContent = result.seoScan.ogSiteNameContent; + coreResult.ogImageContent = result.seoScan.ogImageContent; coreResult.ogTypeContent = result.seoScan.ogTypeContent; coreResult.ogUrlContent = result.seoScan.ogUrlContent; - coreResult.ogImageAltContent = result.seoScan.ogImageAltContent; - coreResult.revisedContent = result.seoScan.revisedContent; - coreResult.lastModifiedContent = result.seoScan.lastModifiedContent; - coreResult.languageContent = result.seoScan.languageContent; - coreResult.dateContent = result.seoScan.dateContent; - coreResult.subjectContent = result.seoScan.subjectContent; - coreResult.ownerContent = result.seoScan.ownerContent; - coreResult.pagenameContent = result.seoScan.pagenameContent; - coreResult.dcTitleContent = result.seoScan.dcTitleContent; - coreResult.ogSiteName = result.seoScan.ogSiteName; - coreResult.itemTypeContent = result.seoScan.itemTypeContent; - coreResult.itemScopeContent = result.seoScan.itemScopeContent; - coreResult.itemPropContent = result.seoScan.itemPropContent; - coreResult.vocabContent = result.seoScan.vocabContent; - coreResult.typeOfContent = result.seoScan.typeOfContent; - coreResult.propertyContent = result.seoScan.propertyContent; - coreResult.contextContent = result.seoScan.contextContent; - coreResult.typeContent = result.seoScan.typeContent; coreResult.htmlLangContent = result.seoScan.htmlLangContent; coreResult.hrefLangContent = result.seoScan.hrefLangContent; - coreResult.meContent = result.seoScan.meContent; - /** - * End experimental fields. - */ // Third-party scan coreResult.thirdPartyServiceCount = @@ -229,54 +187,12 @@ export class CoreResultService { coreResult.searchDetected = null; coreResult.searchgov = null; coreResult.viewportMetaTag = null; - /** - * #852 - * The following fields are experimental and were added in March 2024 for - * prototyping purposes. They are not yet used in the application and may be - * removed in the future. - */ coreResult.metaKeywordsContent = null; - coreResult.metaRobotsContent = null; - coreResult.metaArticleSectionContent = null; - coreResult.metaArticleTagContent = null; - coreResult.ogImageFinalUrl = null; - coreResult.dctermsKeywordsContent = null; - coreResult.dcSubjectContent = null; - coreResult.dctermsSubjectContent = null; - coreResult.dctermsAudienceContent = null; - coreResult.dcTypeContent = null; - coreResult.dctermsTypeContent = null; - coreResult.dcDateContent = null; - coreResult.dcDateCreatedContent = null; - coreResult.dctermsCreatedContent = null; - coreResult.ogLocaleContent = null; - coreResult.ogSiteNameContent = null; + coreResult.ogImageContent = null; coreResult.ogTypeContent = null; coreResult.ogUrlContent = null; - coreResult.ogImageAltContent = null; - coreResult.revisedContent = null; - coreResult.lastModifiedContent = null; - coreResult.languageContent = null; - coreResult.dateContent = null; - coreResult.subjectContent = null; - coreResult.ownerContent = null; - coreResult.pagenameContent = null; - coreResult.dcTitleContent = null; - coreResult.ogSiteName = null; - coreResult.itemTypeContent = null; - coreResult.itemScopeContent = null; - coreResult.itemPropContent = null; - coreResult.vocabContent = null; - coreResult.typeOfContent = null; - coreResult.propertyContent = null; - coreResult.contextContent = null; - coreResult.typeContent = null; coreResult.htmlLangContent = null; coreResult.hrefLangContent = null; - coreResult.meContent = null; - /** - * End experimental fields. - */ } } diff --git a/libs/snapshot/src/serializers/csv-serializer.spec.ts b/libs/snapshot/src/serializers/csv-serializer.spec.ts index e09d37cb..7734f739 100644 --- a/libs/snapshot/src/serializers/csv-serializer.spec.ts +++ b/libs/snapshot/src/serializers/csv-serializer.spec.ts @@ -3,105 +3,18 @@ import { Website } from 'entities/website.entity'; import { CsvSerializer } from './csv-serializer'; describe('CsvSerializer', () => { - let liveColumnOrder: string[]; - - beforeEach(() => { - liveColumnOrder = [ - 'target_url', - 'target_url_domain', - 'target_url_top_level_domain', - 'target_url_redirects', - 'final_url', - 'final_url_domain', - 'final_url_top_level_domain', - 'final_url_website', - 'final_url_live', - 'final_url_status_code', - 'final_url_media_type', - 'final_url_same_domain', - 'final_url_same_website', - 'target_url_agency_owner', - 'target_url_bureau_owner', - 'target_url_branch', - 'target_url_404_test', - 'source_list', - 'public', - 'scan_date', - 'primary_scan_status', - 'accessibility_scan_status', - 'dns_scan_status', - 'not_found_scan_status', - 'performance_scan_status', - 'robots_txt_scan_status', - 'sitemap_xml_scan_status', - 'ipv6', - 'hostname', - 'cms', - 'login_provider', - 'login', - 'site_search', - 'search_dot_gov', - 'dap', - 'dap_parameters', - 'third_party_service_domains', - 'third_party_service_count', - 'cookie_domains', - 'viewport_meta_tag', - 'cumulative_layout_shift', - 'largest_contentful_paint', - 'required_links_url', - 'required_links_text', - 'title', - 'description', - 'og_title', - 'og_description', - 'og_article_published', - 'og_article_modified', - 'canonical_link', - 'main_element_present', - 'robots_txt_detected', - 'robots_txt_target_url_redirects', - 'robots_txt_final_url', - 'robots_txt_final_url_live', - 'robots_txt_final_url_status_code', - 'robots_txt_final_url_media_type', - 'robots_txt_final_url_filesize', - 'robots_txt_crawl_delay', - 'robots_txt_sitemap_locations', - 'sitemap_xml_detected', - 'sitemap_xml_target_url_redirects', - 'sitemap_xml_final_url', - 'sitemap_xml_final_url_live', - 'sitemap_xml_final_url_status_code', - 'sitemap_xml_final_url_media_type', - 'sitemap_xml_final_url_filesize', - 'sitemap_xml_count', - 'sitemap_xml_pdf_count', - 'uswds_favicon', - 'uswds_favicon_in_css', - 'uswds_publicsans_font', - 'uswds_inpage_css', - 'uswds_usa_classes', - 'uswds_string', - 'uswds_string_in_css', - 'uswds_semantic_version', - 'uswds_version', - 'uswds_count', - ]; - }); - it('returns headers when no data is passed', () => { - const serializer = new CsvSerializer(liveColumnOrder); + const serializer = new CsvSerializer(CoreResult.snapshotColumnOrder); const result = serializer.serialize([]); const expectedResult = - '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","og_title","og_description","og_article_published","og_article_modified","canonical_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"'; + '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","keywords","og_title","og_description","og_article_published","og_article_modified","og_image","og_type","og_url","canonical_link","language","language_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"'; expect(result).toEqual(expectedResult); }); it('serializes an array containing one website', () => { - const serializer = new CsvSerializer(liveColumnOrder); + const serializer = new CsvSerializer(CoreResult.snapshotColumnOrder); const website = new Website(); const coreResult = new CoreResult(); website.url = '18f.gov'; @@ -109,13 +22,13 @@ describe('CsvSerializer', () => { const result = serializer.serialize([website]); const expectedResult = - '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","og_title","og_description","og_article_published","og_article_modified","canonical_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; + '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","keywords","og_title","og_description","og_article_published","og_article_modified","og_image","og_type","og_url","canonical_link","language","language_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; expect(result).toEqual(expectedResult); }); it('serializes an array containing more than one website', () => { - const serializer = new CsvSerializer(liveColumnOrder); + const serializer = new CsvSerializer(CoreResult.snapshotColumnOrder); const firstWebsite = new Website(); const firstCoreResult = new CoreResult(); firstWebsite.url = '18f.gov'; @@ -127,13 +40,13 @@ describe('CsvSerializer', () => { const result = serializer.serialize([firstWebsite, secondWebsite]); const expectedResult = - '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","og_title","og_description","og_article_published","og_article_modified","canonical_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,\r\n"nasa.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; + '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","keywords","og_title","og_description","og_article_published","og_article_modified","og_image","og_type","og_url","canonical_link","language","language_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,\r\n"nasa.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; expect(result).toEqual(expectedResult); }); it('serializes an array containing one website with dap parameters', () => { - const serializer = new CsvSerializer(liveColumnOrder); + const serializer = new CsvSerializer(CoreResult.snapshotColumnOrder); const website = new Website(); const coreResult = new CoreResult(); coreResult.dapParameters = 'agency=GSA&subagency=TTS,18F'; @@ -142,13 +55,13 @@ describe('CsvSerializer', () => { const result = serializer.serialize([website]); const expectedResult = - '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","og_title","og_description","og_article_published","og_article_modified","canonical_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"{""agency"":""GSA"",""subagency"":""TTS,18F""}",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; + '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","keywords","og_title","og_description","og_article_published","og_article_modified","og_image","og_type","og_url","canonical_link","language","language_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"{""agency"":""GSA"",""subagency"":""TTS,18F""}",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; expect(result).toEqual(expectedResult); }); it('serializes an array containing more than one website with dap parameters', () => { - const serializer = new CsvSerializer(liveColumnOrder); + const serializer = new CsvSerializer(CoreResult.snapshotColumnOrder); const firstWebsite = new Website(); const firstCoreResult = new CoreResult(); firstCoreResult.dapParameters = 'agency=GSA&subagency=TTS,18F'; @@ -163,13 +76,13 @@ describe('CsvSerializer', () => { const result = serializer.serialize([firstWebsite, secondWebsite]); const expectedResult = - '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","og_title","og_description","og_article_published","og_article_modified","canonical_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"{""agency"":""GSA"",""subagency"":""TTS,18F""}",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,\r\n"mentor.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"{""agency"":""AmeriCorps"",""cto"":""12""}",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; + '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","keywords","og_title","og_description","og_article_published","og_article_modified","og_image","og_type","og_url","canonical_link","language","language_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"{""agency"":""GSA"",""subagency"":""TTS,18F""}",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,\r\n"mentor.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"{""agency"":""AmeriCorps"",""cto"":""12""}",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; expect(result).toEqual(expectedResult); }); it('serializes a csv file and removes newline characters in the process', () => { - const serializer = new CsvSerializer(liveColumnOrder); + const serializer = new CsvSerializer(CoreResult.snapshotColumnOrder); const website = new Website(); const coreResult = new CoreResult(); coreResult.ogDescriptionFinalUrl = 'Yadda \nYadda \nYadda'; @@ -178,13 +91,13 @@ describe('CsvSerializer', () => { const result = serializer.serialize([website]); const expectedResult = - '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","og_title","og_description","og_article_published","og_article_modified","canonical_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"Yadda Yadda Yadda",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; + '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","keywords","og_title","og_description","og_article_published","og_article_modified","og_image","og_type","og_url","canonical_link","language","language_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"Yadda Yadda Yadda",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; expect(result).toEqual(expectedResult); }); it('serializes a csv file and truncates strings that contain more than 5000 characters', () => { - const serializer = new CsvSerializer(liveColumnOrder); + const serializer = new CsvSerializer(CoreResult.snapshotColumnOrder); const website = new Website(); const coreResult = new CoreResult(); const longString = generateLongString(6000); @@ -195,9 +108,9 @@ describe('CsvSerializer', () => { const expectedTruncatedLongString = longString.substring(0, 2000); const result = serializer.serialize([website]); const expectedResult = - '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","og_title","og_description","og_article_published","og_article_modified","canonical_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"' + + '"target_url","target_url_domain","target_url_top_level_domain","target_url_redirects","final_url","final_url_domain","final_url_top_level_domain","final_url_website","final_url_live","final_url_status_code","final_url_media_type","final_url_same_domain","final_url_same_website","target_url_agency_owner","target_url_bureau_owner","target_url_branch","target_url_404_test","source_list","public","scan_date","primary_scan_status","accessibility_scan_status","dns_scan_status","not_found_scan_status","performance_scan_status","robots_txt_scan_status","sitemap_xml_scan_status","ipv6","hostname","cms","login_provider","login","site_search","search_dot_gov","dap","dap_parameters","third_party_service_domains","third_party_service_count","cookie_domains","viewport_meta_tag","cumulative_layout_shift","largest_contentful_paint","required_links_url","required_links_text","title","description","keywords","og_title","og_description","og_article_published","og_article_modified","og_image","og_type","og_url","canonical_link","language","language_link","main_element_present","robots_txt_detected","robots_txt_target_url_redirects","robots_txt_final_url","robots_txt_final_url_live","robots_txt_final_url_status_code","robots_txt_final_url_media_type","robots_txt_final_url_filesize","robots_txt_crawl_delay","robots_txt_sitemap_locations","sitemap_xml_detected","sitemap_xml_target_url_redirects","sitemap_xml_final_url","sitemap_xml_final_url_live","sitemap_xml_final_url_status_code","sitemap_xml_final_url_media_type","sitemap_xml_final_url_filesize","sitemap_xml_count","sitemap_xml_pdf_count","uswds_favicon","uswds_favicon_in_css","uswds_publicsans_font","uswds_inpage_css","uswds_usa_classes","uswds_string","uswds_string_in_css","uswds_semantic_version","uswds_version","uswds_count"\r\n"18f.gov",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"' + expectedTruncatedLongString + - '",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; + '",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'; expect(result).toEqual(expectedResult); }); diff --git a/libs/snapshot/src/serializers/json-serializer.spec.ts b/libs/snapshot/src/serializers/json-serializer.spec.ts index a2a8df48..bf5c0104 100644 --- a/libs/snapshot/src/serializers/json-serializer.spec.ts +++ b/libs/snapshot/src/serializers/json-serializer.spec.ts @@ -4,86 +4,7 @@ import { JsonSerializer } from './json-serializer'; describe('JsonSerializer', () => { it('serializes an array containing one website', () => { - const liveColumnOrder = [ - 'target_url', - 'target_url_domain', - 'target_url_top_level_domain', - 'target_url_redirects', - 'final_url', - 'final_url_domain', - 'final_url_top_level_domain', - 'final_url_website', - 'final_url_live', - 'final_url_status_code', - 'final_url_media_type', - 'final_url_same_domain', - 'final_url_same_website', - 'target_url_agency_owner', - 'target_url_bureau_owner', - 'target_url_branch', - 'target_url_404_test', - 'source_list', - 'public', - 'scan_date', - 'primary_scan_status', - 'accessibility_scan_status', - 'dns_scan_status', - 'not_found_scan_status', - 'performance_scan_status', - 'robots_txt_scan_status', - 'sitemap_xml_scan_status', - 'ipv6', - 'hostname', - 'cms', - 'login_provider', - 'login', - 'site_search', - 'search_dot_gov', - 'dap', - 'dap_parameters', - 'third_party_service_domains', - 'third_party_service_count', - 'cookie_domains', - 'viewport_meta_tag', - 'required_links_url', - 'required_links_text', - 'og_title', - 'og_description', - 'og_article_published', - 'og_article_modified', - 'canonical_link', - 'main_element_present', - 'robots_txt_detected', - 'robots_txt_target_url_redirects', - 'robots_txt_final_url', - 'robots_txt_final_url_live', - 'robots_txt_final_url_status_code', - 'robots_txt_final_url_media_type', - 'robots_txt_final_url_filesize', - 'robots_txt_crawl_delay', - 'robots_txt_sitemap_locations', - 'sitemap_xml_detected', - 'sitemap_xml_target_url_redirects', - 'sitemap_xml_final_url', - 'sitemap_xml_final_url_live', - 'sitemap_xml_final_url_status_code', - 'sitemap_xml_final_url_media_type', - 'sitemap_xml_final_url_filesize', - 'sitemap_xml_count', - 'sitemap_xml_pdf_count', - 'uswds_favicon', - 'uswds_favicon_in_css', - 'uswds_publicsans_font', - 'uswds_inpage_css', - 'uswds_usa_classes', - 'uswds_string', - 'uswds_string_in_css', - 'uswds_semantic_version', - 'uswds_version', - 'uswds_count', - ]; - - const serializer = new JsonSerializer(liveColumnOrder); + const serializer = new JsonSerializer(CoreResult.snapshotColumnOrder); const website = new Website(); const coreResult = new CoreResult(); website.coreResult = coreResult; diff --git a/libs/snapshot/src/snapshot.service.ts b/libs/snapshot/src/snapshot.service.ts index a72c4437..22670b8b 100644 --- a/libs/snapshot/src/snapshot.service.ts +++ b/libs/snapshot/src/snapshot.service.ts @@ -6,6 +6,7 @@ import { Snapshot } from './snapshot'; import { JsonSerializer } from './serializers/json-serializer'; import { CsvSerializer } from './serializers/csv-serializer'; import { ConfigService } from '@nestjs/config'; +import { CoreResult } from 'entities/core-result.entity'; @Injectable() export class SnapshotService { @@ -37,91 +38,8 @@ export class SnapshotService { date.setDate(date.getDate() - 7); const priorDate = date.toISOString(); - const liveColumnOrder = [ - 'target_url', - 'target_url_domain', - 'target_url_top_level_domain', - 'target_url_redirects', - 'final_url', - 'final_url_domain', - 'final_url_top_level_domain', - 'final_url_website', - 'final_url_live', - 'final_url_status_code', - 'final_url_media_type', - 'final_url_same_domain', - 'final_url_same_website', - 'target_url_agency_owner', - 'target_url_bureau_owner', - 'target_url_branch', - 'target_url_404_test', - 'source_list', - 'public', - 'scan_date', - 'primary_scan_status', - 'accessibility_scan_status', - 'dns_scan_status', - 'not_found_scan_status', - 'performance_scan_status', - 'robots_txt_scan_status', - 'sitemap_xml_scan_status', - 'ipv6', - 'hostname', - 'cms', - 'login_provider', - 'login', - 'site_search', - 'search_dot_gov', - 'dap', - 'dap_parameters', - 'third_party_service_domains', - 'third_party_service_count', - 'cookie_domains', - 'viewport_meta_tag', - 'cumulative_layout_shift', - 'largest_contentful_paint', - 'required_links_url', - 'required_links_text', - 'title', - 'description', - 'og_title', - 'og_description', - 'og_article_published', - 'og_article_modified', - 'canonical_link', - 'main_element_present', - 'robots_txt_detected', - 'robots_txt_target_url_redirects', - 'robots_txt_final_url', - 'robots_txt_final_url_live', - 'robots_txt_final_url_status_code', - 'robots_txt_final_url_media_type', - 'robots_txt_final_url_filesize', - 'robots_txt_crawl_delay', - 'robots_txt_sitemap_locations', - 'sitemap_xml_detected', - 'sitemap_xml_target_url_redirects', - 'sitemap_xml_final_url', - 'sitemap_xml_final_url_live', - 'sitemap_xml_final_url_status_code', - 'sitemap_xml_final_url_media_type', - 'sitemap_xml_final_url_filesize', - 'sitemap_xml_count', - 'sitemap_xml_pdf_count', - 'uswds_favicon', - 'uswds_favicon_in_css', - 'uswds_publicsans_font', - 'uswds_inpage_css', - 'uswds_usa_classes', - 'uswds_string', - 'uswds_string_in_css', - 'uswds_semantic_version', - 'uswds_version', - 'uswds_count', - ]; - - await this.liveSnapshot(priorDate, liveColumnOrder); - await this.allSnapshot(priorDate, liveColumnOrder); + await this.liveSnapshot(priorDate, CoreResult.snapshotColumnOrder); + await this.allSnapshot(priorDate, CoreResult.snapshotColumnOrder); } async liveSnapshot(date: string, columns: string[]): Promise { diff --git a/libs/snapshot/src/snapshot.spec.ts b/libs/snapshot/src/snapshot.spec.ts index f01c58d6..4c1cc664 100644 --- a/libs/snapshot/src/snapshot.spec.ts +++ b/libs/snapshot/src/snapshot.spec.ts @@ -8,85 +8,9 @@ import { CsvSerializer } from './serializers/csv-serializer'; describe('Snapshot', () => { let mockStorageService: MockProxy; - let liveColumnOrder: string[]; beforeEach(async () => { mockStorageService = mock(); - liveColumnOrder = [ - 'target_url', - 'target_url_domain', - 'final_url', - 'final_url_domain', - 'final_url_website', - 'canonical_link', - 'final_url_mimetype', - 'final_url_live', - 'target_url_redirects', - 'final_url_same_domain', - 'final_url_same_website', - 'target_url_agency_owner', - 'target_url_bureau_owner', - 'target_url_branch', - 'final_url_status_code', - 'target_url_404_test', - 'scan_date', - 'primary_scan_status', - 'not_found_scan_status', - 'robots_txt_scan_status', - 'sitemap_xml_scan_status', - 'dns_scan_status', - 'dns_ipv6', - 'dns_hostname', - 'source_list', - 'cms', - 'required_links_url', - 'required_links_text', - 'uswds_favicon', - 'uswds_favicon_in_css', - 'uswds_publicsans_font', - 'uswds_count', - 'uswds_usa_classes', - 'uswds_inpage_css', - 'uswds_string', - 'uswds_string_in_css', - 'uswds_semantic_version', - 'uswds_version', - 'og_article_published_final_url', - 'og_article_modified_final_url', - 'og_title_final_url', - 'og_description_final_url', - 'main_element_present_final_url', - 'robots_txt_final_url', - 'robots_txt_detected', - 'robots_txt_final_url_live', - 'robots_txt_target_url_redirects', - 'robots_txt_final_url_status_code', - 'robots_txt_final_url_mimetype', - 'robots_txt_final_url_filesize_in_bytes', - 'robots_txt_crawl_delay', - 'robots_txt_sitemap_locations', - 'sitemap_xml_final_url', - 'sitemap_xml_detected', - 'sitemap_xml_final_url_live', - 'sitemap_xml_target_url_redirects', - 'sitemap_xml_final_url_status_code', - 'sitemap_xml_final_url_mimetype', - 'sitemap_xml_final_url_filesize_in_bytes', - 'sitemap_xml_count', - 'sitemap_xml_pdf_count', - 'third_party_service_domains', - 'third_party_service_count', - 'viewport_meta_tag', - 'cumulative_layout_shift', - 'largest_contentful_paint', - 'cookie_domains', - 'login_provider', - 'login_detected', - 'searchgov', - 'search_detected', - 'dap_detected_final_url', - 'dap_parameters_final_url', - ]; }); it('archives prior snapshot', async () => { @@ -99,7 +23,10 @@ describe('Snapshot', () => { const snapshot = new Snapshot( mockStorageService, - [new JsonSerializer(liveColumnOrder), new CsvSerializer(liveColumnOrder)], + [ + new JsonSerializer(CoreResult.snapshotColumnOrder), + new CsvSerializer(CoreResult.snapshotColumnOrder), + ], [website], dateString, 'weekly-snapshot', @@ -127,7 +54,10 @@ describe('Snapshot', () => { const snapshot = new Snapshot( mockStorageService, - [new JsonSerializer(liveColumnOrder), new CsvSerializer(liveColumnOrder)], + [ + new JsonSerializer(CoreResult.snapshotColumnOrder), + new CsvSerializer(CoreResult.snapshotColumnOrder), + ], [website], dateString, 'weekly-snapshot',