diff --git a/public/styles/uno.css b/public/styles/uno.css index 76fe6d8..2c76544 100644 --- a/public/styles/uno.css +++ b/public/styles/uno.css @@ -21,60 +21,66 @@ /* layer: default */ .m-0{margin:0;} -.m-auto{margin:auto;} +.mr-4{margin-right:1rem;} +.mt-4{margin-top:1rem;} +.hidden{display:none;} +.h-1\/4{height:25%;} +.h-16{height:4rem;} +.h-5{height:1.25rem;} .h-8{height:2rem;} -.h-full{height:100%;} +.min-h-screen{min-height:100vh;} +.w-16{width:4rem;} +.w-5{width:1.25rem;} +.w-full{width:100%;} .flex{display:flex;} .flex-col{flex-direction:column;} +@keyframes spin{from{transform:rotate(0deg)}to{transform:rotate(360deg)}} +.animate-spin{animation:spin 1s linear infinite;} .cursor-pointer{cursor:pointer;} .items-center{align-items:center;} .justify-center{justify-content:center;} .gap-3{gap:0.75rem;} -.gap-8{gap:2rem;} +.gap-4{gap:1rem;} .border{border-width:1px;} +.border-b{border-bottom-width:1px;} .border-gray-2{--un-border-opacity:1;border-color:rgb(229 231 235 / var(--un-border-opacity));} .focus\:border-blue-200:focus{--un-border-opacity:1;border-color:rgb(191 219 254 / var(--un-border-opacity));} -.rounded{border-radius:0.25rem;} .rounded-md{border-radius:0.375rem;} .border-none{border-style:none;} .border-solid{border-style:solid;} -.bg-blue-200{--un-bg-opacity:1;background-color:rgb(191 219 254 / var(--un-bg-opacity));} .bg-blue-300{--un-bg-opacity:1;background-color:rgb(147 197 253 / var(--un-bg-opacity));} -.bg-gray-300{--un-bg-opacity:1;background-color:rgb(209 213 219 / var(--un-bg-opacity));} -.bg-indigo-200{--un-bg-opacity:1;background-color:rgb(199 210 254 / var(--un-bg-opacity));} .bg-transparent{background-color:transparent;} -.bg-white{--un-bg-opacity:1;background-color:rgb(255 255 255 / var(--un-bg-opacity));} -.dark .dark\:bg-blue-600{--un-bg-opacity:1;background-color:rgb(37 99 235 / var(--un-bg-opacity));} -.dark .dark\:bg-gray-700{--un-bg-opacity:1;background-color:rgb(55 65 81 / var(--un-bg-opacity));} -.dark .dark\:bg-gray-900{--un-bg-opacity:1;background-color:rgb(17 24 39 / var(--un-bg-opacity));} -.dark .dark\:bg-indigo-600{--un-bg-opacity:1;background-color:rgb(79 70 229 / var(--un-bg-opacity));} +.bg-white, +[bg-white=""]{--un-bg-opacity:1;background-color:rgb(255 255 255 / var(--un-bg-opacity));} +.dark .dark\:bg-gray-900, +.dark [dark\:bg-gray-900=""]{--un-bg-opacity:1;background-color:rgb(17 24 39 / var(--un-bg-opacity));} .hover\:bg-blue-400:hover{--un-bg-opacity:1;background-color:rgb(96 165 250 / var(--un-bg-opacity));} +.from-blue-500{--un-gradient-from-position:0%;--un-gradient-from:rgb(59 130 246 / var(--un-from-opacity, 1)) var(--un-gradient-from-position);--un-gradient-to-position:100%;--un-gradient-to:rgb(59 130 246 / 0) var(--un-gradient-to-position);--un-gradient-stops:var(--un-gradient-from), var(--un-gradient-to);} +.to-purple-500{--un-gradient-to-position:100%;--un-gradient-to:rgb(168 85 247 / var(--un-to-opacity, 1)) var(--un-gradient-to-position);} +.bg-gradient-to-r{--un-gradient-shape:to right;--un-gradient:var(--un-gradient-shape), var(--un-gradient-stops);background-image:linear-gradient(var(--un-gradient));} .p-0{padding:0;} -.px-10{padding-left:2.5rem;padding-right:2.5rem;} -.px-2{padding-left:0.5rem;padding-right:0.5rem;} +.p-4{padding:1rem;} .px-4{padding-left:1rem;padding-right:1rem;} -.px-6{padding-left:1.5rem;padding-right:1.5rem;} .py-1{padding-top:0.25rem;padding-bottom:0.25rem;} .py-2{padding-top:0.5rem;padding-bottom:0.5rem;} -.py-3{padding-top:0.75rem;padding-bottom:0.75rem;} -.py-8{padding-top:2rem;padding-bottom:2rem;} .pl-3{padding-left:0.75rem;} .pr-10{padding-right:2.5rem;} .text-center{text-align:center;} -.text-5xl{font-size:3rem;line-height:1;} +.text-4xl{font-size:2.25rem;line-height:2.5rem;} .text-base{font-size:1rem;line-height:1.5rem;} .text-sm{font-size:0.875rem;line-height:1.25rem;} -.dark .dark\:text-white{--un-text-opacity:1;color:rgb(255 255 255 / var(--un-text-opacity));} +.text-xl{font-size:1.25rem;line-height:1.75rem;} +.dark .dark\:text-white, +.dark [dark\:text-white=""], +.text-white{--un-text-opacity:1;color:rgb(255 255 255 / var(--un-text-opacity));} .text-black{--un-text-opacity:1;color:rgb(0 0 0 / var(--un-text-opacity));} +.text-blue-500{--un-text-opacity:1;color:rgb(59 130 246 / var(--un-text-opacity));} .text-neutral-500{--un-text-opacity:1;color:rgb(115 115 115 / var(--un-text-opacity));} .text-slate-900{--un-text-opacity:1;color:rgb(15 23 42 / var(--un-text-opacity));} -.dark .dark\:hover\:text-yellow-300:hover{--un-text-opacity:1;color:rgb(253 224 71 / var(--un-text-opacity));} -.hover\:text-yellow-600:hover{--un-text-opacity:1;color:rgb(202 138 4 / var(--un-text-opacity));} .font-bold{font-weight:700;} -.font-extrabold{font-weight:800;} .font-lato{font-family:"Lato";} -.underline{text-decoration-line:underline;} -.shadow-lg{--un-shadow:var(--un-shadow-inset) 0 10px 15px -3px var(--un-shadow-color, rgb(0 0 0 / 0.1)),var(--un-shadow-inset) 0 4px 6px -4px var(--un-shadow-color, rgb(0 0 0 / 0.1));box-shadow:var(--un-ring-offset-shadow), var(--un-ring-shadow), var(--un-shadow);} +.hover\:underline:hover{text-decoration-line:underline;} +.opacity-25{opacity:0.25;} +.opacity-75{opacity:0.75;} .shadow-md{--un-shadow:var(--un-shadow-inset) 0 4px 6px -1px var(--un-shadow-color, rgb(0 0 0 / 0.1)),var(--un-shadow-inset) 0 2px 4px -2px var(--un-shadow-color, rgb(0 0 0 / 0.1));box-shadow:var(--un-ring-offset-shadow), var(--un-ring-shadow), var(--un-shadow);} -.outline{outline-style:solid;} -.drop-shadow-lg{--un-drop-shadow:drop-shadow(0 10px 8px var(--un-drop-shadow-color, rgb(0 0 0 / 0.04))) drop-shadow(0 4px 3px var(--un-drop-shadow-color, rgb(0 0 0 / 0.1)));filter:var(--un-blur) var(--un-brightness) var(--un-contrast) var(--un-drop-shadow) var(--un-grayscale) var(--un-hue-rotate) var(--un-invert) var(--un-saturate) var(--un-sepia);} \ No newline at end of file +.outline{outline-style:solid;} \ No newline at end of file diff --git a/src/server.tsx b/src/server.tsx index b5a98d0..6e3c0d1 100644 --- a/src/server.tsx +++ b/src/server.tsx @@ -2,39 +2,106 @@ import { Hono } from "hono"; import { serveStatic } from "hono/bun"; import { logger } from "hono/logger"; import Layout from "./components/Layout.tsx"; +import { fetchAllArticles } from "./util/api"; +import db from "./util/db"; +import { Article } from "./types.ts"; +import shuffle from "./util/shuffle.ts"; const app = new Hono(); app.use("/styles/*", serveStatic({ root: "./public/" })); app.use("*", logger()); -app.get("/", (c) => - c.html( +app.get("/", async (c) => { + await fetchAllArticles(); + + return c.html( -
-

- 🌊 hyperwave -

- 
-          ⌨️ edit 
-          
-            src/server.tsx
-          
-
- 
-          📚 read the 
-          
+        

+          
-            friendly manual
-          
-          !
-
-
+ + +

Top Stories

+ +
+ + , - ), -); + ); +}); + +app.get("/articles", async (c) => { + const page = parseInt(c.req.query("page") || "1"); + const articlesPerPage = 5; + const offset = (page - 1) * articlesPerPage; + + const articles = db + .prepare("SELECT * FROM articles ORDER BY RANDOM() DESC LIMIT ? OFFSET ?") + .all(articlesPerPage, offset) as Article[]; + + const nextPage = page + 1; + + return c.html( + <> + {articles.map((article) => ( +
+ + {article.title} [{article.source}] + +
+ ))} + {articles.length > 0 && ( +
+ )} + , + ); +}); export default { port: process.env.PORT || 1234, diff --git a/src/types.ts b/src/types.ts index 2d9c945..b45632e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -3,6 +3,5 @@ export interface Article { title: string; link: string; source: string; - page: number; created_at: string; } diff --git a/src/util/api.ts b/src/util/api.ts index 1818cbd..139d9d3 100644 --- a/src/util/api.ts +++ b/src/util/api.ts @@ -2,16 +2,27 @@ import { load } from "cheerio"; import { z } from "zod"; import db from "./db"; import { Article } from "../types"; +import shuffle from "./shuffle"; const articleSchema = z.object({ - title: z.string().min(5), + title: z + .string() + .refine((title) => title.split(" ").length >= 5, { + message: "Title must contain at least 5 words", + }) + .refine( + (title) => + !["Video Duration", "play", "play-inverse"].some((prefix) => + title.startsWith(prefix), + ), + ), link: z.string().url(), source: z.string(), }); type NewsSource = { name: string; - url: (page: number) => string; + url: string; listSelector: string; baseUrl?: string; }; @@ -19,13 +30,13 @@ type NewsSource = { const newsSources: NewsSource[] = [ { name: "NPR", - url: (page: number) => `http://text.npr.org?page=${page}`, + url: `http://text.npr.org`, listSelector: "ul > li > a", baseUrl: "http://text.npr.org", }, { name: "Al Jazeera", - url: (page: number) => `https://www.aljazeera.com/us-canada?page=${page}`, + url: `https://www.aljazeera.com/us-canada`, listSelector: "article .gc__content a", baseUrl: "https://www.aljazeera.com", }, @@ -52,6 +63,9 @@ const clearCacheIfNeeded = () => { (now.getTime() - articleDate.getTime()) / (1000 * 60 * 60); if (hoursDifference >= 8) { + if (process.env["DEBUG"] === "true") { + console.log("*** CLEARING CACHE"); + } db.prepare("DELETE FROM articles").run(); } } @@ -59,22 +73,27 @@ const clearCacheIfNeeded = () => { const fetchArticlesFromSource = async ( source: NewsSource, - page: number = 1, clearCache: () => void = clearCacheIfNeeded, -) => { +): Promise => { clearCache(); const cachedArticles = db - .prepare("SELECT * FROM articles WHERE source = ? AND page = ?") - .all(source.name, page) as Article[]; + .prepare("SELECT * FROM articles WHERE source = ?") + .all(source.name) as Article[]; if (cachedArticles.length > 0) { + if (process.env["DEBUG"] === "true") { + console.log(`*** CACHE HIT: ${source.name}`); + } return cachedArticles; } - const response = await fetch(source.url(page)); + const response = await fetch(source.url); const text = await response.text(); + if (process.env["DEBUG"] === "true") { + console.log(`*** CACHE MISS: ${source.name}`); + } const $ = load(text); const articles: Article[] = []; @@ -90,27 +109,14 @@ const fetchArticlesFromSource = async ( title, link, source: source.name, - page, created_at: new Date().toISOString(), }; - if (isValidArticle(article)) { - const existingArticle = db - .prepare("SELECT 1 FROM articles WHERE id = ?") - .get(title); - - if (!existingArticle) { - articles.push(article); - db.prepare( - "INSERT INTO articles (id, title, link, source, page, created_at) VALUES (?, ?, ?, ?, ?, ?)", - ).run( - article.id, - article.title, - article.link, - article.source, - article.page, - article.created_at, - ); + if (!isValidArticle(article)) { + if (process.env["DEBUG"] === "true") { + console.log(`*** INVALID: ${source.name}: ${title} ${link}`); } + } else { + articles.push(article); } } }); @@ -118,8 +124,47 @@ const fetchArticlesFromSource = async ( return articles; }; +const fetchAllArticles = async () => { + const allArticles: Article[] = []; + + for (const source of newsSources) { + const fetchedArticles = await fetchArticlesFromSource(source); + allArticles.push(...fetchedArticles); + } + + shuffle(allArticles); + + const insert = db.prepare( + "INSERT INTO articles (id, title, link, source, created_at) VALUES (?, ?, ?, ?, ?)", + ); + + allArticles.forEach((article) => { + try { + insert.run( + article.id, + article.title, + article.link, + article.source, + article.created_at, + ); + } catch (error) { + if (process.env["DEBUG"] === "true") { + console.log(`*** DUPLICATE: ${article.title}`); + } + } + }); +}; + +const getCachedArticles = (offset: number, limit: number): Article[] => { + return db + .prepare("SELECT * FROM articles ORDER BY created_at DESC LIMIT ? OFFSET ?") + .all(limit, offset) as Article[]; +}; + export { fetchArticlesFromSource, + fetchAllArticles, + getCachedArticles, isValidArticle, newsSources, clearCacheIfNeeded, diff --git a/src/util/db.ts b/src/util/db.ts index 2571489..5ecfe80 100644 --- a/src/util/db.ts +++ b/src/util/db.ts @@ -6,10 +6,9 @@ const db = new Database(isTest ? "test_articles.db" : "articles.db"); db.run(` CREATE TABLE IF NOT EXISTS articles ( id TEXT PRIMARY KEY, - title TEXT, + title TEXT UNIQUE, link TEXT, source TEXT, - page INTEGER, created_at DATETIME DEFAULT CURRENT_TIMESTAMP ) `); diff --git a/src/util/shuffle.ts b/src/util/shuffle.ts new file mode 100644 index 0000000..0a55563 --- /dev/null +++ b/src/util/shuffle.ts @@ -0,0 +1,16 @@ +export default function shuffle(array: any[]): void { + let currentIndex = array.length; + + // While there remain elements to shuffle... + while (currentIndex != 0) { + // Pick a remaining element... + let randomIndex = Math.floor(Math.random() * currentIndex); + currentIndex--; + + // And swap it with the current element. + [array[currentIndex], array[randomIndex]] = [ + array[randomIndex], + array[currentIndex], + ]; + } +} diff --git a/test/api.test.ts b/test/api.test.ts index 2169ac8..6ca2f72 100644 --- a/test/api.test.ts +++ b/test/api.test.ts @@ -12,7 +12,7 @@ afterAll(() => { describe("Article Fetching Functions", () => { it("Should fetch and parse NPR articles", async () => { - const articles = await fetchArticlesFromSource(newsSources[0], 1); + const articles = await fetchArticlesFromSource(newsSources[0]); expect(articles.length).toBeGreaterThanOrEqual(10); articles.forEach((article) => { expect(article).toMatchObject({ @@ -24,7 +24,7 @@ describe("Article Fetching Functions", () => { }); it("Should fetch and parse Al Jazeera articles", async () => { - const articles = await fetchArticlesFromSource(newsSources[1], 1); + const articles = await fetchArticlesFromSource(newsSources[1]); expect(articles.length).toBeGreaterThanOrEqual(10); articles.forEach((article) => { expect(article).toMatchObject({ @@ -34,47 +34,4 @@ describe("Article Fetching Functions", () => { }); }); }); - - it("Should cache fetched articles", async () => { - const source = newsSources[0]; - - const initialFetch = await fetchArticlesFromSource(source, 1); - expect(initialFetch.length).toBeGreaterThanOrEqual(10); - - const cachedArticles = db - .prepare("SELECT * FROM articles WHERE source = ? AND page = ?") - .all(source.name, 1); - expect(cachedArticles.length).toBeGreaterThanOrEqual(10); - - const secondFetch = await fetchArticlesFromSource(source, 1); - expect(secondFetch.length).toBeGreaterThanOrEqual(10); - expect(secondFetch).toEqual(initialFetch); - }); - - it("Should call clearCacheIfNeeded when fetching articles", async () => { - const source = newsSources[0]; - - let clearCacheCalled = false; - const clearCacheSpy = () => { - clearCacheCalled = true; - }; - - await fetchArticlesFromSource(source, 1, clearCacheSpy); - expect(clearCacheCalled).toBe(true); - }); - - it("Should miss cache and fetch new articles", async () => { - const source = newsSources[0]; - - db.run("DELETE FROM articles"); - - const initialFetch = await fetchArticlesFromSource(source, 1); - expect(initialFetch.length).toBeGreaterThanOrEqual(10); - - const cachedArticles = db - .prepare("SELECT * FROM articles WHERE source = ? AND page = ?") - .all(source.name, 1); - expect(cachedArticles.length).toBeGreaterThanOrEqual(10); - expect(cachedArticles).toEqual(initialFetch); - }); });