Fix wrong course bug (#10)

* make zero trust loading more; remove fp-ts

* add unit  test for updateAssignmentsOfCourse

* lint and format

* remove console.log mergedNewAssignments from updateAssignmentsOfCourse; update config to double spacce for second header

* fix test case

package.json

@@ -2,11 +2,9 @@
   "dependencies": {
     "@prisma/client": "^5.16.1",
     "cheerio": "^1.0.0-rc.12",
-    "discord.js": "^14.15.3",
+    "discord.js": "^14.16.2",
     "dotenv": "^16.3.1",
     "envalid": "^8.0.0",
-    "express": "^4.18.2",
-    "fp-ts": "^2.16.7",
     "node-cache": "^5.1.2",
     "zod": "^3.23.8"
   },
@@ -32,13 +30,14 @@
     "ts-node": "^10.9.2",
     "tsc-alias": "^1.8.10",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.5.3",
+    "typescript": "<5.6.2",
     "typescript-eslint": "^7.15.0"
   },
   "scripts": {
     "migrate": "prisma db push",
     "build": "prisma generate && tsc && tsc-alias",
     "start": "prisma db push && node ./build/start.js",
+    "startTs": "ts-node ./src/start.ts",
     "dev": "prisma generate && prisma db push && nodemon ./src/start.ts",
     "test": "jest",
     "lint": "eslint . --fix",

src/config/config.ts

@@ -9,7 +9,7 @@ export const NEW_ASSIGNMENTS_MESSAGE = '## New Assignments!!'
 export const NEW_ASSIGNMENTS_MESSAGE_SIZE = [...NEW_ASSIGNMENTS_MESSAGE].length
 export const COURSE_MESSAGE_PATTERN = '\n- %s'
 export const ASSIGNMENT_MESSAGE_PATTERN =
-  '\n - [%s](https://www.mycourseville.com/?q=courseville/worksheet/%d/%d)'
+  '\n  - [%s](https://www.mycourseville.com/?q=courseville/worksheet/%d/%d)'
 
 export enum NotifyMessage {
   FetchingError = 'Error fetching, Might be rate limited or server is down',

src/scraper/extractAssignmentsFromCheerio.ts

@@ -1,28 +1,38 @@
 import db, { Assignment } from '../database/database'
-import * as cheerio from 'cheerio'
 
 export default async function extractAssignmentsFromCheerio(
-  mcvID: number,
   $: cheerio.Root
-): Promise<Array<Assignment>> {
+): Promise<[number, Array<Assignment>] | undefined> {
   const assignmentNameNodes = $('tbody tr td:nth-child(2) a').toArray()
   const assignments: Array<Assignment> = []
+  let foundMcvId: number | undefined = undefined
   for (let i = 0; i < assignmentNameNodes.length; i++) {
     const ele = assignmentNameNodes[i]
     const assignmentLink = $(ele).attr('href')
-    const assignmentIdStr: string = assignmentLink!.match(/^.*\/(\d+)$/)![1]
+    const mcvIdAndAssignment = assignmentLink!.match(/^.*\/(\d+)\/(\d+)$/)!
+
+    const currentMcvId: number = parseInt(mcvIdAndAssignment[1])
+    if (foundMcvId == undefined) {
+      foundMcvId = currentMcvId
+    } else if (currentMcvId != foundMcvId) {
+      throw new Error('Unexpected course id')
+    }
+    const assignmentIdStr: string = mcvIdAndAssignment[2]
     const assignmentId: number = parseInt(assignmentIdStr, 10)
     const assignment: Assignment = {
-      mcvCourseID: mcvID,
+      mcvCourseID: foundMcvId,
       assignmentName: $(ele).text(),
       assignmentID: assignmentId,
     }
     const found = await db.assignmentExists(assignment)
     if (!found) {
-      // console.log('found new assignment',assignment)
+      console.log('found new assignment', assignment)
       assignments.push(assignment)
       await db.saveAssignment(assignment)
     }
   }
-  return assignments
+  if (foundMcvId == undefined) {
+    return undefined
+  }
+  return [foundMcvId, assignments]
 }

src/scraper/scrapeAssignmentsOfPage.ts

@@ -1,4 +1,3 @@
-import { option } from 'fp-ts'
 import fetchAndCatch from '../utils/fetchAndCatch'
 import { Assignment } from '../database/database'
 import * as cheerio from 'cheerio'
@@ -9,36 +8,32 @@ import extractAssignmentsFromCheerio from './extractAssignmentsFromCheerio'
  * @throws {InvalidCookieError}
  */
 export default async function scrapeAssignmentsOfPage(
-  mcvID: number,
   next: number
-): Promise<option.Option<Array<Assignment>>> {
-  const optionalResponse = await fetchAndCatch(
+): Promise<[boolean, number, Array<Assignment>] | undefined> {
+  const response = await fetchAndCatch(
     `https://www.mycourseville.com/?q=courseville/ajax/loadmoreassignmentrows`,
     'POST',
     new URLSearchParams({
-      cv_cid: mcvID.toString(),
       next: next.toString(),
     })
   )
-  if (option.isNone(optionalResponse)) return option.none
-  const response = optionalResponse.value
+  if (response == undefined) return undefined
   const resultJson: LoadMoreAssignmentsResponse = await response?.json()
 
   if (resultJson.status == 0) {
-    return option.none
+    return undefined
   }
 
   const $ = cheerio.load(
     '<html><table><tbody>' + resultJson.data.html + '</tbody></table></html>'
   )
 
-  let assignments = await extractAssignmentsFromCheerio(mcvID, $)
-
-  if (resultJson.all == undefined || resultJson.all !== true) {
-    const optionalResult = await scrapeAssignmentsOfPage(mcvID, next + 5)
-    if (option.isSome(optionalResult)) {
-      assignments = assignments.concat(optionalResult.value)
-    }
+  const courseAndAssignment = await extractAssignmentsFromCheerio($)
+  if (courseAndAssignment == undefined) {
+    return undefined
   }
-  return option.some(assignments)
+  const courseId = courseAndAssignment[0]
+  const assignments = courseAndAssignment[1]
+  let hasNext = resultJson.all == undefined || resultJson.all !== true
+  return [hasNext, courseId, assignments]
 }

src/scraper/updateAll.ts

@@ -6,9 +6,8 @@ import {
   NEW_ASSIGNMENTS_MESSAGE_SIZE,
 } from '@/config/config'
 import db, { Assignment, Course } from '../database/database'
-import updateAssignments from './updateAssignments'
+import updateAssignmentsOfCourse from './updateAssignmentsOfCourse'
 import updateCourses from './updateCourses'
-import * as option from 'fp-ts/Option'
 import MutableWrapper from '@/utils/MutableWrapper'
 import { format } from 'util'
 
@@ -20,17 +19,30 @@ import { format } from 'util'
 export async function updateAll(): Promise<Array<string>> {
   await updateCourses()
   const coursesList = await db.getAllCoursesOfTargetSemester()
-  const coursesWithAssignments: Map<Course, Array<Assignment>> = new Map()
+  // const coursesWithAssignments: Map<Course, Array<Assignment>> = new Map()
+  const mcvIdToCourse: Map<number, Course> = new Map()
+  for (let course of coursesList) {
+    mcvIdToCourse.set(course.mcvID, course)
+  }
+  let mcvIdToNewAssignments: Map<number, Array<Assignment>> = new Map()
   for await (const course of coursesList) {
-    const newAssignments: option.Option<Assignment[]> = await updateAssignments(
-      course.mcvID
-    )
-    if (option.isNone(newAssignments) || newAssignments.value.length == 0) {
+    const newAssignments = await updateAssignmentsOfCourse(course.mcvID)
+    if (newAssignments == undefined || newAssignments.size == 0) {
       continue
     }
-    coursesWithAssignments.set(course, newAssignments.value)
+    for (let [mcvId, assignments] of newAssignments) {
+      if (assignments.length == 0) {
+        continue
+      }
+      if (!mcvIdToNewAssignments.has(mcvId)) {
+        mcvIdToNewAssignments.set(mcvId, assignments)
+      } else {
+        mcvIdToNewAssignments.get(mcvId)!.concat(assignments)
+      }
+    }
   }
-  if (coursesWithAssignments.size == 0) {
+
+  if (mcvIdToNewAssignments.size == 0) {
     return []
   }
   const messages: string[] = []
@@ -40,9 +52,12 @@ export async function updateAll(): Promise<Array<string>> {
   const currentMessageSize: MutableWrapper<number> = new MutableWrapper(
     NEW_ASSIGNMENTS_MESSAGE_SIZE
   )
-  for (const [course, assignments] of coursesWithAssignments) {
-    // const newCourseLine = `\n- ${course.title}`
-    const newCourseLine = format(COURSE_MESSAGE_PATTERN, course.title)
+  for (const [mcvId, assignments] of mcvIdToNewAssignments.entries()) {
+    const courseInformation = mcvIdToCourse.get(mcvId)!
+    const newCourseLine = format(
+      COURSE_MESSAGE_PATTERN,
+      courseInformation.title
+    )
     const newAssignmentLineSize = [...newCourseLine].length
     const hasExceeded =
       currentMessageSize.value + newAssignmentLineSize >
@@ -56,7 +71,7 @@ export async function updateAll(): Promise<Array<string>> {
       const newAssignmentLine = format(
         ASSIGNMENT_MESSAGE_PATTERN,
         assignment.assignmentName,
-        course.mcvID,
+        mcvId,
         assignment.assignmentID
       )
       const newAssignmentLineSize = [...newAssignmentLine].length

src/scraper/updateAssignmentsOfCourse.ts

@@ -0,0 +1,59 @@
+import { Assignment } from '../database/database'
+import fetchAndCatch from '../utils/fetchAndCatch'
+import responseToCheerio from '../utils/responseToCheerio'
+import extractAssignmentsFromCheerio from './extractAssignmentsFromCheerio'
+import scrapeAssignmentsOfPage from './scrapeAssignmentsOfPage'
+
+/**
+ * @throws {InvalidCookieError}
+ */
+export default async function updateAssignmentsOfCourse(
+  mcvID: number
+): Promise<Map<number, Array<Assignment>> | undefined> {
+  const result = await fetchAndCatch(
+    `https://www.mycourseville.com/?q=courseville/course/${mcvID}/assignment`,
+    'GET'
+  )
+  const cheerioRootResponse = await responseToCheerio(result)
+  if (cheerioRootResponse == undefined) {
+    return undefined
+  }
+  const mergedNewAssignments: Map<number, Array<Assignment>> = new Map()
+
+  let foundCourseIdAndAssignments =
+    await extractAssignmentsFromCheerio(cheerioRootResponse)
+
+  if (foundCourseIdAndAssignments == undefined) {
+    return undefined
+  }
+  const [foundMcvId, assignments] = foundCourseIdAndAssignments
+  if (assignments.length != 0) {
+    mergedNewAssignments.set(foundMcvId, assignments)
+  }
+
+  let hasNext = true
+  for (let currentAssignmentItems = 5; hasNext; currentAssignmentItems += 5) {
+    const scrapeResult = await scrapeAssignmentsOfPage(currentAssignmentItems)
+
+    if (scrapeResult == undefined) {
+      break
+    }
+
+    const [resultHasNext, resultMcvId, resultAssignments] = scrapeResult
+
+    hasNext = resultHasNext
+    if (resultAssignments.length == 0) {
+      continue
+    }
+    if (!mergedNewAssignments.has(resultMcvId)) {
+      mergedNewAssignments.set(resultMcvId, resultAssignments)
+    } else {
+      const found = mergedNewAssignments.get(resultMcvId)!
+      resultAssignments.forEach(function (v) {
+        found.push(v)
+      }, found)
+    }
+  }
+
+  return mergedNewAssignments
+}

src/scraper/updateCourses.ts

@@ -3,7 +3,6 @@ import { targetYear, targetSemester } from '../config/config'
 import * as cheerio from 'cheerio'
 import db, { Course } from '../database/database'
 import fetchAndCatch from '../utils/fetchAndCatch'
-import { option } from 'fp-ts'
 import { determineYearAndSemester } from './determineYearAndSemester'
 import responseToCheerio from '../utils/responseToCheerio'
 
@@ -12,12 +11,12 @@ import responseToCheerio from '../utils/responseToCheerio'
  */
 export default async function updateCourses(): Promise<void> {
   const result = await fetchAndCatch(`https://www.mycourseville.com/`, 'GET')
-  const optionalCheerioRoot = await responseToCheerio(result)
+  const cheerioRootResponse = await responseToCheerio(result)
 
-  if (option.isNone(optionalCheerioRoot)) {
+  if (cheerioRootResponse == undefined) {
     return
   }
-  const $ = optionalCheerioRoot.value
+  const $ = cheerioRootResponse
 
   if (env.AUTO_DETERMINE_YEAR_AND_SEMESTER) {
     determineYearAndSemester($)

src/server.ts

@@ -8,7 +8,6 @@ import formatDateToBangkok from './utils/formatDateToBangkok'
 import updateHandler from './utils/updateHandler'
 import MutableWrapper from './utils/MutableWrapper'
 import DiscordCommandHandler from './interfaces/DiscordCommandHandler'
-import { isSome } from 'fp-ts/lib/Option'
 
 export const hasEncounteredError: MutableWrapper<boolean> = new MutableWrapper(
   false
@@ -60,7 +59,7 @@ client.on('ready', async () => {
   adminDM = await client.users.createDM(env.ADMIN_USER_ID)
   adminDM.send('server is up!')
 
-  if (isSome(await updateHandler())) {
+  if ((await updateHandler()) != undefined) {
     intervalId.value = setInterval(updateHandler, env.DELAY * 1000)
   }
 })

src/utils/fetchAndCatch.ts

@@ -1,4 +1,3 @@
-import { option } from 'fp-ts'
 import env from '../env/env'
 import { hasEncounteredError } from '../server'
 import errorFetchingNotify from './errorFetchingNotify'
@@ -11,7 +10,7 @@ export default async function fetchAndCatch(
   url: string,
   method: string,
   body?: BodyInit
-): Promise<option.Option<Response>> {
+): Promise<Response | undefined> {
   const response = await fetch(url, {
     method: method,
     headers: {
@@ -26,16 +25,16 @@ export default async function fetchAndCatch(
     await errorFetchingNotify(NotifyMessage.FetchingError)
   })
   if (response == undefined) {
-    return option.none
+    return undefined
   }
   if (response.status != 200) {
     if (hasEncounteredError.value) {
-      return option.none
+      return undefined
     }
     hasEncounteredError.value = true
     await errorFetchingNotify(NotifyMessage.FetchingError)
-    return option.none
+    return undefined
   }
   hasEncounteredError.value = false
-  return option.some(response)
+  return response
 }