feat(login): 🚧 Update friend & thread list WIP

Since Facebook changed some things, we had to adapt this API. Now all data to thread list and friend list comes from main HTML response parsed and loaded. Additional work has to be made (e.g. formatters). Mention #3

index.ts

@@ -5,6 +5,9 @@ import Jar from './lib/jar';
 import cheerio from 'cheerio';
 import Api from './lib/api';
 import { Response } from 'got';
+import { UserInfo } from './lib/types/users';
+import { ThreadInfo } from './lib/types/threads';
+import { formatSingleFriend, formatSingleThread } from './lib/formatting/listFormatters';
 
 const defaultLogRecordSize = 100;
 
@@ -31,8 +34,7 @@ export default async function login(loginData: LoginCredentials, options: ApiOpt
 		autoMarkDelivery: true,
 		autoMarkRead: false,
 		logRecordSize: defaultLogRecordSize,
-		userAgent:
-			'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/600.3.18 (KHTML, like Gecko) Version/8.0.3 Safari/600.3.18'
+		userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0'
 	};
 	setOptions(globalOptions, options);
 
@@ -105,7 +107,7 @@ async function loginHelper(credentials: LoginCredentials, globalOptions: ApiOpti
 		// Open the main page, then we login with the given credentials and finally
 		// load the main page again (it'll give us some IDs that we need)
 		mainPromise = utils
-			.get('https://m.facebook.com/', null, null, globalOptions)
+			.get('https://www.facebook.com/', null, null, globalOptions)
 			.then(utils.saveCookies(jar))
 			.then(makeLogin(jar, credentials.email, credentials.password, globalOptions));
 	} else throw new Error('Argument error: you must specify AppState or email-password credentials');
@@ -171,7 +173,6 @@ async function loginHelper(credentials: LoginCredentials, globalOptions: ApiOpti
 
 function buildAPI(globalOptions: ApiOptions, html: string, jar: Jar) {
 	const userIdCookies = jar.getCookies('https://www.facebook.com').filter(cookie => cookie.key === 'c_user');
-
 	if (userIdCookies.length === 0) {
 		throw new Error(
 			'Error retrieving userID. This can be caused by a lot of things, including having wrong AppState credentials or getting blocked by Facebook for logging in from an unknown location. Try logging in with a browser to verify.'
@@ -203,6 +204,11 @@ function buildAPI(globalOptions: ApiOptions, html: string, jar: Jar) {
 	};
 	const defaultFuncs: Dfs = utils.makeDefaults(html, userID, ctx);
 	const api = new Api(defaultFuncs, ctx);
+
+	// find these lists from the html response
+	api.friendList = findFriendList(html);
+	api.threadList = findThreadList(html);
+
 	return { ctx, defaultFuncs, api };
 }
 
@@ -365,3 +371,36 @@ function makeLogin(jar: Jar, email: string, password: string, loginOptions: ApiO
 		});
 	};
 }
+
+/** Finds & parses a friends list from a whole HTML response. */
+function findFriendList(html: string): UserInfo[] {
+	// search pattern for friend list
+	const pattern = /"?result"?:\s*\{\s*"?data"?:\s*\{\s*"?viewer"?:\s*\{\s*"?bootstrap_keywords"?:\s*\{\s*"?edges"?:\s*/;
+	const res = pattern.exec(html);
+	if (!res) {
+		log.warn('login', 'Failed to load a friend list. Please contact the dev team about this (error code L-09)');
+		return [];
+	}
+	// parse object from JSON: (we don't where a JSON object ends)
+	const parsed = utils.json5parseTillEnd(html.substring(res.index + res[0].length));
+	// parse loaded friend objects to ts-messenger-api's more user-friendly objects:
+	return parsed.map(formatSingleFriend);
+}
+
+function findThreadList(html: string): ThreadInfo[] {
+	// Facebook is perfect. Why? It sends the thread list twice in a single html file.
+	// And what is even better than perfect? The 1st list contains 10 threads & the 2nd contains 20.
+
+	// search pattern for thread list
+	const pattern = /"?message_threads"?:\s*\{\s*"?edges"?:\s*/g;
+	const results = utils.findAllOccurrencesRegex(pattern, html);
+	if (!results.length) {
+		log.warn('login', 'Failed to load a thread list. Please contact the dev team about this (error code L-10)');
+		return [];
+	}
+	const originalThreadList = results
+		.map(res => utils.json5parseTillEnd(html.substring(res.index + res[0].length))) // first, parse all the JSON data
+		.reduce((prev, curr) => (prev.length > curr.length ? prev : curr)); // choose the one list with the most elements
+	// parse loaded thread objects to ts-messenger-api's more user-friendly objects:
+	return originalThreadList.map(formatSingleThread);
+}

lib/api.ts

@@ -49,6 +49,14 @@ export default class Api {
 	private chatOn = true;
 	private foreground = false;
 
+	/** List of user's Facebook friends. Contains the all-available data of every friend.
+	 * This list does not change over time and is assigned after login. */
+	friendList: UserInfo[] = [];
+
+	/** List of user's last threads (text channels). Contains the all-available data of every thread.
+	 * This list does not change over time and is assigned after login. */
+	threadList: ThreadInfo[] = [];
+
 	/** @internal */
 	constructor(defaultFuncs: Dfs, ctx: ApiCtx) {
 		this.ctx = ctx;

lib/formatting/listFormatters.ts

@@ -0,0 +1,15 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
+
+import { ThreadInfo } from '../types/threads';
+import { UserInfo } from '../types/users';
+
+export function formatSingleFriend(originalFriend: any): UserInfo {
+	// TODO: CRITICAL: finish this
+	return originalFriend;
+}
+
+export function formatSingleThread(originalThread: any): ThreadInfo {
+	// TODO: CRITICAL: finish this
+	return originalThread;
+}

lib/utils.ts

@@ -4,6 +4,7 @@ import stream from 'stream';
 import log from 'npmlog';
 import { Cookie } from 'tough-cookie';
 import FormData from 'form-data';
+import JSON5 from 'json5';
 
 import got, { Response } from 'got';
 const gotInstance = got.extend({
@@ -16,10 +17,9 @@ function getHeaders(options: ApiOptions): Record<string, string> {
 	return {
 		'Content-Type': 'application/x-www-form-urlencoded',
 		Referer: 'https://www.facebook.com/',
-		Origin: 'https://www.facebook.com',
-		'User-Agent':
-			options.userAgent ||
-			'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/600.3.18 (KHTML, like Gecko) Version/8.0.3 Safari/600.3.18'
+		Origin: 'www.facebook.com',
+		Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+		'User-Agent': options.userAgent || 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0'
 	};
 }
 
@@ -577,3 +577,40 @@ export function mentionsGetOffsetRecursive(text: string, searchForValues: string
 		);
 	}
 }
+
+/** Parses JSON data in the beginning of specified text parameter.
+ * We sometimes need to parse JSON data from inside an HTML file and we don't know
+ * where the data ends (we can tell only from {} and [] parentheses & assuming there is no new-line character in JSON).
+ * @param text a string starting with the valuable information; can continue with additional characters */
+export function json5parseTillEnd(text: string): any {
+	let parsed: any;
+	try {
+		JSON5.parse(text);
+	} catch (err) {
+		console.error(err);
+		parsed = JSON5.parse(text.substr(0, err.columnNumber - 1));
+	}
+	return parsed;
+}
+
+/** Returns all occurrences of RegEx search in given string. */
+export function findAllOccurrencesRegex(pattern: RegExp, text: string): RegExpExecArray[] {
+	// since I do not sufficiently know how to perform RegEx search, I wrote this help function (Filip, 22/5/2021)
+
+	// if we don't have the "global" RegEx flag, perform the search only once
+	// (otherwise it would continue forever, I've tested that)
+	if (!pattern.flags.includes('g')) {
+		const res = pattern.exec(text);
+		return res ? [pattern.exec(text) as RegExpExecArray] : [];
+	}
+
+	// find all existing occurrences
+	const array: RegExpExecArray[] = [];
+	// eslint-disable-next-line no-constant-condition
+	while (true) {
+		console.log('x');
+		const res = pattern.exec(text);
+		if (res) array.push(res);
+		else return array;
+	}
+}

package.json

@@ -31,6 +31,7 @@
 		"events": "^3.2.0",
 		"form-data": "^4.0.0",
 		"got": "^11.8.1",
+		"json5": "^2.2.0",
 		"mqtt": "^4.2.6",
 		"node-emoji": "^1.10.0",
 		"npmlog": "^4.1.2",