Syncing across devices with Github Gists

Readme.md

@@ -186,6 +186,7 @@ In the extension's `options` (right click on the icon or in the popup's menu) yo
 * **Source filtering**: filter out some paper sources you don't want to record papers from
 * **Custom title function**: provide Javascript code to generate your own web page titles and pdf filenames based on a paper's attributes
 * **Data management**: export/load your memory data and export the bibliography as a `.bib` file
+* **Online Synchronization**: use Github Gists to sync your papers across devices
 
 <p align="center">
   <img src="https://raw.github.com/vict0rsch/PaperMemory/master/extra/imgs/opt_concat.png?raw=true">

gulpfile.js

@@ -39,6 +39,8 @@ function utilsJS() {
             "src/shared/js/utils/levenshtein.js",
             "src/shared/js/utils/bibtexParser.js",
             "src/shared/js/utils/functions.js",
+            "src/shared/js/utils/gist.js",
+            "src/shared/js/utils/sync.js",
             "src/shared/js/utils/data.js",
             "src/shared/js/utils/paper.js",
             "src/shared/js/utils/state.js",

manifest.json

@@ -63,6 +63,8 @@
         "src/shared/js/utils/miniquery.js",
         "src/shared/js/utils/config.js",
         "src/shared/js/utils/functions.js",
+        "src/shared/js/utils/gist.js",
+        "src/shared/js/utils/sync.js",
         "src/shared/js/utils/data.js",
         "src/shared/js/utils/paper.js",
         "src/shared/js/utils/state.js",

package.json

@@ -8,9 +8,10 @@
     "test-cov": "gulp build && cd test/ && ../node_modules/.bin/nyc --reporter=text ../node_modules/.bin/mocha test-*.js",
     "test-storage": "gulp build && cd test/ && ../node_modules/.bin/mocha test-storage.js",
     "test-duplicates": "gulp build && cd test/ && ../node_modules/.bin/mocha test-duplicates.js",
-    "test-no-browser": "gulp build && cd test/ && ../node_modules/.bin/mocha 'test-!(storage|duplicates)*.js'",
-    "test-no-browser-no-build": "cd test/ && ../node_modules/.bin/mocha 'test-!(storage)*.js'",
-    "test-no-browser-no-build-cov": "cd test/ && ../node_modules/.bin/nyc --reporter=text ../node_modules/.bin/mocha 'test-!(storage)*.js'",
+    "test-sync": "gulp build && cd test/ && ../node_modules/.bin/mocha test-sync.js",
+    "test-no-browser": "gulp build && cd test/ && ../node_modules/.bin/mocha 'test-!(storage|duplicates|sync)*.js'",
+    "test-no-browser-no-build": "cd test/ && ../node_modules/.bin/mocha 'test-!(storage|duplicates|sync)*.js'",
+    "test-no-browser-no-build-cov": "cd test/ && ../node_modules/.bin/nyc --reporter=text ../node_modules/.bin/mocha 'test-!(storage|duplicates|sync)*.js'",
     "screenshots": "node_modules/.bin/mocha test/screenshots.js"
   },
   "repository": {
@@ -41,11 +42,12 @@
     "jsdom": "^20.0.0",
     "mocha": "^10.0.0",
     "nyc": "^15.1.0",
+    "puppeteer": "^13.5.2",
     "puppeteer-to-istanbul": "^1.4.0",
     "readline-sync": "^1.4.10",
     "uuid": "^9.0.0"
   },
   "dependencies": {
-    "puppeteer": "^18.0.2"
+    "ora": "5.4.1"
   }
-}
+}

src/background/background.js

@@ -2,6 +2,56 @@ var paperTitles = {};
 var MAX_TITLE_UPDATES = 100;
 var tabStatuses = {};
 
+const badgeOk = () => {
+    chrome.browserAction.setBadgeText({ text: "OK!" });
+    chrome.browserAction.setBadgeBackgroundColor({ color: "rgb(68, 164, 68)" });
+};
+
+const badgeWait = (text) => {
+    chrome.browserAction.setBadgeText({ text });
+    chrome.browserAction.setBadgeBackgroundColor({ color: "rgb(189, 127, 10)" });
+};
+
+const badgeError = () => {
+    chrome.browserAction.setBadgeText({ text: "Error" });
+    chrome.browserAction.setBadgeBackgroundColor({ color: "rgb(195, 40, 56)" });
+};
+
+const badgeClear = (preventTimeout = false) => {
+    if (preventTimeout) {
+        chrome.browserAction.setBadgeText({ text: "" });
+    } else {
+        setTimeout(() => {
+            chrome.browserAction.setBadgeText({ text: "" });
+        }, 2000);
+    }
+};
+
+const initGist = async () => {
+    if (!(await shouldSync())) {
+        warn("Sync disabled.");
+        return;
+    }
+    const start = Date.now();
+    log("Initializing Sync...");
+    badgeWait("Init...");
+    const { ok, error, payload } = await getGist();
+    if (ok) {
+        global.state.gist = payload.gist;
+        global.state.gistDataFile = await getDataFile(global.state.gist);
+        await global.state.gistDataFile.fetchLatest();
+        const duration = (Date.now() - start) / 1e3;
+        logOk(`Sync successfully enabled (${duration}s).`);
+        badgeOk();
+    } else {
+        logError("[initGist]", error);
+        badgeError();
+    }
+    badgeClear();
+};
+
+initGist();
+
 const setFaviconCode = `
 var link;
 if (window.location.href.startsWith("file://")){
@@ -163,6 +213,53 @@ const findCodesForPaper = async (request) => {
     return { ...codes[0], ...code };
 };
 
+const pullSyncPapers = async () => {
+    if (!(await shouldSync())) return;
+    try {
+        badgeWait("Pull...");
+        const start = Date.now();
+        consoleHeader(`Pulling ${String.fromCodePoint("0x23EC")}`);
+        log("Pulling from Github...");
+        await global.state.gistDataFile.fetchLatest();
+        const remotePapers = global.state.gistDataFile.content;
+        log("Pulled papers:", remotePapers);
+        const duration = (Date.now() - start) / 1e3;
+        info(`Pulling from Github... Done (${duration}s)!`);
+        console.groupEnd();
+        badgeOk();
+        badgeClear();
+        return remotePapers;
+    } catch (e) {
+        logError("[pullSyncPapers]", e);
+        badgeError();
+    }
+    badgeClear();
+    console.groupEnd();
+};
+
+const pushSyncPapers = async () => {
+    if (!(await shouldSync())) return;
+    try {
+        const start = Date.now();
+        consoleHeader(`Pushing ${String.fromCodePoint("0x23EB")}`);
+        log("Writing to Github...");
+        badgeWait("Push...");
+        chrome.browserAction.setBadgeBackgroundColor({ color: "rgb(189, 127, 10)" });
+        const papers = (await getStorage("papers")) ?? {};
+        log("Papers to write: ", papers);
+        await global.state.gistDataFile.overwrite(JSON.stringify(papers, null, ""));
+        await global.state.gistDataFile.save();
+        const duration = (Date.now() - start) / 1e3;
+        log(`Writing to Github... Done (${duration}s)!`);
+        badgeOk();
+    } catch (e) {
+        logError("[pushSyncPapers]", e);
+        badgeError();
+    }
+    badgeClear();
+    console.groupEnd();
+};
+
 chrome.runtime.onMessage.addListener((payload, sender, sendResponse) => {
     if (payload.type === "update-title") {
         const { title, url } = payload.options;
@@ -190,6 +287,12 @@ chrome.runtime.onMessage.addListener((payload, sender, sendResponse) => {
         });
     } else if (payload.type === "hello") {
         sendResponse("Connection to background script established.");
+    } else if (payload.type === "writeSync") {
+        pushSyncPapers().then(sendResponse);
+    } else if (payload.type === "pullSync") {
+        pullSyncPapers(payload.gist).then(sendResponse);
+    } else if (payload.type === "restartGist") {
+        initGist().then(sendResponse);
     }
     return true;
 });
@@ -260,3 +363,13 @@ chrome.commands.onCommand.addListener((command) => {
         });
     }
 });
+
+chrome.runtime.onConnect.addListener(function (port) {
+    if (port.name === "PaperMemoryPopupSync") {
+        log("[chrome.runtime.onConnect] Popup connected.");
+        port.onDisconnect.addListener(async function () {
+            log("[chrome.runtime.onConnect] Popup disconnected.");
+            await pushSyncPapers();
+        });
+    }
+});

src/content_scripts/content_script.css

@@ -14,6 +14,7 @@
     justify-content: center;
     align-items: center;
     justify-content: space-between;
+    margin-top: 24px;
 }
 
 .pm-sub-header {