file-server sync: keep backups of remote sqlite3 file

src/packages/file-server/zfs/config.ts

@@ -15,6 +15,10 @@ const FILESYSTEMS = join(DATA, "filesystems");
 // Directory on server where zfs send streams (and tar?) are stored
 const ARCHIVES = join(DATA, "archives");
 
+// Directory to store data used in pulling as part of sync.
+// E.g., this keeps around copies of the sqlite state database of each remote.
+const PULL = join(DATA, "pull");
+
 // Directory for bup
 const BUP = join(DATA, "bup");
 
@@ -25,12 +29,13 @@ export const context = {
   SQLITE3_DATABASE_FILE,
   FILESYSTEMS,
   ARCHIVES,
+  PULL,
   BUP,
 };
 
 // WARNING: this "setContext" is global. It's very useful for **UNIT TESTING**, but
 // for any other use, you want to set this at most once and never again!!!  The reason
-// is because with nodejs you could have async code running all over the place, and 
+// is because with nodejs you could have async code running all over the place, and
 // changing the context out from under it would lead to nonsense and corruption.
 export function setContext({
   namespace,
@@ -45,6 +50,7 @@ export function setContext({
   context.SQLITE3_DATABASE_FILE = databaseFilename(context.DATA);
   context.FILESYSTEMS = join(context.DATA, "filesystems");
   context.ARCHIVES = join(context.DATA, "archives");
+  context.PULL = join(context.DATA, "pull");
   context.BUP = join(context.DATA, "bup");
 }
 

src/packages/file-server/zfs/pull.ts

@@ -28,9 +28,14 @@ import { context } from "./config";
 import { archiveFilesystem, dearchiveFilesystem } from "./archive";
 import { deleteSnapshot } from "./snapshots";
 import { isEqual } from "lodash";
+import { join } from "path";
+import { readdir, unlink } from "fs/promises";
 
 const logger = getLogger("file-server:zfs:pull");
 
+// number of remote backups of db sqlite file to keep.
+const NUM_DB_TO_KEEP = 10;
+
 // This is used for unit testing. It's what fields should match
 // after doing a sync, except snapshots where local is a superset,
 // unless you pull with deleteSnapshots set to true.
@@ -96,7 +101,20 @@ export async function pull({
     cutoff = new Date(Date.now() - 1000 * 60 * 60 * 24 * 7);
   }
   logger.debug("pull: get the remote sqlite database");
-  const remoteDatabase = `${context.DATA}/remote.sqlite3`;
+  await exec({ command: "mkdir", args: ["-p", context.PULL] });
+  const remoteDatabase = join(
+    context.PULL,
+    `${remote}:${prefix}---${new Date().toISOString()}.sqlite3`,
+  );
+  // delete all but the most recent remote database files for this remote/prefix (?).
+  const oldDbFiles = (await readdir(context.PULL))
+    .sort()
+    .filter((x) => x.startsWith(`${remote}:${prefix}---`))
+    .slice(0, -NUM_DB_TO_KEEP);
+  for (const path of oldDbFiles) {
+    await unlink(join(context.PULL, path));
+  }
+
   await exec({
     command: "scp",
     args: [`${remote}:/${databaseFilename(prefix)}`, remoteDatabase],