From d88d7e8a66b1d5104078a0f45ca7af5bfa533bec Mon Sep 17 00:00:00 2001 From: William Hilton Date: Mon, 26 Oct 2020 18:05:46 -0400 Subject: [PATCH 01/43] wip --- package-lock.json | 388 ++++++++++++++++++++++++++++++- package.json | 8 +- src/IdbBackend.js | 6 +- src/PromisifiedFS.js | 22 +- src/YjsBackend.js | 233 +++++++++++++++++++ src/__tests__/YFS.spec.js | 476 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 1112 insertions(+), 21 deletions(-) create mode 100644 src/YjsBackend.js create mode 100755 src/__tests__/YFS.spec.js diff --git a/package-lock.json b/package-lock.json index 26e033b..944604e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -679,6 +679,12 @@ "integrity": "sha512-FZdkNBDqBRHKQ2MEbSC17xnPFOhZxeJ2YGSfr2BKf3sujG49Qe3bB+rGCwQfIaA7WHnGeGkSijX4FuBCdrzW/g==", "dev": true }, + "@zeit/schemas": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@zeit/schemas/-/schemas-2.6.0.tgz", + "integrity": "sha512-uUrgZ8AxS+Lio0fZKAipJjAh415JyrOZowliZAzmnJSsf7piVL5w+G0+gFJ0KSu3QRhvui/7zuvpLz03YjXAhg==", + "dev": true + }, "JSONStream": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", @@ -837,6 +843,15 @@ } } }, + "ansi-align": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", + "integrity": "sha1-w2rsy6VjuJzrVW82kPCx2eNUf38=", + "dev": true, + "requires": { + "string-width": "^2.0.0" + } + }, "ansi-colors": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.4.tgz", @@ -886,6 +901,12 @@ "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "dev": true }, + "arch": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/arch/-/arch-2.1.2.tgz", + "integrity": "sha512-NTBIIbAfkJeIletyABbVtdPgeKfDafR+1mZV/AyyfC1UkVkp9iUjV+wwmqtUgphHYajbI86jejBJp5e+jkGTiQ==", + "dev": true + }, "archiver": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/archiver/-/archiver-2.1.1.tgz", @@ -916,6 +937,12 @@ "readable-stream": "^2.0.0" } }, + "arg": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/arg/-/arg-2.0.0.tgz", + "integrity": "sha512-XxNTUzKnz1ctK3ZIcI2XUPlD96wbHP2nGqkPKpvk/HNRlPveYrXIVSTk9m3LcqOgDPg3B1nMvdV/K8wZd7PG4w==", + "dev": true + }, "argv-formatter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/argv-formatter/-/argv-formatter-1.0.0.tgz", @@ -1317,6 +1344,29 @@ "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==", "dev": true }, + "boxen": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", + "integrity": "sha512-TNPjfTr432qx7yOjQyaXm3dSR0MH9vXp7eT1BFSl/C51g+EFnOR9hTg1IreahGBmDNCehscshe45f+C1TBZbLw==", + "dev": true, + "requires": { + "ansi-align": "^2.0.0", + "camelcase": "^4.0.0", + "chalk": "^2.0.1", + "cli-boxes": "^1.0.0", + "string-width": "^2.0.0", + "term-size": "^1.2.0", + "widest-line": "^2.0.0" + }, + "dependencies": { + "camelcase": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", + "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=", + "dev": true + } + } + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -1788,6 +1838,12 @@ "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", "dev": true }, + "cli-boxes": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", + "integrity": "sha1-T6kXw+WclKAEzWH47lCdplFocUM=", + "dev": true + }, "cli-table": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.1.tgz", @@ -1799,12 +1855,56 @@ "dependencies": { "colors": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true } } }, + "clipboardy": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/clipboardy/-/clipboardy-1.2.3.tgz", + "integrity": "sha512-2WNImOvCRe6r63Gk9pShfkwXsVtKCroMAevIbiae021mS850UkWPbevxsBz3tnvjZIEGvlwaqCPsw+4ulzNgJA==", + "dev": true, + "requires": { + "arch": "^2.1.0", + "execa": "^0.8.0" + }, + "dependencies": { + "cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", + "dev": true, + "requires": { + "lru-cache": "^4.0.1", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "execa": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.8.0.tgz", + "integrity": "sha1-2NdrvBtVIX7RkP1t1J08d07PyNo=", + "dev": true, + "requires": { + "cross-spawn": "^5.0.1", + "get-stream": "^3.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + } + }, + "get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", + "dev": true + } + } + }, "cliui": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz", @@ -1941,6 +2041,46 @@ "readable-stream": "^2.0.0" } }, + "compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dev": true, + "requires": { + "mime-db": ">= 1.43.0 < 2" + }, + "dependencies": { + "mime-db": { + "version": "1.45.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.45.0.tgz", + "integrity": "sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w==", + "dev": true + } + } + }, + "compression": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.3.tgz", + "integrity": "sha512-HSjyBG5N1Nnz7tF2+O7A9XUhyjru71/fwgNb7oIsEVHR0WShfs2tIS/EySLgiTe98aOK18YDlMXpzjCXY/n9mg==", + "dev": true, + "requires": { + "accepts": "~1.3.5", + "bytes": "3.0.0", + "compressible": "~2.0.14", + "debug": "2.6.9", + "on-headers": "~1.0.1", + "safe-buffer": "5.1.2", + "vary": "~1.1.2" + }, + "dependencies": { + "bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=", + "dev": true + } + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1986,6 +2126,12 @@ "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=", "dev": true }, + "content-disposition": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", + "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=", + "dev": true + }, "content-type": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", @@ -3207,6 +3353,23 @@ "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz", "integrity": "sha512-R9bHCvweUxxwkDwhjav5vxpFvdPGlVngtqmx4pIZfSUhM/Q4NiIUHB456BAf+Q1Nwu3HEZYONtu+Rya+af4jiQ==" }, + "fast-url-parser": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/fast-url-parser/-/fast-url-parser-1.1.3.tgz", + "integrity": "sha1-9K8+qfNNiicc9YrSs3WfQx8LMY0=", + "dev": true, + "requires": { + "punycode": "^1.3.2" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", + "dev": true + } + } + }, "fastq": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz", @@ -4110,7 +4273,7 @@ "dependencies": { "split2": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/split2/-/split2-1.0.0.tgz", "integrity": "sha1-UuLiIdiMdfmnP5BVbiY/+WdysxQ=", "dev": true, "requires": { @@ -4810,7 +4973,7 @@ }, "is-obj": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=", "dev": true }, @@ -4910,6 +5073,11 @@ "fast-text-encoding": "^1.0.0" } }, + "isomorphic.js": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.1.5.tgz", + "integrity": "sha512-MkX5lLQApx/8IAIU31PKvpAZosnu2Jqcj1rM8TzxyA4CR96tv3SgMKQNTCxL58G7696Q57zd7ubHV/hTg+5fNA==" + }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -5243,6 +5411,14 @@ "type-check": "~0.3.2" } }, + "lib0": { + "version": "0.2.34", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.34.tgz", + "integrity": "sha512-cqsVIMPgFlDtgQcpkt7HOY6W3sbYPIe3qxMnbRSwHTgiQancgm+TRDPx28mC6GUZ6lG6Nr0bIWf4Nog6dWUNUg==", + "requires": { + "isomorphic.js": "^0.1.3" + } + }, "libbase64": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-0.1.0.tgz", @@ -5674,7 +5850,6 @@ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", "dev": true, - "optional": true, "requires": { "pseudomap": "^1.0.2", "yallist": "^2.1.2" @@ -9972,6 +10147,12 @@ "ee-first": "1.1.1" } }, + "on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "dev": true + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -10346,6 +10527,12 @@ "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, + "path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=", + "dev": true + }, "path-key": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", @@ -10377,6 +10564,12 @@ } } }, + "path-to-regexp": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.2.1.tgz", + "integrity": "sha512-gu9bD6Ta5bwGrrU8muHzVOBFFREpp2iRkVfhBJahwJ6p6Xw20SjT0MxLnwkjOibQmGSYhiUnf2FLe7k+jcFmGQ==", + "dev": true + }, "path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -10624,8 +10817,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", - "dev": true, - "optional": true + "dev": true }, "psl": { "version": "1.1.31", @@ -11000,6 +11192,15 @@ "rc": "^1.2.8" } }, + "registry-url": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", + "integrity": "sha1-PU74cPc93h138M+aOBQyRE4XSUI=", + "dev": true, + "requires": { + "rc": "^1.0.1" + } + }, "remove-trailing-separator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", @@ -11532,6 +11733,82 @@ "integrity": "sha512-AQxrNqu4EXWt03dJdgKXI+Au9+pvEuM5+Nk5g6+TmuxMCkEL03VhZ31HM+VKeaaZbFpDHaoSruiHq4PW9AIrOQ==", "dev": true }, + "serve": { + "version": "11.3.2", + "resolved": "https://registry.npmjs.org/serve/-/serve-11.3.2.tgz", + "integrity": "sha512-yKWQfI3xbj/f7X1lTBg91fXBP0FqjJ4TEi+ilES5yzH0iKJpN5LjNb1YzIfQg9Rqn4ECUS2SOf2+Kmepogoa5w==", + "dev": true, + "requires": { + "@zeit/schemas": "2.6.0", + "ajv": "6.5.3", + "arg": "2.0.0", + "boxen": "1.3.0", + "chalk": "2.4.1", + "clipboardy": "1.2.3", + "compression": "1.7.3", + "serve-handler": "6.1.3", + "update-check": "1.5.2" + }, + "dependencies": { + "ajv": { + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.5.3.tgz", + "integrity": "sha512-LqZ9wY+fx3UMiiPd741yB2pj3hhil+hQc8taf4o2QGRFpWgZ2V5C8HA165DY9sS3fJwsk7uT7ZlFEyC3Ig3lLg==", + "dev": true, + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + } + } + }, + "serve-handler": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/serve-handler/-/serve-handler-6.1.3.tgz", + "integrity": "sha512-FosMqFBNrLyeiIDvP1zgO6YoTzFYHxLDEIavhlmQ+knB2Z7l1t+kGLHkZIDN7UVWqQAmKI3D20A6F6jo3nDd4w==", + "dev": true, + "requires": { + "bytes": "3.0.0", + "content-disposition": "0.5.2", + "fast-url-parser": "1.1.3", + "mime-types": "2.1.18", + "minimatch": "3.0.4", + "path-is-inside": "1.0.2", + "path-to-regexp": "2.2.1", + "range-parser": "1.2.0" + }, + "dependencies": { + "bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=", + "dev": true + }, + "mime-db": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", + "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==", + "dev": true + }, + "mime-types": { + "version": "2.1.18", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", + "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==", + "dev": true, + "requires": { + "mime-db": "~1.33.0" + } + }, + "range-parser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", + "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=", + "dev": true + } + } + }, "set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", @@ -12290,6 +12567,49 @@ } } }, + "term-size": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz", + "integrity": "sha1-RYuDiH8oj8Vtb/+/rSYuJmOO+mk=", + "dev": true, + "requires": { + "execa": "^0.7.0" + }, + "dependencies": { + "cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", + "dev": true, + "requires": { + "lru-cache": "^4.0.1", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "execa": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", + "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", + "dev": true, + "requires": { + "cross-spawn": "^5.0.1", + "get-stream": "^3.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + } + }, + "get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", + "dev": true + } + } + }, "terser": { "version": "3.13.1", "resolved": "https://registry.npmjs.org/terser/-/terser-3.13.1.tgz", @@ -12718,6 +13038,28 @@ "integrity": "sha512-bzpH/oBhoS/QI/YtbkqCg6VEiPYjSZtrHQM6/QnJS6OL9pKUFLqb3aFh4Scvwm45+7iAgiMkLhSbaZxUqmrprw==", "dev": true }, + "update-check": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/update-check/-/update-check-1.5.2.tgz", + "integrity": "sha512-1TrmYLuLj/5ZovwUS7fFd1jMH3NnFDN1y1A8dboedIDt7zs/zJMo6TwwlhYKkSeEwzleeiSBV5/3c9ufAQWDaQ==", + "dev": true, + "requires": { + "registry-auth-token": "3.3.2", + "registry-url": "3.1.0" + }, + "dependencies": { + "registry-auth-token": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.3.2.tgz", + "integrity": "sha512-JL39c60XlzCVgNrO+qq68FoNb56w/m7JYvGR2jT5iR1xBrUA3Mfx5Twk5rqTThPmQKMWydGmq8oFtDlxfrmxnQ==", + "dev": true, + "requires": { + "rc": "^1.1.6", + "safe-buffer": "^5.0.1" + } + } + } + }, "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", @@ -12848,6 +13190,12 @@ "integrity": "sha1-a2GE2mUgzDIEzhtAfKwm2SYJ6/8=", "dev": true }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", + "dev": true + }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", @@ -13030,6 +13378,15 @@ "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", "dev": true }, + "widest-line": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.1.tgz", + "integrity": "sha512-Ba5m9/Fa4Xt9eb2ELXt77JxVDV8w7qQrH0zS/TWSJdLyAwQjWoOzpzj5lwVftDz6n/EOu3tNACS84v509qwnJA==", + "dev": true, + "requires": { + "string-width": "^2.1.1" + } + }, "windows-release": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/windows-release/-/windows-release-3.3.1.tgz", @@ -13183,6 +13540,14 @@ "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", "dev": true }, + "y-indexeddb": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/y-indexeddb/-/y-indexeddb-9.0.5.tgz", + "integrity": "sha512-40VxkqPoK2VxE1vMosS5MfwlHQOvaeLEN89dIkjh7URjZny6bDQOl4yKldaDv9ZosZgYEPyWuWTF3Z92RZ1y+A==", + "requires": { + "lib0": "^0.2.12" + } + }, "y18n": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", @@ -13193,8 +13558,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", - "dev": true, - "optional": true + "dev": true }, "yaml": { "version": "1.10.0", @@ -13255,6 +13619,14 @@ "integrity": "sha1-AI4G2AlDIMNy28L47XagymyKxBk=", "dev": true }, + "yjs": { + "version": "13.4.1", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.4.1.tgz", + "integrity": "sha512-kIh0sprCTzIm2qyr1VsovkvjKzD2GR4WcU/McJpLAEvImCJHA78Q3S6uSLnhZX0i7FQdrLPCRT8DtTPEH73jnw==", + "requires": { + "lib0": "^0.2.33" + } + }, "zip-stream": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-1.2.0.tgz", diff --git a/package.json b/package.json index be27fce..36b6bc4 100644 --- a/package.json +++ b/package.json @@ -13,13 +13,16 @@ "scripts": { "test": "karma start --single-run", "build": "webpack", - "semantic-release": "semantic-release" + "semantic-release": "semantic-release", + "serve": "serve" }, "dependencies": { "@isomorphic-git/idb-keyval": "3.3.2", "isomorphic-textencoder": "1.0.1", "just-debounce-it": "1.1.0", - "just-once": "1.1.0" + "just-once": "1.1.0", + "y-indexeddb": "^9.0.5", + "yjs": "^13.4.1" }, "devDependencies": { "karma": "2.0.5", @@ -38,6 +41,7 @@ "prettier": "^1.15.3", "puppeteer": "^1.10.0", "semantic-release": "16.0.3", + "serve": "^11.3.2", "webpack": "^4.28.2", "webpack-cli": "^3.1.2" }, diff --git a/src/IdbBackend.js b/src/IdbBackend.js index c7af775..2b6db36 100644 --- a/src/IdbBackend.js +++ b/src/IdbBackend.js @@ -12,13 +12,13 @@ module.exports = class IdbBackend { loadSuperblock() { return idb.get("!root", this._store); } - readFile(inode) { + readFileInode(inode) { return idb.get(inode, this._store) } - writeFile(inode, data) { + writeFileInode(inode, data) { return idb.set(inode, data, this._store) } - unlink(inode) { + unlinkInode(inode) { return idb.del(inode, this._store) } wipe() { diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index e11db8c..e200474 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -6,6 +6,7 @@ const CacheFS = require("./CacheFS.js"); const { ENOENT, ENOTEMPTY, ETIMEDOUT } = require("./errors.js"); const IdbBackend = require("./IdbBackend.js"); const HttpBackend = require("./HttpBackend.js") +const YjsBackend = require("./YjsBackend.js") const Mutex = require("./Mutex.js"); const Mutex2 = require("./Mutex2.js"); @@ -65,7 +66,6 @@ module.exports = class PromisifiedFS { } } async init (...args) { - if (this._initPromiseResolve) await this._initPromise; this._initPromise = this._init(...args) return this._initPromise } @@ -78,9 +78,16 @@ module.exports = class PromisifiedFS { lockDbName = name + "_lock", lockStoreName = name + "_lock", defer = false, + yfs = false, } = {}) { await this._gracefulShutdown() this._name = name + if (yfs) { + this._yfs = new YjsBackend(this._name); + this._cache = this._yfs; + this._idb = this._yfs; + return this._yfs._ready; + } this._idb = new IdbBackend(fileDbName, fileStoreName); this._mutex = navigator.locks ? new Mutex2(name) : new Mutex(lockDbName, lockStoreName); this._cache = new CacheFS(name); @@ -90,10 +97,6 @@ module.exports = class PromisifiedFS { this._http = new HttpBackend(url) this._urlauto = !!urlauto } - if (this._initPromiseResolve) { - this._initPromiseResolve(); - this._initPromiseResolve = null; - } // The next comment starting with the "fs is initially activated when constructed"? // That can create contention for the mutex if two threads try to init at the same time // so I've added an option to disable that behavior. @@ -135,6 +138,9 @@ module.exports = class PromisifiedFS { async _activate() { if (!this._initPromise) console.warn(new Error(`Attempted to use LightningFS ${this._name} before it was initialized.`)) await this._initPromise + + if (this._yfs) return + if (this._deactivationTimeout) { clearTimeout(this._deactivationTimeout) this._deactivationTimeout = null @@ -215,7 +221,7 @@ module.exports = class PromisifiedFS { let data = null, stat = null try { stat = this._cache.stat(filepath); - data = await this._idb.readFile(stat.ino) + data = await this._idb.readFileInode(stat.ino) } catch (e) { if (!this._urlauto) throw e } @@ -249,7 +255,7 @@ module.exports = class PromisifiedFS { data = encode(data); } const stat = await this._cache.writeStat(filepath, data.byteLength, { mode }); - await this._idb.writeFile(stat.ino, data) + await this._idb.writeFileInode(stat.ino, data) return null } async unlink(filepath, opts) { @@ -257,7 +263,7 @@ module.exports = class PromisifiedFS { const stat = this._cache.lstat(filepath); this._cache.unlink(filepath); if (stat.type !== 'symlink') { - await this._idb.unlink(stat.ino) + await this._idb.unlinkInode(stat.ino) } return null } diff --git a/src/YjsBackend.js b/src/YjsBackend.js new file mode 100644 index 0000000..5300e02 --- /dev/null +++ b/src/YjsBackend.js @@ -0,0 +1,233 @@ +const Y = require('yjs'); +const { IndexeddbPersistence } = require('y-indexeddb'); + +const path = require("./path.js"); +const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); + +const STAT = '!STAT'; +const DATA = '!DATA'; + +module.exports = class YjsBackend { + constructor(name) { + this._ydoc = new Y.Doc(); + this._yidb = new IndexeddbPersistence(name + '_yjs', this._ydoc); + this._ready = this._yidb.whenSynced.then(async () => { + this._root = this._ydoc.getMap('!root'); + this._inodes = this._ydoc.getMap('!inodes'); + if (!this._root.has("/")) { + const root = new Y.Map(); + root.set(STAT, { mode: 0o777, type: "dir", size: 0, ino: 0, mtimeMs: Date.now() }); + this._inodes.set('0', root); + this._root.set("/", '0'); + } + return 'ready'; + }); + } + get activated () { + return !!this._root + } + autoinc () { + let val = this._maxInode(this._inodes.get("0")) + 1; + return val; + } + _maxInode(map) { + let max = map.get(STAT).ino; + for (let [key, ino] of map) { + if (key === STAT) continue; + if (key === DATA) continue; + const val = this._inodes.get(String(ino)); + if (!val.get) continue; + max = Math.max(max, this._maxInode(val)); + } + return max; + } + _lookup(filepath, follow = true) { + let dir = this._root; + let partialPath = '/' + let parts = path.split(filepath) + for (let i = 0; i < parts.length; ++ i) { + let part = parts[i]; + const ino = dir.get(part); + dir = this._inodes.get(String(ino)); + if (!dir) throw new ENOENT(filepath); + // Follow symlinks + if (follow || i < parts.length - 1) { + const stat = dir.get(STAT) + if (stat.type === 'symlink') { + let target = path.resolve(partialPath, stat.target) + dir = this._lookup(target) + } + if (!partialPath) { + partialPath = part + } else { + partialPath = path.join(partialPath, part) + } + } + } + return dir; + } + mkdir(filepath, { mode }) { + if (filepath === "/") throw new EEXIST(); + let dir = this._lookup(path.dirname(filepath)); + let basename = path.basename(filepath); + if (dir.has(basename)) { + throw new EEXIST(); + } + const ino = this.autoinc(); + let entry = new Y.Map() + let stat = { + mode, + type: "dir", + size: 0, + mtimeMs: Date.now(), + ino, + }; + entry.set(STAT, stat); + this._inodes.set(String(ino), entry); + dir.set(basename, ino); + } + rmdir(filepath) { + let dir = this._lookup(filepath); + if (dir.get(STAT).type !== 'dir') throw new ENOTDIR(); + // check it's empty (size should be 1 for just StatSym) + if (dir.size > 1) throw new ENOTEMPTY(); + // remove from parent + let parent = this._lookup(path.dirname(filepath)); + let basename = path.basename(filepath); + const ino = parent.get(basename) + parent.delete(basename); + this._inodes.delete(ino); + } + readdir(filepath) { + let dir = this._lookup(filepath); + if (dir.get(STAT).type !== 'dir') throw new ENOTDIR(); + return [...dir.keys()].filter(key => key != STAT); + } + writeStat(filepath, size, { mode }) { + let ino; + try { + let oldStat = this.stat(filepath); + if (mode == null) { + mode = oldStat.mode; + } + ino = oldStat.ino; + } catch (err) {} + if (mode == null) { + mode = 0o666; + } + if (ino == null) { + ino = this.autoinc(); + } + let dir = this._lookup(path.dirname(filepath)); + let basename = path.basename(filepath); + let stat = { + mode, + type: "file", + size, + mtimeMs: Date.now(), + ino, + }; + let entry = new Y.Map(); + entry.set(STAT, stat); + dir.set(basename, ino); + this._inodes.set(String(ino), entry); + return stat; + } + unlink(filepath) { + // remove from parent + let parent = this._lookup(path.dirname(filepath)); + let basename = path.basename(filepath); + parent.delete(basename); + } + rename(oldFilepath, newFilepath) { + let basename = path.basename(newFilepath); + // Note: do both lookups before making any changes + // so if lookup throws, we don't lose data (issue #23) + // grab references + let entry = this._lookup(oldFilepath); + let destDir = this._lookup(path.dirname(newFilepath)); + // remove from old parent directory + this.unlink(oldFilepath) + // insert into new parent directory + // TODO: THIS DOESN'T WORK IN YJS (must use New fresh Y.Map object?) + destDir.set(basename, entry); + } + stat(filepath) { + return this._lookup(filepath).get(STAT); + } + lstat(filepath) { + return this._lookup(filepath, false).get(STAT); + } + readlink(filepath) { + return this._lookup(filepath, false).get(STAT).target; + } + symlink(target, filepath) { + let ino, mode; + try { + let oldStat = this.stat(filepath); + if (mode === null) { + mode = oldStat.mode; + } + ino = oldStat.ino; + } catch (err) {} + if (mode == null) { + mode = 0o120000; + } + if (ino == null) { + ino = this.autoinc(); + } + let dir = this._lookup(path.dirname(filepath)); + let basename = path.basename(filepath); + let stat = { + mode, + type: "symlink", + target, + size: 0, + mtimeMs: Date.now(), + ino, + }; + let entry = new Y.Map(); + entry.set(STAT, stat); + dir.set(basename, ino); + this._inodes.set(String(ino), entry); + return stat; + } + _du (dir) { + let size = 0; + for (const [name, ino] of dir.entries()) { + const entry = this._inodes.get(String(ino)); + if (name === STAT) { + size += entry.size; + } else { + size += this._du(entry); + } + } + return size; + } + du (filepath) { + let dir = this._lookup(filepath); + return this._du(dir); + } + + saveSuperblock(superblock) { + return + } + loadSuperblock() { + return + } + readFileInode(inode) { + return this._inodes.get(String(inode)).get(DATA); + } + writeFileInode(inode, data) { + return this._inodes.get(String(inode)).set(DATA, data); + } + unlinkInode(inode) { + return this._inodes.delete(inode) + } + wipe() { + return [...this._root.keys()].map(key => this._root.delete(key)) + } + close() { + return + } +} diff --git a/src/__tests__/YFS.spec.js b/src/__tests__/YFS.spec.js new file mode 100755 index 0000000..a9ffef8 --- /dev/null +++ b/src/__tests__/YFS.spec.js @@ -0,0 +1,476 @@ +import FS from "../index.js"; + +const fs = new FS("testfs-promises2", { wipe: true, yfs: true }).promises; + +const HELLO = new Uint8Array([72, 69, 76, 76, 79]); + +if (!Promise.prototype.finally) { + Promise.prototype.finally = function (onFinally) { + this.then(onFinally, onFinally); + } +} + +describe("YFS module", () => { + describe("mkdir", () => { + it("root directory already exists", (done) => { + fs.mkdir("/").catch(err => { + expect(err).not.toBe(null); + expect(err.code).toEqual("EEXIST"); + done(); + }); + }); + it("create empty directory", done => { + fs.mkdir("/mkdir-test") + .then(() => { + fs.stat("/mkdir-test").then(stat => { + done(); + }); + }) + .catch(err => { + expect(err.code).toEqual("EEXIST"); + done(); + }); + }); + }); + + describe("writeFile", () => { + it("create file", done => { + fs.mkdir("/writeFile").finally(() => { + fs.writeFile("/writeFile/writeFile-uint8.txt", HELLO).then(() => { + fs.stat("/writeFile/writeFile-uint8.txt").then(stats => { + expect(stats.size).toEqual(5); + done(); + }); + }); + }); + }); + it("create file (from string)", done => { + fs.mkdir("/writeFile").finally(() => { + fs.writeFile("/writeFile/writeFile-string.txt", "HELLO").then(() => { + fs.stat("/writeFile/writeFile-string.txt").then(stats => { + expect(stats.size).toEqual(5); + done(); + }); + }); + }); + }); + it("write file perserves old inode", done => { + fs.mkdir("/writeFile").finally(() => { + fs.writeFile("/writeFile/writeFile-inode.txt", "HELLO").then(() => { + fs.stat("/writeFile/writeFile-inode.txt").then(stats => { + let inode = stats.ino; + fs.writeFile("/writeFile/writeFile-inode.txt", "WORLD").then(() => { + fs.stat("/writeFile/writeFile-inode.txt").then(stats => { + expect(stats.ino).toEqual(inode); + done(); + }); + }); + }); + }); + }); + }); + it("write file perserves old mode", done => { + fs.mkdir("/writeFile").finally(() => { + fs.writeFile("/writeFile/writeFile-mode.txt", "HELLO", { mode: 0o635 }).then(() => { + fs.stat("/writeFile/writeFile-mode.txt").then(stats => { + let mode = stats.mode; + expect(mode).toEqual(0o635) + fs.writeFile("/writeFile/writeFile-mode.txt", "WORLD").then(() => { + fs.stat("/writeFile/writeFile-mode.txt").then(stats => { + expect(stats.mode).toEqual(0o635); + done(); + }); + }); + }); + }); + }); + }); + }); + + describe("readFile", () => { + it("read non-existant file throws", done => { + fs.readFile("/readFile/non-existant.txt").catch(err => { + expect(err).not.toBe(null); + done(); + }); + }); + it("read file", done => { + fs.mkdir("/readFile").finally(() => { + fs.writeFile("/readFile/readFile-uint8.txt", "HELLO").then(() => { + fs.readFile("/readFile/readFile-uint8.txt").then(data => { + // instanceof comparisons on Uint8Array's retrieved from IDB are broken in Safari Mobile 11.x (source: https://github.com/dfahlander/Dexie.js/issues/656#issuecomment-391866600) + expect([...data]).toEqual([...HELLO]); + done(); + }); + }); + }); + }); + it("read file (encoding shorthand)", done => { + fs.mkdir("/readFile").finally(() => { + fs.writeFile("/readFile/readFile-encoding-shorthand.txt", "HELLO").then(() => { + fs.readFile("/readFile/readFile-encoding-shorthand.txt", "utf8").then(data => { + expect(data).toEqual("HELLO"); + done(); + }); + }); + }); + }); + it("read file (encoding longhand)", done => { + fs.mkdir("/readFile").finally(() => { + fs.writeFile("/readFile/readFile-encoding-longhand.txt", "HELLO").then(() => { + fs.readFile("/readFile/readFile-encoding-longhand.txt", { encoding: "utf8" }).then(data => { + expect(data).toEqual("HELLO"); + done(); + }); + }); + }); + }); + }); + + describe("readdir", () => { + it("read non-existant dir returns undefined", done => { + fs.readdir("/readdir/non-existant").catch(err => { + expect(err).not.toBe(null); + done(); + }); + }); + it("read root directory", done => { + fs.mkdir("/readdir").finally(() => { + fs.readdir("/").then(data => { + expect(data.includes("readdir")).toBe(true); + done(); + }); + }); + }); + it("read child directory", done => { + fs.mkdir("/readdir").finally(() => { + fs.writeFile("/readdir/1.txt", "").then(() => { + fs.readdir("/readdir").then(data => { + expect(data).toEqual(["1.txt"]) + done(); + }); + }); + }); + }); + it("read a file throws", done => { + fs.mkdir("/readdir2").finally(() => { + fs.writeFile("/readdir2/not-a-dir", "").then(() => { + fs.readdir("/readdir2/not-a-dir").catch(err => { + expect(err).not.toBe(null); + expect(err.code).toBe('ENOTDIR'); + done(); + }); + }) + }) + }); + }); + + describe("rmdir", () => { + it("delete root directory fails", done => { + fs.rmdir("/").catch(err => { + expect(err).not.toBe(null); + expect(err.code).toEqual("ENOTEMPTY"); + done(); + }); + }); + it("delete non-existant directory fails", done => { + fs.rmdir("/rmdir/non-existant").catch(err => { + expect(err).not.toBe(null); + expect(err.code).toEqual("ENOENT"); + done(); + }); + }); + it("delete non-empty directory fails", done => { + fs.mkdir("/rmdir").finally(() => { + fs.mkdir("/rmdir/not-empty").finally(() => { + fs.writeFile("/rmdir/not-empty/file.txt", "").then(() => { + + fs.rmdir("/rmdir/not-empty").catch(err => { + expect(err).not.toBe(null); + expect(err.code).toEqual("ENOTEMPTY"); + done(); + }); + }) + }) + }) + }); + it("delete empty directory", done => { + fs.mkdir("/rmdir").finally(() => { + fs.mkdir("/rmdir/empty").finally(() => { + fs.readdir("/rmdir").then(data => { + let originalSize = data.length; + fs.rmdir("/rmdir/empty").then(() => { + fs.readdir("/rmdir").then(data => { + expect(data.length === originalSize - 1); + expect(data.includes("empty")).toBe(false); + done(); + }); + }); + }); + }); + }); + }); + it("delete a file throws", done => { + fs.mkdir("/rmdir").finally(() => { + fs.writeFile("/rmdir/not-a-dir", "").then(() => { + fs.rmdir("/rmdir/not-a-dir").catch(err => { + expect(err).not.toBe(null); + expect(err.code).toBe('ENOTDIR'); + done(); + }); + }); + }); + }); + }); + + describe("unlink", () => { + it("create and delete file", done => { + fs.mkdir("/unlink").finally(() => { + fs.writeFile("/unlink/file.txt", "").then(() => { + fs.readdir("/unlink").then(data => { + let originalSize = data.length; + fs.unlink("/unlink/file.txt").then(() => { + fs.readdir("/unlink").then(data => { + expect(data.length).toBe(originalSize - 1) + expect(data.includes("file.txt")).toBe(false); + fs.readFile("/unlink/file.txt").catch(err => { + expect(err).not.toBe(null) + expect(err.code).toBe("ENOENT") + done(); + }); + }); + }); + }); + }); + }); + }); + }); + + xdescribe("rename", () => { + it("create and rename file", done => { + fs.mkdir("/rename").finally(() => { + fs.writeFile("/rename/a.txt", "").then(() => { + fs.rename("/rename/a.txt", "/rename/b.txt").then(() => { + fs.readdir("/rename").then(data => { + expect(data.includes("a.txt")).toBe(false); + expect(data.includes("b.txt")).toBe(true); + fs.readFile("/rename/a.txt").catch(err => { + expect(err).not.toBe(null) + expect(err.code).toBe("ENOENT") + fs.readFile("/rename/b.txt", "utf8").then(data => { + expect(data).toBe("") + done(); + }); + }); + }); + }); + }); + }); + }); + it("create and rename directory", done => { + fs.mkdir("/rename").finally(() => { + fs.mkdir("/rename/a").finally(() => { + fs.writeFile("/rename/a/file.txt", "").then(() => { + fs.rename("/rename/a", "/rename/b").then(() => { + fs.readdir("/rename").then(data => { + expect(data.includes("a")).toBe(false); + expect(data.includes("b")).toBe(true); + fs.readFile("/rename/a/file.txt").catch(err => { + expect(err).not.toBe(null) + expect(err.code).toBe("ENOENT") + fs.readFile("/rename/b/file.txt", "utf8").then(data => { + expect(data).toBe("") + done(); + }); + }); + }); + }); + }); + }); + }); + }); + }); + + describe("symlink", () => { + it("symlink a file and read/write to it", done => { + fs.mkdir("/symlink").finally(() => { + fs.writeFile("/symlink/a.txt", "hello").then(() => { + fs.symlink("/symlink/a.txt", "/symlink/b.txt").then(() => { + fs.readFile("/symlink/b.txt", "utf8").then(data => { + expect(data).toBe("hello") + fs.writeFile("/symlink/b.txt", "world").then(() => { + fs.readFile("/symlink/a.txt", "utf8").then(data => { + expect(data).toBe("world"); + done(); + }) + }) + }); + }); + }); + }); + }); + xit("symlink a file and read/write to it (relative)", done => { + fs.mkdir("/symlink").finally(() => { + fs.writeFile("/symlink/a.txt", "hello").then(() => { + fs.symlink("a.txt", "/symlink/b.txt").then(() => { + fs.readFile("/symlink/b.txt", "utf8").then(data => { + expect(data).toBe("hello") + fs.writeFile("/symlink/b.txt", "world").then(() => { + fs.readFile("/symlink/a.txt", "utf8").then(data => { + expect(data).toBe("world"); + done(); + }) + }) + }); + }); + }); + }); + }); + it("symlink a directory and read/write to it", done => { + fs.mkdir("/symlink").finally(() => { + fs.mkdir("/symlink/a").finally(() => { + fs.writeFile("/symlink/a/file.txt", "data").then(() => { + fs.symlink("/symlink/a", "/symlink/b").then(() => { + fs.readdir("/symlink/b").then(data => { + expect(data.includes("file.txt")).toBe(true); + fs.readFile("/symlink/b/file.txt", "utf8").then(data => { + expect(data).toBe("data") + fs.writeFile("/symlink/b/file2.txt", "world").then(() => { + fs.readFile("/symlink/a/file2.txt", "utf8").then(data => { + expect(data).toBe("world"); + done(); + }) + }) + }); + }); + }); + }); + }); + }); + }); + it("symlink a directory and read/write to it (relative)", done => { + fs.mkdir("/symlink").finally(() => { + fs.mkdir("/symlink/a").finally(() => { + fs.mkdir("/symlink/b").finally(() => { + fs.writeFile("/symlink/a/file.txt", "data").then(() => { + fs.symlink("../a", "/symlink/b/c").then(() => { + fs.readdir("/symlink/b/c").then(data => { + expect(data.includes("file.txt")).toBe(true); + fs.readFile("/symlink/b/c/file.txt", "utf8").then(data => { + expect(data).toBe("data") + fs.writeFile("/symlink/b/c/file2.txt", "world").then(() => { + fs.readFile("/symlink/a/file2.txt", "utf8").then(data => { + expect(data).toBe("world"); + done(); + }) + }) + }); + }); + }); + }); + }); + }); + }); + }); + it("unlink doesn't follow symlinks", done => { + fs.mkdir("/symlink").finally(() => { + fs.mkdir("/symlink/del").finally(() => { + fs.writeFile("/symlink/del/file.txt", "data").then(() => { + fs.symlink("/symlink/del/file.txt", "/symlink/del/file2.txt").then(() => { + fs.readdir("/symlink/del").then(data => { + expect(data.includes("file.txt")).toBe(true) + expect(data.includes("file2.txt")).toBe(true) + fs.unlink("/symlink/del/file2.txt").then(data => { + fs.readdir("/symlink/del").then(data => { + expect(data.includes("file.txt")).toBe(true) + expect(data.includes("file2.txt")).toBe(false) + fs.readFile("/symlink/del/file.txt", "utf8").then(data => { + expect(data).toBe("data") + done(); + }) + }); + }); + }); + }); + }); + }); + }); + }); + it("lstat doesn't follow symlinks", done => { + fs.mkdir("/symlink").finally(() => { + fs.mkdir("/symlink/lstat").finally(() => { + fs.writeFile("/symlink/lstat/file.txt", "data").then(() => { + fs.symlink("/symlink/lstat/file.txt", "/symlink/lstat/file2.txt").then(() => { + fs.stat("/symlink/lstat/file2.txt").then(stat => { + expect(stat.isFile()).toBe(true) + expect(stat.isSymbolicLink()).toBe(false) + fs.lstat("/symlink/lstat/file2.txt").then(stat => { + expect(stat.isFile()).toBe(false) + expect(stat.isSymbolicLink()).toBe(true) + done(); + }); + }); + }); + }); + }); + }); + }); + }); + + describe("readlink", () => { + it("readlink returns the target path", done => { + fs.mkdir("/readlink").finally(() => { + fs.writeFile("/readlink/a.txt", "hello").then(() => { + fs.symlink("/readlink/a.txt", "/readlink/b.txt").then(() => { + fs.readlink("/readlink/b.txt", "utf8").then(data => { + expect(data).toBe("/readlink/a.txt") + done(); + }); + }); + }); + }); + }); + it("readlink operates on paths with symlinks", done => { + fs.mkdir("/readlink").finally(() => { + fs.symlink("/readlink", "/readlink/sub").then(() => { + fs.writeFile("/readlink/c.txt", "hello").then(() => { + fs.symlink("/readlink/c.txt", "/readlink/d.txt").then(() => { + fs.readlink("/readlink/sub/d.txt").then(data => { + expect(data).toBe("/readlink/c.txt") + done(); + }); + }); + }); + }); + }); + }); + }); + + xdescribe("du", () => { + it("du returns the total file size of a path", done => { + fs.mkdir("/du").finally(() => { + fs.writeFile("/du/a.txt", "hello").then(() => { + fs.writeFile("/du/b.txt", "hello").then(() => { + fs.mkdir("/du/sub").then(() => { + fs.writeFile("/du/sub/a.txt", "hello").then(() => { + fs.writeFile("/du/sub/b.txt", "hello").then(() => { + fs.du("/du/sub/a.txt").then(size => { + expect(size).toBe(5) + fs.du("/du/sub").then(size => { + expect(size).toBe(10) + fs.du("/du").then(size => { + expect(size).toBe(20) + done(); + }); + }); + }); + }); + }); + }); + }); + }); + }); + }); + }); + +}); From 66bf6660a33d35592b462748b3f57df07239aede Mon Sep 17 00:00:00 2001 From: William Hilton Date: Mon, 26 Oct 2020 23:59:03 -0400 Subject: [PATCH 02/43] wip: thread-safety test --- package-lock.json | 298 +++++++++++++++++++++-- package.json | 1 + src/YjsBackend.js | 23 +- src/__tests__/YFS.spec.js | 52 ++-- src/__tests__/threadsafety-yfs.worker.js | 17 ++ src/__tests__/threadsafety.spec.js | 29 ++- 6 files changed, 367 insertions(+), 53 deletions(-) create mode 100644 src/__tests__/threadsafety-yfs.worker.js diff --git a/package-lock.json b/package-lock.json index 944604e..d1c36ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -695,6 +695,39 @@ "through": ">=2.2.7 <3" } }, + "abstract-leveldown": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", + "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", + "requires": { + "buffer": "^5.5.0", + "immediate": "^3.2.3", + "level-concat-iterator": "~2.0.0", + "level-supports": "~1.0.0", + "xtend": "~4.0.0" + }, + "dependencies": { + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, + "buffer": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.1.tgz", + "integrity": "sha512-2z15UUHpS9/3tk9mY/q+Rl3rydOi7yMp5XWNQnRvoz+mJwiv8brqYwp9a+nOCtma6dwuEIxljD8W3ysVBZ05Vg==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + } + } + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -1086,8 +1119,7 @@ "async-limiter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", - "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==", - "dev": true + "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" }, "asynckit": { "version": "0.4.0", @@ -2498,6 +2530,15 @@ "dev": true, "optional": true }, + "deferred-leveldown": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz", + "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==", + "requires": { + "abstract-leveldown": "~6.2.1", + "inherits": "^2.0.3" + } + }, "define-property": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", @@ -2730,6 +2771,17 @@ "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", "dev": true }, + "encoding-down": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz", + "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==", + "requires": { + "abstract-leveldown": "^6.2.1", + "inherits": "^2.0.3", + "level-codec": "^9.0.0", + "level-errors": "^2.0.0" + } + }, "end-of-stream": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", @@ -2839,7 +2891,6 @@ "version": "0.1.7", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", - "dev": true, "requires": { "prr": "~1.0.1" } @@ -4710,6 +4761,11 @@ "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", "dev": true }, + "immediate": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.3.0.tgz", + "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==" + }, "import-fresh": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", @@ -4785,8 +4841,7 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "ini": { "version": "1.3.5", @@ -5400,6 +5455,168 @@ "invert-kv": "^2.0.0" } }, + "level": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/level/-/level-6.0.1.tgz", + "integrity": "sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw==", + "requires": { + "level-js": "^5.0.0", + "level-packager": "^5.1.0", + "leveldown": "^5.4.0" + } + }, + "level-codec": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.2.tgz", + "integrity": "sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==", + "requires": { + "buffer": "^5.6.0" + }, + "dependencies": { + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, + "buffer": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.1.tgz", + "integrity": "sha512-2z15UUHpS9/3tk9mY/q+Rl3rydOi7yMp5XWNQnRvoz+mJwiv8brqYwp9a+nOCtma6dwuEIxljD8W3ysVBZ05Vg==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + } + } + }, + "level-concat-iterator": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", + "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==" + }, + "level-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", + "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", + "requires": { + "errno": "~0.1.1" + } + }, + "level-iterator-stream": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz", + "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "^3.4.0", + "xtend": "^4.0.2" + }, + "dependencies": { + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + } + } + }, + "level-js": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/level-js/-/level-js-5.0.2.tgz", + "integrity": "sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg==", + "requires": { + "abstract-leveldown": "~6.2.3", + "buffer": "^5.5.0", + "inherits": "^2.0.3", + "ltgt": "^2.1.2" + }, + "dependencies": { + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, + "buffer": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.1.tgz", + "integrity": "sha512-2z15UUHpS9/3tk9mY/q+Rl3rydOi7yMp5XWNQnRvoz+mJwiv8brqYwp9a+nOCtma6dwuEIxljD8W3ysVBZ05Vg==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + } + } + }, + "level-packager": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-5.1.1.tgz", + "integrity": "sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ==", + "requires": { + "encoding-down": "^6.3.0", + "levelup": "^4.3.2" + } + }, + "level-supports": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", + "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", + "requires": { + "xtend": "^4.0.2" + }, + "dependencies": { + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + } + } + }, + "leveldown": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.6.0.tgz", + "integrity": "sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==", + "requires": { + "abstract-leveldown": "~6.2.1", + "napi-macros": "~2.0.0", + "node-gyp-build": "~4.1.0" + } + }, + "levelup": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.4.0.tgz", + "integrity": "sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==", + "requires": { + "deferred-leveldown": "~5.3.0", + "level-errors": "~2.0.0", + "level-iterator-stream": "~4.0.0", + "level-supports": "~1.0.0", + "xtend": "~4.0.0" + } + }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -5526,8 +5743,7 @@ "lodash.debounce": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", - "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=", - "dev": true + "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=" }, "lodash.escaperegexp": { "version": "4.1.2", @@ -5855,6 +6071,11 @@ "yallist": "^2.1.2" } }, + "ltgt": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", + "integrity": "sha1-81ypHEk/e3PaDgdJUwTxezH4fuU=" + }, "macos-release": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.4.0.tgz", @@ -6282,6 +6503,11 @@ "to-regex": "^3.0.1" } }, + "napi-macros": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", + "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==" + }, "negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -6328,6 +6554,11 @@ "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==", "dev": true }, + "node-gyp-build": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", + "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==" + }, "node-libs-browser": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.1.0.tgz", @@ -10801,8 +11032,7 @@ "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=", - "dev": true + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" }, "ps-tree": { "version": "1.2.0", @@ -11371,8 +11601,7 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "safe-regex": { "version": "1.1.0", @@ -12420,7 +12649,6 @@ "version": "1.1.1", "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "requires": { "safe-buffer": "~5.1.0" } @@ -13146,8 +13374,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "dev": true + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "utils-merge": { "version": "1.0.1", @@ -13537,8 +13764,7 @@ "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", - "dev": true + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" }, "y-indexeddb": { "version": "9.0.5", @@ -13548,6 +13774,46 @@ "lib0": "^0.2.12" } }, + "y-leveldb": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/y-leveldb/-/y-leveldb-0.1.0.tgz", + "integrity": "sha512-sMuitVrsAUNh+0b66I42nAuW3lCmez171uP4k0ePcTAJ+c+Iw9w4Yq3wwiyrDMFXBEyQSjSF86Inc23wEvWnxw==", + "requires": { + "level": "^6.0.1", + "lib0": "^0.2.31" + } + }, + "y-protocols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.1.tgz", + "integrity": "sha512-QP3fCM7c2gGfUi2nqf8gspyO4VW23zv3kNqPNdD3wNxMbuNQenMyoDVZYEo12jzR4RQ3aaDfPK62Sf31SVOmfg==", + "requires": { + "lib0": "^0.2.28" + } + }, + "y-websocket": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/y-websocket/-/y-websocket-1.3.5.tgz", + "integrity": "sha512-TTS6bguW53ciLXuuYkgk/YEOEIwsO7Hy8JMej951g2ePBt9Z8j9riGumYQ/79mltig1IE4ZA6DQp6b10woS5Zw==", + "requires": { + "lib0": "^0.2.31", + "lodash.debounce": "^4.0.8", + "ws": "^6.2.1", + "y-leveldb": "^0.1.0", + "y-protocols": "^1.0.0" + }, + "dependencies": { + "ws": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", + "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==", + "optional": true, + "requires": { + "async-limiter": "~1.0.0" + } + } + } + }, "y18n": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", diff --git a/package.json b/package.json index 36b6bc4..cc68eab 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "just-debounce-it": "1.1.0", "just-once": "1.1.0", "y-indexeddb": "^9.0.5", + "y-websocket": "^1.3.5", "yjs": "^13.4.1" }, "devDependencies": { diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 5300e02..87e97a5 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -1,5 +1,6 @@ const Y = require('yjs'); const { IndexeddbPersistence } = require('y-indexeddb'); +// const { WebsocketProvider } = require('y-websocket'); const path = require("./path.js"); const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); @@ -11,6 +12,8 @@ module.exports = class YjsBackend { constructor(name) { this._ydoc = new Y.Doc(); this._yidb = new IndexeddbPersistence(name + '_yjs', this._ydoc); + // WIP: I'm adding this to get the BroadcastChannel functionality for the threadsafety tests can run. + // this._yws = new WebsocketProvider('wss://demos.yjs.dev', name + '_yjs', this._ydoc, { connect: false }); this._ready = this._yidb.whenSynced.then(async () => { this._root = this._ydoc.getMap('!root'); this._inodes = this._ydoc.getMap('!inodes'); @@ -20,6 +23,7 @@ module.exports = class YjsBackend { this._inodes.set('0', root); this._root.set("/", '0'); } + // this._yws.connectBc(); return 'ready'; }); } @@ -140,17 +144,18 @@ module.exports = class YjsBackend { parent.delete(basename); } rename(oldFilepath, newFilepath) { - let basename = path.basename(newFilepath); + let oldBasename = path.basename(oldFilepath); + let newBasename = path.basename(newFilepath); // Note: do both lookups before making any changes // so if lookup throws, we don't lose data (issue #23) // grab references - let entry = this._lookup(oldFilepath); + let entry = this._lookup(path.dirname(oldFilepath)); let destDir = this._lookup(path.dirname(newFilepath)); - // remove from old parent directory - this.unlink(oldFilepath) + let ino = entry.get(oldBasename); // insert into new parent directory - // TODO: THIS DOESN'T WORK IN YJS (must use New fresh Y.Map object?) - destDir.set(basename, entry); + destDir.set(newBasename, ino) + // remove from old parent directory + entry.delete(oldBasename) } stat(filepath) { return this._lookup(filepath).get(STAT); @@ -195,10 +200,10 @@ module.exports = class YjsBackend { _du (dir) { let size = 0; for (const [name, ino] of dir.entries()) { - const entry = this._inodes.get(String(ino)); if (name === STAT) { - size += entry.size; - } else { + size += ino.size; + } else if (name !== DATA) { + const entry = this._inodes.get(String(ino)); size += this._du(entry); } } diff --git a/src/__tests__/YFS.spec.js b/src/__tests__/YFS.spec.js index a9ffef8..3225ea1 100755 --- a/src/__tests__/YFS.spec.js +++ b/src/__tests__/YFS.spec.js @@ -246,7 +246,7 @@ describe("YFS module", () => { }); }); - xdescribe("rename", () => { + describe("rename", () => { it("create and rename file", done => { fs.mkdir("/rename").finally(() => { fs.writeFile("/rename/a.txt", "").then(() => { @@ -309,14 +309,14 @@ describe("YFS module", () => { }); }); }); - xit("symlink a file and read/write to it (relative)", done => { - fs.mkdir("/symlink").finally(() => { - fs.writeFile("/symlink/a.txt", "hello").then(() => { - fs.symlink("a.txt", "/symlink/b.txt").then(() => { - fs.readFile("/symlink/b.txt", "utf8").then(data => { + it("symlink a file and read/write to it (relative)", done => { + fs.mkdir("/symlink-relative").finally(() => { + fs.writeFile("/symlink-relative/a.txt", "hello").then(() => { + fs.symlink("a.txt", "/symlink-relative/b.txt").then(() => { + fs.readFile("/symlink-relative/b.txt", "utf8").then(data => { expect(data).toBe("hello") - fs.writeFile("/symlink/b.txt", "world").then(() => { - fs.readFile("/symlink/a.txt", "utf8").then(data => { + fs.writeFile("/symlink-relative/b.txt", "world").then(() => { + fs.readFile("/symlink-relative/a.txt", "utf8").then(data => { expect(data).toBe("world"); done(); }) @@ -327,16 +327,16 @@ describe("YFS module", () => { }); }); it("symlink a directory and read/write to it", done => { - fs.mkdir("/symlink").finally(() => { - fs.mkdir("/symlink/a").finally(() => { - fs.writeFile("/symlink/a/file.txt", "data").then(() => { - fs.symlink("/symlink/a", "/symlink/b").then(() => { - fs.readdir("/symlink/b").then(data => { + fs.mkdir("/symlink-dir").finally(() => { + fs.mkdir("/symlink-dir/a").finally(() => { + fs.writeFile("/symlink-dir/a/file.txt", "data").then(() => { + fs.symlink("/symlink-dir/a", "/symlink-dir/b").then(() => { + fs.readdir("/symlink-dir/b").then(data => { expect(data.includes("file.txt")).toBe(true); - fs.readFile("/symlink/b/file.txt", "utf8").then(data => { + fs.readFile("/symlink-dir/b/file.txt", "utf8").then(data => { expect(data).toBe("data") - fs.writeFile("/symlink/b/file2.txt", "world").then(() => { - fs.readFile("/symlink/a/file2.txt", "utf8").then(data => { + fs.writeFile("/symlink-dir/b/file2.txt", "world").then(() => { + fs.readFile("/symlink-dir/a/file2.txt", "utf8").then(data => { expect(data).toBe("world"); done(); }) @@ -349,17 +349,17 @@ describe("YFS module", () => { }); }); it("symlink a directory and read/write to it (relative)", done => { - fs.mkdir("/symlink").finally(() => { - fs.mkdir("/symlink/a").finally(() => { - fs.mkdir("/symlink/b").finally(() => { - fs.writeFile("/symlink/a/file.txt", "data").then(() => { - fs.symlink("../a", "/symlink/b/c").then(() => { - fs.readdir("/symlink/b/c").then(data => { + fs.mkdir("/symlink-dir-relative").finally(() => { + fs.mkdir("/symlink-dir-relative/a").finally(() => { + fs.mkdir("/symlink-dir-relative/b").finally(() => { + fs.writeFile("/symlink-dir-relative/a/file.txt", "data").then(() => { + fs.symlink("../a", "/symlink-dir-relative/b/c").then(() => { + fs.readdir("/symlink-dir-relative/b/c").then(data => { expect(data.includes("file.txt")).toBe(true); - fs.readFile("/symlink/b/c/file.txt", "utf8").then(data => { + fs.readFile("/symlink-dir-relative/b/c/file.txt", "utf8").then(data => { expect(data).toBe("data") - fs.writeFile("/symlink/b/c/file2.txt", "world").then(() => { - fs.readFile("/symlink/a/file2.txt", "utf8").then(data => { + fs.writeFile("/symlink-dir-relative/b/c/file2.txt", "world").then(() => { + fs.readFile("/symlink-dir-relative/a/file2.txt", "utf8").then(data => { expect(data).toBe("world"); done(); }) @@ -446,7 +446,7 @@ describe("YFS module", () => { }); }); - xdescribe("du", () => { + describe("du", () => { it("du returns the total file size of a path", done => { fs.mkdir("/du").finally(() => { fs.writeFile("/du/a.txt", "hello").then(() => { diff --git a/src/__tests__/threadsafety-yfs.worker.js b/src/__tests__/threadsafety-yfs.worker.js new file mode 100644 index 0000000..4e9dd30 --- /dev/null +++ b/src/__tests__/threadsafety-yfs.worker.js @@ -0,0 +1,17 @@ +importScripts('http://localhost:9876/base/dist/lightning-fs.min.js'); + +self.fs = new LightningFS("testfs-worker-yfs", { yfs: true }).promises; + +const sleep = ms => new Promise(r => setTimeout(r, ms)) + +const whoAmI = (typeof window === 'undefined' ? (self.name ? self.name : 'worker') : 'main' )+ ': ' + +async function writeFiles () { + console.log(whoAmI + 'write stuff') + // Chrome Mobile 67 and Mobile Safari 11 do not yet support named Workers + let name = self.name || Math.random() + await Promise.all([0, 1, 2, 3, 4].map(i => self.fs.writeFile(`/${name}_${i}.txt`, String(i)))) + self.postMessage({ message: 'COMPLETE' }) +} + +writeFiles() diff --git a/src/__tests__/threadsafety.spec.js b/src/__tests__/threadsafety.spec.js index 0c78a57..dd37516 100644 --- a/src/__tests__/threadsafety.spec.js +++ b/src/__tests__/threadsafety.spec.js @@ -1,10 +1,9 @@ jasmine.DEFAULT_TIMEOUT_INTERVAL = 20000 import FS from "../index.js"; -const fs = new FS("testfs-worker", { wipe: true }).promises; - describe("thread safety", () => { it("launch a bunch of workers", (done) => { + const fs = new FS("testfs-worker", { wipe: true }).promises; let workers = [] let promises = [] let numWorkers = 5 @@ -28,4 +27,30 @@ describe("thread safety", () => { }); }); }); + + xit("launch a bunch of workers (YFS)", (done) => { + const fs = new FS("testfs-worker-yfs", { wipe: true, yfs: true }).promises; + let workers = [] + let promises = [] + let numWorkers = 5 + fs.readdir('/').then(files => { + expect(files.length).toBe(0); + for (let i = 1; i <= numWorkers; i++) { + let promise = new Promise(resolve => { + let worker = new Worker('http://localhost:9876/base/src/__tests__/threadsafety-yfs.worker.js', {name: `worker_yfs_${i}`}) + worker.onmessage = (e) => { + if (e.data && e.data.message === 'COMPLETE') resolve() + } + workers.push(worker) + }) + promises.push(promise) + } + Promise.all(promises).then(() => { + fs.readdir('/').then(files => { + expect(files.length).toBe(5 * numWorkers) + done(); + }); + }); + }); + }); }); From e8395513f919a6d954f133dd2a2de85d78be91d6 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 27 Oct 2020 12:22:05 -0400 Subject: [PATCH 03/43] wip: switch to uuid to prevent conflicts --- package-lock.json | 23 +++++++++++++++++---- package.json | 1 + src/YjsBackend.js | 51 +++++++++++++++++++---------------------------- 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/package-lock.json b/package-lock.json index d1c36ae..ba0d3ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11475,6 +11475,14 @@ "tough-cookie": "~2.4.3", "tunnel-agent": "^0.6.0", "uuid": "^3.3.2" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "dev": true + } } }, "requestretry": { @@ -13383,10 +13391,9 @@ "dev": true }, "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "dev": true + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz", + "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==" }, "uws": { "version": "9.14.0", @@ -13571,6 +13578,14 @@ "requires": { "ansi-colors": "^3.0.0", "uuid": "^3.3.2" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "dev": true + } } }, "webpack-sources": { diff --git a/package.json b/package.json index cc68eab..cd32cd6 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "isomorphic-textencoder": "1.0.1", "just-debounce-it": "1.1.0", "just-once": "1.1.0", + "uuid": "^8.3.1", "y-indexeddb": "^9.0.5", "y-websocket": "^1.3.5", "yjs": "^13.4.1" diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 87e97a5..25c8895 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -1,12 +1,17 @@ const Y = require('yjs'); const { IndexeddbPersistence } = require('y-indexeddb'); // const { WebsocketProvider } = require('y-websocket'); +const { v4: uuidv4 } = require('uuid'); const path = require("./path.js"); const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); -const STAT = '!STAT'; -const DATA = '!DATA'; +// ':' is invalid as a filename character on both Mac and Windows, so these shouldn't conflict with real filenames. +// I can still totally see our own code failing to reject ':' when renaming a file though. +// So for safety, I'm adding NULL because NULL is invalid as a filename character on Linux. And pretty impossible to type using a keyboard. +// So that should handle ANY conceivable craziness. +const STAT = ':S\0'; +const DATA = ':D\0'; module.exports = class YjsBackend { constructor(name) { @@ -19,9 +24,10 @@ module.exports = class YjsBackend { this._inodes = this._ydoc.getMap('!inodes'); if (!this._root.has("/")) { const root = new Y.Map(); - root.set(STAT, { mode: 0o777, type: "dir", size: 0, ino: 0, mtimeMs: Date.now() }); - this._inodes.set('0', root); - this._root.set("/", '0'); + const ino = uuidv4(); + root.set(STAT, { mode: 0o777, type: "dir", size: 0, ino, mtimeMs: Date.now() }); + this._inodes.set(ino, root); + this._root.set("/", ino); } // this._yws.connectBc(); return 'ready'; @@ -30,21 +36,6 @@ module.exports = class YjsBackend { get activated () { return !!this._root } - autoinc () { - let val = this._maxInode(this._inodes.get("0")) + 1; - return val; - } - _maxInode(map) { - let max = map.get(STAT).ino; - for (let [key, ino] of map) { - if (key === STAT) continue; - if (key === DATA) continue; - const val = this._inodes.get(String(ino)); - if (!val.get) continue; - max = Math.max(max, this._maxInode(val)); - } - return max; - } _lookup(filepath, follow = true) { let dir = this._root; let partialPath = '/' @@ -52,7 +43,7 @@ module.exports = class YjsBackend { for (let i = 0; i < parts.length; ++ i) { let part = parts[i]; const ino = dir.get(part); - dir = this._inodes.get(String(ino)); + dir = this._inodes.get(ino); if (!dir) throw new ENOENT(filepath); // Follow symlinks if (follow || i < parts.length - 1) { @@ -77,7 +68,7 @@ module.exports = class YjsBackend { if (dir.has(basename)) { throw new EEXIST(); } - const ino = this.autoinc(); + const ino = uuidv4(); let entry = new Y.Map() let stat = { mode, @@ -87,7 +78,7 @@ module.exports = class YjsBackend { ino, }; entry.set(STAT, stat); - this._inodes.set(String(ino), entry); + this._inodes.set(ino, entry); dir.set(basename, ino); } rmdir(filepath) { @@ -120,7 +111,7 @@ module.exports = class YjsBackend { mode = 0o666; } if (ino == null) { - ino = this.autoinc(); + ino = uuidv4(); } let dir = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); @@ -134,7 +125,7 @@ module.exports = class YjsBackend { let entry = new Y.Map(); entry.set(STAT, stat); dir.set(basename, ino); - this._inodes.set(String(ino), entry); + this._inodes.set(ino, entry); return stat; } unlink(filepath) { @@ -179,7 +170,7 @@ module.exports = class YjsBackend { mode = 0o120000; } if (ino == null) { - ino = this.autoinc(); + ino = uuidv4(); } let dir = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); @@ -194,7 +185,7 @@ module.exports = class YjsBackend { let entry = new Y.Map(); entry.set(STAT, stat); dir.set(basename, ino); - this._inodes.set(String(ino), entry); + this._inodes.set(ino, entry); return stat; } _du (dir) { @@ -203,7 +194,7 @@ module.exports = class YjsBackend { if (name === STAT) { size += ino.size; } else if (name !== DATA) { - const entry = this._inodes.get(String(ino)); + const entry = this._inodes.get(ino); size += this._du(entry); } } @@ -221,10 +212,10 @@ module.exports = class YjsBackend { return } readFileInode(inode) { - return this._inodes.get(String(inode)).get(DATA); + return this._inodes.get(inode).get(DATA); } writeFileInode(inode, data) { - return this._inodes.get(String(inode)).set(DATA, data); + return this._inodes.get(inode).set(DATA, data); } unlinkInode(inode) { return this._inodes.delete(inode) From 34844fbd25211d0950672219255955b79d7d721f Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 27 Oct 2020 16:05:39 -0400 Subject: [PATCH 04/43] chore: move DATA to separate Map --- src/YjsBackend.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 25c8895..5b2a0eb 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -11,7 +11,6 @@ const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); // So for safety, I'm adding NULL because NULL is invalid as a filename character on Linux. And pretty impossible to type using a keyboard. // So that should handle ANY conceivable craziness. const STAT = ':S\0'; -const DATA = ':D\0'; module.exports = class YjsBackend { constructor(name) { @@ -22,6 +21,7 @@ module.exports = class YjsBackend { this._ready = this._yidb.whenSynced.then(async () => { this._root = this._ydoc.getMap('!root'); this._inodes = this._ydoc.getMap('!inodes'); + this._content = this._ydoc.getMap('!content'); if (!this._root.has("/")) { const root = new Y.Map(); const ino = uuidv4(); @@ -193,7 +193,7 @@ module.exports = class YjsBackend { for (const [name, ino] of dir.entries()) { if (name === STAT) { size += ino.size; - } else if (name !== DATA) { + } else { const entry = this._inodes.get(ino); size += this._du(entry); } @@ -212,13 +212,13 @@ module.exports = class YjsBackend { return } readFileInode(inode) { - return this._inodes.get(inode).get(DATA); + return this._content.get(inode); } writeFileInode(inode, data) { - return this._inodes.get(inode).set(DATA, data); + return this._content.set(inode, data); } unlinkInode(inode) { - return this._inodes.delete(inode) + return this._content.delete(inode) } wipe() { return [...this._root.keys()].map(key => this._root.delete(key)) From cb3d014e811f422faa11e4ef74e0ac0090f21c6d Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 27 Oct 2020 16:20:19 -0400 Subject: [PATCH 05/43] fix: Yjs 'Buffer' bug --- src/YjsBackend.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 5b2a0eb..6d5dcce 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -215,6 +215,14 @@ module.exports = class YjsBackend { return this._content.get(inode); } writeFileInode(inode, data) { + if (typeof data === 'string') { + // TODO: Convert to Y.Text + } else { + // Yjs will fail if data.constructor !== Uint8Array + if (data.constructor.name === 'Buffer') { + data = new Uint8Array(data.buffer); + } + } return this._content.set(inode, data); } unlinkInode(inode) { From c369455646cb0e12e63a2f77304dfe0e1b1d7220 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 27 Oct 2020 16:27:35 -0400 Subject: [PATCH 06/43] skip deactivation --- src/PromisifiedFS.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index e200474..2ffa153 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -186,6 +186,7 @@ module.exports = class PromisifiedFS { return this._deactivationPromise } async __deactivate() { + if (this._yfs) return; if (await this._mutex.has()) { await this._saveSuperblock() } From e813eb49fe56e03bc13b5c073c6e6787e14318f7 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 27 Oct 2020 16:46:25 -0400 Subject: [PATCH 07/43] wip: Hey, it kinda works in Studio now! --- src/YjsBackend.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 6d5dcce..0ab3aed 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -1,6 +1,6 @@ const Y = require('yjs'); const { IndexeddbPersistence } = require('y-indexeddb'); -// const { WebsocketProvider } = require('y-websocket'); +const { WebsocketProvider } = require('y-websocket'); const { v4: uuidv4 } = require('uuid'); const path = require("./path.js"); @@ -17,7 +17,7 @@ module.exports = class YjsBackend { this._ydoc = new Y.Doc(); this._yidb = new IndexeddbPersistence(name + '_yjs', this._ydoc); // WIP: I'm adding this to get the BroadcastChannel functionality for the threadsafety tests can run. - // this._yws = new WebsocketProvider('wss://demos.yjs.dev', name + '_yjs', this._ydoc, { connect: false }); + this._yws = new WebsocketProvider('wss://demos.yjs.dev', 'stoplight-v0.0.1-' + name + '_yjs', this._ydoc, { connect: false }); this._ready = this._yidb.whenSynced.then(async () => { this._root = this._ydoc.getMap('!root'); this._inodes = this._ydoc.getMap('!inodes'); @@ -29,7 +29,7 @@ module.exports = class YjsBackend { this._inodes.set(ino, root); this._root.set("/", ino); } - // this._yws.connectBc(); + this._yws.connectBc(); return 'ready'; }); } From ff28f118759d8537fd3bf67bd20be11be9a72f9e Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 27 Oct 2020 20:45:47 -0400 Subject: [PATCH 08/43] hacky, but it works for now --- src/YjsBackend.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 0ab3aed..b331d28 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -25,7 +25,7 @@ module.exports = class YjsBackend { if (!this._root.has("/")) { const root = new Y.Map(); const ino = uuidv4(); - root.set(STAT, { mode: 0o777, type: "dir", size: 0, ino, mtimeMs: Date.now() }); + root.set(STAT, { mode: 0o777, type: "dir", size: 0, ino, mtimeMs: Date.now(), filepath: '/' }); this._inodes.set(ino, root); this._root.set("/", ino); } @@ -76,6 +76,7 @@ module.exports = class YjsBackend { size: 0, mtimeMs: Date.now(), ino, + filepath, }; entry.set(STAT, stat); this._inodes.set(ino, entry); @@ -121,6 +122,7 @@ module.exports = class YjsBackend { size, mtimeMs: Date.now(), ino, + filepath, }; let entry = new Y.Map(); entry.set(STAT, stat); @@ -140,13 +142,18 @@ module.exports = class YjsBackend { // Note: do both lookups before making any changes // so if lookup throws, we don't lose data (issue #23) // grab references - let entry = this._lookup(path.dirname(oldFilepath)); + let srcDir = this._lookup(path.dirname(oldFilepath)); let destDir = this._lookup(path.dirname(newFilepath)); - let ino = entry.get(oldBasename); + let ino = srcDir.get(oldBasename); // insert into new parent directory destDir.set(newBasename, ino) // remove from old parent directory - entry.delete(oldBasename) + srcDir.delete(oldBasename) + // update stat.path + const entry = this._inodes.get(ino); + const stat = entry.get(STAT); + stat.filepath = newFilepath; + entry.set(STAT, stat); } stat(filepath) { return this._lookup(filepath).get(STAT); From cf2bc39ce60a2a7b5ef434db85df923f726e8796 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 27 Oct 2020 21:39:58 -0400 Subject: [PATCH 09/43] wrap in transactions with origins --- src/YjsBackend.js | 56 ++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index b331d28..90f483b 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -69,7 +69,6 @@ module.exports = class YjsBackend { throw new EEXIST(); } const ino = uuidv4(); - let entry = new Y.Map() let stat = { mode, type: "dir", @@ -78,9 +77,12 @@ module.exports = class YjsBackend { ino, filepath, }; - entry.set(STAT, stat); - this._inodes.set(ino, entry); - dir.set(basename, ino); + this._ydoc.transact(() => { + let entry = new Y.Map() + entry.set(STAT, stat); + this._inodes.set(ino, entry); + dir.set(basename, ino); + }, 'mkdir'); } rmdir(filepath) { let dir = this._lookup(filepath); @@ -91,8 +93,10 @@ module.exports = class YjsBackend { let parent = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); const ino = parent.get(basename) - parent.delete(basename); - this._inodes.delete(ino); + this._ydoc.transact(() => { + parent.delete(basename); + this._inodes.delete(ino); + }, 'rmdir'); } readdir(filepath) { let dir = this._lookup(filepath); @@ -124,17 +128,21 @@ module.exports = class YjsBackend { ino, filepath, }; - let entry = new Y.Map(); - entry.set(STAT, stat); - dir.set(basename, ino); - this._inodes.set(ino, entry); + this._ydoc.transact(() => { + let entry = new Y.Map(); + entry.set(STAT, stat); + this._inodes.set(ino, entry); + dir.set(basename, ino); + }, 'writeFile'); return stat; } unlink(filepath) { // remove from parent let parent = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); - parent.delete(basename); + this._ydoc.transact(() => { + parent.delete(basename); + }, 'unlink'); } rename(oldFilepath, newFilepath) { let oldBasename = path.basename(oldFilepath); @@ -145,15 +153,17 @@ module.exports = class YjsBackend { let srcDir = this._lookup(path.dirname(oldFilepath)); let destDir = this._lookup(path.dirname(newFilepath)); let ino = srcDir.get(oldBasename); - // insert into new parent directory - destDir.set(newBasename, ino) - // remove from old parent directory - srcDir.delete(oldBasename) - // update stat.path const entry = this._inodes.get(ino); const stat = entry.get(STAT); - stat.filepath = newFilepath; - entry.set(STAT, stat); + this._ydoc.transact(() => { + // insert into new parent directory + destDir.set(newBasename, ino) + // remove from old parent directory + srcDir.delete(oldBasename) + // update stat.path + stat.filepath = newFilepath; + entry.set(STAT, stat); + }, 'rename'); } stat(filepath) { return this._lookup(filepath).get(STAT); @@ -189,10 +199,12 @@ module.exports = class YjsBackend { mtimeMs: Date.now(), ino, }; - let entry = new Y.Map(); - entry.set(STAT, stat); - dir.set(basename, ino); - this._inodes.set(ino, entry); + this._ydoc.transact(() => { + let entry = new Y.Map(); + entry.set(STAT, stat); + this._inodes.set(ino, entry); + dir.set(basename, ino); + }, 'symlink'); return stat; } _du (dir) { From 9411bded1c4bdff775436e227a790e9c553230ee Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 27 Oct 2020 23:33:15 -0400 Subject: [PATCH 10/43] wip: use computed diffs for updating Y.Text --- package-lock.json | 5 +++++ package.json | 1 + src/PromisifiedFS.js | 3 ++- src/YjsBackend.js | 43 +++++++++++++++++++++++++++++++++++++++---- 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index ba0d3ec..a0ff3ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3309,6 +3309,11 @@ "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", "dev": true }, + "fast-diff": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz", + "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==" + }, "fast-glob": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz", diff --git a/package.json b/package.json index cd32cd6..1fc7836 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ }, "dependencies": { "@isomorphic-git/idb-keyval": "3.3.2", + "fast-diff": "^1.2.0", "isomorphic-textencoder": "1.0.1", "just-debounce-it": "1.1.0", "just-once": "1.1.0", diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index 2ffa153..3ec0857 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -249,6 +249,7 @@ module.exports = class PromisifiedFS { async writeFile(filepath, data, opts) { ;[filepath, opts] = cleanParams(filepath, opts); const { mode, encoding = "utf8" } = opts; + let origData = data if (typeof data === "string") { if (encoding !== "utf8") { throw new Error('Only "utf8" encoding is supported in writeFile'); @@ -256,7 +257,7 @@ module.exports = class PromisifiedFS { data = encode(data); } const stat = await this._cache.writeStat(filepath, data.byteLength, { mode }); - await this._idb.writeFileInode(stat.ino, data) + await this._idb.writeFileInode(stat.ino, data, origData) return null } async unlink(filepath, opts) { diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 90f483b..cacc2d9 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -1,7 +1,9 @@ +const { encode, decode } = require("isomorphic-textencoder"); const Y = require('yjs'); const { IndexeddbPersistence } = require('y-indexeddb'); const { WebsocketProvider } = require('y-websocket'); const { v4: uuidv4 } = require('uuid'); +const diff = require('fast-diff') const path = require("./path.js"); const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); @@ -231,11 +233,44 @@ module.exports = class YjsBackend { return } readFileInode(inode) { - return this._content.get(inode); + let data = this._content.get(inode) + if (data instanceof Y.Text) { + data = encode(data.toString()); + } + return data; } - writeFileInode(inode, data) { - if (typeof data === 'string') { - // TODO: Convert to Y.Text + writeFileInode(inode, data, rawdata) { + if (typeof rawdata === 'string') { + // Update existing Text + const oldData = this._content.get(inode); + if (oldData && oldData instanceof Y.Text) { + const oldString = oldData.toString(); + const changes = diff(oldString, rawdata); + console.log('changes', changes); + let idx = 0; + for (const [kind, string] of changes) { + switch (kind) { + case diff.EQUAL: { + idx += string.length; + break; + } + case diff.DELETE: { + oldData.delete(idx, string.length) + break; + } + case diff.INSERT: { + oldData.insert(idx, string); + idx += string.length; + break; + } + } + } + return; + } else { + // Use new Y.Text + data = new Y.Text(); + data.insert(0, rawdata); + } } else { // Yjs will fail if data.constructor !== Uint8Array if (data.constructor.name === 'Buffer') { From a62dcce315dadcbca268eb4ab8b3ab9c61854b68 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Wed, 28 Oct 2020 15:14:04 -0400 Subject: [PATCH 11/43] wip: cleanup --- package-lock.json | 10 ++++---- package.json | 2 +- src/YjsBackend.js | 65 +++++++++++++++++++++++++---------------------- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/package-lock.json b/package-lock.json index a0ff3ca..3c99f6a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6489,6 +6489,11 @@ "dev": true, "optional": true }, + "nanoid": { + "version": "3.1.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.16.tgz", + "integrity": "sha512-+AK8MN0WHji40lj8AEuwLOvLSbWYApQpre/aFJZD71r43wVRLrOYS4FmJOPQYon1TqB462RzrrxlfA74XRES8w==" + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -13395,11 +13400,6 @@ "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", "dev": true }, - "uuid": { - "version": "8.3.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz", - "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==" - }, "uws": { "version": "9.14.0", "resolved": "https://registry.npmjs.org/uws/-/uws-9.14.0.tgz", diff --git a/package.json b/package.json index 1fc7836..169638d 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "isomorphic-textencoder": "1.0.1", "just-debounce-it": "1.1.0", "just-once": "1.1.0", - "uuid": "^8.3.1", + "nanoid": "^3.1.16", "y-indexeddb": "^9.0.5", "y-websocket": "^1.3.5", "yjs": "^13.4.1" diff --git a/src/YjsBackend.js b/src/YjsBackend.js index cacc2d9..70029a1 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -2,7 +2,7 @@ const { encode, decode } = require("isomorphic-textencoder"); const Y = require('yjs'); const { IndexeddbPersistence } = require('y-indexeddb'); const { WebsocketProvider } = require('y-websocket'); -const { v4: uuidv4 } = require('uuid'); +const { nanoid } = require('nanoid'); const diff = require('fast-diff') const path = require("./path.js"); @@ -12,7 +12,8 @@ const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); // I can still totally see our own code failing to reject ':' when renaming a file though. // So for safety, I'm adding NULL because NULL is invalid as a filename character on Linux. And pretty impossible to type using a keyboard. // So that should handle ANY conceivable craziness. -const STAT = ':S\0'; +const STAT = 's'; +const CHILDREN = 'c'; module.exports = class YjsBackend { constructor(name) { @@ -24,12 +25,15 @@ module.exports = class YjsBackend { this._root = this._ydoc.getMap('!root'); this._inodes = this._ydoc.getMap('!inodes'); this._content = this._ydoc.getMap('!content'); - if (!this._root.has("/")) { - const root = new Y.Map(); - const ino = uuidv4(); - root.set(STAT, { mode: 0o777, type: "dir", size: 0, ino, mtimeMs: Date.now(), filepath: '/' }); - this._inodes.set(ino, root); - this._root.set("/", ino); + if (!this._root.has(CHILDREN)) { + const rootdir = new Y.Map(); + const ino = nanoid(); + rootdir.set(STAT, { mode: 0o777, type: "dir", size: 0, ino, mtimeMs: Date.now(), filepath: '/' }); + rootdir.set(CHILDREN, new Y.Map()); + this._inodes.set(ino, rootdir); + + this._root.set(CHILDREN, new Y.Map()); + this._root.get(CHILDREN).set('/', ino); } this._yws.connectBc(); return 'ready'; @@ -44,7 +48,7 @@ module.exports = class YjsBackend { let parts = path.split(filepath) for (let i = 0; i < parts.length; ++ i) { let part = parts[i]; - const ino = dir.get(part); + const ino = dir.get(CHILDREN).get(part); dir = this._inodes.get(ino); if (!dir) throw new ENOENT(filepath); // Follow symlinks @@ -67,10 +71,10 @@ module.exports = class YjsBackend { if (filepath === "/") throw new EEXIST(); let dir = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); - if (dir.has(basename)) { + if (dir.get(CHILDREN).has(basename)) { throw new EEXIST(); } - const ino = uuidv4(); + const ino = nanoid(); let stat = { mode, type: "dir", @@ -82,28 +86,29 @@ module.exports = class YjsBackend { this._ydoc.transact(() => { let entry = new Y.Map() entry.set(STAT, stat); + entry.set(CHILDREN, new Y.Map()); this._inodes.set(ino, entry); - dir.set(basename, ino); + dir.get(CHILDREN).set(basename, ino); }, 'mkdir'); } rmdir(filepath) { let dir = this._lookup(filepath); if (dir.get(STAT).type !== 'dir') throw new ENOTDIR(); - // check it's empty (size should be 1 for just StatSym) - if (dir.size > 1) throw new ENOTEMPTY(); + // check it's empty + if (dir.get(CHILDREN).size > 0) throw new ENOTEMPTY(); // remove from parent let parent = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); - const ino = parent.get(basename) + const ino = parent.get(CHILDREN).get(basename) this._ydoc.transact(() => { - parent.delete(basename); + parent.get(CHILDREN).delete(basename); this._inodes.delete(ino); }, 'rmdir'); } readdir(filepath) { let dir = this._lookup(filepath); if (dir.get(STAT).type !== 'dir') throw new ENOTDIR(); - return [...dir.keys()].filter(key => key != STAT); + return [...dir.get(CHILDREN).keys()].filter(key => key != STAT); } writeStat(filepath, size, { mode }) { let ino; @@ -118,7 +123,7 @@ module.exports = class YjsBackend { mode = 0o666; } if (ino == null) { - ino = uuidv4(); + ino = nanoid(); } let dir = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); @@ -134,7 +139,7 @@ module.exports = class YjsBackend { let entry = new Y.Map(); entry.set(STAT, stat); this._inodes.set(ino, entry); - dir.set(basename, ino); + dir.get(CHILDREN).set(basename, ino); }, 'writeFile'); return stat; } @@ -143,7 +148,7 @@ module.exports = class YjsBackend { let parent = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); this._ydoc.transact(() => { - parent.delete(basename); + parent.get(CHILDREN).delete(basename); }, 'unlink'); } rename(oldFilepath, newFilepath) { @@ -154,14 +159,14 @@ module.exports = class YjsBackend { // grab references let srcDir = this._lookup(path.dirname(oldFilepath)); let destDir = this._lookup(path.dirname(newFilepath)); - let ino = srcDir.get(oldBasename); + let ino = srcDir.get(CHILDREN).get(oldBasename); const entry = this._inodes.get(ino); const stat = entry.get(STAT); this._ydoc.transact(() => { // insert into new parent directory - destDir.set(newBasename, ino) + destDir.get(CHILDREN).set(newBasename, ino) // remove from old parent directory - srcDir.delete(oldBasename) + srcDir.get(CHILDREN).delete(oldBasename) // update stat.path stat.filepath = newFilepath; entry.set(STAT, stat); @@ -189,7 +194,7 @@ module.exports = class YjsBackend { mode = 0o120000; } if (ino == null) { - ino = uuidv4(); + ino = nanoid(); } let dir = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); @@ -205,16 +210,17 @@ module.exports = class YjsBackend { let entry = new Y.Map(); entry.set(STAT, stat); this._inodes.set(ino, entry); - dir.set(basename, ino); + dir.get(CHILDREN).set(basename, ino); }, 'symlink'); return stat; } _du (dir) { let size = 0; - for (const [name, ino] of dir.entries()) { - if (name === STAT) { - size += ino.size; - } else { + const stat = dir.get(STAT) + if (stat.type === 'file') { + size += stat.size; + } else if (stat.type === 'dir') { + for (const [name, ino] of dir.get(CHILDREN).entries()) { const entry = this._inodes.get(ino); size += this._du(entry); } @@ -246,7 +252,6 @@ module.exports = class YjsBackend { if (oldData && oldData instanceof Y.Text) { const oldString = oldData.toString(); const changes = diff(oldString, rawdata); - console.log('changes', changes); let idx = 0; for (const [kind, string] of changes) { switch (kind) { From 1ea12078cb7352f0efd3186f73e33361ac40a0a6 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Wed, 28 Oct 2020 17:27:14 -0400 Subject: [PATCH 12/43] wip: refactoring --- karma.conf.js | 2 +- src/YjsBackend.js | 111 +++++++++++++++++++++++++++----------- src/__tests__/YFS.spec.js | 6 ++- 3 files changed, 86 insertions(+), 33 deletions(-) diff --git a/karma.conf.js b/karma.conf.js index 0fc9188..cda309c 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -184,7 +184,7 @@ module.exports = function (config) { options.browsers = process.env.TEST_BROWSERS.split(',') } else { options.browsers.push('ChromeHeadlessNoSandbox') - options.browsers.push('FirefoxHeadless') + // options.browsers.push('FirefoxHeadless') } console.log('running with browsers:', options.browsers) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 70029a1..59c4bb0 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -14,6 +14,8 @@ const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); // So that should handle ANY conceivable craziness. const STAT = 's'; const CHILDREN = 'c'; +const PARENT = 'p'; +const BASENAME = 'b'; module.exports = class YjsBackend { constructor(name) { @@ -22,19 +24,22 @@ module.exports = class YjsBackend { // WIP: I'm adding this to get the BroadcastChannel functionality for the threadsafety tests can run. this._yws = new WebsocketProvider('wss://demos.yjs.dev', 'stoplight-v0.0.1-' + name + '_yjs', this._ydoc, { connect: false }); this._ready = this._yidb.whenSynced.then(async () => { - this._root = this._ydoc.getMap('!root'); this._inodes = this._ydoc.getMap('!inodes'); this._content = this._ydoc.getMap('!content'); - if (!this._root.has(CHILDREN)) { + this._ino2path = {}; + this._path2ino = {}; + if (this._inodes.size === 0) { const rootdir = new Y.Map(); const ino = nanoid(); - rootdir.set(STAT, { mode: 0o777, type: "dir", size: 0, ino, mtimeMs: Date.now(), filepath: '/' }); - rootdir.set(CHILDREN, new Y.Map()); + rootdir.set(STAT, { mode: 0o777, type: "dir", size: 0, ino, mtimeMs: Date.now() }); + rootdir.set(PARENT, null); + rootdir.set(BASENAME, '/'); this._inodes.set(ino, rootdir); - - this._root.set(CHILDREN, new Y.Map()); - this._root.get(CHILDREN).set('/', ino); } + for (const ino of this._inodes.keys()) { + this._computePath(ino); + } + console.log(JSON.stringify(this._ino2path, null, 2)) this._yws.connectBc(); return 'ready'; }); @@ -42,14 +47,43 @@ module.exports = class YjsBackend { get activated () { return !!this._root } + _computePath(ino) { + let parts = []; + while (ino != null) { + const dir = this._inodes.get(ino) + if (!dir) break; + parts.unshift(dir.get(BASENAME)) + ino = dir.get(PARENT) + } + const filepath = path.join(parts); + this._ino2path[ino] = filepath; + this._path2ino[filepath] = ino; + return filepath; + } + _childrenOf(id) { + const children = []; + for (const value of this._inodes.values()) { + if (value.get(PARENT) === id) children.push(value); + } + return children; + } + _findChild(id, basename) { + const children = []; + for (const value of this._inodes.values()) { + if (value.get(PARENT) === id && value.get(BASENAME) === basename) return value; + } + return; + } _lookup(filepath, follow = true) { - let dir = this._root; + let dir = null; let partialPath = '/' let parts = path.split(filepath) + // TODO: Actually, given we can reconstruct paths from the bottom up, + // it might be faster to search by matching against the basepath and then + // narrowing that set. The problem would be dealing with symlinks. for (let i = 0; i < parts.length; ++ i) { let part = parts[i]; - const ino = dir.get(CHILDREN).get(part); - dir = this._inodes.get(ino); + dir = this._findChild(dir && dir.get(STAT).ino, part); if (!dir) throw new ENOENT(filepath); // Follow symlinks if (follow || i < parts.length - 1) { @@ -70,9 +104,13 @@ module.exports = class YjsBackend { mkdir(filepath, { mode }) { if (filepath === "/") throw new EEXIST(); let dir = this._lookup(path.dirname(filepath)); + console.log('dir', JSON.stringify(dir.toJSON())); + console.log('ino', dir.get(STAT).ino); let basename = path.basename(filepath); - if (dir.get(CHILDREN).has(basename)) { - throw new EEXIST(); + for (const child of this._childrenOf(dir.get(STAT).ino)) { + if (child.get(BASENAME) === basename) { + throw new EEXIST(); + } } const ino = nanoid(); let stat = { @@ -81,34 +119,36 @@ module.exports = class YjsBackend { size: 0, mtimeMs: Date.now(), ino, - filepath, }; this._ydoc.transact(() => { let entry = new Y.Map() entry.set(STAT, stat); - entry.set(CHILDREN, new Y.Map()); + entry.set(PARENT, dir.get(STAT).ino); + entry.set(BASENAME, basename); this._inodes.set(ino, entry); - dir.get(CHILDREN).set(basename, ino); }, 'mkdir'); + console.log(JSON.stringify(this._inodes.toJSON(), null, 2)); } rmdir(filepath) { let dir = this._lookup(filepath); + console.log('dir', dir.toJSON()); if (dir.get(STAT).type !== 'dir') throw new ENOTDIR(); + const ino = dir.get(STAT).ino; // check it's empty - if (dir.get(CHILDREN).size > 0) throw new ENOTEMPTY(); - // remove from parent - let parent = this._lookup(path.dirname(filepath)); - let basename = path.basename(filepath); - const ino = parent.get(CHILDREN).get(basename) + if (this._childrenOf(ino).length > 0) throw new ENOTEMPTY(); + console.log('its empty'); + // remove from cache + delete this._ino2path[ino]; + delete this._path2ino[filepath]; + // delete inode this._ydoc.transact(() => { - parent.get(CHILDREN).delete(basename); this._inodes.delete(ino); }, 'rmdir'); } readdir(filepath) { let dir = this._lookup(filepath); if (dir.get(STAT).type !== 'dir') throw new ENOTDIR(); - return [...dir.get(CHILDREN).keys()].filter(key => key != STAT); + return this._childrenOf(dir.get(STAT).ino).map(node => node.get(BASENAME)); } writeStat(filepath, size, { mode }) { let ino; @@ -126,6 +166,7 @@ module.exports = class YjsBackend { ino = nanoid(); } let dir = this._lookup(path.dirname(filepath)); + let parentId = dir.get(STAT).ino; let basename = path.basename(filepath); let stat = { mode, @@ -136,19 +177,29 @@ module.exports = class YjsBackend { filepath, }; this._ydoc.transact(() => { - let entry = new Y.Map(); - entry.set(STAT, stat); - this._inodes.set(ino, entry); - dir.get(CHILDREN).set(basename, ino); + let entry = this._inodes.get(ino); + if (!entry) { + entry = new Y.Map(); + entry.set(STAT, stat); + entry.set(PARENT, parentId); + entry.set(BASENAME, basename); + this._inodes.set(ino, entry); + this._computePath(ino); + } else { + entry.set(STAT, stat); + } }, 'writeFile'); return stat; } unlink(filepath) { - // remove from parent - let parent = this._lookup(path.dirname(filepath)); - let basename = path.basename(filepath); + let node = this._lookup(filepath); + // remove from cache + delete this._ino2path[ino]; + delete this._path2ino[filepath]; + // delete inode + const ino = node.get(STAT).ino; this._ydoc.transact(() => { - parent.get(CHILDREN).delete(basename); + this._inodes.delete(ino); }, 'unlink'); } rename(oldFilepath, newFilepath) { diff --git a/src/__tests__/YFS.spec.js b/src/__tests__/YFS.spec.js index 3225ea1..8d2652d 100755 --- a/src/__tests__/YFS.spec.js +++ b/src/__tests__/YFS.spec.js @@ -165,7 +165,7 @@ describe("YFS module", () => { }); }); - describe("rmdir", () => { + fdescribe("rmdir", () => { it("delete root directory fails", done => { fs.rmdir("/").catch(err => { expect(err).not.toBe(null); @@ -199,10 +199,12 @@ describe("YFS module", () => { fs.mkdir("/rmdir/empty").finally(() => { fs.readdir("/rmdir").then(data => { let originalSize = data.length; + console.log('data', data); fs.rmdir("/rmdir/empty").then(() => { fs.readdir("/rmdir").then(data => { - expect(data.length === originalSize - 1); + console.log('data', data); expect(data.includes("empty")).toBe(false); + expect(data.length === originalSize - 1); done(); }); }); From 9c34f5f1ade39e0370f67e248ce60d285ffddd11 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Wed, 28 Oct 2020 19:08:34 -0400 Subject: [PATCH 13/43] finish refactoring --- src/YjsBackend.js | 62 ++++++++++++++------------------------- src/__tests__/YFS.spec.js | 2 +- 2 files changed, 23 insertions(+), 41 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 59c4bb0..7ff1fb1 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -26,8 +26,6 @@ module.exports = class YjsBackend { this._ready = this._yidb.whenSynced.then(async () => { this._inodes = this._ydoc.getMap('!inodes'); this._content = this._ydoc.getMap('!content'); - this._ino2path = {}; - this._path2ino = {}; if (this._inodes.size === 0) { const rootdir = new Y.Map(); const ino = nanoid(); @@ -36,10 +34,6 @@ module.exports = class YjsBackend { rootdir.set(BASENAME, '/'); this._inodes.set(ino, rootdir); } - for (const ino of this._inodes.keys()) { - this._computePath(ino); - } - console.log(JSON.stringify(this._ino2path, null, 2)) this._yws.connectBc(); return 'ready'; }); @@ -56,8 +50,6 @@ module.exports = class YjsBackend { ino = dir.get(PARENT) } const filepath = path.join(parts); - this._ino2path[ino] = filepath; - this._path2ino[filepath] = ino; return filepath; } _childrenOf(id) { @@ -104,8 +96,6 @@ module.exports = class YjsBackend { mkdir(filepath, { mode }) { if (filepath === "/") throw new EEXIST(); let dir = this._lookup(path.dirname(filepath)); - console.log('dir', JSON.stringify(dir.toJSON())); - console.log('ino', dir.get(STAT).ino); let basename = path.basename(filepath); for (const child of this._childrenOf(dir.get(STAT).ino)) { if (child.get(BASENAME) === basename) { @@ -127,19 +117,13 @@ module.exports = class YjsBackend { entry.set(BASENAME, basename); this._inodes.set(ino, entry); }, 'mkdir'); - console.log(JSON.stringify(this._inodes.toJSON(), null, 2)); } rmdir(filepath) { let dir = this._lookup(filepath); - console.log('dir', dir.toJSON()); if (dir.get(STAT).type !== 'dir') throw new ENOTDIR(); const ino = dir.get(STAT).ino; // check it's empty if (this._childrenOf(ino).length > 0) throw new ENOTEMPTY(); - console.log('its empty'); - // remove from cache - delete this._ino2path[ino]; - delete this._path2ino[filepath]; // delete inode this._ydoc.transact(() => { this._inodes.delete(ino); @@ -192,35 +176,26 @@ module.exports = class YjsBackend { return stat; } unlink(filepath) { - let node = this._lookup(filepath); - // remove from cache - delete this._ino2path[ino]; - delete this._path2ino[filepath]; - // delete inode + let node = this._lookup(filepath, false); const ino = node.get(STAT).ino; + // delete inode this._ydoc.transact(() => { this._inodes.delete(ino); }, 'unlink'); } rename(oldFilepath, newFilepath) { - let oldBasename = path.basename(oldFilepath); - let newBasename = path.basename(newFilepath); // Note: do both lookups before making any changes // so if lookup throws, we don't lose data (issue #23) // grab references - let srcDir = this._lookup(path.dirname(oldFilepath)); + let node = this._lookup(oldFilepath); let destDir = this._lookup(path.dirname(newFilepath)); - let ino = srcDir.get(CHILDREN).get(oldBasename); - const entry = this._inodes.get(ino); - const stat = entry.get(STAT); + let basename = path.basename(newFilepath); + // Update parent this._ydoc.transact(() => { - // insert into new parent directory - destDir.get(CHILDREN).set(newBasename, ino) - // remove from old parent directory - srcDir.get(CHILDREN).delete(oldBasename) - // update stat.path - stat.filepath = newFilepath; - entry.set(STAT, stat); + node.set(PARENT, destDir.get(STAT).ino); + if (node.get(BASENAME) !== basename) { + node.set(BASENAME, basename); + } }, 'rename'); } stat(filepath) { @@ -248,6 +223,7 @@ module.exports = class YjsBackend { ino = nanoid(); } let dir = this._lookup(path.dirname(filepath)); + let parentId = dir.get(STAT).ino; let basename = path.basename(filepath); let stat = { mode, @@ -258,10 +234,17 @@ module.exports = class YjsBackend { ino, }; this._ydoc.transact(() => { - let entry = new Y.Map(); - entry.set(STAT, stat); - this._inodes.set(ino, entry); - dir.get(CHILDREN).set(basename, ino); + let entry = this._inodes.get(ino); + if (!entry) { + entry = new Y.Map(); + entry.set(STAT, stat); + entry.set(PARENT, parentId); + entry.set(BASENAME, basename); + this._inodes.set(ino, entry); + this._computePath(ino); + } else { + entry.set(STAT, stat); + } }, 'symlink'); return stat; } @@ -271,8 +254,7 @@ module.exports = class YjsBackend { if (stat.type === 'file') { size += stat.size; } else if (stat.type === 'dir') { - for (const [name, ino] of dir.get(CHILDREN).entries()) { - const entry = this._inodes.get(ino); + for (const entry of this._childrenOf(stat.ino)) { size += this._du(entry); } } diff --git a/src/__tests__/YFS.spec.js b/src/__tests__/YFS.spec.js index 8d2652d..d05fd36 100755 --- a/src/__tests__/YFS.spec.js +++ b/src/__tests__/YFS.spec.js @@ -165,7 +165,7 @@ describe("YFS module", () => { }); }); - fdescribe("rmdir", () => { + describe("rmdir", () => { it("delete root directory fails", done => { fs.rmdir("/").catch(err => { expect(err).not.toBe(null); From baa2f0dce37cbd0808cd445c965a720802170173 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Wed, 28 Oct 2020 22:38:00 -0400 Subject: [PATCH 14/43] solve 'rename' tracking --- src/YjsBackend.js | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 7ff1fb1..73033fc 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -15,7 +15,9 @@ const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); const STAT = 's'; const CHILDREN = 'c'; const PARENT = 'p'; +const PREVPARENT = '-p'; const BASENAME = 'b'; +const PREVBASENAME = '-b'; module.exports = class YjsBackend { constructor(name) { @@ -189,12 +191,24 @@ module.exports = class YjsBackend { // grab references let node = this._lookup(oldFilepath); let destDir = this._lookup(path.dirname(newFilepath)); - let basename = path.basename(newFilepath); // Update parent this._ydoc.transact(() => { - node.set(PARENT, destDir.get(STAT).ino); - if (node.get(BASENAME) !== basename) { - node.set(BASENAME, basename); + const parent = node.get(PARENT); + const newParent = destDir.get(STAT).ino + if (parent !== newParent) { + node.set(PARENT, newParent); + if (node.get(PREVPARENT) !== parent) { + node.set(PREVPARENT, parent); + } + } + + const basename = node.get(BASENAME); + const newBasename = path.basename(newFilepath); + if (basename !== newBasename) { + node.set(BASENAME, newBasename); + if (node.get(PREVBASENAME) !== basename) { + node.set(PREVBASENAME, basename); + } } }, 'rename'); } From cd6d11a307013e78641f382f360cf7808bbc02e0 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Wed, 28 Oct 2020 22:42:35 -0400 Subject: [PATCH 15/43] solve distinguishing 'unlink' and 'rmdir' via soft-delete --- src/YjsBackend.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 73033fc..d8fc49f 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -18,6 +18,7 @@ const PARENT = 'p'; const PREVPARENT = '-p'; const BASENAME = 'b'; const PREVBASENAME = '-b'; +const DELETED = 'd'; module.exports = class YjsBackend { constructor(name) { @@ -57,14 +58,14 @@ module.exports = class YjsBackend { _childrenOf(id) { const children = []; for (const value of this._inodes.values()) { - if (value.get(PARENT) === id) children.push(value); + if (value.get(PARENT) === id && !value.get(DELETED)) children.push(value); } return children; } _findChild(id, basename) { const children = []; for (const value of this._inodes.values()) { - if (value.get(PARENT) === id && value.get(BASENAME) === basename) return value; + if (value.get(PARENT) === id && value.get(BASENAME) === basename && !value.get(DELETED)) return value; } return; } @@ -128,7 +129,7 @@ module.exports = class YjsBackend { if (this._childrenOf(ino).length > 0) throw new ENOTEMPTY(); // delete inode this._ydoc.transact(() => { - this._inodes.delete(ino); + this._inodes.get(ino).set(DELETED, true); }, 'rmdir'); } readdir(filepath) { @@ -182,7 +183,7 @@ module.exports = class YjsBackend { const ino = node.get(STAT).ino; // delete inode this._ydoc.transact(() => { - this._inodes.delete(ino); + this._inodes.get(ino).set(DELETED, true); }, 'unlink'); } rename(oldFilepath, newFilepath) { From a717505493712489c5e551e8cc0d576fa2a534b3 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Fri, 30 Oct 2020 14:17:46 -0400 Subject: [PATCH 16/43] remove y-indexeddb and y-websocket --- karma.conf.js | 2 +- package-lock.json | 306 ++------------------------------------ package.json | 2 - src/PromisifiedFS.js | 2 +- src/YjsBackend.js | 36 ++--- src/__tests__/YFS.spec.js | 4 +- 6 files changed, 34 insertions(+), 318 deletions(-) diff --git a/karma.conf.js b/karma.conf.js index cda309c..0fc9188 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -184,7 +184,7 @@ module.exports = function (config) { options.browsers = process.env.TEST_BROWSERS.split(',') } else { options.browsers.push('ChromeHeadlessNoSandbox') - // options.browsers.push('FirefoxHeadless') + options.browsers.push('FirefoxHeadless') } console.log('running with browsers:', options.browsers) diff --git a/package-lock.json b/package-lock.json index 3c99f6a..72bbf3a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -695,39 +695,6 @@ "through": ">=2.2.7 <3" } }, - "abstract-leveldown": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", - "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", - "requires": { - "buffer": "^5.5.0", - "immediate": "^3.2.3", - "level-concat-iterator": "~2.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - }, - "dependencies": { - "base64-js": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", - "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" - }, - "buffer": { - "version": "5.6.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.1.tgz", - "integrity": "sha512-2z15UUHpS9/3tk9mY/q+Rl3rydOi7yMp5XWNQnRvoz+mJwiv8brqYwp9a+nOCtma6dwuEIxljD8W3ysVBZ05Vg==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - } - } - }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -1119,7 +1086,8 @@ "async-limiter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", - "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" + "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==", + "dev": true }, "asynckit": { "version": "0.4.0", @@ -2530,15 +2498,6 @@ "dev": true, "optional": true }, - "deferred-leveldown": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz", - "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==", - "requires": { - "abstract-leveldown": "~6.2.1", - "inherits": "^2.0.3" - } - }, "define-property": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", @@ -2771,17 +2730,6 @@ "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", "dev": true }, - "encoding-down": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz", - "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==", - "requires": { - "abstract-leveldown": "^6.2.1", - "inherits": "^2.0.3", - "level-codec": "^9.0.0", - "level-errors": "^2.0.0" - } - }, "end-of-stream": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", @@ -2891,6 +2839,7 @@ "version": "0.1.7", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", + "dev": true, "requires": { "prr": "~1.0.1" } @@ -4766,11 +4715,6 @@ "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", "dev": true }, - "immediate": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.3.0.tgz", - "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==" - }, "import-fresh": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", @@ -4846,7 +4790,8 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", + "dev": true }, "ini": { "version": "1.3.5", @@ -5460,168 +5405,6 @@ "invert-kv": "^2.0.0" } }, - "level": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/level/-/level-6.0.1.tgz", - "integrity": "sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw==", - "requires": { - "level-js": "^5.0.0", - "level-packager": "^5.1.0", - "leveldown": "^5.4.0" - } - }, - "level-codec": { - "version": "9.0.2", - "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.2.tgz", - "integrity": "sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==", - "requires": { - "buffer": "^5.6.0" - }, - "dependencies": { - "base64-js": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", - "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" - }, - "buffer": { - "version": "5.6.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.1.tgz", - "integrity": "sha512-2z15UUHpS9/3tk9mY/q+Rl3rydOi7yMp5XWNQnRvoz+mJwiv8brqYwp9a+nOCtma6dwuEIxljD8W3ysVBZ05Vg==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - } - } - }, - "level-concat-iterator": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", - "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==" - }, - "level-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", - "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", - "requires": { - "errno": "~0.1.1" - } - }, - "level-iterator-stream": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz", - "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==", - "requires": { - "inherits": "^2.0.4", - "readable-stream": "^3.4.0", - "xtend": "^4.0.2" - }, - "dependencies": { - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - }, - "xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" - } - } - }, - "level-js": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/level-js/-/level-js-5.0.2.tgz", - "integrity": "sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg==", - "requires": { - "abstract-leveldown": "~6.2.3", - "buffer": "^5.5.0", - "inherits": "^2.0.3", - "ltgt": "^2.1.2" - }, - "dependencies": { - "base64-js": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", - "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" - }, - "buffer": { - "version": "5.6.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.1.tgz", - "integrity": "sha512-2z15UUHpS9/3tk9mY/q+Rl3rydOi7yMp5XWNQnRvoz+mJwiv8brqYwp9a+nOCtma6dwuEIxljD8W3ysVBZ05Vg==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - } - } - }, - "level-packager": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-5.1.1.tgz", - "integrity": "sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ==", - "requires": { - "encoding-down": "^6.3.0", - "levelup": "^4.3.2" - } - }, - "level-supports": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", - "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", - "requires": { - "xtend": "^4.0.2" - }, - "dependencies": { - "xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" - } - } - }, - "leveldown": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.6.0.tgz", - "integrity": "sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==", - "requires": { - "abstract-leveldown": "~6.2.1", - "napi-macros": "~2.0.0", - "node-gyp-build": "~4.1.0" - } - }, - "levelup": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.4.0.tgz", - "integrity": "sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==", - "requires": { - "deferred-leveldown": "~5.3.0", - "level-errors": "~2.0.0", - "level-iterator-stream": "~4.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -5748,7 +5531,8 @@ "lodash.debounce": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", - "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=" + "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=", + "dev": true }, "lodash.escaperegexp": { "version": "4.1.2", @@ -6076,11 +5860,6 @@ "yallist": "^2.1.2" } }, - "ltgt": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", - "integrity": "sha1-81ypHEk/e3PaDgdJUwTxezH4fuU=" - }, "macos-release": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.4.0.tgz", @@ -6513,11 +6292,6 @@ "to-regex": "^3.0.1" } }, - "napi-macros": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", - "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==" - }, "negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -6564,11 +6338,6 @@ "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==", "dev": true }, - "node-gyp-build": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", - "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==" - }, "node-libs-browser": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.1.0.tgz", @@ -11042,7 +10811,8 @@ "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=", + "dev": true }, "ps-tree": { "version": "1.2.0", @@ -11619,7 +11389,8 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true }, "safe-regex": { "version": "1.1.0", @@ -12667,6 +12438,7 @@ "version": "1.1.1", "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, "requires": { "safe-buffer": "~5.1.0" } @@ -13392,7 +13164,8 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "dev": true }, "utils-merge": { "version": "1.0.1", @@ -13784,55 +13557,8 @@ "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" - }, - "y-indexeddb": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/y-indexeddb/-/y-indexeddb-9.0.5.tgz", - "integrity": "sha512-40VxkqPoK2VxE1vMosS5MfwlHQOvaeLEN89dIkjh7URjZny6bDQOl4yKldaDv9ZosZgYEPyWuWTF3Z92RZ1y+A==", - "requires": { - "lib0": "^0.2.12" - } - }, - "y-leveldb": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/y-leveldb/-/y-leveldb-0.1.0.tgz", - "integrity": "sha512-sMuitVrsAUNh+0b66I42nAuW3lCmez171uP4k0ePcTAJ+c+Iw9w4Yq3wwiyrDMFXBEyQSjSF86Inc23wEvWnxw==", - "requires": { - "level": "^6.0.1", - "lib0": "^0.2.31" - } - }, - "y-protocols": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.1.tgz", - "integrity": "sha512-QP3fCM7c2gGfUi2nqf8gspyO4VW23zv3kNqPNdD3wNxMbuNQenMyoDVZYEo12jzR4RQ3aaDfPK62Sf31SVOmfg==", - "requires": { - "lib0": "^0.2.28" - } - }, - "y-websocket": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/y-websocket/-/y-websocket-1.3.5.tgz", - "integrity": "sha512-TTS6bguW53ciLXuuYkgk/YEOEIwsO7Hy8JMej951g2ePBt9Z8j9riGumYQ/79mltig1IE4ZA6DQp6b10woS5Zw==", - "requires": { - "lib0": "^0.2.31", - "lodash.debounce": "^4.0.8", - "ws": "^6.2.1", - "y-leveldb": "^0.1.0", - "y-protocols": "^1.0.0" - }, - "dependencies": { - "ws": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", - "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==", - "optional": true, - "requires": { - "async-limiter": "~1.0.0" - } - } - } + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", + "dev": true }, "y18n": { "version": "4.0.0", diff --git a/package.json b/package.json index 169638d..4c62b33 100644 --- a/package.json +++ b/package.json @@ -23,8 +23,6 @@ "just-debounce-it": "1.1.0", "just-once": "1.1.0", "nanoid": "^3.1.16", - "y-indexeddb": "^9.0.5", - "y-websocket": "^1.3.5", "yjs": "^13.4.1" }, "devDependencies": { diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index 3ec0857..b652a70 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -83,7 +83,7 @@ module.exports = class PromisifiedFS { await this._gracefulShutdown() this._name = name if (yfs) { - this._yfs = new YjsBackend(this._name); + this._yfs = new YjsBackend(yfs.ydoc); this._cache = this._yfs; this._idb = this._yfs; return this._yfs._ready; diff --git a/src/YjsBackend.js b/src/YjsBackend.js index d8fc49f..9279569 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -1,7 +1,5 @@ -const { encode, decode } = require("isomorphic-textencoder"); +const { encode } = require("isomorphic-textencoder"); const Y = require('yjs'); -const { IndexeddbPersistence } = require('y-indexeddb'); -const { WebsocketProvider } = require('y-websocket'); const { nanoid } = require('nanoid'); const diff = require('fast-diff') @@ -13,7 +11,6 @@ const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); // So for safety, I'm adding NULL because NULL is invalid as a filename character on Linux. And pretty impossible to type using a keyboard. // So that should handle ANY conceivable craziness. const STAT = 's'; -const CHILDREN = 'c'; const PARENT = 'p'; const PREVPARENT = '-p'; const BASENAME = 'b'; @@ -21,25 +18,18 @@ const PREVBASENAME = '-b'; const DELETED = 'd'; module.exports = class YjsBackend { - constructor(name) { - this._ydoc = new Y.Doc(); - this._yidb = new IndexeddbPersistence(name + '_yjs', this._ydoc); - // WIP: I'm adding this to get the BroadcastChannel functionality for the threadsafety tests can run. - this._yws = new WebsocketProvider('wss://demos.yjs.dev', 'stoplight-v0.0.1-' + name + '_yjs', this._ydoc, { connect: false }); - this._ready = this._yidb.whenSynced.then(async () => { - this._inodes = this._ydoc.getMap('!inodes'); - this._content = this._ydoc.getMap('!content'); - if (this._inodes.size === 0) { - const rootdir = new Y.Map(); - const ino = nanoid(); - rootdir.set(STAT, { mode: 0o777, type: "dir", size: 0, ino, mtimeMs: Date.now() }); - rootdir.set(PARENT, null); - rootdir.set(BASENAME, '/'); - this._inodes.set(ino, rootdir); - } - this._yws.connectBc(); - return 'ready'; - }); + constructor(ydoc) { + this._ydoc = ydoc; + this._inodes = this._ydoc.getMap('!inodes'); + this._content = this._ydoc.getMap('!content'); + if (this._inodes.size === 0) { + const rootdir = new Y.Map(); + const ino = nanoid(); + rootdir.set(STAT, { mode: 0o777, type: "dir", size: 0, ino, mtimeMs: Date.now() }); + rootdir.set(PARENT, null); + rootdir.set(BASENAME, '/'); + this._inodes.set(ino, rootdir); + } } get activated () { return !!this._root diff --git a/src/__tests__/YFS.spec.js b/src/__tests__/YFS.spec.js index d05fd36..17f2675 100755 --- a/src/__tests__/YFS.spec.js +++ b/src/__tests__/YFS.spec.js @@ -1,6 +1,8 @@ import FS from "../index.js"; +import * as Y from 'yjs'; -const fs = new FS("testfs-promises2", { wipe: true, yfs: true }).promises; +const ydoc = new Y.Doc(); +const fs = new FS("testfs-yjs", { wipe: true, yfs: { ydoc } }).promises; const HELLO = new Uint8Array([72, 69, 76, 76, 79]); From 709d25b678cede8cc929d4075d4d9395ae2c9121 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Fri, 30 Oct 2020 20:45:32 -0400 Subject: [PATCH 17/43] add fs.promises.openYText(filepath) --- src/PromisifiedFS.js | 3 +++ src/YjsBackend.js | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index b652a70..36785fa 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -321,4 +321,7 @@ module.exports = class PromisifiedFS { async du(filepath) { return this._cache.du(filepath); } + async openYText(filepath) { + return this._yfs.openYText(filepath); + } } diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 9279569..2acfc9f 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -269,6 +269,13 @@ module.exports = class YjsBackend { let dir = this._lookup(filepath); return this._du(dir); } + openYText(filepath) { + let node = this._lookup(filepath, false); + let data = this._content.get(node.get(STAT).ino) + if (data instanceof Y.Text) { + return data; + } + } saveSuperblock(superblock) { return From c06480c8f47a2734a274c32a6dc968d36d45abc9 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 3 Nov 2020 11:48:48 -0500 Subject: [PATCH 18/43] rename openYText -> openYType --- src/PromisifiedFS.js | 4 ++-- src/YjsBackend.js | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index 36785fa..c32bfa0 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -321,7 +321,7 @@ module.exports = class PromisifiedFS { async du(filepath) { return this._cache.du(filepath); } - async openYText(filepath) { - return this._yfs.openYText(filepath); + async openYType(filepath) { + return this._yfs.openYType(filepath); } } diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 2acfc9f..726eadd 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -269,10 +269,10 @@ module.exports = class YjsBackend { let dir = this._lookup(filepath); return this._du(dir); } - openYText(filepath) { + openYType(filepath) { let node = this._lookup(filepath, false); let data = this._content.get(node.get(STAT).ino) - if (data instanceof Y.Text) { + if (data instanceof Y.AbstractType) { return data; } } @@ -321,6 +321,8 @@ module.exports = class YjsBackend { data = new Y.Text(); data.insert(0, rawdata); } + } else if (rawdata instanceof Y.AbstractType) { + data = rawdata; } else { // Yjs will fail if data.constructor !== Uint8Array if (data.constructor.name === 'Buffer') { From 83eea87320c726569e405dc138e751ff51f31dd6 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 3 Nov 2020 17:12:56 -0500 Subject: [PATCH 19/43] add yjs dep --- package-lock.json | 6 +++--- package.json | 2 +- src/YjsBackend.js | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 72bbf3a..f6e5fad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13632,9 +13632,9 @@ "dev": true }, "yjs": { - "version": "13.4.1", - "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.4.1.tgz", - "integrity": "sha512-kIh0sprCTzIm2qyr1VsovkvjKzD2GR4WcU/McJpLAEvImCJHA78Q3S6uSLnhZX0i7FQdrLPCRT8DtTPEH73jnw==", + "version": "13.4.2", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.4.2.tgz", + "integrity": "sha512-HtlrDT55db2Gtu09UDijDCCH+7FLG2FX+TRP0ySZJZYPj22qkinS2oAcuzjKbsJ1Ed0RSJGSgUE6ImagFdMJvA==", "requires": { "lib0": "^0.2.33" } diff --git a/package.json b/package.json index 4c62b33..17b0c0a 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "just-debounce-it": "1.1.0", "just-once": "1.1.0", "nanoid": "^3.1.16", - "yjs": "^13.4.1" + "yjs": "^13.4.2" }, "devDependencies": { "karma": "2.0.5", diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 726eadd..df95b52 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -285,7 +285,8 @@ module.exports = class YjsBackend { } readFileInode(inode) { let data = this._content.get(inode) - if (data instanceof Y.Text) { + // instanceof doesn't work because of different Yjs instances? + if (data.constructor && data.constructor.name === 'YText') { data = encode(data.toString()); } return data; From c4ad1704c521d0a93de174ee5855b8c26f2ab664 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Wed, 4 Nov 2020 15:48:23 -0500 Subject: [PATCH 20/43] use user-provided instance of Yjs --- package.json | 4 +++- src/PromisifiedFS.js | 2 +- src/YjsBackend.js | 23 +++++++++++------------ src/__tests__/YFS.spec.js | 2 +- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 17b0c0a..06629f8 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,9 @@ "isomorphic-textencoder": "1.0.1", "just-debounce-it": "1.1.0", "just-once": "1.1.0", - "nanoid": "^3.1.16", + "nanoid": "^3.1.16" + }, + "peerDependencies": { "yjs": "^13.4.2" }, "devDependencies": { diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index c32bfa0..0ec30be 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -83,7 +83,7 @@ module.exports = class PromisifiedFS { await this._gracefulShutdown() this._name = name if (yfs) { - this._yfs = new YjsBackend(yfs.ydoc); + this._yfs = new YjsBackend(yfs.Y, yfs.ydoc); this._cache = this._yfs; this._idb = this._yfs; return this._yfs._ready; diff --git a/src/YjsBackend.js b/src/YjsBackend.js index df95b52..97606e4 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -1,5 +1,4 @@ const { encode } = require("isomorphic-textencoder"); -const Y = require('yjs'); const { nanoid } = require('nanoid'); const diff = require('fast-diff') @@ -18,12 +17,13 @@ const PREVBASENAME = '-b'; const DELETED = 'd'; module.exports = class YjsBackend { - constructor(ydoc) { + constructor(Y, ydoc) { + this.Y = Y; this._ydoc = ydoc; this._inodes = this._ydoc.getMap('!inodes'); this._content = this._ydoc.getMap('!content'); if (this._inodes.size === 0) { - const rootdir = new Y.Map(); + const rootdir = new this.Y.Map(); const ino = nanoid(); rootdir.set(STAT, { mode: 0o777, type: "dir", size: 0, ino, mtimeMs: Date.now() }); rootdir.set(PARENT, null); @@ -104,7 +104,7 @@ module.exports = class YjsBackend { ino, }; this._ydoc.transact(() => { - let entry = new Y.Map() + let entry = new this.Y.Map() entry.set(STAT, stat); entry.set(PARENT, dir.get(STAT).ino); entry.set(BASENAME, basename); @@ -156,7 +156,7 @@ module.exports = class YjsBackend { this._ydoc.transact(() => { let entry = this._inodes.get(ino); if (!entry) { - entry = new Y.Map(); + entry = new this.Y.Map(); entry.set(STAT, stat); entry.set(PARENT, parentId); entry.set(BASENAME, basename); @@ -241,7 +241,7 @@ module.exports = class YjsBackend { this._ydoc.transact(() => { let entry = this._inodes.get(ino); if (!entry) { - entry = new Y.Map(); + entry = new this.Y.Map(); entry.set(STAT, stat); entry.set(PARENT, parentId); entry.set(BASENAME, basename); @@ -272,7 +272,7 @@ module.exports = class YjsBackend { openYType(filepath) { let node = this._lookup(filepath, false); let data = this._content.get(node.get(STAT).ino) - if (data instanceof Y.AbstractType) { + if (data instanceof this.Y.AbstractType) { return data; } } @@ -285,8 +285,7 @@ module.exports = class YjsBackend { } readFileInode(inode) { let data = this._content.get(inode) - // instanceof doesn't work because of different Yjs instances? - if (data.constructor && data.constructor.name === 'YText') { + if (data.constructor && data instanceof this.Y.Text) { data = encode(data.toString()); } return data; @@ -295,7 +294,7 @@ module.exports = class YjsBackend { if (typeof rawdata === 'string') { // Update existing Text const oldData = this._content.get(inode); - if (oldData && oldData instanceof Y.Text) { + if (oldData && oldData instanceof this.Y.Text) { const oldString = oldData.toString(); const changes = diff(oldString, rawdata); let idx = 0; @@ -319,10 +318,10 @@ module.exports = class YjsBackend { return; } else { // Use new Y.Text - data = new Y.Text(); + data = new this.Y.Text(); data.insert(0, rawdata); } - } else if (rawdata instanceof Y.AbstractType) { + } else if (rawdata instanceof this.Y.AbstractType) { data = rawdata; } else { // Yjs will fail if data.constructor !== Uint8Array diff --git a/src/__tests__/YFS.spec.js b/src/__tests__/YFS.spec.js index 17f2675..7f3117f 100755 --- a/src/__tests__/YFS.spec.js +++ b/src/__tests__/YFS.spec.js @@ -2,7 +2,7 @@ import FS from "../index.js"; import * as Y from 'yjs'; const ydoc = new Y.Doc(); -const fs = new FS("testfs-yjs", { wipe: true, yfs: { ydoc } }).promises; +const fs = new FS("testfs-yjs", { wipe: true, yfs: { Y, ydoc } }).promises; const HELLO = new Uint8Array([72, 69, 76, 76, 79]); From ba63f81a080c22c1d981d2ead5e55b7a787a48f3 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Wed, 4 Nov 2020 15:57:27 -0500 Subject: [PATCH 21/43] unset old _yfs if fs is re-initialized --- src/PromisifiedFS.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index 0ec30be..b1ff78c 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -87,6 +87,8 @@ module.exports = class PromisifiedFS { this._cache = this._yfs; this._idb = this._yfs; return this._yfs._ready; + } else { + this._yfs = void 0; } this._idb = new IdbBackend(fileDbName, fileStoreName); this._mutex = navigator.locks ? new Mutex2(name) : new Mutex(lockDbName, lockStoreName); From 2fce2a18b3a114abc392589d4ee973a84dbc510f Mon Sep 17 00:00:00 2001 From: William Hilton Date: Wed, 4 Nov 2020 16:20:37 -0500 Subject: [PATCH 22/43] bind and make sync --- src/PromisifiedFS.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index b1ff78c..2a0e362 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -50,6 +50,7 @@ module.exports = class PromisifiedFS { this.symlink = this._wrap(this.symlink, true) this.backFile = this._wrap(this.backFile, true) this.du = this._wrap(this.du, false); + this.openYType = this.openYType.bind(this); this.saveSuperblock = debounce(() => { this._saveSuperblock(); @@ -323,7 +324,7 @@ module.exports = class PromisifiedFS { async du(filepath) { return this._cache.du(filepath); } - async openYType(filepath) { + openYType(filepath) { return this._yfs.openYType(filepath); } } From 41bca370e44efdf7f07fac5056910d7f187f9725 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Fri, 6 Nov 2020 11:38:33 -0500 Subject: [PATCH 23/43] wip flatten STAT --- src/YjsBackend.js | 54 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 97606e4..56f15cd 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -9,6 +9,11 @@ const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); // I can still totally see our own code failing to reject ':' when renaming a file though. // So for safety, I'm adding NULL because NULL is invalid as a filename character on Linux. And pretty impossible to type using a keyboard. // So that should handle ANY conceivable craziness. +const TYPE = 't'; +const MTIME = 'm'; +const MODE = 'o'; +const CONTENT = 'c'; +const SIZE = 'i'; const STAT = 's'; const PARENT = 'p'; const PREVPARENT = '-p'; @@ -96,15 +101,21 @@ module.exports = class YjsBackend { } } const ino = nanoid(); + const mtimeMs = Date.now(); let stat = { mode, type: "dir", size: 0, - mtimeMs: Date.now(), + mtimeMs, ino, }; this._ydoc.transact(() => { let entry = new this.Y.Map() + entry.set(MODE, mode); + entry.set(TYPE, 'dir'); + entry.set(SIZE, 0); + entry.set(MTIME, mtimeMs); + entry.set(STAT, stat); entry.set(PARENT, dir.get(STAT).ino); entry.set(BASENAME, basename); @@ -145,11 +156,12 @@ module.exports = class YjsBackend { let dir = this._lookup(path.dirname(filepath)); let parentId = dir.get(STAT).ino; let basename = path.basename(filepath); + const mtimeMs = Date.now(); let stat = { mode, type: "file", size, - mtimeMs: Date.now(), + mtimeMs, ino, filepath, }; @@ -157,12 +169,22 @@ module.exports = class YjsBackend { let entry = this._inodes.get(ino); if (!entry) { entry = new this.Y.Map(); + entry.set(MODE, mode); + entry.set(TYPE, 'file'); + entry.set(SIZE, size); + entry.set(MTIME, mtimeMs); + entry.set(STAT, stat); entry.set(PARENT, parentId); entry.set(BASENAME, basename); this._inodes.set(ino, entry); this._computePath(ino); } else { + entry.set(MODE, mode); + entry.set(TYPE, 'file'); + entry.set(SIZE, size); + entry.set(MTIME, mtimeMs); + entry.set(STAT, stat); } }, 'writeFile'); @@ -204,10 +226,26 @@ module.exports = class YjsBackend { }, 'rename'); } stat(filepath) { - return this._lookup(filepath).get(STAT); + const node = this._lookup(filepath); + const stat = { + mode: node.get(MODE), + type: node.get(TYPE), + size: node.get(SIZE), + mtimeMs: node.get(MTIME), + ino: node._item.parentSub, + }; + return stat; } lstat(filepath) { - return this._lookup(filepath, false).get(STAT); + const node = this._lookup(filepath, false); + const stat = { + mode: node.get(MODE), + type: node.get(TYPE), + size: node.get(SIZE), + mtimeMs: node.get(MTIME), + ino: node._item.parentSub, + }; + return stat; } readlink(filepath) { return this._lookup(filepath, false).get(STAT).target; @@ -230,18 +268,24 @@ module.exports = class YjsBackend { let dir = this._lookup(path.dirname(filepath)); let parentId = dir.get(STAT).ino; let basename = path.basename(filepath); + const mtimeMs = Date.now(); let stat = { mode, type: "symlink", target, size: 0, - mtimeMs: Date.now(), + mtimeMs, ino, }; this._ydoc.transact(() => { let entry = this._inodes.get(ino); if (!entry) { entry = new this.Y.Map(); + entry.set(MODE, mode); + entry.set(TYPE, 'symlink'); + entry.set(SIZE, 0); + entry.set(MTIME, mtimeMs); + entry.set(STAT, stat); entry.set(PARENT, parentId); entry.set(BASENAME, basename); From 61018e793e4c02951413c2745615d9ad276ea0bc Mon Sep 17 00:00:00 2001 From: William Hilton Date: Fri, 6 Nov 2020 11:56:55 -0500 Subject: [PATCH 24/43] finish flattening STAT --- src/YjsBackend.js | 66 +++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 56f15cd..7cb8af7 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -5,16 +5,11 @@ const diff = require('fast-diff') const path = require("./path.js"); const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); -// ':' is invalid as a filename character on both Mac and Windows, so these shouldn't conflict with real filenames. -// I can still totally see our own code failing to reject ':' when renaming a file though. -// So for safety, I'm adding NULL because NULL is invalid as a filename character on Linux. And pretty impossible to type using a keyboard. -// So that should handle ANY conceivable craziness. const TYPE = 't'; const MTIME = 'm'; const MODE = 'o'; const CONTENT = 'c'; const SIZE = 'i'; -const STAT = 's'; const PARENT = 'p'; const PREVPARENT = '-p'; const BASENAME = 'b'; @@ -30,7 +25,14 @@ module.exports = class YjsBackend { if (this._inodes.size === 0) { const rootdir = new this.Y.Map(); const ino = nanoid(); - rootdir.set(STAT, { mode: 0o777, type: "dir", size: 0, ino, mtimeMs: Date.now() }); + const mtimeMs = Date.now(); + const mode = 0o777; + + rootdir.set(MODE, mode); + rootdir.set(TYPE, 'dir'); + rootdir.set(SIZE, 0); + rootdir.set(MTIME, mtimeMs); + rootdir.set(PARENT, null); rootdir.set(BASENAME, '/'); this._inodes.set(ino, rootdir); @@ -73,13 +75,12 @@ module.exports = class YjsBackend { // narrowing that set. The problem would be dealing with symlinks. for (let i = 0; i < parts.length; ++ i) { let part = parts[i]; - dir = this._findChild(dir && dir.get(STAT).ino, part); + dir = this._findChild(dir && dir._item.parentSub, part); if (!dir) throw new ENOENT(filepath); // Follow symlinks if (follow || i < parts.length - 1) { - const stat = dir.get(STAT) - if (stat.type === 'symlink') { - let target = path.resolve(partialPath, stat.target) + if (dir.get(TYPE) === 'symlink') { + let target = path.resolve(partialPath, dir.get(CONTENT)) dir = this._lookup(target) } if (!partialPath) { @@ -95,7 +96,7 @@ module.exports = class YjsBackend { if (filepath === "/") throw new EEXIST(); let dir = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); - for (const child of this._childrenOf(dir.get(STAT).ino)) { + for (const child of this._childrenOf(dir._item.parentSub)) { if (child.get(BASENAME) === basename) { throw new EEXIST(); } @@ -116,16 +117,15 @@ module.exports = class YjsBackend { entry.set(SIZE, 0); entry.set(MTIME, mtimeMs); - entry.set(STAT, stat); - entry.set(PARENT, dir.get(STAT).ino); + entry.set(PARENT, dir._item.parentSub); entry.set(BASENAME, basename); this._inodes.set(ino, entry); }, 'mkdir'); } rmdir(filepath) { let dir = this._lookup(filepath); - if (dir.get(STAT).type !== 'dir') throw new ENOTDIR(); - const ino = dir.get(STAT).ino; + if (dir.get(TYPE) !== 'dir') throw new ENOTDIR(); + const ino = dir._item.parentSub; // check it's empty if (this._childrenOf(ino).length > 0) throw new ENOTEMPTY(); // delete inode @@ -135,8 +135,8 @@ module.exports = class YjsBackend { } readdir(filepath) { let dir = this._lookup(filepath); - if (dir.get(STAT).type !== 'dir') throw new ENOTDIR(); - return this._childrenOf(dir.get(STAT).ino).map(node => node.get(BASENAME)); + if (dir.get(TYPE) !== 'dir') throw new ENOTDIR(); + return this._childrenOf(dir._item.parentSub).map(node => node.get(BASENAME)); } writeStat(filepath, size, { mode }) { let ino; @@ -154,7 +154,7 @@ module.exports = class YjsBackend { ino = nanoid(); } let dir = this._lookup(path.dirname(filepath)); - let parentId = dir.get(STAT).ino; + let parentId = dir._item.parentSub; let basename = path.basename(filepath); const mtimeMs = Date.now(); let stat = { @@ -174,7 +174,6 @@ module.exports = class YjsBackend { entry.set(SIZE, size); entry.set(MTIME, mtimeMs); - entry.set(STAT, stat); entry.set(PARENT, parentId); entry.set(BASENAME, basename); this._inodes.set(ino, entry); @@ -184,15 +183,13 @@ module.exports = class YjsBackend { entry.set(TYPE, 'file'); entry.set(SIZE, size); entry.set(MTIME, mtimeMs); - - entry.set(STAT, stat); } }, 'writeFile'); return stat; } unlink(filepath) { let node = this._lookup(filepath, false); - const ino = node.get(STAT).ino; + const ino = node._item.parentSub; // delete inode this._ydoc.transact(() => { this._inodes.get(ino).set(DELETED, true); @@ -207,7 +204,7 @@ module.exports = class YjsBackend { // Update parent this._ydoc.transact(() => { const parent = node.get(PARENT); - const newParent = destDir.get(STAT).ino + const newParent = destDir._item.parentSub if (parent !== newParent) { node.set(PARENT, newParent); if (node.get(PREVPARENT) !== parent) { @@ -248,7 +245,7 @@ module.exports = class YjsBackend { return stat; } readlink(filepath) { - return this._lookup(filepath, false).get(STAT).target; + return this._lookup(filepath, false).get(CONTENT); } symlink(target, filepath) { let ino, mode; @@ -266,7 +263,7 @@ module.exports = class YjsBackend { ino = nanoid(); } let dir = this._lookup(path.dirname(filepath)); - let parentId = dir.get(STAT).ino; + let parentId = dir._item.parentSub; let basename = path.basename(filepath); const mtimeMs = Date.now(); let stat = { @@ -285,25 +282,28 @@ module.exports = class YjsBackend { entry.set(TYPE, 'symlink'); entry.set(SIZE, 0); entry.set(MTIME, mtimeMs); + entry.set(CONTENT, target); - entry.set(STAT, stat); entry.set(PARENT, parentId); entry.set(BASENAME, basename); this._inodes.set(ino, entry); this._computePath(ino); } else { - entry.set(STAT, stat); + entry.set(MODE, mode); + entry.set(TYPE, 'symlink'); + entry.set(SIZE, 0); + entry.set(MTIME, mtimeMs); } }, 'symlink'); return stat; } _du (dir) { let size = 0; - const stat = dir.get(STAT) - if (stat.type === 'file') { - size += stat.size; - } else if (stat.type === 'dir') { - for (const entry of this._childrenOf(stat.ino)) { + const type = dir.get(TYPE) + if (type === 'file') { + size += dir.get(SIZE); + } else if (type === 'dir') { + for (const entry of this._childrenOf(dir._item.parentSub)) { size += this._du(entry); } } @@ -315,7 +315,7 @@ module.exports = class YjsBackend { } openYType(filepath) { let node = this._lookup(filepath, false); - let data = this._content.get(node.get(STAT).ino) + let data = this._content.get(node._item.parentSub) if (data instanceof this.Y.AbstractType) { return data; } From 62ae5853a571312b208839971cfc802475581940 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Fri, 6 Nov 2020 15:12:03 -0500 Subject: [PATCH 25/43] fixes to return correct ino in writeStat --- src/YjsBackend.js | 39 ++++++++++----------------------------- 1 file changed, 10 insertions(+), 29 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 7cb8af7..29ff5fc 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -103,13 +103,6 @@ module.exports = class YjsBackend { } const ino = nanoid(); const mtimeMs = Date.now(); - let stat = { - mode, - type: "dir", - size: 0, - mtimeMs, - ino, - }; this._ydoc.transact(() => { let entry = new this.Y.Map() entry.set(MODE, mode); @@ -141,11 +134,11 @@ module.exports = class YjsBackend { writeStat(filepath, size, { mode }) { let ino; try { - let oldStat = this.stat(filepath); + const node = this._lookup(filepath); if (mode == null) { - mode = oldStat.mode; + mode = node.get(MODE); } - ino = oldStat.ino; + ino = node._item.parentSub; } catch (err) {} if (mode == null) { mode = 0o666; @@ -157,14 +150,7 @@ module.exports = class YjsBackend { let parentId = dir._item.parentSub; let basename = path.basename(filepath); const mtimeMs = Date.now(); - let stat = { - mode, - type: "file", - size, - mtimeMs, - ino, - filepath, - }; + this._ydoc.transact(() => { let entry = this._inodes.get(ino); if (!entry) { @@ -185,6 +171,7 @@ module.exports = class YjsBackend { entry.set(MTIME, mtimeMs); } }, 'writeFile'); + const stat = this.stat(filepath); return stat; } unlink(filepath) { @@ -250,11 +237,11 @@ module.exports = class YjsBackend { symlink(target, filepath) { let ino, mode; try { - let oldStat = this.stat(filepath); + const node = this._lookup(filepath); if (mode === null) { - mode = oldStat.mode; + mode = node.get(MODE); } - ino = oldStat.ino; + ino = node._item.parentSub; } catch (err) {} if (mode == null) { mode = 0o120000; @@ -266,14 +253,7 @@ module.exports = class YjsBackend { let parentId = dir._item.parentSub; let basename = path.basename(filepath); const mtimeMs = Date.now(); - let stat = { - mode, - type: "symlink", - target, - size: 0, - mtimeMs, - ino, - }; + this._ydoc.transact(() => { let entry = this._inodes.get(ino); if (!entry) { @@ -295,6 +275,7 @@ module.exports = class YjsBackend { entry.set(MTIME, mtimeMs); } }, 'symlink'); + const stat = this.lstat(filepath); return stat; } _du (dir) { From 0e07324ac18b84a72647207d0d81cf11a2efda99 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Fri, 6 Nov 2020 16:01:07 -0500 Subject: [PATCH 26/43] store content in inode, remove deleted flag --- src/YjsBackend.js | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 29ff5fc..990d142 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -14,14 +14,12 @@ const PARENT = 'p'; const PREVPARENT = '-p'; const BASENAME = 'b'; const PREVBASENAME = '-b'; -const DELETED = 'd'; module.exports = class YjsBackend { constructor(Y, ydoc) { this.Y = Y; this._ydoc = ydoc; this._inodes = this._ydoc.getMap('!inodes'); - this._content = this._ydoc.getMap('!content'); if (this._inodes.size === 0) { const rootdir = new this.Y.Map(); const ino = nanoid(); @@ -32,6 +30,7 @@ module.exports = class YjsBackend { rootdir.set(TYPE, 'dir'); rootdir.set(SIZE, 0); rootdir.set(MTIME, mtimeMs); + rootdir.set(CONTENT, true); rootdir.set(PARENT, null); rootdir.set(BASENAME, '/'); @@ -55,14 +54,14 @@ module.exports = class YjsBackend { _childrenOf(id) { const children = []; for (const value of this._inodes.values()) { - if (value.get(PARENT) === id && !value.get(DELETED)) children.push(value); + if (value.get(PARENT) === id && value.get(CONTENT)) children.push(value); } return children; } _findChild(id, basename) { const children = []; for (const value of this._inodes.values()) { - if (value.get(PARENT) === id && value.get(BASENAME) === basename && !value.get(DELETED)) return value; + if (value.get(PARENT) === id && value.get(BASENAME) === basename && value.get(CONTENT)) return value; } return; } @@ -109,6 +108,7 @@ module.exports = class YjsBackend { entry.set(TYPE, 'dir'); entry.set(SIZE, 0); entry.set(MTIME, mtimeMs); + entry.set(CONTENT, true); // must be truthy or else directory is in a "deleted" state entry.set(PARENT, dir._item.parentSub); entry.set(BASENAME, basename); @@ -123,7 +123,7 @@ module.exports = class YjsBackend { if (this._childrenOf(ino).length > 0) throw new ENOTEMPTY(); // delete inode this._ydoc.transact(() => { - this._inodes.get(ino).set(DELETED, true); + this._inodes.get(ino).set(CONTENT, false); }, 'rmdir'); } readdir(filepath) { @@ -159,6 +159,7 @@ module.exports = class YjsBackend { entry.set(TYPE, 'file'); entry.set(SIZE, size); entry.set(MTIME, mtimeMs); + entry.set(CONTENT, true); // set to truthy so file isn't in a "deleted" state entry.set(PARENT, parentId); entry.set(BASENAME, basename); @@ -179,7 +180,7 @@ module.exports = class YjsBackend { const ino = node._item.parentSub; // delete inode this._ydoc.transact(() => { - this._inodes.get(ino).set(DELETED, true); + this._inodes.get(ino).set(CONTENT, false); }, 'unlink'); } rename(oldFilepath, newFilepath) { @@ -296,7 +297,7 @@ module.exports = class YjsBackend { } openYType(filepath) { let node = this._lookup(filepath, false); - let data = this._content.get(node._item.parentSub) + let data = node.get(CONTENT) if (data instanceof this.Y.AbstractType) { return data; } @@ -309,7 +310,7 @@ module.exports = class YjsBackend { return } readFileInode(inode) { - let data = this._content.get(inode) + let data = this._inodes.get(inode).get(CONTENT); if (data.constructor && data instanceof this.Y.Text) { data = encode(data.toString()); } @@ -318,7 +319,7 @@ module.exports = class YjsBackend { writeFileInode(inode, data, rawdata) { if (typeof rawdata === 'string') { // Update existing Text - const oldData = this._content.get(inode); + const oldData = this._inodes.get(inode).get(CONTENT); if (oldData && oldData instanceof this.Y.Text) { const oldString = oldData.toString(); const changes = diff(oldString, rawdata); @@ -354,10 +355,10 @@ module.exports = class YjsBackend { data = new Uint8Array(data.buffer); } } - return this._content.set(inode, data); + return this._inodes.get(inode).set(CONTENT, data); } unlinkInode(inode) { - return this._content.delete(inode) + return this._inodes.get(inode).set(CONTENT, false); } wipe() { return [...this._root.keys()].map(key => this._root.delete(key)) From d1af63422131608e052af53473feee20ef206c95 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Fri, 6 Nov 2020 16:16:31 -0500 Subject: [PATCH 27/43] remove SIZE --- src/YjsBackend.js | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 990d142..edcfec4 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -9,7 +9,6 @@ const TYPE = 't'; const MTIME = 'm'; const MODE = 'o'; const CONTENT = 'c'; -const SIZE = 'i'; const PARENT = 'p'; const PREVPARENT = '-p'; const BASENAME = 'b'; @@ -28,7 +27,6 @@ module.exports = class YjsBackend { rootdir.set(MODE, mode); rootdir.set(TYPE, 'dir'); - rootdir.set(SIZE, 0); rootdir.set(MTIME, mtimeMs); rootdir.set(CONTENT, true); @@ -106,7 +104,6 @@ module.exports = class YjsBackend { let entry = new this.Y.Map() entry.set(MODE, mode); entry.set(TYPE, 'dir'); - entry.set(SIZE, 0); entry.set(MTIME, mtimeMs); entry.set(CONTENT, true); // must be truthy or else directory is in a "deleted" state @@ -157,7 +154,6 @@ module.exports = class YjsBackend { entry = new this.Y.Map(); entry.set(MODE, mode); entry.set(TYPE, 'file'); - entry.set(SIZE, size); entry.set(MTIME, mtimeMs); entry.set(CONTENT, true); // set to truthy so file isn't in a "deleted" state @@ -168,7 +164,6 @@ module.exports = class YjsBackend { } else { entry.set(MODE, mode); entry.set(TYPE, 'file'); - entry.set(SIZE, size); entry.set(MTIME, mtimeMs); } }, 'writeFile'); @@ -215,7 +210,7 @@ module.exports = class YjsBackend { const stat = { mode: node.get(MODE), type: node.get(TYPE), - size: node.get(SIZE), + size: this._size(node), mtimeMs: node.get(MTIME), ino: node._item.parentSub, }; @@ -226,7 +221,7 @@ module.exports = class YjsBackend { const stat = { mode: node.get(MODE), type: node.get(TYPE), - size: node.get(SIZE), + size: this._size(node), mtimeMs: node.get(MTIME), ino: node._item.parentSub, }; @@ -261,7 +256,6 @@ module.exports = class YjsBackend { entry = new this.Y.Map(); entry.set(MODE, mode); entry.set(TYPE, 'symlink'); - entry.set(SIZE, 0); entry.set(MTIME, mtimeMs); entry.set(CONTENT, target); @@ -272,7 +266,6 @@ module.exports = class YjsBackend { } else { entry.set(MODE, mode); entry.set(TYPE, 'symlink'); - entry.set(SIZE, 0); entry.set(MTIME, mtimeMs); } }, 'symlink'); @@ -283,7 +276,7 @@ module.exports = class YjsBackend { let size = 0; const type = dir.get(TYPE) if (type === 'file') { - size += dir.get(SIZE); + size += this._size(dir); } else if (type === 'dir') { for (const entry of this._childrenOf(dir._item.parentSub)) { size += this._du(entry); @@ -366,4 +359,18 @@ module.exports = class YjsBackend { close() { return } + + _size(node) { + if (node.get(TYPE) !== 'file') return 0; + + const content = node.get(CONTENT); + + if (content instanceof this.Y.Text || typeof content === 'string') { + return content.length; + } else if (content instanceof Uint8Array) { + return content.byteLength; + } else { + return 0; + } + } } From 39a7019f5f4053a89df8e625af6a75c7f8393e4c Mon Sep 17 00:00:00 2001 From: William Hilton Date: Fri, 6 Nov 2020 21:17:38 -0500 Subject: [PATCH 28/43] benchmark --- src/__tests__/YFS.spec.js | 64 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/src/__tests__/YFS.spec.js b/src/__tests__/YFS.spec.js index 7f3117f..ca07f4b 100755 --- a/src/__tests__/YFS.spec.js +++ b/src/__tests__/YFS.spec.js @@ -1,6 +1,10 @@ +jasmine.DEFAULT_TIMEOUT_INTERVAL = 20000 + import FS from "../index.js"; import * as Y from 'yjs'; +import { find } from 'yjs/src/utils/StructStore'; + const ydoc = new Y.Doc(); const fs = new FS("testfs-yjs", { wipe: true, yfs: { Y, ydoc } }).promises; @@ -12,7 +16,7 @@ if (!Promise.prototype.finally) { } } -describe("YFS module", () => { +fdescribe("YFS module", () => { describe("mkdir", () => { it("root directory already exists", (done) => { fs.mkdir("/").catch(err => { @@ -477,4 +481,62 @@ describe("YFS module", () => { }); }); + fdescribe("benchmark", () => { + it("10 dir x 10 dir x 10 files", done => { + const range = n => [...Array(n).keys()]; + const start = performance.now(); + fs.mkdir(`/benchmark`) + .then(() =>Promise.all(range(10).map( + i => fs.mkdir(`/benchmark/dir${i}`).then( + () => Promise.all(range(10).map( + j => fs.mkdir(`/benchmark/dir${i}/sub${j}`).then( + () => Promise.all(range(100).map( + k => fs.writeFile(`/benchmark/dir${i}/sub${j}/file${k}`, 'A', 'utf8') + )) + ) + )) + ) + ))) + .then(() => fs.du('/benchmark')) + .then(size => { + expect(size).toBe(10000); + const end = performance.now(); + console.log(`TIME: ${end - start}ms`); + // const keys = [...ydoc.getMap('!inodes').keys()]; + const inodes = ydoc.getMap('!inodes'); + let size2 = 0 + const keys = [...inodes.keys()]; + const ids = keys.map(key => inodes.get(key)._item.id); + + const kstart = performance.now() + for (const key of keys) { + const node = inodes.get(key); + const content = node.get('c'); + if (content && content.length) { + size2 += content.length + } + } + const kend = performance.now() + + const idstart = performance.now() + for (const id of ids) { + const item = find(ydoc.store, id) + const node = item.content.type; + const content = node.get('c'); + if (content && content.length) { + size2 += content.length + } + } + const idend = performance.now() + + console.log(`LOOKUP TIME: ${kend - kstart}ms vs ${idend - idstart}ms`); + expect(size2).toBe(size * 2); + + const update = Y.encodeStateAsUpdate(ydoc); + console.log(`YJS SIZE: ${update.byteLength}`); + done(); + }); + }); + }); + }); From d1bf1988b3818b9df1b17db8e666a3b143662933 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Fri, 6 Nov 2020 23:08:07 -0500 Subject: [PATCH 29/43] WIP migrating to client-clock --- src/PromisifiedFS.js | 2 +- src/YjsBackend.js | 201 ++++++++++++++++++++------------------ src/__tests__/YFS.spec.js | 13 ++- 3 files changed, 114 insertions(+), 102 deletions(-) diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index 2a0e362..317ebcf 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -84,7 +84,7 @@ module.exports = class PromisifiedFS { await this._gracefulShutdown() this._name = name if (yfs) { - this._yfs = new YjsBackend(yfs.Y, yfs.ydoc); + this._yfs = new YjsBackend(yfs.Y, yfs.ydoc, yfs.find); this._cache = this._yfs; this._idb = this._yfs; return this._yfs._ready; diff --git a/src/YjsBackend.js b/src/YjsBackend.js index edcfec4..a7d0f63 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -1,5 +1,4 @@ const { encode } = require("isomorphic-textencoder"); -const { nanoid } = require('nanoid'); const diff = require('fast-diff') const path = require("./path.js"); @@ -14,14 +13,37 @@ const PREVPARENT = '-p'; const BASENAME = 'b'; const PREVBASENAME = '-b'; +function ID (client, clock) { + this.client = client; + this.clock = clock; +} + +function serializeID (id) { + const buffer = new ArrayBuffer(16); + const dataview = new DataView(buffer); + dataview.setFloat64(0, id.client); + dataview.setFloat64(8, id.clock); + return new Uint8Array(buffer); +} + +function parseID (arr) { + if (!arr) return arr; + const dataview = new DataView(arr.buffer); + return new ID(dataview.getFloat64(0), dataview.getFloat64(8)); +} + +function sameID (id1, id2) { + return id1.client === id2.client && id1.clock === id2.clock; +} + module.exports = class YjsBackend { - constructor(Y, ydoc) { + constructor(Y, ydoc, find) { this.Y = Y; this._ydoc = ydoc; - this._inodes = this._ydoc.getMap('!inodes'); - if (this._inodes.size === 0) { + this._find = find; + this._inodes = this._ydoc.getArray('!inodes'); + if (this._inodes.length === 0) { const rootdir = new this.Y.Map(); - const ino = nanoid(); const mtimeMs = Date.now(); const mode = 0o777; @@ -32,47 +54,43 @@ module.exports = class YjsBackend { rootdir.set(PARENT, null); rootdir.set(BASENAME, '/'); - this._inodes.set(ino, rootdir); + this._inodes.push([rootdir]); } } get activated () { - return !!this._root + return true } - _computePath(ino) { - let parts = []; - while (ino != null) { - const dir = this._inodes.get(ino) - if (!dir) break; - parts.unshift(dir.get(BASENAME)) - ino = dir.get(PARENT) - } - const filepath = path.join(parts); - return filepath; + _getInode(id) { + const item = this._find(this._ydoc.store, id) + const node = item.content.type; + return node; } _childrenOf(id) { const children = []; - for (const value of this._inodes.values()) { - if (value.get(PARENT) === id && value.get(CONTENT)) children.push(value); + for (const value of this._inodes) { + const parent = parseID(value.get(PARENT)) + if (parent && sameID(parent, id) && value.get(CONTENT)) children.push([value]); } return children; } _findChild(id, basename) { - const children = []; - for (const value of this._inodes.values()) { - if (value.get(PARENT) === id && value.get(BASENAME) === basename && value.get(CONTENT)) return value; + for (const value of this._inodes) { + const parent = parseID(value.get(PARENT)) + if (parent && sameID(parent, id) && value.get(BASENAME) === basename && value.get(CONTENT)) return value; } return; } _lookup(filepath, follow = true) { - let dir = null; + let dir = this._inodes.get(0); + if (filepath === '/') return dir; let partialPath = '/' let parts = path.split(filepath) // TODO: Actually, given we can reconstruct paths from the bottom up, // it might be faster to search by matching against the basepath and then // narrowing that set. The problem would be dealing with symlinks. - for (let i = 0; i < parts.length; ++ i) { + for (let i = 1; i < parts.length; ++ i) { let part = parts[i]; - dir = this._findChild(dir && dir._item.parentSub, part); + dir = this._findChild(dir._item.id, part); if (!dir) throw new ENOENT(filepath); // Follow symlinks if (follow || i < parts.length - 1) { @@ -93,89 +111,84 @@ module.exports = class YjsBackend { if (filepath === "/") throw new EEXIST(); let dir = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); - for (const child of this._childrenOf(dir._item.parentSub)) { + for (const child of this._childrenOf(dir._item.id)) { if (child.get(BASENAME) === basename) { throw new EEXIST(); } } - const ino = nanoid(); const mtimeMs = Date.now(); this._ydoc.transact(() => { - let entry = new this.Y.Map() - entry.set(MODE, mode); - entry.set(TYPE, 'dir'); - entry.set(MTIME, mtimeMs); - entry.set(CONTENT, true); // must be truthy or else directory is in a "deleted" state + let node = new this.Y.Map() + node.set(MODE, mode); + node.set(TYPE, 'dir'); + node.set(MTIME, mtimeMs); + node.set(CONTENT, true); // must be truthy or else directory is in a "deleted" state - entry.set(PARENT, dir._item.parentSub); - entry.set(BASENAME, basename); - this._inodes.set(ino, entry); + node.set(PARENT, serializeID(dir._item.id)); + node.set(BASENAME, basename); + this._inodes.push([node]); + console.log('mkdir', JSON.stringify(node.toJSON())); }, 'mkdir'); } rmdir(filepath) { let dir = this._lookup(filepath); if (dir.get(TYPE) !== 'dir') throw new ENOTDIR(); - const ino = dir._item.parentSub; + const ino = dir._item.id; // check it's empty if (this._childrenOf(ino).length > 0) throw new ENOTEMPTY(); // delete inode this._ydoc.transact(() => { - this._inodes.get(ino).set(CONTENT, false); + dir.set(CONTENT, false); }, 'rmdir'); } readdir(filepath) { let dir = this._lookup(filepath); if (dir.get(TYPE) !== 'dir') throw new ENOTDIR(); - return this._childrenOf(dir._item.parentSub).map(node => node.get(BASENAME)); + return this._childrenOf(dir._item.id).map(node => node.get(BASENAME)); } writeStat(filepath, size, { mode }) { - let ino; + let node try { - const node = this._lookup(filepath); + node = this._lookup(filepath); if (mode == null) { mode = node.get(MODE); } - ino = node._item.parentSub; } catch (err) {} if (mode == null) { mode = 0o666; } - if (ino == null) { - ino = nanoid(); - } let dir = this._lookup(path.dirname(filepath)); - let parentId = dir._item.parentSub; + let parentId = dir._item.id; let basename = path.basename(filepath); const mtimeMs = Date.now(); this._ydoc.transact(() => { - let entry = this._inodes.get(ino); - if (!entry) { - entry = new this.Y.Map(); - entry.set(MODE, mode); - entry.set(TYPE, 'file'); - entry.set(MTIME, mtimeMs); - entry.set(CONTENT, true); // set to truthy so file isn't in a "deleted" state + if (!node) { + node = new this.Y.Map(); + node.set(MODE, mode); + node.set(TYPE, 'file'); + node.set(MTIME, mtimeMs); + node.set(CONTENT, true); // set to truthy so file isn't in a "deleted" state - entry.set(PARENT, parentId); - entry.set(BASENAME, basename); - this._inodes.set(ino, entry); - this._computePath(ino); + node.set(PARENT, serializeID(parentId)); + node.set(BASENAME, basename); + this._inodes.push([node]); + console.log('writeStat', JSON.stringify(node.toJSON())); } else { - entry.set(MODE, mode); - entry.set(TYPE, 'file'); - entry.set(MTIME, mtimeMs); + node.set(MODE, mode); + node.set(TYPE, 'file'); + node.set(MTIME, mtimeMs); } }, 'writeFile'); const stat = this.stat(filepath); + console.log('stat', stat); return stat; } unlink(filepath) { let node = this._lookup(filepath, false); - const ino = node._item.parentSub; // delete inode this._ydoc.transact(() => { - this._inodes.get(ino).set(CONTENT, false); + node.set(CONTENT, false); }, 'unlink'); } rename(oldFilepath, newFilepath) { @@ -186,12 +199,13 @@ module.exports = class YjsBackend { let destDir = this._lookup(path.dirname(newFilepath)); // Update parent this._ydoc.transact(() => { - const parent = node.get(PARENT); - const newParent = destDir._item.parentSub - if (parent !== newParent) { - node.set(PARENT, newParent); - if (node.get(PREVPARENT) !== parent) { - node.set(PREVPARENT, parent); + const parent = parseID(node.get(PARENT)); + const newParent = destDir._item.id + if (!sameID(parent, newParent)) { + node.set(PARENT, serializeID(newParent)); + const prevParent = parseID(node.get(PREVPARENT)); + if (!sameID(prevParent, parent)) { + node.set(PREVPARENT, node.get(PARENT)); } } @@ -212,7 +226,7 @@ module.exports = class YjsBackend { type: node.get(TYPE), size: this._size(node), mtimeMs: node.get(MTIME), - ino: node._item.parentSub, + ino: node._item.id, }; return stat; } @@ -223,7 +237,7 @@ module.exports = class YjsBackend { type: node.get(TYPE), size: this._size(node), mtimeMs: node.get(MTIME), - ino: node._item.parentSub, + ino: node._item.id, }; return stat; } @@ -231,42 +245,36 @@ module.exports = class YjsBackend { return this._lookup(filepath, false).get(CONTENT); } symlink(target, filepath) { - let ino, mode; + let mode, node; try { - const node = this._lookup(filepath); + node = this._lookup(filepath); if (mode === null) { mode = node.get(MODE); } - ino = node._item.parentSub; } catch (err) {} if (mode == null) { mode = 0o120000; } - if (ino == null) { - ino = nanoid(); - } let dir = this._lookup(path.dirname(filepath)); - let parentId = dir._item.parentSub; + let parentId = dir._item.id; let basename = path.basename(filepath); const mtimeMs = Date.now(); this._ydoc.transact(() => { - let entry = this._inodes.get(ino); - if (!entry) { - entry = new this.Y.Map(); - entry.set(MODE, mode); - entry.set(TYPE, 'symlink'); - entry.set(MTIME, mtimeMs); - entry.set(CONTENT, target); + if (!node) { + node = new this.Y.Map(); + node.set(MODE, mode); + node.set(TYPE, 'symlink'); + node.set(MTIME, mtimeMs); + node.set(CONTENT, target); - entry.set(PARENT, parentId); - entry.set(BASENAME, basename); - this._inodes.set(ino, entry); - this._computePath(ino); + node.set(PARENT, serializeID(parentId)); + node.set(BASENAME, basename); + this._inodes.push([node]); } else { - entry.set(MODE, mode); - entry.set(TYPE, 'symlink'); - entry.set(MTIME, mtimeMs); + node.set(MODE, mode); + node.set(TYPE, 'symlink'); + node.set(MTIME, mtimeMs); } }, 'symlink'); const stat = this.lstat(filepath); @@ -278,7 +286,7 @@ module.exports = class YjsBackend { if (type === 'file') { size += this._size(dir); } else if (type === 'dir') { - for (const entry of this._childrenOf(dir._item.parentSub)) { + for (const entry of this._childrenOf(dir._item.id)) { size += this._du(entry); } } @@ -303,7 +311,7 @@ module.exports = class YjsBackend { return } readFileInode(inode) { - let data = this._inodes.get(inode).get(CONTENT); + let data = this._getInode(inode).get(CONTENT); if (data.constructor && data instanceof this.Y.Text) { data = encode(data.toString()); } @@ -312,7 +320,7 @@ module.exports = class YjsBackend { writeFileInode(inode, data, rawdata) { if (typeof rawdata === 'string') { // Update existing Text - const oldData = this._inodes.get(inode).get(CONTENT); + const oldData = this._getInode(inode).get(CONTENT); if (oldData && oldData instanceof this.Y.Text) { const oldString = oldData.toString(); const changes = diff(oldString, rawdata); @@ -348,13 +356,14 @@ module.exports = class YjsBackend { data = new Uint8Array(data.buffer); } } - return this._inodes.get(inode).set(CONTENT, data); + this._getInode(inode).set(CONTENT, data); + return; } unlinkInode(inode) { - return this._inodes.get(inode).set(CONTENT, false); + return this._getInode(inode).set(CONTENT, false); } wipe() { - return [...this._root.keys()].map(key => this._root.delete(key)) + return // TODO } close() { return diff --git a/src/__tests__/YFS.spec.js b/src/__tests__/YFS.spec.js index ca07f4b..6a7d85a 100755 --- a/src/__tests__/YFS.spec.js +++ b/src/__tests__/YFS.spec.js @@ -1,4 +1,4 @@ -jasmine.DEFAULT_TIMEOUT_INTERVAL = 20000 +// jasmine.DEFAULT_TIMEOUT_INTERVAL = 20000 import FS from "../index.js"; import * as Y from 'yjs'; @@ -6,7 +6,7 @@ import * as Y from 'yjs'; import { find } from 'yjs/src/utils/StructStore'; const ydoc = new Y.Doc(); -const fs = new FS("testfs-yjs", { wipe: true, yfs: { Y, ydoc } }).promises; +const fs = new FS("testfs-yjs", { wipe: true, yfs: { Y, ydoc, find } }).promises; const HELLO = new Uint8Array([72, 69, 76, 76, 79]); @@ -16,7 +16,7 @@ if (!Promise.prototype.finally) { } } -fdescribe("YFS module", () => { +describe("YFS module", () => { describe("mkdir", () => { it("root directory already exists", (done) => { fs.mkdir("/").catch(err => { @@ -30,6 +30,8 @@ fdescribe("YFS module", () => { .then(() => { fs.stat("/mkdir-test").then(stat => { done(); + }).catch(err => { + expect(err).toBeUndefined(); }); }) .catch(err => { @@ -43,6 +45,7 @@ fdescribe("YFS module", () => { it("create file", done => { fs.mkdir("/writeFile").finally(() => { fs.writeFile("/writeFile/writeFile-uint8.txt", HELLO).then(() => { + console.log('woot') fs.stat("/writeFile/writeFile-uint8.txt").then(stats => { expect(stats.size).toEqual(5); done(); @@ -93,7 +96,7 @@ fdescribe("YFS module", () => { }); }); - describe("readFile", () => { + fdescribe("readFile", () => { it("read non-existant file throws", done => { fs.readFile("/readFile/non-existant.txt").catch(err => { expect(err).not.toBe(null); @@ -481,7 +484,7 @@ fdescribe("YFS module", () => { }); }); - fdescribe("benchmark", () => { + describe("benchmark", () => { it("10 dir x 10 dir x 10 files", done => { const range = n => [...Array(n).keys()]; const start = performance.now(); From fdd2086e9b2f30e7e9a3c9711fcc9cbd8110a354 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Sat, 7 Nov 2020 14:44:18 -0500 Subject: [PATCH 30/43] finished migrating to client-clock --- src/YjsBackend.js | 5 +---- src/__tests__/YFS.spec.js | 7 +++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index a7d0f63..0096991 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -69,7 +69,7 @@ module.exports = class YjsBackend { const children = []; for (const value of this._inodes) { const parent = parseID(value.get(PARENT)) - if (parent && sameID(parent, id) && value.get(CONTENT)) children.push([value]); + if (parent && sameID(parent, id) && value.get(CONTENT)) children.push(value); } return children; } @@ -127,7 +127,6 @@ module.exports = class YjsBackend { node.set(PARENT, serializeID(dir._item.id)); node.set(BASENAME, basename); this._inodes.push([node]); - console.log('mkdir', JSON.stringify(node.toJSON())); }, 'mkdir'); } rmdir(filepath) { @@ -173,7 +172,6 @@ module.exports = class YjsBackend { node.set(PARENT, serializeID(parentId)); node.set(BASENAME, basename); this._inodes.push([node]); - console.log('writeStat', JSON.stringify(node.toJSON())); } else { node.set(MODE, mode); node.set(TYPE, 'file'); @@ -181,7 +179,6 @@ module.exports = class YjsBackend { } }, 'writeFile'); const stat = this.stat(filepath); - console.log('stat', stat); return stat; } unlink(filepath) { diff --git a/src/__tests__/YFS.spec.js b/src/__tests__/YFS.spec.js index 6a7d85a..f600d1b 100755 --- a/src/__tests__/YFS.spec.js +++ b/src/__tests__/YFS.spec.js @@ -16,7 +16,7 @@ if (!Promise.prototype.finally) { } } -describe("YFS module", () => { +fdescribe("YFS module", () => { describe("mkdir", () => { it("root directory already exists", (done) => { fs.mkdir("/").catch(err => { @@ -96,7 +96,7 @@ describe("YFS module", () => { }); }); - fdescribe("readFile", () => { + describe("readFile", () => { it("read non-existant file throws", done => { fs.readFile("/readFile/non-existant.txt").catch(err => { expect(err).not.toBe(null); @@ -208,7 +208,6 @@ describe("YFS module", () => { fs.mkdir("/rmdir/empty").finally(() => { fs.readdir("/rmdir").then(data => { let originalSize = data.length; - console.log('data', data); fs.rmdir("/rmdir/empty").then(() => { fs.readdir("/rmdir").then(data => { console.log('data', data); @@ -484,7 +483,7 @@ describe("YFS module", () => { }); }); - describe("benchmark", () => { + xdescribe("benchmark", () => { it("10 dir x 10 dir x 10 files", done => { const range = n => [...Array(n).keys()]; const start = performance.now(); From 761b2ce60da0c2c922560cbd19833fbd443e0beb Mon Sep 17 00:00:00 2001 From: William Hilton Date: Sat, 7 Nov 2020 17:26:07 -0500 Subject: [PATCH 31/43] improve client-clock --- src/PromisifiedFS.js | 8 ++++++++ src/YjsBackend.js | 38 ++++++++++++++++++++++++++++---------- src/__tests__/YFS.spec.js | 26 ++++++++------------------ 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index 317ebcf..942d7eb 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -51,6 +51,8 @@ module.exports = class PromisifiedFS { this.backFile = this._wrap(this.backFile, true) this.du = this._wrap(this.du, false); this.openYType = this.openYType.bind(this); + this.getYTypeByIno = this.getYTypeByIno.bind(this); + this.getPathForIno = this.getPathForIno.bind(this); this.saveSuperblock = debounce(() => { this._saveSuperblock(); @@ -327,4 +329,10 @@ module.exports = class PromisifiedFS { openYType(filepath) { return this._yfs.openYType(filepath); } + getYTypeByIno(ino) { + return this._yfs.getYTypeByIno(ino); + } + getPathForIno(ino) { + return this._yfs.getPathForIno(ino); + } } diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 0096991..343f85f 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -3,6 +3,7 @@ const diff = require('fast-diff') const path = require("./path.js"); const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); +const { bton, ntob } = require("./radix64.js"); const TYPE = 't'; const MTIME = 'm'; @@ -19,20 +20,20 @@ function ID (client, clock) { } function serializeID (id) { - const buffer = new ArrayBuffer(16); - const dataview = new DataView(buffer); - dataview.setFloat64(0, id.client); - dataview.setFloat64(8, id.clock); - return new Uint8Array(buffer); + return `${ntob(id.client)}-${ntob(id.clock)}`; } function parseID (arr) { if (!arr) return arr; - const dataview = new DataView(arr.buffer); - return new ID(dataview.getFloat64(0), dataview.getFloat64(8)); + const id = arr.indexOf('-'); + const client = bton(arr.slice(0, id)); + const clock = bton(arr.slice(id + 1)); + return new ID(client, clock); } function sameID (id1, id2) { + if (id1 == null && id2 == null) return true; + if (id1 == null || id2 == null) return false; return id1.client === id2.client && id1.clock === id2.clock; } @@ -60,7 +61,24 @@ module.exports = class YjsBackend { get activated () { return true } - _getInode(id) { + getYTypeByIno(ino) { + let id = typeof ino === 'string' ? parseID(ino) : ino; + const item = this._find(this._ydoc.store, id); + return item.content.type; + } + getPathForIno(ino) { + let id = typeof ino === 'string' ? parseID(ino) : ino; + const parts = []; + while (id !== null) { + const item = this._find(this._ydoc.store, id); + const map = item.content.type; + parts.unshift(map.get(BASENAME)); + id = parseID(map.get(PARENT)); + } + return path.join(...parts); + } + _getInode(ino) { + const id = parseID(ino); const item = this._find(this._ydoc.store, id) const node = item.content.type; return node; @@ -223,7 +241,7 @@ module.exports = class YjsBackend { type: node.get(TYPE), size: this._size(node), mtimeMs: node.get(MTIME), - ino: node._item.id, + ino: serializeID(node._item.id), }; return stat; } @@ -234,7 +252,7 @@ module.exports = class YjsBackend { type: node.get(TYPE), size: this._size(node), mtimeMs: node.get(MTIME), - ino: node._item.id, + ino: serializeID(node._item.id), }; return stat; } diff --git a/src/__tests__/YFS.spec.js b/src/__tests__/YFS.spec.js index f600d1b..a6c9909 100755 --- a/src/__tests__/YFS.spec.js +++ b/src/__tests__/YFS.spec.js @@ -45,7 +45,6 @@ fdescribe("YFS module", () => { it("create file", done => { fs.mkdir("/writeFile").finally(() => { fs.writeFile("/writeFile/writeFile-uint8.txt", HELLO).then(() => { - console.log('woot') fs.stat("/writeFile/writeFile-uint8.txt").then(stats => { expect(stats.size).toEqual(5); done(); @@ -210,7 +209,6 @@ fdescribe("YFS module", () => { let originalSize = data.length; fs.rmdir("/rmdir/empty").then(() => { fs.readdir("/rmdir").then(data => { - console.log('data', data); expect(data.includes("empty")).toBe(false); expect(data.length === originalSize - 1); done(); @@ -484,7 +482,7 @@ fdescribe("YFS module", () => { }); xdescribe("benchmark", () => { - it("10 dir x 10 dir x 10 files", done => { + it("10 dir x 10 dir x 100 files", done => { const range = n => [...Array(n).keys()]; const start = performance.now(); fs.mkdir(`/benchmark`) @@ -505,37 +503,29 @@ fdescribe("YFS module", () => { const end = performance.now(); console.log(`TIME: ${end - start}ms`); // const keys = [...ydoc.getMap('!inodes').keys()]; - const inodes = ydoc.getMap('!inodes'); + const inodes = ydoc.getArray('!inodes'); let size2 = 0 - const keys = [...inodes.keys()]; - const ids = keys.map(key => inodes.get(key)._item.id); - - const kstart = performance.now() - for (const key of keys) { - const node = inodes.get(key); - const content = node.get('c'); - if (content && content.length) { - size2 += content.length - } - } - const kend = performance.now() + const ids = inodes.map(map => map._item.id); const idstart = performance.now() + const idset = new Set() for (const id of ids) { const item = find(ydoc.store, id) const node = item.content.type; const content = node.get('c'); + // idset.add(node.get('p')); if (content && content.length) { size2 += content.length } } const idend = performance.now() - console.log(`LOOKUP TIME: ${kend - kstart}ms vs ${idend - idstart}ms`); - expect(size2).toBe(size * 2); + console.log(`LOOKUP TIME: ${idend - idstart}ms`); + expect(size2).toBe(size); const update = Y.encodeStateAsUpdate(ydoc); console.log(`YJS SIZE: ${update.byteLength}`); + // for (const id of idset) console.log(id); done(); }); }); From dda06b8c78fa332f2a14b6b919f3c646e92aa1b6 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Sat, 7 Nov 2020 19:12:31 -0500 Subject: [PATCH 32/43] simplify (PREV)PARENT and (PREV)BASENAME to single PATH --- src/YjsBackend.js | 71 +++++++++++++++++++-------------------- src/__tests__/YFS.spec.js | 2 +- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/src/YjsBackend.js b/src/YjsBackend.js index 343f85f..32ba24b 100644 --- a/src/YjsBackend.js +++ b/src/YjsBackend.js @@ -9,10 +9,9 @@ const TYPE = 't'; const MTIME = 'm'; const MODE = 'o'; const CONTENT = 'c'; -const PARENT = 'p'; -const PREVPARENT = '-p'; -const BASENAME = 'b'; -const PREVBASENAME = '-b'; +const PATH = 'p'; +const PARENT = 0; +const BASENAME = 1; function ID (client, clock) { this.client = client; @@ -37,6 +36,10 @@ function sameID (id1, id2) { return id1.client === id2.client && id1.clock === id2.clock; } +function ylast (yarr) { + return yarr.get(yarr.length - 1); +} + module.exports = class YjsBackend { constructor(Y, ydoc, find) { this.Y = Y; @@ -53,8 +56,9 @@ module.exports = class YjsBackend { rootdir.set(MTIME, mtimeMs); rootdir.set(CONTENT, true); - rootdir.set(PARENT, null); - rootdir.set(BASENAME, '/'); + const path = new this.Y.Array(); + path.push([[null, '/']]); + rootdir.set(PATH, path); this._inodes.push([rootdir]); } } @@ -72,8 +76,9 @@ module.exports = class YjsBackend { while (id !== null) { const item = this._find(this._ydoc.store, id); const map = item.content.type; - parts.unshift(map.get(BASENAME)); - id = parseID(map.get(PARENT)); + const last = ylast(map.get(PATH)); + parts.unshift(last[BASENAME]); + id = parseID(last[PARENT]); } return path.join(...parts); } @@ -86,15 +91,17 @@ module.exports = class YjsBackend { _childrenOf(id) { const children = []; for (const value of this._inodes) { - const parent = parseID(value.get(PARENT)) + const last = ylast(value.get(PATH)); + const parent = parseID(last[PARENT]); if (parent && sameID(parent, id) && value.get(CONTENT)) children.push(value); } return children; } _findChild(id, basename) { for (const value of this._inodes) { - const parent = parseID(value.get(PARENT)) - if (parent && sameID(parent, id) && value.get(BASENAME) === basename && value.get(CONTENT)) return value; + const last = ylast(value.get(PATH)); + const parent = parseID(last[PARENT]) + if (parent && sameID(parent, id) && last[BASENAME] === basename && value.get(CONTENT)) return value; } return; } @@ -130,7 +137,8 @@ module.exports = class YjsBackend { let dir = this._lookup(path.dirname(filepath)); let basename = path.basename(filepath); for (const child of this._childrenOf(dir._item.id)) { - if (child.get(BASENAME) === basename) { + const last = ylast(child.get(PATH)) + if (last[BASENAME] === basename) { throw new EEXIST(); } } @@ -142,8 +150,9 @@ module.exports = class YjsBackend { node.set(MTIME, mtimeMs); node.set(CONTENT, true); // must be truthy or else directory is in a "deleted" state - node.set(PARENT, serializeID(dir._item.id)); - node.set(BASENAME, basename); + const path = new this.Y.Array(); + path.push([[serializeID(dir._item.id), basename]]); + node.set(PATH, path); this._inodes.push([node]); }, 'mkdir'); } @@ -161,7 +170,7 @@ module.exports = class YjsBackend { readdir(filepath) { let dir = this._lookup(filepath); if (dir.get(TYPE) !== 'dir') throw new ENOTDIR(); - return this._childrenOf(dir._item.id).map(node => node.get(BASENAME)); + return this._childrenOf(dir._item.id).map(node => ylast(node.get(PATH))[BASENAME]); } writeStat(filepath, size, { mode }) { let node @@ -187,8 +196,9 @@ module.exports = class YjsBackend { node.set(MTIME, mtimeMs); node.set(CONTENT, true); // set to truthy so file isn't in a "deleted" state - node.set(PARENT, serializeID(parentId)); - node.set(BASENAME, basename); + const path = new this.Y.Array(); + path.push([[serializeID(parentId), basename]]) + node.set(PATH, path); this._inodes.push([node]); } else { node.set(MODE, mode); @@ -212,26 +222,11 @@ module.exports = class YjsBackend { // grab references let node = this._lookup(oldFilepath); let destDir = this._lookup(path.dirname(newFilepath)); + const basename = path.basename(newFilepath); // Update parent this._ydoc.transact(() => { - const parent = parseID(node.get(PARENT)); - const newParent = destDir._item.id - if (!sameID(parent, newParent)) { - node.set(PARENT, serializeID(newParent)); - const prevParent = parseID(node.get(PREVPARENT)); - if (!sameID(prevParent, parent)) { - node.set(PREVPARENT, node.get(PARENT)); - } - } - - const basename = node.get(BASENAME); - const newBasename = path.basename(newFilepath); - if (basename !== newBasename) { - node.set(BASENAME, newBasename); - if (node.get(PREVBASENAME) !== basename) { - node.set(PREVBASENAME, basename); - } - } + const newParent = serializeID(destDir._item.id); + node.get(PATH).push([[newParent, basename]]); }, 'rename'); } stat(filepath) { @@ -283,13 +278,15 @@ module.exports = class YjsBackend { node.set(MTIME, mtimeMs); node.set(CONTENT, target); - node.set(PARENT, serializeID(parentId)); - node.set(BASENAME, basename); + const path = new this.Y.Array(); + path.push([[serializeID(parentId), basename]]); + node.set(PATH, path); this._inodes.push([node]); } else { node.set(MODE, mode); node.set(TYPE, 'symlink'); node.set(MTIME, mtimeMs); + node.set(CONTENT, target); } }, 'symlink'); const stat = this.lstat(filepath); diff --git a/src/__tests__/YFS.spec.js b/src/__tests__/YFS.spec.js index a6c9909..eee9d99 100755 --- a/src/__tests__/YFS.spec.js +++ b/src/__tests__/YFS.spec.js @@ -16,7 +16,7 @@ if (!Promise.prototype.finally) { } } -fdescribe("YFS module", () => { +describe("YFS module", () => { describe("mkdir", () => { it("root directory already exists", (done) => { fs.mkdir("/").catch(err => { From c936a511df0bc5336c0c11ae62aa4304a07e92ad Mon Sep 17 00:00:00 2001 From: William Hilton Date: Mon, 30 Nov 2020 16:37:52 -0500 Subject: [PATCH 33/43] move cleanParam logic to _wrap --- src/PromisifiedFS.js | 65 ++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index e11db8c..ddf7f9c 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -12,7 +12,7 @@ const Mutex2 = require("./Mutex2.js"); const path = require("./path.js"); const clock = require("./clock.js"); -function cleanParams(filepath, opts) { +function cleanParamsFilepathOpts(filepath, opts, ...rest) { // normalize paths filepath = path.normalize(filepath); // strip out callbacks @@ -25,30 +25,46 @@ function cleanParams(filepath, opts) { encoding: opts, }; } - return [filepath, opts]; + return [filepath, opts, ...rest]; } -function cleanParams2(oldFilepath, newFilepath) { +function cleanParamsFilepathDataOpts(filepath, data, opts, ...rest) { // normalize paths - return [path.normalize(oldFilepath), path.normalize(newFilepath)]; + filepath = path.normalize(filepath); + // strip out callbacks + if (typeof opts === "undefined" || typeof opts === "function") { + opts = {}; + } + // expand string options to encoding options + if (typeof opts === "string") { + opts = { + encoding: opts, + }; + } + return [filepath, data, opts, ...rest]; +} + +function cleanParamsFilepathFilepath(oldFilepath, newFilepath, ...rest) { + // normalize paths + return [path.normalize(oldFilepath), path.normalize(newFilepath), ...rest]; } module.exports = class PromisifiedFS { constructor(name, options) { this.init = this.init.bind(this) - this.readFile = this._wrap(this.readFile, false) - this.writeFile = this._wrap(this.writeFile, true) - this.unlink = this._wrap(this.unlink, true) - this.readdir = this._wrap(this.readdir, false) - this.mkdir = this._wrap(this.mkdir, true) - this.rmdir = this._wrap(this.rmdir, true) - this.rename = this._wrap(this.rename, true) - this.stat = this._wrap(this.stat, false) - this.lstat = this._wrap(this.lstat, false) - this.readlink = this._wrap(this.readlink, false) - this.symlink = this._wrap(this.symlink, true) - this.backFile = this._wrap(this.backFile, true) - this.du = this._wrap(this.du, false); + this.readFile = this._wrap(this.readFile, cleanParamsFilepathOpts, false) + this.writeFile = this._wrap(this.writeFile, cleanParamsFilepathDataOpts, true) + this.unlink = this._wrap(this.unlink, cleanParamsFilepathOpts, true) + this.readdir = this._wrap(this.readdir, cleanParamsFilepathOpts, false) + this.mkdir = this._wrap(this.mkdir, cleanParamsFilepathOpts, true) + this.rmdir = this._wrap(this.rmdir, cleanParamsFilepathOpts, true) + this.rename = this._wrap(this.rename, cleanParamsFilepathFilepath, true) + this.stat = this._wrap(this.stat, cleanParamsFilepathOpts, false) + this.lstat = this._wrap(this.lstat, cleanParamsFilepathOpts, false) + this.readlink = this._wrap(this.readlink, cleanParamsFilepathOpts, false) + this.symlink = this._wrap(this.symlink, cleanParamsFilepathFilepath, true) + this.backFile = this._wrap(this.backFile, cleanParamsFilepathOpts, true) + this.du = this._wrap(this.du, cleanParamsFilepathOpts, false); this.saveSuperblock = debounce(() => { this._saveSuperblock(); @@ -111,9 +127,10 @@ module.exports = class PromisifiedFS { this._gracefulShutdownResolve = null } } - _wrap (fn, mutating) { + _wrap (fn, paramCleaner, mutating) { let i = 0 return async (...args) => { + args = paramCleaner(...args) let op = { name: fn.name, args, @@ -209,7 +226,6 @@ module.exports = class PromisifiedFS { return this._cache.writeStat(filepath, size, opts) } async readFile(filepath, opts) { - ;[filepath, opts] = cleanParams(filepath, opts); const { encoding } = opts; if (encoding && encoding !== 'utf8') throw new Error('Only "utf8" encoding is supported in readFile'); let data = null, stat = null @@ -240,7 +256,6 @@ module.exports = class PromisifiedFS { return data; } async writeFile(filepath, data, opts) { - ;[filepath, opts] = cleanParams(filepath, opts); const { mode, encoding = "utf8" } = opts; if (typeof data === "string") { if (encoding !== "utf8") { @@ -253,7 +268,6 @@ module.exports = class PromisifiedFS { return null } async unlink(filepath, opts) { - ;[filepath, opts] = cleanParams(filepath, opts); const stat = this._cache.lstat(filepath); this._cache.unlink(filepath); if (stat.type !== 'symlink') { @@ -262,17 +276,14 @@ module.exports = class PromisifiedFS { return null } async readdir(filepath, opts) { - ;[filepath, opts] = cleanParams(filepath, opts); return this._cache.readdir(filepath); } async mkdir(filepath, opts) { - ;[filepath, opts] = cleanParams(filepath, opts); const { mode = 0o777 } = opts; await this._cache.mkdir(filepath, { mode }); return null } async rmdir(filepath, opts) { - ;[filepath, opts] = cleanParams(filepath, opts); // Never allow deleting the root directory. if (filepath === "/") { throw new ENOTEMPTY(); @@ -281,31 +292,25 @@ module.exports = class PromisifiedFS { return null; } async rename(oldFilepath, newFilepath) { - ;[oldFilepath, newFilepath] = cleanParams2(oldFilepath, newFilepath); this._cache.rename(oldFilepath, newFilepath); return null; } async stat(filepath, opts) { - ;[filepath, opts] = cleanParams(filepath, opts); const data = this._cache.stat(filepath); return new Stat(data); } async lstat(filepath, opts) { - ;[filepath, opts] = cleanParams(filepath, opts); let data = this._cache.lstat(filepath); return new Stat(data); } async readlink(filepath, opts) { - ;[filepath, opts] = cleanParams(filepath, opts); return this._cache.readlink(filepath); } async symlink(target, filepath) { - ;[target, filepath] = cleanParams2(target, filepath); this._cache.symlink(target, filepath); return null; } async backFile(filepath, opts) { - ;[filepath, opts] = cleanParams(filepath, opts); let size = await this._http.sizeFile(filepath) await this._writeStat(filepath, size, opts) return null From ff5e41418e62fc8b9e52ba9df14c5d7eaf3d3e99 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Mon, 30 Nov 2020 17:49:48 -0500 Subject: [PATCH 34/43] move guts into DefaultBackend --- src/DefaultBackend.js | 180 ++++++++++++++++++++++++++ src/PromisifiedFS.js | 228 +++++++-------------------------- src/__tests__/fallback.spec.js | 2 +- src/index.js | 4 +- 4 files changed, 230 insertions(+), 184 deletions(-) create mode 100644 src/DefaultBackend.js diff --git a/src/DefaultBackend.js b/src/DefaultBackend.js new file mode 100644 index 0000000..11b9b8c --- /dev/null +++ b/src/DefaultBackend.js @@ -0,0 +1,180 @@ +const { encode, decode } = require("isomorphic-textencoder"); +const debounce = require("just-debounce-it"); + +const CacheFS = require("./CacheFS.js"); +const { ENOENT, ENOTEMPTY, ETIMEDOUT } = require("./errors.js"); +const IdbBackend = require("./IdbBackend.js"); +const HttpBackend = require("./HttpBackend.js") +const Mutex = require("./Mutex.js"); +const Mutex2 = require("./Mutex2.js"); + +const path = require("./path.js"); + +module.exports = class DefaultBackend { + constructor() { + this.saveSuperblock = debounce(() => { + this._saveSuperblock(); + }, 500); + } + async init (name, { + wipe, + url, + urlauto, + fileDbName = name, + fileStoreName = name + "_files", + lockDbName = name + "_lock", + lockStoreName = name + "_lock", + } = {}) { + this._name = name + this._idb = new IdbBackend(fileDbName, fileStoreName); + this._mutex = navigator.locks ? new Mutex2(name) : new Mutex(lockDbName, lockStoreName); + this._cache = new CacheFS(name); + this._opts = { wipe, url }; + this._needsWipe = !!wipe; + if (url) { + this._http = new HttpBackend(url) + this._urlauto = !!urlauto + } + } + async activate() { + if (this._cache.activated) return + // Wipe IDB if requested + if (this._needsWipe) { + this._needsWipe = false; + await this._idb.wipe() + await this._mutex.release({ force: true }) + } + if (!(await this._mutex.has())) await this._mutex.wait() + // Attempt to load FS from IDB backend + const root = await this._idb.loadSuperblock() + if (root) { + this._cache.activate(root); + } else if (this._http) { + // If that failed, attempt to load FS from HTTP backend + const text = await this._http.loadSuperblock() + this._cache.activate(text) + await this._saveSuperblock(); + } else { + // If there is no HTTP backend, start with an empty filesystem + this._cache.activate() + } + if (await this._mutex.has()) { + return + } else { + throw new ETIMEDOUT() + } + } + async deactivate() { + if (await this._mutex.has()) { + await this._saveSuperblock() + } + this._cache.deactivate() + try { + await this._mutex.release() + } catch (e) { + console.log(e) + } + await this._idb.close() + } + async _saveSuperblock() { + if (this._cache.activated) { + this._lastSavedAt = Date.now() + await this._idb.saveSuperblock(this._cache._root); + } + } + async _writeStat(filepath, size, opts) { + let dirparts = path.split(path.dirname(filepath)) + let dir = dirparts.shift() + for (let dirpart of dirparts) { + dir = path.join(dir, dirpart) + try { + this._cache.mkdir(dir, { mode: 0o777 }) + } catch (e) {} + } + return this._cache.writeStat(filepath, size, opts) + } + async readFile(filepath, opts) { + const { encoding } = opts; + if (encoding && encoding !== 'utf8') throw new Error('Only "utf8" encoding is supported in readFile'); + let data = null, stat = null + try { + stat = this._cache.stat(filepath); + data = await this._idb.readFile(stat.ino) + } catch (e) { + if (!this._urlauto) throw e + } + if (!data && this._http) { + let lstat = this._cache.lstat(filepath) + while (lstat.type === 'symlink') { + filepath = path.resolve(path.dirname(filepath), lstat.target) + lstat = this._cache.lstat(filepath) + } + data = await this._http.readFile(filepath) + } + if (data) { + if (!stat || stat.size != data.byteLength) { + stat = await this._writeStat(filepath, data.byteLength, { mode: stat ? stat.mode : 0o666 }) + this.saveSuperblock() // debounced + } + if (encoding === "utf8") { + data = decode(data); + } + } + if (!stat) throw new ENOENT(filepath) + return data; + } + async writeFile(filepath, data, opts) { + const { mode, encoding = "utf8" } = opts; + if (typeof data === "string") { + if (encoding !== "utf8") { + throw new Error('Only "utf8" encoding is supported in writeFile'); + } + data = encode(data); + } + const stat = await this._cache.writeStat(filepath, data.byteLength, { mode }); + await this._idb.writeFile(stat.ino, data) + } + async unlink(filepath, opts) { + const stat = this._cache.lstat(filepath); + this._cache.unlink(filepath); + if (stat.type !== 'symlink') { + await this._idb.unlink(stat.ino) + } + } + async readdir(filepath, opts) { + return this._cache.readdir(filepath); + } + async mkdir(filepath, opts) { + const { mode = 0o777 } = opts; + await this._cache.mkdir(filepath, { mode }); + } + async rmdir(filepath, opts) { + // Never allow deleting the root directory. + if (filepath === "/") { + throw new ENOTEMPTY(); + } + this._cache.rmdir(filepath); + } + async rename(oldFilepath, newFilepath) { + this._cache.rename(oldFilepath, newFilepath); + } + async stat(filepath, opts) { + return this._cache.stat(filepath); + } + async lstat(filepath, opts) { + return this._cache.lstat(filepath); + } + async readlink(filepath, opts) { + return this._cache.readlink(filepath); + } + async symlink(target, filepath) { + this._cache.symlink(target, filepath); + } + async backFile(filepath, opts) { + let size = await this._http.sizeFile(filepath) + await this._writeStat(filepath, size, opts) + } + async du(filepath) { + return this._cache.du(filepath); + } +} diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index ddf7f9c..6423c01 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -1,16 +1,7 @@ -const { encode, decode } = require("isomorphic-textencoder"); -const debounce = require("just-debounce-it"); - +const DefaultBackend = require("./DefaultBackend.js"); const Stat = require("./Stat.js"); -const CacheFS = require("./CacheFS.js"); -const { ENOENT, ENOTEMPTY, ETIMEDOUT } = require("./errors.js"); -const IdbBackend = require("./IdbBackend.js"); -const HttpBackend = require("./HttpBackend.js") -const Mutex = require("./Mutex.js"); -const Mutex2 = require("./Mutex2.js"); const path = require("./path.js"); -const clock = require("./clock.js"); function cleanParamsFilepathOpts(filepath, opts, ...rest) { // normalize paths @@ -49,75 +40,53 @@ function cleanParamsFilepathFilepath(oldFilepath, newFilepath, ...rest) { return [path.normalize(oldFilepath), path.normalize(newFilepath), ...rest]; } -module.exports = class PromisifiedFS { - constructor(name, options) { - this.init = this.init.bind(this) - this.readFile = this._wrap(this.readFile, cleanParamsFilepathOpts, false) - this.writeFile = this._wrap(this.writeFile, cleanParamsFilepathDataOpts, true) - this.unlink = this._wrap(this.unlink, cleanParamsFilepathOpts, true) - this.readdir = this._wrap(this.readdir, cleanParamsFilepathOpts, false) - this.mkdir = this._wrap(this.mkdir, cleanParamsFilepathOpts, true) - this.rmdir = this._wrap(this.rmdir, cleanParamsFilepathOpts, true) - this.rename = this._wrap(this.rename, cleanParamsFilepathFilepath, true) - this.stat = this._wrap(this.stat, cleanParamsFilepathOpts, false) - this.lstat = this._wrap(this.lstat, cleanParamsFilepathOpts, false) - this.readlink = this._wrap(this.readlink, cleanParamsFilepathOpts, false) - this.symlink = this._wrap(this.symlink, cleanParamsFilepathFilepath, true) - this.backFile = this._wrap(this.backFile, cleanParamsFilepathOpts, true) - this.du = this._wrap(this.du, cleanParamsFilepathOpts, false); +module.exports = function promises(name, options) { + const pfs = new PromisifiedFS(options); + pfs.init = pfs.init.bind(pfs) + pfs.readFile = pfs._wrap(pfs.readFile, cleanParamsFilepathOpts, false) + pfs.writeFile = pfs._wrap(pfs.writeFile, cleanParamsFilepathDataOpts, true) + pfs.unlink = pfs._wrap(pfs.unlink, cleanParamsFilepathOpts, true) + pfs.readdir = pfs._wrap(pfs.readdir, cleanParamsFilepathOpts, false) + pfs.mkdir = pfs._wrap(pfs.mkdir, cleanParamsFilepathOpts, true) + pfs.rmdir = pfs._wrap(pfs.rmdir, cleanParamsFilepathOpts, true) + pfs.rename = pfs._wrap(pfs.rename, cleanParamsFilepathFilepath, true) + pfs.stat = pfs._wrap(pfs.stat, cleanParamsFilepathOpts, false) + pfs.lstat = pfs._wrap(pfs.lstat, cleanParamsFilepathOpts, false) + pfs.readlink = pfs._wrap(pfs.readlink, cleanParamsFilepathOpts, false) + pfs.symlink = pfs._wrap(pfs.symlink, cleanParamsFilepathFilepath, true) + pfs.backFile = pfs._wrap(pfs.backFile, cleanParamsFilepathOpts, true) + pfs.du = pfs._wrap(pfs.du, cleanParamsFilepathOpts, false); + + if (name) { + pfs.init(name, options) + } + return pfs; +} - this.saveSuperblock = debounce(() => { - this._saveSuperblock(); - }, 500); +class PromisifiedFS { + constructor(options = {}) { + this._backend = options.backend || new DefaultBackend(); this._deactivationPromise = null this._deactivationTimeout = null this._activationPromise = null this._operations = new Set() - - if (name) { - this.init(name, options) - } } async init (...args) { if (this._initPromiseResolve) await this._initPromise; this._initPromise = this._init(...args) return this._initPromise } - async _init (name, { - wipe, - url, - urlauto, - fileDbName = name, - fileStoreName = name + "_files", - lockDbName = name + "_lock", - lockStoreName = name + "_lock", - defer = false, - } = {}) { - await this._gracefulShutdown() - this._name = name - this._idb = new IdbBackend(fileDbName, fileStoreName); - this._mutex = navigator.locks ? new Mutex2(name) : new Mutex(lockDbName, lockStoreName); - this._cache = new CacheFS(name); - this._opts = { wipe, url }; - this._needsWipe = !!wipe; - if (url) { - this._http = new HttpBackend(url) - this._urlauto = !!urlauto - } + async _init (name, options = {}) { + await this._gracefulShutdown(); + + await this._backend.init(name, options); + if (this._initPromiseResolve) { this._initPromiseResolve(); this._initPromiseResolve = null; } - // The next comment starting with the "fs is initially activated when constructed"? - // That can create contention for the mutex if two threads try to init at the same time - // so I've added an option to disable that behavior. - if (!defer) { - // The fs is initially activated when constructed (in order to wipe/save the superblock) - // This is not awaited, because that would create a cycle. - this.stat('/') - } } async _gracefulShutdown () { if (this._operations.size > 0) { @@ -128,7 +97,6 @@ module.exports = class PromisifiedFS { } } _wrap (fn, paramCleaner, mutating) { - let i = 0 return async (...args) => { args = paramCleaner(...args) let op = { @@ -141,7 +109,7 @@ module.exports = class PromisifiedFS { return await fn.apply(this, args) } finally { this._operations.delete(op) - if (mutating) this.saveSuperblock() // this is debounced + if (mutating) this._backend.saveSuperblock() // this is debounced if (this._operations.size === 0) { if (!this._deactivationTimeout) clearTimeout(this._deactivationTimeout) this._deactivationTimeout = setTimeout(this._deactivate.bind(this), 500) @@ -158,164 +126,62 @@ module.exports = class PromisifiedFS { } if (this._deactivationPromise) await this._deactivationPromise this._deactivationPromise = null - if (!this._activationPromise) this._activationPromise = this.__activate() + if (!this._activationPromise) this._activationPromise = this._backend.activate(); await this._activationPromise - if (await this._mutex.has()) { - return - } else { - throw new ETIMEDOUT() - } - } - async __activate() { - if (this._cache.activated) return - // Wipe IDB if requested - if (this._needsWipe) { - this._needsWipe = false; - await this._idb.wipe() - await this._mutex.release({ force: true }) - } - if (!(await this._mutex.has())) await this._mutex.wait() - // Attempt to load FS from IDB backend - const root = await this._idb.loadSuperblock() - if (root) { - this._cache.activate(root); - } else if (this._http) { - // If that failed, attempt to load FS from HTTP backend - const text = await this._http.loadSuperblock() - this._cache.activate(text) - await this._saveSuperblock(); - } else { - // If there is no HTTP backend, start with an empty filesystem - this._cache.activate() - } } async _deactivate() { if (this._activationPromise) await this._activationPromise - if (!this._deactivationPromise) this._deactivationPromise = this.__deactivate() + if (!this._deactivationPromise) this._deactivationPromise = this._backend.deactivate(); this._activationPromise = null if (this._gracefulShutdownResolve) this._gracefulShutdownResolve() return this._deactivationPromise } - async __deactivate() { - if (await this._mutex.has()) { - await this._saveSuperblock() - } - this._cache.deactivate() - try { - await this._mutex.release() - } catch (e) { - console.log(e) - } - await this._idb.close() - } - async _saveSuperblock() { - if (this._cache.activated) { - this._lastSavedAt = Date.now() - await this._idb.saveSuperblock(this._cache._root); - } - } - async _writeStat(filepath, size, opts) { - let dirparts = path.split(path.dirname(filepath)) - let dir = dirparts.shift() - for (let dirpart of dirparts) { - dir = path.join(dir, dirpart) - try { - this._cache.mkdir(dir, { mode: 0o777 }) - } catch (e) {} - } - return this._cache.writeStat(filepath, size, opts) - } async readFile(filepath, opts) { - const { encoding } = opts; - if (encoding && encoding !== 'utf8') throw new Error('Only "utf8" encoding is supported in readFile'); - let data = null, stat = null - try { - stat = this._cache.stat(filepath); - data = await this._idb.readFile(stat.ino) - } catch (e) { - if (!this._urlauto) throw e - } - if (!data && this._http) { - let lstat = this._cache.lstat(filepath) - while (lstat.type === 'symlink') { - filepath = path.resolve(path.dirname(filepath), lstat.target) - lstat = this._cache.lstat(filepath) - } - data = await this._http.readFile(filepath) - } - if (data) { - if (!stat || stat.size != data.byteLength) { - stat = await this._writeStat(filepath, data.byteLength, { mode: stat ? stat.mode : 0o666 }) - this.saveSuperblock() // debounced - } - if (encoding === "utf8") { - data = decode(data); - } - } - if (!stat) throw new ENOENT(filepath) - return data; + return this._backend.readFile(filepath, opts); } async writeFile(filepath, data, opts) { - const { mode, encoding = "utf8" } = opts; - if (typeof data === "string") { - if (encoding !== "utf8") { - throw new Error('Only "utf8" encoding is supported in writeFile'); - } - data = encode(data); - } - const stat = await this._cache.writeStat(filepath, data.byteLength, { mode }); - await this._idb.writeFile(stat.ino, data) + await this._backend.writeFile(filepath, data, opts); return null } async unlink(filepath, opts) { - const stat = this._cache.lstat(filepath); - this._cache.unlink(filepath); - if (stat.type !== 'symlink') { - await this._idb.unlink(stat.ino) - } + await this._backend.unlink(filepath, opts); return null } async readdir(filepath, opts) { - return this._cache.readdir(filepath); + return this._backend.readdir(filepath, opts); } async mkdir(filepath, opts) { - const { mode = 0o777 } = opts; - await this._cache.mkdir(filepath, { mode }); + await this._backend.mkdir(filepath, opts); return null } async rmdir(filepath, opts) { - // Never allow deleting the root directory. - if (filepath === "/") { - throw new ENOTEMPTY(); - } - this._cache.rmdir(filepath); + await this._backend.rmdir(filepath, opts); return null; } async rename(oldFilepath, newFilepath) { - this._cache.rename(oldFilepath, newFilepath); + await this._backend.rename(oldFilepath, newFilepath); return null; } async stat(filepath, opts) { - const data = this._cache.stat(filepath); + const data = await this._backend.stat(filepath, opts); return new Stat(data); } async lstat(filepath, opts) { - let data = this._cache.lstat(filepath); + const data = await this._backend.lstat(filepath, opts); return new Stat(data); } async readlink(filepath, opts) { - return this._cache.readlink(filepath); + return this._backend.readlink(filepath, opts); } async symlink(target, filepath) { - this._cache.symlink(target, filepath); + await this._backend.symlink(target, filepath); return null; } async backFile(filepath, opts) { - let size = await this._http.sizeFile(filepath) - await this._writeStat(filepath, size, opts) + await this._backend.backFile(filepath, opts); return null } async du(filepath) { - return this._cache.du(filepath); + return this._backend.du(filepath); } } diff --git a/src/__tests__/fallback.spec.js b/src/__tests__/fallback.spec.js index a03600b..a5504aa 100644 --- a/src/__tests__/fallback.spec.js +++ b/src/__tests__/fallback.spec.js @@ -4,7 +4,7 @@ const fs = new FS("fallbackfs", { wipe: true, url: 'http://localhost:9876/base/s describe("http fallback", () => { it("sanity check", () => { - expect(fs.promises._http).not.toBeFalsy() + expect(fs.promises._backend._http).not.toBeFalsy() }) it("loads", (done) => { fs.promises._activate().then(() => { diff --git a/src/index.js b/src/index.js index 98558b1..a2fb525 100755 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,6 @@ const once = require("just-once"); -const PromisifiedFS = require('./PromisifiedFS'); +const promises = require('./PromisifiedFS'); function wrapCallback (opts, cb) { if (typeof opts === "function") { @@ -13,7 +13,7 @@ function wrapCallback (opts, cb) { module.exports = class FS { constructor(...args) { - this.promises = new PromisifiedFS(...args) + this.promises = promises(...args) // Needed so things don't break if you destructure fs and pass individual functions around this.init = this.init.bind(this) this.readFile = this.readFile.bind(this) From 78799b28fddaa601e27f6948c9289af41c7d95d2 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 1 Dec 2020 14:31:36 -0500 Subject: [PATCH 35/43] simplify --- src/PromisifiedFS.js | 44 ++++++++++++++++++++------------------------ src/index.js | 4 ++-- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index 6423c01..74f6808 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -40,31 +40,23 @@ function cleanParamsFilepathFilepath(oldFilepath, newFilepath, ...rest) { return [path.normalize(oldFilepath), path.normalize(newFilepath), ...rest]; } -module.exports = function promises(name, options) { - const pfs = new PromisifiedFS(options); - pfs.init = pfs.init.bind(pfs) - pfs.readFile = pfs._wrap(pfs.readFile, cleanParamsFilepathOpts, false) - pfs.writeFile = pfs._wrap(pfs.writeFile, cleanParamsFilepathDataOpts, true) - pfs.unlink = pfs._wrap(pfs.unlink, cleanParamsFilepathOpts, true) - pfs.readdir = pfs._wrap(pfs.readdir, cleanParamsFilepathOpts, false) - pfs.mkdir = pfs._wrap(pfs.mkdir, cleanParamsFilepathOpts, true) - pfs.rmdir = pfs._wrap(pfs.rmdir, cleanParamsFilepathOpts, true) - pfs.rename = pfs._wrap(pfs.rename, cleanParamsFilepathFilepath, true) - pfs.stat = pfs._wrap(pfs.stat, cleanParamsFilepathOpts, false) - pfs.lstat = pfs._wrap(pfs.lstat, cleanParamsFilepathOpts, false) - pfs.readlink = pfs._wrap(pfs.readlink, cleanParamsFilepathOpts, false) - pfs.symlink = pfs._wrap(pfs.symlink, cleanParamsFilepathFilepath, true) - pfs.backFile = pfs._wrap(pfs.backFile, cleanParamsFilepathOpts, true) - pfs.du = pfs._wrap(pfs.du, cleanParamsFilepathOpts, false); +module.exports = class PromisifiedFS { + constructor(name, options = {}) { + this.init = this.init.bind(this) + this.readFile = this._wrap(this.readFile, cleanParamsFilepathOpts, false) + this.writeFile = this._wrap(this.writeFile, cleanParamsFilepathDataOpts, true) + this.unlink = this._wrap(this.unlink, cleanParamsFilepathOpts, true) + this.readdir = this._wrap(this.readdir, cleanParamsFilepathOpts, false) + this.mkdir = this._wrap(this.mkdir, cleanParamsFilepathOpts, true) + this.rmdir = this._wrap(this.rmdir, cleanParamsFilepathOpts, true) + this.rename = this._wrap(this.rename, cleanParamsFilepathFilepath, true) + this.stat = this._wrap(this.stat, cleanParamsFilepathOpts, false) + this.lstat = this._wrap(this.lstat, cleanParamsFilepathOpts, false) + this.readlink = this._wrap(this.readlink, cleanParamsFilepathOpts, false) + this.symlink = this._wrap(this.symlink, cleanParamsFilepathFilepath, true) + this.backFile = this._wrap(this.backFile, cleanParamsFilepathOpts, true) + this.du = this._wrap(this.du, cleanParamsFilepathOpts, false); - if (name) { - pfs.init(name, options) - } - return pfs; -} - -class PromisifiedFS { - constructor(options = {}) { this._backend = options.backend || new DefaultBackend(); this._deactivationPromise = null @@ -72,6 +64,10 @@ class PromisifiedFS { this._activationPromise = null this._operations = new Set() + + if (name) { + this.init(name, options) + } } async init (...args) { if (this._initPromiseResolve) await this._initPromise; diff --git a/src/index.js b/src/index.js index a2fb525..98558b1 100755 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,6 @@ const once = require("just-once"); -const promises = require('./PromisifiedFS'); +const PromisifiedFS = require('./PromisifiedFS'); function wrapCallback (opts, cb) { if (typeof opts === "function") { @@ -13,7 +13,7 @@ function wrapCallback (opts, cb) { module.exports = class FS { constructor(...args) { - this.promises = promises(...args) + this.promises = new PromisifiedFS(...args) // Needed so things don't break if you destructure fs and pass individual functions around this.init = this.init.bind(this) this.readFile = this.readFile.bind(this) From 797d239e7fc227f2c215b266e89c2c736d670651 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 1 Dec 2020 14:47:56 -0500 Subject: [PATCH 36/43] bring back 'defer' param --- src/PromisifiedFS.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index 74f6808..dc1de9a 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -83,6 +83,14 @@ module.exports = class PromisifiedFS { this._initPromiseResolve(); this._initPromiseResolve = null; } + // The next comment starting with the "fs is initially activated when constructed"? + // That can create contention for the mutex if two threads try to init at the same time + // so I've added an option to disable that behavior. + if (!options.defer) { + // The fs is initially activated when constructed (in order to wipe/save the superblock) + // This is not awaited, because that would create a cycle. + this.stat('/') + } } async _gracefulShutdown () { if (this._operations.size > 0) { From ecef48e8da1fa2900b0c7ae05a8b017710de874e Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 1 Dec 2020 14:50:34 -0500 Subject: [PATCH 37/43] drop 'async' from DefaultBackend methods that don't need it --- src/DefaultBackend.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/DefaultBackend.js b/src/DefaultBackend.js index 11b9b8c..ddd4e92 100644 --- a/src/DefaultBackend.js +++ b/src/DefaultBackend.js @@ -82,7 +82,7 @@ module.exports = class DefaultBackend { await this._idb.saveSuperblock(this._cache._root); } } - async _writeStat(filepath, size, opts) { + _writeStat(filepath, size, opts) { let dirparts = path.split(path.dirname(filepath)) let dir = dirparts.shift() for (let dirpart of dirparts) { @@ -141,40 +141,40 @@ module.exports = class DefaultBackend { await this._idb.unlink(stat.ino) } } - async readdir(filepath, opts) { + readdir(filepath, opts) { return this._cache.readdir(filepath); } - async mkdir(filepath, opts) { + mkdir(filepath, opts) { const { mode = 0o777 } = opts; - await this._cache.mkdir(filepath, { mode }); + this._cache.mkdir(filepath, { mode }); } - async rmdir(filepath, opts) { + rmdir(filepath, opts) { // Never allow deleting the root directory. if (filepath === "/") { throw new ENOTEMPTY(); } this._cache.rmdir(filepath); } - async rename(oldFilepath, newFilepath) { + rename(oldFilepath, newFilepath) { this._cache.rename(oldFilepath, newFilepath); } - async stat(filepath, opts) { + stat(filepath, opts) { return this._cache.stat(filepath); } - async lstat(filepath, opts) { + lstat(filepath, opts) { return this._cache.lstat(filepath); } - async readlink(filepath, opts) { + readlink(filepath, opts) { return this._cache.readlink(filepath); } - async symlink(target, filepath) { + symlink(target, filepath) { this._cache.symlink(target, filepath); } async backFile(filepath, opts) { let size = await this._http.sizeFile(filepath) await this._writeStat(filepath, size, opts) } - async du(filepath) { + du(filepath) { return this._cache.du(filepath); } } From 80fcea30b580225aca73b5b59f53a70bda4d0287 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 1 Dec 2020 15:03:20 -0500 Subject: [PATCH 38/43] move YjsBackend to src/backends/yjs --- src/__tests__/threadsafety-yfs.worker.js | 17 ------------ src/__tests__/threadsafety.spec.js | 26 ------------------- src/{ => backends/yjs}/YjsBackend.js | 4 +-- .../yjs/YjsBackend.spec.js} | 9 ++++--- src/{ => backends/yjs}/radix64.js | 0 5 files changed, 7 insertions(+), 49 deletions(-) delete mode 100644 src/__tests__/threadsafety-yfs.worker.js rename src/{ => backends/yjs}/YjsBackend.js (98%) rename src/{__tests__/YFS.spec.js => backends/yjs/YjsBackend.spec.js} (99%) rename src/{ => backends/yjs}/radix64.js (100%) diff --git a/src/__tests__/threadsafety-yfs.worker.js b/src/__tests__/threadsafety-yfs.worker.js deleted file mode 100644 index 4e9dd30..0000000 --- a/src/__tests__/threadsafety-yfs.worker.js +++ /dev/null @@ -1,17 +0,0 @@ -importScripts('http://localhost:9876/base/dist/lightning-fs.min.js'); - -self.fs = new LightningFS("testfs-worker-yfs", { yfs: true }).promises; - -const sleep = ms => new Promise(r => setTimeout(r, ms)) - -const whoAmI = (typeof window === 'undefined' ? (self.name ? self.name : 'worker') : 'main' )+ ': ' - -async function writeFiles () { - console.log(whoAmI + 'write stuff') - // Chrome Mobile 67 and Mobile Safari 11 do not yet support named Workers - let name = self.name || Math.random() - await Promise.all([0, 1, 2, 3, 4].map(i => self.fs.writeFile(`/${name}_${i}.txt`, String(i)))) - self.postMessage({ message: 'COMPLETE' }) -} - -writeFiles() diff --git a/src/__tests__/threadsafety.spec.js b/src/__tests__/threadsafety.spec.js index dd37516..a672bab 100644 --- a/src/__tests__/threadsafety.spec.js +++ b/src/__tests__/threadsafety.spec.js @@ -27,30 +27,4 @@ describe("thread safety", () => { }); }); }); - - xit("launch a bunch of workers (YFS)", (done) => { - const fs = new FS("testfs-worker-yfs", { wipe: true, yfs: true }).promises; - let workers = [] - let promises = [] - let numWorkers = 5 - fs.readdir('/').then(files => { - expect(files.length).toBe(0); - for (let i = 1; i <= numWorkers; i++) { - let promise = new Promise(resolve => { - let worker = new Worker('http://localhost:9876/base/src/__tests__/threadsafety-yfs.worker.js', {name: `worker_yfs_${i}`}) - worker.onmessage = (e) => { - if (e.data && e.data.message === 'COMPLETE') resolve() - } - workers.push(worker) - }) - promises.push(promise) - } - Promise.all(promises).then(() => { - fs.readdir('/').then(files => { - expect(files.length).toBe(5 * numWorkers) - done(); - }); - }); - }); - }); }); diff --git a/src/YjsBackend.js b/src/backends/yjs/YjsBackend.js similarity index 98% rename from src/YjsBackend.js rename to src/backends/yjs/YjsBackend.js index 9929a73..2495842 100644 --- a/src/YjsBackend.js +++ b/src/backends/yjs/YjsBackend.js @@ -1,7 +1,7 @@ const { encode } = require("isomorphic-textencoder"); -const path = require("./path.js"); -const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); +const path = require("../../path.js"); +const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("../../errors.js"); const { bton, ntob } = require("./radix64.js"); const TYPE = 't'; diff --git a/src/__tests__/YFS.spec.js b/src/backends/yjs/YjsBackend.spec.js similarity index 99% rename from src/__tests__/YFS.spec.js rename to src/backends/yjs/YjsBackend.spec.js index 70f5b1f..bf0710c 100755 --- a/src/__tests__/YFS.spec.js +++ b/src/backends/yjs/YjsBackend.spec.js @@ -1,11 +1,12 @@ // jasmine.DEFAULT_TIMEOUT_INTERVAL = 20000 -import FS from "../index.js"; import * as Y from 'yjs'; - -import YjsBackend from '../YjsBackend.js'; import { find } from 'yjs/src/utils/StructStore'; +import FS from "../../index.js"; + +import YjsBackend from './YjsBackend.js'; + const ydoc = new Y.Doc(); const backend = new YjsBackend(Y, ydoc, find); const fs = new FS("testfs-yjs", { wipe: true, backend }).promises; @@ -18,7 +19,7 @@ if (!Promise.prototype.finally) { } } -describe("YFS module", () => { +describe("YjsBackend", () => { describe("mkdir", () => { it("root directory already exists", (done) => { fs.mkdir("/").catch(err => { diff --git a/src/radix64.js b/src/backends/yjs/radix64.js similarity index 100% rename from src/radix64.js rename to src/backends/yjs/radix64.js From 885f6d68be5a1f62d4739ecbe870840b4dfdf575 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 1 Dec 2020 15:12:37 -0500 Subject: [PATCH 39/43] cleanup --- src/__tests__/threadsafety.spec.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/__tests__/threadsafety.spec.js b/src/__tests__/threadsafety.spec.js index a672bab..0c78a57 100644 --- a/src/__tests__/threadsafety.spec.js +++ b/src/__tests__/threadsafety.spec.js @@ -1,9 +1,10 @@ jasmine.DEFAULT_TIMEOUT_INTERVAL = 20000 import FS from "../index.js"; +const fs = new FS("testfs-worker", { wipe: true }).promises; + describe("thread safety", () => { it("launch a bunch of workers", (done) => { - const fs = new FS("testfs-worker", { wipe: true }).promises; let workers = [] let promises = [] let numWorkers = 5 From b9fc54042e40c2cb7912c43498950cd4d26e816e Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 1 Dec 2020 15:14:01 -0500 Subject: [PATCH 40/43] more realistic --- src/backends/yjs/YjsBackend.js | 4 +- src/backends/yjs/errors.js | 21 +++++++ src/backends/yjs/path.js | 107 +++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 2 deletions(-) create mode 100755 src/backends/yjs/errors.js create mode 100755 src/backends/yjs/path.js diff --git a/src/backends/yjs/YjsBackend.js b/src/backends/yjs/YjsBackend.js index 2495842..9929a73 100644 --- a/src/backends/yjs/YjsBackend.js +++ b/src/backends/yjs/YjsBackend.js @@ -1,7 +1,7 @@ const { encode } = require("isomorphic-textencoder"); -const path = require("../../path.js"); -const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("../../errors.js"); +const path = require("./path.js"); +const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); const { bton, ntob } = require("./radix64.js"); const TYPE = 't'; diff --git a/src/backends/yjs/errors.js b/src/backends/yjs/errors.js new file mode 100755 index 0000000..24aa0fe --- /dev/null +++ b/src/backends/yjs/errors.js @@ -0,0 +1,21 @@ +function Err(name) { + return class extends Error { + constructor(...args) { + super(...args); + this.code = name; + if (this.message) { + this.message = name + ": " + this.message; + } else { + this.message = name; + } + } + }; +} + +const EEXIST = Err("EEXIST"); +const ENOENT = Err("ENOENT"); +const ENOTDIR = Err("ENOTDIR"); +const ENOTEMPTY = Err("ENOTEMPTY"); +const ETIMEDOUT = Err("ETIMEDOUT"); + +module.exports = { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY, ETIMEDOUT }; diff --git a/src/backends/yjs/path.js b/src/backends/yjs/path.js new file mode 100755 index 0000000..e45952f --- /dev/null +++ b/src/backends/yjs/path.js @@ -0,0 +1,107 @@ +function normalizePath(path) { + if (path.length === 0) { + return "."; + } + let parts = splitPath(path); + parts = parts.reduce(reducer, []); + return joinPath(...parts); +} + +function resolvePath(...paths) { + let result = ''; + for (let path of paths) { + if (path.startsWith('/')) { + result = path; + } else { + result = normalizePath(joinPath(result, path)); + } + } + return result; +} + +function joinPath(...parts) { + if (parts.length === 0) return ""; + let path = parts.join("/"); + // Replace consecutive '/' + path = path.replace(/\/{2,}/g, "/"); + return path; +} + +function splitPath(path) { + if (path.length === 0) return []; + if (path === "/") return ["/"]; + let parts = path.split("/"); + if (parts[parts.length - 1] === '') { + parts.pop(); + } + if (path[0] === "/") { + // assert(parts[0] === '') + parts[0] = "/"; + } else { + if (parts[0] !== ".") { + parts.unshift("."); + } + } + return parts; +} + +function dirname(path) { + const last = path.lastIndexOf("/"); + if (last === -1) throw new Error(`Cannot get dirname of "${path}"`); + if (last === 0) return "/"; + return path.slice(0, last); +} + +function basename(path) { + if (path === "/") throw new Error(`Cannot get basename of "${path}"`); + const last = path.lastIndexOf("/"); + if (last === -1) return path; + return path.slice(last + 1); +} + +function reducer(ancestors, current) { + // Initial condition + if (ancestors.length === 0) { + ancestors.push(current); + return ancestors; + } + // assert(ancestors.length > 0) + // assert(ancestors[0] === '.' || ancestors[0] === '/') + + // Collapse '.' references + if (current === ".") return ancestors; + + // Collapse '..' references + if (current === "..") { + if (ancestors.length === 1) { + if (ancestors[0] === "/") { + throw new Error("Unable to normalize path - traverses above root directory"); + } + // assert(ancestors[0] === '.') + if (ancestors[0] === ".") { + ancestors.push(current); + return ancestors; + } + } + // assert(ancestors.length > 1) + if (ancestors[ancestors.length - 1] === "..") { + ancestors.push(".."); + return ancestors; + } else { + ancestors.pop(); + return ancestors; + } + } + + ancestors.push(current); + return ancestors; +} + +module.exports = { + join: joinPath, + normalize: normalizePath, + split: splitPath, + basename, + dirname, + resolve: resolvePath, +}; From d648fae07d2fe5626686b67f3cb0e9b8a912e6f6 Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 1 Dec 2020 15:24:39 -0500 Subject: [PATCH 41/43] replace path.js with @stoplight/path (mostly) --- package-lock.json | 6 ++ package.json | 1 + src/backends/yjs/YjsBackend.js | 21 ++++++- src/backends/yjs/path.js | 107 --------------------------------- 4 files changed, 26 insertions(+), 109 deletions(-) delete mode 100755 src/backends/yjs/path.js diff --git a/package-lock.json b/package-lock.json index 56bac51..0629921 100644 --- a/package-lock.json +++ b/package-lock.json @@ -437,6 +437,12 @@ } } }, + "@stoplight/path": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@stoplight/path/-/path-1.3.2.tgz", + "integrity": "sha512-lyIc6JUlUA8Ve5ELywPC8I2Sdnh1zc1zmbYgVarhXIp9YeAB0ReeqmGEOWNtlHkbP2DAA1AL65Wfn2ncjK/jtQ==", + "dev": true + }, "@tootallnate/once": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", diff --git a/package.json b/package.json index f526797..ec7e0e9 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "yjs": "^13.4.2" }, "devDependencies": { + "@stoplight/path": "^1.3.2", "karma": "2.0.5", "karma-browserstack-launcher": "^1.5.1", "karma-chrome-launcher": "3.1.0", diff --git a/src/backends/yjs/YjsBackend.js b/src/backends/yjs/YjsBackend.js index 9929a73..a06ad80 100644 --- a/src/backends/yjs/YjsBackend.js +++ b/src/backends/yjs/YjsBackend.js @@ -1,6 +1,6 @@ const { encode } = require("isomorphic-textencoder"); +const path = require("@stoplight/path"); -const path = require("./path.js"); const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); const { bton, ntob } = require("./radix64.js"); @@ -39,6 +39,23 @@ function ylast (yarr) { return yarr.get(yarr.length - 1); } +function splitPath(path) { + if (path.length === 0) return []; + if (path === "/") return ["/"]; + let parts = path.split("/"); + if (parts[parts.length - 1] === '') { + parts.pop(); + } + if (path[0] === "/") { + parts[0] = "/"; + } else { + if (parts[0] !== ".") { + parts.unshift("."); + } + } + return parts; +} + module.exports = class YjsBackend { constructor(Y, ydoc, find) { this.Y = Y; @@ -117,7 +134,7 @@ module.exports = class YjsBackend { let dir = this._inodes.get(0); if (filepath === '/') return dir; let partialPath = '/' - let parts = path.split(filepath) + let parts = splitPath(filepath) // TODO: Actually, given we can reconstruct paths from the bottom up, // it might be faster to search by matching against the basepath and then // narrowing that set. The problem would be dealing with symlinks. diff --git a/src/backends/yjs/path.js b/src/backends/yjs/path.js deleted file mode 100755 index e45952f..0000000 --- a/src/backends/yjs/path.js +++ /dev/null @@ -1,107 +0,0 @@ -function normalizePath(path) { - if (path.length === 0) { - return "."; - } - let parts = splitPath(path); - parts = parts.reduce(reducer, []); - return joinPath(...parts); -} - -function resolvePath(...paths) { - let result = ''; - for (let path of paths) { - if (path.startsWith('/')) { - result = path; - } else { - result = normalizePath(joinPath(result, path)); - } - } - return result; -} - -function joinPath(...parts) { - if (parts.length === 0) return ""; - let path = parts.join("/"); - // Replace consecutive '/' - path = path.replace(/\/{2,}/g, "/"); - return path; -} - -function splitPath(path) { - if (path.length === 0) return []; - if (path === "/") return ["/"]; - let parts = path.split("/"); - if (parts[parts.length - 1] === '') { - parts.pop(); - } - if (path[0] === "/") { - // assert(parts[0] === '') - parts[0] = "/"; - } else { - if (parts[0] !== ".") { - parts.unshift("."); - } - } - return parts; -} - -function dirname(path) { - const last = path.lastIndexOf("/"); - if (last === -1) throw new Error(`Cannot get dirname of "${path}"`); - if (last === 0) return "/"; - return path.slice(0, last); -} - -function basename(path) { - if (path === "/") throw new Error(`Cannot get basename of "${path}"`); - const last = path.lastIndexOf("/"); - if (last === -1) return path; - return path.slice(last + 1); -} - -function reducer(ancestors, current) { - // Initial condition - if (ancestors.length === 0) { - ancestors.push(current); - return ancestors; - } - // assert(ancestors.length > 0) - // assert(ancestors[0] === '.' || ancestors[0] === '/') - - // Collapse '.' references - if (current === ".") return ancestors; - - // Collapse '..' references - if (current === "..") { - if (ancestors.length === 1) { - if (ancestors[0] === "/") { - throw new Error("Unable to normalize path - traverses above root directory"); - } - // assert(ancestors[0] === '.') - if (ancestors[0] === ".") { - ancestors.push(current); - return ancestors; - } - } - // assert(ancestors.length > 1) - if (ancestors[ancestors.length - 1] === "..") { - ancestors.push(".."); - return ancestors; - } else { - ancestors.pop(); - return ancestors; - } - } - - ancestors.push(current); - return ancestors; -} - -module.exports = { - join: joinPath, - normalize: normalizePath, - split: splitPath, - basename, - dirname, - resolve: resolvePath, -}; From fd464c73d5e67fee1e27e0e580b48c06e39cb9cb Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 1 Dec 2020 15:25:52 -0500 Subject: [PATCH 42/43] remove peerDependency --- package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/package.json b/package.json index ec7e0e9..4f62c1c 100644 --- a/package.json +++ b/package.json @@ -21,9 +21,6 @@ "just-debounce-it": "1.1.0", "just-once": "1.1.0" }, - "peerDependencies": { - "yjs": "^13.4.2" - }, "devDependencies": { "@stoplight/path": "^1.3.2", "karma": "2.0.5", From 81b81775dcc60881c6daacc6acedc1bb566215ec Mon Sep 17 00:00:00 2001 From: William Hilton Date: Tue, 1 Dec 2020 20:14:39 -0500 Subject: [PATCH 43/43] ditch the radix64 for base 36. simpler is gooder --- src/backends/yjs/YjsBackend.js | 11 +++++--- src/backends/yjs/radix64.js | 50 ---------------------------------- 2 files changed, 7 insertions(+), 54 deletions(-) delete mode 100644 src/backends/yjs/radix64.js diff --git a/src/backends/yjs/YjsBackend.js b/src/backends/yjs/YjsBackend.js index a06ad80..85edb64 100644 --- a/src/backends/yjs/YjsBackend.js +++ b/src/backends/yjs/YjsBackend.js @@ -2,7 +2,6 @@ const { encode } = require("isomorphic-textencoder"); const path = require("@stoplight/path"); const { EEXIST, ENOENT, ENOTDIR, ENOTEMPTY } = require("./errors.js"); -const { bton, ntob } = require("./radix64.js"); const TYPE = 't'; const MTIME = 'm'; @@ -17,15 +16,19 @@ function ID (client, clock) { this.clock = clock; } +// https://www.ecma-international.org/ecma-262/#sec-number.prototype.tostring +const MAX_RADIX = 36; + function serializeID (id) { - return `${ntob(id.client)}-${ntob(id.clock)}`; + // Numbers are encoded in base 36 to save space. + return `${id.client.toString(MAX_RADIX)}-${id.clock.toString(MAX_RADIX)}`; } function parseID (arr) { if (!arr) return arr; const id = arr.indexOf('-'); - const client = bton(arr.slice(0, id)); - const clock = bton(arr.slice(id + 1)); + const client = parseInt(arr.slice(0, id), MAX_RADIX); + const clock = parseInt(arr.slice(id + 1), MAX_RADIX); return new ID(client, clock); } diff --git a/src/backends/yjs/radix64.js b/src/backends/yjs/radix64.js deleted file mode 100644 index 25a2844..0000000 --- a/src/backends/yjs/radix64.js +++ /dev/null @@ -1,50 +0,0 @@ -// https://stackoverflow.com/a/48301665/2168416 - -const alphabet = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_.'; - -// binary to string lookup table -const b2s = alphabet.split(''); - -// string to binary lookup table -// 123 == 'z'.charCodeAt(0) + 1 -const s2b = new Array(123); -for (let i = 0; i < alphabet.length; i++) { - s2b[alphabet.charCodeAt(i)] = i; -} - -module.exports = { - // number to base64 - ntob: (number) => { - if (number < 0) return `-${ntob(-number)}`; - - let lo = number >>> 0; - let hi = (number / 4294967296) >>> 0; - - let right = ''; - while (hi > 0) { - right = b2s[0x3f & lo] + right; - lo >>>= 6; - lo |= (0x3f & hi) << 26; - hi >>>= 6; - } - - let left = ''; - do { - left = b2s[0x3f & lo] + left; - lo >>>= 6; - } while (lo > 0); - - return left + right; - }, - // base64 to number - bton: (base64) => { - let number = 0; - const sign = base64.charAt(0) === '-' ? 1 : 0; - - for (let i = sign; i < base64.length; i++) { - number = number * 64 + s2b[base64.charCodeAt(i)]; - } - - return sign ? -number : number; - }, -};