diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3c3629e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/README.md b/README.md new file mode 100644 index 0000000..d15fdf5 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# ssb-blob-prune + +> Prune unused blobs from Scuttlebutt. + +## Table of Contents + +- [Install](#install) +- [Usage](#usage) +- [Maintainers](#maintainers) +- [Contributing](#contributing) +- [License](#license) + +## Install + +```sh +npm -g install ssb-prune-blobs +``` + +## Usage + +If you want to test this out *without* installing, use: + +```sh +npx ssb-blob-prune +``` + +If you've already installed above, you can just use: + +```sh +ssb-blob-prune +``` + +## Maintainers + +[@fraction](https://github.com/fraction) + +## Contributing + +PRs accepted. + +## License + +MIT © 2018 Fraction LLC diff --git a/index.js b/index.js new file mode 100755 index 0000000..2468670 --- /dev/null +++ b/index.js @@ -0,0 +1,81 @@ +#!/usr/bin/env node + +const pull = require('pull-stream') +const ssbClient = require('ssb-client') +const _ = require('deepdash')(require('lodash')) +const ref = require('ssb-ref') +const multiblob = require('multiblob') +const path = require('path') +const debug = require('debug')('ssb-blob-prune') +const os = require('os') + +debug.enabled = true + +const config = { + messagesPerDebug: 1000 +} + +// create a scuttlebot client using default settings +// (server at localhost:8080, using key found at ~/.ssb/secret) +ssbClient(function (err, sbot) { + if (err) { + debug('could not connect to sbot, please ensure that it\'s running') + throw err + } + + debug('connected to sbot') + + const mentionedBlobs = [] + const strayBlobs = [] + let messagesConsumed = 0 + + const debugMessages = debug.extend('messages') + + // stream all messages in all feeds, ordered by publish time + pull( + sbot.createFeedStream(), + pull.drain((msg) => { + // on each + messagesConsumed = messagesConsumed + 1 + if (messagesConsumed % config.messagesPerDebug === 0) { + debugMessages('%d', messagesConsumed) + } + + _.eachDeep(msg, value => { + if (ref.isBlob(value)) { + const sansAmpersand = value.substr(1) + mentionedBlobs.push(sansAmpersand) + } + }) + }, () => { + const debugBlobs = debug.extend('blobs') + debugBlobs('enumerating') + // on done + var blobs = multiblob({ + dir: path.join(os.homedir(), '.ssb', 'blobs'), + alg: 'sha256' + }) + + pull( + blobs.ls(), + pull.drain((blob) => { + if (!mentionedBlobs.includes(blob)) { + debugBlobs('stray %s', blob) + strayBlobs.push(blob) + blobs.rm(blob, (err, val) => { + if (err) throw err + + if (val) debugBlobs('%s', val) + }) + } + }, () => { + debug('previous total: %d', mentionedBlobs.length) + debug('deleted: %d', strayBlobs.length) + const currentTotal = mentionedBlobs.length - strayBlobs.length + debug('current total: %d', currentTotal) + sbot.close() + }) + ) + }) + ) +}) diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..b79c5b1 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,279 @@ +{ + "name": "multiblob-prune", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "blake2s": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/blake2s/-/blake2s-1.0.1.tgz", + "integrity": "sha1-FZiCKjIOzmqkAbqYKVT4L2GwzXs=" + }, + "cont": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cont/-/cont-1.0.3.tgz", + "integrity": "sha1-aHTx6TX8qZ0EjK6qrZoK6wILzOA=", + "requires": { + "continuable": "~1.2.0", + "continuable-para": "~1.2.0", + "continuable-series": "~1.2.0" + } + }, + "continuable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.2.0.tgz", + "integrity": "sha1-CCd0aNQRNiAAdMz4cpQwjRafJbY=" + }, + "continuable-hash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/continuable-hash/-/continuable-hash-0.1.4.tgz", + "integrity": "sha1-gcdNQXcdjJJ4Ph4A5fEbNNbfx4w=", + "requires": { + "continuable": "~1.1.6" + }, + "dependencies": { + "continuable": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.1.8.tgz", + "integrity": "sha1-3Id7R0FghwrjvN6HM2Jo6+UFl9U=" + } + } + }, + "continuable-list": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/continuable-list/-/continuable-list-0.1.6.tgz", + "integrity": "sha1-h88G7FgHFuEN/5X7C4TF8OisrF8=", + "requires": { + "continuable": "~1.1.6" + }, + "dependencies": { + "continuable": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.1.8.tgz", + "integrity": "sha1-3Id7R0FghwrjvN6HM2Jo6+UFl9U=" + } + } + }, + "continuable-para": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/continuable-para/-/continuable-para-1.2.0.tgz", + "integrity": "sha1-RFUQ9klFndD8NchyAVFGEicxxYM=", + "requires": { + "continuable-hash": "~0.1.4", + "continuable-list": "~0.1.5" + } + }, + "continuable-series": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/continuable-series/-/continuable-series-1.2.0.tgz", + "integrity": "sha1-MkM5euk6cdZVswJoNKUVkLlYueg=" + }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, + "deepdash": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/deepdash/-/deepdash-1.5.5.tgz", + "integrity": "sha512-CUI0nOTaM++CydaT6MLg7bDPEddK39qoZvLXaYJAmiNZLdBFwLyCerMaBH2SfbUHyeRL2Ci8qa878NDV9PmF6g==", + "requires": { + "lodash": "^4.17.11" + } + }, + "explain-error": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/explain-error/-/explain-error-1.0.4.tgz", + "integrity": "sha1-p5PTrAytTGq1cemWj7urbLJTKSk=" + }, + "ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + }, + "lodash": { + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + }, + "looper": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/looper/-/looper-4.0.0.tgz", + "integrity": "sha1-dwat7VmpntygbmtUu4bI7BnJUVU=" + }, + "minimist": { + "version": "0.0.8", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "requires": { + "minimist": "0.0.8" + } + }, + "multiblob": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/multiblob/-/multiblob-1.13.2.tgz", + "integrity": "sha512-z1nRV/iLDQQ2ih1lkOmOod4OLaaYk1LayU5mUJTGItekthyuWVWT+uuMp7mI1zXczEVuWky4MaDXuekKHkrJUQ==", + "requires": { + "blake2s": "~1.0.1", + "cont": "~1.0.1", + "explain-error": "~1.0.1", + "mkdirp": "~0.5.0", + "pull-cat": "^1.1.8", + "pull-catch": "^1.0.0", + "pull-defer": "^0.2.2", + "pull-file": "^0.5.0", + "pull-glob": "~1.0.6", + "pull-live": "^1.0.0", + "pull-notify": "^0.1.1", + "pull-paramap": "^1.2.2", + "pull-stream": "^3.6.2", + "pull-write-file": "^0.2.1", + "rc": "~1.2.8", + "rimraf": "~2.2.8", + "stream-to-pull-stream": "^1.7.2" + } + }, + "pull-cat": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/pull-cat/-/pull-cat-1.1.11.tgz", + "integrity": "sha1-tkLdElXaN2pwa220+pYvX9t0wxs=" + }, + "pull-catch": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/pull-catch/-/pull-catch-1.0.0.tgz", + "integrity": "sha1-9YA361woLMtQavn3awAn0zkx5Is=" + }, + "pull-defer": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/pull-defer/-/pull-defer-0.2.3.tgz", + "integrity": "sha512-/An3KE7mVjZCqNhZsr22k1Tx8MACnUnHZZNPSJ0S62td8JtYr/AiRG42Vz7Syu31SoTLUzVIe61jtT/pNdjVYA==" + }, + "pull-file": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/pull-file/-/pull-file-0.5.0.tgz", + "integrity": "sha1-s8pAUwbggvnUUoKIkzutsrZWNls=", + "requires": { + "pull-utf8-decoder": "^1.0.2" + } + }, + "pull-fs": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/pull-fs/-/pull-fs-1.1.6.tgz", + "integrity": "sha1-8YT2p3KLtNlWQTdr6tafb2bfR80=", + "requires": { + "pull-file": "^0.5.0", + "pull-stream": "^3.3.0", + "pull-traverse": "^1.0.3", + "pull-write-file": "^0.2.1" + } + }, + "pull-glob": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/pull-glob/-/pull-glob-1.0.7.tgz", + "integrity": "sha1-7vkV3eZEvdvqjdLgEG1USqy81cI=", + "requires": { + "pull-fs": "~1.1.6", + "pull-stream": "^3.3.0" + } + }, + "pull-live": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pull-live/-/pull-live-1.0.1.tgz", + "integrity": "sha1-pOzuAeMwFV6RJLu89HYfIbOPUfU=", + "requires": { + "pull-cat": "^1.1.9", + "pull-stream": "^3.4.0" + } + }, + "pull-notify": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/pull-notify/-/pull-notify-0.1.1.tgz", + "integrity": "sha1-b4b/ldJwuJw+vyVbYDG3Ay3JnMo=", + "requires": { + "pull-pushable": "^2.0.0" + } + }, + "pull-paramap": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/pull-paramap/-/pull-paramap-1.2.2.tgz", + "integrity": "sha1-UaQZPOnI1yFdla2tReK824STsjo=", + "requires": { + "looper": "^4.0.0" + } + }, + "pull-pushable": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pull-pushable/-/pull-pushable-2.2.0.tgz", + "integrity": "sha1-Xy867UethpGfAbEqLpnW8b13ZYE=" + }, + "pull-stream": { + "version": "3.6.9", + "resolved": "https://registry.npmjs.org/pull-stream/-/pull-stream-3.6.9.tgz", + "integrity": "sha512-hJn4POeBrkttshdNl0AoSCVjMVSuBwuHocMerUdoZ2+oIUzrWHFTwJMlbHND7OiKLVgvz6TFj8ZUVywUMXccbw==" + }, + "pull-traverse": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/pull-traverse/-/pull-traverse-1.0.3.tgz", + "integrity": "sha1-dPtde+f6a9enjpeTPhmbeUWGaTg=" + }, + "pull-utf8-decoder": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pull-utf8-decoder/-/pull-utf8-decoder-1.0.2.tgz", + "integrity": "sha1-p6+iOE0eZBWl1gIFQSbMjeO8vOc=" + }, + "pull-write-file": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/pull-write-file/-/pull-write-file-0.2.4.tgz", + "integrity": "sha1-Q3NErrIYn2XmeO0a838PdgpUU+8=" + }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + } + } + }, + "rimraf": { + "version": "2.2.8", + "resolved": "http://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz", + "integrity": "sha1-5Dm+Kq7jJzIZUnMPmaiSnk/FBYI=" + }, + "stream-to-pull-stream": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/stream-to-pull-stream/-/stream-to-pull-stream-1.7.2.tgz", + "integrity": "sha1-dXYJrhzr0zx0MtSvvjH/eGULnd4=", + "requires": { + "looper": "^3.0.0", + "pull-stream": "^3.2.3" + }, + "dependencies": { + "looper": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/looper/-/looper-3.0.0.tgz", + "integrity": "sha1-LvpUw7HLq6m5Su4uWRSwvlf7t0k=" + } + } + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..6db5bfa --- /dev/null +++ b/package.json @@ -0,0 +1,18 @@ +{ + "name": "ssb-blob-prune", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "deepdash": "^1.5.5", + "multiblob": "^1.13.2" + }, + "bin": { + "ssb-blob-prune": "./index.js" + } +}