diff --git a/cli/package-lock.json b/cli/package-lock.json index 48e076f..68e2538 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -1,17 +1,17 @@ { "name": "@massalabs/deweb-cli", - "version": "0.3.0", + "version": "0.3.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@massalabs/deweb-cli", - "version": "0.3.0", + "version": "0.3.1", "license": "MIT", "dependencies": { "@commander-js/extra-typings": "^12.1.0", "@listr2/prompt-adapter-enquirer": "^2.0.11", - "@massalabs/massa-web3": "5.0.1-dev.20241212140726", + "@massalabs/massa-web3": "5.1.0", "commander": "^12.1.0", "enquirer": "^2.4.1", "js-sha256": "^0.11.0", @@ -1802,9 +1802,10 @@ } }, "node_modules/@massalabs/massa-web3": { - "version": "5.0.1-dev.20241212140726", - "resolved": "https://registry.npmjs.org/@massalabs/massa-web3/-/massa-web3-5.0.1-dev.20241212140726.tgz", - "integrity": "sha512-FlPS6U2ckTYXDlMsnoDHj+aMZJ/pCZt+aWnjMYgl6PNvuMN5nuqrr6c0ryUwF1w3wX1adCPYsyXcYeO7Purzqg==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@massalabs/massa-web3/-/massa-web3-5.1.0.tgz", + "integrity": "sha512-fKlOjKD+F0JoUxLUUfweugt9MrM6P1F4WT80TdhgZ1yIKqguN0bNYsXzF9Wf6xVzljP/D+u1kwSDAQpZ/PZ8yg==", + "license": "MIT", "dependencies": { "@noble/ed25519": "^1.7.3", "@noble/hashes": "^1.2.0", diff --git a/cli/package.json b/cli/package.json index 6973ff7..da32ca3 100644 --- a/cli/package.json +++ b/cli/package.json @@ -28,7 +28,7 @@ "dependencies": { "@commander-js/extra-typings": "^12.1.0", "@listr2/prompt-adapter-enquirer": "^2.0.11", - "@massalabs/massa-web3": "5.0.1-dev.20241212140726", + "@massalabs/massa-web3": "5.1.0", "commander": "^12.1.0", "enquirer": "^2.4.1", "js-sha256": "^0.11.0", diff --git a/cli/src/commands/config.ts b/cli/src/commands/config.ts index 8ed0db2..c4f372e 100644 --- a/cli/src/commands/config.ts +++ b/cli/src/commands/config.ts @@ -1,7 +1,9 @@ -import { PublicApiUrl } from '@massalabs/massa-web3' +import { Address, PublicApiUrl } from '@massalabs/massa-web3' import { OptionValues } from 'commander' import { readFileSync } from 'fs' +import { Metadata } from '../lib/website/models/Metadata' + export const DEFAULT_CHUNK_SIZE = 64000 const DEFAULT_NODE_URL = PublicApiUrl.Buildnet @@ -11,12 +13,25 @@ interface Config { node_url: string chunk_size: number secret_key: string + address: string + metadatas: { [key: string]: string } } export function parseConfigFile(filePath: string): Config { const fileContent = readFileSync(filePath, 'utf-8') try { - return JSON.parse(fileContent) + const config = JSON.parse(fileContent) as Config + + // If address is provided, make sure it's valid + if (config.address) { + try { + Address.fromString(config.address) + } catch (error) { + throw new Error(`Invalid address in config file: ${error}`) + } + } + + return config } catch (error) { throw new Error(`Failed to parse file: ${error}`) } @@ -35,12 +50,26 @@ export function mergeConfigAndOptions( commandOptions.node_url || configOptions.node_url || DEFAULT_NODE_URL, chunk_size: configOptions.chunk_size || DEFAULT_CHUNK_SIZE, secret_key: configOptions.secret_key || '', + address: configOptions.address || '', + metadatas: configOptions.metadatas + ? makeMetadataArray(configOptions.metadatas) + : [], } } +function makeMetadataArray(metadatas: { [key: string]: string }): Metadata[] { + return Object.entries(metadatas).map( + ([key, value]) => new Metadata(key, value) + ) +} + export function setDefaultValues(commandOptions: OptionValues): OptionValues { return { node_url: commandOptions.node_url || DEFAULT_NODE_URL, chunk_size: commandOptions.chunk_size || DEFAULT_CHUNK_SIZE, + wallet: commandOptions.wallet || '', + password: commandOptions.password || '', + address: '', + metadatas: [], } } diff --git a/cli/src/commands/upload.ts b/cli/src/commands/upload.ts index cd021b5..dfdaa58 100644 --- a/cli/src/commands/upload.ts +++ b/cli/src/commands/upload.ts @@ -13,6 +13,8 @@ import { prepareUploadTask } from '../tasks/prepareUpload' import { UploadCtx } from '../tasks/tasks' import { confirmUploadTask, uploadBatchesTask } from '../tasks/upload' +import { divideMetadata } from '../lib/website/metadata' +import { Metadata } from '../lib/website/models/Metadata' import { makeProviderFromNodeURLAndSecret, validateAddress } from './utils' export const uploadCommand = new Command('upload') @@ -41,13 +43,23 @@ export const uploadCommand = new Command('upload') options.noIndex ) - if (options.address) { - const address = options.address + if (options.address || globalOptions.address) { + const address = options.address || (globalOptions.address as string) console.log(`Editing website at address ${address}, no deploy needed`) validateAddress(address) ctx.sc = new SmartContract(provider, address) + + const { updateRequired } = await divideMetadata( + ctx.provider, + ctx.sc.address, + globalOptions.metadatas as Metadata[] + ) + + ctx.metadatas.push(...updateRequired) + } else { + ctx.metadatas.push(...(globalOptions.metadatas as Metadata[])) } const tasksArray = [ diff --git a/cli/src/index.ts b/cli/src/index.ts index cb1e7e6..86134e3 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -14,6 +14,8 @@ import { setDefaultValues, } from './commands/config' +import { Metadata } from './lib/website/models/Metadata' + const version = process.env.VERSION || 'dev' const defaultConfigPath = 'deweb_cli_config.json' @@ -38,6 +40,8 @@ interface OptionValues { node_url: string wallet: string password: string + address: string + metadatas: Metadata[] } const commandOptions: OptionValues = program.opts() as OptionValues diff --git a/cli/src/lib/index/read.ts b/cli/src/lib/index/read.ts index 1b50500..3d9ddae 100644 --- a/cli/src/lib/index/read.ts +++ b/cli/src/lib/index/read.ts @@ -1,6 +1,6 @@ -import { bytesToStr, Provider } from '@massalabs/massa-web3' +import { bytesToStr, PublicProvider } from '@massalabs/massa-web3' -import { addressToOwnerBaseKey } from './keys' +import { addressToOwnerBaseKey, indexByOwnerBaseKey } from './keys' import { getSCAddress } from './utils' /** @@ -10,7 +10,7 @@ import { getSCAddress } from './utils' * @returns The owner of the website */ export async function getWebsiteOwner( - provider: Provider, + provider: PublicProvider, address: string ): Promise { const scAddress = getSCAddress((await provider.networkInfos()).chainId) @@ -25,3 +25,27 @@ export async function getWebsiteOwner( const ownerKeySliced = ownerKey.slice(prefix.length) return bytesToStr(ownerKeySliced) } + +/** + * Get the website addresses owned by an address according to the index smart contract. + * @param provider - PublicProvider instance + * @param owner - The owner's address + * @returns The website addresses owned by the owner + */ +export async function getAddressWebsites( + provider: PublicProvider, + owner: string +): Promise { + const scAddress = getSCAddress((await provider.networkInfos()).chainId) + const prefix = indexByOwnerBaseKey(owner) + + const keys = await provider.getStorageKeys(scAddress, prefix) + if (keys.length === 0) { + return [] + } + + return keys.map((key) => { + const keySliced = key.slice(prefix.length) + return bytesToStr(keySliced) + }) +} diff --git a/cli/src/lib/website/delete.ts b/cli/src/lib/website/delete.ts index 1f3c1ed..87a33d6 100644 --- a/cli/src/lib/website/delete.ts +++ b/cli/src/lib/website/delete.ts @@ -5,7 +5,8 @@ import { SmartContract, } from '@massalabs/massa-web3' -import { getGlobalMetadata, listFiles } from './read' +import { listFiles } from './read' +import { getGlobalMetadata } from './metadata' import { FileDelete } from './models/FileDelete' import { Metadata } from './models/Metadata' diff --git a/cli/src/lib/website/filesInit.ts b/cli/src/lib/website/filesInit.ts index 779e998..81498cd 100644 --- a/cli/src/lib/website/filesInit.ts +++ b/cli/src/lib/website/filesInit.ts @@ -10,23 +10,26 @@ import { strToBytes, U32, } from '@massalabs/massa-web3' + +import { + CallManager, + CallStatus, + CallUpdate, + FunctionCall, +} from '../utils/callManager' import { storageCostForEntry } from '../utils/storage' +import { maxBigInt } from '../utils/utils' +import { FileDelete } from './models/FileDelete' + import { FileInit } from './models/FileInit' import { Metadata } from './models/Metadata' + import { FILE_TAG, fileChunkCountKey, fileLocationKey, globalMetadataKey, } from './storageKeys' -import { FileDelete } from './models/FileDelete' -import { maxBigInt } from '../utils/utils' -import { - FunctionCall, - CallManager, - CallUpdate, - CallStatus, -} from '../utils/callManager' const functionName = 'filesInit' export const batchSize = 32 @@ -154,13 +157,19 @@ export async function sendFilesInits( // TODO: Improve estimation // - If a file is already stored, we don't need to send coins for its hash storage -export async function filesInitCost( +export async function prepareCost( _: SmartContract, files: FileInit[], filesToDelete: FileDelete[], metadatas: Metadata[], metadatasToDelete: Metadata[] -): Promise { +): Promise<{ + filePathListCost: bigint + storageCost: bigint + filesToDeleteCost: bigint + metadatasCost: bigint + metadatasToDeleteCost: bigint +}> { const filePathListCost = files.reduce((acc, chunk) => { return ( acc + @@ -211,6 +220,30 @@ export async function filesInitCost( ) }, 0n) + return { + filePathListCost, + storageCost, + filesToDeleteCost, + metadatasCost, + metadatasToDeleteCost, + } +} + +export async function filesInitCost( + _sc: SmartContract, + files: FileInit[], + filesToDelete: FileDelete[], + metadatas: Metadata[], + metadatasToDelete: Metadata[] +): Promise { + const { + filePathListCost, + storageCost, + filesToDeleteCost, + metadatasCost, + metadatasToDeleteCost, + } = await prepareCost(_sc, files, filesToDelete, metadatas, metadatasToDelete) + return BigInt( filePathListCost + storageCost + diff --git a/cli/src/lib/website/metadata.ts b/cli/src/lib/website/metadata.ts new file mode 100644 index 0000000..1518daa --- /dev/null +++ b/cli/src/lib/website/metadata.ts @@ -0,0 +1,145 @@ +import { + Args, + Operation, + Provider, + PublicProvider, +} from '@massalabs/massa-web3' +import { storageCostForEntry } from '../utils/storage' +import { Metadata } from './models/Metadata' +import { globalMetadataKey } from './storageKeys' + +const SET_GLOBAL_METADATA_FUNCTION = 'setMetadataGlobal' + +export const TITLE_METADATA_KEY = 'TITLE' +export const DESCRIPTION_METADATA_KEY = 'DESCRIPTION' +export const KEYWORD_METADATA_KEY_PREFIX = 'KEYWORD' +export const LAST_UPDATE_KEY = 'LAST_UPDATE' + +/** + * Get the global metadata of a website stored on Massa blockchain + * @param provider - Provider instance + * @param address - Address of the website + * @param prefix - Prefix of the metadata + * @returns - List of Metadata objects + */ +export async function getGlobalMetadata( + provider: PublicProvider, + address: string, + prefix: Uint8Array = new Uint8Array() +): Promise { + const metadataKeys = await provider.getStorageKeys( + address, + globalMetadataKey(prefix) + ) + const metadata = await provider.readStorage(address, metadataKeys) + + return metadata.map((m, index) => { + const metadataKeyBytes = metadataKeys[index].slice( + globalMetadataKey(new Uint8Array()).length + ) + const key = String.fromCharCode(...new Uint8Array(metadataKeyBytes)) + const value = String.fromCharCode(...new Uint8Array(m)) + + return new Metadata(key, value) + }) +} + +/** + * Returns the title, description and keywords of a website stored on Massa blockchain + * Keywords are sorted alphabetically (keyword1, keyword2, keyword3, ...) + * @param provider - Provider instance + * @param address - Address of the website + * @returns - An object with the title, description and keywords + */ +export async function getWebsiteMetadata( + provider: PublicProvider, + address: string +): Promise<{ + title: string + description: string + keywords: string[] +}> { + const metadata = await getGlobalMetadata(provider, address) + + const title = metadata.find((m) => m.key === TITLE_METADATA_KEY)?.value + const description = metadata.find( + (m) => m.key === DESCRIPTION_METADATA_KEY + )?.value + const keywords = metadata + .filter((m) => m.key.startsWith(KEYWORD_METADATA_KEY_PREFIX)) + .sort() + .map((m) => m.value) + + return { title: title ?? '', description: description ?? '', keywords } +} + +/** + * Set a list of global metadata on a website stored on Massa blockchain + * @param provider - Provider instance + * @param address - Address of the website SC + * @param metadatas - List of Metadata objects + */ +export async function setGlobalMetadata( + provider: Provider, + address: string, + metadatas: Metadata[] +): Promise { + const storedGlobalMetadata = await getGlobalMetadata(provider, address) + const changedKeys = metadatas.filter( + (metadata) => + !storedGlobalMetadata.some( + (storedMetadata) => storedMetadata.key === metadata.key + ) + ) + const encoder = new TextEncoder() + const coins = changedKeys.reduce( + (sum, metadata) => + sum + + storageCostForEntry( + BigInt(globalMetadataKey(encoder.encode(metadata.key)).length), + BigInt(metadata.value.length) + ), + 0n + ) + + const args = new Args().addSerializableObjectArray(metadatas).serialize() + + return provider.callSC({ + target: address, + func: SET_GLOBAL_METADATA_FUNCTION, + parameter: args, + coins: coins, + }) +} + +/** + * Divide a list of metadatas into metadatas that require to be updated or not + * @param provider - Provider instance + * @param address - Address of the website SC + * @param metadatas - List of Metadata objects + * @returns - An object with the metadatas that require to be updated and the ones that don't + */ +export async function divideMetadata( + provider: PublicProvider, + address: string, + metadatas: Metadata[] +): Promise<{ updateRequired: Metadata[]; noUpdateRequired: Metadata[] }> { + const storedGlobalMetadata = await getGlobalMetadata(provider, address) + const updateRequired = metadatas.filter( + (metadata) => + !storedGlobalMetadata.some( + (storedMetadata) => + storedMetadata.key === metadata.key && + storedMetadata.value === metadata.value + ) + ) + const noUpdateRequired = metadatas.filter((metadata) => + storedGlobalMetadata.some( + (storedMetadata) => + storedMetadata.key === metadata.key && + storedMetadata.value === metadata.value + ) + ) + + return { updateRequired, noUpdateRequired } +} diff --git a/cli/src/lib/website/read.ts b/cli/src/lib/website/read.ts index 976aa3b..0f586e3 100644 --- a/cli/src/lib/website/read.ts +++ b/cli/src/lib/website/read.ts @@ -1,13 +1,11 @@ -import { Provider, U32 } from '@massalabs/massa-web3' +import { PublicProvider, U32 } from '@massalabs/massa-web3' import { sha256 } from 'js-sha256' import { FILE_LOCATION_TAG, fileChunkCountKey, fileChunkKey, - globalMetadataKey, } from './storageKeys' -import { Metadata } from './models/Metadata' /** * Lists files from the given website on Massa blockchain @@ -15,7 +13,7 @@ import { Metadata } from './models/Metadata' * @returns List of file paths in the website */ export async function listFiles( - provider: Provider, + provider: PublicProvider, scAddress: string ): Promise { const allStorageKeys = await provider.getStorageKeys( @@ -37,7 +35,7 @@ export async function listFiles( * @returns Total number of chunks for the file */ export async function getFileTotalChunks( - provider: Provider, + provider: PublicProvider, scAddress: string, filePath: string ): Promise { @@ -69,7 +67,7 @@ export async function getFileTotalChunks( * @returns - Uint8Array of the file content */ export async function getFileFromAddress( - provider: Provider, + provider: PublicProvider, scAddress: string, filePath: string ): Promise { @@ -104,32 +102,3 @@ export async function getFileFromAddress( return concatenatedArray } - -/** - * Get the metadata of a file from the given website on Massa blockchain - * @param provider - Provider instance - * @param address - Address of the website - * @param prefix - Prefix of the metadata - * @returns - List of Metadata objects - */ -export async function getGlobalMetadata( - provider: Provider, - address: string, - prefix: Uint8Array = new Uint8Array() -): Promise { - const metadataKeys = await provider.getStorageKeys( - address, - globalMetadataKey(prefix) - ) - const metadata = await provider.readStorage(address, metadataKeys) - - return metadata.map((m, index) => { - const metadataKeyBytes = metadataKeys[index].slice( - globalMetadataKey(new Uint8Array()).length - ) - const key = String.fromCharCode(...new Uint8Array(metadataKeyBytes)) - const value = String.fromCharCode(...new Uint8Array(m)) - - return new Metadata(key, value) - }) -} diff --git a/cli/src/lib/website/uploadChunk.ts b/cli/src/lib/website/uploadChunk.ts index 130013e..96ec2dc 100644 --- a/cli/src/lib/website/uploadChunk.ts +++ b/cli/src/lib/website/uploadChunk.ts @@ -4,6 +4,7 @@ import { MAX_GAS_CALL, minBigInt, Operation, + Provider, ReadOnlyCallResult, ReadOnlyParams, SmartContract, @@ -80,17 +81,19 @@ export function makeArgsCoinsFromBatch(batch: Batch): { /** * Estimate the gas cost for each batch - * @param sc - SmartContract instance + * @param provider - Provider instance + * @param address - Address of the smart contract * @param batches - the batches to estimate gas for * @returns the list of UploadBatch with gas estimation */ export async function estimateUploadBatchesGas( - sc: SmartContract, + provider: Provider, + address: string, batches: UploadBatch[] ): Promise { const BATCH_SIZE = 5 - const nodeURL = (await sc.provider.networkInfos()).url + const nodeURL = (await provider.networkInfos()).url if (!nodeURL) { throw new Error('Node URL not found') } @@ -102,10 +105,10 @@ export async function estimateUploadBatchesGas( return { coins: coins, maxGas: MAX_GAS_CALL, - target: sc.address, + target: address, func: 'uploadFileChunks', parameter: args.serialize(), - caller: sc.provider.address, + caller: provider.address, fee: undefined, } }) diff --git a/cli/src/tasks/estimations.ts b/cli/src/tasks/estimations.ts index f91b1a3..3ee3e11 100644 --- a/cli/src/tasks/estimations.ts +++ b/cli/src/tasks/estimations.ts @@ -3,11 +3,11 @@ import { ListrTask } from 'listr2' import { Batch } from '../lib/batcher' import { BatchStatus, UploadBatch } from '../lib/uploadManager' +import { formatBytes } from '../lib/utils/utils' import { computeChunkCost } from '../lib/website/chunk' import { estimateUploadBatchesGas } from '../lib/website/uploadChunk' import { UploadCtx } from './tasks' -import { formatBytes } from '../lib/utils/utils' /** * Create a task to estimate the cost of each batch @@ -82,7 +82,11 @@ export function estimateGasTask(): ListrTask { gas: 0n, })) - ctx.uploadBatches = await estimateUploadBatchesGas(ctx.sc, batches) + ctx.uploadBatches = await estimateUploadBatchesGas( + ctx.provider, + ctx.sc.address, + batches + ) const totalGas = ctx.uploadBatches.reduce( (sum: bigint, batch: UploadBatch) => sum + batch.gas, diff --git a/cli/src/tasks/prepareUpload.ts b/cli/src/tasks/prepareUpload.ts index 2e42cb5..13bdc91 100644 --- a/cli/src/tasks/prepareUpload.ts +++ b/cli/src/tasks/prepareUpload.ts @@ -4,9 +4,10 @@ import { ListrTask } from 'listr2' import { batchSize, - filesInitCost, + prepareCost, sendFilesInits, } from '../lib/website/filesInit' +import { LAST_UPDATE_KEY } from '../lib/website/metadata' import { Metadata } from '../lib/website/models/Metadata' import { UploadCtx } from './tasks' @@ -19,20 +20,38 @@ export function prepareUploadTask(): ListrTask { return { title: 'Prepare upload', task: (ctx: UploadCtx, task) => { - if (ctx.fileInits.length === 0 && ctx.filesToDelete.length === 0) { - task.skip('All files are ready for upload') + if ( + ctx.fileInits.length === 0 && + ctx.filesToDelete.length === 0 && + ctx.metadatas.length === 0 && + ctx.metadatasToDelete.length === 0 + ) { + task.skip('All files are ready for upload, and no metadata changes') return } const utcNowDate = U64.fromNumber(Math.floor(Date.now() / 1000)) - ctx.metadatas.push(new Metadata('LAST_UPDATE', utcNowDate.toString())) + ctx.metadatas.push(new Metadata(LAST_UPDATE_KEY, utcNowDate.toString())) return task.newListr( [ { title: 'Confirm SC preparation', task: async (ctx, subTask) => { + if ( + ctx.fileInits.length !== 0 || + ctx.filesToDelete.length !== 0 + ) { + subTask.output = 'Files changes detected' + } + if ( + ctx.metadatas.length !== 0 || + ctx.metadatasToDelete.length !== 0 + ) { + subTask.output = 'Metadata changes detected' + } + const totalChanges = ctx.fileInits.length + ctx.filesToDelete.length + @@ -40,15 +59,34 @@ export function prepareUploadTask(): ListrTask { ctx.metadatasToDelete.length const estimatedOperations = Math.ceil(totalChanges / batchSize) const minimalFees = ctx.minimalFees * BigInt(estimatedOperations) - const cost = - (await filesInitCost( - ctx.sc, - ctx.fileInits, - ctx.filesToDelete, - ctx.metadatas, - ctx.metadatasToDelete - )) + minimalFees - subTask.output = `SC preparation costs ${formatMas(cost)} MAS (including ${formatMas(minimalFees)} MAS of minimal fees)` + const { + filePathListCost, + storageCost, + filesToDeleteCost, + metadatasCost, + metadatasToDeleteCost, + } = await prepareCost( + ctx.sc, + ctx.fileInits, + ctx.filesToDelete, + ctx.metadatas, + ctx.metadatasToDelete + ) + + const totalCost = + filePathListCost + + storageCost - + filesToDeleteCost + + metadatasCost - + metadatasToDeleteCost + + subTask.output = `Estimated cost of SC preparation:` + subTask.output = ` + Files init: ${formatMas(filePathListCost)} MAS` + subTask.output = ` + Storage: ${formatMas(storageCost)} MAS` + subTask.output = ` - Files to delete: ${formatMas(filesToDeleteCost)} MAS` + subTask.output = ` + Metadatas: ${formatMas(metadatasCost)} MAS` + subTask.output = ` - Metadatas to delete: ${formatMas(metadatasToDeleteCost)} MAS` + subTask.output = `SC preparation costs ${formatMas(totalCost + minimalFees)} MAS (including ${formatMas(minimalFees)} MAS of minimal fees)` if (!ctx.skipConfirm) { const answer = await subTask