diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..d582066 --- /dev/null +++ b/Pipfile @@ -0,0 +1,11 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..e327ba6 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,20 @@ +{ + "_meta": { + "hash": { + "sha256": "b20aaa9718024ec28b53570df5a504bc1765bb9490e31099da47c34bd616c357" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": {}, + "develop": {} +} diff --git a/README.md b/README.md index 6195ec1..23f9abb 100644 --- a/README.md +++ b/README.md @@ -45,25 +45,77 @@ export BLOB_READ_WRITE_TOKEN="vercel_blob_rw_ABC1234XYz" ### Put operation ```sh -vercel_storage blob upload path/to/file.zip +$ vercel_blob put disk_dump.bin +------------------ ---------------------------------------------------------------------------------------------------- +url https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/disk_dump-OTgBsduT0QQcfXImNMZky1NSy3HfML.bin +pathname disk_dump.bin +contentType application/octet-stream +contentDisposition attachment; filename="disk_dump.bin" +------------------ ---------------------------------------------------------------------------------------------------- ``` You can also print the output information as a json: ```sh -vercel_storage --json blob upload path/to/file.zip +$ vercel_blob --json put disk_dump.bin +{'url': 'https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/disk_dump-0eX9NYJnZjO31GsDiwDaQ6TR9QnWkH.bin', 'pathname': 'disk_dump.bin', 'contentType': 'text/plain', 'contentDisposition': 'attachment; filename="disk_dump.bin"'} ``` +By default, Vercel's blob store will insert a randomly generated string to the name of your file. But you can turn off that feature: + +```sh +$ vercel_blob put --no-suffix chart.png +------------------ ----------------------------------------------------------------- +url https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/chart.png +pathname chart.png +contentType image/png +contentDisposition attachment; filename="chart.png" +------------------ ----------------------------------------------------------------- +``` + +### List operation + +The list method returns a list of blob objects in a Blob store. For example, let's upload a file to the bob store: + +```sh +vercel_blob put profile.png +------------------ -------------------------------------------------------------------------------------------------- +url https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/profile-OtpJ1AUIxChAA6UZejVXPA1pkuBw2D.png +pathname profile.png +contentType image/png +contentDisposition attachment; filename="profile.png" +------------------ -------------------------------------------------------------------------------------------------- +``` + +Now you can see your file on your blob store: + +```sh +vercel_blob list +Path name Size in bytes url +----------- --------------- ------------------------------------------------------------------- +profile.png 11211 https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/profile-OtpJ1AUIxChAA6UZejVXPA1pkuBw2D.png +``` + + ### Copy operation ```sh -vercel_storage blob copy new/file/path/file.zip +$ vercel_blob copy https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/disk_dump-0eX9NYJnZjO31GsDiwDaQ6TR9QnWkH.bin file.zip + +------------------ ---------------------------------------------------------------- +url https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip +pathname file.zip +contentType application/octet-stream +contentDisposition attachment; filename="file.zip" +------------------ ---------------------------------------------------------------- + ``` You can also print the output information as a json: ```sh -vercel_storage --json copy new/file/path/file.zip +$ vercel_blob --json copy https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/disk_dump-0eX9NYJnZjO31GsDiwDaQ6TR9QnWkH.bin file.zip +{'url': 'https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip', 'pathname': 'file.zip', 'contentType': 'application/octet-stream', 'contentDisposition': 'attachment; filename="file.zip"'} ``` ### Delete operation @@ -71,11 +123,49 @@ vercel_storage --json copy new/file/path/file.zip The delete operation always suceeds, regardless of whether the blob exists or not. It returns a null payload. ```sh -vercel_storage blob delete +$ vercel_blob delete https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/disk_dump-mSjTcLOIg8hlGNiWpWMUcGqVll1uST.bin ``` +### Head operation + +The head operation returns a blob object's metadata. + + +```sh +$ vercel_blob head https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip +------------------ ---------------------------------------------------------------- +url https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip +pathname file.zip +contentType application/octet-stream +contentDisposition attachment; filename="file.zip" +uploadedAt 2023-11-16T23:53:25.000Z +size 1998 +cacheControl public, max-age=31536000, s-maxage=300 +------------------ ---------------------------------------------------------------- +``` + +As with the other commands, you can generate json output: + +```sh +$ vercel_blob --json head https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip +{'url': 'https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip', 'pathname': 'file.zip', 'contentType': 'application/octet-stream', 'contentDisposition': 'attachment; filename="file.zip"', 'uploadedAt': '2023-11-16T23:53:25.000Z', 'size': 1998, 'cacheControl': 'public, max-age=31536000, s-maxage=300'} +``` + + ## Using vercel_storage in your python code +### List operation + +Note: `vercel_storage` will look for the `BLOB_READ_WRITE_TOKEN` environment variable. If it is not available +it will raise an Exception. + +If you have the token stored somewhere else, you can pass it directly to the put() function like this: + + +```python + resp = blob.list(options={'token': 'ABCD123foobar'}) +``` + ### Put operation ```python @@ -89,7 +179,7 @@ with open(my_file, 'rb') as fp: ) ``` -`vercel_storage` will look for the `BLOB_READ_WRITE_TOKEN` environment variable. If it is not available +Note: `vercel_storage` will look for the `BLOB_READ_WRITE_TOKEN` environment variable. If it is not available it will raise an Exception. If you have the token stored somewhere else, you can pass it directly to the put() function like this: diff --git a/pyproject.toml b/pyproject.toml index fbda930..4c9fb6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,13 +19,16 @@ classifiers = [ ] dynamic = ["version", "description"] requires-python = "~=3.9" - +keywords = ["vercel", "api", "storage"] dependencies = [ "requests >=2.6", "click >= 8.1.7", "tabulate >= 0.9.0" ] +[project.urls] +Source = "https://github.com/misaelnieto/vercel-storage" + [project.optional-dependencies] test = [ "pytest", @@ -33,6 +36,9 @@ test = [ "requests-mock[fixture]", "mypy" ] +packaging = [ + "flit" +] [project.scripts] flit = "flit:main" diff --git a/vercel_storage/.devcontainer/devcontainer.json b/vercel_storage/.devcontainer/devcontainer.json index e9fd75e..c6c75ac 100644 --- a/vercel_storage/.devcontainer/devcontainer.json +++ b/vercel_storage/.devcontainer/devcontainer.json @@ -12,7 +12,7 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pip install -e '.[test]'", + "postCreateCommand": "pip install -e '.[test]'" // Configure tool-specific properties. // "customizations": {}, diff --git a/vercel_storage/blob.py b/vercel_storage/blob.py index 77eeb1f..59467fa 100644 --- a/vercel_storage/blob.py +++ b/vercel_storage/blob.py @@ -1,10 +1,10 @@ -from dataclasses import dataclass from os import getenv from typing import Any, Optional, Union from mimetypes import guess_type import urllib.parse import requests +from tabulate import tabulate from . import ConfigurationError, APIResponseError @@ -14,7 +14,7 @@ API_VERSION = "4" DEFAULT_CACHE_AGE = 365 * 24 * 60 * 60 # 1 Year DEFAULT_ACCESS = "public" - +DEFAULT_PAGE_SIZE = 100 def guess_mime_type(url): @@ -22,21 +22,23 @@ def guess_mime_type(url): def get_token(options: dict): - if not options or not isinstance(options, dict): - _tkn = getenv(TOKEN_ENV, None) - else: - _tkn = options.get("token", None) + _tkn = options.get("token", getenv(TOKEN_ENV, None)) if not _tkn: raise ConfigurationError("Vercel's BLOB_READ_WRITE_TOKEN is not set") return _tkn +def dump_headers(options: dict, headers: dict): + if options.get("debug", False): + print(tabulate([(k, v) for k, v in headers.items()])) + + def _coerce_bool(value): return str(int(bool(value))) def _handle_response(response: requests.Response): - if str(response.status_code) == '200': + if str(response.status_code) == "200": return response.json() raise APIResponseError(f"Oops, something went wrong: {response.json()}") @@ -47,12 +49,15 @@ def put(pathname: str, body: bytes, options: Optional[dict] = None) -> dict: "access": "public", "authorization": f"Bearer {get_token(_opts)}", "x-api-version": API_VERSION, - "x-content-type": _opts.get("contentType", "text/plain"), - "x-add-random-suffix": _coerce_bool(_opts.get("addRandomSuffix", True)), + "x-content-type": guess_mime_type(pathname), "x-cache-control-max-age": _opts.get( "cacheControlMaxAge", str(DEFAULT_CACHE_AGE) ), } + if "no_suffix" in options: + headers["x-add-random-suffix"] = "false" + + dump_headers(options, headers) _resp = requests.put(f"{VERCEL_API_URL}/{pathname}", data=body, headers=headers) return _handle_response(_resp) @@ -72,7 +77,7 @@ def delete( in Read-write token Returns: - None: A delete action is always successful if the blob url exists. + None: A delete action is always successful if the blob url exists. A delete action won't throw if the blob url doesn't exists. """ _opts = dict(options) if options else dict() @@ -81,6 +86,7 @@ def delete( "x-api-version": API_VERSION, "content-type": "application/json", } + dump_headers(options, headers) _resp = requests.post( f"{VERCEL_API_URL}/delete", json={ @@ -104,18 +110,34 @@ def list(options: Optional[dict] = None) -> Any: use when making requests. It defaults to the BLOB_READ_WRITE_TOKEN environment variable when deployed on Vercel as explained in Read-write token - limit - prefix - cursor - mode + limit (Not required): A number specifying the maximum number of + blob objects to return. It defaults to 1000 + prefix (Not required): A string used to filter for blob objects + contained in a specific folder assuming that the folder name was + used in the pathname when the blob object was uploaded + cursor (Not required): A string obtained from a previous response for pagination + of retults + mode (Not required): A string specifying the response format. Can + either be "expanded" (default) or "folded". In folded mode + all blobs that are located inside a folder will be folded into + a single folder string entry Returns: - ??? + Json response """ _opts = dict(options) if options else dict() headers = { "authorization": f"Bearer {get_token(_opts)}", + "limit": _opts.get("limit", str(DEFAULT_PAGE_SIZE)), } + if "prefix" in _opts: + headers["prefix"] = _opts["prefix"] + if "cursor" in _opts: + headers["cursor"] = _opts["cursor"] + if "mode" in _opts: + headers["mode"] = _opts["mode"] + + dump_headers(options, headers) _resp = requests.get( f"{VERCEL_API_URL}", headers=headers, @@ -143,11 +165,8 @@ def head(url: str, options: Optional[dict] = None) -> dict: "authorization": f"Bearer {get_token(_opts)}", "x-api-version": API_VERSION, } - _resp = requests.get( - f"{VERCEL_API_URL}", - headers=headers, - params={'url': url} - ) + dump_headers(options, headers) + _resp = requests.get(f"{VERCEL_API_URL}", headers=headers, params={"url": url}) return _handle_response(_resp) @@ -155,12 +174,12 @@ def copy(from_url: str, to_pathname: str, options: Optional[dict] = None) -> dic """ Copies an existing blob object to a new path inside the blob store. - The contentType and cacheControlMaxAge will not be copied from the source - blob. If the values should be carried over to the copy, they need to be + The contentType and cacheControlMaxAge will not be copied from the source + blob. If the values should be carried over to the copy, they need to be defined again in the options object. - Contrary to put(), addRandomSuffix is false by default. This means no - automatic random id suffix is added to your blob url, unless you pass + Contrary to put(), addRandomSuffix is false by default. This means no + automatic random id suffix is added to your blob url, unless you pass addRandomSuffix: True. This also means copy() overwrites files per default, if the operation targets a pathname that already exists. @@ -173,12 +192,12 @@ def copy(from_url: str, to_pathname: str, options: Optional[dict] = None) -> dic use when making requests. It defaults to the BLOB_READ_WRITE_TOKEN environment variable when deployed on Vercel as explained in Read-write token - contentType (Not required): A string indicating the media type. + contentType (Not required): A string indicating the media type. By default, it's extracted from the to_pathname's extension. addRandomSuffix (Not required): A boolean specifying whether to add a random suffix to the pathname. It defaults to False. cacheControlMaxAge (Not required): A number in seconds to configure - the edge and browser cache. Defaults to one year. See Vercel's + the edge and browser cache. Defaults to one year. See Vercel's caching documentation for more details. """ _opts = dict(options) if options else dict() @@ -192,8 +211,9 @@ def copy(from_url: str, to_pathname: str, options: Optional[dict] = None) -> dic "cacheControlMaxAge", str(DEFAULT_CACHE_AGE) ), } + dump_headers(options, headers) _to = urllib.parse.quote(to_pathname) - resp = requests.put(f"{VERCEL_API_URL}/{_to}", headers=headers, params={ - 'fromUrl': from_url - }) - return _handle_response(resp) \ No newline at end of file + resp = requests.put( + f"{VERCEL_API_URL}/{_to}", headers=headers, params={"fromUrl": from_url} + ) + return _handle_response(resp) diff --git a/vercel_storage/cli.py b/vercel_storage/cli.py index 830a0e2..9de7df5 100644 --- a/vercel_storage/cli.py +++ b/vercel_storage/cli.py @@ -13,12 +13,19 @@ help=f"The Vercel's blob read/write token. If not provided it will take it from the {blob.TOKEN_ENV} environment variable", ) @click.option( - "--json", "-j", "print_json", + "--json", + "-j", + "print_json", help="Print the json response instead of a summary table", - is_flag=True + is_flag=True, +) +@click.option( + "--debug", + help="Will print the request headers sent to the API endpoint", + is_flag=True, ) @click.pass_context -def cli(ctx: click.Context, print_json:bool, token: str = None): +def cli(ctx: click.Context, print_json: bool, debug: bool, token: str = None): # ensure that ctx.obj exists and is a dict (in case `cli()` is called # by means other than the `if` block below) ctx.ensure_object(dict) @@ -31,27 +38,83 @@ def cli(ctx: click.Context, print_json:bool, token: str = None): click.echo(click.style(f"Error: {err}", fg="red"), err=True) sys.exit(1) ctx.obj["print_json"] = print_json + ctx.obj["debug"] = debug @cli.command @click.argument("path", type=click.File(mode="rb")) +@click.option( + "--no-suffix", + help="Will not add a random suffix to the pathname of your file", + is_flag=True, +) @click.pass_context -def put(ctx: click.Context, path: click.File): - resp = blob.put(path.name, path.read(), options={"token": ctx.obj["token"]}) - click.echo(f"New file URL: {resp['url']}") +def put(ctx: click.Context, path: click.File, no_suffix: bool): + _opts = { + "token": ctx.obj["token"], + "debug": ctx.obj["debug"], + } + if no_suffix: + _opts["no_suffix"] = True + resp = blob.put(path.name, path.read(), options=_opts) + if ctx.obj["print_json"]: + click.echo(resp) + else: + click.echo(tabulate([(k, v) for k, v in resp.items()])) @cli.command @click.pass_context @click.argument("urls", nargs=-1, required=True) def delete(ctx: click.Context, urls): - blob.delete(urls, options={"token": ctx.obj["token"]}) + blob.delete(urls, options={"token": ctx.obj["token"], "debug": ctx.obj["debug"]}) @cli.command @click.pass_context -def list(ctx: click.Context): - resp = blob.list(options={"token": ctx.obj["token"]}) +@click.option( + "--limit", + help=f"The maximum number of blob objects to return (defaults to 100)", + type=click.INT, + default=100, +) +@click.option( + "--prefix", + help=f"A string used to filter for blob objects contained in a specific folder assuming that the folder name was used in the pathname when the blob object was uploaded", + type=click.STRING, +) +@click.option( + "--cursor", + help=f"A string obtained from a previous response for pagination of results", + type=click.STRING, +) +@click.option( + "--mode", + help=f"Will change the response format. " + "In folded mode all blobs that are located inside a folder will be folded " + "into a single folder string entry (defaults to expanded)", + type=click.Choice(["expanded", "folded"]), + default="expanded", +) +def list(ctx: click.Context, limit: int, prefix: str, cursor: str, mode: str): + _opts = { + "token": ctx.obj["token"], + "debug": ctx.obj["debug"], + "limit": str(limit), + "mode": mode, + } + + if prefix: + _opts["prefix"] = prefix + + if cursor: + _opts["cursor"] = cursor + + if mode: + _opts["mode"] = mode + + resp = blob.list(options=_opts) + if "blobs" in resp: if ctx.obj["print_json"]: click.echo(resp) @@ -61,30 +124,35 @@ def list(ctx: click.Context): ] click.echo(tabulate(table, headers=["Path name", "Size in bytes", "url"])) - # click.echo(resp) - @cli.command @click.pass_context @click.argument("url", required=True, type=click.STRING) -def head(ctx: click.Context, url:str): - resp = blob.head(url, options={"token": ctx.obj["token"]}) +def head(ctx: click.Context, url: str): + resp = blob.head( + url, options={"token": ctx.obj["token"], "debug": ctx.obj["debug"]} + ) if ctx.obj["print_json"]: click.echo(resp) else: - click.echo(tabulate([(k,v) for k,v in resp.items()])) + click.echo(tabulate([(k, v) for k, v in resp.items()])) @cli.command @click.pass_context @click.argument("from_url", required=True, type=click.STRING) @click.argument("to_pathname", required=True, type=click.STRING) -def copy(ctx: click.Context, from_url:str, to_pathname:str): - resp = blob.copy(from_url, to_pathname, options={"token": ctx.obj["token"]}) +def copy(ctx: click.Context, from_url: str, to_pathname: str): + resp = blob.copy( + from_url, + to_pathname, + options={"token": ctx.obj["token"], "debug": ctx.obj["debug"]}, + ) if ctx.obj["print_json"]: click.echo(resp) else: - click.echo(tabulate([(k,v) for k,v in resp.items()])) + click.echo(tabulate([(k, v) for k, v in resp.items()])) + if __name__ == "__main__": cli()