From 84c14f2a7a8b9f85b9d5563a854dccf3e8957165 Mon Sep 17 00:00:00 2001 From: Noe Nieto Date: Thu, 16 Nov 2023 00:03:12 +0000 Subject: [PATCH 1/7] remove extra comma --- vercel_storage/.devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vercel_storage/.devcontainer/devcontainer.json b/vercel_storage/.devcontainer/devcontainer.json index e9fd75e..c6c75ac 100644 --- a/vercel_storage/.devcontainer/devcontainer.json +++ b/vercel_storage/.devcontainer/devcontainer.json @@ -12,7 +12,7 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pip install -e '.[test]'", + "postCreateCommand": "pip install -e '.[test]'" // Configure tool-specific properties. // "customizations": {}, From e4f3e40dc147ea62ea2581f9bcb3c7a6aa3b59cc Mon Sep 17 00:00:00 2001 From: Noe Nieto Date: Thu, 16 Nov 2023 02:46:30 +0000 Subject: [PATCH 2/7] Pending changes exported from your codespace --- pyproject.toml | 3 ++- vercel_storage/blob.py | 26 ++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fbda930..01c2c9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,8 @@ requires-python = "~=3.9" dependencies = [ "requests >=2.6", "click >= 8.1.7", - "tabulate >= 0.9.0" + "tabulate >= 0.9.0", + "dataclasses-json >= 0.6.2" ] [project.optional-dependencies] diff --git a/vercel_storage/blob.py b/vercel_storage/blob.py index 77eeb1f..e791818 100644 --- a/vercel_storage/blob.py +++ b/vercel_storage/blob.py @@ -1,4 +1,3 @@ -from dataclasses import dataclass from os import getenv from typing import Any, Optional, Union from mimetypes import guess_type @@ -14,7 +13,7 @@ API_VERSION = "4" DEFAULT_CACHE_AGE = 365 * 24 * 60 * 60 # 1 Year DEFAULT_ACCESS = "public" - +DEFAULT_PAGE_SIZE = 100 def guess_mime_type(url): @@ -104,10 +103,17 @@ def list(options: Optional[dict] = None) -> Any: use when making requests. It defaults to the BLOB_READ_WRITE_TOKEN environment variable when deployed on Vercel as explained in Read-write token - limit - prefix - cursor - mode + limit (Not required): A number specifying the maximum number of + blob objects to return. It defaults to 1000 + prefix (Not required): A string used to filter for blob objects + contained in a specific folder assuming that the folder name was + used in the pathname when the blob object was uploaded + cursor (Not required): A string obtained from a previous response for pagination + of retults + mode (Not required): A string specifying the response format. Can + either be "expanded" (default) or "folded". In folded mode + all blobs that are located inside a folder will be folded into + a single folder string entry Returns: ??? @@ -115,7 +121,15 @@ def list(options: Optional[dict] = None) -> Any: _opts = dict(options) if options else dict() headers = { "authorization": f"Bearer {get_token(_opts)}", + "limit": _opts.get('limit', DEFAULT_PAGE_SIZE), } + if 'prefix' in _opts: + headers['prefix'] = _opts['prefix'] + if 'cursor' in _opts: + headers['cursor'] = _opts['cursor'] + if 'mode' in _opts: + headers['mode'] = _opts['mode'] + _resp = requests.get( f"{VERCEL_API_URL}", headers=headers, From de63c7b99e2283a241d569fb07d26a95c14ad64c Mon Sep 17 00:00:00 2001 From: Noe Nieto Date: Wed, 15 Nov 2023 18:49:39 -0800 Subject: [PATCH 3/7] Update pyproject.toml --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 01c2c9a..fbda930 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,8 +23,7 @@ requires-python = "~=3.9" dependencies = [ "requests >=2.6", "click >= 8.1.7", - "tabulate >= 0.9.0", - "dataclasses-json >= 0.6.2" + "tabulate >= 0.9.0" ] [project.optional-dependencies] From 42bd11c285477568f049ec64dc081af834294825 Mon Sep 17 00:00:00 2001 From: Noe Nieto Date: Thu, 16 Nov 2023 00:03:12 +0000 Subject: [PATCH 4/7] remove extra comma --- vercel_storage/.devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vercel_storage/.devcontainer/devcontainer.json b/vercel_storage/.devcontainer/devcontainer.json index e9fd75e..c6c75ac 100644 --- a/vercel_storage/.devcontainer/devcontainer.json +++ b/vercel_storage/.devcontainer/devcontainer.json @@ -12,7 +12,7 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pip install -e '.[test]'", + "postCreateCommand": "pip install -e '.[test]'" // Configure tool-specific properties. // "customizations": {}, From 176c43227ee09fa4919aadb544074efde9ae5c36 Mon Sep 17 00:00:00 2001 From: Noe Nieto Date: Thu, 16 Nov 2023 02:46:30 +0000 Subject: [PATCH 5/7] Pending changes exported from your codespace --- pyproject.toml | 3 ++- vercel_storage/blob.py | 26 ++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fbda930..01c2c9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,8 @@ requires-python = "~=3.9" dependencies = [ "requests >=2.6", "click >= 8.1.7", - "tabulate >= 0.9.0" + "tabulate >= 0.9.0", + "dataclasses-json >= 0.6.2" ] [project.optional-dependencies] diff --git a/vercel_storage/blob.py b/vercel_storage/blob.py index 77eeb1f..e791818 100644 --- a/vercel_storage/blob.py +++ b/vercel_storage/blob.py @@ -1,4 +1,3 @@ -from dataclasses import dataclass from os import getenv from typing import Any, Optional, Union from mimetypes import guess_type @@ -14,7 +13,7 @@ API_VERSION = "4" DEFAULT_CACHE_AGE = 365 * 24 * 60 * 60 # 1 Year DEFAULT_ACCESS = "public" - +DEFAULT_PAGE_SIZE = 100 def guess_mime_type(url): @@ -104,10 +103,17 @@ def list(options: Optional[dict] = None) -> Any: use when making requests. It defaults to the BLOB_READ_WRITE_TOKEN environment variable when deployed on Vercel as explained in Read-write token - limit - prefix - cursor - mode + limit (Not required): A number specifying the maximum number of + blob objects to return. It defaults to 1000 + prefix (Not required): A string used to filter for blob objects + contained in a specific folder assuming that the folder name was + used in the pathname when the blob object was uploaded + cursor (Not required): A string obtained from a previous response for pagination + of retults + mode (Not required): A string specifying the response format. Can + either be "expanded" (default) or "folded". In folded mode + all blobs that are located inside a folder will be folded into + a single folder string entry Returns: ??? @@ -115,7 +121,15 @@ def list(options: Optional[dict] = None) -> Any: _opts = dict(options) if options else dict() headers = { "authorization": f"Bearer {get_token(_opts)}", + "limit": _opts.get('limit', DEFAULT_PAGE_SIZE), } + if 'prefix' in _opts: + headers['prefix'] = _opts['prefix'] + if 'cursor' in _opts: + headers['cursor'] = _opts['cursor'] + if 'mode' in _opts: + headers['mode'] = _opts['mode'] + _resp = requests.get( f"{VERCEL_API_URL}", headers=headers, From aee005ee1d7dd40c0632abcf7233cd63fe1c0dbb Mon Sep 17 00:00:00 2001 From: Noe Nieto Date: Wed, 15 Nov 2023 18:49:39 -0800 Subject: [PATCH 6/7] Update pyproject.toml --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 01c2c9a..fbda930 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,8 +23,7 @@ requires-python = "~=3.9" dependencies = [ "requests >=2.6", "click >= 8.1.7", - "tabulate >= 0.9.0", - "dataclasses-json >= 0.6.2" + "tabulate >= 0.9.0" ] [project.optional-dependencies] From 2de8af5657780b448580d6c07d36565cdcdc3291 Mon Sep 17 00:00:00 2001 From: Noe Nieto Date: Fri, 17 Nov 2023 00:49:21 +0000 Subject: [PATCH 7/7] Release 0.0.1 --- Pipfile | 11 +++++ Pipfile.lock | 20 ++++++++ README.md | 102 ++++++++++++++++++++++++++++++++++++++--- pyproject.toml | 8 +++- vercel_storage/blob.py | 25 +++++++--- vercel_storage/cli.py | 81 +++++++++++++++++++++++++++----- 6 files changed, 221 insertions(+), 26 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..d582066 --- /dev/null +++ b/Pipfile @@ -0,0 +1,11 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..e327ba6 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,20 @@ +{ + "_meta": { + "hash": { + "sha256": "b20aaa9718024ec28b53570df5a504bc1765bb9490e31099da47c34bd616c357" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": {}, + "develop": {} +} diff --git a/README.md b/README.md index 6195ec1..23f9abb 100644 --- a/README.md +++ b/README.md @@ -45,25 +45,77 @@ export BLOB_READ_WRITE_TOKEN="vercel_blob_rw_ABC1234XYz" ### Put operation ```sh -vercel_storage blob upload path/to/file.zip +$ vercel_blob put disk_dump.bin +------------------ ---------------------------------------------------------------------------------------------------- +url https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/disk_dump-OTgBsduT0QQcfXImNMZky1NSy3HfML.bin +pathname disk_dump.bin +contentType application/octet-stream +contentDisposition attachment; filename="disk_dump.bin" +------------------ ---------------------------------------------------------------------------------------------------- ``` You can also print the output information as a json: ```sh -vercel_storage --json blob upload path/to/file.zip +$ vercel_blob --json put disk_dump.bin +{'url': 'https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/disk_dump-0eX9NYJnZjO31GsDiwDaQ6TR9QnWkH.bin', 'pathname': 'disk_dump.bin', 'contentType': 'text/plain', 'contentDisposition': 'attachment; filename="disk_dump.bin"'} ``` +By default, Vercel's blob store will insert a randomly generated string to the name of your file. But you can turn off that feature: + +```sh +$ vercel_blob put --no-suffix chart.png +------------------ ----------------------------------------------------------------- +url https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/chart.png +pathname chart.png +contentType image/png +contentDisposition attachment; filename="chart.png" +------------------ ----------------------------------------------------------------- +``` + +### List operation + +The list method returns a list of blob objects in a Blob store. For example, let's upload a file to the bob store: + +```sh +vercel_blob put profile.png +------------------ -------------------------------------------------------------------------------------------------- +url https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/profile-OtpJ1AUIxChAA6UZejVXPA1pkuBw2D.png +pathname profile.png +contentType image/png +contentDisposition attachment; filename="profile.png" +------------------ -------------------------------------------------------------------------------------------------- +``` + +Now you can see your file on your blob store: + +```sh +vercel_blob list +Path name Size in bytes url +----------- --------------- ------------------------------------------------------------------- +profile.png 11211 https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/profile-OtpJ1AUIxChAA6UZejVXPA1pkuBw2D.png +``` + + ### Copy operation ```sh -vercel_storage blob copy new/file/path/file.zip +$ vercel_blob copy https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/disk_dump-0eX9NYJnZjO31GsDiwDaQ6TR9QnWkH.bin file.zip + +------------------ ---------------------------------------------------------------- +url https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip +pathname file.zip +contentType application/octet-stream +contentDisposition attachment; filename="file.zip" +------------------ ---------------------------------------------------------------- + ``` You can also print the output information as a json: ```sh -vercel_storage --json copy new/file/path/file.zip +$ vercel_blob --json copy https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/disk_dump-0eX9NYJnZjO31GsDiwDaQ6TR9QnWkH.bin file.zip +{'url': 'https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip', 'pathname': 'file.zip', 'contentType': 'application/octet-stream', 'contentDisposition': 'attachment; filename="file.zip"'} ``` ### Delete operation @@ -71,11 +123,49 @@ vercel_storage --json copy new/file/path/file.zip The delete operation always suceeds, regardless of whether the blob exists or not. It returns a null payload. ```sh -vercel_storage blob delete +$ vercel_blob delete https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/disk_dump-mSjTcLOIg8hlGNiWpWMUcGqVll1uST.bin ``` +### Head operation + +The head operation returns a blob object's metadata. + + +```sh +$ vercel_blob head https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip +------------------ ---------------------------------------------------------------- +url https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip +pathname file.zip +contentType application/octet-stream +contentDisposition attachment; filename="file.zip" +uploadedAt 2023-11-16T23:53:25.000Z +size 1998 +cacheControl public, max-age=31536000, s-maxage=300 +------------------ ---------------------------------------------------------------- +``` + +As with the other commands, you can generate json output: + +```sh +$ vercel_blob --json head https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip +{'url': 'https://c6zu0uktwgrh0d3g.public.blob.vercel-storage.com/file.zip', 'pathname': 'file.zip', 'contentType': 'application/octet-stream', 'contentDisposition': 'attachment; filename="file.zip"', 'uploadedAt': '2023-11-16T23:53:25.000Z', 'size': 1998, 'cacheControl': 'public, max-age=31536000, s-maxage=300'} +``` + + ## Using vercel_storage in your python code +### List operation + +Note: `vercel_storage` will look for the `BLOB_READ_WRITE_TOKEN` environment variable. If it is not available +it will raise an Exception. + +If you have the token stored somewhere else, you can pass it directly to the put() function like this: + + +```python + resp = blob.list(options={'token': 'ABCD123foobar'}) +``` + ### Put operation ```python @@ -89,7 +179,7 @@ with open(my_file, 'rb') as fp: ) ``` -`vercel_storage` will look for the `BLOB_READ_WRITE_TOKEN` environment variable. If it is not available +Note: `vercel_storage` will look for the `BLOB_READ_WRITE_TOKEN` environment variable. If it is not available it will raise an Exception. If you have the token stored somewhere else, you can pass it directly to the put() function like this: diff --git a/pyproject.toml b/pyproject.toml index fbda930..4c9fb6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,13 +19,16 @@ classifiers = [ ] dynamic = ["version", "description"] requires-python = "~=3.9" - +keywords = ["vercel", "api", "storage"] dependencies = [ "requests >=2.6", "click >= 8.1.7", "tabulate >= 0.9.0" ] +[project.urls] +Source = "https://github.com/misaelnieto/vercel-storage" + [project.optional-dependencies] test = [ "pytest", @@ -33,6 +36,9 @@ test = [ "requests-mock[fixture]", "mypy" ] +packaging = [ + "flit" +] [project.scripts] flit = "flit:main" diff --git a/vercel_storage/blob.py b/vercel_storage/blob.py index e791818..a48e0b0 100644 --- a/vercel_storage/blob.py +++ b/vercel_storage/blob.py @@ -4,6 +4,7 @@ import urllib.parse import requests +from tabulate import tabulate from . import ConfigurationError, APIResponseError @@ -21,15 +22,17 @@ def guess_mime_type(url): def get_token(options: dict): - if not options or not isinstance(options, dict): - _tkn = getenv(TOKEN_ENV, None) - else: - _tkn = options.get("token", None) + _tkn = options.get("token", getenv(TOKEN_ENV, None)) if not _tkn: raise ConfigurationError("Vercel's BLOB_READ_WRITE_TOKEN is not set") return _tkn +def dump_headers(options: dict, headers: dict): + if options.get("debug", False): + print(tabulate([(k,v) for k,v in headers.items()])) + + def _coerce_bool(value): return str(int(bool(value))) @@ -46,12 +49,15 @@ def put(pathname: str, body: bytes, options: Optional[dict] = None) -> dict: "access": "public", "authorization": f"Bearer {get_token(_opts)}", "x-api-version": API_VERSION, - "x-content-type": _opts.get("contentType", "text/plain"), - "x-add-random-suffix": _coerce_bool(_opts.get("addRandomSuffix", True)), + "x-content-type": guess_mime_type(pathname), "x-cache-control-max-age": _opts.get( "cacheControlMaxAge", str(DEFAULT_CACHE_AGE) ), } + if "no_suffix" in options: + headers["x-add-random-suffix"] = "false" + + dump_headers(options, headers) _resp = requests.put(f"{VERCEL_API_URL}/{pathname}", data=body, headers=headers) return _handle_response(_resp) @@ -80,6 +86,7 @@ def delete( "x-api-version": API_VERSION, "content-type": "application/json", } + dump_headers(options, headers) _resp = requests.post( f"{VERCEL_API_URL}/delete", json={ @@ -130,6 +137,7 @@ def list(options: Optional[dict] = None) -> Any: if 'mode' in _opts: headers['mode'] = _opts['mode'] + dump_headers(options, headers) _resp = requests.get( f"{VERCEL_API_URL}", headers=headers, @@ -157,6 +165,7 @@ def head(url: str, options: Optional[dict] = None) -> dict: "authorization": f"Bearer {get_token(_opts)}", "x-api-version": API_VERSION, } + dump_headers(options, headers) _resp = requests.get( f"{VERCEL_API_URL}", headers=headers, @@ -206,8 +215,10 @@ def copy(from_url: str, to_pathname: str, options: Optional[dict] = None) -> dic "cacheControlMaxAge", str(DEFAULT_CACHE_AGE) ), } + dump_headers(options, headers) _to = urllib.parse.quote(to_pathname) resp = requests.put(f"{VERCEL_API_URL}/{_to}", headers=headers, params={ 'fromUrl': from_url }) - return _handle_response(resp) \ No newline at end of file + return _handle_response(resp) + diff --git a/vercel_storage/cli.py b/vercel_storage/cli.py index 830a0e2..951c1e4 100644 --- a/vercel_storage/cli.py +++ b/vercel_storage/cli.py @@ -17,12 +17,18 @@ help="Print the json response instead of a summary table", is_flag=True ) +@click.option( + "--debug", + help="Will print the request headers sent to the API endpoint", + is_flag=True +) @click.pass_context -def cli(ctx: click.Context, print_json:bool, token: str = None): +def cli(ctx: click.Context, print_json:bool, debug: bool, token: str = None): # ensure that ctx.obj exists and is a dict (in case `cli()` is called # by means other than the `if` block below) ctx.ensure_object(dict) - _opts = {} + _opts = { + } if token: _opts["token"] = token try: @@ -31,27 +37,79 @@ def cli(ctx: click.Context, print_json:bool, token: str = None): click.echo(click.style(f"Error: {err}", fg="red"), err=True) sys.exit(1) ctx.obj["print_json"] = print_json + ctx.obj["debug"] = debug @cli.command @click.argument("path", type=click.File(mode="rb")) +@click.option( + "--no-suffix", + help="Will not add a random suffix to the pathname of your file", + is_flag=True +) @click.pass_context -def put(ctx: click.Context, path: click.File): - resp = blob.put(path.name, path.read(), options={"token": ctx.obj["token"]}) - click.echo(f"New file URL: {resp['url']}") +def put(ctx: click.Context, path: click.File, no_suffix:bool): + _opts = { + "token": ctx.obj["token"], + "debug":ctx.obj["debug"], + } + if no_suffix: + _opts["no_suffix"] = True + resp = blob.put(path.name, path.read(), options=_opts) + if ctx.obj["print_json"]: + click.echo(resp) + else: + click.echo(tabulate([(k,v) for k,v in resp.items()])) @cli.command @click.pass_context @click.argument("urls", nargs=-1, required=True) def delete(ctx: click.Context, urls): - blob.delete(urls, options={"token": ctx.obj["token"]}) + blob.delete(urls, options={"token": ctx.obj["token"], "debug":ctx.obj["debug"]}) @cli.command @click.pass_context -def list(ctx: click.Context): - resp = blob.list(options={"token": ctx.obj["token"]}) +@click.option( + "--limit", + help=f"The maximum number of blob objects to return (defaults to 100)", type=click.INT, default=100 +) +@click.option( + "--prefix", + help=f"A string used to filter for blob objects contained in a specific folder assuming that the folder name was used in the pathname when the blob object was uploaded", type=click.STRING +) +@click.option( + "--cursor", + help=f"A string obtained from a previous response for pagination of results", type=click.STRING +) +@click.option( + "--mode", + help=f'Will change the response format. ' + 'In folded mode all blobs that are located inside a folder will be folded ' + 'into a single folder string entry (defaults to expanded)', + type=click.Choice(["expanded", "folded"]), + default="expanded" +) +def list(ctx: click.Context, limit:int, prefix:str, cursor:str, mode:str): + _opts ={ + "token": ctx.obj["token"], + "debug": ctx.obj["debug"], + "limit": str(limit), + "mode": mode + } + + if prefix: + _opts["prefix"] = prefix + + if cursor: + _opts["cursor"] = cursor + + if mode: + _opts["mode"] = mode + + resp = blob.list(options=_opts) + if "blobs" in resp: if ctx.obj["print_json"]: click.echo(resp) @@ -61,14 +119,12 @@ def list(ctx: click.Context): ] click.echo(tabulate(table, headers=["Path name", "Size in bytes", "url"])) - # click.echo(resp) - @cli.command @click.pass_context @click.argument("url", required=True, type=click.STRING) def head(ctx: click.Context, url:str): - resp = blob.head(url, options={"token": ctx.obj["token"]}) + resp = blob.head(url, options={"token": ctx.obj["token"], "debug":ctx.obj["debug"]}) if ctx.obj["print_json"]: click.echo(resp) else: @@ -80,7 +136,7 @@ def head(ctx: click.Context, url:str): @click.argument("from_url", required=True, type=click.STRING) @click.argument("to_pathname", required=True, type=click.STRING) def copy(ctx: click.Context, from_url:str, to_pathname:str): - resp = blob.copy(from_url, to_pathname, options={"token": ctx.obj["token"]}) + resp = blob.copy(from_url, to_pathname, options={"token": ctx.obj["token"], "debug":ctx.obj["debug"]}) if ctx.obj["print_json"]: click.echo(resp) else: @@ -88,3 +144,4 @@ def copy(ctx: click.Context, from_url:str, to_pathname:str): if __name__ == "__main__": cli() +