From ff568d4fe3ba027a282bc7f81d4b72e31219dca9 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sat, 21 Sep 2019 18:42:46 +0200 Subject: [PATCH] fix #10 add webhooks for instant version updates --- http-update.md | 24 +++++++++++ source/dubregistry/api.d | 70 +++++++++++++++++++++++++++++++ source/dubregistry/dbcontroller.d | 27 ++++++++++++ source/dubregistry/registry.d | 38 ++++++++++++++++- source/dubregistry/web.d | 22 +++++++++- views/my_packages.package.dt | 38 +++++++++++++++-- 6 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 http-update.md diff --git a/http-update.md b/http-update.md new file mode 100644 index 00000000..4658b621 --- /dev/null +++ b/http-update.md @@ -0,0 +1,24 @@ +# Triggering version updates over HTTP + +To queue an update of your package you can use the `POST /api/packages/:packageName/update` endpoint. + +## `POST /api/packages/:packageName/update` + +Queues an update for the specified package. + +Query params: +`secret`: string (optional) provide the secret as query +`header`: string (optional) provide which header is used to check the secret (must start with X-) + +Body params: (application/x-www-form-urlencoded, multipart/form-data or application/json) +`secret`: string (optional) provide the secret as body param + +## `POST /api/packages/:packageName/update/github` + +Queues an update for the specified package. Compatible with GitHub webhooks and only triggers on `create` events. Must pass secret as query param and not in GitHub webhook settings. + +## `POST /api/packages/:packageName/update/gitlab` + +Queues an update for the specified package. Compatible with GitLab webhooks and only triggers on `tag_push` events. The secret is specified in the GitLab control panel. + +Calls `POST /api/packages/:packageName/update` with `header=X-Gitlab-Token` query param after hook parsing. diff --git a/source/dubregistry/api.d b/source/dubregistry/api.d index 116d6e2d..e90ee51f 100644 --- a/source/dubregistry/api.d +++ b/source/dubregistry/api.d @@ -96,6 +96,18 @@ interface IPackages { Json getInfo(string _name, string _version, bool minimize = false); Json[string] getInfos(string[] packages, bool include_dependencies = false, bool minimize = false); + + @path(":name/update") + string postUpdate(string _name, string secret = ""); + + @path(":name/update/github") + @headerParam("event", "X-GitHub-Event") + @queryParam("secret", "secret") + string postUpdateGithub(string _name, string secret, string event, Json hook = Json.init); + + @path(":name/update/gitlab") + @headerParam("secret", "X-Gitlab-Token") + string postUpdateGitlab(string _name, string secret, string object_kind = ""); } class LocalDubRegistryAPI : DubRegistryAPI { @@ -176,6 +188,52 @@ override { .check!(r => r !is null)(HTTPStatus.notFound, "None of the packages were found") .byKeyValue.map!(p => tuple(p.key, p.value.info)).assocArray; } + + @before!updateSecretReader("secret") + string postUpdate(string _name, string secret = "") + { + if (!secret.length) + return "No secret sent"; + + string expected = m_registry.getPackageSecret(_name); + if (!expected.length || secret != expected) + return "Secret doesn't match"; + + m_registry.triggerPackageUpdate(_name); + return "Queued package update"; + } + + string postUpdateGithub(string _name, string secret, string event, Json hook = Json.init) + { + if (event == "create") { + return postUpdate(_name, secret); + } else if (event == "ping") { + enforceBadRequest(hook.type == Json.Type.object, "hook is not of type json"); + auto eventsObj = *enforceBadRequest("events" in hook, "no events object sent in hook object"); + enforceBadRequest(eventsObj.type == Json.Type.array, "Hook events must be of type array"); + auto events = eventsObj.get!(Json[]); + + foreach (ev; events) + if (ev.type == Json.Type.string && ev.get!string == "create") + return "valid"; + + string expected = m_registry.getPackageSecret(_name); + if (expected.length && secret.length && secret == expected) + m_registry.addPackageError(_name, + "GitHub hook configuration is invalid. Hook is missing 'create' event. (Tags or branches)"); + + return "invalid hook - create event missing"; + } else + return "ignored event " ~ event; + } + + string postUpdateGitlab(string _name, string secret, string object_kind) + { + if (object_kind != "tag_push") + return "ignored event " ~ object_kind; + + return postUpdate(_name, secret); + } } private: @@ -187,6 +245,18 @@ private: } } +private string updateSecretReader(scope HTTPServerRequest req, scope HTTPServerResponse res) +{ + string header = req.query.get("header", ""); + if (header.length) + return req.headers.get(header); + + string ret = req.contentType == "application/json" ? req.json["secret"].get!string : req.form.get("secret", ""); + if (ret.length) + return ret; + + return req.query.get("secret", ""); +} private auto ref T check(alias cond, T)(auto ref T t, HTTPStatus status, string msg) { diff --git a/source/dubregistry/dbcontroller.d b/source/dubregistry/dbcontroller.d index 3320bbad..0db3d2e0 100644 --- a/source/dubregistry/dbcontroller.d +++ b/source/dubregistry/dbcontroller.d @@ -196,6 +196,16 @@ class DbController { m_packages.update(["name": packname], ["$set": ["errors": error]]); } + void addPackageError(string packname, string error) + { + m_packages.update(["name": packname], ["$push": ["errors": error]]); + } + + void clearPackageErrors(string packname) + { + m_packages.update(["name": packname], ["$set": ["errors": Bson.emptyArray]]); + } + void setPackageCategories(string packname, string[] categories...) { m_packages.update(["name": packname], ["$set": ["categories": categories]]); @@ -251,6 +261,21 @@ class DbController { return data.get.data.rawData; } + string getPackageSecret(string packname) + { + auto ret = m_packages.findOne(["name": packname]).tryIndex("secret"); + if (ret.isNull) return null; + else return ret.get.get!string; + } + + void setPackageSecret(string packname, string secret) + { + if (secret.length) + m_packages.update(["name": packname], ["$set": ["secret": secret]]); + else + m_packages.update(["name": packname], ["$unset": ["secret": 0]]); + } + void addVersion(string packname, DbPackageVersion ver) { assert(ver.version_.startsWith("~") || ver.version_.isValidVersion()); @@ -496,6 +521,8 @@ struct DbPackage { @optional BsonObjectID logo; // reference to m_files @optional string documentationURL; @optional float textScore = 0; // for FTS textScore in searchPackages + /// Random secret string used for package management API (webhooks) + @optional string secret; } struct DbRepository { diff --git a/source/dubregistry/registry.d b/source/dubregistry/registry.d index ee769f31..15278a04 100644 --- a/source/dubregistry/registry.d +++ b/source/dubregistry/registry.d @@ -239,7 +239,7 @@ class DubRegistry { .map!(pack => getPackageInfo(pack, flags)); } - auto getPackageInfo(DbPackage pack, PackageInfoFlags flags = PackageInfoFlags.none) + PackageInfo getPackageInfo(DbPackage pack, PackageInfoFlags flags = PackageInfoFlags.none) { auto rep = getRepository(pack.repository); @@ -261,6 +261,8 @@ class DubRegistry { } if (flags & PackageInfoFlags.includeErrors) nfo["errors"] = serializeToJson(pack.errors); + if (flags & PackageInfoFlags.includeAdmin) + ret.secret = pack.secret; ret.info = nfo; @@ -429,6 +431,38 @@ class DubRegistry { return m_db.getPackageLogo(pack_name, rev); } + void addPackageError(string pack_name, string error) + { + m_db.addPackageError(pack_name, error); + } + + void clearPackageErrors(string pack_name) + { + m_db.clearPackageErrors(pack_name); + } + + void unsetPackageSecret(string pack_name) + { + m_db.setPackageSecret(pack_name, null); + } + + string getPackageSecret(string pack_name) + { + return m_db.getPackageSecret(pack_name); + } + + void regenPackageSecret(string pack_name) + { + import std.ascii : lowerHexDigits; + import std.random : uniform; + + char[24] token; + foreach (ref c; token) + c = lowerHexDigits[uniform(0, $)]; + + m_db.setPackageSecret(pack_name, token[].idup); + } + void updatePackages() { logDiagnostic("Triggering package update..."); @@ -733,6 +767,7 @@ struct PackageVersionInfo { struct PackageInfo { PackageVersionInfo[] versions; BsonObjectID logo; + string secret; Json info; /// JSON package information, as reported to the client } @@ -743,6 +778,7 @@ enum PackageInfoFlags includeDependencies = 1 << 0, /// include package info of dependencies includeErrors = 1 << 1, /// include package errors minimize = 1 << 2, /// return only minimal information (for dependency resolver) + includeAdmin = 1 << 3, /// include package admin information such as secrets } /// Computes a package score from given package stats and global distributions of those stats. diff --git a/source/dubregistry/web.d b/source/dubregistry/web.d index 77a3d9c7..80dc0360 100644 --- a/source/dubregistry/web.d +++ b/source/dubregistry/web.d @@ -649,13 +649,13 @@ class DubRegistryFullWebFrontend : DubRegistryWebFrontend { { enforceUserPackage(_user, _packname); auto packageName = _packname; - auto nfo = m_registry.getPackageInfo(packageName, PackageInfoFlags.includeErrors); + auto nfo = m_registry.getPackageInfo(packageName, PackageInfoFlags.includeErrors | PackageInfoFlags.includeAdmin); if (nfo.info.type == Json.Type.null_) return; auto categories = m_categories; auto registry = m_registry; auto user = _user; auto error = _error; - render!("my_packages.package.dt", packageName, categories, user, registry, error); + render!("my_packages.package.dt", packageName, categories, user, registry, error, nfo); } @auth @path("/my_packages/:packname/update") @@ -754,6 +754,24 @@ class DubRegistryFullWebFrontend : DubRegistryWebFrontend { redirect("/my_packages/"~_packname); } + @auth @path("/my_packages/:packname/regen_secret") + void postPackageRegenSecret(string _packname, User _user) + { + enforceUserPackage(_user, _packname); + m_registry.regenPackageSecret(_packname); + + redirect("/my_packages/"~_packname~"#repository"); + } + + @auth @path("/my_packages/:packname/unset_secret") + void postPackageUnsetSecret(string _packname, User _user) + { + enforceUserPackage(_user, _packname); + m_registry.unsetPackageSecret(_packname); + + redirect("/my_packages/"~_packname~"#repository"); + } + @path("/docs/commandline") void getCommandLineDocs() { diff --git a/views/my_packages.package.dt b/views/my_packages.package.dt index 8e5363c3..44999d17 100644 --- a/views/my_packages.package.dt +++ b/views/my_packages.package.dt @@ -8,8 +8,7 @@ block body - import dubregistry.registry : PackageInfoFlags; - import dubregistry.web : Category; - - auto pack = registry.getPackageInfo(packageName, PackageInfoFlags.includeErrors).info; - + - auto pack = nfo.info; h1 Edit package #[a.blind(href="#{req.rootDir}packages/#{packageName}")= packageName] - auto latest = pack["versions"].length ? pack["versions"][pack["versions"].length-1] : Json(null); - if (latest.type == Json.Type.Object) @@ -82,9 +81,10 @@ block body p Package is scheduled for an automatic update check. Still have to wait for one more package. - else if (update_check_index > 1) p Package is scheduled for an automatic update check. Still have to wait for #{update_check_index} other packages to complete. - form.inputForm(method="POST", action="#{req.rootDir}my_packages/#{packageName}/update") - p + .updates + form.inputForm(method="POST", action="#{req.rootDir}my_packages/#{packageName}/update") button(type="submit") Trigger manual update + a(href="#repository"): button Setup update hook section.pkgconfig#tab-categories a#categories(name="categories") @@ -116,6 +116,36 @@ block body a#repository(name="repository") h2 Repository + h3 Update Hooks + p You can POST to a special endpoint on the dub registry to trigger a manual package update. + p Setup these hooks in GitHub, GitLab or some other service you use to trigger when you push a new tag to make them available on the registry as quickly as possible. + - auto secret = nfo.secret; + .inputForm + - if (secret.length) + - auto apiUrl = req.fullURL; + - apiUrl.path = typeof(apiUrl.path).init; + p + label(for="generic") Generic update webhook: (POST) + input#generic(type="text", readonly, value="#{apiUrl}/api/packages/#{packageName}/update") + p + label(for="generic") GitHub webhook: + input#generic(type="text", readonly, value="#{apiUrl}/api/packages/#{packageName}/update/github?secret=#{secret}") + p + label(for="generic") GitLab webhook: + input#generic(type="text", readonly, value="#{apiUrl}/api/packages/#{packageName}/update/gitlab") + hr + p + label(for="generic") Secret: + input#generic(type="text", readonly, value=secret) + hr + form(method="POST", action="#{req.rootDir}my_packages/#{packageName}/unset_secret", onsubmit="confirm('Are you sure you want to revoke the secret and disable webhook support? Existing webhooks will not work anymore.')") + button(type="submit") Revoke Webhooks + form(method="POST", action="#{req.rootDir}my_packages/#{packageName}/regen_secret", onsubmit="confirm('Are you sure you want to regenerate the secret? Existing webhooks will not work anymore.')") + button(type="submit") Invalidate Secret + - else + form(method="POST", action="#{req.rootDir}my_packages/#{packageName}/regen_secret") + button(type="submit") Enable Webhooks + h3 Transfer Repository .inputForm form(method="POST", action="#{req.rootDir}my_packages/#{packageName}/set_repository")