Merge pull request #278 from MartinNowak/pkg_dependencies

add API to get info for package and dependencies

dub.sdl

@@ -7,7 +7,7 @@ authors "Sönke Ludwig" "Martin Nowak" "Anton Fediushin aka ohdatboi" \
 license "BSL-1.0"
 
 dependency "vibe-d" version="~>0.8.4-alpha.1"
-dependency "dub" version="~>1.7.0"
+dependency "dub" version="~>1.9.0"
 dependency "userman" version="~>0.3.2"
 subConfiguration "dub" "library-nonet"
 

dub.selections.json

@@ -5,7 +5,7 @@
 		"botan": "1.12.9",
 		"botan-math": "1.0.3",
 		"diet-ng": "1.5.0",
-		"dub": "1.7.2",
+		"dub": "1.9.0",
 		"eventcore": "0.8.35",
 		"libasync": "0.8.3",
 		"libev": "5.0.0+4.04",

source/app.d

@@ -128,6 +128,7 @@ void main()
 	settings.bindAddresses = ["127.0.0.1"];
 	settings.port = 8005;
 	settings.sessionStore = new MemorySessionStore;
+	settings.useCompressionIfPossible = true;
 	readOption("bind", &settings.bindAddresses[0], "Sets the address used for serving.");
 	readOption("port|p", &settings.port, "Sets the port used for serving.");
 

source/dubregistry/api.d

@@ -11,6 +11,7 @@ import dubregistry.registry;
 import std.algorithm.iteration : map;
 import std.array : array;
 import std.exception : enforce;
+import std.typecons : Flag, Yes, No;
 import vibe.data.json : Json;
 import vibe.http.router;
 import vibe.inet.url;
@@ -71,6 +72,7 @@ interface DubRegistryAPI {
 
 struct SearchResult { string name, description, version_; }
 struct DownloadStats { DbDownloadStats downloads; }
+alias Version = string;
 
 interface IPackages {
 @safe:
@@ -88,10 +90,12 @@ interface IPackages {
 	DownloadStats getStats(string _name, string _version);
 
 	@path(":name/info")
-	Json getInfo(string _name);
+	Json getInfo(string _name, bool minimize = false);
 
 	@path(":name/:version/info")
-	Json getInfo(string _name, string _version);
+	Json getInfo(string _name, string _version, bool minimize = false);
+
+	Json[string] getInfos(string[] packages, bool include_dependencies = false, bool minimize = false);
 }
 
 class LocalDubRegistryAPI : DubRegistryAPI {
@@ -147,16 +151,30 @@ override {
 		}
 	}
 
-	Json getInfo(string name) {
-		return m_registry.getPackageInfo(rootOf(name))
-			.check!(r => r.info.type != Json.Type.null_)(HTTPStatus.notFound, "Package/Version not found")
+	Json getInfo(string name, bool minimize = false) {
+		immutable flags = minimize ? PackageInfoFlags.minimize : PackageInfoFlags.none;
+		return m_registry.getPackageInfo(rootOf(name), flags)
+			.check!(r => r.info.type != Json.Type.undefined)(HTTPStatus.notFound, "Package/Version not found")
 			.info;
 	}
 
-	Json getInfo(string name, string ver) {
-		return m_registry.getPackageVersionInfo(rootOf(name), ver)
+	Json getInfo(string name, string ver, bool minimize = false) {
+		immutable flags = minimize ? PackageInfoFlags.minimize : PackageInfoFlags.none;
+		return m_registry.getPackageVersionInfo(rootOf(name), ver, flags)
 			.check!(r => r.type != Json.Type.null_)(HTTPStatus.notFound, "Package/Version not found");
 	}
+
+	Json[string] getInfos(string[] packages, bool include_dependencies = false, bool minimize = false)
+	{
+		import std.array : assocArray;
+
+		auto flags = minimize ? PackageInfoFlags.minimize : PackageInfoFlags.none;
+		if (include_dependencies)
+			flags |= PackageInfoFlags.includeDependencies;
+		return m_registry.getPackageInfosRecursive(packages, flags)
+			.check!(r => r !is null)(HTTPStatus.notFound, "None of the packages were found")
+			.byKeyValue.map!(p => tuple(p.key, p.value.info)).assocArray;
+	}
 }
 
 private:

source/dubregistry/registry.d

@@ -195,25 +195,27 @@ class DubRegistry {
 		return m_db.aggregateDownloadStats(packid, ver);
 	}
 
-	Json getPackageVersionInfo(string packname, string ver)
+	Json getPackageVersionInfo(string packname, string ver, PackageInfoFlags flags)
 	{
 		if (ver == "latest") ver = getLatestVersion(packname);
 		if (!m_db.hasVersion(packname, ver)) return Json(null);
-		return m_db.getVersionInfo(packname, ver).serializeToJson();
+		auto ret = m_db.getVersionInfo(packname, ver).serializeToJson();
+		if (flags & PackageInfoFlags.minimize) ret.remove("readme");
+		return ret;
 	}
 
 	string getLatestVersion(string packname)
 	{
 		return m_db.getLatestVersion(packname);
 	}
 
-	PackageInfo getPackageInfo(string packname, bool include_errors = false)
+	PackageInfo getPackageInfo(string packname, PackageInfoFlags flags = PackageInfoFlags.none)
 	{
 		DbPackage pack;
 		try pack = m_db.getPackage(packname);
 		catch(Exception) return PackageInfo.init;
 
-		return getPackageInfo(pack, include_errors);
+		return getPackageInfo(pack, flags);
 	}
 
 	/** Gets information about multiple packages at once.
@@ -231,44 +233,71 @@ class DubRegistry {
 			An unordered input range of `PackageInfo` values is returned,
 			corresponding to all or part of the packages of the given names.
 	*/
-	auto getPackageInfos(scope string[] pack_names, bool include_errors = false)
+	auto getPackageInfos(scope string[] pack_names, PackageInfoFlags flags = PackageInfoFlags.none)
 	{
 		return m_db.getPackages(pack_names)
-			.map!(pack => getPackageInfo(pack, include_errors));
+			.map!(pack => getPackageInfo(pack, flags));
 	}
 
-	PackageInfo getPackageInfo(DbPackage pack, bool include_errors)
+	auto getPackageInfo(DbPackage pack, PackageInfoFlags flags = PackageInfoFlags.none)
 	{
 		auto rep = getRepository(pack.repository);
 
 		PackageInfo ret;
-		ret.versions = pack.versions.map!(v => getPackageVersionInfo(v, rep)).array;
+		ret.versions = pack.versions.map!(v => getPackageVersionInfo(v, rep, flags)).array;
 		ret.logo = pack.logo;
 
 		Json nfo = Json.emptyObject;
-		nfo["id"] = pack._id.toString();
-		nfo["dateAdded"] = pack._id.timeStamp.toISOExtString();
-		nfo["owner"] = pack.owner.toString();
-		nfo["name"] = pack.name;
 		nfo["versions"] = Json(ret.versions.map!(v => v.info).array);
-		nfo["repository"] = serializeToJson(pack.repository);
-		nfo["categories"] = serializeToJson(pack.categories);
-		nfo["documentationURL"] = pack.documentationURL;
-		if(include_errors) nfo["errors"] = serializeToJson(pack.errors);
+		if (!(flags & PackageInfoFlags.minimize))
+		{
+			nfo["name"] = pack.name;
+			nfo["id"] = pack._id.toString();
+			nfo["dateAdded"] = pack._id.timeStamp.toISOExtString();
+			nfo["owner"] = pack.owner.toString();
+			nfo["repository"] = serializeToJson(pack.repository);
+			nfo["categories"] = serializeToJson(pack.categories);
+			nfo["documentationURL"] = pack.documentationURL;
+		}
+		if (flags & PackageInfoFlags.includeErrors)
+			nfo["errors"] = serializeToJson(pack.errors);
 
 		ret.info = nfo;
 
 		return ret;
 	}
 
-	private PackageVersionInfo getPackageVersionInfo(DbPackageVersion v, Repository rep)
+	private PackageVersionInfo getPackageVersionInfo(DbPackageVersion v, Repository rep, PackageInfoFlags flags)
 	{
 		// JSON package version info as reported to the client
-		auto nfo = v.info.get!(Json[string]).dup;
+		Json[string] nfo;
+		if (flags & PackageInfoFlags.minimize)
+		{
+			// only keep information relevant for dependency resolution
+			static Json[string] keepDeps(in Json info)
+			{
+				Json[string] ret;
+				ret["name"] = info["name"];
+				ret["dependencies"] = info["dependencies"];
+				auto cfgs = info["configurations"].opt!(Json[])
+					.map!(cfg => Json(["name": cfg["name"], "dependencies": cfg["dependencies"]]));
+				if (!cfgs.empty)
+					ret["configurations"] = cfgs.array.Json;
+				return ret;
+			}
+			nfo = keepDeps(v.info);
+			auto subpkgs = v.info["subPackages"].opt!(Json[]).map!(subpkg => Json(keepDeps(subpkg)));
+			if (!subpkgs.empty)
+				nfo["subPackages"] = subpkgs.array.Json;
+		}
+		else
+		{
+			nfo = v.info.get!(Json[string]).dup;
+			nfo["date"] = v.date.toISOExtString();
+			nfo["readme"] = v.readme;
+			nfo["commitID"] = v.commitID;
+		}
 		nfo["version"] = v.version_;
-		nfo["date"] = v.date.toISOExtString();
-		nfo["readme"] = v.readme;
-		nfo["commitID"] = v.commitID;
 
 		PackageVersionInfo ret;
 		ret.info = Json(nfo);
@@ -279,6 +308,62 @@ class DubRegistry {
 		return ret;
 	}
 
+	PackageInfo[string] getPackageInfosRecursive(string[] packnames, PackageInfoFlags flags)
+	{
+		import dub.recipe.packagerecipe : getBasePackageName;
+
+		PackageInfo[string] infos;
+		void[0][string] visited;
+		foreach (packname; packnames)
+		{
+			logDebug("getPackageInfosRecursive: %s", packname);
+			getPackageInfosRecursive(packname, flags, infos, visited);
+		}
+		logDebug("getPackageInfosRecursive for %s returned %s", packnames, infos.byKey);
+		return infos;
+	}
+
+	private void getPackageInfosRecursive(string packname, PackageInfoFlags flags, ref PackageInfo[string] infos, ref void[0][string] visited)
+	{
+		import dub.recipe.packagerecipe : getBasePackageName, getSubPackageName;
+		import std.range : dropOne;
+		import std.algorithm.searching : find;
+
+		if (packname in visited)
+			return;
+		visited[packname] = typeof(visited[packname]).init;
+
+		auto basepkg = getBasePackageName(packname);
+		auto p = basepkg in infos;
+		if (p is null)
+			p = &(infos[basepkg] = getPackageInfo(basepkg, flags));
+
+		if (!(flags & PackageInfoFlags.includeDependencies))
+			return;
+
+		void addDeps(Json info)
+		{
+			foreach (dep, _; info["dependencies"].opt!(Json[string]))
+				getPackageInfosRecursive(dep, flags, infos, visited);
+			foreach (cfg; info["configurations"].opt!(Json[]))
+				foreach (dep, _; cfg["dependencies"].opt!(Json[string]))
+					getPackageInfosRecursive(dep, flags, infos, visited);
+		}
+
+	Louter: foreach (v; p.versions)
+		{
+			auto info = v.info;
+			foreach (subpkg; packname.splitter(":").dropOne)
+			{
+				auto sp = info["subPackages"].opt!(Json[]).find!(sp => sp["name"] == subpkg);
+				if (sp.empty)
+					continue Louter;
+				info = sp.front;
+			}
+			addDeps(info);
+		}
+	}
+
 	string getReadme(Json version_info, DbRepository repository)
 	{
 		auto readme = version_info["readme"].opt!string;
@@ -650,6 +735,15 @@ struct PackageInfo {
 	Json info; /// JSON package information, as reported to the client
 }
 
+/// flags to customize getPackageInfo* methods
+enum PackageInfoFlags
+{
+	none,
+	includeDependencies = 1 << 0, /// include package info of dependencies
+	includeErrors = 1 << 1, /// include package errors
+	minimize = 1 << 2, /// return only minimal information (for dependency resolver)
+}
+
 /// Computes a package score from given package stats and global distributions of those stats.
 private float computeScore(DownDist, RepoDist)(in ref DbPackageStats stats, DownDist downDist, RepoDist repoDist)
 @safe {

source/dubregistry/web.d

@@ -650,7 +650,7 @@ class DubRegistryFullWebFrontend : DubRegistryWebFrontend {
 	{
 		enforceUserPackage(_user, _packname);
 		auto packageName = _packname;
-		auto nfo = m_registry.getPackageInfo(packageName);
+		auto nfo = m_registry.getPackageInfo(packageName, PackageInfoFlags.includeErrors);
 		if (nfo.info.type == Json.Type.null_) return;
 		auto categories = m_categories;
 		auto registry = m_registry;

views/my_packages.dt

@@ -7,6 +7,7 @@ block body
 	- import vibe.data.json;
 	- import std.algorithm;
 	- import std.array : array;
+	- import dubregistry.registry : PackageInfoFlags;
 
 	- auto packages = registry.getPackages(user.id);
 
@@ -25,7 +26,7 @@ block body
 				button(type="submit") Register new package
 		.packageList
 			- foreach (p; sort!((a, b) => a < b)(packages.array))
-				- auto pack = registry.getPackageInfo(p, true).info;
+				- auto pack = registry.getPackageInfo(p, PackageInfoFlags.includeErrors).info;
 				- auto latest = pack["versions"].length ? pack["versions"][pack["versions"].length-1] : Json(null);
 				.row
 					strong.cell

views/my_packages.package.dt

@@ -5,8 +5,10 @@ block title
 
 block body
 	- import vibe.data.json;
+	- import dubregistry.registry : PackageInfoFlags;
 	- import dubregistry.web : Category;
-	- auto pack = registry.getPackageInfo(packageName, true).info;
+
+	- auto pack = registry.getPackageInfo(packageName, PackageInfoFlags.includeErrors).info;
 
 	h1 Edit package #[a.blind(href="#{req.rootDir}packages/#{packageName}")= packageName]
 		- auto latest = pack["versions"].length ? pack["versions"][pack["versions"].length-1] : Json(null);