diff --git a/package_managers/homebrew/jq/dependencies.jq b/package_managers/homebrew/jq/dependencies.jq index 9e7181a..56b9055 100644 --- a/package_managers/homebrew/jq/dependencies.jq +++ b/package_managers/homebrew/jq/dependencies.jq @@ -1,37 +1,30 @@ -# build_dependencies -# dependencies -# test_dependencies -# optional_dependencies -# uses_from_macos # TODO: variations (linux only, by architecture) -# all of the above are the fields that contain dependency info for Homebrew -# uses from macos sometimes specifies build / test -- right now logging that as macos only - -[.[] | -.name as $name | -( - (.uses_from_macos // []) | - map({ - package: $name, - dependency_type: "uses_from_macos", - dependency: (if type == "object" then keys[0] else . end) - }) -), -( - (.dependencies // []) | - map({package: $name, dependency_type: "dependency", dependency: .}) -), -( - (.test_dependencies // []) | - map({package: $name, dependency_type: "test_dependency", dependency: .}) -), -( - (.optional_dependencies // []) | - map({package: $name, dependency_type: "optional_dependency", dependency: .}) -), -( - (.build_dependencies // []) | - map({package: $name, dependency_type: "build_dependency", dependency: .}) -) -| .[]] \ No newline at end of file +[.[] | { + package_name: .name, + build_deps: .build_dependencies, + runtime_deps: .dependencies, + recommended_deps: .recommended_dependencies, + test_deps: .test_dependencies, + optional_deps: .optional_dependencies, + uses_from_macos: .uses_from_macos +} | + # here's where we'd substitute the depends_on_type ids, for each depends_on type ids + # the `[]` at the end is to ensure that we're exploding the arrays, so each dependency gets its own row! + {package_name: .package_name, depends_on_type: $build_deps_type_id, depends_on: .build_deps[]}, + {package_name: .package_name, depends_on_type: $runtime_deps_type_id, depends_on: .runtime_deps[]}, + {package_name: .package_name, depends_on_type: $recommended_deps_type_id, depends_on: .recommended_deps[]}, + {package_name: .package_name, depends_on_type: $test_deps_type_id, depends_on: .test_deps[]}, + {package_name: .package_name, depends_on_type: $optional_deps_type_id, depends_on: .optional_deps[]}, + {package_name: .package_name, depends_on_type: $uses_from_macos_type_id, depends_on: .uses_from_macos[]} + | + # now, filter out the null dependencies + select(.depends_on != null) | + # and only look at the ones that are strings TODO: some are JSONs? + select(.depends_on | type == "string") | + # generate the sql statements! + "INSERT INTO dependencies (version_id, dependency_id, depends_on_type_id) VALUES ( + (SELECT id FROM versions WHERE import_id = '" + .package_name + "'), + (SELECT id FROM packages WHERE name = '" + .depends_on + "'), + '" + .depends_on_type + "');" +] | join("\n") \ No newline at end of file diff --git a/package_managers/homebrew/jq/package_url.jq b/package_managers/homebrew/jq/package_url.jq index f1f767a..04b50df 100644 --- a/package_managers/homebrew/jq/package_url.jq +++ b/package_managers/homebrew/jq/package_url.jq @@ -1,8 +1,18 @@ +# mapping package to urls is straightforward +# but, in the first normal form we've gotta do the mapping ourselves +# luckily, homebrew is small enough that we can push some of that work to the db + [.[] | { package_name: .name, homepage_url: .homepage, source_url: .urls.stable.url -} | [ - {package_name: .package_name, url: .homepage_url}, - {package_name: .package_name, url: .source_url} -] | .[]] +} | + # here's where we substitute the url type ids, for each url type + {package_name: .package_name, type: $homepage_url_type_id, url: .homepage_url}, + {package_name: .package_name, type: $source_url_type_id, url: .source_url} + | + # and here we say "for each url, generate an insert statement" + "INSERT INTO package_urls (package_id, url_id) VALUES ( + (SELECT id FROM packages WHERE name = '" + .package_name + "'), + (SELECT id FROM urls WHERE url = '" + .url + "' AND url_type_id = '" + .type + "'));" +] | join("\n") diff --git a/package_managers/homebrew/jq/packages.jq b/package_managers/homebrew/jq/packages.jq index b070648..42cef0c 100644 --- a/package_managers/homebrew/jq/packages.jq +++ b/package_managers/homebrew/jq/packages.jq @@ -1,2 +1,13 @@ -# we just need the name for the packages models -[.[] | {name: .name, derived_id: ("homebrew/" + .name), import_id: .name, readme: null}] \ No newline at end of file + +[.[] | + "INSERT INTO packages (name, derived_id, import_id, package_manager_id) VALUES ('" + + # for every single row, extract the name => it's the only key we need from Homebrew + (.name) + "', '" + + # the derived_id is the package manager name + "/" + the package name, which enforces + # uniqueness on the packages table + ("homebrew/" + .name) + "', '" + + # the import_id is the same as the package name (used for joins) + .name + "', '" + + # the package manager ID is passed in as a variable + $package_manager_id + "');" +] | join("\n") diff --git a/package_managers/homebrew/jq/urls.jq b/package_managers/homebrew/jq/urls.jq index c51c2fc..3d904c5 100644 --- a/package_managers/homebrew/jq/urls.jq +++ b/package_managers/homebrew/jq/urls.jq @@ -1,12 +1,18 @@ -# homepage is at the main key -# source is inside stable, and it's the tarball +# from our sources.json, we're extracting homepage and source: + # homepage is at the main key + # source is inside stable, and it's the tarball + +# for every single row, extract the homepage and source: [.[] | { homepage: .homepage, source: .urls.stable.url } | to_entries | map({ +# `map` basically explodes the json, creating two rows for each JSON object name: .key, url: .value -}) | .[] | { - url: .url, - url_type: .name -}] \ No newline at end of file +}) | .[] | +# and here, we can generate our SQL statement! + "INSERT INTO urls (url, url_type_id) VALUES ('" + + .url + "', '" + + if .name == "source" then $source_url_type_id else $homepage_url_type_id end + "');" +] | join("\n") diff --git a/package_managers/homebrew/jq/versions.jq b/package_managers/homebrew/jq/versions.jq index 8d76b0d..413d05c 100644 --- a/package_managers/homebrew/jq/versions.jq +++ b/package_managers/homebrew/jq/versions.jq @@ -1,16 +1,21 @@ # homebrew has the problem where there are no versions # we're gonna assume the version available is the latest -# and we'll deal with that later # TODO: `downloads: .analytics.install_on_request."365d".[$name]` # above gives us the downloads for the last 365 days # not available in the full JSON API # TODO: there are also a problem of versioned formulae + +# TODO: licenses is in source.json, but we need a long-term mapping solution + [.[] | .name as $name | { version: .versions.stable, - import_id: .name, - license: .license -}] \ No newline at end of file + import_id: .name +} | +"INSERT INTO versions (version, package_id) VALUES ('" + + .version + "', '" + + .import_id + "');" +] | join("\n") diff --git a/package_managers/homebrew/sed/dependencies.sed b/package_managers/homebrew/sed/dependencies.sed deleted file mode 100644 index 3a4e234..0000000 --- a/package_managers/homebrew/sed/dependencies.sed +++ /dev/null @@ -1,2 +0,0 @@ -1d -s%"\([^"]*\)","\([^"]*\)",*%INSERT INTO dependencies (version_id, dependency_id, dependency_type_id) SELECT (SELECT id FROM versions WHERE import_id = '\3'), (SELECT id FROM packages WHERE derived_id = 'homebrew/\1'), (SELECT id FROM dependency_types WHERE name = '\2');% diff --git a/package_managers/homebrew/sed/package_url.sed b/package_managers/homebrew/sed/package_url.sed deleted file mode 100644 index ecc6f33..0000000 --- a/package_managers/homebrew/sed/package_url.sed +++ /dev/null @@ -1,2 +0,0 @@ -1d -s%"\([^"]*\)","\([^"]*\)",*%INSERT INTO package_urls (package_id, url_id) SELECT (SELECT id FROM packages WHERE derived_id = 'homebrew/\1'), (SELECT id FROM urls WHERE url = '\2') ON CONFLICT ("package_id", "url_id") DO NOTHING;% diff --git a/package_managers/homebrew/sed/packages.sed b/package_managers/homebrew/sed/packages.sed deleted file mode 100644 index 6373b11..0000000 --- a/package_managers/homebrew/sed/packages.sed +++ /dev/null @@ -1,2 +0,0 @@ -1d -s/"\([^"]*\)","\([^"]*\)","\([^"]*\)",*/INSERT INTO packages (derived_id, import_id, name, package_manager_id) VALUES ('\1', '\2', '\3', '@@HOMEBREW_ID@@') ON CONFLICT ("derived_id") DO NOTHING;/ \ No newline at end of file diff --git a/package_managers/homebrew/sed/urls.sed b/package_managers/homebrew/sed/urls.sed deleted file mode 100644 index 16f7639..0000000 --- a/package_managers/homebrew/sed/urls.sed +++ /dev/null @@ -1,4 +0,0 @@ - - -1d -s/"\([^"]*\)","\([^"]*\)",*/INSERT INTO urls (url, url_type_id) VALUES ('\1', (SELECT id FROM url_types WHERE "name" = '\2')) ON CONFLICT ("url", "url_type_id") DO NOTHING;/ \ No newline at end of file diff --git a/package_managers/homebrew/sed/versions.sed b/package_managers/homebrew/sed/versions.sed deleted file mode 100644 index d3453b7..0000000 --- a/package_managers/homebrew/sed/versions.sed +++ /dev/null @@ -1,2 +0,0 @@ -1d -s%"\([^"]*\)","\([^"]*\)","\([^"]*\)",*%INSERT INTO versions (import_id, package_id, "version") VALUES ('\1', (SELECT id FROM packages WHERE derived_id = 'homebrew/\1'), '\3') ON CONFLICT ("package_id", "version") DO NOTHING;%