diff --git a/repology/update/__init__.py b/repology/update/__init__.py index 4f58812b3..9f5579eef 100644 --- a/repology/update/__init__.py +++ b/repology/update/__init__.py @@ -161,18 +161,18 @@ def _finish_update(self) -> None: self._logger.log('updating redirects') self._database.update_redirects(self._enable_partial_update, self._enable_explicit_analyze) + self._logger.log('updating cpe information') + self._database.update_cpe(self._enable_explicit_analyze) + + self._logger.log('updating vulnerabilities') + self._database.update_vulnerabilities() + # Note: before this, packages table still contains old versions of packages, # while new versions reside in incoming_packages temporary table self._logger.log('applying updated packages') self._database.update_apply_packages(self._enable_partial_update, self._enable_explicit_analyze) # Note: after this, packages table contain new versions of packages - self._logger.log('updating cpe information') - self._database.update_cpe(self._enable_partial_update, self._enable_explicit_analyze) - - self._logger.log('updating vulnerabilities') - self._database.update_vulnerabilities() - self._logger.log('updating binding table repo_metapackages') self._database.update_binding_repo_metapackages(self._enable_partial_update, self._enable_explicit_analyze) diff --git a/sql.d/update/update_cpe.sql b/sql.d/update/update_cpe.sql index 697858e50..f4fec5976 100644 --- a/sql.d/update/update_cpe.sql +++ b/sql.d/update/update_cpe.sql @@ -16,16 +16,10 @@ -- along with repology. If not, see . -------------------------------------------------------------------------------- --- @param partial=False -- @param analyze=True -------------------------------------------------------------------------------- -{% set packages = 'incoming_packages' if partial else 'packages' %} - DELETE FROM project_cpe -{% if partial %} -WHERE effname IN (SELECT effname FROM changed_projects) -{% endif %} -; +WHERE effname IN (SELECT effname FROM changed_projects); INSERT INTO project_cpe ( effname, @@ -36,7 +30,7 @@ SELECT DISTINCT effname, cpe_vendor, cpe_product -FROM {{ packages }} +FROM incoming_packages WHERE cpe_vendor IS NOT NULL AND cpe_product IS NOT NULL; {% if analyze %} diff --git a/sql.d/update/update_vulnerabilities.sql b/sql.d/update/update_vulnerabilities.sql index 5bd9a3536..6be80bc7f 100644 --- a/sql.d/update/update_vulnerabilities.sql +++ b/sql.d/update/update_vulnerabilities.sql @@ -15,38 +15,25 @@ -- You should have received a copy of the GNU General Public License -- along with repology. If not, see . -WITH target AS ( - SELECT - id, - bool_or( - EXISTS ( - -- XXX: this lookup is rather slow because vulnerabilities may contains a lot - -- of rows per vendor/product; to fix this, we need to extend index onto version - -- field, but for this we need to improve postgresql-libversion first - SELECT * - FROM vulnerabilities_simplified AS vulnerabilities - WHERE - vulnerabilities.cpe_vendor = project_cpe.cpe_vendor AND - vulnerabilities.cpe_product = project_cpe.cpe_product AND - coalesce( - version_compare2(packages.version, vulnerabilities.start_version) > - CASE WHEN vulnerabilities.start_version_excluded THEN 0 ELSE -1 END, - true - ) AND - version_compare2(packages.version, vulnerabilities.end_version) < - CASE WHEN vulnerabilities.end_version_excluded THEN 0 ELSE 1 END - ) - ) AS vulnerable - FROM packages INNER JOIN project_cpe USING(effname) - WHERE - packages.versionclass != 10 -- ROLLING - GROUP BY id -) -UPDATE packages +UPDATE incoming_packages SET - flags = (flags & ~(1 << 16)) | (1 << 16) * vulnerable::integer -FROM - target + flags = flags | (1 << 16) WHERE - packages.id = target.id AND (flags & (1 << 16))::boolean != vulnerable; - + versionclass != 10 -- ROLLING + AND EXISTS ( + -- XXX: this lookup is rather slow because vulnerabilities may contains a lot + -- of rows per vendor/product; to fix this, we need to extend index onto version + -- field, but for this we need to improve postgresql-libversion first + SELECT * + FROM vulnerabilities_simplified AS vulnerabilities INNER JOIN project_cpe USING (cpe_vendor, cpe_product) + WHERE + project_cpe.effname = incoming_packages.effname AND + coalesce( + version_compare2(incoming_packages.version, vulnerabilities.start_version) > + CASE WHEN vulnerabilities.start_version_excluded THEN 0 ELSE -1 END, + true + ) AND + version_compare2(incoming_packages.version, vulnerabilities.end_version) < + CASE WHEN vulnerabilities.end_version_excluded THEN 0 ELSE 1 END + ) +;