Make vulnerability update bits affect incoming packages only (#15)

repology · May 6, 2020 · b8fdc8f · b8fdc8f
1 parent cc510f6
commit b8fdc8f
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 47 deletions.
diff --git a/repology/update/__init__.py b/repology/update/__init__.py
@@ -161,18 +161,18 @@ def _finish_update(self) -> None:
         self._logger.log('updating redirects')
         self._database.update_redirects(self._enable_partial_update, self._enable_explicit_analyze)
 
+        self._logger.log('updating cpe information')
+        self._database.update_cpe(self._enable_explicit_analyze)
+
+        self._logger.log('updating vulnerabilities')
+        self._database.update_vulnerabilities()
+
         # Note: before this, packages table still contains old versions of packages,
         # while new versions reside in incoming_packages temporary table
         self._logger.log('applying updated packages')
         self._database.update_apply_packages(self._enable_partial_update, self._enable_explicit_analyze)
         # Note: after this, packages table contain new versions of packages
 
-        self._logger.log('updating cpe information')
-        self._database.update_cpe(self._enable_partial_update, self._enable_explicit_analyze)
-
-        self._logger.log('updating vulnerabilities')
-        self._database.update_vulnerabilities()
-
         self._logger.log('updating binding table repo_metapackages')
         self._database.update_binding_repo_metapackages(self._enable_partial_update, self._enable_explicit_analyze)
 

diff --git a/sql.d/update/update_cpe.sql b/sql.d/update/update_cpe.sql
@@ -16,16 +16,10 @@
 -- along with repology.  If not, see <http://www.gnu.org/licenses/>.
 
 --------------------------------------------------------------------------------
--- @param partial=False
 -- @param analyze=True
 --------------------------------------------------------------------------------
-{% set packages = 'incoming_packages' if partial else 'packages' %}
-
 DELETE FROM project_cpe
-{% if partial %}
-WHERE effname IN (SELECT effname FROM changed_projects)
-{% endif %}
-;
+WHERE effname IN (SELECT effname FROM changed_projects);
 
 INSERT INTO project_cpe (
 	effname,
@@ -36,7 +30,7 @@ SELECT DISTINCT
     effname,
     cpe_vendor,
 	cpe_product
-FROM {{ packages }}
+FROM incoming_packages
 WHERE cpe_vendor IS NOT NULL AND cpe_product IS NOT NULL;
 
 {% if analyze %}

diff --git a/sql.d/update/update_vulnerabilities.sql b/sql.d/update/update_vulnerabilities.sql
@@ -15,38 +15,25 @@
 -- You should have received a copy of the GNU General Public License
 -- along with repology.  If not, see <http://www.gnu.org/licenses/>.
 
-WITH target AS (
-	SELECT
-		id,
-		bool_or(
-			EXISTS (
-				-- XXX: this lookup is rather slow because vulnerabilities may contains a lot
-				-- of rows per vendor/product; to fix this, we need to extend index onto version
-				-- field, but for this we need to improve postgresql-libversion first
-				SELECT *
-				FROM vulnerabilities_simplified AS vulnerabilities
-				WHERE
-					vulnerabilities.cpe_vendor = project_cpe.cpe_vendor AND
-					vulnerabilities.cpe_product = project_cpe.cpe_product AND
-					coalesce(
-						version_compare2(packages.version, vulnerabilities.start_version) >
-						CASE WHEN vulnerabilities.start_version_excluded THEN 0 ELSE -1 END,
-						true
-					) AND
-					version_compare2(packages.version, vulnerabilities.end_version) <
-					CASE WHEN vulnerabilities.end_version_excluded THEN 0 ELSE 1 END
-			)
-		) AS vulnerable
-	FROM packages INNER JOIN project_cpe USING(effname)
-	WHERE
-		packages.versionclass != 10 -- ROLLING
-	GROUP BY id
-)
-UPDATE packages
+UPDATE incoming_packages
 SET
-	flags = (flags & ~(1 << 16)) | (1 << 16) * vulnerable::integer
-FROM
-	target
+	flags = flags | (1 << 16)
 WHERE
-	packages.id = target.id AND (flags & (1 << 16))::boolean != vulnerable;
-
+	versionclass != 10 -- ROLLING
+	AND EXISTS (
+		-- XXX: this lookup is rather slow because vulnerabilities may contains a lot
+		-- of rows per vendor/product; to fix this, we need to extend index onto version
+		-- field, but for this we need to improve postgresql-libversion first
+		SELECT *
+		FROM vulnerabilities_simplified AS vulnerabilities INNER JOIN project_cpe USING (cpe_vendor, cpe_product)
+		WHERE
+			project_cpe.effname = incoming_packages.effname AND
+			coalesce(
+				version_compare2(incoming_packages.version, vulnerabilities.start_version) >
+				CASE WHEN vulnerabilities.start_version_excluded THEN 0 ELSE -1 END,
+				true
+			) AND
+			version_compare2(incoming_packages.version, vulnerabilities.end_version) <
+			CASE WHEN vulnerabilities.end_version_excluded THEN 0 ELSE 1 END
+	)
+;