From c37f0e45969d38ba3f1673919bd3e0f1fc733d92 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 21 Dec 2022 18:20:13 +0530 Subject: [PATCH 1/5] Migrate xen importer Signed-off-by: Tushar Goel --- vulnerabilities/importers/__init__.py | 2 + vulnerabilities/importers/xen.py | 80 +++++++------------ vulnerabilities/references.py | 10 +++ .../parse-advisory-xen-expected.json | 17 ++++ vulnerabilities/tests/test_data/xen_data.json | 42 ++++++++++ vulnerabilities/tests/test_xen.py | 20 +++++ 6 files changed, 119 insertions(+), 52 deletions(-) create mode 100644 vulnerabilities/tests/test_data/parse-advisory-xen-expected.json create mode 100644 vulnerabilities/tests/test_data/xen_data.json create mode 100644 vulnerabilities/tests/test_xen.py diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index d0c473ea5..c5e87ae88 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -31,6 +31,7 @@ from vulnerabilities.importers import retiredotnet from vulnerabilities.importers import suse_scores from vulnerabilities.importers import ubuntu +from vulnerabilities.importers import xen IMPORTERS_REGISTRY = [ nginx.NginxImporter, @@ -57,6 +58,7 @@ suse_scores.SUSESeverityScoreImporter, elixir_security.ElixirSecurityImporter, apache_tomcat.ApacheTomcatImporter, + xen.XenImporter, ] IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} diff --git a/vulnerabilities/importers/xen.py b/vulnerabilities/importers/xen.py index b704e62c5..41cc4ed22 100644 --- a/vulnerabilities/importers/xen.py +++ b/vulnerabilities/importers/xen.py @@ -13,60 +13,36 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.utils import create_etag +from vulnerabilities.references import XsaReference +from vulnerabilities.utils import fetch_response from vulnerabilities.utils import is_cve class XenImporter(Importer): - # CONFIG_CLASS = XenDBConfiguration - def updated_advisories(self): - advisories = [] - if create_etag(data_src=self, url=self.config.db_url, etag_key="etag"): - advisories.extend(self.to_advisories(fetch(self.config.db_url))) - - return self.batch_advisories(advisories) - - def create_etag(self, url): - etag = requests.head(url).headers.get("etag") - if not etag: - return True - - elif url in self.config.etags: - if self.config.etags[url] == etag: - return False - - self.config.etags[url] = etag - return True - - @staticmethod - def to_advisories(xen_db): - advisories = [] - for xsa in xen_db[0]["xsas"]: - reference = get_xen_references(xsa["xsa"]) - title = xsa.get("title", [""]) - for cve in xsa.get("cve", [""]): - if not is_cve(cve): - cve = "" - - advisories.append( - AdvisoryData( - vulnerability_id=cve, - summary=title, - references=[reference], - ) - ) - return advisories - - -def get_xen_references(xsa_id): - return Reference( - reference_id="XSA-" + xsa_id, - url="https://xenbits.xen.org/xsa/advisory-{}.html".format(xsa_id), - ) - - -def fetch(url): - response = requests.get(url).content - return json.loads(response) + url = "https://xenbits.xen.org/xsa/xsa.json" + spdx_license_expression = "" + license_url = "" + + def advisory_data(self): + data = fetch_response(self.url).json() + if not len(data): + return [] + xsas = data[0].get("xsas") or [] + for xsa in xsas: + yield from self.to_advisories(xsa) + + def to_advisories(self, xsa): + xsa_id = xsa.get("xsa") + references = [] + if xsa_id: + references.append(XsaReference.from_num_id(num_id=xsa_id)) + title = xsa.get("title") + for cve in xsa.get("cve") or []: + if not is_cve(cve): + continue + yield AdvisoryData( + aliases=[cve], + summary=title, + references=references, + ) diff --git a/vulnerabilities/references.py b/vulnerabilities/references.py index bec716be2..96e5a4c99 100644 --- a/vulnerabilities/references.py +++ b/vulnerabilities/references.py @@ -28,6 +28,16 @@ def from_id(cls, xsa_id): url=f"https://xenbits.xen.org/xsa/advisory-{numid}.html", ) + @classmethod + def from_num_id(cls, num_id): + """ + Return a new XsaReference from an XSA-XXXX id. + """ + return cls( + reference_id=f"XSA-{num_id}", + url=f"https://xenbits.xen.org/xsa/advisory-{num_id}.html", + ) + class ZbxReference(Reference): """ diff --git a/vulnerabilities/tests/test_data/parse-advisory-xen-expected.json b/vulnerabilities/tests/test_data/parse-advisory-xen-expected.json new file mode 100644 index 000000000..2acffa444 --- /dev/null +++ b/vulnerabilities/tests/test_data/parse-advisory-xen-expected.json @@ -0,0 +1,17 @@ +[ + { + "aliases": [ + "CVE-2012-5510" + ], + "summary": "Grant table version switch list corruption vulnerability", + "affected_packages": [], + "references": [ + { + "reference_id": "XSA-26", + "url": "https://xenbits.xen.org/xsa/advisory-26.html", + "severities": [] + } + ], + "date_published": null + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/xen_data.json b/vulnerabilities/tests/test_data/xen_data.json new file mode 100644 index 000000000..dcea34b91 --- /dev/null +++ b/vulnerabilities/tests/test_data/xen_data.json @@ -0,0 +1,42 @@ +{ + "cve": [ + "CVE-2012-5510" + ], + "files": [ + { + "name": "xsa26-4.1.patch", + "project": "xen", + "upstream": "Xen", + "versions": [ + [ + "4.1.x" + ] + ] + }, + { + "name": "xsa26-4.2.patch", + "project": "xen", + "upstream": "Xen", + "versions": [ + [ + "4.2.x" + ] + ] + }, + { + "name": "xsa26-unstable.patch", + "project": "xen", + "upstream": "Xen", + "versions": [ + [ + "xen-unstable" + ] + ] + } + ], + "public_time": "2012-12-03 17:51", + "title": "Grant table version switch list corruption vulnerability", + "version": 3, + "version_time": "2012-12-03 17:51", + "xsa": "26" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_xen.py b/vulnerabilities/tests/test_xen.py new file mode 100644 index 000000000..4e0278d5a --- /dev/null +++ b/vulnerabilities/tests/test_xen.py @@ -0,0 +1,20 @@ +import json +import os + +from vulnerabilities.importers.xen import XenImporter +from vulnerabilities.tests import util_tests + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join( + BASE_DIR, + "test_data", +) + + +def test_xen_to_advisories(): + with open(os.path.join(TEST_DATA, "xen_data.json")) as f: + raw_data = json.load(f) + advisories = XenImporter().to_advisories(raw_data) + result = [data.to_dict() for data in advisories] + expected_file = os.path.join(TEST_DATA, f"parse-advisory-xen-expected.json") + util_tests.check_results_against_json(result, expected_file) From 8a65a9731b49bbbe5c66a73d7917118bb4de2e44 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 5 Jan 2023 13:02:53 +0530 Subject: [PATCH 2/5] Address review comments Signed-off-by: Tushar Goel --- vulnerabilities/importers/xen.py | 25 +++++++++++++++++-------- vulnerabilities/tests/test_xen.py | 9 +++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/importers/xen.py b/vulnerabilities/importers/xen.py index 41cc4ed22..8e97a4cf7 100644 --- a/vulnerabilities/importers/xen.py +++ b/vulnerabilities/importers/xen.py @@ -7,10 +7,6 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -import json - -import requests - from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.references import XsaReference @@ -21,14 +17,27 @@ class XenImporter(Importer): url = "https://xenbits.xen.org/xsa/xsa.json" - spdx_license_expression = "" - license_url = "" + spdx_license_expression = "GPL-2" + license_url = "https://wiki.xenproject.org/wiki/Xen_FAQ_General" def advisory_data(self): data = fetch_response(self.url).json() - if not len(data): + # The data looks like this + # [ + # { + # "xsas": [ + # { + # "cve": [ + # "CVE-2012-5510" + # ], + # "title": "XSA-1: Xen security advisory", + # } + # ] + # } + # ] + if not data: return [] - xsas = data[0].get("xsas") or [] + xsas = data[0]["xsas"] for xsa in xsas: yield from self.to_advisories(xsa) diff --git a/vulnerabilities/tests/test_xen.py b/vulnerabilities/tests/test_xen.py index 4e0278d5a..d9e913f68 100644 --- a/vulnerabilities/tests/test_xen.py +++ b/vulnerabilities/tests/test_xen.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import json import os From fd0ae947e4b03c93bffca7bbeb358611b8eec50c Mon Sep 17 00:00:00 2001 From: Tushar Goel <34160672+TG1999@users.noreply.github.com> Date: Thu, 26 Jan 2023 13:29:36 +0530 Subject: [PATCH 3/5] Apply suggestions from code review Signed-off-by: Tushar Goel --- vulnerabilities/importers/xen.py | 3 ++- vulnerabilities/references.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/importers/xen.py b/vulnerabilities/importers/xen.py index 8e97a4cf7..519df6bb8 100644 --- a/vulnerabilities/importers/xen.py +++ b/vulnerabilities/importers/xen.py @@ -45,9 +45,10 @@ def to_advisories(self, xsa): xsa_id = xsa.get("xsa") references = [] if xsa_id: - references.append(XsaReference.from_num_id(num_id=xsa_id)) + references.append(XsaReference.from_number(number=xsa_id)) title = xsa.get("title") for cve in xsa.get("cve") or []: + #TODO: https://github.com/nexB/vulnerablecode/issues/981 if not is_cve(cve): continue yield AdvisoryData( diff --git a/vulnerabilities/references.py b/vulnerabilities/references.py index 96e5a4c99..87b45b9f6 100644 --- a/vulnerabilities/references.py +++ b/vulnerabilities/references.py @@ -29,13 +29,13 @@ def from_id(cls, xsa_id): ) @classmethod - def from_num_id(cls, num_id): + def from_number(cls, number): """ - Return a new XsaReference from an XSA-XXXX id. + Return a new XsaReference from an XSA number. """ return cls( - reference_id=f"XSA-{num_id}", - url=f"https://xenbits.xen.org/xsa/advisory-{num_id}.html", + reference_id=f"XSA-{number}", + url=f"https://xenbits.xen.org/xsa/advisory-{number}.html", ) From d0edf65bb7636dc53ff14d79678e998208d20576 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 26 Jan 2023 13:36:48 +0530 Subject: [PATCH 4/5] Adjust tests according to rebase Signed-off-by: Tushar Goel --- vulnerabilities/importers/xen.py | 2 +- .../tests/test_data/parse-advisory-xen-expected.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/xen.py b/vulnerabilities/importers/xen.py index 519df6bb8..d89f72c44 100644 --- a/vulnerabilities/importers/xen.py +++ b/vulnerabilities/importers/xen.py @@ -48,7 +48,7 @@ def to_advisories(self, xsa): references.append(XsaReference.from_number(number=xsa_id)) title = xsa.get("title") for cve in xsa.get("cve") or []: - #TODO: https://github.com/nexB/vulnerablecode/issues/981 + # TODO: https://github.com/nexB/vulnerablecode/issues/981 if not is_cve(cve): continue yield AdvisoryData( diff --git a/vulnerabilities/tests/test_data/parse-advisory-xen-expected.json b/vulnerabilities/tests/test_data/parse-advisory-xen-expected.json index 2acffa444..040c75fe1 100644 --- a/vulnerabilities/tests/test_data/parse-advisory-xen-expected.json +++ b/vulnerabilities/tests/test_data/parse-advisory-xen-expected.json @@ -12,6 +12,7 @@ "severities": [] } ], - "date_published": null + "date_published": null, + "weaknesses": [] } ] \ No newline at end of file From 94c2c8cd71f719596915459975fc672abe640281 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 26 Jan 2023 13:41:23 +0530 Subject: [PATCH 5/5] Add license for xen importer Signed-off-by: Tushar Goel --- vulnerabilities/importers/xen.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/xen.py b/vulnerabilities/importers/xen.py index d89f72c44..07c62d1f9 100644 --- a/vulnerabilities/importers/xen.py +++ b/vulnerabilities/importers/xen.py @@ -17,8 +17,33 @@ class XenImporter(Importer): url = "https://xenbits.xen.org/xsa/xsa.json" - spdx_license_expression = "GPL-2" - license_url = "https://wiki.xenproject.org/wiki/Xen_FAQ_General" + spdx_license_expression = "LicenseRef-scancode-other-permissive" + notice = """ + From: George Dunlap + Date: Wed, Jan 25, 2023 at 4:57 PM + Subject: Re: Usage of Xen Security Data in VulnerableCode + To: Tushar Goel + Cc: Andrew Cooper , xen-devel@lists.xenproject.org , Xen Security , Philippe Ombredanne , + + On Thu, Jan 19, 2023 at 1:10 PM Tushar Goel wrote: + > + > Hi Andrew, + > + > > Maybe we want to make it CC-BY-4 to require people to reference back to + > > the canonical upstream ? + > Thanks for your response, can we have a more declarative statement on + > the license from your end + > and also can you please provide your acknowledgement over the usage of + > Xen security data in vulnerablecode. + + + Hey Tushar, + Informally, the Xen Project Security Team is happy for you to include the data from xsa.json in your open-source vulnerability database. As a courtesy we'd request that it be documented where the information came from. (I think if the data includes links to then advisories on our website, that will suffice.) + Formally, we're not copyright lawyers; but we don't think there's anything copyright-able in the xsa.json: There is no editorial or creative control in the generation of that file; it's just a collection of facts which you could re-generate by scanning all the advisories. (In fact that's exactly how the file is created; i.e., the collection of advisory texts is our "source of truth".) + We do have "Officially license all advisory text as CC-BY-4" on our to-do list; if you'd be more comfortable with an official license for xsa.json as well, we can add that to the list. + + -George + """ def advisory_data(self): data = fetch_response(self.url).json()