Fix: Standardize User-Agent headers across Importers (#2122)

aboutcode/federated/__init__.py

@@ -26,6 +26,8 @@
 from packageurl import normalize_subpath
 from packageurl import normalize_version
 
+from django.conf import settings
+
 __version__ = "0.1.0"
 
 """
@@ -559,7 +561,7 @@ def from_url(
             federation_name=name,
             config_filename=cls.CONFIG_FILENAME,
         )
-        headers = {"User-Agent": "AboutCode/FederatedCode"}
+        headers = {"User-Agent": settings.VC_USER_AGENT}
         response = requests.get(url=rcf_url, headers=headers)
         if not response.ok:
             raise Exception(f"Failed to fetch Federation config: {rcf_url}")

vulnerabilities/importers/apache_httpd.py

@@ -27,6 +27,7 @@
 from vulnerabilities.utils import create_weaknesses_list
 from vulnerabilities.utils import cwe_regex
 from vulnerabilities.utils import get_item
+from django.conf import settings
 
 logger = logging.getLogger(__name__)
 
@@ -41,7 +42,10 @@ class ApacheHTTPDImporter(Importer):
     def advisory_data(self):
         links = fetch_links(self.base_url)
         for link in links:
-            data = requests.get(link).json()
+            data = requests.get(
+                link,
+                headers={'User-Agent': settings.VC_USER_AGENT}
+            ).json()
             yield self.to_advisory(data)
 
     def to_advisory(self, data):
@@ -150,7 +154,10 @@ def to_version_ranges(self, versions_data, fixed_versions):
 
 def fetch_links(url):
     links = []
-    data = requests.get(url).content
+    data = requests.get(
+        url,
+        headers={'User-Agent': settings.VC_USER_AGENT}
+    ).content
     soup = BeautifulSoup(data, features="lxml")
     for tag in soup.find_all("a"):
         link = tag.get("href")

vulnerabilities/importers/apache_kafka.py

@@ -20,6 +20,7 @@
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
+from django.conf import settings
 
 logger = logging.getLogger(__name__)
 
@@ -99,7 +100,10 @@ class ApacheKafkaImporter(Importer):
 
     @staticmethod
     def fetch_advisory_page(self):
-        page = requests.get(self.GH_PAGE_URL)
+        page = requests.get(
+            self.GH_PAGE_URL,
+            headers={'User-Agent': settings.VC_USER_AGENT}
+        )
         return page.content
 
     def advisory_data(self):

vulnerabilities/importers/apache_tomcat.py

@@ -27,6 +27,7 @@
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.severity_systems import APACHE_TOMCAT
+from django.conf import settings
 
 LOGGER = logging.getLogger(__name__)
 
@@ -126,15 +127,21 @@ def fetch_advisory_pages(self):
         """
         links = self.fetch_advisory_links("https://tomcat.apache.org/security")
         for page_url in links:
-            yield page_url, requests.get(page_url).content
+            yield page_url, requests.get(
+                page_url,
+                headers={'User-Agent': settings.VC_USER_AGENT}
+            ).content
 
     def fetch_advisory_links(self, url):
         """
         Yield the URLs of each Tomcat version security-related page.
         Each page link is in the form of `https://tomcat.apache.org/security-10.html`,
         for instance, for v10.
         """
-        data = requests.get(url).content
+        data = requests.get(
+            url,
+            headers={'User-Agent': settings.VC_USER_AGENT}
+        ).content
         soup = BeautifulSoup(data, features="lxml")
         for tag in soup.find_all("a"):
             link = tag.get("href")

vulnerabilities/importers/debian.py

@@ -27,6 +27,7 @@
 from vulnerabilities.utils import create_weaknesses_list
 from vulnerabilities.utils import dedupe
 from vulnerabilities.utils import get_item
+from django.conf import settings
 
 logger = logging.getLogger(__name__)
 
@@ -83,7 +84,10 @@ class DebianImporter(Importer):
     importer_name = "Debian Importer"
 
     def get_response(self):
-        response = requests.get(self.api_url)
+        response = requests.get(
+            self.api_url,
+            headers={'User-Agent': settings.VC_USER_AGENT}
+        )
         if response.status_code == 200:
             return response.json()
         raise Exception(

vulnerabilities/importers/debian_oval.py

@@ -14,7 +14,7 @@
 import requests
 
 from vulnerabilities.importer import OvalImporter
-
+from django.conf import settings
 
 class DebianOvalImporter(OvalImporter):
 
@@ -68,7 +68,10 @@ def _fetch(self):
         for release in releases:
             file_url = f"https://www.debian.org/security/oval/oval-definitions-{release}.xml.bz2"
             self.data_url = file_url
-            resp = requests.get(file_url).content
+            resp = requests.get(
+                file_url,
+                headers={'User-Agent': settings.VC_USER_AGENT}
+            ).content
             extracted = bz2.decompress(resp)
             yield (
                 {"type": "deb", "namespace": "debian", "qualifiers": {"distro": release}},

vulnerabilities/importers/gsd.py

@@ -22,6 +22,7 @@
 from vulnerabilities.importer import Reference
 from vulnerabilities.utils import build_description
 from vulnerabilities.utils import dedupe
+from django.conf import settings
 
 logger = logging.getLogger(__name__)
 
@@ -32,7 +33,10 @@ class GSDImporter:  # TODO inherit from Importer
     url = "https://codeload.github.com/cloudsecurityalliance/gsd-database/zip/refs/heads/main"
 
     def advisory_data(self) -> Iterable[AdvisoryData]:
-        response = requests.get(self.url).content
+        response = requests.get(
+            self.url,
+            headers={'User-Agent': settings.VC_USER_AGENT}
+        ).content
         with ZipFile(BytesIO(response)) as zip_file:
             for file_name in zip_file.namelist():
                 if file_name == "gsd-database-main/allowlist.json" or not file_name.endswith(

vulnerabilities/importers/mattermost.py

@@ -25,6 +25,7 @@
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.package_managers import GitHubTagsAPI
+from django.conf import settings
 
 SECURITY_UPDATES_URL = "https://mattermost.com/security-updates"
 MM_REPO = {
@@ -36,13 +37,13 @@
 
 class MattermostDataSource(Importer):
     def updated_advisories(self):
-        # FIXME: Change after this https://forum.mattermost.org/t/mattermost-website-returning-403-when-headers-contain-the-word-python/11412
         self.set_api()
         data = requests.get(
-            SECURITY_UPDATES_URL, headers={"user-agent": "aboutcode/vulnerablecode"}
+            SECURITY_UPDATES_URL, 
+            headers={"User-Agent": settings.VC_USER_AGENT},
         ).content
         return self.batch_advisories(self.to_advisories(data))
-
+    
     def set_api(self):
         self.version_api = GitHubTagsAPI()
         asyncio.run(

vulnerabilities/importers/openssl.py

@@ -25,6 +25,7 @@
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.severity_systems import SCORING_SYSTEMS
+from django.conf import settings
 
 logger = logging.getLogger(__name__)
 
@@ -36,7 +37,10 @@ class OpensslImporter(Importer):
     importer_name = "OpenSSL Importer"
 
     def fetch(self):
-        response = requests.get(url=self.url)
+        response = requests.get(
+            url=self.url,
+            headers={'User-Agent': settings.VC_USER_AGENT}
+        )
         if not response.status_code == 200:
             logger.error(f"Error while fetching {self.url}: {response.status_code}")
             return

vulnerabilities/importers/postgresql.py

@@ -21,7 +21,7 @@
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
-
+from django.conf import settings
 
 class PostgreSQLImporter(Importer):
 
@@ -37,7 +37,10 @@ def advisory_data(self):
         while True:
             unvisited_urls = known_urls - visited_urls
             for url in unvisited_urls:
-                data = requests.get(url).content
+                data = requests.get(
+                    url,
+                    headers={'User-Agent': settings.VC_USER_AGENT}
+                ).content
                 data_by_url[url] = data
                 visited_urls.add(url)
                 known_urls.update(find_advisory_urls(data))

vulnerabilities/importers/suse_backports.py

@@ -15,12 +15,15 @@
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
 from vulnerabilities.utils import create_etag
-
+from django.conf import settings
 
 class SUSEBackportsImporter(Importer):
     @staticmethod
     def get_all_urls_of_backports(url):
-        r = requests.get(url)
+        r = requests.get(
+            url,
+            headers={'User-Agent': settings.VC_USER_AGENT}
+        )
         soup = BeautifulSoup(r.content, "lxml")
         for a_tag in soup.find_all("a", href=True):
             if a_tag["href"].endswith(".yaml") and a_tag["href"].startswith("backports"):
@@ -38,7 +41,10 @@ def updated_advisories(self):
     def _fetch_yaml(self, url):
 
         try:
-            resp = requests.get(url)
+            resp = requests.get(
+                url,
+                headers={'User-Agent': settings.VC_USER_AGENT}
+            )
             resp.raise_for_status()
             return saneyaml.load(resp.content)
 

vulnerabilities/importers/suse_oval.py

@@ -15,6 +15,7 @@
 from bs4 import BeautifulSoup
 
 from vulnerabilities.importer import OvalImporter
+from django.conf import settings
 
 
 class SuseOvalImporter(OvalImporter):
@@ -27,7 +28,10 @@ def __init__(self, *args, **kwargs):
         self.translations = {"less than": "<", "equals": "=", "greater than or equal": ">="}
 
     def _fetch(self):
-        page = requests.get(self.base_url).text
+        page = requests.get(
+            self.base_url, 
+            headers={'User-Agent': settings.VC_USER_AGENT}
+        ).text
         soup = BeautifulSoup(page, "lxml")
 
         suse_oval_files = [
@@ -37,7 +41,10 @@ def _fetch(self):
         ]
 
         for suse_file in filter(suse_oval_files):
-            response = requests.get(suse_file)
+            response = requests.get(
+                suse_file,
+                headers={'User-Agent': settings.VC_USER_AGENT}
+            )
 
             extracted = gzip.decompress(response.content)
             yield (

vulnerabilities/importers/ubuntu.py

@@ -14,6 +14,7 @@
 import requests
 
 from vulnerabilities.importer import OvalImporter
+from django.conf import settings
 
 logger = logging.getLogger(__name__)
 
@@ -77,7 +78,10 @@ def _fetch(self):
             file_url = f"{base_url}/com.ubuntu.{release}.cve.oval.xml.bz2"  # nopep8
             self.data_url = file_url
             logger.info(f"Fetching Ubuntu Oval: {file_url}")
-            response = requests.get(file_url)
+            response = requests.get(
+                file_url,
+                headers={'User-Agent': settings.VC_USER_AGENT}
+            )
             if response.status_code != requests.codes.ok:
                 logger.error(
                     f"Failed to fetch Ubuntu Oval: HTTP {response.status_code} : {file_url}"

vulnerabilities/importers/ubuntu_usn.py

@@ -16,7 +16,7 @@
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.utils import is_cve
-
+from django.conf import settings
 
 class UbuntuUSNImporter(Importer):
     db_url = "https://usn.ubuntu.com/usn-db/database-all.json.bz2"
@@ -97,7 +97,10 @@ def get_usn_reference(usn_id):
 
 
 def fetch(url):
-    response = requests.get(url).content
+    response = requests.get(
+        url,
+        headers={'User-Agent': settings.VC_USER_AGENT}
+    ).content
     raw_data = bz2.decompress(response)
 
     return json.loads(raw_data)

vulnerabilities/management/commands/commit_export.py

@@ -23,6 +23,7 @@
 
 from vulnerablecode.settings import ALLOWED_HOSTS
 from vulnerablecode.settings import VULNERABLECODE_VERSION
+from django.conf import settings
 
 logger = logging.getLogger(__name__)
 
@@ -161,7 +162,11 @@ def create_pull_request(self, repo_url, branch, title, body, token):
             raise ValueError("Invalid GitHub repo URL")
 
         url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/pulls"
-        headers = {"Authorization": f"token {token}", "Accept": "application/vnd.github.v3+json"}
+        headers = {
+            "Authorization": f"token {token}", 
+            "Accept": "application/vnd.github.v3+json",
+            "User-Agent": VC_USER_AGENT  # <--- ADD THIS LINE
+        }
         data = {"title": title, "head": branch, "base": "main", "body": body}
 
         response = requests.post(url, headers=headers, json=data)

vulnerabilities/pipelines/enhance_with_exploitdb.py

@@ -23,6 +23,7 @@
 from vulnerabilities.models import VulnerabilityRelatedReference
 from vulnerabilities.pipelines import VulnerableCodePipeline
 
+from django.conf import settings
 
 class ExploitDBImproverPipeline(VulnerableCodePipeline):
     """
@@ -47,7 +48,10 @@ def fetch_exploits(self):
         self.log(f"Fetching {exploit_db_url}")
 
         try:
-            response = requests.get(exploit_db_url)
+            response = requests.get(
+                exploit_db_url,
+                headers={'User-Agent': settings.VC_USER_AGENT}
+            )
             response.raise_for_status()
         except requests.exceptions.HTTPError as http_err:
             self.log(

vulnerabilities/pipelines/enhance_with_kev.py

@@ -17,6 +17,7 @@
 from vulnerabilities.models import Exploit
 from vulnerabilities.pipelines import VulnerableCodePipeline
 
+from django.conf import settings
 
 class VulnerabilityKevPipeline(VulnerableCodePipeline):
     """
@@ -39,7 +40,10 @@ def fetch_exploits(self):
         self.log(f"Fetching {kev_url}")
 
         try:
-            response = requests.get(kev_url)
+            response = requests.get(
+                kev_url,
+                headers={'User-Agent': settings.VC_USER_AGENT}
+            )
             response.raise_for_status()
         except requests.exceptions.HTTPError as http_err:
             self.log(