From 301e841b644e876324e310e203d90e7c7c1bffa7 Mon Sep 17 00:00:00 2001 From: tim <46972822+regulartim@users.noreply.github.com> Date: Fri, 3 Oct 2025 10:57:26 +0200 Subject: [PATCH 1/5] Include mass scanners in advanced API by default. Closes #580 (#581) * Partly revert "added mass scanner exclusion as default" This reverts commit f953887509b695f64c5259405440656857989cfd. * adapt tests * add "tor exit nodes" to default excludes * add test case for tor exit node inclusion * add test case for tor exit node inclusion (ii) * fix syntax * rename method --- api/views/feeds.py | 11 +++++------ api/views/utils.py | 17 +++++++++-------- tests/__init__.py | 22 ++++++++++++++++++++++ tests/test_scoring_utils.py | 7 ++++--- tests/test_views.py | 26 +++++++++++++++++++++----- 5 files changed, 61 insertions(+), 22 deletions(-) diff --git a/api/views/feeds.py b/api/views/feeds.py index f4ac4762..5e309d11 100644 --- a/api/views/feeds.py +++ b/api/views/feeds.py @@ -26,6 +26,7 @@ def feeds(request, feed_type, attack_type, prioritize, format_): prioritize (str): Prioritization mechanism to use (e.g., recent, persistent). format_ (str): Desired format of the response (e.g., json, csv, txt). include_mass_scanners (bool): query parameter flag to include IOCs that are known mass scanners. + include_tor_exit_nodes (bool): query parameter flag to include IOCs that are known tor exit nodes. Returns: Response: The HTTP response with formatted IOC data. @@ -33,9 +34,8 @@ def feeds(request, feed_type, attack_type, prioritize, format_): logger.info(f"request /api/feeds with params: feed type: {feed_type}, " f"attack_type: {attack_type}, prioritization: {prioritize}, format: {format_}") feed_params = FeedRequestParams({"feed_type": feed_type, "attack_type": attack_type, "format_": format_}) + feed_params.apply_default_filters(request.query_params) feed_params.set_prioritization(prioritize) - if request.query_params and "include_mass_scanners" in request.query_params: - feed_params.include_mass_scanners() valid_feed_types = get_valid_feed_types() iocs_queryset = get_queryset(request, feed_params, valid_feed_types) @@ -58,9 +58,8 @@ def feeds_pagination(request): feed_params = FeedRequestParams(request.query_params) feed_params.format = "json" + feed_params.apply_default_filters(request.query_params) feed_params.set_prioritization(request.query_params.get("prioritize")) - if request.query_params and "include_mass_scanners" in request.query_params: - feed_params.include_mass_scanners() valid_feed_types = get_valid_feed_types() iocs_queryset = get_queryset(request, feed_params, valid_feed_types) @@ -83,8 +82,8 @@ def feeds_advanced(request): attack_type (str): Type of attack to filter. (supported: `scanner`, `payload_request`, `all`; default: `all`) max_age (int): Maximum number of days since last occurrence. E.g. an IOC that was last seen 4 days ago is excluded by default. (default: 3) min_days_seen (int): Minimum number of days on which an IOC must have been seen. (default: 1) - include_reputation (str): `;`-separated list of reputation values to include, e.g. `known attacker` or `known attacker;` to include IOCs without reputation. (default: include all) this has precedence over exclusion - exclude_reputation (str): `;`-separated list of reputation values to exclude, e.g. `mass scanner` or `mass scanner;bot, crawler`. (default: exclude mass scanners) + include_reputation (str): `;`-separated list of reputation values to include, e.g. `known attacker` or `known attacker;` to include IOCs without reputation. (default: include all) + exclude_reputation (str): `;`-separated list of reputation values to exclude, e.g. `mass scanner` or `mass scanner;bot, crawler`. (default: exclude none) feed_size (int): Number of IOC items to return. (default: 5000) ordering (str): Field to order results by, with optional `-` prefix for descending. (default: `-last_seen`) verbose (bool): `true` to include IOC properties that contain a lot of data, e.g. the list of days it was seen. (default: `false`) diff --git a/api/views/utils.py b/api/views/utils.py index 28d54932..39c2ae1c 100644 --- a/api/views/utils.py +++ b/api/views/utils.py @@ -75,10 +75,14 @@ def __init__(self, query_params: dict): self.paginate = query_params.get("paginate", "false").lower() self.format = query_params.get("format_", "json").lower() self.feed_type_sorting = None - self.exclude_reputation.append("mass scanner") - def include_mass_scanners(self): - self.exclude_reputation.remove("mass scanner") + def apply_default_filters(self, query_params): + if not query_params: + query_params = dict() + if "include_mass_scanners" not in query_params: + self.exclude_reputation.append("mass scanner") + if "include_tor_exit_nodes" not in query_params: + self.exclude_reputation.append("tor exit node") def set_prioritization(self, prioritize: str): match prioritize: @@ -90,7 +94,7 @@ def set_prioritization(self, prioritize: str): self.ordering = "-last_seen" case "persistent": self.max_age = "14" - self.min_days_seen: "10" + self.min_days_seen = "10" if "feed_type" in self.ordering: self.feed_type_sorting = self.ordering self.ordering = "-attack_count" @@ -155,14 +159,11 @@ def get_queryset(request, feed_params, valid_feed_types): query_dict["number_of_days_seen__gte"] = int(feed_params.min_days_seen) if feed_params.include_reputation: query_dict["ip_reputation__in"] = feed_params.include_reputation - for reputation_type in feed_params.include_reputation: - if reputation_type in feed_params.exclude_reputation: - feed_params.exclude_reputation.remove(reputation_type) iocs = ( IOC.objects.filter(**query_dict) .filter(Q(cowrie=True) | Q(log4j=True) | Q(general_honeypot__active=True)) - .exclude(Q() if "nothing" in feed_params.exclude_reputation else Q(ip_reputation__in=feed_params.exclude_reputation)) + .exclude(ip_reputation__in=feed_params.exclude_reputation) .annotate(value=F("name")) .annotate(honeypots=ArrayAgg("general_honeypot__name")) .order_by(feed_params.ordering)[: int(feed_params.feed_size)] diff --git a/tests/__init__.py b/tests/__init__.py index 2ede5af4..a84a7b98 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -59,6 +59,28 @@ def setUpTestData(cls): expected_interactions=11.1, ) + cls.ioc_3 = IOC.objects.create( + name="100.100.100.100", + type=iocType.IP.value, + first_seen=cls.current_time, + last_seen=cls.current_time, + days_seen=[cls.current_time], + number_of_days_seen=1, + attack_count=1, + interaction_count=1, + log4j=False, + cowrie=True, + scanner=True, + payload_request=True, + related_urls=[], + ip_reputation="tor exit node", + asn="12345", + destination_ports=[22, 23, 24], + login_attempts=1, + recurrence_probability=0.1, + expected_interactions=11.1, + ) + cls.ioc.general_honeypot.add(cls.heralding) # FEEDS cls.ioc.general_honeypot.add(cls.ciscoasa) # FEEDS cls.ioc.save() diff --git a/tests/test_scoring_utils.py b/tests/test_scoring_utils.py index e97cd8a3..0febc5c6 100644 --- a/tests/test_scoring_utils.py +++ b/tests/test_scoring_utils.py @@ -78,7 +78,7 @@ class TestFeatExtraction(CustomTestCase): def test_data_retrieval(self): """Test with sample IoCs""" data = get_current_data() - self.assertEqual(len(data), 2) + self.assertEqual(len(data), 3) def test_feature_extraction(self): """Test with sample IoCs""" @@ -92,8 +92,9 @@ def test_feature_extraction(self): self.assertEqual(len(feature["days_seen"]), 1) self.assertEqual(str(feature["days_seen"][0]), today) self.assertEqual(feature["asn"], "12345") - self.assertEqual(set(feature["honeypots"]), set(["heralding", "ciscoasa", "log4j", "cowrie"])) - self.assertEqual(feature["honeypot_count"], 4) + self.assertTrue(len(feature["honeypots"]) > 0) + self.assertTrue(set(feature["honeypots"]).issubset({"heralding", "ciscoasa", "log4j", "cowrie"})) + self.assertEqual(feature["honeypot_count"], len(feature["honeypots"])) self.assertEqual(feature["destination_port_count"], 3) self.assertEqual(feature["days_seen_count"], 1) self.assertEqual(feature["active_timespan"], 1) diff --git a/tests/test_views.py b/tests/test_views.py index fbb5cdc1..6cabad7d 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -95,11 +95,21 @@ def test_200_feeds_pagination(self): self.assertEqual(response.json()["count"], 1) self.assertEqual(response.json()["total_pages"], 1) - def test_200_feeds_pagination_inclusion(self): + def test_200_feeds_pagination_inclusion_mass(self): response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&include_mass_scanners") self.assertEqual(response.status_code, 200) self.assertEqual(response.json()["count"], 2) + def test_200_feeds_pagination_inclusion_tor(self): + response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&include_tor_exit_nodes") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], 2) + + def test_200_feeds_pagination_inclusion_mass_and_tor(self): + response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&include_mass_scanners&include_tor_exit_nodes") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], 3) + def test_400_feeds_pagination(self): response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=test&age=recent") self.assertEqual(response.status_code, 400) @@ -139,7 +149,7 @@ def test_400_feeds(self): def test_200_feeds_pagination(self): response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1") self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["count"], 1) + self.assertEqual(response.json()["count"], 3) self.assertEqual(response.json()["total_pages"], 1) def test_200_feeds_pagination_include(self): @@ -148,8 +158,14 @@ def test_200_feeds_pagination_include(self): self.assertEqual(response.json()["count"], 1) self.assertEqual(response.json()["total_pages"], 1) - def test_200_feeds_pagination_exclude_nothing(self): - response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1&exclude_reputation=nothing") + def test_200_feeds_pagination_exclude_mass(self): + response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1&exclude_reputation=mass%20scanner") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], 2) + self.assertEqual(response.json()["total_pages"], 1) + + def test_200_feeds_pagination_exclude_tor(self): + response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1&exclude_reputation=tor%20exit%20node") self.assertEqual(response.status_code, 200) self.assertEqual(response.json()["count"], 2) self.assertEqual(response.json()["total_pages"], 1) @@ -203,7 +219,7 @@ def test_200_feed_types(self): self.assertEqual(response.json()[0]["Heralding"], 2) self.assertEqual(response.json()[0]["Ciscoasa"], 2) self.assertEqual(response.json()[0]["Log4j"], 2) - self.assertEqual(response.json()[0]["Cowrie"], 2) + self.assertEqual(response.json()[0]["Cowrie"], 3) self.assertEqual(response.json()[0]["Tanner"], 0) From d7ad7e490dabff7cf728c3cdb618a49b8c6b8ccd Mon Sep 17 00:00:00 2001 From: tim <46972822+regulartim@users.noreply.github.com> Date: Fri, 3 Oct 2025 10:58:43 +0200 Subject: [PATCH 2/5] Upgrade Django to 5.2. Closes #502 (#579) * bump django-rest-email-auth * bump django to 5.2 * bump postgres to 18 (this requires manual manual intervention when upgrading GreedyBear) --- docker/default.yml | 2 +- requirements/project-requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/default.yml b/docker/default.yml index ef3fa5b3..0907c88d 100644 --- a/docker/default.yml +++ b/docker/default.yml @@ -4,7 +4,7 @@ x-no-healthcheck: &no-healthcheck services: postgres: - image: library/postgres:13-alpine + image: library/postgres:18-alpine container_name: greedybear_postgres volumes: - postgres_data:/var/lib/postgresql/data/ diff --git a/requirements/project-requirements.txt b/requirements/project-requirements.txt index 7760bb54..051b7987 100644 --- a/requirements/project-requirements.txt +++ b/requirements/project-requirements.txt @@ -3,9 +3,9 @@ celery==5.5.3 # if you change this, update the documentation elasticsearch-dsl==8.18.0 -Django==4.2.24 +Django==5.2.7 djangorestframework==3.16.1 -django-rest-email-auth==4.0.3 +django-rest-email-auth==5.0.0 django-ses==4.4.0 psycopg2-binary==2.9.10 From edc9d5f4609acff8aaf9449f046af4e4ce077e53 Mon Sep 17 00:00:00 2001 From: tim <46972822+regulartim@users.noreply.github.com> Date: Fri, 3 Oct 2025 12:15:59 +0200 Subject: [PATCH 3/5] Link to admin interface for staff users. Closes #529 (#582) * remove restriction to only show link to superusers * fix indentation --- frontend/src/layouts/widget/UserMenu.jsx | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/frontend/src/layouts/widget/UserMenu.jsx b/frontend/src/layouts/widget/UserMenu.jsx index 544d2dfa..6c7f4863 100644 --- a/frontend/src/layouts/widget/UserMenu.jsx +++ b/frontend/src/layouts/widget/UserMenu.jsx @@ -29,11 +29,9 @@ function UserMenu(props) { {/* Django Admin Interface */} - {isSuperuser && ( - - Django Admin Interface - - )} + + Django Admin Interface + {/* API Access/Sessions */} API Access / Sessions From f54354bbee5fef7fbce8c97b50adb6dec88e1047 Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 3 Oct 2025 16:11:57 +0200 Subject: [PATCH 4/5] bump 2.0.0 --- .env_template | 2 +- docker/.version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env_template b/.env_template index 7dd11f0d..5ce5883a 100644 --- a/.env_template +++ b/.env_template @@ -13,4 +13,4 @@ COMPOSE_FILE=docker/default.yml:docker/local.override.yml #COMPOSE_FILE=docker/default.yml:docker/local.override.yml:docker/elasticsearch.yml # If you want to run a specific version, populate this -# REACT_APP_INTELOWL_VERSION="1.6.8" +# REACT_APP_INTELOWL_VERSION="2.0.0" diff --git a/docker/.version b/docker/.version index 26031632..cf4e5aed 100644 --- a/docker/.version +++ b/docker/.version @@ -1 +1 @@ -REACT_APP_GREEDYBEAR_VERSION="1.6.8" \ No newline at end of file +REACT_APP_GREEDYBEAR_VERSION="2.0.0" \ No newline at end of file From 310f1e1dc4f8320cbea47bc7bb30ae82183e85d8 Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 3 Oct 2025 16:25:35 +0200 Subject: [PATCH 5/5] adapt CI --- .github/workflows/pull_request_automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request_automation.yml b/.github/workflows/pull_request_automation.yml index 5b86f09b..03b7c81c 100644 --- a/.github/workflows/pull_request_automation.yml +++ b/.github/workflows/pull_request_automation.yml @@ -70,7 +70,7 @@ jobs: postgres_db: greedybear_db postgres_user: user postgres_password: password - postgres_version: 13 + postgres_version: 18 use_memcached: false use_elastic_search: false use_rabbitmq: true