diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 7735ad816..6c388c891 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -32,6 +32,7 @@ ) from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2 from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2 +from vulnerabilities.pipelines.v2_improvers import yara_rules from vulnerabilities.utils import create_registry IMPROVERS_REGISTRY = create_registry( @@ -72,5 +73,6 @@ unfurl_version_range_v2.UnfurlVersionRangePipeline, compute_advisory_todo.ComputeToDo, collect_ssvc_trees.CollectSSVCPipeline, + yara_rules.YaraRulesImproverPipeline, ] ) diff --git a/vulnerabilities/migrations/0106_detectionrule.py b/vulnerabilities/migrations/0106_detectionrule.py new file mode 100644 index 000000000..bfd1a5b86 --- /dev/null +++ b/vulnerabilities/migrations/0106_detectionrule.py @@ -0,0 +1,68 @@ +# Generated by Django 4.2.25 on 2025-12-16 12:27 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0105_packagecommitpatch_patch_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="DetectionRule", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "rule_type", + models.CharField( + choices=[ + ("yara", "Yara"), + ("yara-x", "Yara-X"), + ("sigma", "Sigma"), + ("clamav", "ClamAV"), + ("suricata", "Suricata"), + ], + help_text="The type of the detection rule content (e.g., YARA, Sigma).", + max_length=50, + ), + ), + ( + "source_url", + models.URLField( + help_text="URL to the original source or reference for this rule.", + max_length=1024, + ), + ), + ( + "rule_metadata", + models.JSONField( + blank=True, + help_text="Additional structured data such as tags, or author information.", + null=True, + ), + ), + ( + "rule_text", + models.TextField(help_text="The content of the detection signature."), + ), + ( + "advisory", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="detection_rules", + to="vulnerabilities.advisoryv2", + ), + ), + ], + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 4c2cf5499..0c498453c 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -3489,3 +3489,45 @@ def __str__(self): class Meta: unique_together = ("vector", "source_advisory") + + +class DetectionRuleTypes(models.TextChoices): + """Defines the supported formats for security detection rules.""" + + YARA = "yara", "Yara" + YARA_X = "yara-x", "Yara-X" + SIGMA = "sigma", "Sigma" + CLAMAV = "clamav", "ClamAV" + SURICATA = "suricata", "Suricata" + + +class DetectionRule(models.Model): + """ + A Detection Rule is code used to identify malicious activity or security threats. + """ + + rule_type = models.CharField( + max_length=50, + choices=DetectionRuleTypes.choices, + help_text="The type of the detection rule content (e.g., YARA, Sigma).", + ) + + source_url = models.URLField( + max_length=1024, help_text="URL to the original source or reference for this rule." + ) + + rule_metadata = models.JSONField( + null=True, + blank=True, + help_text="Additional structured data such as tags, or author information.", + ) + + rule_text = models.TextField(help_text="The content of the detection signature.") + + advisory = models.ForeignKey( + AdvisoryV2, + related_name="detection_rules", + on_delete=models.SET_NULL, + null=True, + blank=True, + ) diff --git a/vulnerabilities/pipelines/v2_improvers/yara_rules.py b/vulnerabilities/pipelines/v2_improvers/yara_rules.py new file mode 100644 index 000000000..e65d7e6c7 --- /dev/null +++ b/vulnerabilities/pipelines/v2_improvers/yara_rules.py @@ -0,0 +1,119 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from pathlib import Path + +from aboutcode.pipeline import LoopProgress +from fetchcode.vcs import fetch_via_vcs + +from vulnerabilities.models import DetectionRule +from vulnerabilities.models import DetectionRuleTypes +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class YaraRulesImproverPipeline(VulnerableCodePipeline): + pipeline_id = "yara_rules" + + repo_urls = [ + "git+https://github.com/elastic/protections-artifacts", + "git+https://github.com/Yara-Rules/rules", + "git+https://github.com/Xumeiquer/yara-forensics", + "git+https://github.com/reversinglabs/reversinglabs-yara-rules", + "git+https://github.com/advanced-threat-research/Yara-Rules", + "git+https://github.com/bartblaze/Yara-rules", + "git+https://github.com/godaddy/yara-rules", # archived + "git+https://github.com/SupportIntelligence/Icewater", + "git+https://github.com/jeFF0Falltrades/YARA-Signatures", + "git+https://github.com/tjnel/yara_repo", + "git+https://github.com/JPCERTCC/jpcert-yara", + "git+https://github.com/mikesxrs/Open-Source-YARA-rules", + "git+https://github.com/fboldewin/YARA-rules", + "git+https://github.com/h3x2b/yara-rules", + ] + + license_urls = """ + https://github.com/elastic/protections-artifacts/blob/main/LICENSE.txt + https://github.com/Yara-Rules/rules/blob/master/LICENSE + https://github.com/Xumeiquer/yara-forensics/blob/master/LICENSE + https://github.com/reversinglabs/reversinglabs-yara-rules/blob/develop/LICENSE + https://github.com/advanced-threat-research/Yara-Rules/blob/master/LICENSE + https://github.com/bartblaze/Yara-rules/blob/master/LICENSE + https://github.com/godaddy/yara-rules/blob/master/LICENSE.md + https://github.com/SupportIntelligence/Icewater/blob/master/LICENSE + https://github.com/jeFF0Falltrades/YARA-Signatures/blob/master/LICENSE.md + https://github.com/tjnel/yara_repo/blob/master/LICENSE + https://github.com/JPCERTCC/jpcert-yara/blob/main/LICENSE + + NO-LICENSE: https://github.com/mikesxrs/Open-Source-YARA-rules/ + NO-LICENSE: https://github.com/fboldewin/YARA-rules + NO-LICENSE: https://github.com/h3x2b/yara-rules + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.vcs_responses = [] + + @classmethod + def steps(cls): + return ( + cls.clone_repos, + cls.collect_and_store_rules, + cls.clean_downloads, + ) + + def clone_repos(self): + for url in self.repo_urls: + self.log(f"Cloning `{url}`") + try: + response = fetch_via_vcs(url) + if response: + self.vcs_responses.append((response, url)) + except Exception as e: + self.log(f"Failed to clone {url}: {e}") + + def collect_and_store_rules(self): + for vcs_response, repo_url in self.vcs_responses: + base_directory = Path(vcs_response.dest_dir) + yara_files = [ + p + for p in base_directory.rglob("*") + if p.suffix in (".yar", ".yara") and p.is_file() + ] + + rules_count = len(yara_files) + self.log(f"Processing {rules_count:,d} rules from {repo_url}") + + progress = LoopProgress(total_iterations=rules_count, logger=self.log) + for file_path in progress.iter(yara_files): + if not file_path.exists() or not file_path.is_file(): + self.log( + f"Skipping file as it no longer exists or is not a file: {file_path}", + level="warning", + ) + continue + + raw_text = file_path.read_text(encoding="utf-8", errors="ignore") + if not raw_text: + continue + + DetectionRule.objects.update_or_create( + rule_text=raw_text, + rule_type=DetectionRuleTypes.YARA, + advisory=None, + ) + + def clean_downloads(self): + for vcs_response, _ in self.vcs_responses: + if vcs_response: + self.log(f"Removing cloned repository: {vcs_response.dest_dir}") + vcs_response.delete() + + self.vcs_responses = [] + + def on_failure(self): + self.clean_downloads() diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_yara.py b/vulnerabilities/tests/pipelines/v2_improvers/test_yara.py new file mode 100644 index 000000000..6daa0268f --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_yara.py @@ -0,0 +1,32 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from vulnerabilities.models import DetectionRule +from vulnerabilities.pipelines.v2_improvers.yara_rules import YaraRulesImproverPipeline + +BASE_DIR = Path(__file__).resolve().parent +TEST_REPO_DIR = (BASE_DIR / "../../test_data/yara").resolve() + + +@pytest.mark.django_db +def test_collect_and_store_rules_from_test_repo_dir(): + mock_vcs_response = MagicMock() + mock_vcs_response.dest_dir = str(TEST_REPO_DIR) + + improver = YaraRulesImproverPipeline() + improver.vcs_responses = [(mock_vcs_response, "https://github.com/mock/repo")] + improver.collect_and_store_rules() + + assert DetectionRule.objects.exists() + assert DetectionRule.objects.count() == 4 diff --git a/vulnerabilities/tests/test_data/yara/Linux_Backdoor_Bash.yar b/vulnerabilities/tests/test_data/yara/Linux_Backdoor_Bash.yar new file mode 100644 index 000000000..54447b4b6 --- /dev/null +++ b/vulnerabilities/tests/test_data/yara/Linux_Backdoor_Bash.yar @@ -0,0 +1,45 @@ +rule Windows_Trojan_Xeno_f92ffb82 { + meta: + author = "Elastic Security" + id = "f92ffb82-b743-4df1-9d6b-2afa3b7bb61f" + fingerprint = "2ae1aebd652afb7da5799f46883205b1f3a5c5b01e975b526640407d9bd0d22c" + creation_date = "2024-10-10" + last_modified = "2024-10-24" + threat_name = "Windows.Trojan.Xeno" + reference_sample = "22dbdbcdd4c8b6899006f9f07e87c19b6a2947eeff8cc89c653309379b388cf4" + severity = 50 + arch_context = "x86" + scan_context = "file, memory" + license = "Elastic License v2" + os = "windows" + strings: + $a1 = { 28 00 00 0A 7D 0E 00 00 04 02 7B 0E 00 00 04 28 29 00 00 0A 07 7B 03 00 00 04 02 7B 0E 00 00 04 6F 2A 00 00 0A 3A F2 00 00 00 02 7B 07 00 00 04 02 7B 09 00 00 04 6F 32 00 00 06 6F 2B 00 00 0A } + condition: + all of them +} + +rule Windows_Trojan_Xeno_89f9f060 { + meta: + author = "Elastic Security" + id = "89f9f060-afc8-427d-ad36-3672016efdf6" + fingerprint = "ddc5bf8c6d5140cb9ea2fbd9b6f1aaab60f506dcd6161a26961958efa4aa42e1" + creation_date = "2024-10-25" + last_modified = "2024-11-26" + threat_name = "Windows.Trojan.Xeno" + reference_sample = "b74733d68e95220ab0630a68ddf973b0c959fd421628e639c1b91e465ba9299b" + severity = 100 + arch_context = "x86" + scan_context = "file, memory" + license = "Elastic License v2" + os = "windows" + strings: + $sc_1 = { 8B 44 24 04 89 C6 FF 56 08 68 00 04 00 00 6A 08 50 FF 16 89 C3 8B 06 89 83 2C 01 00 00 8B 46 04 89 83 30 01 00 00 8B 46 08 89 83 34 01 00 00 8B 46 0C 89 83 38 } + $sc_2 = { 55 48 89 E5 48 83 EC 40 49 89 CC 41 FF 54 24 10 48 89 C1 BA 08 00 00 00 41 B8 00 04 00 00 41 FF 14 24 48 89 C3 49 8B 04 24 48 89 83 90 01 00 00 49 8B 44 24 08 } + $str_1 = "SharpInjector" ascii fullword + $str_2 = "HEAVENSGATE_NON_OPERATIONAL" ascii fullword + $str_3 = "ChromeDecryptor" ascii fullword + $str_4 = "DataExtractionStructs" ascii fullword + $str_5 = "XenoStealer" ascii fullword + condition: + (($sc_1 or $sc_2) and ($str_1 or $str_2)) and (1 of ($str_3, $str_4, $str_5)) +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/yara/Linux_Backdoor_Python.yar b/vulnerabilities/tests/test_data/yara/Linux_Backdoor_Python.yar new file mode 100644 index 000000000..40c03b711 --- /dev/null +++ b/vulnerabilities/tests/test_data/yara/Linux_Backdoor_Python.yar @@ -0,0 +1,19 @@ +rule Linux_Backdoor_Python_00606bac { + meta: + author = "Elastic Security" + id = "00606bac-83eb-4a58-82d2-e4fd16d30846" + fingerprint = "cce1d0e7395a74c04f15ff95f6de7fd7d5f46ede83322b832df74133912c0b17" + creation_date = "2021-01-12" + last_modified = "2021-09-16" + threat_name = "Linux.Backdoor.Python" + reference_sample = "b3e3728d43535f47a1c15b915c2d29835d9769a9dc69eb1b16e40d5ba1b98460" + severity = 100 + arch_context = "x86" + scan_context = "file, memory" + license = "Elastic License v2" + os = "linux" + strings: + $a = { F4 01 83 45 F8 01 8B 45 F8 0F B6 00 84 C0 75 F2 83 45 F8 01 8B } + condition: + all of them +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/yara/Linux_Exploit_CVE_2022_0847.yar b/vulnerabilities/tests/test_data/yara/Linux_Exploit_CVE_2022_0847.yar new file mode 100644 index 000000000..5199c81f2 --- /dev/null +++ b/vulnerabilities/tests/test_data/yara/Linux_Exploit_CVE_2022_0847.yar @@ -0,0 +1,27 @@ +rule Linux_Exploit_CVE_2022_0847_e831c285 { + meta: + author = "Elastic Security" + id = "e831c285-b2b9-49f3-a87c-3deb806e31e4" + fingerprint = "376b791f9bb5f48d0f41ead4e48b5bcc74cb68002bb7c170760428ace169457e" + creation_date = "2022-03-10" + last_modified = "2022-03-14" + threat_name = "Linux.Exploit.CVE-2022-0847" + reference_sample = "c6b2cef2f2bc04e3ae33e0d368eb39eb5ea38d1bca390df47f7096117c1aecca" + severity = 100 + arch_context = "x86" + scan_context = "file, memory" + license = "Elastic License v2" + os = "linux" + strings: + $pp = "prepare_pipe" + $s1 = "splice failed" + $s2 = "short splice" + $s3 = "short write" + $s4 = "hijacking suid binary" + $s5 = "Usage: %s TARGETFILE OFFSET DATA" + $s6 = "Usage: %s SUID" + $bs1 = { B8 00 10 00 00 81 7D EC 00 10 00 00 0F 46 45 EC 89 45 FC 8B 55 FC 48 8B 45 D8 48 83 C0 04 8B 00 48 8D 35 } + $bs2 = { B8 00 10 00 00 81 7D F0 00 10 00 00 0F 46 45 F0 89 45 F8 8B 55 F8 48 8B 45 D8 8B 00 48 } + condition: + ($pp and 2 of ($s*)) or (all of ($bs*)) +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/yara/general.yara b/vulnerabilities/tests/test_data/yara/general.yara new file mode 100644 index 000000000..d293f93b7 --- /dev/null +++ b/vulnerabilities/tests/test_data/yara/general.yara @@ -0,0 +1,67 @@ +// yara rules that can cross boundaries between the various sets/types... more general detection signatures +import "pe" + +rule wiper_unique_strings +{ + meta: + copyright = "2015 Novetta Solutions" + author = "Novetta Threat Research & Interdiction Group - trig@novetta.com" + company = "novetta" + + strings: + $a = "C!@I#%VJSIEOTQWPVz034vuA" + $b = "BAISEO%$2fas9vQsfvx%$" + $c = "1.2.7.f-hanba-win64-v1" + $d = "md %s© %s\\*.* %s" + $e = "%sd.e%sc n%ssh%srewa%s ad%s po%sop%sing T%s %d \"%s\"" + $f = "Ge.tVol. .umeIn..for mati.onW" + + condition: + $a or $b or $c or $d or $e or $f +} + + +rule wiper_encoded_strings +{ + meta: + copyright = "2015 Novetta Solutions" + author = "Novetta Threat Research & Interdiction Group - trig@novetta.com" + company = "novetta" + + strings: + $scr = {89 D4 C4 D5 00 00 00} + $explorer = {E2 DF D7 CB C8 D5 C2 D5 89 C2 DF C2 00 00 00 } + $kernel32 = {CC C2 D5 C9 C2 CB 94 95 89 C3 CB CB 00 00 } + + condition: + $scr or $explorer or $kernel32 +} + + +rule createP2P +{ + meta: + copyright = "2015 Novetta Solutions" + author = "Novetta Threat Research & Interdiction Group - trig@novetta.com" + + strings: + $ = "CreatP2P Thread" wide + + condition: + any of them +} + +rule firewallOpener +{ + meta: + copyright = "2015 Novetta Solutions" + author = "Novetta Threat Research & Interdiction Group - trig@novetta.com" + + strings: + $ = "%sd.e%sc n%ssh%srewa%s ad%s po%sop%sing T%s %d \"%s\"" + + condition: + any of them + +} +