Merge branch 'dev' into 3.0

Author: github-actions

.github/workflows/distro_tests.yml

@@ -63,7 +63,7 @@ jobs:
           export BBOT_DISTRO_TESTS=true
           poetry env use python3.11
           poetry install
-          poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .
+          poetry run pytest --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/tests.yml

@@ -36,7 +36,7 @@ jobs:
           poetry run ruff format --check
       - name: Run tests
         run: |
-          poetry run pytest -vv --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
+          poetry run pytest -vv --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v4

bbot/cli.py

@@ -199,7 +199,7 @@ async def _main():
             if sys.stdin.isatty():
                 # warn if any targets belong directly to a cloud provider
                 if not scan.preset.strict_scope:
-                    for event in scan.target.seeds.events:
+                    for event in scan.target.seeds.event_seeds:
                         if event.type == "DNS_NAME":
                             cloudcheck_result = scan.helpers.cloudcheck(event.host)
                             if cloudcheck_result:

bbot/core/config/logger.py

@@ -60,7 +60,6 @@ def __init__(self, core):
         self._loggers = None
         self._log_handlers = None
         self._log_level = None
-        self.root_logger = logging.getLogger()
         self.core_logger = logging.getLogger("bbot")
         self.core = core
 
@@ -83,14 +82,6 @@ def cleanup_logging(self):
         with suppress(Exception):
             self.queue_handler.close()
 
-        # Clean root logger
-        root_logger = logging.getLogger()
-        for handler in list(root_logger.handlers):
-            with suppress(Exception):
-                root_logger.removeHandler(handler)
-            with suppress(Exception):
-                handler.close()
-
         # Clean all other loggers
         for logger in logging.Logger.manager.loggerDict.values():
             if hasattr(logger, "handlers"):  # Logger, not PlaceHolder
@@ -111,8 +102,7 @@ def setup_queue_handler(self, logging_queue=None, log_level=logging.DEBUG):
             self.queue = logging_queue
         self.queue_handler = logging.handlers.QueueHandler(logging_queue)
 
-        self.root_logger.addHandler(self.queue_handler)
-
+        self.core_logger.addHandler(self.queue_handler)
         self.core_logger.setLevel(log_level)
         # disable asyncio logging for child processes
         if not SHARED_INTERPRETER_STATE.is_main_process:
@@ -215,11 +205,13 @@ def log_handlers(self):
             if not mkdir(log_dir, raise_error=False):
                 error_and_exit(f"Failure creating or error writing to BBOT logs directory ({log_dir})")
 
-            # Main log file (compressed)
-            main_handler = GzipRotatingFileHandler(f"{log_dir}/bbot.log", when="d", interval=1, backupCount=14)
+            # Main log file
+            main_handler = GzipRotatingFileHandler(f"{log_dir}/bbot.log", maxBytes=1024 * 1024 * 100, backupCount=100)
 
-            # Separate log file for debugging (compressed)
-            debug_handler = GzipRotatingFileHandler(f"{log_dir}/bbot.debug.log", when="d", interval=1, backupCount=14)
+            # Separate log file for debugging
+            debug_handler = GzipRotatingFileHandler(
+                f"{log_dir}/bbot.debug.log", maxBytes=1024 * 1024 * 100, backupCount=100
+            )
 
             # Log to stderr
             stderr_handler = logging.StreamHandler(sys.stderr)

bbot/core/event/base.py

@@ -19,8 +19,8 @@
 from urllib.parse import urlparse, urljoin, parse_qs
 
 
-from .helpers import *
 from bbot.errors import *
+from .helpers import EventSeed
 from bbot.core.helpers import (
     extract_words,
     is_domain,
@@ -111,18 +111,66 @@ class BaseEvent:
     # Bypass scope checking and dns resolution, distribute immediately to modules
     # This is useful for "end-of-line" events like FINDING and VULNERABILITY
     _quick_emit = False
-    # Whether this event has been retroactively marked as part of an important discovery chain
-    _graph_important = False
-    # Disables certain data validations
-    _dummy = False
     # Data validation, if data is a dictionary
     _data_validator = None
     # Whether to increment scope distance if the child and parent hosts are the same
+    # Normally we don't want this, since scope distance only increases if the host changes
+    # But for some events like SOCIAL media profiles, this is required to prevent spidering all of facebook.com
     _scope_distance_increment_same_host = False
     # Don't allow duplicates to occur within a parent chain
     # In other words, don't emit the event if the same one already exists in its discovery context
     _suppress_chain_dupes = False
 
+    # using __slots__ dramatically reduces memory usage in large scans
+    __slots__ = [
+        # Core identification attributes
+        "_uuid",
+        "_id",
+        "_hash",
+        "_data",
+        "_data_hash",
+        # Host-related attributes
+        "__host",
+        "_host_original",
+        "_port",
+        # Parent-related attributes
+        "_parent",
+        "_parent_id",
+        "_parent_uuid",
+        # Event metadata
+        "_type",
+        "_tags",
+        "_omit",
+        "__words",
+        "_priority",
+        "_scope_distance",
+        "_module_priority",
+        "_graph_important",
+        "_resolved_hosts",
+        "_discovery_context",
+        "_discovery_context_regex",
+        "_stats_recorded",
+        "_internal",
+        "_confidence",
+        "_dummy",
+        "_module",
+        # DNS-related attributes
+        "dns_children",
+        "raw_dns_records",
+        "dns_resolve_distance",
+        # Web-related attributes
+        "web_spider_distance",
+        "parsed_url",
+        "url_extension",
+        "num_redirects",
+        # File-related attributes
+        "_data_path",
+        # Public attributes
+        "module",
+        "scan",
+        "timestamp",
+    ]
+
     def __init__(
         self,
         data,
@@ -131,7 +179,6 @@ def __init__(
         context=None,
         module=None,
         scan=None,
-        scans=None,
         tags=None,
         confidence=100,
         timestamp=None,
@@ -150,7 +197,6 @@ def __init__(
             parent (BaseEvent, optional): Parent event that led to this event's discovery. Defaults to None.
             module (str, optional): Module that discovered the event. Defaults to None.
             scan (Scan, optional): BBOT Scan object. Required unless _dummy is True. Defaults to None.
-            scans (list of Scan, optional): BBOT Scan objects, used primarily when unserializing an Event from the database. Defaults to None.
             tags (list of str, optional): Descriptive tags for the event. Defaults to None.
             confidence (int, optional): Confidence level for the event, on a scale of 1-100. Defaults to 100.
             timestamp (datetime, optional): Time of event discovery. Defaults to current UTC time.
@@ -176,6 +222,7 @@ def __init__(
         self._host_original = None
         self._scope_distance = None
         self._module_priority = None
+        self._graph_important = False
         self._resolved_hosts = set()
         self.dns_children = {}
         self.raw_dns_records = {}
@@ -206,12 +253,6 @@ def __init__(
         self.scan = scan
         if (not self.scan) and (not self._dummy):
             raise ValidationError("Must specify scan")
-        # self.scans holds a list of scan IDs from scans that encountered this event
-        self.scans = []
-        if scans is not None:
-            self.scans = scans
-        if self.scan:
-            self.scans = list(set([self.scan.id] + self.scans))
 
         try:
             self.data = self._sanitize_data(data)
@@ -1350,7 +1391,7 @@ def sanitize_data(self, data):
         return validators.validate_email(data)
 
     def _host(self):
-        data = str(self.data).split("@")[-1]
+        data = str(self.data).rsplit("@", 1)[-1]
         host, self._port = split_host_port(data)
         return host
 
@@ -1654,7 +1695,6 @@ def make_event(
     context=None,
     module=None,
     scan=None,
-    scans=None,
     tags=None,
     confidence=100,
     dummy=False,
@@ -1714,12 +1754,11 @@ def make_event(
         tags = [tags]
     tags = set(tags)
 
+    # if data is already an event, update it with the user's kwargs
     if is_event(data):
         event = copy(data)
         if scan is not None and not event.scan:
             event.scan = scan
-        if scans is not None and not event.scans:
-            event.scans = scans
         if module is not None:
             event.module = module
         if parent is not None:
@@ -1733,8 +1772,11 @@ def make_event(
         event_type = data.type
         return event
     else:
+        # if event_type is not provided, autodetect it
         if event_type is None:
-            event_type, data = get_event_type(data)
+            event_seed = EventSeed(data)
+            event_type = event_seed.type
+            data = event_seed.data
             if not dummy:
                 log.debug(f'Autodetected event type "{event_type}" based on data: "{data}"')
 
@@ -1778,7 +1820,6 @@ def make_event(
             context=context,
             module=module,
             scan=scan,
-            scans=scans,
             tags=tags,
             confidence=confidence,
             _dummy=dummy,
@@ -1812,7 +1853,6 @@ def event_from_json(j):
         event_type = j["type"]
         kwargs = {
             "event_type": event_type,
-            "scans": j.get("scans", []),
             "tags": j.get("tags", []),
             "confidence": j.get("confidence", 100),
             "context": j.get("discovery_context", None),