Merge pull request #3 from BlueBirdBack/reliability-config-and-twitter-guardrails

feat: implement env config + add Nitter fetch guardrails

README.md

@@ -210,8 +210,8 @@ DeepReader uses sensible defaults out of the box. Configuration can be customize
 
 | Variable | Default | Description |
 |----------|---------|-------------|
-| `DEEPREEDER_MEMORY_PATH` | `../../memory/inbox/` | Where to save ingested content |
-| `DEEPREEDER_LOG_LEVEL` | `INFO` | Logging verbosity |
+| `DEEPREEDER_MEMORY_PATH` | `../../memory/inbox/` | Where to save ingested content (absolute path, or relative to repo root) |
+| `DEEPREEDER_LOG_LEVEL` | `INFO` | Logging verbosity (`DEBUG`, `INFO`, `WARNING`, `ERROR`) |
 
 ---
 

deepreader_skill/SKILL.md

@@ -69,8 +69,8 @@ word_count: 350
 
 | Variable | Default | Description |
 |----------|---------|-------------|
-| `DEEPREEDER_MEMORY_PATH` | `../../memory/inbox/` | Where to save ingested content |
-| `DEEPREEDER_LOG_LEVEL` | `INFO` | Logging verbosity |
+| `DEEPREEDER_MEMORY_PATH` | `../../memory/inbox/` | Where to save ingested content (absolute path, or relative to repo root) |
+| `DEEPREEDER_LOG_LEVEL` | `INFO` | Logging verbosity (`DEBUG`, `INFO`, `WARNING`, `ERROR`) |
 
 ## How it works
 

deepreader_skill/__init__.py

@@ -24,6 +24,7 @@ Supported URL types:
 from __future__ import annotations
 
 import logging
+import os
 from typing import Any
 
 from .core.router import ParserRouter
@@ -45,7 +46,10 @@ if not logger.handlers:
         )
     )
     logger.addHandler(handler)
-    logger.setLevel(logging.INFO)
+
+_log_level_name = os.getenv("DEEPREEDER_LOG_LEVEL", "INFO").upper()
+_log_level = getattr(logging, _log_level_name, logging.INFO)
+logger.setLevel(_log_level)
 
 
 # ---------------------------------------------------------------------------

deepreader_skill/core/storage.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import json
 import logging
+import os
 import re
 import uuid
 from datetime import datetime, timezone
@@ -62,12 +63,25 @@ class StorageManager:
                         directory.  Defaults to ``../../memory/inbox/``
                         relative to this file.
         """
-        if memory_dir is None:
-            # Default: ../../memory/inbox/ relative to the skill package
-            skill_root = Path(__file__).resolve().parent.parent
-            self._memory_dir = skill_root.parent.parent / "memory" / "inbox"
+        # Base directory used for resolving relative paths
+        skill_root = Path(__file__).resolve().parent.parent
+        repo_root = skill_root.parent.parent
+
+        env_memory_dir = os.getenv("DEEPREEDER_MEMORY_PATH", "").strip()
+
+        if memory_dir is not None:
+            target = Path(memory_dir)
+        elif env_memory_dir:
+            target = Path(env_memory_dir).expanduser()
         else:
-            self._memory_dir = Path(memory_dir)
+            # Default: ../../memory/inbox/ relative to the skill package
+            target = repo_root / "memory" / "inbox"
+
+        # Resolve relative paths against repository root for predictable behavior
+        if not target.is_absolute():
+            target = (repo_root / target).resolve()
+
+        self._memory_dir = target
 
         logger.info("StorageManager target directory: %s", self._memory_dir)
 

deepreader_skill/parsers/twitter.py

@@ -75,6 +75,8 @@ class TwitterParser(BaseParser):
 
     # Maximum Nitter instances to try for reply extraction.
     max_nitter_retries: int = 3
+    max_nitter_response_bytes: int = 3_000_000
+    _nitter_allowed_content_types = ("text/html", "application/xhtml+xml")
 
     def can_handle(self, url: str) -> bool:
         """Return ``True`` for twitter.com / x.com URLs."""
@@ -310,13 +312,40 @@ class TwitterParser(BaseParser):
 
     def _parse_nitter_page(self, original_url: str, nitter_url: str) -> ParseResult:
         """Fetch and parse a single Nitter page."""
-        resp = requests.get(
+        from ..core.utils import validate_external_url
+
+        with requests.get(
             nitter_url,
             headers=self._get_headers(),
             timeout=self.timeout,
-        )
-        resp.raise_for_status()
-        soup = BeautifulSoup(resp.text, "lxml")
+            allow_redirects=True,
+            stream=True,
+        ) as resp:
+            resp.raise_for_status()
+
+            final_url = resp.url or nitter_url
+            safe, reason = validate_external_url(final_url)
+            if not safe:
+                raise requests.RequestException(f"Blocked redirect target: {reason}")
+
+            content_type = (resp.headers.get("Content-Type") or "").lower()
+            if content_type and not any(ct in content_type for ct in self._nitter_allowed_content_types):
+                raise requests.RequestException(f"Unsupported content type: {content_type}")
+
+            body = bytearray()
+            for chunk in resp.iter_content(chunk_size=8192):
+                if not chunk:
+                    continue
+                body.extend(chunk)
+                if len(body) > self.max_nitter_response_bytes:
+                    raise requests.RequestException(
+                        f"Nitter response exceeds {self.max_nitter_response_bytes} bytes limit"
+                    )
+
+            encoding = resp.encoding or resp.apparent_encoding or "utf-8"
+            html = body.decode(encoding, errors="replace")
+
+        soup = BeautifulSoup(html, "lxml")
 
         tweet_div = soup.find("div", class_="tweet-content") or soup.find(
             "div", class_="main-tweet"