admin/OpenClaw-DeepReeder
1
0
Fork 0
mirror of https://github.com/astonysh/OpenClaw-DeepReeder.git synced 2026-08-14 00:57:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(parser): add Firecrawl fallback for paywalled and bot-protected content

Browse Source 
Integrates the Firecrawl API into GenericParser. If native HTTP requests fail
(e.g., 403 Forbidden) or returning nothing due to paywalls, we automatically
fallback to Firecrawl to extract Markdown content using pseudosearch engine features.

Requires FIRECRAWL_API_KEY environment variable to activate.
This commit is contained in:
Tony Li
2026-02-22 15:41:34 +01:00
parent b700e86ee4
commit 0c91f0068e
2 changed files with 93 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1
View File
@@ -216,6 +216,7 @@ DeepReader uses sensible defaults out of the box. Configuration can be customize
|----------|---------|-------------|
| `DEEPREEDER_MEMORY_PATH` | `../../memory/inbox/` | Where to save ingested content (absolute path, or relative to repo root) |
| `DEEPREEDER_LOG_LEVEL` | `INFO` | Logging verbosity (`DEBUG`, `INFO`, `WARNING`, `ERROR`) |
| `FIRECRAWL_API_KEY` | `""` | Optional. If set, used as a fallback to scrape paywalled/blocked content via [Firecrawl](https://firecrawl.dev) |
---
deepreader_skill/parsers/generic.py
+92 -23
View File
@@ -32,37 +32,49 @@ class GenericParser(BaseParser):
)
def parse(self, url: str) -> ParseResult:
"""Fetch *url* and extract the article body via trafilatura."""
"""Fetch *url* and extract the article body via trafilatura, with Firecrawl fallback."""
last_error = ""
try:
html = self._fetch_html(url)
if not html:
return ParseResult.failure(url, "Failed to download page content.")
last_error = "Failed to download page content."
else:
# ------------------------------------------------------------------
# Primary: trafilatura (best for article-style pages)
# ------------------------------------------------------------------
result = self._extract_with_trafilatura(url, html)
if result and result.success and result.content:
logger.info("Trafilatura extracted %d chars from %s", len(result.content), url)
return result
# ------------------------------------------------------------------
# Primary: trafilatura (best for article-style pages)
# ------------------------------------------------------------------
result = self._extract_with_trafilatura(url, html)
if result and result.success and result.content:
logger.info("Trafilatura extracted %d chars from %s", len(result.content), url)
return result
# ------------------------------------------------------------------
# Fallback: BeautifulSoup heuristic extraction
# ------------------------------------------------------------------
logger.info("Trafilatura returned empty, trying BeautifulSoup fallback for %s", url)
result = self._extract_with_beautifulsoup(url, html)
if result and result.success and result.content:
logger.info("BS4 extracted %d chars from %s", len(result.content), url)
return result
# ------------------------------------------------------------------
# Fallback: BeautifulSoup heuristic extraction
# ------------------------------------------------------------------
logger.info("Trafilatura returned empty, trying BeautifulSoup fallback for %s", url)
result = self._extract_with_beautifulsoup(url, html)
if result and result.success and result.content:
logger.info("BS4 extracted %d chars from %s", len(result.content), url)
return result
return ParseResult.failure(url, "Could not extract meaningful content from the page.")
last_error = "Could not extract meaningful content from the page."
except requests.RequestException as exc:
logger.error("HTTP error for %s: %s", url, exc)
return ParseResult.failure(url, f"HTTP request failed: {exc}")
logger.warning("HTTP error for %s: %s", url, exc)
last_error = f"HTTP request failed: {exc}"
except Exception as exc: # noqa: BLE001
logger.exception("Unexpected error parsing %s", url)
return ParseResult.failure(url, f"Unexpected error: {exc}")
logger.warning("Unexpected error parsing %s", url)
last_error = f"Unexpected error: {exc}"
# ------------------------------------------------------------------
# Final Fallback: Firecrawl (bypasses bot protections, paywalls)
# ------------------------------------------------------------------
logger.info("Local extraction failed. Trying Firecrawl fallback for %s", url)
fc_result = self._extract_with_firecrawl(url)
if fc_result and fc_result.success:
logger.info("Firecrawl successfully extracted content for %s", url)
return fc_result
return ParseResult.failure(url, f"All extraction methods failed. Local error: {last_error}")
# ------------------------------------------------------------------
# Internal helpers
@@ -188,3 +200,60 @@ class GenericParser(BaseParser):
soup = BeautifulSoup(html, "lxml")
tag = soup.find("title")
return tag.get_text(strip=True) if tag else ""
def _extract_with_firecrawl(self, url: str) -> ParseResult | None:
"""Fallback extraction using Firecrawl API to bypass blocks/paywalls."""
import os
api_key = os.getenv("FIRECRAWL_API_KEY")
if not api_key:
logger.info("FIRECRAWL_API_KEY not set. Skipping Firecrawl fallback.")
return None
logger.info("Sending %s to Firecrawl API...", url)
try:
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}
payload = {
"url": url,
"formats": ["markdown"]
}
response = requests.post(
"https://api.firecrawl.dev/v1/scrape",
json=payload,
headers=headers,
timeout=self.timeout * 2,
)
response.raise_for_status()
data = response.json()
if data.get("success"):
doc_data = data.get("data", {})
markdown = doc_data.get("markdown", "")
if not markdown:
return None
metadata = doc_data.get("metadata", {})
title = metadata.get("title", "")
author = metadata.get("author", "")
from ..core.utils import clean_text, generate_excerpt
content = clean_text(markdown)
if len(content) < 50:
return None
return ParseResult(
url=url,
title=title,
content=content,
author=author,
excerpt=generate_excerpt(content),
tags=["firecrawl-fallback"],
)
except Exception as exc:
logger.warning("Firecrawl fallback HTTP request failed: %s", exc)
return None