From a45f09dfe3859d80c3e63fc19a178bf8b7c14d2e Mon Sep 17 00:00:00 2001 From: Tony Li Date: Mon, 16 Feb 2026 14:09:53 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=A6=20Integrate=20FxTwitter=20API=20+?= =?UTF-8?q?=20multilingual=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twitter/X Parser: - Primary: FxTwitter API (zero-dep, structured JSON) - Regular tweets with engagement stats - X Articles (long-form content) with full text - Quoted tweets with nested content - Media extraction (images, videos, GIFs) - Fallback: Nitter instances (reply thread extraction) - Credits: inspired by x-tweet-fetcher by ythx-101 README: - Updated with FxTwitter integration details - Added credits section - Added multilingual versions: - 🇨🇳 中文 (README_zh.md) - 🇪🇸 Español (README_es.md) - 🇰🇷 한국어 (README_ko.md) - 🇯🇵 日本語 (README_ja.md) - 🇸🇦 العربية (README_ar.md) - 🇫🇷 Français (README_fr.md) --- README.md | 77 ++++-- README_ar.md | 170 ++++++++++++ README_es.md | 150 +++++++++++ README_fr.md | 150 +++++++++++ README_ja.md | 150 +++++++++++ README_ko.md | 150 +++++++++++ README_zh.md | 150 +++++++++++ deepreader_skill/parsers/twitter.py | 384 +++++++++++++++++++--------- 8 files changed, 1234 insertions(+), 147 deletions(-) create mode 100644 README_ar.md create mode 100644 README_es.md create mode 100644 README_fr.md create mode 100644 README_ja.md create mode 100644 README_ko.md create mode 100644 README_zh.md diff --git a/README.md b/README.md index 9eb6cb2..70705d2 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ DeepReeder intercepts URLs from user messages, scrapes content intelligently using specialized parsers, formats it into clean Markdown with YAML frontmatter, and saves it to the agent's long-term memory. +🌍 **Translations**: [中文](README_zh.md) · [Español](README_es.md) · [한국어](README_ko.md) · [日本語](README_ja.md) · [العربية](README_ar.md) · [Français](README_fr.md) + --- ## ✨ Features @@ -11,9 +13,25 @@ DeepReeder intercepts URLs from user messages, scrapes content intelligently usi | Parser | Sources | Method | |--------|---------|--------| | 🌐 **Generic** | Blogs, articles, docs | [Trafilatura](https://trafilatura.readthedocs.io/) with BeautifulSoup fallback | -| 🐦 **Twitter / X** | Tweets & threads | Nitter instance proxying | +| 🐦 **Twitter / X** | Tweets, threads, X Articles | **FxTwitter API** (primary) + Nitter (fallback) | | 🎬 **YouTube** | Video transcripts | [youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api) | +### 🐦 Twitter / X — Deep Integration + +Powered by [FxTwitter](https://github.com/FxEmbed/FxEmbed) API with Nitter fallback. Inspired by [x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher). + +| Content Type | Support | +|-------------|---------| +| Regular tweets | ✅ Full text + engagement stats | +| Long tweets (Twitter Blue) | ✅ Full text | +| X Articles (long-form) | ✅ Complete article text + word count | +| Quoted tweets | ✅ Nested content included | +| Media (images, video, GIF) | ✅ URLs extracted | +| Reply threads | ✅ Via Nitter fallback (first 5) | +| Engagement stats | ✅ ❤️ likes, 🔁 RTs, 👁️ views, 🔖 bookmarks | + +**No API keys. No login. No rate limits.** + ### Output Format Every piece of content is saved as a `.md` file with structured YAML frontmatter: @@ -62,6 +80,10 @@ from deepreader_skill import run result = run("Check out this article: https://example.com/blog/post") print(result) +# Process a tweet (uses FxTwitter API automatically) +result = run("Interesting thread: https://x.com/elonmusk/status/123456") +print(result) + # Process multiple URLs at once result = run(""" Here are some links: @@ -79,18 +101,18 @@ print(result) ✅ How to Build AI Agents Source: https://example.com/article - Saved to: memory/inbox/20260216_120000_how-to-build-ai-agents.md - Content: 3200 characters + Saved to: memory/inbox/2026-02-16_how-to-build-ai-agents.md + Content: 3,200 characters + +✅ Tweet by @elonmusk (Mon Feb 16 12:00:00 +0000 2026) + Source: https://x.com/elonmusk/status/123456 + Saved to: memory/inbox/2026-02-16_tweet-by-elonmusk.md + Content: 480 characters ✅ Rick Astley - Never Gonna Give You Up Source: https://youtube.com/watch?v=dQw4w9WgXcQ - Saved to: memory/inbox/20260216_120001_rick-astley-never-gonna.md - Content: 15000 characters - -✅ @user's tweet - Source: https://x.com/user/status/123456 - Saved to: memory/inbox/20260216_120002_user-tweet.md - Content: 280 characters + Saved to: memory/inbox/2026-02-16_rick-astley-never-gonna.md + Content: 15,000 characters ``` --- @@ -107,10 +129,21 @@ deepreader_skill/ │ ├── storage.py # Markdown file generation & saving │ └── utils.py # URL extraction & helper utilities └── parsers/ - ├── base.py # Abstract base parser & ParseResult model - ├── generic.py # Generic article/blog parser - ├── twitter.py # Twitter/X parser (via Nitter) - └── youtube.py # YouTube transcript parser + ├── base.py # Abstract base parser & ParseResult model + ├── generic.py # Generic article/blog parser (Trafilatura) + ├── twitter.py # Twitter/X parser (FxTwitter + Nitter) + └── youtube.py # YouTube transcript parser +``` + +### Twitter Parser Strategy + +``` +URL detected → FxTwitter API (primary) + ↓ success? → ✅ Rich result (stats, media, articles) + ↓ failure? + Nitter instances (fallback) + ↓ success? → ✅ Basic result + reply threads + ↓ failure? → ❌ Graceful error with diagnostics ``` --- @@ -126,6 +159,15 @@ DeepReeder uses sensible defaults out of the box. Configuration can be customize --- +## 🙏 Credits + +- **[FxTwitter / FixTweet](https://github.com/FxEmbed/FxEmbed)** — Public API for fetching Twitter/X content +- **[x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher)** — Inspiration for the FxTwitter integration approach +- **[Trafilatura](https://trafilatura.readthedocs.io/)** — Robust web content extraction +- **[youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api)** — YouTube transcript fetching + +--- + ## 🤝 Contributing Contributions are welcome! Feel free to: @@ -144,13 +186,6 @@ This project is licensed under the **MIT License** — see the [LICENSE](LICENSE --- -## 🔗 Links - -- **Repository**: [github.com/astonysh/OpenClaw-DeepReeder](https://github.com/astonysh/OpenClaw-DeepReeder) -- **Issues**: [github.com/astonysh/OpenClaw-DeepReeder/issues](https://github.com/astonysh/OpenClaw-DeepReeder/issues) - ---- -

Built with 🦞 by OpenClaw

diff --git a/README_ar.md b/README_ar.md new file mode 100644 index 0000000..4e18e17 --- /dev/null +++ b/README_ar.md @@ -0,0 +1,170 @@ +
+ +# 🦞 OpenClaw DeepReeder + +> **محرك استيعاب محتوى الويب الذاتي لوكلاء الذكاء الاصطناعي.** + +يعترض DeepReeder عناوين URL من رسائل المستخدم، ويستخرج المحتوى بذكاء باستخدام محللات متخصصة، ويحوله إلى Markdown نظيف مع بيانات YAML الوصفية، ويحفظه في الذاكرة طويلة المدى للوكيل. + +🌍 **الترجمات**: [English](README.md) · [中文](README_zh.md) · [Español](README_es.md) · [한국어](README_ko.md) · [日本語](README_ja.md) · [Français](README_fr.md) + +--- + +## ✨ الميزات + +| المحلل | المصادر | الطريقة | +|--------|---------|---------| +| 🌐 **عام** | مدونات، مقالات، وثائق | [Trafilatura](https://trafilatura.readthedocs.io/) مع BeautifulSoup احتياطي | +| 🐦 **Twitter / X** | تغريدات، سلاسل، مقالات X | **FxTwitter API** (رئيسي) + Nitter (احتياطي) | +| 🎬 **YouTube** | نصوص الفيديو | [youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api) | + +### 🐦 Twitter / X — تكامل عميق + +مدعوم بواجهة برمجة تطبيقات [FxTwitter](https://github.com/FxEmbed/FxEmbed). مستلهم من [x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher). + +| نوع المحتوى | الدعم | +|-------------|-------| +| التغريدات العادية | ✅ نص كامل + إحصائيات التفاعل | +| التغريدات الطويلة (Twitter Blue) | ✅ نص كامل | +| مقالات X (محتوى طويل) | ✅ مقال كامل + عدد الكلمات | +| التغريدات المقتبسة | ✅ محتوى متداخل مضمّن | +| الوسائط (صور، فيديو، GIF) | ✅ استخراج الروابط | +| سلاسل الردود | ✅ عبر Nitter الاحتياطي (أول 5) | +| إحصائيات التفاعل | ✅ ❤️ إعجابات، 🔁 إعادة تغريد، 👁️ مشاهدات، 🔖 إشارات مرجعية | + +**بدون مفاتيح API. بدون تسجيل دخول. بدون حدود للسرعة.** + +--- + +## 📦 التثبيت + +
+ +```bash +# استنساخ المستودع +git clone https://github.com/astonysh/OpenClaw-DeepReeder.git +cd OpenClaw-DeepReeder + +# إنشاء بيئة افتراضية +python3 -m venv .venv +source .venv/bin/activate + +# تثبيت التبعيات +pip install -e . +``` + +
+ +--- + +## 🚀 البداية السريعة + +
+ +```python +from deepreader_skill import run + +# معالجة عنوان URL واحد +result = run("اطلع على هذا المقال: https://example.com/blog/post") +print(result) + +# معالجة تغريدة (يستخدم FxTwitter API تلقائياً) +result = run("سلسلة مثيرة: https://x.com/elonmusk/status/123456") +print(result) + +# معالجة عناوين URL متعددة دفعة واحدة +result = run(""" + إليك بعض الروابط: + https://example.com/article + https://youtube.com/watch?v=dQw4w9WgXcQ + https://x.com/user/status/123456 +""") +print(result) +``` + +
+ +--- + +## 🏗️ الهيكل + +
+ +``` +deepreader_skill/ +├── __init__.py # نقطة الدخول — دالة run() +├── manifest.json # بيانات المهارة الوصفية وإعدادات المشغلات +├── requirements.txt # قائمة التبعيات +├── core/ +│ ├── router.py # منطق توجيه URL → المحلل +│ ├── storage.py # إنشاء وحفظ ملفات Markdown +│ └── utils.py # استخراج URL ودوال مساعدة +└── parsers/ + ├── base.py # المحلل الأساسي المجرد ونموذج ParseResult + ├── generic.py # محلل المقالات/المدونات العام + ├── twitter.py # محلل Twitter/X (FxTwitter + Nitter) + └── youtube.py # محلل نصوص YouTube +``` + +
+ +### استراتيجية محلل Twitter + +
+ +``` +اكتشاف URL → FxTwitter API (رئيسي) + ↓ نجاح؟ → ✅ نتيجة غنية (إحصائيات، وسائط، مقالات) + ↓ فشل؟ + مثيلات Nitter (احتياطي) + ↓ نجاح؟ → ✅ نتيجة أساسية + سلاسل الردود + ↓ فشل؟ → ❌ رسالة خطأ ودية مع التشخيص +``` + +
+ +--- + +## 🔧 الإعدادات + +يعمل DeepReeder مباشرة مع إعدادات افتراضية معقولة. يمكن تخصيص الإعدادات عبر متغيرات البيئة: + +| المتغير | الافتراضي | الوصف | +|---------|-----------|-------| +| `DEEPREEDER_MEMORY_PATH` | `../../memory/inbox/` | مسار حفظ المحتوى | +| `DEEPREEDER_LOG_LEVEL` | `INFO` | مستوى تفصيل السجلات | + +--- + +## 🙏 شكر وتقدير + +- **[FxTwitter / FixTweet](https://github.com/FxEmbed/FxEmbed)** — واجهة برمجة تطبيقات عامة لجلب محتوى Twitter/X +- **[x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher)** — مصدر إلهام لنهج تكامل FxTwitter +- **[Trafilatura](https://trafilatura.readthedocs.io/)** — استخراج محتوى الويب القوي +- **[youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api)** — جلب نصوص YouTube + +--- + +## 🤝 المساهمة + +المساهمات مرحب بها! يمكنك: + +1. عمل Fork للمستودع +2. إنشاء فرع ميزة (`git checkout -b feature/amazing-parser`) +3. التزام بتغييراتك (`git commit -m 'إضافة محلل رائع'`) +4. دفع الفرع (`git push origin feature/amazing-parser`) +5. فتح Pull Request + +--- + +## 📄 الترخيص + +هذا المشروع مرخص بموجب **رخصة MIT** — راجع ملف [LICENSE](LICENSE) للتفاصيل. + +--- + +

+ صنع بـ 🦞 بواسطة OpenClaw +

+ +
diff --git a/README_es.md b/README_es.md new file mode 100644 index 0000000..8b073cf --- /dev/null +++ b/README_es.md @@ -0,0 +1,150 @@ +# 🦞 OpenClaw DeepReeder + +> **Motor autónomo de ingestión de contenido web para agentes de IA.** + +DeepReeder intercepta URLs de los mensajes de usuario, extrae contenido de forma inteligente usando parsers especializados, lo formatea en Markdown limpio con metadatos YAML frontmatter, y lo guarda en la memoria a largo plazo del agente. + +🌍 **Traducciones**: [English](README.md) · [中文](README_zh.md) · [한국어](README_ko.md) · [日本語](README_ja.md) · [العربية](README_ar.md) · [Français](README_fr.md) + +--- + +## ✨ Características + +| Parser | Fuentes | Método | +|--------|---------|--------| +| 🌐 **Genérico** | Blogs, artículos, documentación | [Trafilatura](https://trafilatura.readthedocs.io/) con fallback BeautifulSoup | +| 🐦 **Twitter / X** | Tweets, hilos, X Articles | **FxTwitter API** (principal) + Nitter (fallback) | +| 🎬 **YouTube** | Transcripciones de vídeo | [youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api) | + +### 🐦 Twitter / X — Integración Profunda + +Impulsado por la API de [FxTwitter](https://github.com/FxEmbed/FxEmbed). Inspirado en [x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher). + +| Tipo de Contenido | Soporte | +|-------------------|---------| +| Tweets regulares | ✅ Texto completo + estadísticas | +| Tweets largos (Twitter Blue) | ✅ Texto completo | +| X Articles (contenido largo) | ✅ Artículo completo + recuento de palabras | +| Tweets citados | ✅ Contenido anidado incluido | +| Medios (imágenes, vídeo, GIF) | ✅ URLs extraídas | +| Hilos de respuestas | ✅ Vía Nitter fallback (primeras 5) | +| Estadísticas de interacción | ✅ ❤️ likes, 🔁 RTs, 👁️ vistas, 🔖 marcadores | + +**Sin claves API. Sin inicio de sesión. Sin límites de velocidad.** + +--- + +## 📦 Instalación + +```bash +# Clonar el repositorio +git clone https://github.com/astonysh/OpenClaw-DeepReeder.git +cd OpenClaw-DeepReeder + +# Crear entorno virtual +python3 -m venv .venv +source .venv/bin/activate + +# Instalar dependencias +pip install -e . +``` + +--- + +## 🚀 Inicio Rápido + +```python +from deepreader_skill import run + +# Procesar una sola URL +result = run("Mira este artículo: https://example.com/blog/post") +print(result) + +# Procesar un tweet (usa FxTwitter API automáticamente) +result = run("Hilo interesante: https://x.com/elonmusk/status/123456") +print(result) + +# Procesar múltiples URLs a la vez +result = run(""" + Aquí hay algunos enlaces: + https://example.com/article + https://youtube.com/watch?v=dQw4w9WgXcQ + https://x.com/user/status/123456 +""") +print(result) +``` + +--- + +## 🏗️ Arquitectura + +``` +deepreader_skill/ +├── __init__.py # Punto de entrada — función run() +├── manifest.json # Metadatos del skill y configuración de triggers +├── requirements.txt # Dependencias +├── core/ +│ ├── router.py # Lógica de enrutamiento URL → Parser +│ ├── storage.py # Generación y guardado de archivos Markdown +│ └── utils.py # Extracción de URLs y utilidades +└── parsers/ + ├── base.py # Parser base abstracto y modelo ParseResult + ├── generic.py # Parser genérico de artículos/blogs + ├── twitter.py # Parser Twitter/X (FxTwitter + Nitter) + └── youtube.py # Parser de transcripciones de YouTube +``` + +### Estrategia del Parser de Twitter + +``` +URL detectada → FxTwitter API (principal) + ↓ ¿éxito? → ✅ Resultado enriquecido (stats, media, artículos) + ↓ ¿fallo? + Instancias Nitter (fallback) + ↓ ¿éxito? → ✅ Resultado básico + hilos de respuestas + ↓ ¿fallo? → ❌ Error descriptivo con diagnóstico +``` + +--- + +## 🔧 Configuración + +DeepReeder funciona listo para usar con valores predeterminados sensatos. Se puede personalizar mediante variables de entorno: + +| Variable | Predeterminado | Descripción | +|----------|---------------|-------------| +| `DEEPREEDER_MEMORY_PATH` | `../../memory/inbox/` | Ruta para guardar contenido | +| `DEEPREEDER_LOG_LEVEL` | `INFO` | Nivel de detalle del registro | + +--- + +## 🙏 Créditos + +- **[FxTwitter / FixTweet](https://github.com/FxEmbed/FxEmbed)** — API pública para obtener contenido de Twitter/X +- **[x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher)** — Inspiración para la integración de FxTwitter +- **[Trafilatura](https://trafilatura.readthedocs.io/)** — Extracción robusta de contenido web +- **[youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api)** — Obtención de transcripciones de YouTube + +--- + +## 🤝 Contribuir + +¡Las contribuciones son bienvenidas! + +1. Haz fork del repositorio +2. Crea una rama de funcionalidad (`git checkout -b feature/parser-increible`) +3. Haz commit de tus cambios (`git commit -m 'Agregar parser increíble'`) +4. Haz push a la rama (`git push origin feature/parser-increible`) +5. Abre un Pull Request + +--- + +## 📄 Licencia + +Este proyecto está licenciado bajo la **Licencia MIT** — consulta el archivo [LICENSE](LICENSE) para más detalles. + +--- + +

+ Construido con 🦞 por OpenClaw +

diff --git a/README_fr.md b/README_fr.md new file mode 100644 index 0000000..5f21311 --- /dev/null +++ b/README_fr.md @@ -0,0 +1,150 @@ +# 🦞 OpenClaw DeepReeder + +> **Moteur autonome d'ingestion de contenu web pour agents IA.** + +DeepReeder intercepte les URLs des messages utilisateur, extrait le contenu intelligemment à l'aide de parsers spécialisés, le formate en Markdown propre avec des métadonnées YAML frontmatter, et le sauvegarde dans la mémoire à long terme de l'agent. + +🌍 **Traductions** : [English](README.md) · [中文](README_zh.md) · [Español](README_es.md) · [한국어](README_ko.md) · [日本語](README_ja.md) · [العربية](README_ar.md) + +--- + +## ✨ Fonctionnalités + +| Parser | Sources | Méthode | +|--------|---------|---------| +| 🌐 **Générique** | Blogs, articles, documentation | [Trafilatura](https://trafilatura.readthedocs.io/) avec fallback BeautifulSoup | +| 🐦 **Twitter / X** | Tweets, fils, X Articles | **FxTwitter API** (principal) + Nitter (fallback) | +| 🎬 **YouTube** | Transcriptions vidéo | [youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api) | + +### 🐦 Twitter / X — Intégration Approfondie + +Propulsé par l'API [FxTwitter](https://github.com/FxEmbed/FxEmbed). Inspiré par [x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher). + +| Type de Contenu | Support | +|----------------|---------| +| Tweets classiques | ✅ Texte complet + statistiques d'engagement | +| Tweets longs (Twitter Blue) | ✅ Texte complet | +| X Articles (contenu long) | ✅ Article complet + nombre de mots | +| Tweets cités | ✅ Contenu imbriqué inclus | +| Médias (images, vidéo, GIF) | ✅ URLs extraites | +| Fils de réponses | ✅ Via Nitter fallback (5 premières) | +| Statistiques d'engagement | ✅ ❤️ likes, 🔁 RTs, 👁️ vues, 🔖 signets | + +**Sans clé API. Sans connexion. Sans limite de débit.** + +--- + +## 📦 Installation + +```bash +# Cloner le dépôt +git clone https://github.com/astonysh/OpenClaw-DeepReeder.git +cd OpenClaw-DeepReeder + +# Créer un environnement virtuel +python3 -m venv .venv +source .venv/bin/activate + +# Installer les dépendances +pip install -e . +``` + +--- + +## 🚀 Démarrage Rapide + +```python +from deepreader_skill import run + +# Traiter une seule URL +result = run("Regarde cet article : https://example.com/blog/post") +print(result) + +# Traiter un tweet (utilise automatiquement l'API FxTwitter) +result = run("Fil intéressant : https://x.com/elonmusk/status/123456") +print(result) + +# Traiter plusieurs URLs en une fois +result = run(""" + Voici quelques liens : + https://example.com/article + https://youtube.com/watch?v=dQw4w9WgXcQ + https://x.com/user/status/123456 +""") +print(result) +``` + +--- + +## 🏗️ Architecture + +``` +deepreader_skill/ +├── __init__.py # Point d'entrée — fonction run() +├── manifest.json # Métadonnées du skill et configuration des triggers +├── requirements.txt # Liste des dépendances +├── core/ +│ ├── router.py # Logique de routage URL → Parser +│ ├── storage.py # Génération et sauvegarde des fichiers Markdown +│ └── utils.py # Extraction d'URLs et fonctions utilitaires +└── parsers/ + ├── base.py # Parser de base abstrait et modèle ParseResult + ├── generic.py # Parser générique d'articles/blogs + ├── twitter.py # Parser Twitter/X (FxTwitter + Nitter) + └── youtube.py # Parser de transcriptions YouTube +``` + +### Stratégie du Parser Twitter + +``` +URL détectée → FxTwitter API (principal) + ↓ succès ? → ✅ Résultat enrichi (stats, médias, articles) + ↓ échec ? + Instances Nitter (fallback) + ↓ succès ? → ✅ Résultat basique + fils de réponses + ↓ échec ? → ❌ Message d'erreur explicatif avec diagnostic +``` + +--- + +## 🔧 Configuration + +DeepReeder fonctionne immédiatement avec des valeurs par défaut raisonnables. La configuration peut être personnalisée via des variables d'environnement : + +| Variable | Par défaut | Description | +|----------|-----------|-------------| +| `DEEPREEDER_MEMORY_PATH` | `../../memory/inbox/` | Chemin de sauvegarde du contenu | +| `DEEPREEDER_LOG_LEVEL` | `INFO` | Niveau de verbosité des journaux | + +--- + +## 🙏 Remerciements + +- **[FxTwitter / FixTweet](https://github.com/FxEmbed/FxEmbed)** — API publique pour récupérer le contenu Twitter/X +- **[x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher)** — Inspiration pour l'approche d'intégration FxTwitter +- **[Trafilatura](https://trafilatura.readthedocs.io/)** — Extraction robuste de contenu web +- **[youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api)** — Récupération de transcriptions YouTube + +--- + +## 🤝 Contribuer + +Les contributions sont les bienvenues ! + +1. Forkez le dépôt +2. Créez une branche de fonctionnalité (`git checkout -b feature/parser-genial`) +3. Commitez vos changements (`git commit -m 'Ajouter un parser génial'`) +4. Poussez la branche (`git push origin feature/parser-genial`) +5. Ouvrez une Pull Request + +--- + +## 📄 Licence + +Ce projet est sous licence **MIT** — consultez le fichier [LICENSE](LICENSE) pour plus de détails. + +--- + +

+ Construit avec 🦞 par OpenClaw +

diff --git a/README_ja.md b/README_ja.md new file mode 100644 index 0000000..34d24a6 --- /dev/null +++ b/README_ja.md @@ -0,0 +1,150 @@ +# 🦞 OpenClaw DeepReeder + +> **AIエージェント向け自律型Webコンテンツ取り込みエンジン。** + +DeepReederはユーザーメッセージからURLを自動検出し、専用パーサーを使ってコンテンツをインテリジェントにスクレイピングし、YAMLフロントマター付きのクリーンなMarkdownに変換して、エージェントの長期メモリに保存します。 + +🌍 **翻訳**: [English](README.md) · [中文](README_zh.md) · [Español](README_es.md) · [한국어](README_ko.md) · [العربية](README_ar.md) · [Français](README_fr.md) + +--- + +## ✨ 機能 + +| パーサー | ソース | 方法 | +|---------|--------|------| +| 🌐 **汎用** | ブログ、記事、ドキュメント | [Trafilatura](https://trafilatura.readthedocs.io/) + BeautifulSoup フォールバック | +| 🐦 **Twitter / X** | ツイート、スレッド、Xアーティクル | **FxTwitter API**（メイン）+ Nitter（フォールバック） | +| 🎬 **YouTube** | 動画字幕 | [youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api) | + +### 🐦 Twitter / X — ディープインテグレーション + +[FxTwitter](https://github.com/FxEmbed/FxEmbed) APIベース。[x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher)にインスパイアされました。 + +| コンテンツタイプ | サポート | +|----------------|---------| +| 通常のツイート | ✅ 全文 + エンゲージメント統計 | +| 長文ツイート（Twitter Blue） | ✅ 全文 | +| Xアーティクル（長文コンテンツ） | ✅ 完全な記事 + 単語数 | +| 引用ツイート | ✅ ネストされたコンテンツ含む | +| メディア（画像、動画、GIF） | ✅ URL抽出 | +| リプライスレッド | ✅ Nitterフォールバック経由（最初の5件） | +| エンゲージメント統計 | ✅ ❤️ いいね、🔁 RT、👁️ 閲覧、🔖 ブックマーク | + +**APIキー不要。ログイン不要。レート制限なし。** + +--- + +## 📦 インストール + +```bash +# リポジトリをクローン +git clone https://github.com/astonysh/OpenClaw-DeepReeder.git +cd OpenClaw-DeepReeder + +# 仮想環境を作成 +python3 -m venv .venv +source .venv/bin/activate + +# 依存関係をインストール +pip install -e . +``` + +--- + +## 🚀 クイックスタート + +```python +from deepreader_skill import run + +# 単一URLを処理 +result = run("この記事をチェック: https://example.com/blog/post") +print(result) + +# ツイートを処理（自動的にFxTwitter APIを使用） +result = run("興味深いスレッド: https://x.com/elonmusk/status/123456") +print(result) + +# 複数のURLを一括処理 +result = run(""" + いくつかのリンクがあります: + https://example.com/article + https://youtube.com/watch?v=dQw4w9WgXcQ + https://x.com/user/status/123456 +""") +print(result) +``` + +--- + +## 🏗️ アーキテクチャ + +``` +deepreader_skill/ +├── __init__.py # エントリポイント — run() 関数 +├── manifest.json # スキルメタデータとトリガー設定 +├── requirements.txt # 依存関係リスト +├── core/ +│ ├── router.py # URL → パーサールーティングロジック +│ ├── storage.py # Markdownファイル生成・保存 +│ └── utils.py # URL抽出とユーティリティ関数 +└── parsers/ + ├── base.py # 抽象基底パーサーとParseResultモデル + ├── generic.py # 汎用記事/ブログパーサー + ├── twitter.py # Twitter/Xパーサー（FxTwitter + Nitter） + └── youtube.py # YouTube字幕パーサー +``` + +### Twitterパーサー戦略 + +``` +URL検出 → FxTwitter API（メイン） + ↓ 成功？ → ✅ リッチな結果（統計、メディア、記事） + ↓ 失敗？ + Nitterインスタンス（フォールバック） + ↓ 成功？ → ✅ 基本結果 + リプライスレッド + ↓ 失敗？ → ❌ 診断付きエラーメッセージ +``` + +--- + +## 🔧 設定 + +DeepReederはデフォルト設定ですぐに使えます。環境変数でカスタマイズ可能です： + +| 変数 | デフォルト | 説明 | +|------|-----------|------| +| `DEEPREEDER_MEMORY_PATH` | `../../memory/inbox/` | コンテンツの保存先 | +| `DEEPREEDER_LOG_LEVEL` | `INFO` | ログの詳細レベル | + +--- + +## 🙏 クレジット + +- **[FxTwitter / FixTweet](https://github.com/FxEmbed/FxEmbed)** — Twitter/Xコンテンツ取得用パブリックAPI +- **[x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher)** — FxTwitter統合アプローチのインスピレーション +- **[Trafilatura](https://trafilatura.readthedocs.io/)** — 高性能Webコンテンツ抽出 +- **[youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api)** — YouTube字幕取得 + +--- + +## 🤝 コントリビューション + +コントリビューション歓迎です！ + +1. リポジトリをForkします +2. フィーチャーブランチを作成 (`git checkout -b feature/amazing-parser`) +3. 変更をコミット (`git commit -m '素晴らしいパーサーを追加'`) +4. ブランチをプッシュ (`git push origin feature/amazing-parser`) +5. Pull Requestを開きます + +--- + +## 📄 ライセンス + +このプロジェクトは**MITライセンス**の下でライセンスされています — 詳細は[LICENSE](LICENSE)ファイルをご覧ください。 + +--- + +

+ OpenClawが🦞で構築 +

diff --git a/README_ko.md b/README_ko.md new file mode 100644 index 0000000..1582a4d --- /dev/null +++ b/README_ko.md @@ -0,0 +1,150 @@ +# 🦞 OpenClaw DeepReeder + +> **AI 에이전트를 위한 자율 웹 콘텐츠 수집 엔진.** + +DeepReeder는 사용자 메시지에서 URL을 자동으로 감지하고, 전문 파서를 사용하여 콘텐츠를 지능적으로 스크래핑하며, YAML 프론트매터가 포함된 깔끔한 Markdown으로 변환하여 에이전트의 장기 메모리에 저장합니다. + +🌍 **번역**: [English](README.md) · [中文](README_zh.md) · [Español](README_es.md) · [日本語](README_ja.md) · [العربية](README_ar.md) · [Français](README_fr.md) + +--- + +## ✨ 기능 + +| 파서 | 소스 | 방법 | +|------|------|------| +| 🌐 **범용** | 블로그, 기사, 문서 | [Trafilatura](https://trafilatura.readthedocs.io/) + BeautifulSoup 대체 | +| 🐦 **Twitter / X** | 트윗, 스레드, X 아티클 | **FxTwitter API** (주력) + Nitter (대체) | +| 🎬 **YouTube** | 동영상 자막 | [youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api) | + +### 🐦 Twitter / X — 심층 통합 + +[FxTwitter](https://github.com/FxEmbed/FxEmbed) API 기반. [x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher)에서 영감을 받았습니다. + +| 콘텐츠 유형 | 지원 | +|------------|------| +| 일반 트윗 | ✅ 전체 텍스트 + 참여 통계 | +| 긴 트윗 (Twitter Blue) | ✅ 전체 텍스트 | +| X 아티클 (장문) | ✅ 전체 기사 + 단어 수 | +| 인용 트윗 | ✅ 중첩 콘텐츠 포함 | +| 미디어 (이미지, 동영상, GIF) | ✅ URL 추출 | +| 답글 스레드 | ✅ Nitter 대체를 통해 (처음 5개) | +| 참여 통계 | ✅ ❤️ 좋아요, 🔁 리트윗, 👁️ 조회, 🔖 북마크 | + +**API 키 불필요. 로그인 불필요. 속도 제한 없음.** + +--- + +## 📦 설치 + +```bash +# 저장소 클론 +git clone https://github.com/astonysh/OpenClaw-DeepReeder.git +cd OpenClaw-DeepReeder + +# 가상 환경 생성 +python3 -m venv .venv +source .venv/bin/activate + +# 의존성 설치 +pip install -e . +``` + +--- + +## 🚀 빠른 시작 + +```python +from deepreader_skill import run + +# 단일 URL 처리 +result = run("이 기사를 확인하세요: https://example.com/blog/post") +print(result) + +# 트윗 처리 (자동으로 FxTwitter API 사용) +result = run("흥미로운 스레드: https://x.com/elonmusk/status/123456") +print(result) + +# 여러 URL 한번에 처리 +result = run(""" + 여기 몇 가지 링크가 있습니다: + https://example.com/article + https://youtube.com/watch?v=dQw4w9WgXcQ + https://x.com/user/status/123456 +""") +print(result) +``` + +--- + +## 🏗️ 아키텍처 + +``` +deepreader_skill/ +├── __init__.py # 진입점 — run() 함수 +├── manifest.json # 스킬 메타데이터 및 트리거 설정 +├── requirements.txt # 의존성 목록 +├── core/ +│ ├── router.py # URL → 파서 라우팅 로직 +│ ├── storage.py # Markdown 파일 생성 및 저장 +│ └── utils.py # URL 추출 및 유틸리티 함수 +└── parsers/ + ├── base.py # 추상 기본 파서 및 ParseResult 모델 + ├── generic.py # 범용 기사/블로그 파서 + ├── twitter.py # Twitter/X 파서 (FxTwitter + Nitter) + └── youtube.py # YouTube 자막 파서 +``` + +### Twitter 파서 전략 + +``` +URL 감지 → FxTwitter API (주력) + ↓ 성공? → ✅ 풍부한 결과 (통계, 미디어, 기사) + ↓ 실패? + Nitter 인스턴스 (대체) + ↓ 성공? → ✅ 기본 결과 + 답글 스레드 + ↓ 실패? → ❌ 친절한 오류 메시지 및 진단 +``` + +--- + +## 🔧 설정 + +DeepReeder는 합리적인 기본값으로 바로 사용할 수 있습니다. 환경 변수로 설정을 변경할 수 있습니다: + +| 변수 | 기본값 | 설명 | +|------|--------|------| +| `DEEPREEDER_MEMORY_PATH` | `../../memory/inbox/` | 콘텐츠 저장 경로 | +| `DEEPREEDER_LOG_LEVEL` | `INFO` | 로깅 상세 수준 | + +--- + +## 🙏 크레딧 + +- **[FxTwitter / FixTweet](https://github.com/FxEmbed/FxEmbed)** — Twitter/X 콘텐츠 가져오기용 공개 API +- **[x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher)** — FxTwitter 통합 접근 방식에 영감 +- **[Trafilatura](https://trafilatura.readthedocs.io/)** — 강력한 웹 콘텐츠 추출 +- **[youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api)** — YouTube 자막 가져오기 + +--- + +## 🤝 기여 + +기여를 환영합니다! + +1. 저장소를 Fork 합니다 +2. 기능 브랜치를 생성합니다 (`git checkout -b feature/amazing-parser`) +3. 변경 사항을 커밋합니다 (`git commit -m '놀라운 파서 추가'`) +4. 브랜치를 푸시합니다 (`git push origin feature/amazing-parser`) +5. Pull Request를 엽니다 + +--- + +## 📄 라이선스 + +이 프로젝트는 **MIT 라이선스**에 따라 라이선스가 부여됩니다 — 자세한 내용은 [LICENSE](LICENSE) 파일을 참조하세요. + +--- + +

+ OpenClaw에서 🦞 로 만들었습니다 +

diff --git a/README_zh.md b/README_zh.md new file mode 100644 index 0000000..bd59e5b --- /dev/null +++ b/README_zh.md @@ -0,0 +1,150 @@ +# 🦞 OpenClaw DeepReeder + +> **面向 AI 智能体的自主网页内容摄取引擎。** + +DeepReeder 自动拦截用户消息中的 URL，使用专用解析器智能抓取内容，将其格式化为带有 YAML 前置信息的干净 Markdown，并保存到智能体的长期记忆中。 + +🌍 **其他语言**: [English](README.md) · [Español](README_es.md) · [한국어](README_ko.md) · [日本語](README_ja.md) · [العربية](README_ar.md) · [Français](README_fr.md) + +--- + +## ✨ 功能特性 + +| 解析器 | 来源 | 方法 | +|--------|------|------| +| 🌐 **通用** | 博客、文章、文档 | [Trafilatura](https://trafilatura.readthedocs.io/) + BeautifulSoup 备用方案 | +| 🐦 **Twitter / X** | 推文、线程、X 文章 | **FxTwitter API**（主力） + Nitter（备用） | +| 🎬 **YouTube** | 视频字幕 | [youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api) | + +### 🐦 Twitter / X — 深度整合 + +基于 [FxTwitter](https://github.com/FxEmbed/FxEmbed) API，灵感来自 [x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher)。 + +| 内容类型 | 支持 | +|---------|------| +| 普通推文 | ✅ 全文 + 互动数据 | +| 长推文（Twitter Blue） | ✅ 完整文本 | +| X 文章（长文） | ✅ 完整文章 + 字数统计 | +| 引用推文 | ✅ 嵌套内容 | +| 媒体（图片、视频、GIF） | ✅ URL 提取 | +| 回复线程 | ✅ 通过 Nitter 备用方案（前5条） | +| 互动数据 | ✅ ❤️ 喜欢、🔁 转发、👁️ 浏览、🔖 书签 | + +**无需 API 密钥。无需登录。无速率限制。** + +--- + +## 📦 安装 + +```bash +# 克隆仓库 +git clone https://github.com/astonysh/OpenClaw-DeepReeder.git +cd OpenClaw-DeepReeder + +# 创建虚拟环境 +python3 -m venv .venv +source .venv/bin/activate + +# 安装依赖 +pip install -e . +``` + +--- + +## 🚀 快速开始 + +```python +from deepreader_skill import run + +# 处理单个 URL +result = run("看看这篇文章: https://example.com/blog/post") +print(result) + +# 处理推文（自动使用 FxTwitter API） +result = run("有趣的推文: https://x.com/elonmusk/status/123456") +print(result) + +# 批量处理多个 URL +result = run(""" + 这里有一些链接: + https://example.com/article + https://youtube.com/watch?v=dQw4w9WgXcQ + https://x.com/user/status/123456 +""") +print(result) +``` + +--- + +## 🏗️ 架构 + +``` +deepreader_skill/ +├── __init__.py # 入口 — run() 函数 +├── manifest.json # 技能元数据与触发配置 +├── requirements.txt # 依赖列表 +├── core/ +│ ├── router.py # URL → 解析器路由逻辑 +│ ├── storage.py # Markdown 文件生成与保存 +│ └── utils.py # URL 提取和工具函数 +└── parsers/ + ├── base.py # 抽象基类与 ParseResult 模型 + ├── generic.py # 通用文章/博客解析器 + ├── twitter.py # Twitter/X 解析器（FxTwitter + Nitter） + └── youtube.py # YouTube 字幕解析器 +``` + +### Twitter 解析器策略 + +``` +检测到 URL → FxTwitter API（主力） + ↓ 成功？ → ✅ 丰富结果（数据、媒体、文章） + ↓ 失败？ + Nitter 实例（备用） + ↓ 成功？ → ✅ 基础结果 + 回复线程 + ↓ 失败？ → ❌ 友好的错误信息与诊断 +``` + +--- + +## 🔧 配置 + +DeepReeder 开箱即用，使用合理的默认值。可通过环境变量自定义配置： + +| 变量 | 默认值 | 说明 | +|------|--------|------| +| `DEEPREEDER_MEMORY_PATH` | `../../memory/inbox/` | 保存内容的路径 | +| `DEEPREEDER_LOG_LEVEL` | `INFO` | 日志级别 | + +--- + +## 🙏 致谢 + +- **[FxTwitter / FixTweet](https://github.com/FxEmbed/FxEmbed)** — 获取 Twitter/X 内容的公共 API +- **[x-tweet-fetcher](https://github.com/ythx-101/x-tweet-fetcher)** — FxTwitter 整合方案的灵感来源 +- **[Trafilatura](https://trafilatura.readthedocs.io/)** — 强大的网页内容提取工具 +- **[youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api)** — YouTube 字幕获取 + +--- + +## 🤝 贡献 + +欢迎贡献！您可以： + +1. Fork 仓库 +2. 创建功能分支 (`git checkout -b feature/amazing-parser`) +3. 提交更改 (`git commit -m '添加新解析器'`) +4. 推送分支 (`git push origin feature/amazing-parser`) +5. 提交 Pull Request + +--- + +## 📄 许可证 + +本项目基于 **MIT 许可证** — 查看 [LICENSE](LICENSE) 文件获取详情。 + +--- + +

+ 由 OpenClaw 用 🦞 构建 +

diff --git a/deepreader_skill/parsers/twitter.py b/deepreader_skill/parsers/twitter.py index e55b454..ce83a87 100644 --- a/deepreader_skill/parsers/twitter.py +++ b/deepreader_skill/parsers/twitter.py @@ -1,32 +1,42 @@ """ DeepReader Skill - Twitter / X Parser ======================================= -Strategy-pattern implementation for reading tweets: +Hybrid strategy for reading tweets: -1. **Primary**: Rotate through public Nitter instances to fetch tweet - content without any API keys. -2. **Fallback**: Gracefully degrade with informative guidance on how to - plug in a scraping service (ZenRows, ScrapingBee) or browser cookies. +1. **Primary**: FxTwitter public API — zero dependencies, reliable, + supports regular tweets, long tweets, X Articles, quoted tweets, + media, and engagement stats. +2. **Fallback**: Rotate through public Nitter instances to fetch tweet + content (especially useful for fetching reply threads). -Why Nitter? ------------ -Twitter's official API is paywalled and rate-limited. Nitter is an -open-source alternative frontend that serves tweets as plain HTML, -making extraction trivial. However, public instances may go down, -so we rotate through several and retry. +Why FxTwitter first? +--------------------- +FxTwitter (api.fxtwitter.com) is a public, maintenance-free API that +returns structured JSON. It is far more reliable than Nitter instances +which frequently go offline. It also returns rich metadata (stats, +media, articles) that Nitter does not expose. -Extending with a paid scraping service ---------------------------------------- -If all Nitter instances fail, you can integrate a proxy/rendering -service. See the ``_fallback_scrape`` method for detailed guidance -on where to plug in ZenRows or browser cookies. +Nitter as fallback +------------------- +Nitter HTML scraping is retained as a secondary strategy, primarily +because it can extract reply threads (first N replies), which FxTwitter +does not provide. + +Credits +-------- +FxTwitter integration inspired by `x-tweet-fetcher` by ythx-101: +https://github.com/ythx-101/x-tweet-fetcher """ from __future__ import annotations +import json import logging import random import re +import time +import urllib.error +import urllib.request from urllib.parse import urlparse import requests @@ -39,7 +49,7 @@ logger = logging.getLogger("deepreader.parsers.twitter") # --------------------------------------------------------------------------- # Known public Nitter instances (community-maintained) -# Update this list periodically – instances come and go. +# Used only as fallback for reply-thread extraction. # --------------------------------------------------------------------------- NITTER_INSTANCES: list[str] = [ "https://nitter.privacydev.net", @@ -54,13 +64,17 @@ NITTER_INSTANCES: list[str] = [ class TwitterParser(BaseParser): - """Parse tweets from Twitter / X via Nitter relay instances.""" + """Parse tweets from Twitter / X. + + Primary: FxTwitter API (structured JSON, zero deps, rich metadata). + Fallback: Nitter HTML scraping (reply threads). + """ name = "twitter" - timeout = 20 + timeout = 30 - # Maximum number of Nitter instances to try before giving up. - max_retries: int = 4 + # Maximum Nitter instances to try for reply extraction. + max_nitter_retries: int = 3 def can_handle(self, url: str) -> bool: """Return ``True`` for twitter.com / x.com URLs.""" @@ -68,49 +82,233 @@ class TwitterParser(BaseParser): return is_twitter_url(url) def parse(self, url: str) -> ParseResult: - """Attempt to read a tweet via Nitter, with graceful fallbacks.""" - tweet_path = self._extract_tweet_path(url) - if not tweet_path: + """Attempt to read a tweet — FxTwitter first, Nitter fallback.""" + tweet_info = self._extract_tweet_info(url) + if not tweet_info: return ParseResult.failure( url, "Could not extract a valid tweet path from this URL. " "Expected format: https://twitter.com/user/status/123456", ) - # Shuffle instances to spread load and improve resilience. + username, tweet_id = tweet_info + + # ----- Strategy 1: FxTwitter API (primary) ----- + result = self._parse_fxtwitter(url, username, tweet_id) + if result.success: + return result + + logger.warning( + "FxTwitter failed for %s, trying Nitter fallback: %s", + url, result.error, + ) + + # ----- Strategy 2: Nitter HTML scraping (fallback) ----- + tweet_path = f"{username}/status/{tweet_id}" + nitter_result = self._parse_nitter_fallback(url, tweet_path) + if nitter_result.success: + return nitter_result + + # ----- Both strategies failed ----- + return ParseResult.failure( + url, + f"⚠️ All strategies failed for this tweet.\n" + f"FxTwitter error: {result.error}\n" + f"Nitter error: {nitter_result.error}\n\n" + f"The tweet URL was: {url}\n" + f"This may be a deleted/private tweet, or a temporary service outage.", + ) + + # ================================================================== + # FxTwitter API — Primary Strategy + # ================================================================== + + def _parse_fxtwitter( + self, original_url: str, username: str, tweet_id: str, + ) -> ParseResult: + """Fetch tweet via the FxTwitter public JSON API.""" + api_url = f"https://api.fxtwitter.com/{username}/status/{tweet_id}" + + max_attempts = 2 + last_error = "" + + for attempt in range(max_attempts): + try: + req = urllib.request.Request( + api_url, + headers={"User-Agent": "Mozilla/5.0"}, + ) + with urllib.request.urlopen(req, timeout=self.timeout) as resp: + data = json.loads(resp.read().decode()) + + if data.get("code") != 200: + last_error = ( + f"FxTwitter returned code {data.get('code')}: " + f"{data.get('message', 'Unknown')}" + ) + return ParseResult.failure(original_url, last_error) + + return self._build_result_from_fxtwitter(original_url, data) + + except urllib.error.HTTPError as exc: + last_error = f"HTTP {exc.code}: {exc.reason}" + return ParseResult.failure(original_url, last_error) + + except urllib.error.URLError: + last_error = "Network error: failed to reach FxTwitter API" + if attempt < max_attempts - 1: + time.sleep(1) + continue + + except Exception as exc: # noqa: BLE001 + last_error = f"Unexpected error: {exc}" + logger.warning("FxTwitter unexpected error: %s", exc) + break + + return ParseResult.failure(original_url, last_error) + + def _build_result_from_fxtwitter( + self, original_url: str, data: dict, + ) -> ParseResult: + """Transform FxTwitter JSON response into a ParseResult.""" + from ..core.utils import clean_text, generate_excerpt + + tweet = data["tweet"] + author = tweet.get("author", {}).get("name", "") + screen_name = tweet.get("author", {}).get("screen_name", "") + created_at = tweet.get("created_at", "") + tweet_text = tweet.get("text", "") + is_article = bool(tweet.get("article")) + is_note = tweet.get("is_note_tweet", False) + + # --- Engagement stats --- + stats_line = ( + f"❤️ {tweet.get('likes', 0):,} " + f"🔁 {tweet.get('retweets', 0):,} " + f"🔖 {tweet.get('bookmarks', 0):,} " + f"👁️ {tweet.get('views', 0):,} " + f"💬 {tweet.get('replies', 0):,}" + ) + + content_parts: list[str] = [] + + # --- X Article (long-form content) --- + if is_article: + article = tweet["article"] + article_title = article.get("title", "") + article_blocks = article.get("content", {}).get("blocks", []) + article_text = "\n\n".join( + b.get("text", "") for b in article_blocks if b.get("text", "") + ) + word_count = len(article_text.split()) + + title = f"📝 {article_title}" if article_title else f"X Article by @{screen_name}" + + content_parts.append(f"# {article_title}\n") + content_parts.append(f"**By @{screen_name}** ({author}) · {created_at}\n") + content_parts.append(f"📊 {stats_line}\n") + content_parts.append(f"📐 {word_count:,} words\n") + content_parts.append("---\n") + content_parts.append(article_text) + + else: + # --- Regular tweet / note tweet --- + title = f"Tweet by @{screen_name}" + if created_at: + title += f" ({created_at})" + + content_parts.append(f"**@{screen_name}** ({author})\n") + content_parts.append(f"🕐 {created_at}\n") + content_parts.append(f"📊 {stats_line}\n") + content_parts.append("---\n") + content_parts.append(tweet_text) + + if is_note: + content_parts.append("\n\n> 📝 *This is a Note Tweet (long-form)*") + + # --- Quoted tweet --- + quote = tweet.get("quote") + if quote: + qt_author = quote.get("author", {}).get("screen_name", "unknown") + qt_text = quote.get("text", "") + content_parts.append("\n\n---\n### 🔁 Quoted Tweet\n") + content_parts.append(f"> **@{qt_author}**: {qt_text}\n") + qt_stats = ( + f"> ❤️ {quote.get('likes', 0):,} " + f"🔁 {quote.get('retweets', 0):,} " + f"👁️ {quote.get('views', 0):,}" + ) + content_parts.append(qt_stats) + + # --- Media --- + media = tweet.get("media", {}) + all_media = media.get("all", []) + if all_media: + content_parts.append("\n\n---\n### 🖼️ Media\n") + for i, item in enumerate(all_media, 1): + media_type = item.get("type", "unknown") + media_url = item.get("url", "") + if media_type == "photo": + content_parts.append(f"![Image {i}]({media_url})\n") + elif media_type == "video": + content_parts.append(f"🎬 Video: [{media_url}]({media_url})\n") + elif media_type == "gif": + content_parts.append(f"🎞️ GIF: [{media_url}]({media_url})\n") + + full_content = clean_text("\n".join(content_parts)) + + tags = ["twitter"] + if is_article: + tags.append("x-article") + if is_note: + tags.append("note-tweet") + if quote: + tags.append("quote-tweet") + if all_media: + tags.append("has-media") + + return ParseResult( + url=original_url, + title=title, + content=full_content, + author=f"@{screen_name}" if screen_name else author, + excerpt=generate_excerpt(full_content), + tags=tags, + ) + + # ================================================================== + # Nitter HTML Scraping — Fallback Strategy + # ================================================================== + + def _parse_nitter_fallback(self, original_url: str, tweet_path: str) -> ParseResult: + """Try Nitter instances as a fallback (useful for reply threads).""" instances = random.sample( NITTER_INSTANCES, - min(self.max_retries, len(NITTER_INSTANCES)), + min(self.max_nitter_retries, len(NITTER_INSTANCES)), ) last_error = "" for instance in instances: nitter_url = f"{instance}/{tweet_path}" - logger.info("Trying Nitter instance: %s", nitter_url) + logger.info("Trying Nitter fallback: %s", nitter_url) try: - result = self._parse_nitter(url, nitter_url) + result = self._parse_nitter_page(original_url, nitter_url) if result.success: return result last_error = result.error except requests.RequestException as exc: last_error = str(exc) - logger.warning("Nitter instance %s failed: %s", instance, exc) - continue + logger.warning("Nitter %s failed: %s", instance, exc) except Exception as exc: # noqa: BLE001 last_error = str(exc) - logger.warning("Unexpected error with %s: %s", instance, exc) - continue + logger.warning("Nitter unexpected error: %s", exc) - # ------------------------------------------------------------------ - # All Nitter instances failed → fallback - # ------------------------------------------------------------------ - return self._fallback_scrape(url, last_error) + return ParseResult.failure( + original_url, + f"All Nitter instances failed. Last error: {last_error}", + ) - # ------------------------------------------------------------------ - # Nitter HTML Parsing - # ------------------------------------------------------------------ - - def _parse_nitter(self, original_url: str, nitter_url: str) -> ParseResult: + def _parse_nitter_page(self, original_url: str, nitter_url: str) -> ParseResult: """Fetch and parse a single Nitter page.""" resp = requests.get( nitter_url, @@ -120,40 +318,36 @@ class TwitterParser(BaseParser): resp.raise_for_status() soup = BeautifulSoup(resp.text, "lxml") - # --- Tweet body --- tweet_div = soup.find("div", class_="tweet-content") or soup.find( "div", class_="main-tweet" ) if not tweet_div: return ParseResult.failure( original_url, - f"Nitter page loaded but no tweet content found at {nitter_url}", + f"Nitter page loaded but no content found at {nitter_url}", ) tweet_text = tweet_div.get_text(separator="\n", strip=True) - # --- Author --- author_tag = soup.find("a", class_="fullname") or soup.find( "span", class_="username" ) author = author_tag.get_text(strip=True) if author_tag else "" - # --- Timestamp --- date_tag = soup.find("span", class_="tweet-date") timestamp = "" if date_tag: a_tag = date_tag.find("a") timestamp = a_tag.get("title", "") if a_tag else date_tag.get_text(strip=True) - # Build a nice title title = f"Tweet by {author}" if author else "Tweet" if timestamp: title += f" ({timestamp})" - # Collect reply context if present + # Collect reply context replies: list[str] = [] reply_divs = soup.find_all("div", class_="reply") - for rd in reply_divs[:5]: # limit to first 5 replies + for rd in reply_divs[:5]: reply_content = rd.find("div", class_="tweet-content") if reply_content: replies.append(reply_content.get_text(separator=" ", strip=True)) @@ -174,92 +368,30 @@ class TwitterParser(BaseParser): content=full_content, author=author, excerpt=generate_excerpt(full_content), - tags=["twitter"], + tags=["twitter", "nitter-fallback"], ) - # ------------------------------------------------------------------ - # Fallback Strategy - # ------------------------------------------------------------------ - - def _fallback_scrape(self, url: str, last_error: str) -> ParseResult: - """Produce a graceful degradation result with integration guidance. - - .. rubric:: How to extend with a paid scraping service - - **Option A – ZenRows / ScrapingBee:** - - 1. Sign up at https://www.zenrows.com/ or https://www.scrapingbee.com/ - 2. Obtain your API key. - 3. Replace the body of this method with:: - - import requests - api_key = "YOUR_ZENROWS_API_KEY" - params = { - "url": url, - "apikey": api_key, - "js_render": "true", - "premium_proxy": "true", - } - resp = requests.get("https://api.zenrows.com/v1/", params=params) - html = resp.text - # Then parse 'html' with BeautifulSoup to extract tweet text. - - **Option B – Browser Cookies:** - - 1. Export your Twitter session cookies (e.g. with the *EditThisCookie* - browser extension) as a Netscape-format ``cookies.txt`` file. - 2. Place the file at ``deepreader_skill/twitter_cookies.txt``. - 3. Modify ``_fetch_with_cookies()`` below to load and send them:: - - import http.cookiejar - jar = http.cookiejar.MozillaCookieJar("twitter_cookies.txt") - jar.load() - session = requests.Session() - session.cookies = jar - resp = session.get(url, headers=self._get_headers()) - - **Option C – Playwright / Selenium headless browser:** - - For the most reliable extraction, you can use a headless browser. - This is heavier but handles JavaScript-rendered content:: - - from playwright.sync_api import sync_playwright - with sync_playwright() as p: - browser = p.chromium.launch(headless=True) - page = browser.new_page() - page.goto(url, wait_until="networkidle") - html = page.content() - browser.close() - # Then parse 'html' as above. - - Returns a :class:`ParseResult` with ``success=False`` and the - guidance embedded in the error message. - """ - error_msg = ( - f"⚠️ All Nitter instances failed for this tweet.\n" - f"Last error: {last_error}\n\n" - f"The tweet URL was: {url}\n\n" - f"💡 To improve Twitter support, consider:\n" - f" 1. Updating the NITTER_INSTANCES list in twitter.py\n" - f" 2. Integrating a paid scraping service (ZenRows/ScrapingBee)\n" - f" 3. Using browser cookies for authenticated access\n" - f" See the _fallback_scrape() docstring for detailed instructions." - ) - logger.warning("Twitter fallback triggered for %s", url) - return ParseResult.failure(url, error_msg) - - # ------------------------------------------------------------------ - # URL Utilities - # ------------------------------------------------------------------ + # ================================================================== + # URL Utilities + # ================================================================== @staticmethod - def _extract_tweet_path(url: str) -> str | None: - """Extract the tweet path (``user/status/id``) from a Twitter URL. + def _extract_tweet_info(url: str) -> tuple[str, str] | None: + """Extract (username, tweet_id) from a Twitter URL. Returns ``None`` if the URL doesn't match the expected pattern. """ + match = re.search( + r"(?:x\.com|twitter\.com)/([a-zA-Z0-9_]{1,15})/status/(\d+)", url, + ) + if match: + return match.group(1), match.group(2) + return None + + @staticmethod + def _extract_tweet_path(url: str) -> str | None: + """Legacy: extract ``user/status/id`` path from a Twitter URL.""" parsed = urlparse(url) - # Match patterns like /username/status/1234567890 match = re.match(r"^/([^/]+)/status/(\d+)", parsed.path) if match: return f"{match.group(1)}/status/{match.group(2)}"