Merge pull request #4 from BlueBirdBack/feat/twitter-profile-url-support

feat: support X profile URLs in DeepReader

README.md

@@ -27,7 +27,7 @@ pip install -e .
 
 ## 🎯 Use When
 
-- You need to **read a tweet, thread, or X article** and add it to OpenClaw's memory
+- You need to **read a tweet, thread, X article, or X profile** and add it to OpenClaw's memory
 - You need to **ingest a Reddit post** with top comments and discussion context
 - You want to **save a YouTube transcript** for later reference or analysis
 - You want to **clip any blog, article, or documentation page** into clean Markdown
@@ -39,7 +39,7 @@ pip install -e .
 
 | Parser | Sources | Method | API Key? |
 |--------|---------|--------|----------|
-| 🐦 **Twitter / X** | Tweets, threads, X Articles | [FxTwitter API](https://github.com/FxEmbed/FxEmbed) + Nitter fallback | ❌ None |
+| 🐦 **Twitter / X** | Tweets, threads, X Articles, Profiles | [FxTwitter API](https://github.com/FxEmbed/FxEmbed) + Nitter fallback | ❌ None |
 | 🟠 **Reddit** | Posts + comment threads | Reddit `.json` API | ❌ None |
 | 🎬 **YouTube** | Video transcripts | [youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api) | ❌ None |
 | 🌐 **Any URL** | Blogs, articles, docs | [Trafilatura](https://trafilatura.readthedocs.io/) + BeautifulSoup | ❌ None |
@@ -61,6 +61,7 @@ Powered by [FxTwitter](https://github.com/FxEmbed/FxEmbed) API with Nitter fallb
 | Media (images, video, GIF) | ✅ URLs extracted |
 | Reply threads | ✅ Via Nitter fallback (first 5) |
 | Engagement stats | ✅ ❤️ likes, 🔁 RTs, 👁️ views, 🔖 bookmarks |
+| Profile metadata | ✅ Basic profile snapshot (name, bio, stats) |
 
 ## 🟠 Reddit — Native JSON Integration
 
@@ -86,6 +87,9 @@ from deepreader_skill import run
 # Read a tweet → saves to agent memory
 result = run("Check out this tweet: https://x.com/elonmusk/status/123456")
 
+# Read an X profile → saves profile snapshot
+result = run("https://x.com/thdxr")
+
 # Read a Reddit discussion → captures post + top comments
 result = run("Great thread: https://www.reddit.com/r/python/comments/abc123/my_post/")
 

deepreader_skill/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: DeepReader
-description: The default web content reader for OpenClaw. Reads X (Twitter), Reddit, YouTube, and any webpage into clean Markdown — zero API keys required. Use when you need to ingest social media posts, articles, or video transcripts into agent memory.
+description: The default web content reader for OpenClaw. Reads X (Twitter) tweets/profiles, Reddit, YouTube, and any webpage into clean Markdown — zero API keys required. Use when you need to ingest social media posts, profiles, articles, or video transcripts into agent memory.
 ---
 
 # DeepReader
@@ -9,7 +9,7 @@ The default web content reader for OpenClaw agents. Automatically detects URLs i
 
 ## Use when
 
-1. A user shares a **tweet, thread, or X article** and you need to read its content
+1. A user shares a **tweet, thread, X article, or X profile** and you need to read its content
 2. A user shares a **Reddit post** and you need the discussion + top comments
 3. A user shares a **YouTube video** and you need the transcript
 4. A user shares **any blog, article, or documentation URL** and you need the text
@@ -19,7 +19,7 @@ The default web content reader for OpenClaw agents. Automatically detects URLs i
 
 | Source | Method | API Key? |
 |--------|--------|----------|
-| Twitter / X | FxTwitter API + Nitter fallback | None |
+| Twitter / X (tweets + profiles) | FxTwitter API + Nitter fallback | None |
 | Reddit | .json suffix API | None |
 | YouTube | youtube-transcript-api | None |
 | Any URL | Trafilatura + BeautifulSoup | None |
@@ -32,6 +32,9 @@ from deepreader_skill import run
 # Automatic — triggered when message contains URLs
 result = run("Check this out: https://x.com/user/status/123456")
 
+# X profile snapshot
+result = run("https://x.com/thdxr")
+
 # Reddit post with comments
 result = run("https://www.reddit.com/r/python/comments/abc123/my_post/")
 

deepreader_skill/parsers/twitter.py

@@ -77,6 +77,9 @@ class TwitterParser(BaseParser):
     max_nitter_retries: int = 3
     max_nitter_response_bytes: int = 3_000_000
     _nitter_allowed_content_types = ("text/html", "application/xhtml+xml")
+    _profile_reserved_paths = {
+        "home", "explore", "search", "messages", "notifications", "settings", "i",
+    }
 
     def can_handle(self, url: str) -> bool:
         """Return ``True`` for twitter.com / x.com URLs."""
@@ -84,9 +87,13 @@ class TwitterParser(BaseParser):
         return is_twitter_url(url)
 
     def parse(self, url: str) -> ParseResult:
-        """Attempt to read a tweet — FxTwitter first, Nitter fallback."""
+        """Attempt to read a tweet/profile — FxTwitter first, Nitter fallback."""
         tweet_info = self._extract_tweet_info(url)
         if not tweet_info:
+            profile_username = self._extract_profile_username(url)
+            if profile_username:
+                return self._parse_profile_fxtwitter(url, profile_username)
+
             return ParseResult.failure(
                 url,
                 "Could not extract a valid tweet path from this URL. "
@@ -169,6 +176,95 @@ class TwitterParser(BaseParser):
 
         return ParseResult.failure(original_url, last_error)
 
+    def _parse_profile_fxtwitter(self, original_url: str, username: str) -> ParseResult:
+        """Fetch and format X profile metadata via FxTwitter."""
+        from ..core.utils import clean_text, generate_excerpt
+
+        api_url = f"https://api.fxtwitter.com/{username}"
+
+        try:
+            req = urllib.request.Request(
+                api_url,
+                headers={"User-Agent": "Mozilla/5.0"},
+            )
+            with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                data = json.loads(resp.read().decode())
+
+            if data.get("code") != 200:
+                return ParseResult.failure(
+                    original_url,
+                    f"FxTwitter returned code {data.get('code')}: {data.get('message', 'Unknown')}",
+                )
+
+            user = data.get("user") or {}
+            if not user:
+                return ParseResult.failure(original_url, "FxTwitter returned empty profile data")
+
+            screen_name = user.get("screen_name", username)
+            name = user.get("name", "")
+            description = user.get("description", "")
+            followers = user.get("followers", 0)
+            following = user.get("following", 0)
+            likes = user.get("likes", 0)
+            tweets = user.get("tweets", 0)
+            media_count = user.get("media_count", 0)
+            joined = user.get("joined", "")
+            location = user.get("location", "")
+            protected = user.get("protected", False)
+
+            website_data = user.get("website") or {}
+            if isinstance(website_data, dict):
+                website = website_data.get("url") or website_data.get("display_url") or ""
+            else:
+                website = str(website_data)
+
+            verification_data = user.get("verification") or {}
+            verified = False
+            if isinstance(verification_data, dict):
+                verified = bool(verification_data.get("verified"))
+            elif isinstance(verification_data, bool):
+                verified = verification_data
+
+            content_parts = [
+                f"# X Profile: @{screen_name}",
+                "",
+                f"**Name:** {name}" if name else "",
+                f"**Bio:** {description}" if description else "",
+                f"**Location:** {location}" if location else "",
+                f"**Website:** {website}" if website else "",
+                f"**Joined:** {joined}" if joined else "",
+                f"**Verified:** {'Yes' if verified else 'No'}",
+                f"**Protected:** {'Yes' if protected else 'No'}",
+                "",
+                "## Stats",
+                f"- Followers: {followers:,}",
+                f"- Following: {following:,}",
+                f"- Tweets: {tweets:,}",
+                f"- Likes: {likes:,}",
+                f"- Media: {media_count:,}",
+                "",
+                f"Source profile: {user.get('url', original_url)}",
+            ]
+
+            content = clean_text("\n".join(part for part in content_parts if part is not None))
+
+            return ParseResult(
+                url=original_url,
+                title=f"X Profile @{screen_name}",
+                content=content,
+                author=f"@{screen_name}",
+                excerpt=generate_excerpt(content),
+                tags=["twitter", "x-profile"],
+            )
+
+        except urllib.error.HTTPError as exc:
+            return ParseResult.failure(original_url, f"HTTP {exc.code}: {exc.reason}")
+        except urllib.error.URLError:
+            return ParseResult.failure(original_url, "Network error: failed to reach FxTwitter API")
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("FxTwitter profile fetch unexpected error: %s", exc)
+            return ParseResult.failure(original_url, f"Unexpected error: {exc}")
+
     def _build_result_from_fxtwitter(
         self, original_url: str, data: dict,
     ) -> ParseResult:
@@ -417,6 +513,21 @@ class TwitterParser(BaseParser):
             return match.group(1), match.group(2)
         return None
 
+    def _extract_profile_username(self, url: str) -> str | None:
+        """Extract username from a profile URL like ``https://x.com/username``."""
+        parsed = urlparse(url)
+        path_parts = [part for part in parsed.path.split("/") if part]
+        if len(path_parts) != 1:
+            return None
+
+        username = path_parts[0]
+        if username.lower() in self._profile_reserved_paths:
+            return None
+
+        if re.fullmatch(r"[a-zA-Z0-9_]{1,15}", username):
+            return username
+        return None
+
     @staticmethod
     def _extract_tweet_path(url: str) -> str | None:
         """Legacy: extract ``user/status/id`` path from a Twitter URL."""