From 0eef81088c14d09beec1201340f7f75b592b2363 Mon Sep 17 00:00:00 2001
From: Bobby Cude <bcude@users.noreply.github.com>
Date: Fri, 15 May 2026 22:26:20 -0500
Subject: [PATCH] fix: whitespace-flexible self-repeat dedup in JSONL parser

Gateway v4 inserts varying whitespace (extra newlines) between repeated
copies of assistant text, so exact byte-level N-copy detection missed
them. Now handles whitespace-flexible boundaries when checking for
self-repeated text.
---
 api/src/services/openclaw/jsonlParser.ts | 57 ++++++++++++++++++++----
 1 file changed, 48 insertions(+), 9 deletions(-)

diff --git a/api/src/services/openclaw/jsonlParser.ts b/api/src/services/openclaw/jsonlParser.ts
index c60062b..4a25604 100644
--- a/api/src/services/openclaw/jsonlParser.ts
+++ b/api/src/services/openclaw/jsonlParser.ts
@@ -150,18 +150,57 @@ export function extractUserText(raw: string): string {
  */
 function deduplicateSelfRepeat(text: string): string {
   if (!text || text.length < 40) return text;
+  // Normalize: collapse runs of newlines to single newline for matching,
+  // but return the original first segment (untouched) on match.
+  const normalized = text.replace(/\n{2,}/g, '\n');
   for (let n = 2; n <= 6; n++) {
-    if (text.length % n !== 0) continue;
-    const segLen = text.length / n;
-    const seg = text.slice(0, segLen);
-    let isRepeat = true;
-    for (let i = 1; i < n; i++) {
-      if (text.slice(i * segLen, (i + 1) * segLen) !== seg) {
-        isRepeat = false;
-        break;
+    // Try exact division on normalized text
+    if (normalized.length % n === 0) {
+      const segLen = normalized.length / n;
+      const seg = normalized.slice(0, segLen);
+      let isRepeat = true;
+      for (let i = 1; i < n; i++) {
+        if (normalized.slice(i * segLen, (i + 1) * segLen) !== seg) {
+          isRepeat = false;
+          break;
+        }
+      }
+      if (isRepeat) {
+        // Return the original (un-normalized) first segment
+        // Find where the first copy ends in the original text
+        const firstCopyEnd = text.indexOf(seg.slice(-20)) + 20;
+        // Safer: just split by the segment and return first match
+        return text.slice(0, text.length / n).trim();
+      }
+    }
+    // Also try with flexible boundaries: check if the first ~1/n of the
+    // text repeats by searching for it later in the string
+    const approxLen = Math.floor(text.length / n);
+    for (let fuzz = -2; fuzz <= 2; fuzz++) {
+      const tryLen = approxLen + fuzz;
+      if (tryLen < 20 || tryLen >= text.length) continue;
+      const candidate = text.slice(0, tryLen).trim();
+      if (!candidate) continue;
+      // Check if the rest of the text is just repeats of candidate (with whitespace flex)
+      let pos = tryLen;
+      let copies = 1;
+      while (pos < text.length) {
+        // Skip whitespace between copies
+        while (pos < text.length && /\s/.test(text[pos])) pos++;
+        if (pos >= text.length) break;
+        if (text.startsWith(candidate, pos)) {
+          copies++;
+          pos += candidate.length;
+        } else {
+          break;
+        }
+      }
+      // Allow trailing whitespace
+      const remaining = text.slice(pos).trim();
+      if (copies === n && remaining.length === 0) {
+        return candidate;
       }
     }
-    if (isRepeat) return seg;
   }
   return text;
 }