From 6ccd3948387d6222ac2f93e1b42e9ceab5896fe1 Mon Sep 17 00:00:00 2001 From: luccast <2213102+luccast@users.noreply.github.com> Date: Mon, 26 Jan 2026 20:25:12 -0500 Subject: [PATCH] Add persistence feature to monitor - Introduced a persistence service to enable data retention across sessions. - Updated SettingsPanel to manage persistence state and actions. - Implemented hydration from server persistence in collections. - Enhanced TRPC router with persistence-related queries and mutations. - Updated monitor page to handle persistence status and actions. --- .gitignore | 3 + package-lock.json | 8 +- src/components/monitor/SettingsPanel.tsx | 74 ++++++++- src/integrations/clawdbot/collections.ts | 23 +++ src/integrations/clawdbot/persistence.ts | 196 +++++++++++++++++++++++ src/integrations/trpc/router.ts | 38 ++++- src/routes/monitor/index.tsx | 95 ++++++++++- 7 files changed, 429 insertions(+), 8 deletions(-) create mode 100644 src/integrations/clawdbot/persistence.ts diff --git a/.gitignore b/.gitignore index cfcc2db..e17169c 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ src/routeTree.gen.ts documents/* .tanstack/tmp/* + +# Persistence data +data/ diff --git a/package-lock.json b/package-lock.json index 572f6d9..15d18d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,10 +1,12 @@ { "name": "crabwalk", + "version": "1.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "crabwalk", + "version": "1.0.2", "dependencies": { "@tanstack/db": "^0.5.0", "@tanstack/react-db": "^0.1.0", @@ -37,8 +39,7 @@ "tailwindcss": "^4.0.0", "typescript": "^5.7.0", "vite": "^7.0.0" - }, - "version": "1.0.2" + } }, "node_modules/@babel/code-frame": { "version": "7.28.6", @@ -5525,6 +5526,5 @@ "url": "https://github.com/sponsors/wooorm" } } - }, - "version": "1.0.2" + } } diff --git a/src/components/monitor/SettingsPanel.tsx b/src/components/monitor/SettingsPanel.tsx index 4a23fb8..23a53d1 100644 --- a/src/components/monitor/SettingsPanel.tsx +++ b/src/components/monitor/SettingsPanel.tsx @@ -1,6 +1,6 @@ import { useState } from 'react' import { motion, AnimatePresence } from 'framer-motion' -import { Settings, X, History, Wifi, WifiOff, RefreshCw, Terminal, Download, Trash2, Database } from 'lucide-react' +import { Settings, X, History, Wifi, WifiOff, RefreshCw, Terminal, Download, Trash2, Database, HardDrive, Play, Square } from 'lucide-react' interface SettingsPanelProps { connected: boolean @@ -8,6 +8,10 @@ interface SettingsPanelProps { debugMode: boolean logCollection: boolean logCount: number + persistenceEnabled: boolean + persistenceStartedAt: number | null + persistenceSessionCount: number + persistenceActionCount: number onHistoricalModeChange: (enabled: boolean) => void onDebugModeChange: (enabled: boolean) => void onLogCollectionChange: (enabled: boolean) => void @@ -16,6 +20,9 @@ interface SettingsPanelProps { onConnect: () => void onDisconnect: () => void onRefresh: () => void + onPersistenceStart: () => void + onPersistenceStop: () => void + onPersistenceClear: () => void } export function SettingsPanel({ @@ -24,6 +31,10 @@ export function SettingsPanel({ debugMode, logCollection, logCount, + persistenceEnabled, + persistenceStartedAt, + persistenceSessionCount, + persistenceActionCount, onHistoricalModeChange, onDebugModeChange, onLogCollectionChange, @@ -32,6 +43,9 @@ export function SettingsPanel({ onConnect, onDisconnect, onRefresh, + onPersistenceStart, + onPersistenceStop, + onPersistenceClear, }: SettingsPanelProps) { const [open, setOpen] = useState(false) @@ -171,6 +185,64 @@ export function SettingsPanel({ + {/* Background Service */} +
+
+ + + Background Service + +
+ +

+ > persist data across refreshes +

+ + {persistenceEnabled && persistenceStartedAt && ( +
+ > running since {new Date(persistenceStartedAt).toLocaleTimeString()} +
+ )} + +
+
+ > {persistenceSessionCount} sessions +
+
+ > {persistenceActionCount} actions +
+
+ +
+ {persistenceEnabled ? ( + + ) : ( + + )} +
+ + +
+ {/* Log collection */}
diff --git a/src/integrations/clawdbot/collections.ts b/src/integrations/clawdbot/collections.ts index a93a300..7246a0b 100644 --- a/src/integrations/clawdbot/collections.ts +++ b/src/integrations/clawdbot/collections.ts @@ -158,3 +158,26 @@ export function clearCollections() { actionsCollection.delete(action.id) } } + +// Hydrate collections from server persistence +export function hydrateFromServer( + sessions: MonitorSession[], + actions: MonitorAction[] +) { + // First clear existing data + clearCollections() + + // Insert all sessions + for (const session of sessions) { + sessionsCollection.insert(session) + } + + // Insert all actions (rebuild runSessionMap as we go) + for (const action of actions) { + // Learn runId → sessionKey mapping + if (action.sessionKey && !action.sessionKey.includes('lifecycle')) { + runSessionMap.set(action.runId, action.sessionKey) + } + actionsCollection.insert(action) + } +} diff --git a/src/integrations/clawdbot/persistence.ts b/src/integrations/clawdbot/persistence.ts new file mode 100644 index 0000000..4466491 --- /dev/null +++ b/src/integrations/clawdbot/persistence.ts @@ -0,0 +1,196 @@ +import fs from 'fs' +import path from 'path' +import type { MonitorSession, MonitorAction } from './protocol' + +const DATA_DIR = path.join(process.cwd(), 'data') +const SESSIONS_FILE = path.join(DATA_DIR, 'sessions.json') +const ACTIONS_FILE = path.join(DATA_DIR, 'actions.jsonl') +const STATE_FILE = path.join(DATA_DIR, 'state.json') +const MAX_ACTIONS = 10000 + +interface PersistenceState { + enabled: boolean + startedAt: number | null +} + +class PersistenceService { + private sessions: Map = new Map() + private actions: MonitorAction[] = [] + private enabled = false + private startedAt: number | null = null + + constructor() { + this.ensureDataDir() + this.loadState() + this.loadData() + } + + private ensureDataDir() { + if (!fs.existsSync(DATA_DIR)) { + fs.mkdirSync(DATA_DIR, { recursive: true }) + } + } + + private loadState() { + try { + if (fs.existsSync(STATE_FILE)) { + const data = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')) as PersistenceState + this.enabled = data.enabled + this.startedAt = data.startedAt + } + } catch { + // ignore + } + } + + private saveState() { + const state: PersistenceState = { + enabled: this.enabled, + startedAt: this.startedAt, + } + fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2)) + } + + private loadData() { + // Load sessions + try { + if (fs.existsSync(SESSIONS_FILE)) { + const data = JSON.parse(fs.readFileSync(SESSIONS_FILE, 'utf-8')) as MonitorSession[] + for (const session of data) { + this.sessions.set(session.key, session) + } + } + } catch { + // ignore + } + + // Load actions (JSONL) + try { + if (fs.existsSync(ACTIONS_FILE)) { + const content = fs.readFileSync(ACTIONS_FILE, 'utf-8') + const lines = content.trim().split('\n').filter(Boolean) + for (const line of lines) { + try { + const action = JSON.parse(line) as MonitorAction + this.actions.push(action) + } catch { + // skip bad lines + } + } + // Trim to max if needed + if (this.actions.length > MAX_ACTIONS) { + this.actions = this.actions.slice(-MAX_ACTIONS) + this.saveActions() + } + } + } catch { + // ignore + } + } + + private saveSessions() { + const data = Array.from(this.sessions.values()) + fs.writeFileSync(SESSIONS_FILE, JSON.stringify(data, null, 2)) + } + + private saveActions() { + const content = this.actions.map((a) => JSON.stringify(a)).join('\n') + fs.writeFileSync(ACTIONS_FILE, content) + } + + private appendAction(action: MonitorAction) { + fs.appendFileSync(ACTIONS_FILE, JSON.stringify(action) + '\n') + } + + get isEnabled() { + return this.enabled + } + + start(): { enabled: boolean; startedAt: number } { + this.enabled = true + this.startedAt = Date.now() + this.saveState() + console.log('[persistence] started') + return { enabled: true, startedAt: this.startedAt } + } + + stop(): { enabled: boolean } { + this.enabled = false + this.startedAt = null + this.saveState() + console.log('[persistence] stopped') + return { enabled: false } + } + + getStatus(): { + enabled: boolean + startedAt: number | null + sessionCount: number + actionCount: number + } { + return { + enabled: this.enabled, + startedAt: this.startedAt, + sessionCount: this.sessions.size, + actionCount: this.actions.length, + } + } + + upsertSession(session: MonitorSession) { + if (!this.enabled) return + this.sessions.set(session.key, session) + this.saveSessions() + } + + addAction(action: MonitorAction) { + if (!this.enabled) return + + // Check if action already exists (by id) + const existingIdx = this.actions.findIndex((a) => a.id === action.id) + if (existingIdx >= 0) { + // Update existing action + this.actions[existingIdx] = action + this.saveActions() + } else { + // Add new action + this.actions.push(action) + this.appendAction(action) + + // Rotate if over limit + if (this.actions.length > MAX_ACTIONS) { + this.actions = this.actions.slice(-MAX_ACTIONS) + this.saveActions() + } + } + } + + hydrate(): { sessions: MonitorSession[]; actions: MonitorAction[] } { + return { + sessions: Array.from(this.sessions.values()), + actions: [...this.actions], + } + } + + clear(): { cleared: boolean } { + this.sessions.clear() + this.actions = [] + try { + if (fs.existsSync(SESSIONS_FILE)) fs.unlinkSync(SESSIONS_FILE) + if (fs.existsSync(ACTIONS_FILE)) fs.unlinkSync(ACTIONS_FILE) + } catch { + // ignore + } + console.log('[persistence] cleared all data') + return { cleared: true } + } +} + +// Singleton instance +let instance: PersistenceService | null = null + +export function getPersistenceService(): PersistenceService { + if (!instance) { + instance = new PersistenceService() + } + return instance +} diff --git a/src/integrations/trpc/router.ts b/src/integrations/trpc/router.ts index 7b6378a..2c9e6df 100644 --- a/src/integrations/trpc/router.ts +++ b/src/integrations/trpc/router.ts @@ -3,6 +3,7 @@ import { observable } from '@trpc/server/observable' import superjson from 'superjson' import { z } from 'zod' import { getClawdbotClient } from '~/integrations/clawdbot/client' +import { getPersistenceService } from '~/integrations/clawdbot/persistence' import { parseEventFrame, sessionInfoToMonitor, @@ -114,14 +115,18 @@ const clawdbotRouter = router({ ) .query(async ({ input }) => { const client = getClawdbotClient() + const persistence = getPersistenceService() if (!client.connected) { return { sessions: [], error: 'Not connected' } } try { const sessions = await client.listSessions(input) - return { - sessions: sessions.map(sessionInfoToMonitor), + const monitorSessions = sessions.map(sessionInfoToMonitor) + // Persist sessions if service is enabled + for (const session of monitorSessions) { + persistence.upsertSession(session) } + return { sessions: monitorSessions } } catch (error) { return { sessions: [], @@ -137,6 +142,7 @@ const clawdbotRouter = router({ action?: MonitorAction }>((emit) => { const client = getClawdbotClient() + const persistence = getPersistenceService() const unsubscribe = client.onEvent((event) => { // Collect raw event when log collection is enabled @@ -161,6 +167,8 @@ const clawdbotRouter = router({ emit.next({ type: 'session', session: parsed.session }) } if (parsed.action) { + // Persist action if service is enabled + persistence.addAction(parsed.action) emit.next({ type: 'action', action: parsed.action }) } } @@ -171,6 +179,32 @@ const clawdbotRouter = router({ } }) }), + + // Persistence service + persistenceStatus: publicProcedure.query(() => { + const persistence = getPersistenceService() + return persistence.getStatus() + }), + + persistenceStart: publicProcedure.mutation(() => { + const persistence = getPersistenceService() + return persistence.start() + }), + + persistenceStop: publicProcedure.mutation(() => { + const persistence = getPersistenceService() + return persistence.stop() + }), + + persistenceHydrate: publicProcedure.query(() => { + const persistence = getPersistenceService() + return persistence.hydrate() + }), + + persistenceClear: publicProcedure.mutation(() => { + const persistence = getPersistenceService() + return persistence.clear() + }), }) export const appRouter = router({ diff --git a/src/routes/monitor/index.tsx b/src/routes/monitor/index.tsx index ca46b8b..79f973a 100644 --- a/src/routes/monitor/index.tsx +++ b/src/routes/monitor/index.tsx @@ -11,6 +11,7 @@ import { addAction, updateSessionStatus, clearCollections, + hydrateFromServer, } from '~/integrations/clawdbot' import { ActionGraph, @@ -67,6 +68,12 @@ function MonitorPage() { const [logCount, setLogCount] = useState(0) const [selectedSession, setSelectedSession] = useState(null) + // Persistence service state + const [persistenceEnabled, setPersistenceEnabled] = useState(false) + const [persistenceStartedAt, setPersistenceStartedAt] = useState(null) + const [persistenceSessionCount, setPersistenceSessionCount] = useState(0) + const [persistenceActionCount, setPersistenceActionCount] = useState(0) + // Sidebar collapse state - default to collapsed const [sidebarCollapsed, setSidebarCollapsed] = useState(true) @@ -78,11 +85,24 @@ function MonitorPage() { const actions = actionsQuery.data ?? [] - // Check connection status on mount + // Check connection status and persistence on mount useEffect(() => { checkStatus() + checkPersistenceStatus() }, []) + const checkPersistenceStatus = async () => { + try { + const status = await trpc.clawdbot.persistenceStatus.query() + setPersistenceEnabled(status.enabled) + setPersistenceStartedAt(status.startedAt) + setPersistenceSessionCount(status.sessionCount) + setPersistenceActionCount(status.actionCount) + } catch { + // ignore + } + } + const checkStatus = async () => { try { const status = await trpc.clawdbot.status.query() @@ -101,6 +121,8 @@ function MonitorPage() { setConnected(true) setRetryCount(0) setConnecting(false) + // Hydrate from persistence if enabled + await hydrateFromPersistence() await loadSessions() return } @@ -115,6 +137,23 @@ function MonitorPage() { } } + const hydrateFromPersistence = async () => { + try { + const status = await trpc.clawdbot.persistenceStatus.query() + if (status.sessionCount > 0 || status.actionCount > 0) { + const data = await trpc.clawdbot.persistenceHydrate.query() + hydrateFromServer(data.sessions, data.actions) + console.log(`[monitor] hydrated ${data.sessions.length} sessions, ${data.actions.length} actions`) + } + setPersistenceEnabled(status.enabled) + setPersistenceStartedAt(status.startedAt) + setPersistenceSessionCount(status.sessionCount) + setPersistenceActionCount(status.actionCount) + } catch (e) { + console.error('Failed to hydrate:', e) + } + } + const handleDisconnect = async () => { try { await trpc.clawdbot.disconnect.mutate() @@ -196,6 +235,37 @@ function MonitorPage() { } } + const handlePersistenceStart = async () => { + try { + const result = await trpc.clawdbot.persistenceStart.mutate() + setPersistenceEnabled(result.enabled) + setPersistenceStartedAt(result.startedAt) + } catch (e) { + console.error('Failed to start persistence:', e) + } + } + + const handlePersistenceStop = async () => { + try { + const result = await trpc.clawdbot.persistenceStop.mutate() + setPersistenceEnabled(result.enabled) + setPersistenceStartedAt(null) + } catch (e) { + console.error('Failed to stop persistence:', e) + } + } + + const handlePersistenceClear = async () => { + try { + await trpc.clawdbot.persistenceClear.mutate() + setPersistenceSessionCount(0) + setPersistenceActionCount(0) + clearCollections() + } catch (e) { + console.error('Failed to clear persistence:', e) + } + } + // Poll log count while collecting useEffect(() => { if (!logCollection) return @@ -210,6 +280,22 @@ function MonitorPage() { return () => clearInterval(interval) }, [logCollection]) + // Poll persistence status + useEffect(() => { + const interval = setInterval(async () => { + try { + const status = await trpc.clawdbot.persistenceStatus.query() + setPersistenceEnabled(status.enabled) + setPersistenceStartedAt(status.startedAt) + setPersistenceSessionCount(status.sessionCount) + setPersistenceActionCount(status.actionCount) + } catch { + // ignore + } + }, 5000) + return () => clearInterval(interval) + }, []) + const handleToggleSidebar = useCallback(() => { setSidebarCollapsed((prev) => !prev) }, []) @@ -313,6 +399,10 @@ function MonitorPage() { debugMode={debugMode} logCollection={logCollection} logCount={logCount} + persistenceEnabled={persistenceEnabled} + persistenceStartedAt={persistenceStartedAt} + persistenceSessionCount={persistenceSessionCount} + persistenceActionCount={persistenceActionCount} onHistoricalModeChange={handleHistoricalModeChange} onDebugModeChange={handleDebugModeChange} onLogCollectionChange={handleLogCollectionChange} @@ -321,6 +411,9 @@ function MonitorPage() { onConnect={handleConnect} onDisconnect={handleDisconnect} onRefresh={handleRefresh} + onPersistenceStart={handlePersistenceStart} + onPersistenceStop={handlePersistenceStop} + onPersistenceClear={handlePersistenceClear} />