diff --git a/.gitignore b/.gitignore index cfcc2db..e17169c 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ src/routeTree.gen.ts documents/* .tanstack/tmp/* + +# Persistence data +data/ diff --git a/package-lock.json b/package-lock.json index 572f6d9..15d18d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,10 +1,12 @@ { "name": "crabwalk", + "version": "1.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "crabwalk", + "version": "1.0.2", "dependencies": { "@tanstack/db": "^0.5.0", "@tanstack/react-db": "^0.1.0", @@ -37,8 +39,7 @@ "tailwindcss": "^4.0.0", "typescript": "^5.7.0", "vite": "^7.0.0" - }, - "version": "1.0.2" + } }, "node_modules/@babel/code-frame": { "version": "7.28.6", @@ -5525,6 +5526,5 @@ "url": "https://github.com/sponsors/wooorm" } } - }, - "version": "1.0.2" + } } diff --git a/src/components/monitor/SettingsPanel.tsx b/src/components/monitor/SettingsPanel.tsx index 4a23fb8..1527054 100644 --- a/src/components/monitor/SettingsPanel.tsx +++ b/src/components/monitor/SettingsPanel.tsx @@ -1,6 +1,5 @@ -import { useState } from 'react' import { motion, AnimatePresence } from 'framer-motion' -import { Settings, X, History, Wifi, WifiOff, RefreshCw, Terminal, Download, Trash2, Database } from 'lucide-react' +import { Settings, X, History, Wifi, WifiOff, RefreshCw, Terminal, Download, Trash2, Database, HardDrive, Play, Square } from 'lucide-react' interface SettingsPanelProps { connected: boolean @@ -8,6 +7,12 @@ interface SettingsPanelProps { debugMode: boolean logCollection: boolean logCount: number + persistenceEnabled: boolean + persistenceStartedAt: number | null + persistenceSessionCount: number + persistenceActionCount: number + open: boolean + onOpenChange: (open: boolean) => void onHistoricalModeChange: (enabled: boolean) => void onDebugModeChange: (enabled: boolean) => void onLogCollectionChange: (enabled: boolean) => void @@ -16,6 +21,9 @@ interface SettingsPanelProps { onConnect: () => void onDisconnect: () => void onRefresh: () => void + onPersistenceStart: () => void + onPersistenceStop: () => void + onPersistenceClear: () => void } export function SettingsPanel({ @@ -24,6 +32,12 @@ export function SettingsPanel({ debugMode, logCollection, logCount, + persistenceEnabled, + persistenceStartedAt, + persistenceSessionCount, + persistenceActionCount, + open, + onOpenChange, onHistoricalModeChange, onDebugModeChange, onLogCollectionChange, @@ -32,13 +46,15 @@ export function SettingsPanel({ onConnect, onDisconnect, onRefresh, + onPersistenceStart, + onPersistenceStop, + onPersistenceClear, }: SettingsPanelProps) { - const [open, setOpen] = useState(false) return ( <> + {/* Background Service */} +
+
+ + + Background Service + +
+ +

+ > persist data across refreshes +

+ + {persistenceEnabled && persistenceStartedAt && ( +
+ > running since {new Date(persistenceStartedAt).toLocaleTimeString()} +
+ )} + +
+
+ > {persistenceSessionCount} sessions +
+
+ > {persistenceActionCount} actions +
+
+ +
+ {persistenceEnabled ? ( + + ) : ( + + )} +
+ + +
+ {/* Log collection */}
diff --git a/src/integrations/clawdbot/collections.ts b/src/integrations/clawdbot/collections.ts index a93a300..9531be9 100644 --- a/src/integrations/clawdbot/collections.ts +++ b/src/integrations/clawdbot/collections.ts @@ -158,3 +158,24 @@ export function clearCollections() { actionsCollection.delete(action.id) } } + +// Hydrate collections from server persistence +export function hydrateFromServer( + sessions: MonitorSession[], + actions: MonitorAction[] +) { + // First clear existing data + clearCollections() + + // Insert all sessions + for (const session of sessions) { + sessionsCollection.insert(session) + } + + // Replay actions through addAction to apply aggregation logic + // Sort by timestamp to ensure correct order + const sortedActions = [...actions].sort((a, b) => a.timestamp - b.timestamp) + for (const action of sortedActions) { + addAction(action) + } +} diff --git a/src/integrations/clawdbot/persistence.ts b/src/integrations/clawdbot/persistence.ts new file mode 100644 index 0000000..eea376b --- /dev/null +++ b/src/integrations/clawdbot/persistence.ts @@ -0,0 +1,200 @@ +import fs from 'fs' +import path from 'path' +import type { MonitorSession, MonitorAction } from './protocol' + +const DATA_DIR = path.join(process.cwd(), 'data') +const SESSIONS_FILE = path.join(DATA_DIR, 'sessions.json') +const ACTIONS_FILE = path.join(DATA_DIR, 'actions.jsonl') +const STATE_FILE = path.join(DATA_DIR, 'state.json') +const MAX_ACTIONS = 10000 + +interface PersistenceState { + enabled: boolean + startedAt: number | null +} + +class PersistenceService { + private sessions: Map = new Map() + private actions: MonitorAction[] = [] + private enabled = false + private startedAt: number | null = null + + constructor() { + this.ensureDataDir() + this.loadState() + this.loadData() + // Auto-start by default if no state file exists + if (!this.enabled && !fs.existsSync(STATE_FILE)) { + this.start() + } + } + + private ensureDataDir() { + if (!fs.existsSync(DATA_DIR)) { + fs.mkdirSync(DATA_DIR, { recursive: true }) + } + } + + private loadState() { + try { + if (fs.existsSync(STATE_FILE)) { + const data = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')) as PersistenceState + this.enabled = data.enabled + this.startedAt = data.startedAt + } + } catch { + // ignore + } + } + + private saveState() { + const state: PersistenceState = { + enabled: this.enabled, + startedAt: this.startedAt, + } + fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2)) + } + + private loadData() { + // Load sessions + try { + if (fs.existsSync(SESSIONS_FILE)) { + const data = JSON.parse(fs.readFileSync(SESSIONS_FILE, 'utf-8')) as MonitorSession[] + for (const session of data) { + this.sessions.set(session.key, session) + } + } + } catch { + // ignore + } + + // Load actions (JSONL) + try { + if (fs.existsSync(ACTIONS_FILE)) { + const content = fs.readFileSync(ACTIONS_FILE, 'utf-8') + const lines = content.trim().split('\n').filter(Boolean) + for (const line of lines) { + try { + const action = JSON.parse(line) as MonitorAction + this.actions.push(action) + } catch { + // skip bad lines + } + } + // Trim to max if needed + if (this.actions.length > MAX_ACTIONS) { + this.actions = this.actions.slice(-MAX_ACTIONS) + this.saveActions() + } + } + } catch { + // ignore + } + } + + private saveSessions() { + const data = Array.from(this.sessions.values()) + fs.writeFileSync(SESSIONS_FILE, JSON.stringify(data, null, 2)) + } + + private saveActions() { + const content = this.actions.map((a) => JSON.stringify(a)).join('\n') + fs.writeFileSync(ACTIONS_FILE, content) + } + + private appendAction(action: MonitorAction) { + fs.appendFileSync(ACTIONS_FILE, JSON.stringify(action) + '\n') + } + + get isEnabled() { + return this.enabled + } + + start(): { enabled: boolean; startedAt: number } { + this.enabled = true + this.startedAt = Date.now() + this.saveState() + console.log('[persistence] started') + return { enabled: true, startedAt: this.startedAt } + } + + stop(): { enabled: boolean } { + this.enabled = false + this.startedAt = null + this.saveState() + console.log('[persistence] stopped') + return { enabled: false } + } + + getStatus(): { + enabled: boolean + startedAt: number | null + sessionCount: number + actionCount: number + } { + return { + enabled: this.enabled, + startedAt: this.startedAt, + sessionCount: this.sessions.size, + actionCount: this.actions.length, + } + } + + upsertSession(session: MonitorSession) { + if (!this.enabled) return + this.sessions.set(session.key, session) + this.saveSessions() + } + + addAction(action: MonitorAction) { + if (!this.enabled) return + + // Check if action already exists (by id) + const existingIdx = this.actions.findIndex((a) => a.id === action.id) + if (existingIdx >= 0) { + // Update existing action + this.actions[existingIdx] = action + this.saveActions() + } else { + // Add new action + this.actions.push(action) + this.appendAction(action) + + // Rotate if over limit + if (this.actions.length > MAX_ACTIONS) { + this.actions = this.actions.slice(-MAX_ACTIONS) + this.saveActions() + } + } + } + + hydrate(): { sessions: MonitorSession[]; actions: MonitorAction[] } { + return { + sessions: Array.from(this.sessions.values()), + actions: [...this.actions], + } + } + + clear(): { cleared: boolean } { + this.sessions.clear() + this.actions = [] + try { + if (fs.existsSync(SESSIONS_FILE)) fs.unlinkSync(SESSIONS_FILE) + if (fs.existsSync(ACTIONS_FILE)) fs.unlinkSync(ACTIONS_FILE) + } catch { + // ignore + } + console.log('[persistence] cleared all data') + return { cleared: true } + } +} + +// Singleton instance +let instance: PersistenceService | null = null + +export function getPersistenceService(): PersistenceService { + if (!instance) { + instance = new PersistenceService() + } + return instance +} diff --git a/src/integrations/trpc/router.ts b/src/integrations/trpc/router.ts index 7b6378a..2c9e6df 100644 --- a/src/integrations/trpc/router.ts +++ b/src/integrations/trpc/router.ts @@ -3,6 +3,7 @@ import { observable } from '@trpc/server/observable' import superjson from 'superjson' import { z } from 'zod' import { getClawdbotClient } from '~/integrations/clawdbot/client' +import { getPersistenceService } from '~/integrations/clawdbot/persistence' import { parseEventFrame, sessionInfoToMonitor, @@ -114,14 +115,18 @@ const clawdbotRouter = router({ ) .query(async ({ input }) => { const client = getClawdbotClient() + const persistence = getPersistenceService() if (!client.connected) { return { sessions: [], error: 'Not connected' } } try { const sessions = await client.listSessions(input) - return { - sessions: sessions.map(sessionInfoToMonitor), + const monitorSessions = sessions.map(sessionInfoToMonitor) + // Persist sessions if service is enabled + for (const session of monitorSessions) { + persistence.upsertSession(session) } + return { sessions: monitorSessions } } catch (error) { return { sessions: [], @@ -137,6 +142,7 @@ const clawdbotRouter = router({ action?: MonitorAction }>((emit) => { const client = getClawdbotClient() + const persistence = getPersistenceService() const unsubscribe = client.onEvent((event) => { // Collect raw event when log collection is enabled @@ -161,6 +167,8 @@ const clawdbotRouter = router({ emit.next({ type: 'session', session: parsed.session }) } if (parsed.action) { + // Persist action if service is enabled + persistence.addAction(parsed.action) emit.next({ type: 'action', action: parsed.action }) } } @@ -171,6 +179,32 @@ const clawdbotRouter = router({ } }) }), + + // Persistence service + persistenceStatus: publicProcedure.query(() => { + const persistence = getPersistenceService() + return persistence.getStatus() + }), + + persistenceStart: publicProcedure.mutation(() => { + const persistence = getPersistenceService() + return persistence.start() + }), + + persistenceStop: publicProcedure.mutation(() => { + const persistence = getPersistenceService() + return persistence.stop() + }), + + persistenceHydrate: publicProcedure.query(() => { + const persistence = getPersistenceService() + return persistence.hydrate() + }), + + persistenceClear: publicProcedure.mutation(() => { + const persistence = getPersistenceService() + return persistence.clear() + }), }) export const appRouter = router({ diff --git a/src/routes/monitor/index.tsx b/src/routes/monitor/index.tsx index ca46b8b..95794af 100644 --- a/src/routes/monitor/index.tsx +++ b/src/routes/monitor/index.tsx @@ -2,7 +2,7 @@ import { useState, useEffect, useCallback } from 'react' import { createFileRoute, Link } from '@tanstack/react-router' import { useLiveQuery } from '@tanstack/react-db' import { motion } from 'framer-motion' -import { ArrowLeft, Loader2 } from 'lucide-react' +import { ArrowLeft, Loader2, HardDrive } from 'lucide-react' import { trpc } from '~/integrations/trpc/client' import { sessionsCollection, @@ -11,6 +11,7 @@ import { addAction, updateSessionStatus, clearCollections, + hydrateFromServer, } from '~/integrations/clawdbot' import { ActionGraph, @@ -67,9 +68,18 @@ function MonitorPage() { const [logCount, setLogCount] = useState(0) const [selectedSession, setSelectedSession] = useState(null) + // Persistence service state + const [persistenceEnabled, setPersistenceEnabled] = useState(false) + const [persistenceStartedAt, setPersistenceStartedAt] = useState(null) + const [persistenceSessionCount, setPersistenceSessionCount] = useState(0) + const [persistenceActionCount, setPersistenceActionCount] = useState(0) + // Sidebar collapse state - default to collapsed const [sidebarCollapsed, setSidebarCollapsed] = useState(true) + // Settings panel state + const [settingsOpen, setSettingsOpen] = useState(false) + // Live queries from TanStack DB collections const sessionsQuery = useLiveQuery(sessionsCollection) const actionsQuery = useLiveQuery(actionsCollection) @@ -78,11 +88,24 @@ function MonitorPage() { const actions = actionsQuery.data ?? [] - // Check connection status on mount + // Check connection status and persistence on mount useEffect(() => { checkStatus() + checkPersistenceStatus() }, []) + const checkPersistenceStatus = async () => { + try { + const status = await trpc.clawdbot.persistenceStatus.query() + setPersistenceEnabled(status.enabled) + setPersistenceStartedAt(status.startedAt) + setPersistenceSessionCount(status.sessionCount) + setPersistenceActionCount(status.actionCount) + } catch { + // ignore + } + } + const checkStatus = async () => { try { const status = await trpc.clawdbot.status.query() @@ -101,6 +124,8 @@ function MonitorPage() { setConnected(true) setRetryCount(0) setConnecting(false) + // Hydrate from persistence if enabled + await hydrateFromPersistence() await loadSessions() return } @@ -115,6 +140,23 @@ function MonitorPage() { } } + const hydrateFromPersistence = async () => { + try { + const status = await trpc.clawdbot.persistenceStatus.query() + if (status.sessionCount > 0 || status.actionCount > 0) { + const data = await trpc.clawdbot.persistenceHydrate.query() + hydrateFromServer(data.sessions, data.actions) + console.log(`[monitor] hydrated ${data.sessions.length} sessions, ${data.actions.length} actions`) + } + setPersistenceEnabled(status.enabled) + setPersistenceStartedAt(status.startedAt) + setPersistenceSessionCount(status.sessionCount) + setPersistenceActionCount(status.actionCount) + } catch (e) { + console.error('Failed to hydrate:', e) + } + } + const handleDisconnect = async () => { try { await trpc.clawdbot.disconnect.mutate() @@ -196,6 +238,37 @@ function MonitorPage() { } } + const handlePersistenceStart = async () => { + try { + const result = await trpc.clawdbot.persistenceStart.mutate() + setPersistenceEnabled(result.enabled) + setPersistenceStartedAt(result.startedAt) + } catch (e) { + console.error('Failed to start persistence:', e) + } + } + + const handlePersistenceStop = async () => { + try { + const result = await trpc.clawdbot.persistenceStop.mutate() + setPersistenceEnabled(result.enabled) + setPersistenceStartedAt(null) + } catch (e) { + console.error('Failed to stop persistence:', e) + } + } + + const handlePersistenceClear = async () => { + try { + await trpc.clawdbot.persistenceClear.mutate() + setPersistenceSessionCount(0) + setPersistenceActionCount(0) + clearCollections() + } catch (e) { + console.error('Failed to clear persistence:', e) + } + } + // Poll log count while collecting useEffect(() => { if (!logCollection) return @@ -210,6 +283,22 @@ function MonitorPage() { return () => clearInterval(interval) }, [logCollection]) + // Poll persistence status + useEffect(() => { + const interval = setInterval(async () => { + try { + const status = await trpc.clawdbot.persistenceStatus.query() + setPersistenceEnabled(status.enabled) + setPersistenceStartedAt(status.startedAt) + setPersistenceSessionCount(status.sessionCount) + setPersistenceActionCount(status.actionCount) + } catch { + // ignore + } + }, 5000) + return () => clearInterval(interval) + }, []) + const handleToggleSidebar = useCallback(() => { setSidebarCollapsed((prev) => !prev) }, []) @@ -294,6 +383,25 @@ function MonitorPage() { )} + {/* Persistence indicator */} + + {/* Stats display */}
@@ -313,6 +421,12 @@ function MonitorPage() { debugMode={debugMode} logCollection={logCollection} logCount={logCount} + persistenceEnabled={persistenceEnabled} + persistenceStartedAt={persistenceStartedAt} + persistenceSessionCount={persistenceSessionCount} + persistenceActionCount={persistenceActionCount} + open={settingsOpen} + onOpenChange={setSettingsOpen} onHistoricalModeChange={handleHistoricalModeChange} onDebugModeChange={handleDebugModeChange} onLogCollectionChange={handleLogCollectionChange} @@ -321,6 +435,9 @@ function MonitorPage() { onConnect={handleConnect} onDisconnect={handleDisconnect} onRefresh={handleRefresh} + onPersistenceStart={handlePersistenceStart} + onPersistenceStop={handlePersistenceStop} + onPersistenceClear={handlePersistenceClear} />