Add persistence feature to monitor

- Introduced a persistence service to enable data retention across sessions.
- Updated SettingsPanel to manage persistence state and actions.
- Implemented hydration from server persistence in collections.
- Enhanced TRPC router with persistence-related queries and mutations.
- Updated monitor page to handle persistence status and actions.
This commit is contained in:
luccast
2026-01-26 20:25:12 -05:00
parent 37a4db51de
commit 6ccd394838
7 changed files with 429 additions and 8 deletions
+3
View File
@@ -34,3 +34,6 @@ src/routeTree.gen.ts
documents/*
.tanstack/tmp/*
# Persistence data
data/
+4 -4
View File
@@ -1,10 +1,12 @@
{
"name": "crabwalk",
"version": "1.0.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "crabwalk",
"version": "1.0.2",
"dependencies": {
"@tanstack/db": "^0.5.0",
"@tanstack/react-db": "^0.1.0",
@@ -37,8 +39,7 @@
"tailwindcss": "^4.0.0",
"typescript": "^5.7.0",
"vite": "^7.0.0"
},
"version": "1.0.2"
}
},
"node_modules/@babel/code-frame": {
"version": "7.28.6",
@@ -5525,6 +5526,5 @@
"url": "https://github.com/sponsors/wooorm"
}
}
},
"version": "1.0.2"
}
}
+73 -1
View File
@@ -1,6 +1,6 @@
import { useState } from 'react'
import { motion, AnimatePresence } from 'framer-motion'
import { Settings, X, History, Wifi, WifiOff, RefreshCw, Terminal, Download, Trash2, Database } from 'lucide-react'
import { Settings, X, History, Wifi, WifiOff, RefreshCw, Terminal, Download, Trash2, Database, HardDrive, Play, Square } from 'lucide-react'
interface SettingsPanelProps {
connected: boolean
@@ -8,6 +8,10 @@ interface SettingsPanelProps {
debugMode: boolean
logCollection: boolean
logCount: number
persistenceEnabled: boolean
persistenceStartedAt: number | null
persistenceSessionCount: number
persistenceActionCount: number
onHistoricalModeChange: (enabled: boolean) => void
onDebugModeChange: (enabled: boolean) => void
onLogCollectionChange: (enabled: boolean) => void
@@ -16,6 +20,9 @@ interface SettingsPanelProps {
onConnect: () => void
onDisconnect: () => void
onRefresh: () => void
onPersistenceStart: () => void
onPersistenceStop: () => void
onPersistenceClear: () => void
}
export function SettingsPanel({
@@ -24,6 +31,10 @@ export function SettingsPanel({
debugMode,
logCollection,
logCount,
persistenceEnabled,
persistenceStartedAt,
persistenceSessionCount,
persistenceActionCount,
onHistoricalModeChange,
onDebugModeChange,
onLogCollectionChange,
@@ -32,6 +43,9 @@ export function SettingsPanel({
onConnect,
onDisconnect,
onRefresh,
onPersistenceStart,
onPersistenceStop,
onPersistenceClear,
}: SettingsPanelProps) {
const [open, setOpen] = useState(false)
@@ -171,6 +185,64 @@ export function SettingsPanel({
</button>
</div>
{/* Background Service */}
<div className="panel-retro p-4">
<div className="flex items-center gap-3 mb-2">
<HardDrive size={18} className={persistenceEnabled ? 'text-neon-mint' : 'text-shell-500'} />
<span className="font-display text-sm font-medium text-gray-200 uppercase tracking-wide">
Background Service
</span>
</div>
<p className="font-console text-[10px] text-shell-500 mb-3">
<span className="text-crab-600">&gt;</span> persist data across refreshes
</p>
{persistenceEnabled && persistenceStartedAt && (
<div className="font-console text-[10px] text-neon-mint mb-2">
<span className="text-crab-600">&gt;</span> running since {new Date(persistenceStartedAt).toLocaleTimeString()}
</div>
)}
<div className="font-console text-[10px] text-shell-400 mb-3 space-y-1">
<div>
<span className="text-crab-600">&gt;</span> {persistenceSessionCount} sessions
</div>
<div>
<span className="text-crab-600">&gt;</span> {persistenceActionCount} actions
</div>
</div>
<div className="flex gap-2 mb-2">
{persistenceEnabled ? (
<button
onClick={onPersistenceStop}
className="flex-1 flex items-center justify-center gap-2 px-4 py-2 font-display text-xs uppercase tracking-wide bg-crab-600 hover:bg-crab-500 text-white rounded-lg transition-all"
>
<Square size={12} />
Stop
</button>
) : (
<button
onClick={onPersistenceStart}
className="flex-1 flex items-center justify-center gap-2 px-4 py-2 font-display text-xs uppercase tracking-wide bg-neon-mint/20 hover:bg-neon-mint/30 text-neon-mint rounded-lg transition-all"
>
<Play size={12} />
Start
</button>
)}
</div>
<button
onClick={onPersistenceClear}
disabled={persistenceSessionCount === 0 && persistenceActionCount === 0}
className="w-full flex items-center justify-center gap-2 px-3 py-2 font-display text-xs uppercase tracking-wide bg-shell-800 hover:bg-crab-900/50 rounded-lg transition-all disabled:opacity-40 disabled:cursor-not-allowed"
>
<Trash2 size={12} />
Clear Stored Data
</button>
</div>
{/* Log collection */}
<div className="panel-retro p-4">
<div className="flex items-center gap-3 mb-2">
+23
View File
@@ -158,3 +158,26 @@ export function clearCollections() {
actionsCollection.delete(action.id)
}
}
// Hydrate collections from server persistence
export function hydrateFromServer(
sessions: MonitorSession[],
actions: MonitorAction[]
) {
// First clear existing data
clearCollections()
// Insert all sessions
for (const session of sessions) {
sessionsCollection.insert(session)
}
// Insert all actions (rebuild runSessionMap as we go)
for (const action of actions) {
// Learn runId → sessionKey mapping
if (action.sessionKey && !action.sessionKey.includes('lifecycle')) {
runSessionMap.set(action.runId, action.sessionKey)
}
actionsCollection.insert(action)
}
}
+196
View File
@@ -0,0 +1,196 @@
import fs from 'fs'
import path from 'path'
import type { MonitorSession, MonitorAction } from './protocol'
const DATA_DIR = path.join(process.cwd(), 'data')
const SESSIONS_FILE = path.join(DATA_DIR, 'sessions.json')
const ACTIONS_FILE = path.join(DATA_DIR, 'actions.jsonl')
const STATE_FILE = path.join(DATA_DIR, 'state.json')
const MAX_ACTIONS = 10000
interface PersistenceState {
enabled: boolean
startedAt: number | null
}
class PersistenceService {
private sessions: Map<string, MonitorSession> = new Map()
private actions: MonitorAction[] = []
private enabled = false
private startedAt: number | null = null
constructor() {
this.ensureDataDir()
this.loadState()
this.loadData()
}
private ensureDataDir() {
if (!fs.existsSync(DATA_DIR)) {
fs.mkdirSync(DATA_DIR, { recursive: true })
}
}
private loadState() {
try {
if (fs.existsSync(STATE_FILE)) {
const data = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')) as PersistenceState
this.enabled = data.enabled
this.startedAt = data.startedAt
}
} catch {
// ignore
}
}
private saveState() {
const state: PersistenceState = {
enabled: this.enabled,
startedAt: this.startedAt,
}
fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2))
}
private loadData() {
// Load sessions
try {
if (fs.existsSync(SESSIONS_FILE)) {
const data = JSON.parse(fs.readFileSync(SESSIONS_FILE, 'utf-8')) as MonitorSession[]
for (const session of data) {
this.sessions.set(session.key, session)
}
}
} catch {
// ignore
}
// Load actions (JSONL)
try {
if (fs.existsSync(ACTIONS_FILE)) {
const content = fs.readFileSync(ACTIONS_FILE, 'utf-8')
const lines = content.trim().split('\n').filter(Boolean)
for (const line of lines) {
try {
const action = JSON.parse(line) as MonitorAction
this.actions.push(action)
} catch {
// skip bad lines
}
}
// Trim to max if needed
if (this.actions.length > MAX_ACTIONS) {
this.actions = this.actions.slice(-MAX_ACTIONS)
this.saveActions()
}
}
} catch {
// ignore
}
}
private saveSessions() {
const data = Array.from(this.sessions.values())
fs.writeFileSync(SESSIONS_FILE, JSON.stringify(data, null, 2))
}
private saveActions() {
const content = this.actions.map((a) => JSON.stringify(a)).join('\n')
fs.writeFileSync(ACTIONS_FILE, content)
}
private appendAction(action: MonitorAction) {
fs.appendFileSync(ACTIONS_FILE, JSON.stringify(action) + '\n')
}
get isEnabled() {
return this.enabled
}
start(): { enabled: boolean; startedAt: number } {
this.enabled = true
this.startedAt = Date.now()
this.saveState()
console.log('[persistence] started')
return { enabled: true, startedAt: this.startedAt }
}
stop(): { enabled: boolean } {
this.enabled = false
this.startedAt = null
this.saveState()
console.log('[persistence] stopped')
return { enabled: false }
}
getStatus(): {
enabled: boolean
startedAt: number | null
sessionCount: number
actionCount: number
} {
return {
enabled: this.enabled,
startedAt: this.startedAt,
sessionCount: this.sessions.size,
actionCount: this.actions.length,
}
}
upsertSession(session: MonitorSession) {
if (!this.enabled) return
this.sessions.set(session.key, session)
this.saveSessions()
}
addAction(action: MonitorAction) {
if (!this.enabled) return
// Check if action already exists (by id)
const existingIdx = this.actions.findIndex((a) => a.id === action.id)
if (existingIdx >= 0) {
// Update existing action
this.actions[existingIdx] = action
this.saveActions()
} else {
// Add new action
this.actions.push(action)
this.appendAction(action)
// Rotate if over limit
if (this.actions.length > MAX_ACTIONS) {
this.actions = this.actions.slice(-MAX_ACTIONS)
this.saveActions()
}
}
}
hydrate(): { sessions: MonitorSession[]; actions: MonitorAction[] } {
return {
sessions: Array.from(this.sessions.values()),
actions: [...this.actions],
}
}
clear(): { cleared: boolean } {
this.sessions.clear()
this.actions = []
try {
if (fs.existsSync(SESSIONS_FILE)) fs.unlinkSync(SESSIONS_FILE)
if (fs.existsSync(ACTIONS_FILE)) fs.unlinkSync(ACTIONS_FILE)
} catch {
// ignore
}
console.log('[persistence] cleared all data')
return { cleared: true }
}
}
// Singleton instance
let instance: PersistenceService | null = null
export function getPersistenceService(): PersistenceService {
if (!instance) {
instance = new PersistenceService()
}
return instance
}
+36 -2
View File
@@ -3,6 +3,7 @@ import { observable } from '@trpc/server/observable'
import superjson from 'superjson'
import { z } from 'zod'
import { getClawdbotClient } from '~/integrations/clawdbot/client'
import { getPersistenceService } from '~/integrations/clawdbot/persistence'
import {
parseEventFrame,
sessionInfoToMonitor,
@@ -114,14 +115,18 @@ const clawdbotRouter = router({
)
.query(async ({ input }) => {
const client = getClawdbotClient()
const persistence = getPersistenceService()
if (!client.connected) {
return { sessions: [], error: 'Not connected' }
}
try {
const sessions = await client.listSessions(input)
return {
sessions: sessions.map(sessionInfoToMonitor),
const monitorSessions = sessions.map(sessionInfoToMonitor)
// Persist sessions if service is enabled
for (const session of monitorSessions) {
persistence.upsertSession(session)
}
return { sessions: monitorSessions }
} catch (error) {
return {
sessions: [],
@@ -137,6 +142,7 @@ const clawdbotRouter = router({
action?: MonitorAction
}>((emit) => {
const client = getClawdbotClient()
const persistence = getPersistenceService()
const unsubscribe = client.onEvent((event) => {
// Collect raw event when log collection is enabled
@@ -161,6 +167,8 @@ const clawdbotRouter = router({
emit.next({ type: 'session', session: parsed.session })
}
if (parsed.action) {
// Persist action if service is enabled
persistence.addAction(parsed.action)
emit.next({ type: 'action', action: parsed.action })
}
}
@@ -171,6 +179,32 @@ const clawdbotRouter = router({
}
})
}),
// Persistence service
persistenceStatus: publicProcedure.query(() => {
const persistence = getPersistenceService()
return persistence.getStatus()
}),
persistenceStart: publicProcedure.mutation(() => {
const persistence = getPersistenceService()
return persistence.start()
}),
persistenceStop: publicProcedure.mutation(() => {
const persistence = getPersistenceService()
return persistence.stop()
}),
persistenceHydrate: publicProcedure.query(() => {
const persistence = getPersistenceService()
return persistence.hydrate()
}),
persistenceClear: publicProcedure.mutation(() => {
const persistence = getPersistenceService()
return persistence.clear()
}),
})
export const appRouter = router({
+94 -1
View File
@@ -11,6 +11,7 @@ import {
addAction,
updateSessionStatus,
clearCollections,
hydrateFromServer,
} from '~/integrations/clawdbot'
import {
ActionGraph,
@@ -67,6 +68,12 @@ function MonitorPage() {
const [logCount, setLogCount] = useState(0)
const [selectedSession, setSelectedSession] = useState<string | null>(null)
// Persistence service state
const [persistenceEnabled, setPersistenceEnabled] = useState(false)
const [persistenceStartedAt, setPersistenceStartedAt] = useState<number | null>(null)
const [persistenceSessionCount, setPersistenceSessionCount] = useState(0)
const [persistenceActionCount, setPersistenceActionCount] = useState(0)
// Sidebar collapse state - default to collapsed
const [sidebarCollapsed, setSidebarCollapsed] = useState(true)
@@ -78,11 +85,24 @@ function MonitorPage() {
const actions = actionsQuery.data ?? []
// Check connection status on mount
// Check connection status and persistence on mount
useEffect(() => {
checkStatus()
checkPersistenceStatus()
}, [])
const checkPersistenceStatus = async () => {
try {
const status = await trpc.clawdbot.persistenceStatus.query()
setPersistenceEnabled(status.enabled)
setPersistenceStartedAt(status.startedAt)
setPersistenceSessionCount(status.sessionCount)
setPersistenceActionCount(status.actionCount)
} catch {
// ignore
}
}
const checkStatus = async () => {
try {
const status = await trpc.clawdbot.status.query()
@@ -101,6 +121,8 @@ function MonitorPage() {
setConnected(true)
setRetryCount(0)
setConnecting(false)
// Hydrate from persistence if enabled
await hydrateFromPersistence()
await loadSessions()
return
}
@@ -115,6 +137,23 @@ function MonitorPage() {
}
}
const hydrateFromPersistence = async () => {
try {
const status = await trpc.clawdbot.persistenceStatus.query()
if (status.sessionCount > 0 || status.actionCount > 0) {
const data = await trpc.clawdbot.persistenceHydrate.query()
hydrateFromServer(data.sessions, data.actions)
console.log(`[monitor] hydrated ${data.sessions.length} sessions, ${data.actions.length} actions`)
}
setPersistenceEnabled(status.enabled)
setPersistenceStartedAt(status.startedAt)
setPersistenceSessionCount(status.sessionCount)
setPersistenceActionCount(status.actionCount)
} catch (e) {
console.error('Failed to hydrate:', e)
}
}
const handleDisconnect = async () => {
try {
await trpc.clawdbot.disconnect.mutate()
@@ -196,6 +235,37 @@ function MonitorPage() {
}
}
const handlePersistenceStart = async () => {
try {
const result = await trpc.clawdbot.persistenceStart.mutate()
setPersistenceEnabled(result.enabled)
setPersistenceStartedAt(result.startedAt)
} catch (e) {
console.error('Failed to start persistence:', e)
}
}
const handlePersistenceStop = async () => {
try {
const result = await trpc.clawdbot.persistenceStop.mutate()
setPersistenceEnabled(result.enabled)
setPersistenceStartedAt(null)
} catch (e) {
console.error('Failed to stop persistence:', e)
}
}
const handlePersistenceClear = async () => {
try {
await trpc.clawdbot.persistenceClear.mutate()
setPersistenceSessionCount(0)
setPersistenceActionCount(0)
clearCollections()
} catch (e) {
console.error('Failed to clear persistence:', e)
}
}
// Poll log count while collecting
useEffect(() => {
if (!logCollection) return
@@ -210,6 +280,22 @@ function MonitorPage() {
return () => clearInterval(interval)
}, [logCollection])
// Poll persistence status
useEffect(() => {
const interval = setInterval(async () => {
try {
const status = await trpc.clawdbot.persistenceStatus.query()
setPersistenceEnabled(status.enabled)
setPersistenceStartedAt(status.startedAt)
setPersistenceSessionCount(status.sessionCount)
setPersistenceActionCount(status.actionCount)
} catch {
// ignore
}
}, 5000)
return () => clearInterval(interval)
}, [])
const handleToggleSidebar = useCallback(() => {
setSidebarCollapsed((prev) => !prev)
}, [])
@@ -313,6 +399,10 @@ function MonitorPage() {
debugMode={debugMode}
logCollection={logCollection}
logCount={logCount}
persistenceEnabled={persistenceEnabled}
persistenceStartedAt={persistenceStartedAt}
persistenceSessionCount={persistenceSessionCount}
persistenceActionCount={persistenceActionCount}
onHistoricalModeChange={handleHistoricalModeChange}
onDebugModeChange={handleDebugModeChange}
onLogCollectionChange={handleLogCollectionChange}
@@ -321,6 +411,9 @@ function MonitorPage() {
onConnect={handleConnect}
onDisconnect={handleDisconnect}
onRefresh={handleRefresh}
onPersistenceStart={handlePersistenceStart}
onPersistenceStop={handlePersistenceStop}
onPersistenceClear={handlePersistenceClear}
/>
</div>
</header>