feat: add dedupe command

src/commands/registry.ts

@@ -5,6 +5,7 @@ import { pickCommand } from "./stdlib/pick.js";
 import { tableCommand } from "./stdlib/table.js";
 import { whereCommand } from "./stdlib/where.js";
 import { sortCommand } from "./stdlib/sort.js";
+import { dedupeCommand } from "./stdlib/dedupe.js";
 import { approveCommand } from "./stdlib/approve.js";
 import { clawdInvokeCommand } from "./stdlib/clawd_invoke.js";
 import { stateGetCommand, stateSetCommand } from "./stdlib/state.js";
@@ -27,6 +28,7 @@ export function createDefaultRegistry() {
     tableCommand,
     whereCommand,
     sortCommand,
+    dedupeCommand,
     approveCommand,
     clawdInvokeCommand,
     stateGetCommand,

src/commands/stdlib/dedupe.ts

@@ -0,0 +1,52 @@
+function getByPath(obj: any, path: string): any {
+  if (!path) return obj;
+  const parts = path.split('.').filter(Boolean);
+  let cur: any = obj;
+  for (const p of parts) {
+    if (cur == null) return undefined;
+    cur = cur[p];
+  }
+  return cur;
+}
+
+export const dedupeCommand = {
+  name: 'dedupe',
+  meta: {
+    description: 'Remove duplicate items, keeping first occurrence (stable)',
+    argsSchema: {
+      type: 'object',
+      properties: {
+        key: { type: 'string', description: 'Dot-path key used for identity (defaults to whole item)' },
+        _: { type: 'array', items: { type: 'string' } },
+      },
+      required: [],
+    },
+    sideEffects: [],
+  },
+  help() {
+    return (
+      `dedupe — remove duplicate items (stable)\n\n` +
+      `Usage:\n` +
+      `  ... | dedupe\n` +
+      `  ... | dedupe --key id\n\n` +
+      `Notes:\n` +
+      `  - Keeps the first occurrence.\n`
+    );
+  },
+  async run({ input, args }: any) {
+    const key = typeof args.key === 'string' ? args.key : undefined;
+    const seen = new Set<string>();
+
+    return {
+      output: (async function* () {
+        for await (const item of input) {
+          const id = key ? getByPath(item, key) : item;
+          const k = JSON.stringify(id);
+          if (seen.has(k)) continue;
+          seen.add(k);
+          yield item;
+        }
+      })(),
+    };
+  },
+};

test/dedupe.test.ts

@@ -0,0 +1,49 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import { runPipeline } from '../src/runtime.js';
+import { createDefaultRegistry } from '../src/commands/registry.js';
+import { parsePipeline } from '../src/parser.js';
+
+async function run(pipelineText: string, input: any[]) {
+  const pipeline = parsePipeline(pipelineText);
+  const registry = createDefaultRegistry();
+  const res = await runPipeline({
+    pipeline,
+    registry,
+    stdin: process.stdin,
+    stdout: process.stdout,
+    stderr: process.stderr,
+    env: process.env,
+    mode: 'tool',
+    input: (async function* () { for (const x of input) yield x; })(),
+  });
+  return res.items;
+}
+
+test('dedupe removes duplicate primitives (stable)', async () => {
+  const out = await run('dedupe', [1, 2, 1, 3, 2]);
+  assert.deepEqual(out, [1, 2, 3]);
+});
+
+test('dedupe supports --key', async () => {
+  const input = [
+    { id: 'a', v: 1 },
+    { id: 'b', v: 2 },
+    { id: 'a', v: 3 },
+  ];
+  const out = await run('dedupe --key id', input);
+  assert.deepEqual(out, [input[0], input[1]]);
+});
+
+test('dedupe treats undefined keys as a key value', async () => {
+  const input = [
+    { id: undefined, v: 1 },
+    { id: undefined, v: 2 },
+    { id: 'x', v: 3 },
+  ];
+  const out = await run('dedupe --key id', input);
+  assert.equal(out.length, 2);
+  assert.equal(out[0].v, 1);
+  assert.equal(out[1].v, 3);
+});