mirror of
https://github.com/moeru-ai/airi.git
synced 2026-08-14 08:52:42 +00:00
feat(airi-plugin-web-extension): integrated to server-sdk, now supports to read context from browser
This commit is contained in:
Neko Ayaka
@@ -3,3 +3,17 @@
|
||||
> Read what you are reading!
|
||||
|
||||
This is a plugin for the AIRI to understand what you are reading, looking at, or listening to on the web.
|
||||
|
||||
## What it does now
|
||||
|
||||
- Captures page + video context from YouTube and Bilibili.
|
||||
- Extracts subtitles from text tracks or DOM overlays.
|
||||
- Sends context updates and optional `spark:notify` events to the character.
|
||||
- Exposes a popup to configure WebSocket, toggles, and quick status.
|
||||
|
||||
## Quick start
|
||||
|
||||
1. `pnpm -F @proj-airi/airi-plugin-web-extension dev`
|
||||
2. Load the unpacked extension from `.wxt/dev` in your browser.
|
||||
3. Open the popup to set the WebSocket URL (default: `ws://localhost:6121/ws`).
|
||||
4. Watch a YouTube/Bilibili video and confirm the popup shows the detected title/subtitle.
|
||||
|
||||
@@ -1,4 +1,159 @@
|
||||
import type {
|
||||
BackgroundToContentMessage,
|
||||
ContentToBackgroundMessage,
|
||||
ExtensionSettings,
|
||||
PopupToBackgroundMessage,
|
||||
} from '../src/shared/types'
|
||||
|
||||
import {
|
||||
createClientState,
|
||||
ensureClient,
|
||||
handlePageContext,
|
||||
handleSubtitle,
|
||||
handleVideoContext,
|
||||
toStatus,
|
||||
} from '../src/background/client'
|
||||
import { loadSettings, saveSettings } from '../src/background/storage'
|
||||
import { DEFAULT_SETTINGS, STORAGE_KEY } from '../src/shared/constants'
|
||||
import { detectSiteFromUrl } from '../src/shared/sites'
|
||||
|
||||
const state = createClientState()
|
||||
|
||||
let settings: ExtensionSettings = { ...DEFAULT_SETTINGS }
|
||||
let lastVideoNotifyKey = ''
|
||||
let lastStatusSentAt = 0
|
||||
let connectionKey = ''
|
||||
|
||||
async function refreshClient() {
|
||||
const nextKey = `${settings.enabled}:${settings.wsUrl}:${settings.token}`
|
||||
if (nextKey !== connectionKey) {
|
||||
connectionKey = nextKey
|
||||
if (state.client)
|
||||
state.client.close()
|
||||
state.client = null
|
||||
state.connected = false
|
||||
}
|
||||
await ensureClient(state, settings)
|
||||
}
|
||||
|
||||
function buildNotifyKey(payload: { url: string, title?: string, videoId?: string }) {
|
||||
return [payload.videoId, payload.title, payload.url].filter(Boolean).join('|')
|
||||
}
|
||||
|
||||
function shouldNotifyVideo(payload: { url: string, title?: string, videoId?: string }) {
|
||||
const key = buildNotifyKey(payload)
|
||||
if (!key || key === lastVideoNotifyKey)
|
||||
return false
|
||||
lastVideoNotifyKey = key
|
||||
return true
|
||||
}
|
||||
|
||||
function emitStatus() {
|
||||
const now = Date.now()
|
||||
if (now - lastStatusSentAt < 300)
|
||||
return
|
||||
|
||||
lastStatusSentAt = now
|
||||
void browser.runtime.sendMessage({ type: 'background:status', payload: toStatus(state, settings) }).catch(() => {})
|
||||
}
|
||||
|
||||
async function updateSettings(partial: Partial<ExtensionSettings>) {
|
||||
settings = await saveSettings(partial)
|
||||
await refreshClient()
|
||||
emitStatus()
|
||||
}
|
||||
|
||||
async function init() {
|
||||
settings = await loadSettings()
|
||||
await refreshClient()
|
||||
emitStatus()
|
||||
}
|
||||
|
||||
function handleContentMessage(message: ContentToBackgroundMessage) {
|
||||
switch (message.type) {
|
||||
case 'content:page': {
|
||||
const payload = {
|
||||
...message.payload,
|
||||
site: message.payload.site === 'unknown' ? detectSiteFromUrl(message.payload.url) : message.payload.site,
|
||||
}
|
||||
handlePageContext(state, settings, payload)
|
||||
emitStatus()
|
||||
break
|
||||
}
|
||||
case 'content:video': {
|
||||
const payload = {
|
||||
...message.payload,
|
||||
site: message.payload.site === 'unknown' ? detectSiteFromUrl(message.payload.url) : message.payload.site,
|
||||
}
|
||||
handleVideoContext(state, settings, payload, { notify: shouldNotifyVideo(payload) })
|
||||
emitStatus()
|
||||
break
|
||||
}
|
||||
case 'content:subtitle': {
|
||||
const payload = {
|
||||
...message.payload,
|
||||
site: message.payload.site === 'unknown' ? detectSiteFromUrl(message.payload.url) : message.payload.site,
|
||||
}
|
||||
handleSubtitle(state, settings, payload)
|
||||
emitStatus()
|
||||
break
|
||||
}
|
||||
case 'content:vision:frame': {
|
||||
state.lastVisionFrameAt = Date.now()
|
||||
emitStatus()
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function handlePopupMessage(message: PopupToBackgroundMessage) {
|
||||
switch (message.type) {
|
||||
case 'popup:get-status':
|
||||
return toStatus(state, settings)
|
||||
case 'popup:update-settings':
|
||||
await updateSettings(message.payload)
|
||||
return toStatus(state, settings)
|
||||
case 'popup:toggle-enabled':
|
||||
await updateSettings({ enabled: message.payload })
|
||||
return toStatus(state, settings)
|
||||
case 'popup:request-vision-frame': {
|
||||
const message: BackgroundToContentMessage = { type: 'background:request-vision-frame' }
|
||||
const tabs = await browser.tabs.query({ active: true, currentWindow: true })
|
||||
const tab = tabs[0]
|
||||
if (tab?.id != null) {
|
||||
await browser.tabs.sendMessage(tab.id, message).catch(() => {})
|
||||
}
|
||||
return toStatus(state, settings)
|
||||
}
|
||||
case 'popup:clear-error':
|
||||
state.lastError = undefined
|
||||
emitStatus()
|
||||
return toStatus(state, settings)
|
||||
}
|
||||
}
|
||||
|
||||
export default defineBackground(() => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Hello background!', { id: browser.runtime.id })
|
||||
void init()
|
||||
|
||||
browser.runtime.onMessage.addListener((message: ContentToBackgroundMessage | PopupToBackgroundMessage) => {
|
||||
if (message && typeof message === 'object' && 'type' in message) {
|
||||
if (message.type.startsWith('content:')) {
|
||||
handleContentMessage(message as ContentToBackgroundMessage)
|
||||
return
|
||||
}
|
||||
|
||||
if (message.type.startsWith('popup:')) {
|
||||
return handlePopupMessage(message as PopupToBackgroundMessage)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
browser.storage.onChanged.addListener((changes) => {
|
||||
if (changes[STORAGE_KEY]) {
|
||||
const next = changes[STORAGE_KEY].newValue as ExtensionSettings | undefined
|
||||
settings = { ...DEFAULT_SETTINGS, ...next }
|
||||
void refreshClient()
|
||||
emitStatus()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
import { startContentObserver } from '../src/content'
|
||||
|
||||
export default defineContentScript({
|
||||
matches: ['*://*.google.com/*'],
|
||||
matches: [
|
||||
'*://*.youtube.com/*',
|
||||
'*://*.youtu.be/*',
|
||||
'*://*.bilibili.com/*',
|
||||
'*://*.b23.tv/*',
|
||||
],
|
||||
runAt: 'document_idle',
|
||||
main() {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Hello content.')
|
||||
startContentObserver()
|
||||
},
|
||||
})
|
||||
|
||||
@@ -1,15 +1,53 @@
|
||||
<script lang="ts" setup>
|
||||
import HelloWorld from '../../components/HelloWorld.vue'
|
||||
import { Button, Callout } from '@proj-airi/ui'
|
||||
import { onMounted } from 'vue'
|
||||
|
||||
import {
|
||||
HeaderPopup,
|
||||
PreferenceCapture,
|
||||
SettingsConnection,
|
||||
VisualizeLiveVision,
|
||||
} from './components'
|
||||
import { usePopupStore } from './stores'
|
||||
|
||||
const popup = usePopupStore()
|
||||
|
||||
onMounted(() => popup.init())
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div flex justify-center>
|
||||
<a href="https://wxt.dev" target="_blank">
|
||||
<img src="/wxt.svg" h-24 p-6 transition-filter duration-300 will-change-filter class="hover:drop-shadow-[0_0_2em_#54bc4ae0]" alt="WXT logo">
|
||||
</a>
|
||||
<a href="https://vuejs.org/" target="_blank">
|
||||
<img src="@/assets/vue.svg" h-24 p-6 transition-filter duration-300 will-change-filter class="hover:drop-shadow-[0_0_2em_#42b883aa]" alt="Vue logo">
|
||||
</a>
|
||||
</div>
|
||||
<HelloWorld msg="WXT + Vue" />
|
||||
<main :class="['flex', 'flex-col', 'gap-4', 'w-full']">
|
||||
<HeaderPopup :syncing="popup.syncing.value" :connected="popup.connected.value" @refresh="popup.refresh" />
|
||||
|
||||
<Callout v-if="popup.lastError.value" theme="orange" label="Connection error">
|
||||
<div :class="['flex', 'items-start', 'gap-2']">
|
||||
<div :class="['flex-1', 'text-xs', 'leading-snug', 'opacity-80']">
|
||||
{{ popup.lastError.value }}
|
||||
</div>
|
||||
<Button variant="danger" size="sm" @click="popup.clearLastError">
|
||||
Clear
|
||||
</Button>
|
||||
</div>
|
||||
</Callout>
|
||||
|
||||
<PreferenceCapture
|
||||
v-model:send-page-context="popup.form.sendPageContext"
|
||||
v-model:send-video-context="popup.form.sendVideoContext"
|
||||
v-model:send-subtitles="popup.form.sendSubtitles"
|
||||
v-model:send-spark-notify="popup.form.sendSparkNotify"
|
||||
v-model:enable-vision="popup.form.enableVision"
|
||||
@capture="popup.captureFrame"
|
||||
/>
|
||||
|
||||
<VisualizeLiveVision :last-video="popup.lastVideo.value" :last-subtitle="popup.lastSubtitle.value" />
|
||||
|
||||
<SettingsConnection
|
||||
v-model:ws-url="popup.form.wsUrl"
|
||||
v-model:token="popup.form.token"
|
||||
:enabled="popup.form.enabled"
|
||||
:syncing="popup.syncing.value"
|
||||
@toggle="popup.toggle"
|
||||
@apply="popup.applySettings"
|
||||
/>
|
||||
</main>
|
||||
</template>
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export { default as HeaderPopup } from './popup.vue'
|
||||
@@ -0,0 +1,35 @@
|
||||
<script lang="ts" setup>
|
||||
defineProps<{
|
||||
syncing: boolean
|
||||
connected: boolean
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
(event: 'refresh'): void
|
||||
}>()
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<header :class="['flex', 'items-center', 'justify-between', 'gap-3']">
|
||||
<div :class="['flex', 'flex-col', 'gap-1']">
|
||||
<h1 :class="['text-lg', 'font-600', 'tracking-tight']">
|
||||
AIRI Web Extension
|
||||
</h1>
|
||||
<p :class="['text-xs', 'opacity-70']">
|
||||
Context provider for AIRI Stage
|
||||
</p>
|
||||
</div>
|
||||
<div :class="['flex', 'items-center', 'gap-2']">
|
||||
<button
|
||||
:disabled="syncing"
|
||||
:class="['transition', syncing ? 'opacity-60 cursor-not-allowed' : '']"
|
||||
@click="emit('refresh')"
|
||||
>
|
||||
<div :class="[syncing ? 'i-svg-spinners:ring-resize' : 'i-solar:refresh-linear', 'size-4']" />
|
||||
</button>
|
||||
<span :class="['px-2', 'py-1', 'rounded-full', 'text-xs', 'font-600', connected ? 'bg-emerald-400/20' : 'bg-rose-400/20', connected ? 'text-emerald-100' : 'text-rose-100']">
|
||||
{{ connected ? 'Connected' : 'Offline' }}
|
||||
</span>
|
||||
</div>
|
||||
</header>
|
||||
</template>
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from './header'
|
||||
export * from './sections'
|
||||
@@ -0,0 +1,3 @@
|
||||
export { default as SettingsConnection } from './settings/connection.vue'
|
||||
export { default as PreferenceCapture } from './settings/preference-capture.vue'
|
||||
export { default as VisualizeLiveVision } from './visualize-live-vision.vue'
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
<script lang="ts" setup>
|
||||
import { Button, FieldInput } from '@proj-airi/ui'
|
||||
|
||||
defineProps<{
|
||||
enabled: boolean
|
||||
syncing: boolean
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
(event: 'toggle'): void
|
||||
(event: 'apply'): void
|
||||
}>()
|
||||
const wsUrlModel = defineModel<string>('ws-url', { required: true })
|
||||
const tokenModel = defineModel<string>('token', { required: true })
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<section :class="['rounded-2xl', 'bg-white/6', 'border', 'border-white/10', 'p-3', 'flex', 'flex-col', 'gap-3']">
|
||||
<div :class="['flex', 'items-center', 'justify-between']">
|
||||
<h2 :class="['text-sm', 'font-600']">
|
||||
Connection
|
||||
</h2>
|
||||
<Button variant="secondary" size="sm" @click="emit('toggle')">
|
||||
{{ enabled ? 'Disable' : 'Enable' }}
|
||||
</Button>
|
||||
</div>
|
||||
<FieldInput v-model="wsUrlModel" label="WebSocket URL" placeholder="ws://localhost:6121/ws" />
|
||||
<FieldInput v-model="tokenModel" label="Access Token" placeholder="optional" />
|
||||
<Button variant="primary" size="sm" :disabled="syncing" @click="emit('apply')">
|
||||
Apply settings
|
||||
</Button>
|
||||
</section>
|
||||
</template>
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
<script lang="ts" setup>
|
||||
import { Button, FieldCheckbox } from '@proj-airi/ui'
|
||||
|
||||
const emit = defineEmits<{
|
||||
(event: 'capture'): void
|
||||
}>()
|
||||
const sendPageContextModel = defineModel<boolean>('send-page-context', { required: true })
|
||||
const sendVideoContextModel = defineModel<boolean>('send-video-context', { required: true })
|
||||
const sendSubtitlesModel = defineModel<boolean>('send-subtitles', { required: true })
|
||||
const sendSparkNotifyModel = defineModel<boolean>('send-spark-notify', { required: true })
|
||||
const enableVisionModel = defineModel<boolean>('enable-vision', { required: true })
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<section :class="['rounded-2xl', 'bg-white/6', 'border', 'border-white/10', 'p-3', 'flex', 'flex-col', 'gap-3']">
|
||||
<h2 :class="['text-sm', 'font-600']">
|
||||
Capture Controls
|
||||
</h2>
|
||||
<div :class="['grid', 'grid-cols-1', 'gap-3']">
|
||||
<FieldCheckbox v-model="sendPageContextModel" label="Page context" />
|
||||
<FieldCheckbox v-model="sendVideoContextModel" label="Video context" />
|
||||
<FieldCheckbox v-model="sendSubtitlesModel" label="Subtitles" />
|
||||
<FieldCheckbox v-model="sendSparkNotifyModel" label="Notify character" />
|
||||
<FieldCheckbox v-model="enableVisionModel" label="Vision capture (manual)" />
|
||||
</div>
|
||||
<Button
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
:disabled="!enableVision"
|
||||
@click="emit('capture')"
|
||||
>
|
||||
Capture frame
|
||||
</Button>
|
||||
</section>
|
||||
</template>
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
<script lang="ts" setup>
|
||||
import type { SubtitlePayload, VideoContextPayload } from '../../../../src/shared/types'
|
||||
|
||||
defineProps<{
|
||||
lastVideo?: VideoContextPayload
|
||||
lastSubtitle?: SubtitlePayload
|
||||
}>()
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<section :class="['rounded-2xl', 'bg-white/6', 'border', 'border-white/10', 'p-3', 'flex', 'flex-col', 'gap-3']">
|
||||
<h2 :class="['text-sm', 'font-600']">
|
||||
Live Preview
|
||||
</h2>
|
||||
<div :class="['text-xs', 'flex', 'flex-col', 'gap-2', 'leading-relaxed']">
|
||||
<div :class="['opacity-80']">
|
||||
{{ lastVideo?.title || 'No active video detected yet.' }}
|
||||
</div>
|
||||
<div v-if="lastVideo" :class="['opacity-60']">
|
||||
{{ lastVideo.channel ? `Channel: ${lastVideo.channel}` : 'Channel: unknown' }}
|
||||
</div>
|
||||
<div v-if="lastVideo?.url" :class="['opacity-50', 'break-all']">
|
||||
{{ lastVideo.url }}
|
||||
</div>
|
||||
<div v-if="lastSubtitle?.text" :class="['mt-2', 'bg-black/30', 'border', 'border-white/10', 'rounded-lg', 'p-2']">
|
||||
“{{ lastSubtitle.text }}”
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</template>
|
||||
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Default Popup Title</title>
|
||||
<title>AIRI Web Extension</title>
|
||||
<meta name="manifest.type" content="browser_action" />
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export * from './popup'
|
||||
@@ -0,0 +1,128 @@
|
||||
import type { ExtensionSettings, ExtensionStatus } from '../../../src/shared/types'
|
||||
|
||||
import { createGlobalState } from '@vueuse/core'
|
||||
import { computed, reactive, ref, watch } from 'vue'
|
||||
|
||||
import { clearError, onBackgroundStatus, requestStatus, requestVisionFrame, toggleEnabled, updateSettings } from '../../../src/popup/bridge'
|
||||
|
||||
const STORAGE_KEY = 'airi-popup-settings'
|
||||
|
||||
export const usePopupStore = createGlobalState(() => {
|
||||
const status = ref<ExtensionStatus | null>(null)
|
||||
const syncing = ref(true)
|
||||
const initialized = ref(false)
|
||||
|
||||
const form = reactive<ExtensionSettings>({
|
||||
wsUrl: '',
|
||||
token: '',
|
||||
enabled: true,
|
||||
sendPageContext: true,
|
||||
sendVideoContext: true,
|
||||
sendSubtitles: true,
|
||||
sendSparkNotify: true,
|
||||
enableVision: false,
|
||||
})
|
||||
|
||||
const connected = computed(() => status.value?.connected ?? false)
|
||||
const lastVideo = computed(() => status.value?.lastVideo)
|
||||
const lastSubtitle = computed(() => status.value?.lastSubtitle)
|
||||
const lastError = computed(() => status.value?.lastError)
|
||||
|
||||
function hydrate(next: ExtensionStatus) {
|
||||
status.value = next
|
||||
Object.assign(form, next.settings)
|
||||
}
|
||||
|
||||
function loadStoredSettings() {
|
||||
try {
|
||||
const raw = localStorage.getItem(STORAGE_KEY)
|
||||
if (!raw)
|
||||
return
|
||||
const parsed = JSON.parse(raw) as Partial<ExtensionSettings>
|
||||
Object.assign(form, parsed)
|
||||
}
|
||||
catch {
|
||||
localStorage.removeItem(STORAGE_KEY)
|
||||
}
|
||||
}
|
||||
|
||||
function persistSettings() {
|
||||
const payload: ExtensionSettings = { ...form }
|
||||
localStorage.setItem(STORAGE_KEY, JSON.stringify(payload))
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
syncing.value = true
|
||||
try {
|
||||
const next = await requestStatus()
|
||||
hydrate(next)
|
||||
}
|
||||
finally {
|
||||
syncing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function applySettings() {
|
||||
syncing.value = true
|
||||
try {
|
||||
const next = await updateSettings({ ...form })
|
||||
hydrate(next)
|
||||
}
|
||||
finally {
|
||||
syncing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function toggle() {
|
||||
syncing.value = true
|
||||
try {
|
||||
const next = await toggleEnabled(!form.enabled)
|
||||
hydrate(next)
|
||||
}
|
||||
finally {
|
||||
syncing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function captureFrame() {
|
||||
syncing.value = true
|
||||
try {
|
||||
const next = await requestVisionFrame()
|
||||
hydrate(next)
|
||||
}
|
||||
finally {
|
||||
syncing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function clearLastError() {
|
||||
const next = await clearError()
|
||||
hydrate(next)
|
||||
}
|
||||
|
||||
function init() {
|
||||
if (initialized.value)
|
||||
return
|
||||
initialized.value = true
|
||||
loadStoredSettings()
|
||||
watch(form, persistSettings, { deep: true })
|
||||
void refresh()
|
||||
onBackgroundStatus(hydrate)
|
||||
}
|
||||
|
||||
return {
|
||||
status,
|
||||
syncing,
|
||||
form,
|
||||
connected,
|
||||
lastVideo,
|
||||
lastSubtitle,
|
||||
lastError,
|
||||
init,
|
||||
refresh,
|
||||
applySettings,
|
||||
toggle,
|
||||
captureFrame,
|
||||
clearLastError,
|
||||
}
|
||||
})
|
||||
@@ -1,11 +1,10 @@
|
||||
:root {
|
||||
font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
|
||||
line-height: 1.5;
|
||||
font-weight: 400;
|
||||
--bg-color-light: rgb(255 255 255);
|
||||
--bg-color-dark: rgb(18 18 18);
|
||||
--bg-color: var(--bg-color-light);
|
||||
|
||||
color-scheme: light dark;
|
||||
color: rgba(255, 255, 255, 0.87);
|
||||
background-color: #242424;
|
||||
background-color: var(--bg-color-dark);
|
||||
|
||||
font-synthesis: none;
|
||||
text-rendering: optimizeLegibility;
|
||||
@@ -14,67 +13,23 @@
|
||||
-webkit-text-size-adjust: 100%;
|
||||
}
|
||||
|
||||
a {
|
||||
font-weight: 500;
|
||||
color: #646cff;
|
||||
text-decoration: inherit;
|
||||
}
|
||||
a:hover {
|
||||
color: #535bf2;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
display: flex;
|
||||
place-items: center;
|
||||
min-width: 320px;
|
||||
min-width: 600px;
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 3.2em;
|
||||
line-height: 1.1;
|
||||
}
|
||||
|
||||
button {
|
||||
border-radius: 8px;
|
||||
border: 1px solid transparent;
|
||||
padding: 0.6em 1.2em;
|
||||
font-size: 1em;
|
||||
font-weight: 500;
|
||||
font-family: inherit;
|
||||
background-color: #1a1a1a;
|
||||
cursor: pointer;
|
||||
transition: border-color 0.25s;
|
||||
}
|
||||
button:hover {
|
||||
border-color: #646cff;
|
||||
}
|
||||
button:focus,
|
||||
button:focus-visible {
|
||||
outline: 4px auto -webkit-focus-ring-color;
|
||||
}
|
||||
|
||||
.card {
|
||||
padding: 2em;
|
||||
}
|
||||
|
||||
#app {
|
||||
width: 100%;
|
||||
max-width: 1280px;
|
||||
margin: 0 auto;
|
||||
padding: 2rem;
|
||||
text-align: center;
|
||||
padding: 1rem;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: light) {
|
||||
:root {
|
||||
color: #213547;
|
||||
background-color: #ffffff;
|
||||
}
|
||||
a:hover {
|
||||
color: #747bff;
|
||||
}
|
||||
button {
|
||||
background-color: #f9f9f9;
|
||||
background-color: var(--bg-color-light);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,11 +15,17 @@
|
||||
"postinstall": "wxt prepare"
|
||||
},
|
||||
"dependencies": {
|
||||
"vue": "catalog:"
|
||||
"@vueuse/core": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@iconify-json/solar": "^1.2.5",
|
||||
"@iconify-json/svg-spinners": "^1.2.4",
|
||||
"@proj-airi/server-sdk": "workspace:^",
|
||||
"@proj-airi/ui": "workspace:^",
|
||||
"@unocss/reset": "^66.5.11",
|
||||
"@wxt-dev/module-vue": "^1.0.3",
|
||||
"nanoid": "^5.1.6",
|
||||
"vue": "catalog:",
|
||||
"vue-tsc": "^3.1.8",
|
||||
"wxt": "^0.20.13"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
import type { ContextUpdate } from '@proj-airi/server-sdk'
|
||||
|
||||
import type { ExtensionSettings, ExtensionStatus, PageContextPayload, SubtitlePayload, VideoContextPayload } from '../shared/types'
|
||||
|
||||
import { Client, ContextUpdateStrategy } from '@proj-airi/server-sdk'
|
||||
import { nanoid } from 'nanoid'
|
||||
|
||||
import packageJSON from '../../package.json'
|
||||
|
||||
const PLUGIN_NAME = 'proj-airi:plugin-web-extension'
|
||||
|
||||
export interface ClientState {
|
||||
client: Client | null
|
||||
connected: boolean
|
||||
lastError?: string
|
||||
lastPage?: PageContextPayload
|
||||
lastVideo?: VideoContextPayload
|
||||
lastSubtitle?: SubtitlePayload
|
||||
lastVisionFrameAt?: number
|
||||
}
|
||||
|
||||
export function createClientState(): ClientState {
|
||||
return {
|
||||
client: null,
|
||||
connected: false,
|
||||
}
|
||||
}
|
||||
|
||||
function createIdentity() {
|
||||
return {
|
||||
plugin: PLUGIN_NAME,
|
||||
instanceId: nanoid(),
|
||||
version: typeof packageJSON.version === 'string' ? packageJSON.version : undefined,
|
||||
labels: {
|
||||
runtime: 'web-extension',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function toStatus(state: ClientState, settings: ExtensionSettings): ExtensionStatus {
|
||||
return {
|
||||
connected: state.connected,
|
||||
lastError: state.lastError,
|
||||
settings,
|
||||
lastPage: state.lastPage,
|
||||
lastVideo: state.lastVideo,
|
||||
lastSubtitle: state.lastSubtitle,
|
||||
lastVisionFrameAt: state.lastVisionFrameAt,
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureClient(state: ClientState, settings: ExtensionSettings) {
|
||||
if (!settings.enabled) {
|
||||
disconnectClient(state)
|
||||
return
|
||||
}
|
||||
|
||||
if (state.client) {
|
||||
return
|
||||
}
|
||||
|
||||
const client = new Client({
|
||||
name: PLUGIN_NAME,
|
||||
url: settings.wsUrl,
|
||||
token: settings.token || undefined,
|
||||
identity: createIdentity(),
|
||||
possibleEvents: ['context:update', 'spark:notify', 'spark:emit'],
|
||||
autoConnect: false,
|
||||
autoReconnect: true,
|
||||
onError: (error) => {
|
||||
state.connected = false
|
||||
state.lastError = error instanceof Error ? error.message : String(error)
|
||||
},
|
||||
onClose: () => {
|
||||
state.connected = false
|
||||
},
|
||||
})
|
||||
|
||||
state.client = client
|
||||
|
||||
try {
|
||||
await client.connect()
|
||||
state.connected = true
|
||||
state.lastError = undefined
|
||||
}
|
||||
catch (error) {
|
||||
state.connected = false
|
||||
state.lastError = error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
}
|
||||
|
||||
export function disconnectClient(state: ClientState) {
|
||||
if (!state.client)
|
||||
return
|
||||
|
||||
state.client.close()
|
||||
state.client = null
|
||||
state.connected = false
|
||||
}
|
||||
|
||||
function sendContextUpdate(state: ClientState, update: Omit<ContextUpdate, 'id' | 'contextId'> & Partial<Pick<ContextUpdate, 'id' | 'contextId'>>) {
|
||||
if (!state.client || !state.connected)
|
||||
return
|
||||
|
||||
const id = update.id ?? nanoid()
|
||||
state.client.send({
|
||||
type: 'context:update',
|
||||
data: {
|
||||
id,
|
||||
contextId: update.contextId ?? id,
|
||||
...update,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
function sendSparkNotify(state: ClientState, data: { headline: string, note?: string, payload?: Record<string, unknown> }) {
|
||||
if (!state.client || !state.connected)
|
||||
return
|
||||
|
||||
state.client.send({
|
||||
type: 'spark:notify',
|
||||
data: {
|
||||
id: nanoid(),
|
||||
eventId: nanoid(),
|
||||
kind: 'ping',
|
||||
urgency: 'soon',
|
||||
headline: data.headline,
|
||||
note: data.note,
|
||||
payload: data.payload,
|
||||
destinations: ['character'],
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function handlePageContext(state: ClientState, settings: ExtensionSettings, payload: PageContextPayload) {
|
||||
state.lastPage = payload
|
||||
|
||||
if (!settings.enabled || !settings.sendPageContext)
|
||||
return
|
||||
|
||||
sendContextUpdate(state, {
|
||||
strategy: ContextUpdateStrategy.ReplaceSelf,
|
||||
lane: 'web:page',
|
||||
text: `User is browsing: ${payload.title} (${payload.url}).`,
|
||||
metadata: {
|
||||
source: 'web-extension',
|
||||
site: payload.site,
|
||||
url: payload.url,
|
||||
title: payload.title,
|
||||
description: payload.description,
|
||||
language: payload.language,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function handleVideoContext(
|
||||
state: ClientState,
|
||||
settings: ExtensionSettings,
|
||||
payload: VideoContextPayload,
|
||||
options?: { notify?: boolean },
|
||||
) {
|
||||
state.lastVideo = payload
|
||||
|
||||
if (!settings.enabled || !settings.sendVideoContext)
|
||||
return
|
||||
|
||||
const headline = payload.title
|
||||
? `User is watching: ${payload.title}`
|
||||
: 'User is watching a video'
|
||||
|
||||
if (settings.sendSparkNotify && options?.notify !== false && payload.title) {
|
||||
sendSparkNotify(state, {
|
||||
headline,
|
||||
note: payload.channel ? `Channel: ${payload.channel}` : undefined,
|
||||
payload: {
|
||||
site: payload.site,
|
||||
url: payload.url,
|
||||
title: payload.title,
|
||||
channel: payload.channel,
|
||||
videoId: payload.videoId,
|
||||
durationSec: payload.durationSec,
|
||||
currentTimeSec: payload.currentTimeSec,
|
||||
isPlaying: payload.isPlaying,
|
||||
isLive: payload.isLive,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
sendContextUpdate(state, {
|
||||
strategy: ContextUpdateStrategy.ReplaceSelf,
|
||||
lane: 'web:video',
|
||||
text: [
|
||||
headline,
|
||||
payload.channel ? `Channel: ${payload.channel}.` : undefined,
|
||||
payload.currentTimeSec != null
|
||||
? `Progress: ${Math.floor(payload.currentTimeSec)}s${payload.durationSec ? ` / ${Math.floor(payload.durationSec)}s` : ''}.`
|
||||
: undefined,
|
||||
payload.url ? `URL: ${payload.url}.` : undefined,
|
||||
].filter(Boolean).join(' '),
|
||||
metadata: {
|
||||
source: 'web-extension',
|
||||
site: payload.site,
|
||||
url: payload.url,
|
||||
title: payload.title,
|
||||
channel: payload.channel,
|
||||
videoId: payload.videoId,
|
||||
durationSec: payload.durationSec,
|
||||
currentTimeSec: payload.currentTimeSec,
|
||||
isPlaying: payload.isPlaying,
|
||||
playbackRate: payload.playbackRate,
|
||||
isLive: payload.isLive,
|
||||
playerSize: payload.playerSize,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function handleSubtitle(state: ClientState, settings: ExtensionSettings, payload: SubtitlePayload) {
|
||||
state.lastSubtitle = payload
|
||||
|
||||
if (!settings.enabled || !settings.sendSubtitles)
|
||||
return
|
||||
|
||||
sendContextUpdate(state, {
|
||||
strategy: ContextUpdateStrategy.ReplaceSelf,
|
||||
lane: 'web:subtitle',
|
||||
text: `Subtitle: ${payload.text}`,
|
||||
metadata: {
|
||||
source: 'web-extension',
|
||||
site: payload.site,
|
||||
url: payload.url,
|
||||
title: payload.title,
|
||||
videoId: payload.videoId,
|
||||
language: payload.language,
|
||||
startMs: payload.startMs,
|
||||
endMs: payload.endMs,
|
||||
isAuto: payload.isAuto,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { ExtensionSettings } from '../shared/types'
|
||||
|
||||
import { DEFAULT_SETTINGS, STORAGE_KEY } from '../shared/constants'
|
||||
|
||||
export async function loadSettings(): Promise<ExtensionSettings> {
|
||||
const stored = await browser.storage.local.get(STORAGE_KEY)
|
||||
const value = stored[STORAGE_KEY] as ExtensionSettings | undefined
|
||||
return {
|
||||
...DEFAULT_SETTINGS,
|
||||
...value,
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveSettings(partial: Partial<ExtensionSettings>): Promise<ExtensionSettings> {
|
||||
const next = {
|
||||
...DEFAULT_SETTINGS,
|
||||
...(await loadSettings()),
|
||||
...partial,
|
||||
}
|
||||
|
||||
await browser.storage.local.set({ [STORAGE_KEY]: next })
|
||||
return next
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
import type { BackgroundToContentMessage, ContentToBackgroundMessage, PageContextPayload, SubtitlePayload, VideoContextPayload, VideoSite, VisionFramePayload } from '../shared/types'
|
||||
|
||||
import { detectSiteFromUrl, extractVideoId, normalizeText } from '../shared/sites'
|
||||
|
||||
const VIDEO_PROGRESS_INTERVAL = 15000
|
||||
const TITLE_POLL_INTERVAL = 2000
|
||||
const SUBTITLE_DEDUPE_WINDOW = 2000
|
||||
|
||||
const lastPayloadByType = new Map<string, string>()
|
||||
|
||||
function safeSend(message: ContentToBackgroundMessage) {
|
||||
const serialized = JSON.stringify(message.payload)
|
||||
const lastSerialized = lastPayloadByType.get(message.type)
|
||||
if (serialized === lastSerialized)
|
||||
return
|
||||
|
||||
lastPayloadByType.set(message.type, serialized)
|
||||
void browser.runtime.sendMessage(message).catch(() => {})
|
||||
}
|
||||
|
||||
function buildPageContext(site: VideoSite): PageContextPayload {
|
||||
const description = normalizeText(document.querySelector('meta[name="description"]')?.getAttribute('content'))
|
||||
const ogDescription = normalizeText(document.querySelector('meta[property="og:description"]')?.getAttribute('content'))
|
||||
|
||||
return {
|
||||
site,
|
||||
url: location.href,
|
||||
title: normalizeText(document.title),
|
||||
description: description || ogDescription || undefined,
|
||||
language: document.documentElement.lang || undefined,
|
||||
}
|
||||
}
|
||||
|
||||
function buildVideoContext(site: VideoSite, video: HTMLVideoElement, includeProgress = false): VideoContextPayload {
|
||||
const title = normalizeText(findVideoTitle(site))
|
||||
const channel = normalizeText(findChannelName(site))
|
||||
const url = location.href
|
||||
const videoId = extractVideoId(site, url)
|
||||
const durationSec = Number.isFinite(video.duration) ? Math.floor(video.duration) : undefined
|
||||
const currentTimeSec = includeProgress && Number.isFinite(video.currentTime) ? Math.floor(video.currentTime) : undefined
|
||||
const rect = video.getBoundingClientRect()
|
||||
|
||||
return {
|
||||
site,
|
||||
url,
|
||||
title: title || normalizeText(document.title),
|
||||
channel: channel || undefined,
|
||||
videoId,
|
||||
durationSec,
|
||||
currentTimeSec,
|
||||
isPlaying: !video.paused && !video.ended,
|
||||
isMuted: video.muted,
|
||||
volume: Number.isFinite(video.volume) ? Number(video.volume.toFixed(2)) : undefined,
|
||||
playbackRate: Number.isFinite(video.playbackRate) ? Number(video.playbackRate.toFixed(2)) : undefined,
|
||||
playerSize: rect.width && rect.height ? { width: Math.round(rect.width), height: Math.round(rect.height) } : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
function findVideoTitle(site: VideoSite) {
|
||||
if (site === 'youtube') {
|
||||
return (
|
||||
document.querySelector('ytd-watch-metadata h1 yt-formatted-string')?.textContent
|
||||
|| document.querySelector('h1.title yt-formatted-string')?.textContent
|
||||
|| document.querySelector('h1.title')?.textContent
|
||||
)
|
||||
}
|
||||
|
||||
if (site === 'bilibili') {
|
||||
return (
|
||||
document.querySelector('h1.video-title')?.textContent
|
||||
|| document.querySelector('.video-title')?.textContent
|
||||
|| document.querySelector('h1')?.textContent
|
||||
)
|
||||
}
|
||||
|
||||
return document.querySelector('h1')?.textContent
|
||||
}
|
||||
|
||||
function findChannelName(site: VideoSite) {
|
||||
if (site === 'youtube') {
|
||||
return (
|
||||
document.querySelector('#channel-name a')?.textContent
|
||||
|| document.querySelector('ytd-channel-name a')?.textContent
|
||||
|| document.querySelector('ytd-channel-name')?.textContent
|
||||
)
|
||||
}
|
||||
|
||||
if (site === 'bilibili') {
|
||||
return (
|
||||
document.querySelector('.up-name')?.textContent
|
||||
|| document.querySelector('.username')?.textContent
|
||||
|| document.querySelector('.up-info .name')?.textContent
|
||||
)
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
function observeTextTracks(site: VideoSite, video: HTMLVideoElement, onSubtitle: (payload: SubtitlePayload) => void) {
|
||||
const seen = new Map<string, number>()
|
||||
|
||||
const handleCueChange = (track: TextTrack) => {
|
||||
const cues = Array.from(track.activeCues ?? []) as TextTrackCue[]
|
||||
for (const cue of cues) {
|
||||
const text = normalizeText((cue as VTTCue).text ?? '')
|
||||
if (!text)
|
||||
continue
|
||||
|
||||
const key = `${text}:${Math.floor(cue.startTime * 1000)}`
|
||||
const now = Date.now()
|
||||
const lastSeen = seen.get(key)
|
||||
if (lastSeen && now - lastSeen < SUBTITLE_DEDUPE_WINDOW)
|
||||
continue
|
||||
|
||||
seen.set(key, now)
|
||||
onSubtitle({
|
||||
site,
|
||||
url: location.href,
|
||||
title: normalizeText(findVideoTitle(site)) || undefined,
|
||||
videoId: extractVideoId(site, location.href),
|
||||
text,
|
||||
language: (track.language || track.label || undefined),
|
||||
startMs: Math.floor(cue.startTime * 1000),
|
||||
endMs: Math.floor(cue.endTime * 1000),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const attach = () => {
|
||||
const tracks = Array.from(video.textTracks ?? [])
|
||||
for (const track of tracks) {
|
||||
if (track.kind && !['subtitles', 'captions'].includes(track.kind))
|
||||
continue
|
||||
|
||||
if (track.mode === 'disabled')
|
||||
track.mode = 'hidden'
|
||||
track.oncuechange = () => handleCueChange(track)
|
||||
}
|
||||
}
|
||||
|
||||
attach()
|
||||
|
||||
const observer = new MutationObserver(() => attach())
|
||||
observer.observe(video, { attributes: true, childList: true, subtree: true })
|
||||
|
||||
return () => observer.disconnect()
|
||||
}
|
||||
|
||||
function observeSubtitleDom(site: VideoSite, onSubtitle: (payload: SubtitlePayload) => void) {
|
||||
let selector = ''
|
||||
if (site === 'youtube')
|
||||
selector = '.caption-window .caption-window-text, .ytp-caption-segment'
|
||||
if (site === 'bilibili')
|
||||
selector = '.bpx-player-subtitle-panel-text, .bpx-player-subtitle-text'
|
||||
|
||||
if (!selector)
|
||||
return () => {}
|
||||
|
||||
let lastText = ''
|
||||
|
||||
const read = () => {
|
||||
const nodes = Array.from(document.querySelectorAll(selector))
|
||||
const text = normalizeText(nodes.map(node => node.textContent).join(' '))
|
||||
if (!text || text === lastText)
|
||||
return
|
||||
|
||||
lastText = text
|
||||
onSubtitle({
|
||||
site,
|
||||
url: location.href,
|
||||
title: normalizeText(findVideoTitle(site)) || undefined,
|
||||
videoId: extractVideoId(site, location.href),
|
||||
text,
|
||||
})
|
||||
}
|
||||
|
||||
const observer = new MutationObserver(read)
|
||||
observer.observe(document.documentElement, { childList: true, subtree: true })
|
||||
|
||||
const interval = window.setInterval(read, 1200)
|
||||
|
||||
return () => {
|
||||
observer.disconnect()
|
||||
window.clearInterval(interval)
|
||||
}
|
||||
}
|
||||
|
||||
function captureVisionFrame(site: VideoSite, video: HTMLVideoElement): VisionFramePayload | null {
|
||||
const canvas = document.createElement('canvas')
|
||||
const width = Math.min(480, Math.max(1, Math.floor(video.videoWidth)))
|
||||
const height = Math.min(270, Math.max(1, Math.floor(video.videoHeight)))
|
||||
|
||||
if (!width || !height)
|
||||
return null
|
||||
|
||||
canvas.width = width
|
||||
canvas.height = height
|
||||
|
||||
const ctx = canvas.getContext('2d')
|
||||
if (!ctx)
|
||||
return null
|
||||
|
||||
try {
|
||||
ctx.drawImage(video, 0, 0, width, height)
|
||||
return {
|
||||
site,
|
||||
url: location.href,
|
||||
videoId: extractVideoId(site, location.href),
|
||||
title: normalizeText(findVideoTitle(site)) || undefined,
|
||||
capturedAt: Date.now(),
|
||||
width,
|
||||
height,
|
||||
dataUrl: canvas.toDataURL('image/jpeg', 0.6),
|
||||
}
|
||||
}
|
||||
catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function observeVideo(site: VideoSite) {
|
||||
let video: HTMLVideoElement | null = null
|
||||
let stopTracks: (() => void) | null = null
|
||||
let stopDomSubtitles: (() => void) | null = null
|
||||
let listenersAttached = false
|
||||
|
||||
const sendVideo = (includeProgress: boolean) => {
|
||||
if (!video)
|
||||
return
|
||||
|
||||
safeSend({ type: 'content:video', payload: buildVideoContext(site, video, includeProgress) })
|
||||
}
|
||||
|
||||
const sendPage = () => {
|
||||
safeSend({ type: 'content:page', payload: buildPageContext(site) })
|
||||
}
|
||||
|
||||
const attach = () => {
|
||||
const found = document.querySelector('video') as HTMLVideoElement | null
|
||||
if (!found || found === video)
|
||||
return
|
||||
|
||||
if (video && listenersAttached) {
|
||||
video.removeEventListener('play', onPlayback)
|
||||
video.removeEventListener('pause', onPlayback)
|
||||
video.removeEventListener('loadedmetadata', onPlayback)
|
||||
listenersAttached = false
|
||||
}
|
||||
|
||||
video = found
|
||||
stopTracks?.()
|
||||
stopDomSubtitles?.()
|
||||
|
||||
stopTracks = observeTextTracks(site, video, payload => safeSend({ type: 'content:subtitle', payload }))
|
||||
stopDomSubtitles = observeSubtitleDom(site, payload => safeSend({ type: 'content:subtitle', payload }))
|
||||
|
||||
sendPage()
|
||||
sendVideo(false)
|
||||
}
|
||||
|
||||
const interval = window.setInterval(attach, 1000)
|
||||
|
||||
const progressInterval = window.setInterval(() => {
|
||||
if (!video)
|
||||
return
|
||||
sendVideo(true)
|
||||
}, VIDEO_PROGRESS_INTERVAL)
|
||||
|
||||
const titleInterval = window.setInterval(() => {
|
||||
sendPage()
|
||||
sendVideo(false)
|
||||
}, TITLE_POLL_INTERVAL)
|
||||
|
||||
const onPlayback = () => sendVideo(true)
|
||||
|
||||
const cleanup = () => {
|
||||
window.clearInterval(interval)
|
||||
window.clearInterval(progressInterval)
|
||||
window.clearInterval(titleInterval)
|
||||
if (video) {
|
||||
video.removeEventListener('play', onPlayback)
|
||||
video.removeEventListener('pause', onPlayback)
|
||||
video.removeEventListener('loadedmetadata', onPlayback)
|
||||
listenersAttached = false
|
||||
}
|
||||
stopTracks?.()
|
||||
stopDomSubtitles?.()
|
||||
}
|
||||
|
||||
const attachListeners = () => {
|
||||
if (!video)
|
||||
return
|
||||
if (listenersAttached)
|
||||
return
|
||||
|
||||
video.addEventListener('play', onPlayback)
|
||||
video.addEventListener('pause', onPlayback)
|
||||
video.addEventListener('loadedmetadata', onPlayback)
|
||||
listenersAttached = true
|
||||
}
|
||||
|
||||
const observer = new MutationObserver(() => {
|
||||
attach()
|
||||
attachListeners()
|
||||
})
|
||||
|
||||
observer.observe(document.documentElement, { childList: true, subtree: true })
|
||||
|
||||
attach()
|
||||
attachListeners()
|
||||
|
||||
return () => {
|
||||
cleanup()
|
||||
observer.disconnect()
|
||||
}
|
||||
}
|
||||
|
||||
export function startContentObserver() {
|
||||
const site = detectSiteFromUrl(location.href)
|
||||
safeSend({ type: 'content:page', payload: buildPageContext(site) })
|
||||
const stopVideo = observeVideo(site)
|
||||
|
||||
browser.runtime.onMessage.addListener((message: BackgroundToContentMessage) => {
|
||||
if (message.type === 'background:request-vision-frame') {
|
||||
const video = document.querySelector('video') as HTMLVideoElement | null
|
||||
if (!video)
|
||||
return
|
||||
|
||||
const frame = captureVisionFrame(site, video)
|
||||
if (frame)
|
||||
safeSend({ type: 'content:vision:frame', payload: frame })
|
||||
}
|
||||
})
|
||||
|
||||
return () => {
|
||||
stopVideo?.()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { BackgroundToPopupMessage, ExtensionSettings, ExtensionStatus, PopupToBackgroundMessage } from '../shared/types'
|
||||
|
||||
export async function requestStatus(): Promise<ExtensionStatus> {
|
||||
return await browser.runtime.sendMessage({ type: 'popup:get-status' } satisfies PopupToBackgroundMessage)
|
||||
}
|
||||
|
||||
export async function updateSettings(partial: Partial<ExtensionSettings>): Promise<ExtensionStatus> {
|
||||
return await browser.runtime.sendMessage({ type: 'popup:update-settings', payload: partial } satisfies PopupToBackgroundMessage)
|
||||
}
|
||||
|
||||
export async function toggleEnabled(enabled: boolean): Promise<ExtensionStatus> {
|
||||
return await browser.runtime.sendMessage({ type: 'popup:toggle-enabled', payload: enabled } satisfies PopupToBackgroundMessage)
|
||||
}
|
||||
|
||||
export async function requestVisionFrame(): Promise<ExtensionStatus> {
|
||||
return await browser.runtime.sendMessage({ type: 'popup:request-vision-frame' } satisfies PopupToBackgroundMessage)
|
||||
}
|
||||
|
||||
export async function clearError(): Promise<ExtensionStatus> {
|
||||
return await browser.runtime.sendMessage({ type: 'popup:clear-error' } satisfies PopupToBackgroundMessage)
|
||||
}
|
||||
|
||||
export function onBackgroundStatus(callback: (status: ExtensionStatus) => void) {
|
||||
const listener = (message: BackgroundToPopupMessage) => {
|
||||
if (message?.type === 'background:status')
|
||||
callback(message.payload)
|
||||
}
|
||||
|
||||
browser.runtime.onMessage.addListener(listener)
|
||||
|
||||
return () => browser.runtime.onMessage.removeListener(listener)
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { ExtensionSettings } from './types'
|
||||
|
||||
export const DEFAULT_WS_URL = 'ws://localhost:6121/ws'
|
||||
|
||||
export const DEFAULT_SETTINGS: ExtensionSettings = {
|
||||
wsUrl: DEFAULT_WS_URL,
|
||||
token: '',
|
||||
enabled: true,
|
||||
sendPageContext: true,
|
||||
sendVideoContext: true,
|
||||
sendSubtitles: true,
|
||||
sendSparkNotify: true,
|
||||
enableVision: false,
|
||||
}
|
||||
|
||||
export const STORAGE_KEY = 'airi:web-extension:settings'
|
||||
@@ -0,0 +1,43 @@
|
||||
import type { VideoSite } from './types'
|
||||
|
||||
export function detectSiteFromUrl(url: string): VideoSite {
|
||||
try {
|
||||
const parsed = new URL(url)
|
||||
const host = parsed.hostname
|
||||
if (host.includes('youtube.com') || host.includes('youtu.be'))
|
||||
return 'youtube'
|
||||
if (host.includes('bilibili.com') || host.includes('b23.tv'))
|
||||
return 'bilibili'
|
||||
return 'unknown'
|
||||
}
|
||||
catch {
|
||||
return 'unknown'
|
||||
}
|
||||
}
|
||||
|
||||
export function extractVideoId(site: VideoSite, url: string): string | undefined {
|
||||
try {
|
||||
const parsed = new URL(url)
|
||||
if (site === 'youtube') {
|
||||
if (parsed.hostname.includes('youtu.be'))
|
||||
return parsed.pathname.replace('/', '') || undefined
|
||||
return parsed.searchParams.get('v') || undefined
|
||||
}
|
||||
if (site === 'bilibili') {
|
||||
const parts = parsed.pathname.split('/').filter(Boolean)
|
||||
const videoIndex = parts.findIndex(part => part === 'video')
|
||||
if (videoIndex >= 0)
|
||||
return parts[videoIndex + 1]
|
||||
return parts[0]
|
||||
}
|
||||
}
|
||||
catch {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function normalizeText(value: string | null | undefined) {
|
||||
return value?.replace(/\s+/g, ' ').trim() || ''
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
export type VideoSite = 'youtube' | 'bilibili' | 'unknown'
|
||||
|
||||
export interface PageContextPayload {
|
||||
site: VideoSite
|
||||
url: string
|
||||
title: string
|
||||
description?: string
|
||||
language?: string
|
||||
}
|
||||
|
||||
export interface VideoContextPayload {
|
||||
site: VideoSite
|
||||
url: string
|
||||
title: string
|
||||
channel?: string
|
||||
videoId?: string
|
||||
durationSec?: number
|
||||
currentTimeSec?: number
|
||||
isPlaying?: boolean
|
||||
isMuted?: boolean
|
||||
volume?: number
|
||||
playbackRate?: number
|
||||
isLive?: boolean
|
||||
playerSize?: { width: number, height: number }
|
||||
}
|
||||
|
||||
export interface SubtitlePayload {
|
||||
site: VideoSite
|
||||
url: string
|
||||
videoId?: string
|
||||
title?: string
|
||||
text: string
|
||||
language?: string
|
||||
startMs?: number
|
||||
endMs?: number
|
||||
isAuto?: boolean
|
||||
}
|
||||
|
||||
export interface VisionFramePayload {
|
||||
site: VideoSite
|
||||
url: string
|
||||
videoId?: string
|
||||
title?: string
|
||||
capturedAt: number
|
||||
width: number
|
||||
height: number
|
||||
dataUrl: string
|
||||
}
|
||||
|
||||
export type ContentToBackgroundMessage
|
||||
= | { type: 'content:page', payload: PageContextPayload }
|
||||
| { type: 'content:video', payload: VideoContextPayload }
|
||||
| { type: 'content:subtitle', payload: SubtitlePayload }
|
||||
| { type: 'content:vision:frame', payload: VisionFramePayload }
|
||||
|
||||
export interface ExtensionSettings {
|
||||
wsUrl: string
|
||||
token: string
|
||||
enabled: boolean
|
||||
sendPageContext: boolean
|
||||
sendVideoContext: boolean
|
||||
sendSubtitles: boolean
|
||||
sendSparkNotify: boolean
|
||||
enableVision: boolean
|
||||
}
|
||||
|
||||
export interface ExtensionStatus {
|
||||
connected: boolean
|
||||
lastError?: string
|
||||
settings: ExtensionSettings
|
||||
lastPage?: PageContextPayload
|
||||
lastVideo?: VideoContextPayload
|
||||
lastSubtitle?: SubtitlePayload
|
||||
lastVisionFrameAt?: number
|
||||
}
|
||||
|
||||
export type PopupToBackgroundMessage
|
||||
= | { type: 'popup:get-status' }
|
||||
| { type: 'popup:update-settings', payload: Partial<ExtensionSettings> }
|
||||
| { type: 'popup:toggle-enabled', payload: boolean }
|
||||
| { type: 'popup:request-vision-frame' }
|
||||
| { type: 'popup:clear-error' }
|
||||
|
||||
export type BackgroundToPopupMessage
|
||||
= | { type: 'background:status', payload: ExtensionStatus }
|
||||
|
||||
export type BackgroundToContentMessage
|
||||
= | { type: 'background:request-vision-frame' }
|
||||
@@ -9,6 +9,20 @@ type VitePlugin = NonNullable<WxtViteConfig['plugins']>[number]
|
||||
// See https://wxt.dev/api/config.html
|
||||
export default defineConfig({
|
||||
modules: ['@wxt-dev/module-vue'],
|
||||
manifest: {
|
||||
name: 'AIRI Web Extension',
|
||||
description: 'Capture web context (videos, pages, subtitles) for Project AIRI.',
|
||||
permissions: ['storage', 'tabs'],
|
||||
host_permissions: [
|
||||
'*://*.youtube.com/*',
|
||||
'*://*.youtu.be/*',
|
||||
'*://*.bilibili.com/*',
|
||||
'*://*.b23.tv/*',
|
||||
],
|
||||
action: {
|
||||
default_title: 'AIRI Web Extension',
|
||||
},
|
||||
},
|
||||
vite: () => {
|
||||
return {
|
||||
plugins: [
|
||||
|
||||
Generated
+30
-6
@@ -67,7 +67,7 @@ catalogs:
|
||||
specifier: ^3.0.3
|
||||
version: 3.0.3
|
||||
'@vueuse/core':
|
||||
specifier: ^14.1.0
|
||||
specifier: 14.1.0
|
||||
version: 14.1.0
|
||||
'@xsai-ext/providers':
|
||||
specifier: ^0.4.0-beta.13
|
||||
@@ -3001,16 +3001,34 @@ importers:
|
||||
|
||||
plugins/airi-plugin-web-extension:
|
||||
dependencies:
|
||||
vue:
|
||||
'@vueuse/core':
|
||||
specifier: 'catalog:'
|
||||
version: 3.5.26(typescript@5.9.3)
|
||||
version: 14.1.0(vue@3.5.26(typescript@5.9.3))
|
||||
devDependencies:
|
||||
'@iconify-json/solar':
|
||||
specifier: ^1.2.5
|
||||
version: 1.2.5
|
||||
'@iconify-json/svg-spinners':
|
||||
specifier: ^1.2.4
|
||||
version: 1.2.4
|
||||
'@proj-airi/server-sdk':
|
||||
specifier: workspace:^
|
||||
version: link:../../packages/server-sdk
|
||||
'@proj-airi/ui':
|
||||
specifier: workspace:^
|
||||
version: link:../../packages/ui
|
||||
'@unocss/reset':
|
||||
specifier: ^66.5.11
|
||||
version: 66.5.11
|
||||
'@wxt-dev/module-vue':
|
||||
specifier: ^1.0.3
|
||||
version: 1.0.3(vite@7.3.0(@types/node@24.10.4)(jiti@2.6.1)(less@4.5.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.9.3))(wxt@0.20.13(@types/node@24.10.4)(jiti@2.6.1)(less@4.5.1)(lightningcss@1.30.2)(rollup@4.54.0)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))
|
||||
version: 1.0.3(vite@8.0.0-beta.5(@types/node@24.10.4)(esbuild@0.25.12)(jiti@2.6.1)(less@4.5.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.9.3))(wxt@0.20.13(@types/node@24.10.4)(jiti@2.6.1)(less@4.5.1)(lightningcss@1.30.2)(rollup@4.54.0)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))
|
||||
nanoid:
|
||||
specifier: ^5.1.6
|
||||
version: 5.1.6
|
||||
vue:
|
||||
specifier: 'catalog:'
|
||||
version: 3.5.26(typescript@5.9.3)
|
||||
vue-tsc:
|
||||
specifier: ^3.1.8
|
||||
version: 3.2.1(typescript@5.9.3)
|
||||
@@ -22449,6 +22467,12 @@ snapshots:
|
||||
vite: 7.3.0(@types/node@24.10.4)(jiti@2.6.1)(less@4.5.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)
|
||||
vue: 3.5.26(typescript@5.9.3)
|
||||
|
||||
'@vitejs/plugin-vue@6.0.3(vite@8.0.0-beta.5(@types/node@24.10.4)(esbuild@0.25.12)(jiti@2.6.1)(less@4.5.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.9.3))':
|
||||
dependencies:
|
||||
'@rolldown/pluginutils': 1.0.0-beta.53
|
||||
vite: 8.0.0-beta.5(@types/node@24.10.4)(esbuild@0.25.12)(jiti@2.6.1)(less@4.5.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)
|
||||
vue: 3.5.26(typescript@5.9.3)
|
||||
|
||||
'@vitest/browser-playwright@4.0.16(bufferutil@4.1.0)(playwright@1.57.0)(utf-8-validate@5.0.10)(vite@7.3.0(@types/node@24.10.4)(jiti@2.6.1)(less@4.5.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))(vitest@4.0.16)':
|
||||
dependencies:
|
||||
'@vitest/browser': 4.0.16(bufferutil@4.1.0)(utf-8-validate@5.0.10)(vite@7.3.0(@types/node@24.10.4)(jiti@2.6.1)(less@4.5.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))(vitest@4.0.16)
|
||||
@@ -23148,9 +23172,9 @@ snapshots:
|
||||
'@types/filesystem': 0.0.36
|
||||
'@types/har-format': 1.2.16
|
||||
|
||||
'@wxt-dev/module-vue@1.0.3(vite@7.3.0(@types/node@24.10.4)(jiti@2.6.1)(less@4.5.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.9.3))(wxt@0.20.13(@types/node@24.10.4)(jiti@2.6.1)(less@4.5.1)(lightningcss@1.30.2)(rollup@4.54.0)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))':
|
||||
'@wxt-dev/module-vue@1.0.3(vite@8.0.0-beta.5(@types/node@24.10.4)(esbuild@0.25.12)(jiti@2.6.1)(less@4.5.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.9.3))(wxt@0.20.13(@types/node@24.10.4)(jiti@2.6.1)(less@4.5.1)(lightningcss@1.30.2)(rollup@4.54.0)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))':
|
||||
dependencies:
|
||||
'@vitejs/plugin-vue': 6.0.3(vite@7.3.0(@types/node@24.10.4)(jiti@2.6.1)(less@4.5.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.9.3))
|
||||
'@vitejs/plugin-vue': 6.0.3(vite@8.0.0-beta.5(@types/node@24.10.4)(esbuild@0.25.12)(jiti@2.6.1)(less@4.5.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.9.3))
|
||||
wxt: 0.20.13(@types/node@24.10.4)(jiti@2.6.1)(less@4.5.1)(lightningcss@1.30.2)(rollup@4.54.0)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)
|
||||
transitivePeerDependencies:
|
||||
- vite
|
||||
|
||||
+1
-1
@@ -43,7 +43,7 @@ catalog:
|
||||
'@types/hast': ^3.0.4
|
||||
'@types/splitpanes': ^2.2.6
|
||||
'@types/unist': ^3.0.3
|
||||
'@vueuse/core': ^14.1.0
|
||||
'@vueuse/core': 14.1.0
|
||||
'@xsai-ext/providers': ^0.4.0-beta.13
|
||||
'@xsai/embed': ^0.4.0-beta.13
|
||||
'@xsai/generate-speech': 0.4.0-beta.13
|
||||
|
||||
Reference in New Issue
Block a user