admin/Claw3D
1
0
Fork 0
mirror of https://github.com/iamlukethedev/Claw3D.git synced 2026-08-14 00:58:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pivot to self-hosted AI seam and improve mesh draft

Browse Source 
Co-authored-by: Luke The Dev <iamlukethedev@users.noreply.github.com>
This commit is contained in:
Cursor Agent
2026-04-19 04:05:25 +00:00
co-authored by Luke The Dev
parent dcfc93c285
commit bef154fb9a
10 changed files with 271 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app/api/studio-world/route.ts
+12 -12
View File
@@ -12,8 +12,8 @@ import {
import { buildOfficeMapFromStudioProject } from "@/lib/studio-world/office";
import {
buildRealAiSummary,
createMeshyImageTo3dTask,
getMeshyImageTo3dTask,
createSelfHostedImageTo3dTask,
getSelfHostedImageTo3dTask,
isRealStudioAiEnabled,
resolveStudioAiProvider,
} from "@/lib/studio-world/provider";
@@ -59,7 +59,7 @@ const parseFocus = (value: unknown): StudioWorldFocus =>
value === "assets" || value === "animation" ? value : "world";
const parseProvider = (value: unknown): StudioWorldGenerationProvider =>
value === "meshy" ? "meshy" : "local";
value === "self_hosted" ? "self_hosted" : "local";
const parseGenerationInput = (value: unknown): StudioGenerationInput | null => {
if (!isRecord(value)) return null;
@@ -109,7 +109,7 @@ const parseGenerationInput = (value: unknown): StudioGenerationInput | null => {
const buildProviderAvailability = (): StudioProviderAvailability => {
const provider = resolveStudioAiProvider();
if (provider === "meshy") {
if (provider === "self_hosted") {
const enabled = isRealStudioAiEnabled();
return {
provider,
@@ -117,7 +117,7 @@ const buildProviderAvailability = (): StudioProviderAvailability => {
configured: true,
message: enabled
? "Real AI image-to-3D is enabled."
: "Meshy is configured but disabled until CLAW3D_STUDIO_ENABLE_REAL_AI is enabled.",
: "Self-hosted AI is configured but disabled until CLAW3D_STUDIO_ENABLE_REAL_AI is enabled.",
};
}
return {
@@ -235,15 +235,15 @@ export async function GET(request: Request) {
{ status: 400 },
);
}
if (project.externalModel.provider !== "meshy") {
if (project.externalModel.provider !== "self_hosted") {
return NextResponse.json(
{ error: "Unsupported provider for task status." },
{ status: 400 },
);
}
const task = await getMeshyImageTo3dTask(project.externalModel.taskId);
const task = await getSelfHostedImageTo3dTask(project.externalModel.taskId);
const updatedProject = updateStudioProjectExternalModel(projectId, {
provider: "meshy",
provider: "self_hosted",
taskId: task.id,
status:
task.status === "PENDING"
@@ -368,19 +368,19 @@ export async function POST(request: Request) {
);
}
if (
input.provider === "meshy" &&
input.provider === "self_hosted" &&
input.sourceImage &&
isRealStudioAiEnabled()
) {
const taskId = await createMeshyImageTo3dTask({
const taskId = await createSelfHostedImageTo3dTask({
sourceImage: input.sourceImage,
prompt: input.prompt,
mode: "ai_image_to_3d",
mode: "image_mesh",
});
const project = createStudioPendingProject({
input: {
...input,
provider: "meshy",
provider: "self_hosted",
},
providerTaskId: taskId,
});
src/features/studio-world/preview/StudioWorldPreview.tsx
+1 -1
View File
@@ -120,7 +120,7 @@ export function StudioWorldPreview({
referenceImage = null,
project = null,
}: StudioWorldPreviewProps) {
const isRemoteAiProject = project?.provider === "meshy";
const isRemoteAiProject = project?.provider === "self_hosted";
const remoteReady = Boolean(project?.externalModel?.glbUrl);
const remoteThumbnailUrl = project?.externalModel?.thumbnailUrl ?? null;
const previewLabel = isRemoteAiProject
src/features/studio-world/preview/scene-utils.ts
+18
View File
@@ -59,6 +59,12 @@ export const buildAssetGeometry = (kind: StudioWorldAssetDraft["kind"]) => {
if (kind === "avatar_orb") {
return new THREE.OctahedronGeometry(0.82, 0);
}
if (kind === "heightfield_panel") {
return new THREE.BoxGeometry(1, 1, 0.12);
}
if (kind === "billboard_frame") {
return new THREE.BoxGeometry(1, 1, 1);
}
return new THREE.BoxGeometry(1, 1, 1);
};
@@ -143,6 +149,18 @@ const createAssetMesh = (asset: StudioWorldAssetDraft) => {
return group;
}
if (asset.kind === "heightfield_panel") {
const mesh = new THREE.Mesh(buildAssetGeometry(asset.kind), material);
mesh.scale.set(asset.scale[0], asset.scale[1], asset.scale[2]);
return mesh;
}
if (asset.kind === "billboard_frame") {
const mesh = new THREE.Mesh(buildAssetGeometry(asset.kind), material);
mesh.scale.set(asset.scale[0], asset.scale[1], asset.scale[2]);
return mesh;
}
if (asset.kind === "arch") {
const group = new THREE.Group();
const legGeometry = new THREE.BoxGeometry(0.24, 1, 0.24);
src/features/studio-world/screens/StudioWorldScreen.tsx
+23 -25
View File
@@ -264,7 +264,7 @@ export function StudioWorldScreen() {
setError(null);
setStatusLine(
body.project.latestJob.status === "pending"
? `Submitted ${body.project.name} to real AI generation.`
? `Submitted ${body.project.name} to self-hosted AI generation.`
: `Generated ${body.project.name}.`,
);
} catch (generationError) {
@@ -281,7 +281,7 @@ export function StudioWorldScreen() {
const handleSyncProject = async (projectId: string) => {
setBusy(true);
setStatusLine("Syncing provider task status.");
setStatusLine("Syncing self-hosted AI task status.");
try {
const response = await fetch(
`/api/studio-world?action=task-status&projectId=${encodeURIComponent(projectId)}`,
@@ -289,7 +289,7 @@ export function StudioWorldScreen() {
);
const body = (await response.json()) as StudioWorldResponse;
if (!response.ok || !body.project) {
throw new Error(body.error || "Failed to sync provider task.");
throw new Error(body.error || "Failed to sync self-hosted AI task.");
}
setProjects((current) =>
current.map((entry) => (entry.id === body.project!.id ? body.project! : entry)),
@@ -297,14 +297,14 @@ export function StudioWorldScreen() {
setSelectedProjectId(body.project.id);
setError(null);
if (body.providerTask?.status === "SUCCEEDED") {
setStatusLine("Provider task synced. Remote GLB is ready.");
setStatusLine("AI task synced. Remote GLB is ready.");
} else if (body.providerTask?.status === "FAILED" || body.providerTask?.status === "CANCELED") {
setStatusLine(body.providerTask.taskErrorMessage || "Provider task failed.");
setStatusLine(body.providerTask.taskErrorMessage || "AI task failed.");
} else {
setStatusLine(`Provider task is ${body.providerTask?.status?.toLowerCase() ?? "in progress"}.`);
setStatusLine(`AI task is ${body.providerTask?.status?.toLowerCase() ?? "in progress"}.`);
}
} catch (syncError) {
setError(syncError instanceof Error ? syncError.message : "Failed to sync provider task.");
setError(syncError instanceof Error ? syncError.message : "Failed to sync self-hosted AI task.");
setStatusLine(null);
} finally {
setBusy(false);
@@ -376,20 +376,20 @@ export function StudioWorldScreen() {
const handleExportProviderGlb = async (project: StudioProjectRecord) => {
const glbUrl = project.externalModel?.glbUrl?.trim() ?? "";
if (!glbUrl) {
setError("Provider GLB is not ready yet.");
setError("AI-generated GLB is not ready yet.");
return;
}
setBusy(true);
setStatusLine("Downloading provider GLB.");
setStatusLine("Downloading AI-generated GLB.");
try {
await downloadFileFromUrl({
url: glbUrl,
filename: `${project.name.trim().replace(/\s+/g, "-").toLowerCase() || "studio-provider"}.glb`,
filename: `${project.name.trim().replace(/\s+/g, "-").toLowerCase() || "studio-ai"}.glb`,
});
setError(null);
setStatusLine("Provider GLB downloaded.");
setStatusLine("AI-generated GLB downloaded.");
} catch (downloadError) {
setError(downloadError instanceof Error ? downloadError.message : "Failed to download provider GLB.");
setError(downloadError instanceof Error ? downloadError.message : "Failed to download AI-generated GLB.");
setStatusLine(null);
} finally {
setBusy(false);
@@ -457,7 +457,7 @@ export function StudioWorldScreen() {
<div className="font-medium text-foreground">AI provider status</div>
<div className="mt-1">
{providerAvailability?.message ??
"Local generation is available. Configure a real provider to enable model-backed image-to-3D."}
"Local generation is available. Configure a self-hosted AI provider to enable model-backed image-to-3D."}
</div>
</div>
<div className="mt-5 space-y-4">
@@ -580,7 +580,7 @@ export function StudioWorldScreen() {
</label>
<div className="grid gap-3 sm:grid-cols-2">
<label className="block">
<span className="mb-1.5 block text-xs font-medium text-foreground">Generation provider</span>
<span className="mb-1.5 block text-xs font-medium text-foreground">Generation backend</span>
<select
className="ui-input w-full"
value={provider}
@@ -588,10 +588,10 @@ export function StudioWorldScreen() {
>
<option value="local">Local Studio</option>
<option
value="meshy"
value="self_hosted"
disabled={!providerAvailability?.available}
>
Meshy AI
Self-hosted AI
</option>
</select>
</label>
@@ -692,10 +692,10 @@ export function StudioWorldScreen() {
</div>
<div className="mt-3 space-y-2 text-sm text-muted-foreground">
<div className="rounded-lg border border-border/50 bg-surface-1/50 px-3 py-2">Direct GLB download.</div>
<div className="rounded-lg border border-border/50 bg-surface-1/50 px-3 py-2">Provider GLB download when remote AI finishes.</div>
<div className="rounded-lg border border-border/50 bg-surface-1/50 px-3 py-2">AI-generated GLB download when the self-hosted job finishes.</div>
<div className="rounded-lg border border-border/50 bg-surface-1/50 px-3 py-2">GLB manifest download.</div>
<div className="rounded-lg border border-border/50 bg-surface-1/50 px-3 py-2">Publish to Claw3D office layout.</div>
<div className="rounded-lg border border-border/50 bg-surface-1/50 px-3 py-2">Manual provider task sync for remote AI jobs.</div>
<div className="rounded-lg border border-border/50 bg-surface-1/50 px-3 py-2">Manual task sync for self-hosted AI jobs.</div>
</div>
</div>
</div>
@@ -762,9 +762,7 @@ export function StudioWorldScreen() {
? "image avatar"
: project.mode === "image_mesh"
? "image mesh"
: project.mode === "ai_image_to_3d"
? "ai image-to-3d"
: "text scene"}
: "text scene"}
</span>
<span className="rounded-full bg-muted px-2 py-1 font-mono text-[10px] uppercase tracking-[0.14em] text-muted-foreground">
{project.provider}
@@ -776,8 +774,8 @@ export function StudioWorldScreen() {
</div>
{project.externalModel ? (
<div className="mt-3 rounded-xl border border-border/60 bg-surface-1/35 p-2 text-xs text-muted-foreground">
Provider task {project.externalModel.status} • {project.externalModel.progress}%.
{project.externalModel.glbUrl ? " GLB ready from provider." : ""}
AI task {project.externalModel.status} • {project.externalModel.progress}%.
{project.externalModel.glbUrl ? " GLB ready from self-hosted AI." : ""}
{project.externalModel.errorMessage ? ` ${project.externalModel.errorMessage}` : ""}
</div>
) : null}
@@ -785,7 +783,7 @@ export function StudioWorldScreen() {
<div className="mt-3 overflow-hidden rounded-xl border border-border/60 bg-surface-1/35">
<Image
src={project.externalModel.thumbnailUrl}
alt={`${project.name} provider thumbnail`}
alt={`${project.name} AI thumbnail`}
width={320}
height={180}
className="h-32 w-full object-cover"
@@ -837,7 +835,7 @@ export function StudioWorldScreen() {
onClick={() => void handleExportProviderGlb(project)}
disabled={busy}
>
Download provider GLB
Download AI GLB
</button>
) : null}
<button
src/lib/studio-world/generator.ts
+67 -31
View File
@@ -430,6 +430,14 @@ const buildImageMeshDraft = (input: StudioGenerationInput): StudioWorldDraft =>
const panelWidth = clamp(4.6 * aspectRatio, 2.6, 6.8);
const panelHeight = clamp(5.2 / Math.max(aspectRatio, 0.65), 3.8, 7.2);
const grayscale = Array.isArray(sourceImage.intensitySamples) && sourceImage.intensitySamples.length > 0
? sourceImage.intensitySamples
: [0.2, 0.5, 0.8, 0.4];
const reliefColumns = Math.max(4, Math.round(Math.sqrt(grayscale.length)));
const reliefRows = Math.max(4, Math.ceil(grayscale.length / reliefColumns));
const cellWidth = panelWidth / reliefColumns;
const cellHeight = panelHeight / reliefRows;
const assets: StudioWorldAssetDraft[] = [
{
id: "mesh_base",
@@ -446,58 +454,86 @@ const buildImageMeshDraft = (input: StudioGenerationInput): StudioWorldDraft =>
id: "mesh_panel",
name: "Image mesh panel",
kind: "avatar_torso",
position: [0, 2.7, 0],
scale: [panelWidth, panelHeight, 0.4],
position: [0, 2.8, -0.18],
scale: [panelWidth * 1.02, panelHeight * 1.02, 0.18],
rotationY: 0,
color: palette.structure,
emissive: null,
animation: "none",
},
];
for (let row = 0; row < reliefRows; row += 1) {
for (let col = 0; col < reliefColumns; col += 1) {
const index = row * reliefColumns + col;
const intensity = grayscale[index] ?? grayscale[grayscale.length - 1] ?? 0.5;
const centeredX = -panelWidth / 2 + cellWidth * col + cellWidth / 2;
const centeredY = panelHeight / 2 - cellHeight * row - cellHeight / 2;
const depth = 0.08 + intensity * 0.82;
const colorIndex = Math.min(
sourceImage.palette.length - 1,
Math.floor(intensity * Math.max(sourceImage.palette.length, 1)),
);
const color =
sourceImage.palette[colorIndex] ??
(intensity > 0.66
? imageNotes.accessory
: intensity > 0.33
? imageNotes.outfitTrim
: imageNotes.outfitMain);
assets.push({
id: `mesh_voxel_${row}_${col}`,
name: `Relief voxel ${row}-${col}`,
kind: "crate",
position: [round2(centeredX), round2(2.8 + centeredY), round2(depth * 0.42)],
scale: [
round2(cellWidth * 0.92),
round2(cellHeight * 0.92),
round2(depth),
],
rotationY: 0,
color,
emissive: intensity > 0.84 ? imageNotes.accessory : null,
animation: "none",
});
}
}
assets.push(
{
id: "mesh_head_form",
name: "Portrait mass",
kind: "avatar_head",
position: [0, 4.15, 0.25],
scale: [panelWidth * 0.42, panelHeight * 0.28, 0.44],
id: "mesh_frame_top",
name: "Mesh frame top",
kind: "arch",
position: [0, 2.8 + panelHeight / 2 + 0.45, -0.22],
scale: [panelWidth * 0.62, 0.42, 0.18],
rotationY: 0,
color: imageNotes.skinLike,
color: imageNotes.accessory,
emissive: null,
animation: "none",
},
{
id: "mesh_hair_form",
name: "Hair crest",
kind: "avatar_hair",
position: [0.05, 5.15, 0.2],
scale: [panelWidth * 0.36, panelHeight * 0.26, 0.34],
id: "mesh_frame_bottom",
name: "Mesh frame bottom",
kind: "platform",
position: [0, 2.8 - panelHeight / 2 - 0.35, -0.22],
scale: [panelWidth * 1.06, 0.22, 0.28],
rotationY: 0,
color: imageNotes.hairLike,
color: imageNotes.outfitTrim,
emissive: null,
animation: input.focus === "animation" ? "pulse" : "none",
animation: "none",
},
{
id: "mesh_accent_strip",
name: "Accent strip",
kind: "avatar_accessory",
position: [0, 2.8, 0.32],
scale: [panelWidth * 0.52, 0.8, 0.12],
rotationY: 0,
color: imageNotes.accessory,
emissive: imageNotes.accessory,
animation: "pulse",
},
{
id: "mesh_companion",
name: "Floating detail",
id: "mesh_light",
name: "Mesh halo light",
kind: "avatar_orb",
position: [panelWidth * 0.75, 4.7, -0.8],
scale: [0.92, 0.92, 0.92],
position: [panelWidth * 0.74, 2.8 + panelHeight * 0.34, -0.62],
scale: [0.74, 0.74, 0.74],
rotationY: 0,
color: imageNotes.outfitTrim,
emissive: imageNotes.accessory,
animation: "spin",
},
];
);
return {
mode: "image_mesh",
src/lib/studio-world/image-analysis.ts
+52
View File
@@ -150,3 +150,55 @@ export const buildAvatarImageNotes = (image: StudioSourceImageRecord) => {
backdrop: image.palette[3] ?? darken(primary, 0.5),
};
};
export const buildImageIntensitySamples = (buffer: Buffer) => {
const samples: number[] = [];
const step = Math.max(4, Math.floor(buffer.length / 4096));
for (let index = 0; index + 2 < buffer.length; index += step) {
const red = buffer[index] ?? 0;
const green = buffer[index + 1] ?? 0;
const blue = buffer[index + 2] ?? 0;
const intensity = clamp(
Math.round(((red + green + blue) / (255 * 3)) * 1000) / 1000,
0,
1,
);
samples.push(intensity);
}
if (samples.length === 0) {
return [0.25, 0.5, 0.75, 0.5];
}
return samples.slice(0, 256);
};
const samplePixel = (buffer: Buffer, offset: number) => {
const red = buffer[offset] ?? 0;
const green = buffer[offset + 1] ?? red;
const blue = buffer[offset + 2] ?? green;
return { red, green, blue };
};
export const buildImageSampleGridFromBuffer = (buffer: Buffer, cells = 12) => {
const samples: number[][] = [];
const length = buffer.length;
if (length <= 3) {
return Array.from({ length: cells }, () =>
Array.from({ length: cells }, () => 0.5),
);
}
for (let row = 0; row < cells; row += 1) {
const rowValues: number[] = [];
for (let col = 0; col < cells; col += 1) {
const normalizedIndex = (row * cells + col) / Math.max(cells * cells - 1, 1);
const offset = Math.min(
Math.max(0, Math.floor(normalizedIndex * (length - 3))),
length - 3,
);
const { red, green, blue } = samplePixel(buffer, offset);
const luminance = (red * 0.2126 + green * 0.7152 + blue * 0.0722) / 255;
rowValues.push(clamp(Math.round(luminance * 1000) / 1000, 0, 1));
}
samples.push(rowValues);
}
return samples;
};
src/lib/studio-world/provider.ts
+66 -57
View File
@@ -5,7 +5,7 @@ import type {
StudioWorldGenerationMode,
} from "@/lib/studio-world/types";
export type StudioAiProviderKind = "none" | "meshy";
export type StudioAiProviderKind = "none" | "self_hosted";
export type StudioAiTaskStatus =
| "PENDING"
@@ -27,11 +27,11 @@ export type StudioAiTaskRecord = {
updatedAt: string;
};
type MeshyCreateResponse = {
type SelfHostedCreateResponse = {
result?: string;
};
type MeshyTaskResponse = {
type SelfHostedTaskResponse = {
id?: string;
status?: string;
progress?: number;
@@ -44,7 +44,7 @@ type MeshyTaskResponse = {
};
};
const MESHY_API_BASE_URL = "https://api.meshy.ai/openapi/v1";
const SELF_HOSTED_API_BASE_URL = "http://127.0.0.1:3333/openapi/v1";
const isEnabled = (value: string | undefined) => {
const normalized = (value ?? "").trim().toLowerCase();
@@ -52,8 +52,8 @@ const isEnabled = (value: string | undefined) => {
};
export const resolveStudioAiProvider = (): StudioAiProviderKind => {
if (process.env.MESHY_API_KEY?.trim()) {
return "meshy";
if (process.env.CLAW3D_STUDIO_PROVIDER_URL?.trim()) {
return "self_hosted";
}
return "none";
};
@@ -64,17 +64,17 @@ export const isRealStudioAiEnabled = () =>
export const buildStudioAiProviderAvailability = (): StudioProviderAvailability => {
const provider = resolveStudioAiProvider();
if (provider === "meshy") {
if (provider === "self_hosted") {
const enabled = isRealStudioAiEnabled();
const apiKey = process.env.MESHY_API_KEY?.trim() ?? "";
const providerUrl = process.env.CLAW3D_STUDIO_PROVIDER_URL?.trim() ?? "";
return {
provider: "meshy",
provider: "self_hosted",
available: enabled,
configured: Boolean(apiKey),
usingTestMode: apiKey === "msy_dummy_api_key_for_test_mode_12345678",
configured: Boolean(providerUrl),
usingTestMode: false,
message: enabled
? "Real AI image-to-3D is enabled."
: "Meshy is configured but disabled until CLAW3D_STUDIO_ENABLE_REAL_AI is enabled.",
? "Self-hosted AI image-to-3D is enabled."
: "A self-hosted provider is configured but disabled until CLAW3D_STUDIO_ENABLE_REAL_AI is enabled.",
};
}
return {
@@ -82,16 +82,17 @@ export const buildStudioAiProviderAvailability = (): StudioProviderAvailability
available: false,
configured: false,
usingTestMode: false,
message: "No real AI provider is configured. Studio will use local generators.",
message: "No self-hosted AI provider is configured. Studio will use local generators.",
};
};
const assertMeshyConfigured = () => {
const apiKey = process.env.MESHY_API_KEY?.trim();
if (!apiKey) {
throw new Error("MESHY_API_KEY is not configured.");
const resolveSelfHostedProviderConfig = () => {
const baseUrl = process.env.CLAW3D_STUDIO_PROVIDER_URL?.trim() || SELF_HOSTED_API_BASE_URL;
const apiKey = process.env.CLAW3D_STUDIO_PROVIDER_API_KEY?.trim() ?? "";
if (!baseUrl) {
throw new Error("CLAW3D_STUDIO_PROVIDER_URL is not configured.");
}
return apiKey;
return { baseUrl, apiKey };
};
const mapStatus = (status: string | undefined): StudioAiTaskStatus => {
@@ -109,66 +110,74 @@ const mapStatus = (status: string | undefined): StudioAiTaskStatus => {
const buildDataUri = (image: StudioSourceImageRecord) => image.dataUrl;
export const createMeshyImageTo3dTask = async (params: {
export const createSelfHostedImageTo3dTask = async (params: {
sourceImage: StudioSourceImageRecord;
prompt: string;
mode: StudioWorldGenerationMode;
}) => {
const apiKey = assertMeshyConfigured();
const response = await fetch(`${MESHY_API_BASE_URL}/image-to-3d`, {
const { baseUrl, apiKey } = resolveSelfHostedProviderConfig();
const payload = {
image_url: buildDataUri(params.sourceImage),
model_type: params.mode === "image_mesh" ? "standard" : "lowpoly",
ai_model: "latest",
should_texture: true,
enable_pbr: false,
remove_lighting: true,
image_enhancement: true,
target_formats: ["glb"],
should_remesh: params.mode === "image_mesh",
...(params.mode === "image_mesh"
? {
topology: "triangle",
target_polycount: 30000,
}
: {}),
...(params.prompt.trim()
? {
texture_prompt: params.prompt.trim().slice(0, 600),
}
: {}),
};
const response = await fetch(`${baseUrl}/image-to-3d`, {
method: "POST",
headers: {
Authorization: `Bearer ${apiKey}`,
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
"Content-Type": "application/json",
},
body: JSON.stringify({
image_url: buildDataUri(params.sourceImage),
model_type: params.mode === "image_mesh" ? "standard" : "lowpoly",
ai_model: "latest",
should_texture: true,
enable_pbr: false,
remove_lighting: true,
image_enhancement: true,
target_formats: ["glb"],
should_remesh: params.mode === "image_mesh",
...(params.mode === "image_mesh"
? {
topology: "triangle",
target_polycount: 30000,
}
: {}),
...(params.prompt.trim()
? {
texture_prompt: params.prompt.trim().slice(0, 600),
}
: {}),
}),
body: JSON.stringify(payload),
});
const body = (await response.json()) as MeshyCreateResponse;
const rawBody = await response.text();
let body: SelfHostedCreateResponse = {};
try {
body = JSON.parse(rawBody) as SelfHostedCreateResponse;
} catch {
body = {};
}
if (!response.ok || !body.result) {
throw new Error("Failed to create Meshy image-to-3D task.");
const diagnostic = rawBody.trim() || `${response.status} ${response.statusText}`;
throw new Error(`Failed to create self-hosted image-to-3D task. ${diagnostic}`);
}
return body.result;
};
export const getMeshyImageTo3dTask = async (
export const getSelfHostedImageTo3dTask = async (
taskId: string,
): Promise<StudioAiTaskRecord> => {
const apiKey = assertMeshyConfigured();
const response = await fetch(`${MESHY_API_BASE_URL}/image-to-3d/${encodeURIComponent(taskId)}`, {
const { baseUrl, apiKey } = resolveSelfHostedProviderConfig();
const response = await fetch(`${baseUrl}/image-to-3d/${encodeURIComponent(taskId)}`, {
headers: {
Authorization: `Bearer ${apiKey}`,
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
},
cache: "no-store",
});
const body = (await response.json()) as MeshyTaskResponse;
const body = (await response.json()) as SelfHostedTaskResponse;
if (!response.ok || !body.id) {
throw new Error("Failed to fetch Meshy task.");
throw new Error("Failed to fetch self-hosted provider task.");
}
const now = new Date().toISOString();
return {
id: body.id,
provider: "meshy",
provider: "self_hosted",
mode: body.model_urls?.glb ? "image_mesh" : "image_avatar",
status: mapStatus(body.status),
progress:
@@ -183,7 +192,7 @@ export const getMeshyImageTo3dTask = async (
};
};
export const waitForMeshyImageTo3dTask = async (params: {
export const waitForSelfHostedImageTo3dTask = async (params: {
taskId: string;
timeoutMs?: number;
pollIntervalMs?: number;
@@ -191,7 +200,7 @@ export const waitForMeshyImageTo3dTask = async (params: {
const timeoutAt = Date.now() + (params.timeoutMs ?? 10 * 60_000);
const pollIntervalMs = params.pollIntervalMs ?? 5000;
while (Date.now() < timeoutAt) {
const task = await getMeshyImageTo3dTask(params.taskId);
const task = await getSelfHostedImageTo3dTask(params.taskId);
if (
task.status === "SUCCEEDED" ||
task.status === "FAILED" ||
@@ -201,7 +210,7 @@ export const waitForMeshyImageTo3dTask = async (params: {
}
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
}
throw new Error("Timed out waiting for Meshy image-to-3D task.");
throw new Error("Timed out waiting for self-hosted image-to-3D task.");
};
export const buildRealAiSummary = (params: {
src/lib/studio-world/store.ts
+19 -11
View File
@@ -8,6 +8,7 @@ import {
} from "@/lib/studio-world/generator";
import {
buildImagePaletteFromBuffer,
buildImageIntensitySamples,
resolveImageSize,
} from "@/lib/studio-world/image-analysis";
import type {
@@ -96,16 +97,19 @@ const normalizeStore = (value: unknown): StudioProjectsStore => {
seed: asNumber(entry.seed, 0),
mode:
entry.mode === "image_avatar" ||
entry.mode === "image_mesh" ||
entry.mode === "ai_image_to_3d"
entry.mode === "image_mesh"
? entry.mode
: "text_scene",
provider: entry.provider === "meshy" ? "meshy" : "local",
provider:
entry.provider === "self_hosted" ? "self_hosted" : "local",
createdAt,
updatedAt,
latestJob: {
id: asString(entry.latestJob.id, "job"),
provider: entry.latestJob.provider === "meshy" ? "meshy" : "local",
provider:
entry.latestJob.provider === "self_hosted"
? "self_hosted"
: "local",
status:
entry.latestJob.status === "pending" ||
entry.latestJob.status === "in_progress" ||
@@ -118,8 +122,7 @@ const normalizeStore = (value: unknown): StudioProjectsStore => {
assetCount: asNumber(entry.latestJob.assetCount, 0),
mode:
entry.latestJob.mode === "image_avatar" ||
entry.latestJob.mode === "image_mesh" ||
entry.latestJob.mode === "ai_image_to_3d"
entry.latestJob.mode === "image_mesh"
? entry.latestJob.mode
: "text_scene",
progress:
@@ -149,7 +152,10 @@ const normalizeStore = (value: unknown): StudioProjectsStore => {
sceneDraft: entry.sceneDraft as StudioProjectRecord["sceneDraft"],
externalModel: isRecord(entry.externalModel)
? ({
provider: entry.externalModel.provider === "meshy" ? "meshy" : "local",
provider:
entry.externalModel.provider === "self_hosted"
? "self_hosted"
: "local",
taskId: asString(entry.externalModel.taskId),
status:
entry.externalModel.status === "pending" ||
@@ -264,6 +270,7 @@ export const createStudioSourceImage = (params: {
storagePath,
dataUrl: `data:${params.mimeType};base64,${params.buffer.toString("base64")}`,
palette,
intensitySamples: buildImageIntensitySamples(params.buffer),
};
};
@@ -272,7 +279,8 @@ export const createStudioProject = (input: StudioGenerationInput) => {
const createdAt = new Date().toISOString();
const seed = resolveGenerationSeed(input);
const sceneDraft = buildStudioWorldDraft(input);
const provider: StudioWorldGenerationProvider = input.provider === "meshy" ? "meshy" : "local";
const provider: StudioWorldGenerationProvider =
input.provider === "self_hosted" ? "self_hosted" : "local";
const latestJob: StudioGenerationJobRecord = {
id: createJobId(),
provider,
@@ -320,15 +328,15 @@ export const createStudioPendingProject = (params: {
sourceImage: params.input.sourceImage,
imageMode: params.input.imageMode === "mesh" ? "mesh" : "avatar",
});
const mode = "ai_image_to_3d";
const provider: StudioWorldGenerationProvider = "meshy";
const mode = "image_mesh";
const provider: StudioWorldGenerationProvider = "self_hosted";
const latestJob: StudioGenerationJobRecord = {
id: createJobId(),
provider,
status: "pending",
createdAt,
finishedAt: createdAt,
summary: "Submitted real AI image-to-3D task.",
summary: "Submitted self-hosted image-to-3D task.",
assetCount: 0,
mode,
progress: 0,
src/lib/studio-world/types.ts
+5 -3
View File
@@ -27,6 +27,8 @@ export type StudioWorldAssetKind =
| "beacon"
| "crate"
| "portal"
| "heightfield_panel"
| "billboard_frame"
| "avatar_head"
| "avatar_hair"
| "avatar_torso"
@@ -37,10 +39,9 @@ export type StudioWorldAssetKind =
export type StudioWorldGenerationMode =
| "text_scene"
| "image_avatar"
| "image_mesh"
| "ai_image_to_3d";
| "image_mesh";
export type StudioWorldGenerationProvider = "local" | "meshy";
export type StudioWorldGenerationProvider = "local" | "self_hosted";
export type StudioSourceImageRecord = {
id: string;
@@ -52,6 +53,7 @@ export type StudioSourceImageRecord = {
storagePath: string;
dataUrl: string;
palette: string[];
intensitySamples?: number[];
};
export type StudioWorldAssetDraft = {
tests/unit/studioWorldRoute.test.ts
+8 -7
View File
@@ -248,10 +248,10 @@ describe("studio world route", () => {
).toBe(true);
});
it("submits a real AI image-to-3D task when Meshy is configured", async () => {
tempDir = makeTempDir("studio-world-meshy-route");
it("submits a real AI image-to-3D task when a self-hosted provider is configured", async () => {
tempDir = makeTempDir("studio-world-self-hosted-route");
process.env.OPENCLAW_STATE_DIR = tempDir;
process.env.MESHY_API_KEY = "msy_dummy_api_key_for_test_mode_12345678";
process.env.CLAW3D_STUDIO_PROVIDER_URL = "http://provider.test/openapi/v1";
process.env.CLAW3D_STUDIO_ENABLE_REAL_AI = "true";
const pngBytes = Uint8Array.from([
@@ -303,7 +303,8 @@ describe("studio world route", () => {
style: "realistic",
scale: "medium",
focus: "assets",
provider: "meshy",
provider: "self_hosted",
imageMode: "mesh",
sourceImage: uploadBody.sourceImage,
},
}),
@@ -320,13 +321,13 @@ describe("studio world route", () => {
};
expect(response.status).toBe(200);
expect(body.project?.provider).toBe("meshy");
expect(body.project?.mode).toBe("ai_image_to_3d");
expect(body.project?.provider).toBe("self_hosted");
expect(body.project?.mode).toBe("image_mesh");
expect(body.project?.latestJob?.status).toBe("pending");
expect(body.project?.latestJob?.providerTaskId).toBe("task_test_123");
expect(body.project?.externalModel?.taskId).toBe("task_test_123");
expect(body.project?.externalModel?.status).toBe("pending");
expect(body.providerAvailability?.provider).toBe("meshy");
expect(body.providerAvailability?.provider).toBe("self_hosted");
expect(body.providerAvailability?.available).toBe(true);
});
});