From b13d70f2dbaa69733177fbeb2ee8a3d25b84b9c5 Mon Sep 17 00:00:00 2001 From: Yuzu Date: Sat, 16 May 2026 11:51:44 +0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20=20feat:=20transparency,=20clipboar?= =?UTF-8?q?d=20paste,=20face=20detection=20optimization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/web/components/canvas-renderer.test.tsx | 63 +++++ apps/web/components/canvas-renderer.tsx | 162 ++++++++--- apps/web/components/editor/editor-header.tsx | 2 +- apps/web/hooks/use-face-detection.ts | 88 ++++-- apps/web/lib/export-png.ts | 42 ++- apps/web/lib/face-blur-renderer.test.ts | 270 ++++++++++++++++++- apps/web/lib/face-blur-renderer.ts | 215 ++++++++++++++- apps/web/lib/face-detection-utils.ts | 34 ++- apps/web/lib/face-detection-worker.ts | 260 ++++++++++++++++++ apps/web/lib/face-ml.ts | 188 ++++++------- apps/web/messages/en.json | 3 +- apps/web/messages/ja.json | 3 +- apps/web/messages/th.json | 3 +- apps/web/next-env.d.ts | 2 +- apps/web/next.config.mjs | 2 +- apps/web/store/editor-store.ts | 64 ++++- 16 files changed, 1186 insertions(+), 215 deletions(-) create mode 100644 apps/web/components/canvas-renderer.test.tsx create mode 100644 apps/web/lib/face-detection-worker.ts diff --git a/apps/web/components/canvas-renderer.test.tsx b/apps/web/components/canvas-renderer.test.tsx new file mode 100644 index 0000000..667d987 --- /dev/null +++ b/apps/web/components/canvas-renderer.test.tsx @@ -0,0 +1,63 @@ +import React from "react"; +import { fireEvent, render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { CanvasRenderer } from "./canvas-renderer"; +import type { Layer } from "@pien-studio/types"; + +vi.mock("next/image", () => ({ + default: ({ alt, src, unoptimized: _unoptimized, ...props }: React.ImgHTMLAttributes & { unoptimized?: boolean }) => React.createElement("img", { alt, src, ...props }), +})); + +vi.mock("../hooks/use-translations", () => ({ + useTranslations: () => ({ t: (key: string) => key }), +})); + +class ResizeObserverMock { + observe() {} + disconnect() {} +} + +describe("CanvasRenderer", () => { + beforeEach(() => { + vi.stubGlobal("ResizeObserver", ResizeObserverMock); + Object.defineProperty(HTMLElement.prototype, "setPointerCapture", { configurable: true, value: vi.fn() }); + }); + + it("keeps the rotation handle interactive", () => { + const layer: Layer = { + id: "layer-1", + type: "image", + sourceUri: "data:image/png;base64,test", + x: 0, + y: 0, + width: 100, + height: 100, + scale: 1, + rotation: 0, + opacity: 1, + }; + const onRotateLayer = vi.fn(); + + render( + undefined} + onMoveLayer={() => undefined} + onResizeLayer={() => undefined} + onRotateLayer={onRotateLayer} + isDark={false} + />, + ); + + const rotateHandle = screen.getByTitle("editor.rotate"); + expect(rotateHandle).toHaveClass("pointer-events-auto"); + + fireEvent(rotateHandle, new MouseEvent("pointerdown", { bubbles: true, button: 0, clientX: 50, clientY: 0 })); + fireEvent(rotateHandle, new MouseEvent("pointermove", { bubbles: true, clientX: 100, clientY: 50 })); + + expect(onRotateLayer).toHaveBeenCalledWith(layer.id, 90); + }); +}); diff --git a/apps/web/components/canvas-renderer.tsx b/apps/web/components/canvas-renderer.tsx index 28fd263..3debe5c 100644 --- a/apps/web/components/canvas-renderer.tsx +++ b/apps/web/components/canvas-renderer.tsx @@ -3,9 +3,12 @@ import React from "react"; import NextImage from "next/image"; import { RotateCw } from "lucide-react"; -import { CANVAS_HANDLE_BASE_SIZE, CANVAS_ROTATE_HANDLE_BASE_SIZE } from "../lib/editor-constants"; +import { + CANVAS_HANDLE_BASE_SIZE, + CANVAS_ROTATE_HANDLE_BASE_SIZE, +} from "../lib/editor-constants"; import { buildFaceLabelOverlays } from "../lib/canvas-geometry"; -import { renderFaceBlurRegions } from "../lib/face-blur-renderer"; +import { renderImageWithFaceBlur } from "../lib/face-blur-renderer"; import { useCanvasInteractions } from "../hooks/use-canvas-interactions"; import { useTranslations } from "../hooks/use-translations"; import type { FaceBlurMethod, Layer } from "@pien-studio/types"; @@ -27,7 +30,13 @@ interface CanvasRendererProps { onContextMenu?: (x: number, y: number) => void; isDark: boolean; tool?: "pointer" | "hand" | "face"; - faceDetections?: { x: number; y: number; width: number; height: number; label?: string }[]; + faceDetections?: { + x: number; + y: number; + width: number; + height: number; + label?: string; + }[]; faceOverlayLayerId?: string | null; faceBlurPreview?: { layerId: string; @@ -49,7 +58,13 @@ function BlurredImageLayer({ faceBlurOverride?: { method: FaceBlurMethod; amount: number; - regions: { x: number; y: number; width: number; height: number; censorColor?: string }[]; + regions: { + x: number; + y: number; + width: number; + height: number; + censorColor?: string; + }[]; censorColor?: string; } | null; }) { @@ -65,10 +80,8 @@ function BlurredImageLayer({ const cw = canvas.width; const ch = canvas.height; ctx.clearRect(0, 0, cw, ch); - ctx.drawImage(image, 0, 0, cw, ch); const blur = faceBlurOverride ?? layer.faceBlur; - if (!blur || blur.regions.length === 0) return; - renderFaceBlurRegions(ctx, image, blur, cw, ch); + renderImageWithFaceBlur(ctx, image, blur, cw, ch); }, [faceBlurOverride, layer.faceBlur]); React.useEffect(() => { @@ -91,7 +104,14 @@ function BlurredImageLayer({ draw(); }, [draw, width, height]); - return ; + return ( + + ); } export function CanvasRenderer({ @@ -147,15 +167,35 @@ export function CanvasRenderer({ }); const faceLabelOverlays = React.useMemo(() => { - if (tool !== "face" || !faceOverlayLayerId || faceDetections.length === 0) return []; - return buildFaceLabelOverlays(layers, faceOverlayLayerId, faceDetections, viewport); - }, [faceDetections, faceOverlayLayerId, layers, tool, viewport.scale, viewport.x, viewport.y]); + if (tool !== "face" || !faceOverlayLayerId || faceDetections.length === 0) + return []; + return buildFaceLabelOverlays( + layers, + faceOverlayLayerId, + faceDetections, + viewport, + ); + }, [ + faceDetections, + faceOverlayLayerId, + layers, + tool, + viewport.scale, + viewport.x, + viewport.y, + ]); return (
{ if (tool === "pointer" && e.button === 0) onSelectLayer(null); onContainerPointerDown(e); @@ -183,18 +223,26 @@ export function CanvasRenderer({ willChange: "transform", transform: `translate(${viewport.x}px, ${viewport.y}px) scale(${viewport.scale})`, transformOrigin: "0 0", - boxShadow: "0 0 0 1px var(--color-accent-strong)", - background: isDark ? "#17181b" : "#ffffff", + outline: `${2 / viewport.scale}px solid var(--color-accent-strong)`, + background: + "linear-gradient(45deg, #d0d0d0 25%, transparent 25%), linear-gradient(-45deg, #d0d0d0 25%, transparent 25%), linear-gradient(45deg, transparent 75%, #d0d0d0 75%), linear-gradient(-45deg, transparent 75%, #d0d0d0 75%)", + backgroundSize: "16px 16px", + backgroundPosition: "0 0, 0 8px, 8px -8px, -8px 0px", }} > {layers.map((layer, idx) => { const isSelected = layer.id === selectedLayerId; const isImage = layer.type === "image"; - const layerWidth = layer.width ?? (isImage ? Math.round(200 * layer.scale) : undefined); - const layerHeight = layer.height ?? (isImage ? Math.round(150 * layer.scale) : undefined); + const layerWidth = + layer.width ?? + (isImage ? Math.round(200 * layer.scale) : undefined); + const layerHeight = + layer.height ?? + (isImage ? Math.round(150 * layer.scale) : undefined); const handleSize = CANVAS_HANDLE_BASE_SIZE / viewport.scale; const handleSizePx = `${handleSize}px`; - const largeHandleSize = CANVAS_ROTATE_HANDLE_BASE_SIZE / viewport.scale; + const largeHandleSize = + CANVAS_ROTATE_HANDLE_BASE_SIZE / viewport.scale; const largeHandleSizePx = `${largeHandleSize}px`; return ( @@ -209,8 +257,12 @@ export function CanvasRenderer({ transform: `rotate(${layer.rotation}deg)`, opacity: layer.opacity, cursor: "move", - border: isSelected ? "2px solid var(--color-accent-strong)" : "1px dashed transparent", - outline: isSelected ? "2px solid var(--color-accent-strong)" : "none", + border: isSelected + ? "2px solid var(--color-accent-strong)" + : "1px dashed transparent", + outline: isSelected + ? "2px solid var(--color-accent-strong)" + : "none", outlineOffset: "2px", zIndex: idx, }} @@ -218,13 +270,19 @@ export function CanvasRenderer({ onClick={() => onSelectLayer(layer.id)} > {isImage && layer.sourceUri ? ( - (faceBlurPreview && faceBlurPreview.layerId === layer.id && faceBlurPreview.regions.length > 0) || + (faceBlurPreview && + faceBlurPreview.layerId === layer.id && + faceBlurPreview.regions.length > 0) || (layer.faceBlur && layer.faceBlur.regions.length > 0) ? ( ) : ( onResizeHandleDown(e, layer, "tl")} title={t("editor.resize")} />
diff --git a/apps/web/components/editor/editor-header.tsx b/apps/web/components/editor/editor-header.tsx index da40ccc..2b1ecd3 100644 --- a/apps/web/components/editor/editor-header.tsx +++ b/apps/web/components/editor/editor-header.tsx @@ -176,12 +176,12 @@ function FileMenu({ }) { return (
+
-
); diff --git a/apps/web/hooks/use-face-detection.ts b/apps/web/hooks/use-face-detection.ts index b8263c7..b7849e9 100644 --- a/apps/web/hooks/use-face-detection.ts +++ b/apps/web/hooks/use-face-detection.ts @@ -1,6 +1,10 @@ import React from "react"; import type { EditorToolId } from "../store/editor-store"; -import { buildFacePreviews, loadImageFromUri, toFaceDetectionOverlays } from "../lib/face-detection-utils"; +import { + buildFacePreviews, + loadImageFromUri, + toFaceDetectionOverlays, +} from "../lib/face-detection-utils"; export type FaceDetectionOverlay = { x: number; @@ -34,21 +38,35 @@ type UseFaceDetectionOptions = { }; export function useFaceDetection(options: UseFaceDetectionOptions) { - const { tool, selectedLayerId, selectedImageLayer, activeLayerStillSelected } = options; + const { + tool, + selectedLayerId, + selectedImageLayer, + activeLayerStillSelected, + } = options; const selectedImageLayerId = selectedImageLayer?.id ?? null; const selectedImageSourceUri = selectedImageLayer?.sourceUri ?? null; const selectedImageWidth = selectedImageLayer?.width; const selectedImageHeight = selectedImageLayer?.height; - const [faceDetections, setFaceDetections] = React.useState([]); - const [faceDetectionsLayerId, setFaceDetectionsLayerId] = React.useState(null); - const [faceStatus, setFaceStatus] = React.useState<"idle" | "detecting" | "unsupported">("idle"); + const [faceDetections, setFaceDetections] = React.useState< + FaceDetectionOverlay[] + >([]); + const [faceDetectionsLayerId, setFaceDetectionsLayerId] = React.useState< + string | null + >(null); + const [faceStatus, setFaceStatus] = React.useState< + "idle" | "detecting" | "unsupported" + >("idle"); const [facePreviews, setFacePreviews] = React.useState([]); - const resetFaceState = React.useCallback((status: "idle" | "detecting" | "unsupported" = "idle") => { - setFaceDetections((prev) => (prev.length === 0 ? prev : [])); - setFaceDetectionsLayerId((prev) => (prev === null ? prev : null)); - setFaceStatus((prev) => (prev === status ? prev : status)); - }, []); + const resetFaceState = React.useCallback( + (status: "idle" | "detecting" | "unsupported" = "idle") => { + setFaceDetections((prev) => (prev.length === 0 ? prev : [])); + setFaceDetectionsLayerId((prev) => (prev === null ? prev : null)); + setFaceStatus((prev) => (prev === status ? prev : status)); + }, + [], + ); React.useEffect(() => { let canceled = false; @@ -60,21 +78,18 @@ export function useFaceDetection(options: UseFaceDetectionOptions) { return; } - resetFaceState("detecting"); - const image = await loadImageFromUri(selectedImageSourceUri); - if (!image) { - if (!canceled) { - resetFaceState("idle"); - } - return; - } - try { + resetFaceState("detecting"); const { detectFaceBoxes } = await import("../lib/face-ml"); - const faces = await detectFaceBoxes(image); + const result = await detectFaceBoxes(selectedImageSourceUri); if (canceled) return; if (!activeLayerStillSelected(layerId) || tool !== "face") return; - const overlays = toFaceDetectionOverlays(faces, image, selectedImageWidth, selectedImageHeight); + const overlays = toFaceDetectionOverlays( + result.faces, + result, + selectedImageWidth, + selectedImageHeight, + ); setFaceDetections(overlays); setFaceDetectionsLayerId(layerId); setFaceStatus((prev) => (prev === "idle" ? prev : "idle")); @@ -90,7 +105,15 @@ export function useFaceDetection(options: UseFaceDetectionOptions) { return () => { canceled = true; }; - }, [activeLayerStillSelected, resetFaceState, selectedImageHeight, selectedImageLayerId, selectedImageSourceUri, selectedImageWidth, tool]); + }, [ + activeLayerStillSelected, + resetFaceState, + selectedImageHeight, + selectedImageLayerId, + selectedImageSourceUri, + selectedImageWidth, + tool, + ]); React.useEffect(() => { if (tool !== "face") { @@ -108,7 +131,11 @@ export function useFaceDetection(options: UseFaceDetectionOptions) { let canceled = false; async function generateFacePreviews() { - if (tool !== "face" || !selectedImageSourceUri || faceDetections.length === 0) { + if ( + tool !== "face" || + !selectedImageSourceUri || + faceDetections.length === 0 + ) { setFacePreviews([]); return; } @@ -119,7 +146,12 @@ export function useFaceDetection(options: UseFaceDetectionOptions) { return; } - const previews = buildFacePreviews(image, faceDetections, selectedImageWidth, selectedImageHeight); + const previews = buildFacePreviews( + image, + faceDetections, + selectedImageWidth, + selectedImageHeight, + ); if (!canceled) setFacePreviews(previews); } @@ -128,7 +160,13 @@ export function useFaceDetection(options: UseFaceDetectionOptions) { return () => { canceled = true; }; - }, [faceDetections, selectedImageHeight, selectedImageSourceUri, selectedImageWidth, tool]); + }, [ + faceDetections, + selectedImageHeight, + selectedImageSourceUri, + selectedImageWidth, + tool, + ]); return { faceDetections, faceDetectionsLayerId, facePreviews, faceStatus }; } diff --git a/apps/web/lib/export-png.ts b/apps/web/lib/export-png.ts index 525c785..88a273d 100644 --- a/apps/web/lib/export-png.ts +++ b/apps/web/lib/export-png.ts @@ -1,5 +1,5 @@ import type { Layer, Project } from "@pien-studio/types"; -import { renderFaceBlurRegions } from "./face-blur-renderer"; +import { renderImageWithFaceBlur } from "./face-blur-renderer"; type ExportOptions = { isDark: boolean; @@ -20,7 +20,11 @@ function loadImage(src: string) { }); } -function drawFallbackLayer(ctx: CanvasRenderingContext2D, layer: Layer, isDark: boolean) { +function drawFallbackLayer( + ctx: CanvasRenderingContext2D, + layer: Layer, + isDark: boolean, +) { const text = layer.name ?? layer.type; const width = Math.max(80, layer.width ?? 120); const height = Math.max(34, layer.height ?? 40); @@ -45,15 +49,24 @@ function drawFallbackLayer(ctx: CanvasRenderingContext2D, layer: Layer, isDark: ctx.stroke(); ctx.fillStyle = isDark ? "#d7dae0" : "#1f2430"; - ctx.font = "600 12px ui-sans-serif, system-ui, -apple-system, Segoe UI, sans-serif"; + ctx.font = + "600 12px ui-sans-serif, system-ui, -apple-system, Segoe UI, sans-serif"; ctx.textAlign = "center"; ctx.textBaseline = "middle"; ctx.fillText(text, width / 2, height / 2); } -async function drawLayer(ctx: CanvasRenderingContext2D, layer: Layer, isDark: boolean) { - const width = layer.width ?? (layer.type === "image" ? Math.round(200 * layer.scale) : 120); - const height = layer.height ?? (layer.type === "image" ? Math.round(150 * layer.scale) : 40); +async function drawLayer( + ctx: CanvasRenderingContext2D, + layer: Layer, + isDark: boolean, +) { + const width = + layer.width ?? + (layer.type === "image" ? Math.round(200 * layer.scale) : 120); + const height = + layer.height ?? + (layer.type === "image" ? Math.round(150 * layer.scale) : 40); ctx.save(); ctx.globalAlpha = clampOpacity(layer.opacity); @@ -64,10 +77,7 @@ async function drawLayer(ctx: CanvasRenderingContext2D, layer: Layer, isDark: bo if ((layer.type === "image" || layer.type === "sticker") && layer.sourceUri) { try { const image = await loadImage(layer.sourceUri); - ctx.drawImage(image, 0, 0, width, height); - if (layer.faceBlur && layer.faceBlur.regions.length > 0) { - renderFaceBlurRegions(ctx, image, layer.faceBlur, width, height); - } + renderImageWithFaceBlur(ctx, image, layer.faceBlur, width, height); } catch { drawFallbackLayer(ctx, layer, isDark); } @@ -78,8 +88,14 @@ async function drawLayer(ctx: CanvasRenderingContext2D, layer: Layer, isDark: bo ctx.restore(); } -export async function exportProjectAsPng(project: Project, options: ExportOptions) { - const pixelRatio = Math.max(1, Math.floor(options.pixelRatio ?? window.devicePixelRatio ?? 1)); +export async function exportProjectAsPng( + project: Project, + options: ExportOptions, +) { + const pixelRatio = Math.max( + 1, + Math.floor(options.pixelRatio ?? window.devicePixelRatio ?? 1), + ); const { width, height } = project.canvas; const canvas = document.createElement("canvas"); canvas.width = width * pixelRatio; @@ -89,8 +105,6 @@ export async function exportProjectAsPng(project: Project, options: ExportOption if (!ctx) throw new Error("Cannot create export canvas context"); ctx.scale(pixelRatio, pixelRatio); - ctx.fillStyle = options.isDark ? "#17181b" : "#ffffff"; - ctx.fillRect(0, 0, width, height); for (const layer of project.layers) { await drawLayer(ctx, layer, options.isDark); diff --git a/apps/web/lib/face-blur-renderer.test.ts b/apps/web/lib/face-blur-renderer.test.ts index 9fda8d4..5e55396 100644 --- a/apps/web/lib/face-blur-renderer.test.ts +++ b/apps/web/lib/face-blur-renderer.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it, vi } from "vitest"; -import { renderFaceBlurRegions } from "./face-blur-renderer"; +import { + renderFaceBlurRegions, + renderImageWithFaceBlur, +} from "./face-blur-renderer"; function makeContext() { return { @@ -27,7 +30,16 @@ describe("renderFaceBlurRegions", () => { { method: "gaussian", amount: 24, - regions: [{ x: 120, y: 80, width: 300, height: 200, sourceWidth: 1200, sourceHeight: 800 }], + regions: [ + { + x: 120, + y: 80, + width: 300, + height: 200, + sourceWidth: 1200, + sourceHeight: 800, + }, + ], }, 600, 400, @@ -35,7 +47,17 @@ describe("renderFaceBlurRegions", () => { expect(ctx.save).toHaveBeenCalledOnce(); expect(ctx.filter).toBe("blur(24px)"); - expect(ctx.drawImage).toHaveBeenCalledWith(image, 120, 80, 300, 200, 60, 40, 150, 100); + expect(ctx.drawImage).toHaveBeenCalledWith( + image, + 120, + 80, + 300, + 200, + 60, + 40, + 150, + 100, + ); expect(ctx.restore).toHaveBeenCalledOnce(); }); @@ -46,13 +68,22 @@ describe("renderFaceBlurRegions", () => { const ctx = makeContext(); const image = makeImage(); const sampleDrawImage = vi.fn(); - const sampleCtx = { imageSmoothingEnabled: true, drawImage: sampleDrawImage } as unknown as CanvasRenderingContext2D; - const sampleCanvas = { width: 0, height: 0, getContext: vi.fn(() => sampleCtx) } as unknown as HTMLCanvasElement; + const sampleCtx = { + imageSmoothingEnabled: true, + drawImage: sampleDrawImage, + } as unknown as CanvasRenderingContext2D; + const sampleCanvas = { + width: 0, + height: 0, + getContext: vi.fn(() => sampleCtx), + } as unknown as HTMLCanvasElement; const nativeCreateElement = doc.createElement.bind(doc); - const createElement = vi.spyOn(doc, "createElement").mockImplementation((tagName: string) => { - if (tagName === "canvas") return sampleCanvas; - return nativeCreateElement(tagName); - }); + const createElement = vi + .spyOn(doc, "createElement") + .mockImplementation((tagName: string) => { + if (tagName === "canvas") return sampleCanvas; + return nativeCreateElement(tagName); + }); renderFaceBlurRegions( ctx, @@ -60,7 +91,16 @@ describe("renderFaceBlurRegions", () => { { method: "pixelate", amount: 10, - regions: [{ x: 200, y: 100, width: 160, height: 120, sourceWidth: 1200, sourceHeight: 800 }], + regions: [ + { + x: 200, + y: 100, + width: 160, + height: 120, + sourceWidth: 1200, + sourceHeight: 800, + }, + ], }, 600, 400, @@ -68,8 +108,28 @@ describe("renderFaceBlurRegions", () => { expect(sampleCanvas.width).toBe(16); expect(sampleCanvas.height).toBe(12); - expect(sampleDrawImage).toHaveBeenCalledWith(image, 200, 100, 160, 120, 0, 0, 16, 12); - expect(ctx.drawImage).toHaveBeenCalledWith(sampleCanvas, 0, 0, 16, 12, 100, 50, 80, 60); + expect(sampleDrawImage).toHaveBeenCalledWith( + image, + 200, + 100, + 160, + 120, + 0, + 0, + 16, + 12, + ); + expect(ctx.drawImage).toHaveBeenCalledWith( + sampleCanvas, + 0, + 0, + 16, + 12, + 100, + 50, + 80, + 60, + ); createElement.mockRestore(); }); @@ -83,7 +143,17 @@ describe("renderFaceBlurRegions", () => { method: "censor", amount: 20, censorColor: "#ff0000", - regions: [{ x: 20, y: 30, width: 40, height: 50, sourceWidth: 1200, sourceHeight: 800, censorColor: "#00ff00" }], + regions: [ + { + x: 20, + y: 30, + width: 40, + height: 50, + sourceWidth: 1200, + sourceHeight: 800, + censorColor: "#00ff00", + }, + ], }, 600, 400, @@ -108,6 +178,178 @@ describe("renderFaceBlurRegions", () => { 400, ); - expect(ctx.drawImage).toHaveBeenCalledWith(image, 400, 480, 1200, 800, 100, 120, 300, 200); + expect(ctx.drawImage).toHaveBeenCalledWith( + image, + 400, + 480, + 1200, + 800, + 100, + 120, + 300, + 200, + ); + }); + + it("applies blur to source-sized image before drawing the resized layer", () => { + const doc = globalThis.document; + expect(doc).toBeDefined(); + if (!doc) return; + const ctx = makeContext(); + const image = makeImage(); + const sourceDrawImage = vi.fn(); + const sourceCtx = { + ...makeContext(), + drawImage: sourceDrawImage, + } as unknown as CanvasRenderingContext2D; + const sourceCanvas = { + width: 0, + height: 0, + getContext: vi.fn(() => sourceCtx), + } as unknown as HTMLCanvasElement; + const nativeCreateElement = doc.createElement.bind(doc); + const createElement = vi + .spyOn(doc, "createElement") + .mockImplementation((tagName: string) => { + if (tagName === "canvas") return sourceCanvas; + return nativeCreateElement(tagName); + }); + + renderImageWithFaceBlur( + ctx, + image, + { + method: "gaussian", + amount: 24, + regions: [ + { + x: 120, + y: 80, + width: 300, + height: 200, + sourceWidth: 1200, + sourceHeight: 800, + }, + ], + }, + 600, + 400, + ); + + expect(sourceCanvas.width).toBe(1200); + expect(sourceCanvas.height).toBe(800); + expect(sourceDrawImage).toHaveBeenNthCalledWith(1, image, 0, 0, 1200, 800); + expect(sourceDrawImage).toHaveBeenNthCalledWith( + 2, + image, + 120, + 80, + 300, + 200, + 120, + 80, + 300, + 200, + ); + expect(ctx.drawImage).toHaveBeenCalledWith( + sourceCanvas, + 0, + 0, + 1200, + 800, + 0, + 0, + 600, + 400, + ); + createElement.mockRestore(); + }); + + it("uses a canvas-filter fallback for gaussian blur when filters are unavailable", () => { + const doc = globalThis.document; + expect(doc).toBeDefined(); + if (!doc) return; + const ctx = makeContext(); + delete (ctx as Partial).filter; + const image = makeImage(); + const regionDrawImage = vi.fn(); + const blurDrawImage = vi.fn(); + const regionCtx = { + ...makeContext(), + clearRect: vi.fn(), + drawImage: regionDrawImage, + } as unknown as CanvasRenderingContext2D; + const blurCtx = { + ...makeContext(), + clearRect: vi.fn(), + drawImage: blurDrawImage, + } as unknown as CanvasRenderingContext2D; + const regionCanvas = { + width: 0, + height: 0, + getContext: vi.fn(() => regionCtx), + } as unknown as HTMLCanvasElement; + const blurCanvas = { + width: 0, + height: 0, + getContext: vi.fn(() => blurCtx), + } as unknown as HTMLCanvasElement; + const nativeCreateElement = doc.createElement.bind(doc); + const createElement = vi + .spyOn(doc, "createElement") + .mockImplementation((tagName: string) => { + if (tagName !== "canvas") return nativeCreateElement(tagName); + return createElement.mock.calls.length === 1 + ? regionCanvas + : blurCanvas; + }); + + renderFaceBlurRegions( + ctx, + image, + { + method: "gaussian", + amount: 24, + regions: [ + { + x: 120, + y: 80, + width: 300, + height: 200, + sourceWidth: 1200, + sourceHeight: 800, + }, + ], + }, + 600, + 400, + ); + + expect(ctx.save).not.toHaveBeenCalled(); + expect(regionCanvas.width).toBe(150); + expect(regionCanvas.height).toBe(100); + expect(regionDrawImage).toHaveBeenCalledWith( + image, + 120, + 80, + 300, + 200, + 0, + 0, + 150, + 100, + ); + expect(ctx.drawImage).toHaveBeenCalledWith( + regionCanvas, + 0, + 0, + 150, + 100, + 60, + 40, + 150, + 100, + ); + createElement.mockRestore(); }); }); diff --git a/apps/web/lib/face-blur-renderer.ts b/apps/web/lib/face-blur-renderer.ts index 4dea30c..680acfa 100644 --- a/apps/web/lib/face-blur-renderer.ts +++ b/apps/web/lib/face-blur-renderer.ts @@ -21,16 +21,126 @@ function drawPixelatedRegion( const sampleCtx = sampleCanvas.getContext("2d"); if (!sampleCtx) return; sampleCtx.imageSmoothingEnabled = false; - sampleCtx.drawImage(source, sourceX, sourceY, sourceWidth, sourceHeight, 0, 0, sampleCanvas.width, sampleCanvas.height); + sampleCtx.drawImage( + source, + sourceX, + sourceY, + sourceWidth, + sourceHeight, + 0, + 0, + sampleCanvas.width, + sampleCanvas.height, + ); ctx.imageSmoothingEnabled = false; - ctx.drawImage(sampleCanvas, 0, 0, sampleCanvas.width, sampleCanvas.height, targetX, targetY, targetWidth, targetHeight); + ctx.drawImage( + sampleCanvas, + 0, + 0, + sampleCanvas.width, + sampleCanvas.height, + targetX, + targetY, + targetWidth, + targetHeight, + ); ctx.imageSmoothingEnabled = true; } +function canUseCanvasFilter(ctx: CanvasRenderingContext2D) { + return "filter" in ctx && typeof ctx.filter === "string"; +} + +function drawBlurredRegionFallback( + ctx: CanvasRenderingContext2D, + source: CanvasImageSource, + sourceX: number, + sourceY: number, + sourceWidth: number, + sourceHeight: number, + targetX: number, + targetY: number, + targetWidth: number, + targetHeight: number, + amount: number, +) { + const regionCanvas = document.createElement("canvas"); + regionCanvas.width = Math.max(1, Math.round(targetWidth)); + regionCanvas.height = Math.max(1, Math.round(targetHeight)); + const regionCtx = regionCanvas.getContext("2d"); + if (!regionCtx) return; + + regionCtx.drawImage( + source, + sourceX, + sourceY, + sourceWidth, + sourceHeight, + 0, + 0, + regionCanvas.width, + regionCanvas.height, + ); + + const scale = Math.max(0.04, Math.min(0.5, 1 / Math.max(2, amount / 2))); + const blurCanvas = document.createElement("canvas"); + blurCanvas.width = Math.max(1, Math.round(regionCanvas.width * scale)); + blurCanvas.height = Math.max(1, Math.round(regionCanvas.height * scale)); + const blurCtx = blurCanvas.getContext("2d"); + if (!blurCtx) return; + + blurCtx.imageSmoothingEnabled = true; + blurCtx.drawImage(regionCanvas, 0, 0, blurCanvas.width, blurCanvas.height); + regionCtx.imageSmoothingEnabled = true; + for (let i = 0; i < 3; i++) { + regionCtx.clearRect(0, 0, regionCanvas.width, regionCanvas.height); + regionCtx.drawImage( + blurCanvas, + 0, + 0, + blurCanvas.width, + blurCanvas.height, + 0, + 0, + regionCanvas.width, + regionCanvas.height, + ); + blurCtx.clearRect(0, 0, blurCanvas.width, blurCanvas.height); + blurCtx.drawImage( + regionCanvas, + 0, + 0, + regionCanvas.width, + regionCanvas.height, + 0, + 0, + blurCanvas.width, + blurCanvas.height, + ); + } + + ctx.drawImage( + regionCanvas, + 0, + 0, + regionCanvas.width, + regionCanvas.height, + targetX, + targetY, + targetWidth, + targetHeight, + ); +} + export function renderFaceBlurRegions( ctx: CanvasRenderingContext2D, image: HTMLImageElement, - blur: { method: FaceBlurSettings["method"]; amount: number; regions: BlurRegion[]; censorColor?: string }, + blur: { + method: FaceBlurSettings["method"]; + amount: number; + regions: BlurRegion[]; + censorColor?: string; + }, targetWidth: number, targetHeight: number, ): void { @@ -49,10 +159,18 @@ export function renderFaceBlurRegions( const y = Math.max(0, Math.floor(region.y * scaleY)); const w = Math.max(1, Math.floor(region.width * scaleX)); const h = Math.max(1, Math.floor(region.height * scaleY)); - const sx0 = hasSourceDims ? region.x : Math.max(0, Math.floor(region.x * legacyScaleX)); - const sy0 = hasSourceDims ? region.y : Math.max(0, Math.floor(region.y * legacyScaleY)); - const sw = hasSourceDims ? region.width : Math.max(1, Math.floor(region.width * legacyScaleX)); - const sh = hasSourceDims ? region.height : Math.max(1, Math.floor(region.height * legacyScaleY)); + const sx0 = hasSourceDims + ? region.x + : Math.max(0, Math.floor(region.x * legacyScaleX)); + const sy0 = hasSourceDims + ? region.y + : Math.max(0, Math.floor(region.y * legacyScaleY)); + const sw = hasSourceDims + ? region.width + : Math.max(1, Math.floor(region.width * legacyScaleX)); + const sh = hasSourceDims + ? region.height + : Math.max(1, Math.floor(region.height * legacyScaleY)); if (blur.method === "censor") { ctx.fillStyle = region.censorColor ?? blur.censorColor ?? "#111111"; @@ -66,9 +184,84 @@ export function renderFaceBlurRegions( continue; } - ctx.save(); - ctx.filter = `blur(${blur.amount}px)`; - ctx.drawImage(image, sx0, sy0, sw, sh, x, y, w, h); - ctx.restore(); + if (canUseCanvasFilter(ctx)) { + ctx.save(); + ctx.filter = `blur(${blur.amount}px)`; + ctx.drawImage(image, sx0, sy0, sw, sh, x, y, w, h); + ctx.restore(); + continue; + } + + drawBlurredRegionFallback( + ctx, + image, + sx0, + sy0, + sw, + sh, + x, + y, + w, + h, + blur.amount, + ); } } + +export function renderImageWithFaceBlur( + ctx: CanvasRenderingContext2D, + image: HTMLImageElement, + blur: + | { + method: FaceBlurSettings["method"]; + amount: number; + regions: BlurRegion[]; + censorColor?: string; + } + | undefined, + targetWidth: number, + targetHeight: number, +): void { + if (!blur || blur.regions.length === 0) { + ctx.drawImage(image, 0, 0, targetWidth, targetHeight); + return; + } + + const canBlurAtSourceSize = blur.regions.every( + (region) => (region.sourceWidth ?? 0) > 0 && (region.sourceHeight ?? 0) > 0, + ); + if (!canBlurAtSourceSize) { + ctx.drawImage(image, 0, 0, targetWidth, targetHeight); + renderFaceBlurRegions(ctx, image, blur, targetWidth, targetHeight); + return; + } + + const sourceCanvas = document.createElement("canvas"); + sourceCanvas.width = Math.max(1, image.naturalWidth); + sourceCanvas.height = Math.max(1, image.naturalHeight); + const sourceCtx = sourceCanvas.getContext("2d"); + if (!sourceCtx) { + ctx.drawImage(image, 0, 0, targetWidth, targetHeight); + return; + } + + sourceCtx.drawImage(image, 0, 0, sourceCanvas.width, sourceCanvas.height); + renderFaceBlurRegions( + sourceCtx, + image, + blur, + sourceCanvas.width, + sourceCanvas.height, + ); + ctx.drawImage( + sourceCanvas, + 0, + 0, + sourceCanvas.width, + sourceCanvas.height, + 0, + 0, + targetWidth, + targetHeight, + ); +} diff --git a/apps/web/lib/face-detection-utils.ts b/apps/web/lib/face-detection-utils.ts index 1ed7060..30c9a8b 100644 --- a/apps/web/lib/face-detection-utils.ts +++ b/apps/web/lib/face-detection-utils.ts @@ -1,6 +1,11 @@ -import type { FaceDetectionOverlay, FacePreview } from "../hooks/use-face-detection"; +import type { + FaceDetectionOverlay, + FacePreview, +} from "../hooks/use-face-detection"; -export async function loadImageFromUri(uri: string): Promise { +export async function loadImageFromUri( + uri: string, +): Promise { const image = new Image(); image.crossOrigin = "anonymous"; await new Promise((resolve) => { @@ -13,8 +18,15 @@ export async function loadImageFromUri(uri: string): Promise, - image: HTMLImageElement, + faces: Array<{ + x: number; + y: number; + width: number; + height: number; + gender?: string; + genderScore?: number; + }>, + image: { naturalWidth: number; naturalHeight: number }, layerWidth?: number, layerHeight?: number, ): FaceDetectionOverlay[] { @@ -25,8 +37,11 @@ export function toFaceDetectionOverlays( return faces.map((face, index) => { const genderLabel = face.gender ?? "unknown"; - const scoreLabel = face.genderScore != null ? `${Math.round(face.genderScore * 100)}%` : ""; - const label = scoreLabel ? `Person ${index + 1} - ${genderLabel} ${scoreLabel}` : `Person ${index + 1} - ${genderLabel}`; + const scoreLabel = + face.genderScore != null ? `${Math.round(face.genderScore * 100)}%` : ""; + const label = scoreLabel + ? `Person ${index + 1} - ${genderLabel} ${scoreLabel}` + : `Person ${index + 1} - ${genderLabel}`; return { x: face.x * scaleX, @@ -71,7 +86,10 @@ export function buildFacePreviews( const ctx = canvas.getContext("2d"); if (!ctx) return { id: `face-${index + 1}`, src: "" }; ctx.drawImage(image, sx, sy, cw, ch, 0, 0, canvas.width, canvas.height); - return { id: `face-${index + 1}`, src: canvas.toDataURL("image/jpeg", 0.9) }; + return { + id: `face-${index + 1}`, + src: canvas.toDataURL("image/jpeg", 0.9), + }; }) .filter((preview) => preview.src); -} \ No newline at end of file +} diff --git a/apps/web/lib/face-detection-worker.ts b/apps/web/lib/face-detection-worker.ts new file mode 100644 index 0000000..a0ac3fe --- /dev/null +++ b/apps/web/lib/face-detection-worker.ts @@ -0,0 +1,260 @@ +import type { FaceBox, FaceDetectionResult } from "./face-ml"; + +type FaceDetectionWorkerRequest = { + id: number; + sourceUri: string; +}; + +type FaceDetectionWorkerResponse = { + id: number; + result?: FaceDetectionResult; + error?: string; +}; + +type TinyFaceOptions = { inputSize: number; scoreThreshold: number }; +type FaceApiDet = { + gender: string; + genderProbability: number; + detection: { box: { x: number; y: number; width: number; height: number } }; +}; +type FaceApiRuntime = { + tf: { + browser: { + fromPixels: (pixels: ImageData) => Tensor3DLike; + }; + }; + faceapi: { + detectAllFaces: ( + img: unknown, + options: TinyFaceOptions, + ) => { + withFaceLandmarks: (useTinyLandmarkNet: boolean) => { + withAgeAndGender: () => Promise; + }; + }; + }; + TinyFaceDetectorOptions: new (options: TinyFaceOptions) => TinyFaceOptions; +}; +type Tensor3DLike = { + dispose: () => void; +}; + +let faceApiModelsPromise: Promise | null = null; + +self.onmessage = async (event: MessageEvent) => { + const { id, sourceUri } = event.data; + let bitmap: ImageBitmap | null = null; + let tensor: Tensor3DLike | null = null; + + try { + const runtime = await loadFaceApiModels(); + bitmap = await loadImageBitmap(sourceUri); + const imageData = await imageBitmapToImageData(bitmap); + tensor = runtime.tf.browser.fromPixels(imageData); + const faces = await runDetectorPasses( + tensor, + runtime.faceapi, + runtime.TinyFaceDetectorOptions, + ); + const result = { + faces, + naturalWidth: bitmap.width, + naturalHeight: bitmap.height, + }; + + self.postMessage({ id, result } satisfies FaceDetectionWorkerResponse); + } catch (err) { + self.postMessage({ + id, + error: err instanceof Error ? err.message : String(err), + } satisfies FaceDetectionWorkerResponse); + } finally { + tensor?.dispose(); + bitmap?.close(); + } +}; + +async function loadFaceApiModels(): Promise { + if (faceApiModelsPromise) return faceApiModelsPromise; + + faceApiModelsPromise = (async () => { + const tf = await import("@tensorflow/tfjs"); + await tf.ready(); + + const faceapi = await import("@vladmandic/face-api"); + faceapi.env.setEnv({ + Canvas: OffscreenCanvas, + CanvasRenderingContext2D: OffscreenCanvasRenderingContext2D, + Image: class WorkerImage {}, + ImageData, + Video: class WorkerVideo {}, + createCanvasElement: () => new OffscreenCanvas(1, 1), + createImageElement: () => { + throw new Error( + "HTMLImageElement is unavailable in face detection worker", + ); + }, + createVideoElement: () => { + throw new Error( + "HTMLVideoElement is unavailable in face detection worker", + ); + }, + fetch, + readFile: () => { + throw new Error("readFile is unavailable in face detection worker"); + }, + } as unknown as Parameters[0]); + + await Promise.all([ + faceapi.nets.tinyFaceDetector.loadFromUri( + "https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights", + ), + faceapi.nets.faceLandmark68TinyNet.loadFromUri( + "https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights", + ), + faceapi.nets.ageGenderNet.loadFromUri( + "https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights", + ), + ]); + + const TinyFaceDetectorOptions = ( + faceapi as unknown as { + TinyFaceDetectorOptions: new ( + options: TinyFaceOptions, + ) => TinyFaceOptions; + } + ).TinyFaceDetectorOptions; + + return { + tf, + faceapi, + TinyFaceDetectorOptions, + } as unknown as FaceApiRuntime; + })(); + + return faceApiModelsPromise; +} + +async function loadImageBitmap(sourceUri: string): Promise { + if (typeof createImageBitmap === "undefined") { + throw new Error("createImageBitmap is unavailable in this browser worker"); + } + + const response = await fetch(sourceUri); + if (!response.ok) { + throw new Error( + `Failed to load image for face detection: ${response.status}`, + ); + } + + return createImageBitmap(await response.blob()); +} + +async function imageBitmapToImageData(bitmap: ImageBitmap): Promise { + if (typeof OffscreenCanvas === "undefined") { + throw new Error("OffscreenCanvas is unavailable in this browser worker"); + } + + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const ctx = canvas.getContext("2d", { willReadFrequently: true }); + if (!ctx) throw new Error("Could not prepare image for face detection"); + ctx.drawImage(bitmap, 0, 0); + return ctx.getImageData(0, 0, bitmap.width, bitmap.height); +} + +async function runDetectorPasses( + image: Tensor3DLike, + faceapi: FaceApiRuntime["faceapi"], + TinyFaceDetectorOptions: FaceApiRuntime["TinyFaceDetectorOptions"], +): Promise { + const passes = [ + { inputSize: 320, scoreThreshold: 0.5 }, + { inputSize: 416, scoreThreshold: 0.45 }, + { inputSize: 512, scoreThreshold: 0.5 }, + { inputSize: 608, scoreThreshold: 0.45 }, + { inputSize: 736, scoreThreshold: 0.4 }, + { inputSize: 864, scoreThreshold: 0.35 }, + ]; + + const allDets = await Promise.all( + passes.map(({ inputSize, scoreThreshold }) => + faceapi + .detectAllFaces( + image, + new TinyFaceDetectorOptions({ inputSize, scoreThreshold }), + ) + .withFaceLandmarks(true) + .withAgeAndGender(), + ), + ); + + const flat = allDets.flat(); + if (flat.length === 0) return []; + + function iou( + a: { x: number; y: number; width: number; height: number }, + b: { x: number; y: number; width: number; height: number }, + ) { + const ix = Math.max(a.x, b.x); + const iy = Math.max(a.y, b.y); + const ix2 = Math.min(a.x + a.width, b.x + b.width); + const iy2 = Math.min(a.y + a.height, b.y + b.height); + const inter = Math.max(0, ix2 - ix) * Math.max(0, iy2 - iy); + const union = a.width * a.height + b.width * b.height - inter; + return union > 0 ? inter / union : 0; + } + + function avgGender(dets: FaceApiDet[]): { + gender: "male" | "female" | undefined; + score: number; + } { + let maleScore = 0; + let femaleScore = 0; + let count = 0; + for (const det of dets) { + if (det.gender === "male") maleScore += det.genderProbability; + else if (det.gender === "female") femaleScore += det.genderProbability; + count++; + } + if (count === 0) return { gender: undefined, score: 0 }; + const avgMale = maleScore / count; + const avgFemale = femaleScore / count; + if (avgMale > avgFemale) return { gender: "male", score: avgMale }; + if (avgFemale > avgMale) return { gender: "female", score: avgFemale }; + return { gender: undefined, score: 0 }; + } + + const clusters: FaceApiDet[][] = []; + for (const det of flat) { + const b = det.detection?.box; + if (!b) continue; + let matched = false; + for (const cluster of clusters) { + if (cluster.some((c) => iou(c.detection.box, b) > 0.4)) { + cluster.push(det); + matched = true; + break; + } + } + if (!matched) clusters.push([det]); + } + + return clusters.map((group) => { + const largest = [...group].sort( + (a, b) => + (b.detection?.box?.width ?? 0) * (b.detection?.box?.height ?? 0) - + (a.detection?.box?.width ?? 0) * (a.detection?.box?.height ?? 0), + )[0]; + const box = largest.detection.box; + const { gender, score: genderScore } = avgGender(group); + + return { + x: box.x, + y: box.y, + width: box.width, + height: box.height, + gender, + genderScore, + }; + }); +} diff --git a/apps/web/lib/face-ml.ts b/apps/web/lib/face-ml.ts index 9cfb413..aa07a15 100644 --- a/apps/web/lib/face-ml.ts +++ b/apps/web/lib/face-ml.ts @@ -8,116 +8,100 @@ export type FaceBox = { score?: number; }; -export async function detectFaceBoxes(image: HTMLImageElement): Promise { - const tf = await import("@tensorflow/tfjs"); - await tf.ready(); +export type FaceDetectionResult = { + faces: FaceBox[]; + naturalWidth: number; + naturalHeight: number; +}; - const faceapi = await import("@vladmandic/face-api"); - await Promise.all([ - faceapi.nets.tinyFaceDetector.loadFromUri("https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights"), - faceapi.nets.faceLandmark68TinyNet.loadFromUri("https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights"), - faceapi.nets.ageGenderNet.loadFromUri("https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights"), - ]); +type FaceDetectionWorkerRequest = { + id: number; + sourceUri: string; +}; - type TinyFaceOptions = { inputSize: number; scoreThreshold: number }; - const TinyFaceDetectorOptions = ( - faceapi as unknown as { TinyFaceDetectorOptions: new (options: TinyFaceOptions) => TinyFaceOptions } - ).TinyFaceDetectorOptions; +type FaceDetectionWorkerResponse = { + id: number; + result?: FaceDetectionResult; + error?: string; +}; - const passes = [ - { inputSize: 320, scoreThreshold: 0.5 }, - { inputSize: 416, scoreThreshold: 0.45 }, - { inputSize: 512, scoreThreshold: 0.5 }, - { inputSize: 608, scoreThreshold: 0.45 }, - { inputSize: 736, scoreThreshold: 0.4 }, - { inputSize: 864, scoreThreshold: 0.35 }, - ]; +let faceDetectionWorker: Worker | null = null; +let faceDetectionWorkerDisabled = false; +let workerRequestId = 0; +const workerRequests = new Map< + number, + { + resolve: (result: FaceDetectionResult) => void; + reject: (error: Error) => void; + } +>(); - type FaceApiDet = { - gender: string; - genderProbability: number; - detection: { box: { x: number; y: number; width: number; height: number } }; - }; - - const allDets = await Promise.all( - passes.map(({ inputSize, scoreThreshold }) => - (faceapi as unknown as { - detectAllFaces: ( - img: HTMLImageElement, - options: TinyFaceOptions - ) => { - withFaceLandmarks: (useTinyLandmarkNet: boolean) => { - withAgeAndGender: () => Promise; - }; - }; - }) - .detectAllFaces(image, new TinyFaceDetectorOptions({ inputSize, scoreThreshold })) - .withFaceLandmarks(true) - .withAgeAndGender() - ) - ); - - const flat = allDets.flat(); - if (flat.length === 0) return []; - - function iou(a: { x: number; y: number; width: number; height: number }, b: { x: number; y: number; width: number; height: number }) { - const ix = Math.max(a.x, b.x); - const iy = Math.max(a.y, b.y); - const ix2 = Math.min(a.x + a.width, b.x + b.width); - const iy2 = Math.min(a.y + a.height, b.y + b.height); - const inter = Math.max(0, ix2 - ix) * Math.max(0, iy2 - iy); - const union = a.width * a.height + b.width * b.height - inter; - return union > 0 ? inter / union : 0; +export async function detectFaceBoxes( + sourceUri: string, +): Promise { + if (typeof Worker === "undefined" || faceDetectionWorkerDisabled) { + throw new Error("Face detection worker is unavailable"); } - function avgGender(dets: FaceApiDet[]): { gender: "male" | "female" | undefined; score: number } { - let maleScore = 0; - let femaleScore = 0; - let count = 0; - for (const det of dets) { - if (det.gender === "male") maleScore += det.genderProbability; - else if (det.gender === "female") femaleScore += det.genderProbability; - count++; - } - if (count === 0) return { gender: undefined, score: 0 }; - const avgMale = maleScore / count; - const avgFemale = femaleScore / count; - if (avgMale > avgFemale) return { gender: "male", score: avgMale }; - if (avgFemale > avgMale) return { gender: "female", score: avgFemale }; - return { gender: undefined, score: 0 }; + try { + return await detectFaceBoxesInWorker(sourceUri); + } catch (err) { + faceDetectionWorkerDisabled = true; + faceDetectionWorker?.terminate(); + faceDetectionWorker = null; + console.warn("[face-detection] worker failed:", err); + throw err; } +} - const clusters: FaceApiDet[][] = []; - for (const det of flat) { - const b = det.detection?.box; - if (!b) continue; - let matched = false; - for (const cluster of clusters) { - if (cluster.some((c) => iou(c.detection.box, b) > 0.4)) { - cluster.push(det); - matched = true; - break; - } - } - if (!matched) clusters.push([det]); - } +function detectFaceBoxesInWorker( + sourceUri: string, +): Promise { + const worker = getFaceDetectionWorker(); + const id = ++workerRequestId; - return clusters.map((group) => { - const largest = [...group].sort( - (a, b) => - (b.detection?.box?.width ?? 0) * (b.detection?.box?.height ?? 0) - - (a.detection?.box?.width ?? 0) * (a.detection?.box?.height ?? 0) - )[0]; - const box = largest.detection.box; - const { gender, score: genderScore } = avgGender(group); - - return { - x: box.x, - y: box.y, - width: box.width, - height: box.height, - gender, - genderScore, - }; + return new Promise((resolve, reject) => { + workerRequests.set(id, { resolve, reject }); + worker.postMessage({ id, sourceUri } satisfies FaceDetectionWorkerRequest); }); } + +function getFaceDetectionWorker(): Worker { + if (faceDetectionWorker) return faceDetectionWorker; + + faceDetectionWorker = new Worker( + new URL("./face-detection-worker.ts", import.meta.url), + { type: "module" }, + ); + faceDetectionWorker.onmessage = ( + event: MessageEvent, + ) => { + const request = workerRequests.get(event.data.id); + if (!request) return; + workerRequests.delete(event.data.id); + if (event.data.error) { + faceDetectionWorkerDisabled = true; + faceDetectionWorker?.terminate(); + faceDetectionWorker = null; + request.reject(new Error(event.data.error)); + return; + } + + if (!event.data.result) { + request.reject(new Error("Face detection worker returned no result")); + return; + } + + request.resolve(event.data.result); + }; + faceDetectionWorker.onerror = (event) => { + const error = new Error(event.message || "Face detection worker failed"); + faceDetectionWorkerDisabled = true; + workerRequests.forEach((request) => request.reject(error)); + workerRequests.clear(); + faceDetectionWorker?.terminate(); + faceDetectionWorker = null; + }; + + return faceDetectionWorker; +} diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 55064e6..88db35c 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -109,6 +109,7 @@ "ui": { "locale": "Locale", "darkMode": "Dark mode", - "lightMode": "Light mode" + "lightMode": "Light mode", + "systemMode": "System" } } diff --git a/apps/web/messages/ja.json b/apps/web/messages/ja.json index f9e52f1..0ba8fbd 100644 --- a/apps/web/messages/ja.json +++ b/apps/web/messages/ja.json @@ -109,6 +109,7 @@ "ui": { "locale": "言語", "darkMode": "ダークモード", - "lightMode": "ライトモード" + "lightMode": "ライトモード", + "systemMode": "システム" } } diff --git a/apps/web/messages/th.json b/apps/web/messages/th.json index e738eec..8c1f253 100644 --- a/apps/web/messages/th.json +++ b/apps/web/messages/th.json @@ -109,6 +109,7 @@ "ui": { "locale": "ภาษา", "darkMode": "โหมดมืด", - "lightMode": "โหมดสว่าง" + "lightMode": "โหมดสว่าง", + "systemMode": "ระบบ" } } diff --git a/apps/web/next-env.d.ts b/apps/web/next-env.d.ts index 9edff1c..c4b7818 100644 --- a/apps/web/next-env.d.ts +++ b/apps/web/next-env.d.ts @@ -1,6 +1,6 @@ /// /// -import "./.next/types/routes.d.ts"; +import "./.next/dev/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index 5ed4a3d..112a78d 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -4,4 +4,4 @@ const withNextIntl = createNextIntlPlugin(); export default withNextIntl({ reactStrictMode: true, -}); +}); \ No newline at end of file diff --git a/apps/web/store/editor-store.ts b/apps/web/store/editor-store.ts index b945fd8..1524cd4 100644 --- a/apps/web/store/editor-store.ts +++ b/apps/web/store/editor-store.ts @@ -284,14 +284,68 @@ export const useEditorStore = create((set, get) => ({ return withCommittedProject(state, nextProject, { clipboardLayer: cloneLayer(layer) }); }), - pasteLayer: () => - set((state) => { - if (!state.clipboardLayer) return state; + pasteLayer: () => { + const state = get(); + if (!state.clipboardLayer) { + navigator.clipboard.read().then(async (clipboardItems) => { + for (const item of clipboardItems) { + for (const type of item.types) { + if (type.startsWith("image/")) { + const blob = await item.getType(type); + const reader = new FileReader(); + const dataUrl = await new Promise((resolve, reject) => { + reader.onload = () => resolve(typeof reader.result === "string" ? reader.result : ""); + reader.onerror = reject; + reader.readAsDataURL(blob); + }); + const imageSize = await new Promise<{ width: number; height: number }>((resolve) => { + const image = new Image(); + image.onload = () => resolve({ width: image.naturalWidth, height: image.naturalHeight }); + image.onerror = () => + resolve({ width: DEFAULT_IMAGE_IMPORT.fallbackWidth, height: DEFAULT_IMAGE_IMPORT.fallbackHeight }); + image.src = dataUrl; + }); + const layer = createLayer("image", { + name: "Image", + sourceUri: dataUrl, + x: DEFAULT_IMAGE_IMPORT.offsetX, + y: DEFAULT_IMAGE_IMPORT.offsetY, + width: Math.max(1, Math.round(imageSize.width)), + height: Math.max(1, Math.round(imageSize.height)), + }); + set((s) => withCommittedProject(s, addLayer(s.project, layer), { selectedLayerId: layer.id })); + return; + } + } + } + navigator.clipboard.readText().then((text) => { + if (text) { + const trimmed = text.trim(); + if (trimmed.startsWith("data:image") || trimmed.startsWith("http") || trimmed.startsWith("blob:")) { + const image = new Image(); + image.onload = () => { + const layer = createLayer("image", { + name: "Image", + sourceUri: trimmed, + x: DEFAULT_IMAGE_IMPORT.offsetX, + y: DEFAULT_IMAGE_IMPORT.offsetY, + width: Math.max(1, Math.round(image.naturalWidth)), + height: Math.max(1, Math.round(image.naturalHeight)), + }); + set((s) => withCommittedProject(s, addLayer(s.project, layer), { selectedLayerId: layer.id })); + }; + image.src = trimmed; + } + } + }); + }).catch(() => {}); + return; + } const base = state.clipboardLayer; const pasted: Layer = { ...base, id: crypto.randomUUID(), x: base.x + 20, y: base.y + 20 }; const nextProject = addLayer(state.project, pasted); - return withCommittedProject(state, nextProject, { selectedLayerId: pasted.id }); - }), + set((s) => withCommittedProject(s, nextProject, { selectedLayerId: pasted.id })); + }, resetProject: () => { releaseProjectObjectUrls(get().project);