YuzuZensai/Pien-Studio
1
0
Fork 0
mirror of https://github.com/YuzuZensai/Pien-Studio.git synced 2026-09-02 14:18:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

✨ feat: transparency, clipboard paste, face detection optimization

Browse Source
This commit is contained in:
yuzu
2026-05-16 11:53:57 +07:00
parent 80744f40ba
commit b13d70f2db
16 changed files with 1186 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/web/components/canvas-renderer.test.tsx
+63
View File
@@ -0,0 +1,63 @@
import React from "react";
import { fireEvent, render, screen } from "@testing-library/react";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { CanvasRenderer } from "./canvas-renderer";
import type { Layer } from "@pien-studio/types";
vi.mock("next/image", () => ({
default: ({ alt, src, unoptimized: _unoptimized, ...props }: React.ImgHTMLAttributes<HTMLImageElement> & { unoptimized?: boolean }) => React.createElement("img", { alt, src, ...props }),
}));
vi.mock("../hooks/use-translations", () => ({
useTranslations: () => ({ t: (key: string) => key }),
}));
class ResizeObserverMock {
observe() {}
disconnect() {}
}
describe("CanvasRenderer", () => {
beforeEach(() => {
vi.stubGlobal("ResizeObserver", ResizeObserverMock);
Object.defineProperty(HTMLElement.prototype, "setPointerCapture", { configurable: true, value: vi.fn() });
});
it("keeps the rotation handle interactive", () => {
const layer: Layer = {
id: "layer-1",
type: "image",
sourceUri: "data:image/png;base64,test",
x: 0,
y: 0,
width: 100,
height: 100,
scale: 1,
rotation: 0,
opacity: 1,
};
const onRotateLayer = vi.fn();
render(
<CanvasRenderer
layers={[layer]}
canvasWidth={200}
canvasHeight={200}
selectedLayerId={layer.id}
onSelectLayer={() => undefined}
onMoveLayer={() => undefined}
onResizeLayer={() => undefined}
onRotateLayer={onRotateLayer}
isDark={false}
/>,
);
const rotateHandle = screen.getByTitle("editor.rotate");
expect(rotateHandle).toHaveClass("pointer-events-auto");
fireEvent(rotateHandle, new MouseEvent("pointerdown", { bubbles: true, button: 0, clientX: 50, clientY: 0 }));
fireEvent(rotateHandle, new MouseEvent("pointermove", { bubbles: true, clientX: 100, clientY: 50 }));
expect(onRotateLayer).toHaveBeenCalledWith(layer.id, 90);
});
});
apps/web/components/canvas-renderer.tsx
+132 -30
View File
@@ -3,9 +3,12 @@
import React from "react";
import NextImage from "next/image";
import { RotateCw } from "lucide-react";
import { CANVAS_HANDLE_BASE_SIZE, CANVAS_ROTATE_HANDLE_BASE_SIZE } from "../lib/editor-constants";
import {
CANVAS_HANDLE_BASE_SIZE,
CANVAS_ROTATE_HANDLE_BASE_SIZE,
} from "../lib/editor-constants";
import { buildFaceLabelOverlays } from "../lib/canvas-geometry";
import { renderFaceBlurRegions } from "../lib/face-blur-renderer";
import { renderImageWithFaceBlur } from "../lib/face-blur-renderer";
import { useCanvasInteractions } from "../hooks/use-canvas-interactions";
import { useTranslations } from "../hooks/use-translations";
import type { FaceBlurMethod, Layer } from "@pien-studio/types";
@@ -27,7 +30,13 @@ interface CanvasRendererProps {
onContextMenu?: (x: number, y: number) => void;
isDark: boolean;
tool?: "pointer" | "hand" | "face";
faceDetections?: { x: number; y: number; width: number; height: number; label?: string }[];
faceDetections?: {
x: number;
y: number;
width: number;
height: number;
label?: string;
}[];
faceOverlayLayerId?: string | null;
faceBlurPreview?: {
layerId: string;
@@ -49,7 +58,13 @@ function BlurredImageLayer({
faceBlurOverride?: {
method: FaceBlurMethod;
amount: number;
regions: { x: number; y: number; width: number; height: number; censorColor?: string }[];
regions: {
x: number;
y: number;
width: number;
height: number;
censorColor?: string;
}[];
censorColor?: string;
} | null;
}) {
@@ -65,10 +80,8 @@ function BlurredImageLayer({
const cw = canvas.width;
const ch = canvas.height;
ctx.clearRect(0, 0, cw, ch);
ctx.drawImage(image, 0, 0, cw, ch);
const blur = faceBlurOverride ?? layer.faceBlur;
if (!blur || blur.regions.length === 0) return;
renderFaceBlurRegions(ctx, image, blur, cw, ch);
renderImageWithFaceBlur(ctx, image, blur, cw, ch);
}, [faceBlurOverride, layer.faceBlur]);
React.useEffect(() => {
@@ -91,7 +104,14 @@ function BlurredImageLayer({
draw();
}, [draw, width, height]);
return <canvas ref={canvasRef} width={Math.max(1, Math.round(width))} height={Math.max(1, Math.round(height))} className="pointer-events-none h-full w-full rounded object-cover" />;
return (
<canvas
ref={canvasRef}
width={Math.max(1, Math.round(width))}
height={Math.max(1, Math.round(height))}
className="pointer-events-none h-full w-full rounded object-cover"
/>
);
}
export function CanvasRenderer({
@@ -147,15 +167,35 @@ export function CanvasRenderer({
});
const faceLabelOverlays = React.useMemo(() => {
if (tool !== "face" || !faceOverlayLayerId || faceDetections.length === 0) return [];
return buildFaceLabelOverlays(layers, faceOverlayLayerId, faceDetections, viewport);
}, [faceDetections, faceOverlayLayerId, layers, tool, viewport.scale, viewport.x, viewport.y]);
if (tool !== "face" || !faceOverlayLayerId || faceDetections.length === 0)
return [];
return buildFaceLabelOverlays(
layers,
faceOverlayLayerId,
faceDetections,
viewport,
);
}, [
faceDetections,
faceOverlayLayerId,
layers,
tool,
viewport.scale,
viewport.x,
viewport.y,
]);
return (
<div
ref={containerRef}
className="relative overflow-hidden"
style={{ width: "100%", height: "100%", touchAction: "none", userSelect: "none", cursor: tool === "hand" || isSpacePan ? "grab" : "default" }}
style={{
width: "100%",
height: "100%",
touchAction: "none",
userSelect: "none",
cursor: tool === "hand" || isSpacePan ? "grab" : "default",
}}
onPointerDown={(e) => {
if (tool === "pointer" && e.button === 0) onSelectLayer(null);
onContainerPointerDown(e);
@@ -183,18 +223,26 @@ export function CanvasRenderer({
willChange: "transform",
transform: `translate(${viewport.x}px, ${viewport.y}px) scale(${viewport.scale})`,
transformOrigin: "0 0",
boxShadow: "0 0 0 1px var(--color-accent-strong)",
background: isDark ? "#17181b" : "#ffffff",
outline: `${2 / viewport.scale}px solid var(--color-accent-strong)`,
background:
"linear-gradient(45deg, #d0d0d0 25%, transparent 25%), linear-gradient(-45deg, #d0d0d0 25%, transparent 25%), linear-gradient(45deg, transparent 75%, #d0d0d0 75%), linear-gradient(-45deg, transparent 75%, #d0d0d0 75%)",
backgroundSize: "16px 16px",
backgroundPosition: "0 0, 0 8px, 8px -8px, -8px 0px",
}}
>
{layers.map((layer, idx) => {
const isSelected = layer.id === selectedLayerId;
const isImage = layer.type === "image";
const layerWidth = layer.width ?? (isImage ? Math.round(200 * layer.scale) : undefined);
const layerHeight = layer.height ?? (isImage ? Math.round(150 * layer.scale) : undefined);
const layerWidth =
layer.width ??
(isImage ? Math.round(200 * layer.scale) : undefined);
const layerHeight =
layer.height ??
(isImage ? Math.round(150 * layer.scale) : undefined);
const handleSize = CANVAS_HANDLE_BASE_SIZE / viewport.scale;
const handleSizePx = `${handleSize}px`;
const largeHandleSize = CANVAS_ROTATE_HANDLE_BASE_SIZE / viewport.scale;
const largeHandleSize =
CANVAS_ROTATE_HANDLE_BASE_SIZE / viewport.scale;
const largeHandleSizePx = `${largeHandleSize}px`;
return (
@@ -209,8 +257,12 @@ export function CanvasRenderer({
transform: `rotate(${layer.rotation}deg)`,
opacity: layer.opacity,
cursor: "move",
border: isSelected ? "2px solid var(--color-accent-strong)" : "1px dashed transparent",
outline: isSelected ? "2px solid var(--color-accent-strong)" : "none",
border: isSelected
? "2px solid var(--color-accent-strong)"
: "1px dashed transparent",
outline: isSelected
? "2px solid var(--color-accent-strong)"
: "none",
outlineOffset: "2px",
zIndex: idx,
}}
@@ -218,13 +270,19 @@ export function CanvasRenderer({
onClick={() => onSelectLayer(layer.id)}
>
{isImage && layer.sourceUri ? (
(faceBlurPreview && faceBlurPreview.layerId === layer.id && faceBlurPreview.regions.length > 0) ||
(faceBlurPreview &&
faceBlurPreview.layerId === layer.id &&
faceBlurPreview.regions.length > 0) ||
(layer.faceBlur && layer.faceBlur.regions.length > 0) ? (
<BlurredImageLayer
layer={layer}
width={layerWidth ?? 1}
height={layerHeight ?? 1}
faceBlurOverride={faceBlurPreview && faceBlurPreview.layerId === layer.id ? faceBlurPreview : null}
faceBlurOverride={
faceBlurPreview && faceBlurPreview.layerId === layer.id
? faceBlurPreview
: null
}
/>
) : (
<NextImage
@@ -271,41 +329,85 @@ export function CanvasRenderer({
<button
type="button"
className="absolute rounded-full border-2 border-white/80 bg-[var(--color-accent-strong)] shadow"
style={{ width: handleSizePx, height: handleSizePx, top: -handleSize / 2, left: -handleSize / 2 }}
style={{
width: handleSizePx,
height: handleSizePx,
top: -handleSize / 2,
left: -handleSize / 2,
}}
onPointerDown={(e) => onResizeHandleDown(e, layer, "tl")}
title={t("editor.resize")}
/>
<button
type="button"
className="absolute rounded-full border-2 border-white/80 bg-[var(--color-accent-strong)] shadow"
style={{ width: handleSizePx, height: handleSizePx, top: -handleSize / 2, right: -handleSize / 2 }}
style={{
width: handleSizePx,
height: handleSizePx,
top: -handleSize / 2,
right: -handleSize / 2,
}}
onPointerDown={(e) => onResizeHandleDown(e, layer, "tr")}
title={t("editor.resize")}
/>
<button
type="button"
className="absolute rounded-full border-2 border-white/80 bg-[var(--color-accent-strong)] shadow"
style={{ width: handleSizePx, height: handleSizePx, bottom: -handleSize / 2, left: -handleSize / 2 }}
style={{
width: handleSizePx,
height: handleSizePx,
bottom: -handleSize / 2,
left: -handleSize / 2,
}}
onPointerDown={(e) => onResizeHandleDown(e, layer, "bl")}
title={t("editor.resize")}
/>
<button
type="button"
className="absolute rounded-full border-2 border-white/80 bg-[var(--color-accent-strong)] shadow"
style={{ width: handleSizePx, height: handleSizePx, bottom: -handleSize / 2, right: -handleSize / 2 }}
style={{
width: handleSizePx,
height: handleSizePx,
bottom: -handleSize / 2,
right: -handleSize / 2,
}}
onPointerDown={(e) => onResizeHandleDown(e, layer, "br")}
title={t("editor.resize")}
/>
<div className="absolute pointer-events-none" style={{ top: -largeHandleSize, left: "50%", transform: "translateX(-50%)", height: largeHandleSize }}>
<div className="w-px bg-[var(--color-accent-strong)]" style={{ width: "1px", height: "100%", marginLeft: "0px" }} />
<div
className="absolute pointer-events-none"
style={{
top: -largeHandleSize,
left: "50%",
transform: "translateX(-50%)",
height: largeHandleSize,
}}
>
<div
className="w-px bg-[var(--color-accent-strong)]"
style={{
width: "1px",
height: "100%",
marginLeft: "0px",
}}
/>
<button
type="button"
className="absolute top-0 left-1/2 -translate-x-1/2 rounded-full border-2 border-white/90 bg-[var(--color-accent-strong)] shadow flex items-center justify-center"
style={{ width: largeHandleSizePx, height: largeHandleSizePx }}
className="pointer-events-auto absolute top-0 left-1/2 -translate-x-1/2 rounded-full border-2 border-white/90 bg-[var(--color-accent-strong)] shadow flex items-center justify-center"
style={{
width: largeHandleSizePx,
height: largeHandleSizePx,
}}
onPointerDown={(e) => onRotateHandleDown(e, layer)}
title={t("editor.rotate")}
>
<RotateCw className="text-white" style={{ width: handleSize * 0.6, height: handleSize * 0.6 }} />
<RotateCw
className="text-white"
style={{
width: handleSize * 0.6,
height: handleSize * 0.6,
}}
/>
</button>
</div>
</>
apps/web/components/editor/editor-header.tsx
+1 -1
View File
@@ -176,12 +176,12 @@ function FileMenu({
}) {
return (
<div className="space-y-1">
<button type="button" onClick={onImportImage} className={`w-full rounded px-2 py-1 text-left ${hoverSubtleClass(isDark)}`}>{labels.importImage}</button>
<button type="button" onClick={onSave} className={`w-full rounded px-2 py-1 text-left ${hoverSubtleClass(isDark)}`}>{labels.save}</button>
<div className={`my-1 h-px ${dividerClass(isDark)}`} />
<button type="button" onClick={onExportPng} className={`w-full rounded px-2 py-1 text-left ${hoverSubtleClass(isDark)}`}>{labels.exportPng}</button>
<button type="button" onClick={onExportProjectFile} className={`w-full rounded px-2 py-1 text-left ${hoverSubtleClass(isDark)}`}>{labels.exportProjectFile}</button>
<div className={`my-1 h-px ${dividerClass(isDark)}`} />
<button type="button" onClick={onImportImage} className={`w-full rounded px-2 py-1 text-left ${hoverSubtleClass(isDark)}`}>{labels.importImage}</button>
<button type="button" onClick={onOpenCanvasSize} className={`w-full rounded px-2 py-1 text-left ${hoverSubtleClass(isDark)}`}>{labels.canvasSize} ({canvasWidth} x {canvasHeight})</button>
</div>
);
apps/web/hooks/use-face-detection.ts
+60 -22
View File
@@ -1,6 +1,10 @@
import React from "react";
import type { EditorToolId } from "../store/editor-store";
import { buildFacePreviews, loadImageFromUri, toFaceDetectionOverlays } from "../lib/face-detection-utils";
import {
buildFacePreviews,
loadImageFromUri,
toFaceDetectionOverlays,
} from "../lib/face-detection-utils";
export type FaceDetectionOverlay = {
x: number;
@@ -34,21 +38,35 @@ type UseFaceDetectionOptions = {
};
export function useFaceDetection(options: UseFaceDetectionOptions) {
const { tool, selectedLayerId, selectedImageLayer, activeLayerStillSelected } = options;
const {
tool,
selectedLayerId,
selectedImageLayer,
activeLayerStillSelected,
} = options;
const selectedImageLayerId = selectedImageLayer?.id ?? null;
const selectedImageSourceUri = selectedImageLayer?.sourceUri ?? null;
const selectedImageWidth = selectedImageLayer?.width;
const selectedImageHeight = selectedImageLayer?.height;
const [faceDetections, setFaceDetections] = React.useState<FaceDetectionOverlay[]>([]);
const [faceDetectionsLayerId, setFaceDetectionsLayerId] = React.useState<string | null>(null);
const [faceStatus, setFaceStatus] = React.useState<"idle" | "detecting" | "unsupported">("idle");
const [faceDetections, setFaceDetections] = React.useState<
FaceDetectionOverlay[]
>([]);
const [faceDetectionsLayerId, setFaceDetectionsLayerId] = React.useState<
string | null
>(null);
const [faceStatus, setFaceStatus] = React.useState<
"idle" | "detecting" | "unsupported"
>("idle");
const [facePreviews, setFacePreviews] = React.useState<FacePreview[]>([]);
const resetFaceState = React.useCallback((status: "idle" | "detecting" | "unsupported" = "idle") => {
const resetFaceState = React.useCallback(
(status: "idle" | "detecting" | "unsupported" = "idle") => {
setFaceDetections((prev) => (prev.length === 0 ? prev : []));
setFaceDetectionsLayerId((prev) => (prev === null ? prev : null));
setFaceStatus((prev) => (prev === status ? prev : status));
}, []);
},
[],
);
React.useEffect(() => {
let canceled = false;
@@ -60,21 +78,18 @@ export function useFaceDetection(options: UseFaceDetectionOptions) {
return;
}
resetFaceState("detecting");
const image = await loadImageFromUri(selectedImageSourceUri);
if (!image) {
if (!canceled) {
resetFaceState("idle");
}
return;
}
try {
resetFaceState("detecting");
const { detectFaceBoxes } = await import("../lib/face-ml");
const faces = await detectFaceBoxes(image);
const result = await detectFaceBoxes(selectedImageSourceUri);
if (canceled) return;
if (!activeLayerStillSelected(layerId) || tool !== "face") return;
const overlays = toFaceDetectionOverlays(faces, image, selectedImageWidth, selectedImageHeight);
const overlays = toFaceDetectionOverlays(
result.faces,
result,
selectedImageWidth,
selectedImageHeight,
);
setFaceDetections(overlays);
setFaceDetectionsLayerId(layerId);
setFaceStatus((prev) => (prev === "idle" ? prev : "idle"));
@@ -90,7 +105,15 @@ export function useFaceDetection(options: UseFaceDetectionOptions) {
return () => {
canceled = true;
};
}, [activeLayerStillSelected, resetFaceState, selectedImageHeight, selectedImageLayerId, selectedImageSourceUri, selectedImageWidth, tool]);
}, [
activeLayerStillSelected,
resetFaceState,
selectedImageHeight,
selectedImageLayerId,
selectedImageSourceUri,
selectedImageWidth,
tool,
]);
React.useEffect(() => {
if (tool !== "face") {
@@ -108,7 +131,11 @@ export function useFaceDetection(options: UseFaceDetectionOptions) {
let canceled = false;
async function generateFacePreviews() {
if (tool !== "face" || !selectedImageSourceUri || faceDetections.length === 0) {
if (
tool !== "face" ||
!selectedImageSourceUri ||
faceDetections.length === 0
) {
setFacePreviews([]);
return;
}
@@ -119,7 +146,12 @@ export function useFaceDetection(options: UseFaceDetectionOptions) {
return;
}
const previews = buildFacePreviews(image, faceDetections, selectedImageWidth, selectedImageHeight);
const previews = buildFacePreviews(
image,
faceDetections,
selectedImageWidth,
selectedImageHeight,
);
if (!canceled) setFacePreviews(previews);
}
@@ -128,7 +160,13 @@ export function useFaceDetection(options: UseFaceDetectionOptions) {
return () => {
canceled = true;
};
}, [faceDetections, selectedImageHeight, selectedImageSourceUri, selectedImageWidth, tool]);
}, [
faceDetections,
selectedImageHeight,
selectedImageSourceUri,
selectedImageWidth,
tool,
]);
return { faceDetections, faceDetectionsLayerId, facePreviews, faceStatus };
}
apps/web/lib/export-png.ts
+28 -14
View File
@@ -1,5 +1,5 @@
import type { Layer, Project } from "@pien-studio/types";
import { renderFaceBlurRegions } from "./face-blur-renderer";
import { renderImageWithFaceBlur } from "./face-blur-renderer";
type ExportOptions = {
isDark: boolean;
@@ -20,7 +20,11 @@ function loadImage(src: string) {
});
}
function drawFallbackLayer(ctx: CanvasRenderingContext2D, layer: Layer, isDark: boolean) {
function drawFallbackLayer(
ctx: CanvasRenderingContext2D,
layer: Layer,
isDark: boolean,
) {
const text = layer.name ?? layer.type;
const width = Math.max(80, layer.width ?? 120);
const height = Math.max(34, layer.height ?? 40);
@@ -45,15 +49,24 @@ function drawFallbackLayer(ctx: CanvasRenderingContext2D, layer: Layer, isDark:
ctx.stroke();
ctx.fillStyle = isDark ? "#d7dae0" : "#1f2430";
ctx.font = "600 12px ui-sans-serif, system-ui, -apple-system, Segoe UI, sans-serif";
ctx.font =
"600 12px ui-sans-serif, system-ui, -apple-system, Segoe UI, sans-serif";
ctx.textAlign = "center";
ctx.textBaseline = "middle";
ctx.fillText(text, width / 2, height / 2);
}
async function drawLayer(ctx: CanvasRenderingContext2D, layer: Layer, isDark: boolean) {
const width = layer.width ?? (layer.type === "image" ? Math.round(200 * layer.scale) : 120);
const height = layer.height ?? (layer.type === "image" ? Math.round(150 * layer.scale) : 40);
async function drawLayer(
ctx: CanvasRenderingContext2D,
layer: Layer,
isDark: boolean,
) {
const width =
layer.width ??
(layer.type === "image" ? Math.round(200 * layer.scale) : 120);
const height =
layer.height ??
(layer.type === "image" ? Math.round(150 * layer.scale) : 40);
ctx.save();
ctx.globalAlpha = clampOpacity(layer.opacity);
@@ -64,10 +77,7 @@ async function drawLayer(ctx: CanvasRenderingContext2D, layer: Layer, isDark: bo
if ((layer.type === "image" || layer.type === "sticker") && layer.sourceUri) {
try {
const image = await loadImage(layer.sourceUri);
ctx.drawImage(image, 0, 0, width, height);
if (layer.faceBlur && layer.faceBlur.regions.length > 0) {
renderFaceBlurRegions(ctx, image, layer.faceBlur, width, height);
}
renderImageWithFaceBlur(ctx, image, layer.faceBlur, width, height);
} catch {
drawFallbackLayer(ctx, layer, isDark);
}
@@ -78,8 +88,14 @@ async function drawLayer(ctx: CanvasRenderingContext2D, layer: Layer, isDark: bo
ctx.restore();
}
export async function exportProjectAsPng(project: Project, options: ExportOptions) {
const pixelRatio = Math.max(1, Math.floor(options.pixelRatio ?? window.devicePixelRatio ?? 1));
export async function exportProjectAsPng(
project: Project,
options: ExportOptions,
) {
const pixelRatio = Math.max(
1,
Math.floor(options.pixelRatio ?? window.devicePixelRatio ?? 1),
);
const { width, height } = project.canvas;
const canvas = document.createElement("canvas");
canvas.width = width * pixelRatio;
@@ -89,8 +105,6 @@ export async function exportProjectAsPng(project: Project, options: ExportOption
if (!ctx) throw new Error("Cannot create export canvas context");
ctx.scale(pixelRatio, pixelRatio);
ctx.fillStyle = options.isDark ? "#17181b" : "#ffffff";
ctx.fillRect(0, 0, width, height);
for (const layer of project.layers) {
await drawLayer(ctx, layer, options.isDark);
apps/web/lib/face-blur-renderer.test.ts
+253 -11
View File
@@ -1,5 +1,8 @@
import { describe, expect, it, vi } from "vitest";
import { renderFaceBlurRegions } from "./face-blur-renderer";
import {
renderFaceBlurRegions,
renderImageWithFaceBlur,
} from "./face-blur-renderer";
function makeContext() {
return {
@@ -27,7 +30,16 @@ describe("renderFaceBlurRegions", () => {
{
method: "gaussian",
amount: 24,
regions: [{ x: 120, y: 80, width: 300, height: 200, sourceWidth: 1200, sourceHeight: 800 }],
regions: [
{
x: 120,
y: 80,
width: 300,
height: 200,
sourceWidth: 1200,
sourceHeight: 800,
},
],
},
600,
400,
@@ -35,7 +47,17 @@ describe("renderFaceBlurRegions", () => {
expect(ctx.save).toHaveBeenCalledOnce();
expect(ctx.filter).toBe("blur(24px)");
expect(ctx.drawImage).toHaveBeenCalledWith(image, 120, 80, 300, 200, 60, 40, 150, 100);
expect(ctx.drawImage).toHaveBeenCalledWith(
image,
120,
80,
300,
200,
60,
40,
150,
100,
);
expect(ctx.restore).toHaveBeenCalledOnce();
});
@@ -46,10 +68,19 @@ describe("renderFaceBlurRegions", () => {
const ctx = makeContext();
const image = makeImage();
const sampleDrawImage = vi.fn();
const sampleCtx = { imageSmoothingEnabled: true, drawImage: sampleDrawImage } as unknown as CanvasRenderingContext2D;
const sampleCanvas = { width: 0, height: 0, getContext: vi.fn(() => sampleCtx) } as unknown as HTMLCanvasElement;
const sampleCtx = {
imageSmoothingEnabled: true,
drawImage: sampleDrawImage,
} as unknown as CanvasRenderingContext2D;
const sampleCanvas = {
width: 0,
height: 0,
getContext: vi.fn(() => sampleCtx),
} as unknown as HTMLCanvasElement;
const nativeCreateElement = doc.createElement.bind(doc);
const createElement = vi.spyOn(doc, "createElement").mockImplementation((tagName: string) => {
const createElement = vi
.spyOn(doc, "createElement")
.mockImplementation((tagName: string) => {
if (tagName === "canvas") return sampleCanvas;
return nativeCreateElement(tagName);
});
@@ -60,7 +91,16 @@ describe("renderFaceBlurRegions", () => {
{
method: "pixelate",
amount: 10,
regions: [{ x: 200, y: 100, width: 160, height: 120, sourceWidth: 1200, sourceHeight: 800 }],
regions: [
{
x: 200,
y: 100,
width: 160,
height: 120,
sourceWidth: 1200,
sourceHeight: 800,
},
],
},
600,
400,
@@ -68,8 +108,28 @@ describe("renderFaceBlurRegions", () => {
expect(sampleCanvas.width).toBe(16);
expect(sampleCanvas.height).toBe(12);
expect(sampleDrawImage).toHaveBeenCalledWith(image, 200, 100, 160, 120, 0, 0, 16, 12);
expect(ctx.drawImage).toHaveBeenCalledWith(sampleCanvas, 0, 0, 16, 12, 100, 50, 80, 60);
expect(sampleDrawImage).toHaveBeenCalledWith(
image,
200,
100,
160,
120,
0,
0,
16,
12,
);
expect(ctx.drawImage).toHaveBeenCalledWith(
sampleCanvas,
0,
0,
16,
12,
100,
50,
80,
60,
);
createElement.mockRestore();
});
@@ -83,7 +143,17 @@ describe("renderFaceBlurRegions", () => {
method: "censor",
amount: 20,
censorColor: "#ff0000",
regions: [{ x: 20, y: 30, width: 40, height: 50, sourceWidth: 1200, sourceHeight: 800, censorColor: "#00ff00" }],
regions: [
{
x: 20,
y: 30,
width: 40,
height: 50,
sourceWidth: 1200,
sourceHeight: 800,
censorColor: "#00ff00",
},
],
},
600,
400,
@@ -108,6 +178,178 @@ describe("renderFaceBlurRegions", () => {
400,
);
expect(ctx.drawImage).toHaveBeenCalledWith(image, 400, 480, 1200, 800, 100, 120, 300, 200);
expect(ctx.drawImage).toHaveBeenCalledWith(
image,
400,
480,
1200,
800,
100,
120,
300,
200,
);
});
it("applies blur to source-sized image before drawing the resized layer", () => {
const doc = globalThis.document;
expect(doc).toBeDefined();
if (!doc) return;
const ctx = makeContext();
const image = makeImage();
const sourceDrawImage = vi.fn();
const sourceCtx = {
...makeContext(),
drawImage: sourceDrawImage,
} as unknown as CanvasRenderingContext2D;
const sourceCanvas = {
width: 0,
height: 0,
getContext: vi.fn(() => sourceCtx),
} as unknown as HTMLCanvasElement;
const nativeCreateElement = doc.createElement.bind(doc);
const createElement = vi
.spyOn(doc, "createElement")
.mockImplementation((tagName: string) => {
if (tagName === "canvas") return sourceCanvas;
return nativeCreateElement(tagName);
});
renderImageWithFaceBlur(
ctx,
image,
{
method: "gaussian",
amount: 24,
regions: [
{
x: 120,
y: 80,
width: 300,
height: 200,
sourceWidth: 1200,
sourceHeight: 800,
},
],
},
600,
400,
);
expect(sourceCanvas.width).toBe(1200);
expect(sourceCanvas.height).toBe(800);
expect(sourceDrawImage).toHaveBeenNthCalledWith(1, image, 0, 0, 1200, 800);
expect(sourceDrawImage).toHaveBeenNthCalledWith(
2,
image,
120,
80,
300,
200,
120,
80,
300,
200,
);
expect(ctx.drawImage).toHaveBeenCalledWith(
sourceCanvas,
0,
0,
1200,
800,
0,
0,
600,
400,
);
createElement.mockRestore();
});
it("uses a canvas-filter fallback for gaussian blur when filters are unavailable", () => {
const doc = globalThis.document;
expect(doc).toBeDefined();
if (!doc) return;
const ctx = makeContext();
delete (ctx as Partial<CanvasRenderingContext2D>).filter;
const image = makeImage();
const regionDrawImage = vi.fn();
const blurDrawImage = vi.fn();
const regionCtx = {
...makeContext(),
clearRect: vi.fn(),
drawImage: regionDrawImage,
} as unknown as CanvasRenderingContext2D;
const blurCtx = {
...makeContext(),
clearRect: vi.fn(),
drawImage: blurDrawImage,
} as unknown as CanvasRenderingContext2D;
const regionCanvas = {
width: 0,
height: 0,
getContext: vi.fn(() => regionCtx),
} as unknown as HTMLCanvasElement;
const blurCanvas = {
width: 0,
height: 0,
getContext: vi.fn(() => blurCtx),
} as unknown as HTMLCanvasElement;
const nativeCreateElement = doc.createElement.bind(doc);
const createElement = vi
.spyOn(doc, "createElement")
.mockImplementation((tagName: string) => {
if (tagName !== "canvas") return nativeCreateElement(tagName);
return createElement.mock.calls.length === 1
? regionCanvas
: blurCanvas;
});
renderFaceBlurRegions(
ctx,
image,
{
method: "gaussian",
amount: 24,
regions: [
{
x: 120,
y: 80,
width: 300,
height: 200,
sourceWidth: 1200,
sourceHeight: 800,
},
],
},
600,
400,
);
expect(ctx.save).not.toHaveBeenCalled();
expect(regionCanvas.width).toBe(150);
expect(regionCanvas.height).toBe(100);
expect(regionDrawImage).toHaveBeenCalledWith(
image,
120,
80,
300,
200,
0,
0,
150,
100,
);
expect(ctx.drawImage).toHaveBeenCalledWith(
regionCanvas,
0,
0,
150,
100,
60,
40,
150,
100,
);
createElement.mockRestore();
});
});
apps/web/lib/face-blur-renderer.ts
+200 -7
View File
@@ -21,16 +21,126 @@ function drawPixelatedRegion(
const sampleCtx = sampleCanvas.getContext("2d");
if (!sampleCtx) return;
sampleCtx.imageSmoothingEnabled = false;
sampleCtx.drawImage(source, sourceX, sourceY, sourceWidth, sourceHeight, 0, 0, sampleCanvas.width, sampleCanvas.height);
sampleCtx.drawImage(
source,
sourceX,
sourceY,
sourceWidth,
sourceHeight,
0,
0,
sampleCanvas.width,
sampleCanvas.height,
);
ctx.imageSmoothingEnabled = false;
ctx.drawImage(sampleCanvas, 0, 0, sampleCanvas.width, sampleCanvas.height, targetX, targetY, targetWidth, targetHeight);
ctx.drawImage(
sampleCanvas,
0,
0,
sampleCanvas.width,
sampleCanvas.height,
targetX,
targetY,
targetWidth,
targetHeight,
);
ctx.imageSmoothingEnabled = true;
}
function canUseCanvasFilter(ctx: CanvasRenderingContext2D) {
return "filter" in ctx && typeof ctx.filter === "string";
}
function drawBlurredRegionFallback(
ctx: CanvasRenderingContext2D,
source: CanvasImageSource,
sourceX: number,
sourceY: number,
sourceWidth: number,
sourceHeight: number,
targetX: number,
targetY: number,
targetWidth: number,
targetHeight: number,
amount: number,
) {
const regionCanvas = document.createElement("canvas");
regionCanvas.width = Math.max(1, Math.round(targetWidth));
regionCanvas.height = Math.max(1, Math.round(targetHeight));
const regionCtx = regionCanvas.getContext("2d");
if (!regionCtx) return;
regionCtx.drawImage(
source,
sourceX,
sourceY,
sourceWidth,
sourceHeight,
0,
0,
regionCanvas.width,
regionCanvas.height,
);
const scale = Math.max(0.04, Math.min(0.5, 1 / Math.max(2, amount / 2)));
const blurCanvas = document.createElement("canvas");
blurCanvas.width = Math.max(1, Math.round(regionCanvas.width * scale));
blurCanvas.height = Math.max(1, Math.round(regionCanvas.height * scale));
const blurCtx = blurCanvas.getContext("2d");
if (!blurCtx) return;
blurCtx.imageSmoothingEnabled = true;
blurCtx.drawImage(regionCanvas, 0, 0, blurCanvas.width, blurCanvas.height);
regionCtx.imageSmoothingEnabled = true;
for (let i = 0; i < 3; i++) {
regionCtx.clearRect(0, 0, regionCanvas.width, regionCanvas.height);
regionCtx.drawImage(
blurCanvas,
0,
0,
blurCanvas.width,
blurCanvas.height,
0,
0,
regionCanvas.width,
regionCanvas.height,
);
blurCtx.clearRect(0, 0, blurCanvas.width, blurCanvas.height);
blurCtx.drawImage(
regionCanvas,
0,
0,
regionCanvas.width,
regionCanvas.height,
0,
0,
blurCanvas.width,
blurCanvas.height,
);
}
ctx.drawImage(
regionCanvas,
0,
0,
regionCanvas.width,
regionCanvas.height,
targetX,
targetY,
targetWidth,
targetHeight,
);
}
export function renderFaceBlurRegions(
ctx: CanvasRenderingContext2D,
image: HTMLImageElement,
blur: { method: FaceBlurSettings["method"]; amount: number; regions: BlurRegion[]; censorColor?: string },
blur: {
method: FaceBlurSettings["method"];
amount: number;
regions: BlurRegion[];
censorColor?: string;
},
targetWidth: number,
targetHeight: number,
): void {
@@ -49,10 +159,18 @@ export function renderFaceBlurRegions(
const y = Math.max(0, Math.floor(region.y * scaleY));
const w = Math.max(1, Math.floor(region.width * scaleX));
const h = Math.max(1, Math.floor(region.height * scaleY));
const sx0 = hasSourceDims ? region.x : Math.max(0, Math.floor(region.x * legacyScaleX));
const sy0 = hasSourceDims ? region.y : Math.max(0, Math.floor(region.y * legacyScaleY));
const sw = hasSourceDims ? region.width : Math.max(1, Math.floor(region.width * legacyScaleX));
const sh = hasSourceDims ? region.height : Math.max(1, Math.floor(region.height * legacyScaleY));
const sx0 = hasSourceDims
? region.x
: Math.max(0, Math.floor(region.x * legacyScaleX));
const sy0 = hasSourceDims
? region.y
: Math.max(0, Math.floor(region.y * legacyScaleY));
const sw = hasSourceDims
? region.width
: Math.max(1, Math.floor(region.width * legacyScaleX));
const sh = hasSourceDims
? region.height
: Math.max(1, Math.floor(region.height * legacyScaleY));
if (blur.method === "censor") {
ctx.fillStyle = region.censorColor ?? blur.censorColor ?? "#111111";
@@ -66,9 +184,84 @@ export function renderFaceBlurRegions(
continue;
}
if (canUseCanvasFilter(ctx)) {
ctx.save();
ctx.filter = `blur(${blur.amount}px)`;
ctx.drawImage(image, sx0, sy0, sw, sh, x, y, w, h);
ctx.restore();
continue;
}
drawBlurredRegionFallback(
ctx,
image,
sx0,
sy0,
sw,
sh,
x,
y,
w,
h,
blur.amount,
);
}
}
export function renderImageWithFaceBlur(
ctx: CanvasRenderingContext2D,
image: HTMLImageElement,
blur:
| {
method: FaceBlurSettings["method"];
amount: number;
regions: BlurRegion[];
censorColor?: string;
}
| undefined,
targetWidth: number,
targetHeight: number,
): void {
if (!blur || blur.regions.length === 0) {
ctx.drawImage(image, 0, 0, targetWidth, targetHeight);
return;
}
const canBlurAtSourceSize = blur.regions.every(
(region) => (region.sourceWidth ?? 0) > 0 && (region.sourceHeight ?? 0) > 0,
);
if (!canBlurAtSourceSize) {
ctx.drawImage(image, 0, 0, targetWidth, targetHeight);
renderFaceBlurRegions(ctx, image, blur, targetWidth, targetHeight);
return;
}
const sourceCanvas = document.createElement("canvas");
sourceCanvas.width = Math.max(1, image.naturalWidth);
sourceCanvas.height = Math.max(1, image.naturalHeight);
const sourceCtx = sourceCanvas.getContext("2d");
if (!sourceCtx) {
ctx.drawImage(image, 0, 0, targetWidth, targetHeight);
return;
}
sourceCtx.drawImage(image, 0, 0, sourceCanvas.width, sourceCanvas.height);
renderFaceBlurRegions(
sourceCtx,
image,
blur,
sourceCanvas.width,
sourceCanvas.height,
);
ctx.drawImage(
sourceCanvas,
0,
0,
sourceCanvas.width,
sourceCanvas.height,
0,
0,
targetWidth,
targetHeight,
);
}
apps/web/lib/face-detection-utils.ts
+25 -7
View File
@@ -1,6 +1,11 @@
import type { FaceDetectionOverlay, FacePreview } from "../hooks/use-face-detection";
import type {
FaceDetectionOverlay,
FacePreview,
} from "../hooks/use-face-detection";
export async function loadImageFromUri(uri: string): Promise<HTMLImageElement | null> {
export async function loadImageFromUri(
uri: string,
): Promise<HTMLImageElement | null> {
const image = new Image();
image.crossOrigin = "anonymous";
await new Promise<void>((resolve) => {
@@ -13,8 +18,15 @@ export async function loadImageFromUri(uri: string): Promise<HTMLImageElement |
}
export function toFaceDetectionOverlays(
faces: Array<{ x: number; y: number; width: number; height: number; gender?: string; genderScore?: number }>,
image: HTMLImageElement,
faces: Array<{
x: number;
y: number;
width: number;
height: number;
gender?: string;
genderScore?: number;
}>,
image: { naturalWidth: number; naturalHeight: number },
layerWidth?: number,
layerHeight?: number,
): FaceDetectionOverlay[] {
@@ -25,8 +37,11 @@ export function toFaceDetectionOverlays(
return faces.map((face, index) => {
const genderLabel = face.gender ?? "unknown";
const scoreLabel = face.genderScore != null ? `${Math.round(face.genderScore * 100)}%` : "";
const label = scoreLabel ? `Person ${index + 1} - ${genderLabel} ${scoreLabel}` : `Person ${index + 1} - ${genderLabel}`;
const scoreLabel =
face.genderScore != null ? `${Math.round(face.genderScore * 100)}%` : "";
const label = scoreLabel
? `Person ${index + 1} - ${genderLabel} ${scoreLabel}`
: `Person ${index + 1} - ${genderLabel}`;
return {
x: face.x * scaleX,
@@ -71,7 +86,10 @@ export function buildFacePreviews(
const ctx = canvas.getContext("2d");
if (!ctx) return { id: `face-${index + 1}`, src: "" };
ctx.drawImage(image, sx, sy, cw, ch, 0, 0, canvas.width, canvas.height);
return { id: `face-${index + 1}`, src: canvas.toDataURL("image/jpeg", 0.9) };
return {
id: `face-${index + 1}`,
src: canvas.toDataURL("image/jpeg", 0.9),
};
})
.filter((preview) => preview.src);
}
apps/web/lib/face-detection-worker.ts
+260
View File
@@ -0,0 +1,260 @@
import type { FaceBox, FaceDetectionResult } from "./face-ml";
type FaceDetectionWorkerRequest = {
id: number;
sourceUri: string;
};
type FaceDetectionWorkerResponse = {
id: number;
result?: FaceDetectionResult;
error?: string;
};
type TinyFaceOptions = { inputSize: number; scoreThreshold: number };
type FaceApiDet = {
gender: string;
genderProbability: number;
detection: { box: { x: number; y: number; width: number; height: number } };
};
type FaceApiRuntime = {
tf: {
browser: {
fromPixels: (pixels: ImageData) => Tensor3DLike;
};
};
faceapi: {
detectAllFaces: (
img: unknown,
options: TinyFaceOptions,
) => {
withFaceLandmarks: (useTinyLandmarkNet: boolean) => {
withAgeAndGender: () => Promise<FaceApiDet[]>;
};
};
};
TinyFaceDetectorOptions: new (options: TinyFaceOptions) => TinyFaceOptions;
};
type Tensor3DLike = {
dispose: () => void;
};
let faceApiModelsPromise: Promise<FaceApiRuntime> | null = null;
self.onmessage = async (event: MessageEvent<FaceDetectionWorkerRequest>) => {
const { id, sourceUri } = event.data;
let bitmap: ImageBitmap | null = null;
let tensor: Tensor3DLike | null = null;
try {
const runtime = await loadFaceApiModels();
bitmap = await loadImageBitmap(sourceUri);
const imageData = await imageBitmapToImageData(bitmap);
tensor = runtime.tf.browser.fromPixels(imageData);
const faces = await runDetectorPasses(
tensor,
runtime.faceapi,
runtime.TinyFaceDetectorOptions,
);
const result = {
faces,
naturalWidth: bitmap.width,
naturalHeight: bitmap.height,
};
self.postMessage({ id, result } satisfies FaceDetectionWorkerResponse);
} catch (err) {
self.postMessage({
id,
error: err instanceof Error ? err.message : String(err),
} satisfies FaceDetectionWorkerResponse);
} finally {
tensor?.dispose();
bitmap?.close();
}
};
async function loadFaceApiModels(): Promise<FaceApiRuntime> {
if (faceApiModelsPromise) return faceApiModelsPromise;
faceApiModelsPromise = (async () => {
const tf = await import("@tensorflow/tfjs");
await tf.ready();
const faceapi = await import("@vladmandic/face-api");
faceapi.env.setEnv({
Canvas: OffscreenCanvas,
CanvasRenderingContext2D: OffscreenCanvasRenderingContext2D,
Image: class WorkerImage {},
ImageData,
Video: class WorkerVideo {},
createCanvasElement: () => new OffscreenCanvas(1, 1),
createImageElement: () => {
throw new Error(
"HTMLImageElement is unavailable in face detection worker",
);
},
createVideoElement: () => {
throw new Error(
"HTMLVideoElement is unavailable in face detection worker",
);
},
fetch,
readFile: () => {
throw new Error("readFile is unavailable in face detection worker");
},
} as unknown as Parameters<typeof faceapi.env.setEnv>[0]);
await Promise.all([
faceapi.nets.tinyFaceDetector.loadFromUri(
"https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights",
),
faceapi.nets.faceLandmark68TinyNet.loadFromUri(
"https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights",
),
faceapi.nets.ageGenderNet.loadFromUri(
"https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights",
),
]);
const TinyFaceDetectorOptions = (
faceapi as unknown as {
TinyFaceDetectorOptions: new (
options: TinyFaceOptions,
) => TinyFaceOptions;
}
).TinyFaceDetectorOptions;
return {
tf,
faceapi,
TinyFaceDetectorOptions,
} as unknown as FaceApiRuntime;
})();
return faceApiModelsPromise;
}
async function loadImageBitmap(sourceUri: string): Promise<ImageBitmap> {
if (typeof createImageBitmap === "undefined") {
throw new Error("createImageBitmap is unavailable in this browser worker");
}
const response = await fetch(sourceUri);
if (!response.ok) {
throw new Error(
`Failed to load image for face detection: ${response.status}`,
);
}
return createImageBitmap(await response.blob());
}
async function imageBitmapToImageData(bitmap: ImageBitmap): Promise<ImageData> {
if (typeof OffscreenCanvas === "undefined") {
throw new Error("OffscreenCanvas is unavailable in this browser worker");
}
const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
const ctx = canvas.getContext("2d", { willReadFrequently: true });
if (!ctx) throw new Error("Could not prepare image for face detection");
ctx.drawImage(bitmap, 0, 0);
return ctx.getImageData(0, 0, bitmap.width, bitmap.height);
}
async function runDetectorPasses(
image: Tensor3DLike,
faceapi: FaceApiRuntime["faceapi"],
TinyFaceDetectorOptions: FaceApiRuntime["TinyFaceDetectorOptions"],
): Promise<FaceBox[]> {
const passes = [
{ inputSize: 320, scoreThreshold: 0.5 },
{ inputSize: 416, scoreThreshold: 0.45 },
{ inputSize: 512, scoreThreshold: 0.5 },
{ inputSize: 608, scoreThreshold: 0.45 },
{ inputSize: 736, scoreThreshold: 0.4 },
{ inputSize: 864, scoreThreshold: 0.35 },
];
const allDets = await Promise.all(
passes.map(({ inputSize, scoreThreshold }) =>
faceapi
.detectAllFaces(
image,
new TinyFaceDetectorOptions({ inputSize, scoreThreshold }),
)
.withFaceLandmarks(true)
.withAgeAndGender(),
),
);
const flat = allDets.flat();
if (flat.length === 0) return [];
function iou(
a: { x: number; y: number; width: number; height: number },
b: { x: number; y: number; width: number; height: number },
) {
const ix = Math.max(a.x, b.x);
const iy = Math.max(a.y, b.y);
const ix2 = Math.min(a.x + a.width, b.x + b.width);
const iy2 = Math.min(a.y + a.height, b.y + b.height);
const inter = Math.max(0, ix2 - ix) * Math.max(0, iy2 - iy);
const union = a.width * a.height + b.width * b.height - inter;
return union > 0 ? inter / union : 0;
}
function avgGender(dets: FaceApiDet[]): {
gender: "male" | "female" | undefined;
score: number;
} {
let maleScore = 0;
let femaleScore = 0;
let count = 0;
for (const det of dets) {
if (det.gender === "male") maleScore += det.genderProbability;
else if (det.gender === "female") femaleScore += det.genderProbability;
count++;
}
if (count === 0) return { gender: undefined, score: 0 };
const avgMale = maleScore / count;
const avgFemale = femaleScore / count;
if (avgMale > avgFemale) return { gender: "male", score: avgMale };
if (avgFemale > avgMale) return { gender: "female", score: avgFemale };
return { gender: undefined, score: 0 };
}
const clusters: FaceApiDet[][] = [];
for (const det of flat) {
const b = det.detection?.box;
if (!b) continue;
let matched = false;
for (const cluster of clusters) {
if (cluster.some((c) => iou(c.detection.box, b) > 0.4)) {
cluster.push(det);
matched = true;
break;
}
}
if (!matched) clusters.push([det]);
}
return clusters.map((group) => {
const largest = [...group].sort(
(a, b) =>
(b.detection?.box?.width ?? 0) * (b.detection?.box?.height ?? 0) -
(a.detection?.box?.width ?? 0) * (a.detection?.box?.height ?? 0),
)[0];
const box = largest.detection.box;
const { gender, score: genderScore } = avgGender(group);
return {
x: box.x,
y: box.y,
width: box.width,
height: box.height,
gender,
genderScore,
};
});
}
apps/web/lib/face-ml.ts
+83 -99
View File
@@ -8,116 +8,100 @@ export type FaceBox = {
score?: number;
};
export async function detectFaceBoxes(image: HTMLImageElement): Promise<FaceBox[]> {
const tf = await import("@tensorflow/tfjs");
await tf.ready();
const faceapi = await import("@vladmandic/face-api");
await Promise.all([
faceapi.nets.tinyFaceDetector.loadFromUri("https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights"),
faceapi.nets.faceLandmark68TinyNet.loadFromUri("https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights"),
faceapi.nets.ageGenderNet.loadFromUri("https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights"),
]);
type TinyFaceOptions = { inputSize: number; scoreThreshold: number };
const TinyFaceDetectorOptions = (
faceapi as unknown as { TinyFaceDetectorOptions: new (options: TinyFaceOptions) => TinyFaceOptions }
).TinyFaceDetectorOptions;
const passes = [
{ inputSize: 320, scoreThreshold: 0.5 },
{ inputSize: 416, scoreThreshold: 0.45 },
{ inputSize: 512, scoreThreshold: 0.5 },
{ inputSize: 608, scoreThreshold: 0.45 },
{ inputSize: 736, scoreThreshold: 0.4 },
{ inputSize: 864, scoreThreshold: 0.35 },
];
type FaceApiDet = {
gender: string;
genderProbability: number;
detection: { box: { x: number; y: number; width: number; height: number } };
export type FaceDetectionResult = {
faces: FaceBox[];
naturalWidth: number;
naturalHeight: number;
};
const allDets = await Promise.all(
passes.map(({ inputSize, scoreThreshold }) =>
(faceapi as unknown as {
detectAllFaces: (
img: HTMLImageElement,
options: TinyFaceOptions
) => {
withFaceLandmarks: (useTinyLandmarkNet: boolean) => {
withAgeAndGender: () => Promise<FaceApiDet[]>;
type FaceDetectionWorkerRequest = {
id: number;
sourceUri: string;
};
type FaceDetectionWorkerResponse = {
id: number;
result?: FaceDetectionResult;
error?: string;
};
})
.detectAllFaces(image, new TinyFaceDetectorOptions({ inputSize, scoreThreshold }))
.withFaceLandmarks(true)
.withAgeAndGender()
)
);
const flat = allDets.flat();
if (flat.length === 0) return [];
let faceDetectionWorker: Worker | null = null;
let faceDetectionWorkerDisabled = false;
let workerRequestId = 0;
const workerRequests = new Map<
number,
{
resolve: (result: FaceDetectionResult) => void;
reject: (error: Error) => void;
}
>();
function iou(a: { x: number; y: number; width: number; height: number }, b: { x: number; y: number; width: number; height: number }) {
const ix = Math.max(a.x, b.x);
const iy = Math.max(a.y, b.y);
const ix2 = Math.min(a.x + a.width, b.x + b.width);
const iy2 = Math.min(a.y + a.height, b.y + b.height);
const inter = Math.max(0, ix2 - ix) * Math.max(0, iy2 - iy);
const union = a.width * a.height + b.width * b.height - inter;
return union > 0 ? inter / union : 0;
export async function detectFaceBoxes(
sourceUri: string,
): Promise<FaceDetectionResult> {
if (typeof Worker === "undefined" || faceDetectionWorkerDisabled) {
throw new Error("Face detection worker is unavailable");
}
function avgGender(dets: FaceApiDet[]): { gender: "male" | "female" | undefined; score: number } {
let maleScore = 0;
let femaleScore = 0;
let count = 0;
for (const det of dets) {
if (det.gender === "male") maleScore += det.genderProbability;
else if (det.gender === "female") femaleScore += det.genderProbability;
count++;
try {
return await detectFaceBoxesInWorker(sourceUri);
} catch (err) {
faceDetectionWorkerDisabled = true;
faceDetectionWorker?.terminate();
faceDetectionWorker = null;
console.warn("[face-detection] worker failed:", err);
throw err;
}
if (count === 0) return { gender: undefined, score: 0 };
const avgMale = maleScore / count;
const avgFemale = femaleScore / count;
if (avgMale > avgFemale) return { gender: "male", score: avgMale };
if (avgFemale > avgMale) return { gender: "female", score: avgFemale };
return { gender: undefined, score: 0 };
}
const clusters: FaceApiDet[][] = [];
for (const det of flat) {
const b = det.detection?.box;
if (!b) continue;
let matched = false;
for (const cluster of clusters) {
if (cluster.some((c) => iou(c.detection.box, b) > 0.4)) {
cluster.push(det);
matched = true;
break;
}
}
if (!matched) clusters.push([det]);
}
function detectFaceBoxesInWorker(
sourceUri: string,
): Promise<FaceDetectionResult> {
const worker = getFaceDetectionWorker();
const id = ++workerRequestId;
return clusters.map((group) => {
const largest = [...group].sort(
(a, b) =>
(b.detection?.box?.width ?? 0) * (b.detection?.box?.height ?? 0) -
(a.detection?.box?.width ?? 0) * (a.detection?.box?.height ?? 0)
)[0];
const box = largest.detection.box;
const { gender, score: genderScore } = avgGender(group);
return {
x: box.x,
y: box.y,
width: box.width,
height: box.height,
gender,
genderScore,
};
return new Promise<FaceDetectionResult>((resolve, reject) => {
workerRequests.set(id, { resolve, reject });
worker.postMessage({ id, sourceUri } satisfies FaceDetectionWorkerRequest);
});
}
function getFaceDetectionWorker(): Worker {
if (faceDetectionWorker) return faceDetectionWorker;
faceDetectionWorker = new Worker(
new URL("./face-detection-worker.ts", import.meta.url),
{ type: "module" },
);
faceDetectionWorker.onmessage = (
event: MessageEvent<FaceDetectionWorkerResponse>,
) => {
const request = workerRequests.get(event.data.id);
if (!request) return;
workerRequests.delete(event.data.id);
if (event.data.error) {
faceDetectionWorkerDisabled = true;
faceDetectionWorker?.terminate();
faceDetectionWorker = null;
request.reject(new Error(event.data.error));
return;
}
if (!event.data.result) {
request.reject(new Error("Face detection worker returned no result"));
return;
}
request.resolve(event.data.result);
};
faceDetectionWorker.onerror = (event) => {
const error = new Error(event.message || "Face detection worker failed");
faceDetectionWorkerDisabled = true;
workerRequests.forEach((request) => request.reject(error));
workerRequests.clear();
faceDetectionWorker?.terminate();
faceDetectionWorker = null;
};
return faceDetectionWorker;
}
apps/web/messages/en.json
+2 -1
View File
@@ -109,6 +109,7 @@
"ui": {
"locale": "Locale",
"darkMode": "Dark mode",
"lightMode": "Light mode"
"lightMode": "Light mode",
"systemMode": "System"
}
}
apps/web/messages/ja.json
+2 -1
View File
@@ -109,6 +109,7 @@
"ui": {
"locale": "言語",
"darkMode": "ダークモード",
"lightMode": "ライトモード"
"lightMode": "ライトモード",
"systemMode": "システム"
}
}
apps/web/messages/th.json
+2 -1
View File
@@ -109,6 +109,7 @@
"ui": {
"locale": "ภาษา",
"darkMode": "โหมดมืด",
"lightMode": "โหมดสว่าง"
"lightMode": "โหมดสว่าง",
"systemMode": "ระบบ"
}
}
apps/web/next-env.d.ts
Vendored
+1 -1
View File
@@ -1,6 +1,6 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />
import "./.next/types/routes.d.ts";
import "./.next/dev/types/routes.d.ts";
// NOTE: This file should not be edited
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
apps/web/store/editor-store.ts
+59 -5
View File
@@ -284,14 +284,68 @@ export const useEditorStore = create<EditorState>((set, get) => ({
return withCommittedProject(state, nextProject, { clipboardLayer: cloneLayer(layer) });
}),
pasteLayer: () =>
set((state) => {
if (!state.clipboardLayer) return state;
pasteLayer: () => {
const state = get();
if (!state.clipboardLayer) {
navigator.clipboard.read().then(async (clipboardItems) => {
for (const item of clipboardItems) {
for (const type of item.types) {
if (type.startsWith("image/")) {
const blob = await item.getType(type);
const reader = new FileReader();
const dataUrl = await new Promise<string>((resolve, reject) => {
reader.onload = () => resolve(typeof reader.result === "string" ? reader.result : "");
reader.onerror = reject;
reader.readAsDataURL(blob);
});
const imageSize = await new Promise<{ width: number; height: number }>((resolve) => {
const image = new Image();
image.onload = () => resolve({ width: image.naturalWidth, height: image.naturalHeight });
image.onerror = () =>
resolve({ width: DEFAULT_IMAGE_IMPORT.fallbackWidth, height: DEFAULT_IMAGE_IMPORT.fallbackHeight });
image.src = dataUrl;
});
const layer = createLayer("image", {
name: "Image",
sourceUri: dataUrl,
x: DEFAULT_IMAGE_IMPORT.offsetX,
y: DEFAULT_IMAGE_IMPORT.offsetY,
width: Math.max(1, Math.round(imageSize.width)),
height: Math.max(1, Math.round(imageSize.height)),
});
set((s) => withCommittedProject(s, addLayer(s.project, layer), { selectedLayerId: layer.id }));
return;
}
}
}
navigator.clipboard.readText().then((text) => {
if (text) {
const trimmed = text.trim();
if (trimmed.startsWith("data:image") || trimmed.startsWith("http") || trimmed.startsWith("blob:")) {
const image = new Image();
image.onload = () => {
const layer = createLayer("image", {
name: "Image",
sourceUri: trimmed,
x: DEFAULT_IMAGE_IMPORT.offsetX,
y: DEFAULT_IMAGE_IMPORT.offsetY,
width: Math.max(1, Math.round(image.naturalWidth)),
height: Math.max(1, Math.round(image.naturalHeight)),
});
set((s) => withCommittedProject(s, addLayer(s.project, layer), { selectedLayerId: layer.id }));
};
image.src = trimmed;
}
}
});
}).catch(() => {});
return;
}
const base = state.clipboardLayer;
const pasted: Layer = { ...base, id: crypto.randomUUID(), x: base.x + 20, y: base.y + 20 };
const nextProject = addLayer(state.project, pasted);
return withCommittedProject(state, nextProject, { selectedLayerId: pasted.id });
}),
set((s) => withCommittedProject(s, nextProject, { selectedLayerId: pasted.id }));
},
resetProject: () => {
releaseProjectObjectUrls(get().project);