mirror of
https://github.com/YuzuZensai/Pien-Studio.git
synced 2026-09-02 14:18:35 +00:00
✨ feat: transparency, clipboard paste, face detection optimization
This commit is contained in:
+86
-102
@@ -8,116 +8,100 @@ export type FaceBox = {
|
||||
score?: number;
|
||||
};
|
||||
|
||||
export async function detectFaceBoxes(image: HTMLImageElement): Promise<FaceBox[]> {
|
||||
const tf = await import("@tensorflow/tfjs");
|
||||
await tf.ready();
|
||||
export type FaceDetectionResult = {
|
||||
faces: FaceBox[];
|
||||
naturalWidth: number;
|
||||
naturalHeight: number;
|
||||
};
|
||||
|
||||
const faceapi = await import("@vladmandic/face-api");
|
||||
await Promise.all([
|
||||
faceapi.nets.tinyFaceDetector.loadFromUri("https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights"),
|
||||
faceapi.nets.faceLandmark68TinyNet.loadFromUri("https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights"),
|
||||
faceapi.nets.ageGenderNet.loadFromUri("https://cdn.jsdelivr.net/gh/justadudewhohacks/face-api.js@master/weights"),
|
||||
]);
|
||||
type FaceDetectionWorkerRequest = {
|
||||
id: number;
|
||||
sourceUri: string;
|
||||
};
|
||||
|
||||
type TinyFaceOptions = { inputSize: number; scoreThreshold: number };
|
||||
const TinyFaceDetectorOptions = (
|
||||
faceapi as unknown as { TinyFaceDetectorOptions: new (options: TinyFaceOptions) => TinyFaceOptions }
|
||||
).TinyFaceDetectorOptions;
|
||||
type FaceDetectionWorkerResponse = {
|
||||
id: number;
|
||||
result?: FaceDetectionResult;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
const passes = [
|
||||
{ inputSize: 320, scoreThreshold: 0.5 },
|
||||
{ inputSize: 416, scoreThreshold: 0.45 },
|
||||
{ inputSize: 512, scoreThreshold: 0.5 },
|
||||
{ inputSize: 608, scoreThreshold: 0.45 },
|
||||
{ inputSize: 736, scoreThreshold: 0.4 },
|
||||
{ inputSize: 864, scoreThreshold: 0.35 },
|
||||
];
|
||||
let faceDetectionWorker: Worker | null = null;
|
||||
let faceDetectionWorkerDisabled = false;
|
||||
let workerRequestId = 0;
|
||||
const workerRequests = new Map<
|
||||
number,
|
||||
{
|
||||
resolve: (result: FaceDetectionResult) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
>();
|
||||
|
||||
type FaceApiDet = {
|
||||
gender: string;
|
||||
genderProbability: number;
|
||||
detection: { box: { x: number; y: number; width: number; height: number } };
|
||||
};
|
||||
|
||||
const allDets = await Promise.all(
|
||||
passes.map(({ inputSize, scoreThreshold }) =>
|
||||
(faceapi as unknown as {
|
||||
detectAllFaces: (
|
||||
img: HTMLImageElement,
|
||||
options: TinyFaceOptions
|
||||
) => {
|
||||
withFaceLandmarks: (useTinyLandmarkNet: boolean) => {
|
||||
withAgeAndGender: () => Promise<FaceApiDet[]>;
|
||||
};
|
||||
};
|
||||
})
|
||||
.detectAllFaces(image, new TinyFaceDetectorOptions({ inputSize, scoreThreshold }))
|
||||
.withFaceLandmarks(true)
|
||||
.withAgeAndGender()
|
||||
)
|
||||
);
|
||||
|
||||
const flat = allDets.flat();
|
||||
if (flat.length === 0) return [];
|
||||
|
||||
function iou(a: { x: number; y: number; width: number; height: number }, b: { x: number; y: number; width: number; height: number }) {
|
||||
const ix = Math.max(a.x, b.x);
|
||||
const iy = Math.max(a.y, b.y);
|
||||
const ix2 = Math.min(a.x + a.width, b.x + b.width);
|
||||
const iy2 = Math.min(a.y + a.height, b.y + b.height);
|
||||
const inter = Math.max(0, ix2 - ix) * Math.max(0, iy2 - iy);
|
||||
const union = a.width * a.height + b.width * b.height - inter;
|
||||
return union > 0 ? inter / union : 0;
|
||||
export async function detectFaceBoxes(
|
||||
sourceUri: string,
|
||||
): Promise<FaceDetectionResult> {
|
||||
if (typeof Worker === "undefined" || faceDetectionWorkerDisabled) {
|
||||
throw new Error("Face detection worker is unavailable");
|
||||
}
|
||||
|
||||
function avgGender(dets: FaceApiDet[]): { gender: "male" | "female" | undefined; score: number } {
|
||||
let maleScore = 0;
|
||||
let femaleScore = 0;
|
||||
let count = 0;
|
||||
for (const det of dets) {
|
||||
if (det.gender === "male") maleScore += det.genderProbability;
|
||||
else if (det.gender === "female") femaleScore += det.genderProbability;
|
||||
count++;
|
||||
}
|
||||
if (count === 0) return { gender: undefined, score: 0 };
|
||||
const avgMale = maleScore / count;
|
||||
const avgFemale = femaleScore / count;
|
||||
if (avgMale > avgFemale) return { gender: "male", score: avgMale };
|
||||
if (avgFemale > avgMale) return { gender: "female", score: avgFemale };
|
||||
return { gender: undefined, score: 0 };
|
||||
try {
|
||||
return await detectFaceBoxesInWorker(sourceUri);
|
||||
} catch (err) {
|
||||
faceDetectionWorkerDisabled = true;
|
||||
faceDetectionWorker?.terminate();
|
||||
faceDetectionWorker = null;
|
||||
console.warn("[face-detection] worker failed:", err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const clusters: FaceApiDet[][] = [];
|
||||
for (const det of flat) {
|
||||
const b = det.detection?.box;
|
||||
if (!b) continue;
|
||||
let matched = false;
|
||||
for (const cluster of clusters) {
|
||||
if (cluster.some((c) => iou(c.detection.box, b) > 0.4)) {
|
||||
cluster.push(det);
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!matched) clusters.push([det]);
|
||||
}
|
||||
function detectFaceBoxesInWorker(
|
||||
sourceUri: string,
|
||||
): Promise<FaceDetectionResult> {
|
||||
const worker = getFaceDetectionWorker();
|
||||
const id = ++workerRequestId;
|
||||
|
||||
return clusters.map((group) => {
|
||||
const largest = [...group].sort(
|
||||
(a, b) =>
|
||||
(b.detection?.box?.width ?? 0) * (b.detection?.box?.height ?? 0) -
|
||||
(a.detection?.box?.width ?? 0) * (a.detection?.box?.height ?? 0)
|
||||
)[0];
|
||||
const box = largest.detection.box;
|
||||
const { gender, score: genderScore } = avgGender(group);
|
||||
|
||||
return {
|
||||
x: box.x,
|
||||
y: box.y,
|
||||
width: box.width,
|
||||
height: box.height,
|
||||
gender,
|
||||
genderScore,
|
||||
};
|
||||
return new Promise<FaceDetectionResult>((resolve, reject) => {
|
||||
workerRequests.set(id, { resolve, reject });
|
||||
worker.postMessage({ id, sourceUri } satisfies FaceDetectionWorkerRequest);
|
||||
});
|
||||
}
|
||||
|
||||
function getFaceDetectionWorker(): Worker {
|
||||
if (faceDetectionWorker) return faceDetectionWorker;
|
||||
|
||||
faceDetectionWorker = new Worker(
|
||||
new URL("./face-detection-worker.ts", import.meta.url),
|
||||
{ type: "module" },
|
||||
);
|
||||
faceDetectionWorker.onmessage = (
|
||||
event: MessageEvent<FaceDetectionWorkerResponse>,
|
||||
) => {
|
||||
const request = workerRequests.get(event.data.id);
|
||||
if (!request) return;
|
||||
workerRequests.delete(event.data.id);
|
||||
if (event.data.error) {
|
||||
faceDetectionWorkerDisabled = true;
|
||||
faceDetectionWorker?.terminate();
|
||||
faceDetectionWorker = null;
|
||||
request.reject(new Error(event.data.error));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!event.data.result) {
|
||||
request.reject(new Error("Face detection worker returned no result"));
|
||||
return;
|
||||
}
|
||||
|
||||
request.resolve(event.data.result);
|
||||
};
|
||||
faceDetectionWorker.onerror = (event) => {
|
||||
const error = new Error(event.message || "Face detection worker failed");
|
||||
faceDetectionWorkerDisabled = true;
|
||||
workerRequests.forEach((request) => request.reject(error));
|
||||
workerRequests.clear();
|
||||
faceDetectionWorker?.terminate();
|
||||
faceDetectionWorker = null;
|
||||
};
|
||||
|
||||
return faceDetectionWorker;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user