From 8b34e589bc0db541bff7dbd2b3b1345c1013a206 Mon Sep 17 00:00:00 2001 From: Yuzu Date: Wed, 4 Feb 2026 12:37:40 +0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat:=20video/gif=20partial=20suppo?= =?UTF-8?q?rt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/App.tsx | 24 +- src/components/StatusApp.tsx | 27 ++- src/lib/ble/ble-client.ts | 4 +- src/lib/core/beambox-uploader.ts | 122 +++++++++- src/lib/processing/frame-extractor.ts | 194 +++++++++++++++ src/lib/processing/index.ts | 5 + src/lib/processing/media-detector.test.ts | 144 ++++++++++++ src/lib/processing/media-detector.ts | 192 +++++++++++++++ .../protocol/builders/payload-builder.test.ts | 102 +++++++- src/lib/protocol/builders/payload-builder.ts | 49 ++-- src/lib/protocol/builders/xv4-header.test.ts | 124 ++++++++++ src/lib/protocol/builders/xv4-header.ts | 221 ++++++++++++++++++ src/lib/protocol/index.ts | 1 + 13 files changed, 1150 insertions(+), 59 deletions(-) create mode 100644 src/lib/processing/frame-extractor.ts create mode 100644 src/lib/processing/media-detector.test.ts create mode 100644 src/lib/processing/media-detector.ts create mode 100644 src/lib/protocol/builders/xv4-header.test.ts create mode 100644 src/lib/protocol/builders/xv4-header.ts diff --git a/src/components/App.tsx b/src/components/App.tsx index 2fd33a2..625ca72 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -1,11 +1,7 @@ -import React from "react"; -import { Box, Text } from "ink"; +import React, { useEffect } from "react"; +import { Box, Text, useApp } from "ink"; import Spinner from "ink-spinner"; -import { - Header, - UploadProgress, - ConnectionStatus, -} from "./index.ts"; +import { Header, UploadProgress, ConnectionStatus } from "./index.ts"; import { useUpload } from "../hooks/useUpload.ts"; import type { UploadOptions } from "../cli/types.ts"; @@ -15,6 +11,7 @@ export interface AppProps { } export const App: React.FC = ({ options, verbose }) => { + const { exit } = useApp(); const { status, message, @@ -25,6 +22,19 @@ export const App: React.FC = ({ options, verbose }) => { connectionSteps, } = useUpload(options, verbose); + // Exit the app when done + useEffect(() => { + if (status === "success" || status === "error") { + // Give time for the final render, then exit + const timer = setTimeout(() => { + exit(); + // Force exit since noble keeps handles open + process.exit(status === "error" ? 1 : 0); + }, 100); + return () => clearTimeout(timer); + } + }, [status, exit]); + return (
diff --git a/src/components/StatusApp.tsx b/src/components/StatusApp.tsx index 34528be..0d83172 100644 --- a/src/components/StatusApp.tsx +++ b/src/components/StatusApp.tsx @@ -1,5 +1,5 @@ -import React from "react"; -import { Box, Text } from "ink"; +import React, { useEffect } from "react"; +import { Box, Text, useApp } from "ink"; import Spinner from "ink-spinner"; import { Header, Status, ConnectionStatus } from "./index.ts"; import { useDeviceStatus } from "../hooks/useDeviceStatus.ts"; @@ -11,13 +11,22 @@ export interface StatusAppProps { } export const StatusApp: React.FC = ({ options }) => { - const { - loading, - error, - deviceStatus, - notifications, - connectionSteps, - } = useDeviceStatus(options); + const { exit } = useApp(); + const { loading, error, deviceStatus, notifications, connectionSteps } = + useDeviceStatus(options); + + // Exit the app when done + useEffect(() => { + if (!loading) { + // Give time for the final render, then exit + const timer = setTimeout(() => { + exit(); + // Force exit since noble keeps handles open + process.exit(error ? 1 : 0); + }, 100); + return () => clearTimeout(timer); + } + }, [loading, error, exit]); return ( diff --git a/src/lib/ble/ble-client.ts b/src/lib/ble/ble-client.ts index ed5ffe3..8cb4a5b 100644 --- a/src/lib/ble/ble-client.ts +++ b/src/lib/ble/ble-client.ts @@ -595,11 +595,13 @@ export class BleUploader { /** * Send image data packets to device with batched acknowledgment waiting * @param fullData Complete image data payload + * @param packetType Packet type for header * @param onProgress Progress callback with (progress, status) * @returns True if successful */ public async sendData( fullData: Buffer, + packetType: PacketType, onProgress?: (progress: number, status?: string) => void, ): Promise { if (!this.writeCharacteristic) { @@ -638,7 +640,7 @@ export class BleUploader { chunk, totalChunks, remainingPackets, - this.protocolConfig.cmdSubtype, + packetType, ); logger.info( diff --git a/src/lib/core/beambox-uploader.ts b/src/lib/core/beambox-uploader.ts index f1ac8c1..9a85d8b 100644 --- a/src/lib/core/beambox-uploader.ts +++ b/src/lib/core/beambox-uploader.ts @@ -7,9 +7,12 @@ import { DEFAULT_BLE_CONFIG, DEFAULT_PROTOCOL_CONFIG, DEFAULT_IMAGE_CONFIG, + PacketType, } from "../protocol/index.ts"; import { BleUploader } from "../ble/ble-client.ts"; import { ImageProcessor } from "../processing/image-processor.ts"; +import { MediaDetector } from "../processing/media-detector.ts"; +import { FrameExtractor } from "../processing/frame-extractor.ts"; import { PayloadBuilder } from "../protocol/index.ts"; import { logger } from "../utils/logger.ts"; import { UploadError } from "../utils/errors.ts"; @@ -18,7 +21,7 @@ export interface UploadOptions { imagePath?: string; imageData?: Buffer; targetSize?: [number, number]; - onProgress?: (progress: number) => void; + onProgress?: (progress: number, status?: string) => void; } /** @@ -68,7 +71,12 @@ export class BeamBoxUploader { } /** - * Upload an image to the device + * Upload an image, GIF, or video to the device + * + * Automatically detects file type and uses appropriate upload method: + * - Static images: Type 6 (IMAGE) - single frame + * - Animated GIFs: Type 5 (DYNAMIC_AMBIENCE), frames extracted + * - Videos: Type 5 (DYNAMIC_AMBIENCE), frames extracted * * @param options Upload options * @returns True if upload successful @@ -82,7 +90,25 @@ export class BeamBoxUploader { const effectiveSize = targetSize ?? this.imageConfig.defaultSize; - // Prepare JPEG data + // Detect file type if path provided + let isAnimated = false; + if (imagePath) { + const mediaInfo = await MediaDetector.detectFromFile(imagePath); + isAnimated = mediaInfo.type === "gif" || mediaInfo.type === "video"; + + logger.info( + `Detected file type: ${mediaInfo.type} (${mediaInfo.mimeType})`, + ); + + if (isAnimated) { + logger.info("File is animated, using Type 5 (DYNAMIC_AMBIENCE)"); + return await this.uploadAnimation(imagePath, effectiveSize, onProgress); + } else { + logger.info("File is static image, using Type 6 (IMAGE)"); + } + } + + // Handle static image upload (Type 6) let jpegData: Buffer; if (imageData) { jpegData = imageData; @@ -100,7 +126,10 @@ export class BeamBoxUploader { await this.sleep(1000); // Step 1: Send image info packet to announce upload - const imageInfoPayload = this.payloadBuilder.buildImageInfo(); + const imageInfoPayload = this.payloadBuilder.buildImageInfo( + PacketType.IMAGE, + 1, + ); await this.ble.sendImageInfo(imageInfoPayload); logger.info("Sent image info packet, proceeding to data transfer"); @@ -108,6 +137,7 @@ export class BeamBoxUploader { const fullData = this.payloadBuilder.buildImageData( jpegData, effectiveSize, + PacketType.IMAGE, ); const prefixLen = fullData.length - jpegData.length; logger.info( @@ -115,15 +145,15 @@ export class BeamBoxUploader { ); // Send data in chunks with protocol packets - const ok = await this.ble.sendData(fullData, onProgress); + const ok = await this.ble.sendData(fullData, PacketType.IMAGE, onProgress); if (!ok) { logger.error("Upload reported error"); return false; } - // Wait for final response - if (!(await this.ble.waitForResponse(5.0))) { + // Wait for all acknowledgments + if (!(await this.ble.waitForResponse(onProgress))) { logger.error("Upload timeout waiting for response"); return false; } @@ -131,6 +161,80 @@ export class BeamBoxUploader { return true; } + /** + * Upload an animated GIF or video as Type 5 (DYNAMIC_AMBIENCE) + * + * @param filePath Path to the GIF or video file + * @param targetSize Target size for frames + * @param onProgress Progress callback + * @returns True if upload successful + */ + private async uploadAnimation( + filePath: string, + targetSize: [number, number], + onProgress?: (progress: number) => void, + ): Promise { + // Extract frames from the file + logger.info("Extracting frames from animation..."); + const frames = await FrameExtractor.extractFrames(filePath, { + maxFrames: 100, + targetSize, + }); + + logger.info(`Extracted ${frames.length} frames`); + + // Calculate frame interval + const intervalMs = await FrameExtractor.calculateFrameInterval( + filePath, + frames.length, + ); + logger.info(`Using frame interval: ${intervalMs}ms`); + + // Wait for device to be ready + logger.info("Waiting for device to be fully ready..."); + await this.sleep(1000); + + // Step 1: Send image info packet (Type 6 for the info) + const imageInfoPayload = this.payloadBuilder.buildImageInfo( + PacketType.IMAGE, + 1, + ); + await this.ble.sendImageInfo(imageInfoPayload); + logger.info("Sent animation info packet, proceeding to data transfer"); + + // Step 2: Build and send animation data payload (Type 5) + const fullData = this.payloadBuilder.buildAnimationData( + frames, + intervalMs, + targetSize, + ); + + logger.info( + `Animation payload bytes: total=${fullData.length}, frames=${frames.length}`, + ); + + // Send data in chunks with protocol packets using DYNAMIC_AMBIENCE packet type + const ok = await this.ble.sendData( + fullData, + PacketType.DYNAMIC_AMBIENCE, + onProgress, + ); + + if (!ok) { + logger.error("Animation upload reported error"); + return false; + } + + // Wait for all acknowledgments + if (!(await this.ble.waitForResponse(onProgress))) { + logger.error("Animation upload timeout waiting for response"); + return false; + } + + logger.info("Animation upload completed successfully"); + return true; + } + /** * Upload an image from a file * @param imagePath Path to the image file @@ -141,7 +245,7 @@ export class BeamBoxUploader { public async uploadImageFromFile( imagePath: string, targetSize?: [number, number], - onProgress?: (progress: number) => void, + onProgress?: (progress: number, status?: string) => void, ): Promise { return await this.upload({ imagePath, targetSize, onProgress }); } @@ -156,7 +260,7 @@ export class BeamBoxUploader { public async uploadCheckerboard( targetSize?: [number, number], squares?: number, - onProgress?: (progress: number) => void, + onProgress?: (progress: number, status?: string) => void, ): Promise { const effectiveSize = targetSize ?? this.imageConfig.defaultSize; const effectiveSquares = squares ?? this.imageConfig.checkerboardSquares; diff --git a/src/lib/processing/frame-extractor.ts b/src/lib/processing/frame-extractor.ts new file mode 100644 index 0000000..fb00274 --- /dev/null +++ b/src/lib/processing/frame-extractor.ts @@ -0,0 +1,194 @@ +import { exec } from "child_process"; +import { promisify } from "util"; +import { mkdtemp, readdir, readFile, rm } from "fs/promises"; +import { tmpdir } from "os"; +import { join } from "path"; +import type { XV4Frame } from "../protocol/index.ts"; +import { ImageProcessingError } from "../utils/errors.ts"; +import { logger } from "../utils/logger.ts"; + +const execAsync = promisify(exec); + +export interface FrameExtractionOptions { + /** Maximum number of frames to extract (default: 100) */ + maxFrames?: number; + /** Target FPS for extraction (default: extract all frames) */ + fps?: number; + /** Target size for frames [width, height] */ + targetSize?: [number, number]; +} + +/** + * Extract frames from animated GIFs and videos using ffmpeg + */ +export class FrameExtractor { + /** + * Extract frames from a GIF or video file + * @param filePath Path to the GIF or video file + * @param options Extraction options + * @returns Array of XV4 frames ready for upload + */ + static async extractFrames( + filePath: string, + options: FrameExtractionOptions = {}, + ): Promise { + const { + maxFrames = 100, + fps = null, + targetSize = [368, 368], + } = options; + + // Create temporary directory for frames + const tempDir = await mkdtemp(join(tmpdir(), "beambox-frames-")); + + try { + logger.info(`Extracting frames from ${filePath} to ${tempDir}`); + + // Build ffmpeg command + const outputPattern = join(tempDir, "frame_%05d.jpg"); + let ffmpegCmd = `ffmpeg -i "${filePath}"`; + + // Build filter chain + let filters: string[] = []; + + // Add FPS filter if specified + if (fps !== null) { + filters.push(`fps=${fps}`); + } + + // Add scaling and padding + filters.push(`scale=${targetSize[0]}:${targetSize[1]}:force_original_aspect_ratio=decrease`); + filters.push(`pad=${targetSize[0]}:${targetSize[1]}:(ow-iw)/2:(oh-ih)/2`); + + // Combine all filters + if (filters.length > 0) { + ffmpegCmd += ` -vf "${filters.join(',')}"`; + } + + // Add frame limit + ffmpegCmd += ` -vframes ${maxFrames}`; + + // Add quality settings + ffmpegCmd += ` -q:v 2`; // High quality JPEG + + ffmpegCmd += ` "${outputPattern}"`; + + logger.info(`Running: ${ffmpegCmd}`); + + // Execute ffmpeg + const { stdout, stderr } = await execAsync(ffmpegCmd); + + if (stderr && !stderr.includes("frame=")) { + logger.warn(`ffmpeg stderr: ${stderr}`); + } + + // Read extracted frames + const files = await readdir(tempDir); + const frameFiles = files + .filter((f) => f.startsWith("frame_") && f.endsWith(".jpg")) + .sort(); + + if (frameFiles.length === 0) { + throw new ImageProcessingError( + "No frames were extracted. Ensure the file is a valid GIF or video and ffmpeg is installed.", + ); + } + + logger.info(`Extracted ${frameFiles.length} frames`); + + // Load frames into XV4Frame format + const frames: XV4Frame[] = []; + for (const file of frameFiles) { + const framePath = join(tempDir, file); + const data = await readFile(framePath); + + // Extract frame number from filename (e.g., "frame_00001.jpg" -> "frame_00001") + const name = file.replace(".jpg", ""); + + frames.push({ + name, + data, + }); + } + + return frames; + } catch (error) { + if (error instanceof Error && error.message.includes("ENOENT")) { + throw new ImageProcessingError( + "ffmpeg not found. Please install ffmpeg to extract frames from GIFs and videos.\n" + + "Install: sudo apt-get install ffmpeg (Linux) or brew install ffmpeg (macOS)", + ); + } + throw new ImageProcessingError(`Failed to extract frames: ${error}`); + } finally { + // Clean up temporary directory + try { + await rm(tempDir, { recursive: true, force: true }); + } catch (error) { + logger.warn(`Failed to clean up temp directory: ${error}`); + } + } + } + + /** + * Get frame rate of a video file + * @param filePath Path to the video file + * @returns Frame rate in fps + */ + static async getFrameRate(filePath: string): Promise { + try { + const cmd = `ffprobe -v quiet -select_streams v:0 -show_entries stream=r_frame_rate -of csv=p=0 "${filePath}"`; + const { stdout } = await execAsync(cmd); + + // Parse fraction (e.g., "30/1" or "30000/1001") + const parts = stdout.trim().split("/"); + if (parts.length === 2) { + const num = parseInt(parts[0]!); + const den = parseInt(parts[1]!); + return num / den; + } + + return 30; // Default fallback + } catch (error) { + logger.warn(`Failed to get frame rate: ${error}`); + return 30; // Default fallback + } + } + + /** + * Get duration of a video file in seconds + * @param filePath Path to the video file + * @returns Duration in seconds + */ + static async getDuration(filePath: string): Promise { + try { + const cmd = `ffprobe -v quiet -show_entries format=duration -of csv=p=0 "${filePath}"`; + const { stdout } = await execAsync(cmd); + return parseFloat(stdout.trim()); + } catch (error) { + logger.warn(`Failed to get duration: ${error}`); + return 0; + } + } + + /** + * Calculate recommended frame interval in milliseconds based on extracted frames and original duration + * @param filePath Path to the source file + * @param extractedFrameCount Number of frames that were extracted + * @returns Recommended interval in milliseconds + */ + static async calculateFrameInterval( + filePath: string, + extractedFrameCount: number, + ): Promise { + const duration = await this.getDuration(filePath); + + if (duration > 0 && extractedFrameCount > 1) { + // Calculate interval to maintain original playback speed + const intervalMs = (duration * 1000) / extractedFrameCount; + return Math.max(20, Math.round(intervalMs)); // Minimum 20ms (50fps) + } + + return 50; // Default 50ms (20fps) + } +} diff --git a/src/lib/processing/index.ts b/src/lib/processing/index.ts index def3f55..f0f0adb 100644 --- a/src/lib/processing/index.ts +++ b/src/lib/processing/index.ts @@ -1 +1,6 @@ export { ImageProcessor } from "./image-processor.ts"; +export { MediaDetector, type MediaInfo } from "./media-detector.ts"; +export { + FrameExtractor, + type FrameExtractionOptions, +} from "./frame-extractor.ts"; diff --git a/src/lib/processing/media-detector.test.ts b/src/lib/processing/media-detector.test.ts new file mode 100644 index 0000000..282977e --- /dev/null +++ b/src/lib/processing/media-detector.test.ts @@ -0,0 +1,144 @@ +import { describe, it, expect } from "bun:test"; +import { MediaDetector } from "./media-detector.ts"; + +describe("MediaDetector", () => { + describe("detectFromBuffer", () => { + it("should detect JPEG from magic bytes", () => { + const buffer = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]); + const info = MediaDetector.detectFromBuffer(buffer); + + expect(info.type).toBe("image"); + expect(info.mimeType).toBe("image/jpeg"); + expect(info.extension).toBe("jpg"); + }); + + it("should detect PNG from magic bytes", () => { + const buffer = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, + ]); + const info = MediaDetector.detectFromBuffer(buffer); + + expect(info.type).toBe("image"); + expect(info.mimeType).toBe("image/png"); + expect(info.extension).toBe("png"); + }); + + it("should detect GIF from magic bytes", () => { + const buffer = Buffer.from([ + 0x47, 0x49, 0x46, 0x38, 0x39, 0x61, // GIF89a + ]); + const info = MediaDetector.detectFromBuffer(buffer); + + expect(info.type).toBe("gif"); + expect(info.mimeType).toBe("image/gif"); + expect(info.extension).toBe("gif"); + }); + + it("should detect MP4 from ftyp box", () => { + const buffer = Buffer.from([ + 0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70, // ftyp + 0x69, 0x73, 0x6f, 0x6d, // isom + ]); + const info = MediaDetector.detectFromBuffer(buffer); + + expect(info.type).toBe("video"); + expect(info.mimeType).toBe("video/mp4"); + expect(info.extension).toBe("mp4"); + }); + + it("should detect WebM from magic bytes", () => { + const buffer = Buffer.from([0x1a, 0x45, 0xdf, 0xa3]); + const info = MediaDetector.detectFromBuffer(buffer); + + expect(info.type).toBe("video"); + expect(info.mimeType).toBe("video/webm"); + expect(info.extension).toBe("webm"); + }); + + it("should detect AVI from RIFF header", () => { + const buffer = Buffer.from([ + 0x52, 0x49, 0x46, 0x46, // RIFF + 0x00, 0x00, 0x00, 0x00, + 0x41, 0x56, 0x49, 0x20, // AVI + ]); + const info = MediaDetector.detectFromBuffer(buffer); + + expect(info.type).toBe("video"); + expect(info.mimeType).toBe("video/x-msvideo"); + expect(info.extension).toBe("avi"); + }); + + it("should detect WebP from magic bytes", () => { + const buffer = Buffer.from([ + 0x52, 0x49, 0x46, 0x46, // RIFF + 0x00, 0x00, 0x00, 0x00, + 0x57, 0x45, 0x42, 0x50, // WEBP + ]); + const info = MediaDetector.detectFromBuffer(buffer); + + expect(info.type).toBe("image"); + expect(info.mimeType).toBe("image/webp"); + expect(info.extension).toBe("webp"); + }); + + it("should detect BMP from magic bytes", () => { + const buffer = Buffer.from([0x42, 0x4d]); + const info = MediaDetector.detectFromBuffer(buffer); + + expect(info.type).toBe("image"); + expect(info.mimeType).toBe("image/bmp"); + expect(info.extension).toBe("bmp"); + }); + + it("should use extension hint for MOV files", () => { + const buffer = Buffer.from([0x00, 0x00, 0x00, 0x00]); + const info = MediaDetector.detectFromBuffer(buffer, "mov"); + + expect(info.type).toBe("video"); + expect(info.mimeType).toBe("video/quicktime"); + expect(info.extension).toBe("mov"); + }); + + it("should default to image for unknown types", () => { + const buffer = Buffer.from([0x00, 0x00, 0x00, 0x00]); + const info = MediaDetector.detectFromBuffer(buffer); + + expect(info.type).toBe("image"); + expect(info.mimeType).toBe("application/octet-stream"); + }); + }); + + describe("isAnimatedBuffer", () => { + it("should return true for GIF", () => { + const buffer = Buffer.from([0x47, 0x49, 0x46, 0x38]); + expect(MediaDetector.isAnimatedBuffer(buffer)).toBe(true); + }); + + it("should return true for MP4", () => { + const buffer = Buffer.from([ + 0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70, + ]); + expect(MediaDetector.isAnimatedBuffer(buffer)).toBe(true); + }); + + it("should return true for WebM", () => { + const buffer = Buffer.from([0x1a, 0x45, 0xdf, 0xa3]); + expect(MediaDetector.isAnimatedBuffer(buffer)).toBe(true); + }); + + it("should return false for JPEG", () => { + const buffer = Buffer.from([0xff, 0xd8, 0xff, 0xe0]); + expect(MediaDetector.isAnimatedBuffer(buffer)).toBe(false); + }); + + it("should return false for PNG", () => { + const buffer = Buffer.from([0x89, 0x50, 0x4e, 0x47]); + expect(MediaDetector.isAnimatedBuffer(buffer)).toBe(false); + }); + + it("should return true for MOV with extension hint", () => { + const buffer = Buffer.from([0x00, 0x00, 0x00, 0x00]); + expect(MediaDetector.isAnimatedBuffer(buffer, "mov")).toBe(true); + }); + }); +}); diff --git a/src/lib/processing/media-detector.ts b/src/lib/processing/media-detector.ts new file mode 100644 index 0000000..b4a655a --- /dev/null +++ b/src/lib/processing/media-detector.ts @@ -0,0 +1,192 @@ +import { readFile } from "fs/promises"; + +/** + * Media type detection result + */ +export interface MediaInfo { + /** Type of media file */ + type: "image" | "gif" | "video"; + /** MIME type */ + mimeType: string; + /** File extension */ + extension: string; +} + +/** + * Detect media file type from file path or buffer + */ +export class MediaDetector { + /** + * Detect media type from file path + * @param filePath Path to the media file + * @returns Media info + */ + static async detectFromFile(filePath: string): Promise { + const buffer = await readFile(filePath); + const extension = filePath.split(".").pop()?.toLowerCase() || ""; + + return this.detectFromBuffer(buffer, extension); + } + + /** + * Detect media type from buffer + * @param buffer File data buffer + * @param extension Optional file extension hint + * @returns Media info + */ + static detectFromBuffer(buffer: Buffer, extension: string = ""): MediaInfo { + // Check magic bytes for file type + const magic = buffer.subarray(0, 12); + + // GIF: 47 49 46 38 (GIF8) + if (magic[0] === 0x47 && magic[1] === 0x49 && magic[2] === 0x46) { + return { + type: "gif", + mimeType: "image/gif", + extension: "gif", + }; + } + + // Video formats + // MP4: starts with ftyp box (offset 4-7: 66 74 79 70 = "ftyp") + if ( + magic[4] === 0x66 && + magic[5] === 0x74 && + magic[6] === 0x79 && + magic[7] === 0x70 + ) { + return { + type: "video", + mimeType: "video/mp4", + extension: "mp4", + }; + } + + // WebM: 1A 45 DF A3 + if ( + magic[0] === 0x1a && + magic[1] === 0x45 && + magic[2] === 0xdf && + magic[3] === 0xa3 + ) { + return { + type: "video", + mimeType: "video/webm", + extension: "webm", + }; + } + + // AVI: 52 49 46 46 ... 41 56 49 20 (RIFF...AVI ) + if ( + magic[0] === 0x52 && + magic[1] === 0x49 && + magic[2] === 0x46 && + magic[3] === 0x46 && + magic[8] === 0x41 && + magic[9] === 0x56 && + magic[10] === 0x49 + ) { + return { + type: "video", + mimeType: "video/x-msvideo", + extension: "avi", + }; + } + + // MOV/QuickTime: similar to MP4 but with different ftyp subtypes + if (extension === "mov" || extension === "qt") { + return { + type: "video", + mimeType: "video/quicktime", + extension: "mov", + }; + } + + // MKV: 1A 45 DF A3 (same as WebM but different codec) + if (extension === "mkv") { + return { + type: "video", + mimeType: "video/x-matroska", + extension: "mkv", + }; + } + + // JPEG: FF D8 FF + if (magic[0] === 0xff && magic[1] === 0xd8 && magic[2] === 0xff) { + return { + type: "image", + mimeType: "image/jpeg", + extension: "jpg", + }; + } + + // PNG: 89 50 4E 47 0D 0A 1A 0A + if ( + magic[0] === 0x89 && + magic[1] === 0x50 && + magic[2] === 0x4e && + magic[3] === 0x47 + ) { + return { + type: "image", + mimeType: "image/png", + extension: "png", + }; + } + + // WebP: 52 49 46 46 ... 57 45 42 50 (RIFF...WEBP) + if ( + magic[0] === 0x52 && + magic[1] === 0x49 && + magic[2] === 0x46 && + magic[3] === 0x46 && + magic[8] === 0x57 && + magic[9] === 0x45 && + magic[10] === 0x42 && + magic[11] === 0x50 + ) { + return { + type: "image", + mimeType: "image/webp", + extension: "webp", + }; + } + + // BMP: 42 4D + if (magic[0] === 0x42 && magic[1] === 0x4d) { + return { + type: "image", + mimeType: "image/bmp", + extension: "bmp", + }; + } + + // Default to image for unknown types + return { + type: "image", + mimeType: "application/octet-stream", + extension: extension || "bin", + }; + } + + /** + * Check if a file is animated (GIF or video) + * @param filePath Path to the file + * @returns True if file is animated + */ + static async isAnimated(filePath: string): Promise { + const info = await this.detectFromFile(filePath); + return info.type === "gif" || info.type === "video"; + } + + /** + * Check if a buffer is animated (GIF or video) + * @param buffer File data buffer + * @param extension Optional file extension hint + * @returns True if buffer is animated + */ + static isAnimatedBuffer(buffer: Buffer, extension: string = ""): boolean { + const info = this.detectFromBuffer(buffer, extension); + return info.type === "gif" || info.type === "video"; + } +} diff --git a/src/lib/protocol/builders/payload-builder.test.ts b/src/lib/protocol/builders/payload-builder.test.ts index b2a1cc2..3e49a22 100644 --- a/src/lib/protocol/builders/payload-builder.test.ts +++ b/src/lib/protocol/builders/payload-builder.test.ts @@ -3,6 +3,7 @@ import { PayloadBuilder } from "./payload-builder.ts"; import { PacketType } from "../packet-types.ts"; import { DEFAULT_PROTOCOL_CONFIG } from "../interfaces/defaults.ts"; import type { ProtocolConfig } from "../interfaces/config.ts"; +import type { XV4Frame } from "./xv4-header.ts"; import { expectHex, createTestJpeg, @@ -142,19 +143,102 @@ describe("PayloadBuilder", () => { }); }); - describe("buildInitPayload()", () => { - test("throws error (not implemented)", () => { - expect(() => createBuilder().buildInitPayload()).toThrow(); + describe("buildAnimationData()", () => { + test("format starts with {\"type\":5,\"data\"", () => { + const frames: XV4Frame[] = [ + { + name: "frame_00001", + data: createTestJpeg(100), + }, + ]; + const payload = createBuilder().buildAnimationData(frames, 50, [368, 368]); + const text = payload.toString("utf-8", 0, 15); + expect(text).toMatch(/^\{"type":5,"data/); }); - test("error message mentions Type 5/DYNAMIC_AMBIENCE", () => { - expect(() => createBuilder().buildInitPayload()).toThrow( - /DYNAMIC_AMBIENCE|Type 5/i - ); + test("has correct prefix", () => { + const frames: XV4Frame[] = [ + { + name: "frame_00001", + data: createTestJpeg(100), + }, + ]; + const payload = createBuilder().buildAnimationData(frames, 50, [368, 368]); + const prefix = payload.toString("utf-8", 0, 17); + expect(prefix).toBe('{"type":5,"data":'); }); - test("error message mentions not implemented", () => { - expect(() => createBuilder().buildInitPayload()).toThrow(/not.*implemented/i); + test("suffix is '}'", () => { + const frames: XV4Frame[] = [ + { + name: "frame_00001", + data: createTestJpeg(100), + }, + ]; + const payload = createBuilder().buildAnimationData(frames, 50, [368, 368]); + const lastByte = payload.toString("utf-8", payload.length - 1); + expect(lastByte).toBe("}"); + }); + + test("xV4 header present after prefix", () => { + const frames: XV4Frame[] = [ + { + name: "frame_00001", + data: createTestJpeg(100), + }, + ]; + const payload = createBuilder().buildAnimationData(frames, 50, [368, 368]); + const prefixLen = '{"type":5,"data":'.length; + const xv4Sig = payload.toString("utf-8", prefixLen, prefixLen + 3); + expect(xv4Sig).toBe("xV4"); + }); + + test("works with multiple frames", () => { + const frames: XV4Frame[] = [ + { + name: "frame_00001", + data: createTestJpeg(100), + }, + { + name: "frame_00002", + data: createTestJpeg(150), + }, + { + name: "frame_00003", + data: createTestJpeg(120), + }, + ]; + const payload = createBuilder().buildAnimationData(frames, 50, [368, 368]); + + // Check it has proper structure + const prefixLen = '{"type":5,"data":'.length; + const xv4Sig = payload.toString("utf-8", prefixLen, prefixLen + 3); + expect(xv4Sig).toBe("xV4"); + + // Check frame count in xV4 header (offset 8, u32 LE) + const frameCount = payload.readUInt32LE(prefixLen + 8); + expect(frameCount).toBe(3); + }); + + test("custom interval creates correct timing string", () => { + const frames: XV4Frame[] = [ + { + name: "frame_00001", + data: createTestJpeg(100), + }, + ]; + const payload = createBuilder().buildAnimationData(frames, 100, [368, 368]); + + // The timing string is embedded in the xV4 container + const prefixLen = '{"type":5,"data":'.length; + const xv4Start = prefixLen; + + // Timing string is at offset 16 (xV4 + ver + unk + pad + count + size) + const timingString = payload + .subarray(xv4Start + 16, xv4Start + 35) + .toString("utf8") + .replace(/\0.*$/, ""); + expect(timingString).toBe("output/100ms"); }); }); diff --git a/src/lib/protocol/builders/payload-builder.ts b/src/lib/protocol/builders/payload-builder.ts index 0c94db5..fb87ae7 100644 --- a/src/lib/protocol/builders/payload-builder.ts +++ b/src/lib/protocol/builders/payload-builder.ts @@ -1,6 +1,7 @@ import type { ProtocolConfig } from "../interfaces/config.ts"; import { PacketType } from "../packet-types.ts"; import { IMBHeaderBuilder } from "./imb-header.ts"; +import { XV4HeaderBuilder, type XV4Frame } from "./xv4-header.ts"; /** * Builder for creating protocol payloads and packets for image uploads @@ -39,33 +40,33 @@ export class PayloadBuilder { } /** - * Build initialization payload for dynamic ambience mode (animations, gallery) + * Build animation data payload for dynamic ambience mode * - * Type 5 (DYNAMIC_AMBIENCE) is used for: - * - Video uploads (frames extracted via FFmpeg) - * - GIF uploads (frames separated) - * - Gallery/slideshow mode (multiple images converted to animation) - * - * Process: - * 1. Extract/convert frames using FFmpeg - * 2. Send info packet: {"type":6,"number":1} (uses buildImageInfo) - * 3. Send data packet: {"type":5,"data":} - * - * Animation data format (needs implementation): - * - Signature: "xV4" (0x78 0x56 0x34) - * - Frame timing: "output/50ms" or similar - * - Frame references: "frame_00001", "frame_00002", etc. - * - Multiple JPEG frames embedded - * - * @throws {Error} Dynamic ambience feature is not yet implemented - * @deprecated This feature is not implemented yet + * @param frames Array of frames with names and JPEG data + * @param intervalMs Frame interval in milliseconds (default: 50ms = 20fps) + * @param targetSize Image dimensions [width, height] (default: [368, 368]) + * @returns Animation data payload bytes in format: {"type":5,"data":} */ - public buildInitPayload(): Buffer { - throw new Error( - "Dynamic ambience (PacketType.DYNAMIC_AMBIENCE) is not yet implemented. " + - "This feature is required for video/GIF upload and gallery/slideshow mode. " + - "Only static single image upload (PacketType.IMAGE) is currently supported.", + public buildAnimationData( + frames: XV4Frame[], + intervalMs: number = 50, + targetSize: [number, number] = [368, 368], + ): Buffer { + const dataPrefix = Buffer.from( + `{"type":${PacketType.DYNAMIC_AMBIENCE},"data":`, + "utf-8", ); + const dataSuffix = Buffer.from("}", "utf-8"); + + // Build xV4 animation container + const xv4Container = XV4HeaderBuilder.build( + frames, + intervalMs, + targetSize[0], + targetSize[1], + ); + + return Buffer.concat([dataPrefix, xv4Container, dataSuffix]); } /** diff --git a/src/lib/protocol/builders/xv4-header.test.ts b/src/lib/protocol/builders/xv4-header.test.ts new file mode 100644 index 0000000..5e56e4f --- /dev/null +++ b/src/lib/protocol/builders/xv4-header.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect } from "bun:test"; +import { XV4HeaderBuilder, type XV4Frame } from "./xv4-header.ts"; + +describe("XV4HeaderBuilder", () => { + describe("build", () => { + it("should create a valid xV4 container with single frame", () => { + const frames: XV4Frame[] = [ + { + name: "frame_00001", + data: Buffer.from([0xff, 0xd8, 0xff, 0xe0]), // Fake JPEG header + }, + ]; + + const container = XV4HeaderBuilder.build(frames, 50, 368, 368); + + // Check signature + expect(container.subarray(0, 3).toString("ascii")).toBe("xV4"); + + // Check version + expect(container.readUInt8(3)).toBe(0x12); + + // Check unknown byte + expect(container.readUInt8(4)).toBe(0x48); + + // Check frame count (offset 8, u32 LE) + expect(container.readUInt32LE(8)).toBe(1); + }); + + it("should create a valid xV4 container with multiple frames", () => { + const frames: XV4Frame[] = [ + { + name: "frame_00001", + data: Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]), + }, + { + name: "frame_00002", + data: Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x11]), + }, + { + name: "frame_00003", + data: Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x12]), + }, + ]; + + const container = XV4HeaderBuilder.build(frames, 50, 368, 368); + + // Check frame count (offset 8, u32 LE) + expect(container.readUInt32LE(8)).toBe(3); + + // Validate it's a proper xV4 container + expect(XV4HeaderBuilder.validate(container)).toBe(true); + }); + + it("should include correct timing string", () => { + const frames: XV4Frame[] = [ + { + name: "frame_00001", + data: Buffer.from([0xff, 0xd8, 0xff, 0xe0]), + }, + ]; + + const container = XV4HeaderBuilder.build(frames, 100, 368, 368); + + // Timing string is after: xV4 (3) + ver (1) + unk (1) + pad (3) + count (4) + size (4) = offset 16 + const timingString = container + .subarray(16, 35) + .toString("utf8") + .replace(/\0.*$/, ""); + expect(timingString).toBe("output/100ms"); + }); + + it("should throw error with no frames", () => { + expect(() => { + XV4HeaderBuilder.build([], 50, 368, 368); + }).toThrow("At least one frame is required"); + }); + }); + + describe("validate", () => { + it("should validate a proper xV4 container", () => { + const frames: XV4Frame[] = [ + { + name: "frame_00001", + data: Buffer.from([0xff, 0xd8, 0xff, 0xe0]), + }, + ]; + + const container = XV4HeaderBuilder.build(frames, 50, 368, 368); + expect(XV4HeaderBuilder.validate(container)).toBe(true); + }); + + it("should reject buffer that is too small", () => { + const buffer = Buffer.alloc(10); + expect(XV4HeaderBuilder.validate(buffer)).toBe(false); + }); + + it("should reject buffer with wrong signature", () => { + const buffer = Buffer.alloc(20); + buffer.write("ABC", 0); + buffer.writeUInt8(0x12, 3); + buffer.writeUInt8(0x48, 4); + + expect(XV4HeaderBuilder.validate(buffer)).toBe(false); + }); + + it("should reject buffer with wrong version", () => { + const buffer = Buffer.alloc(20); + buffer.write("xV4", 0); + buffer.writeUInt8(0x99, 3); // Wrong version + buffer.writeUInt8(0x48, 4); + + expect(XV4HeaderBuilder.validate(buffer)).toBe(false); + }); + + it("should reject buffer with wrong unknown byte", () => { + const buffer = Buffer.alloc(20); + buffer.write("xV4", 0); + buffer.writeUInt8(0x12, 3); + buffer.writeUInt8(0x99, 4); // Wrong unknown byte + + expect(XV4HeaderBuilder.validate(buffer)).toBe(false); + }); + }); +}); diff --git a/src/lib/protocol/builders/xv4-header.ts b/src/lib/protocol/builders/xv4-header.ts new file mode 100644 index 0000000..692e13d --- /dev/null +++ b/src/lib/protocol/builders/xv4-header.ts @@ -0,0 +1,221 @@ +/** + * xV4 Animation header builder + * + * The xV4 format is a animation container + * + * ## Format Structure + * + * ``` + * [Magic: "xV4"] [3 bytes] + * [Version] [1 byte] - Always 0x12 + * [Unknown] [1 byte] - Always 0x48 + * [Padding] [3 bytes] - Always 0x00 0x00 0x00 + * [Frame count] [4 bytes, LE u32] + * [Total JPEG size] [4 bytes, LE u32] - Sum of all frame data sizes + * [Timing string] [variable, null-terminated] - e.g., "output/50ms\0" + * [Frame data offset] [4 bytes, LE u32] - Offset to where JPEG data starts + * [Frame table] [variable] - Frame entries (name + size) + * [Footer] [16 bytes] - Frame sizes (first 2), unknown, dimensions + * [JPEG data] [variable] - Concatenated JPEG frames + * ``` + * + * ## Frame Table Entry + * + * Each entry consists of: + * - Frame name (dot-terminated string): e.g., "frame_00001." + * - Frame size (4 bytes, LE u32): Size of this JPEG frame in bytes + * + */ + +export interface XV4Frame { + /** Frame name without extension, e.g., "frame_00001" */ + name: string; + /** JPEG image data for this frame */ + data: Buffer; +} + +export class XV4HeaderBuilder { + private static readonly SIGNATURE = Buffer.from("xV4"); + private static readonly VERSION = 0x12; + private static readonly UNKNOWN_BYTE = 0x48; + + /** + * Build complete xV4 animation container with frames + * + * @param frames Array of frames with names and JPEG data + * @param intervalMs Frame interval in milliseconds (e.g., 50 for 20fps) + * @param width Image width (default: 368) + * @param height Image height (default: 368) + * @returns Complete xV4 animation buffer ready to send + */ + static build( + frames: XV4Frame[], + intervalMs: number = 50, + width: number = 368, + height: number = 368, + ): Buffer { + if (frames.length === 0) { + throw new Error("At least one frame is required"); + } + + // Calculate frame count + const frameCount = frames.length; + + // Total JPEG size field is it's frame_count * 1000 + const totalSizeField = frameCount * 1000; + + const totalJpegSize = frames.reduce( + (sum, frame) => sum + frame.data.length, + 0, + ); + + // Build timing string with null terminator + const timingStr = `output/${intervalMs}ms`; + const timingBuffer = Buffer.from(timingStr + "\0", "utf-8"); + + // Calculate frame table size (each entry: name + "." + 4-byte size) + let frameTableSize = 0; + for (const frame of frames) { + frameTableSize += Buffer.from(frame.name + ".", "utf-8").length + 4; + } + + // Footer size (from capture: 2 frame sizes + unknown + dimensions) + const footerSize = 16; + + // Header structure: + const headerSize = + 3 + // signature "xV4" + 1 + // version 0x12 + 1 + // unknown 0x48 + 3 + // padding 0x00 0x00 0x00 + 4 + // frame count (u32 LE) + 4 + // total JPEG size (u32 LE) + timingBuffer.length + // timing string with null terminator + 4; // frame data offset (u32 LE) + + // Frame data offset + const frameDataOffsetValue = 0x2c000 + (headerSize - 8); + + // Actual container size: header + frame table + footer + JPEG data + // (We don't actually pad to frameDataOffsetValue in our container) + const actualDataOffset = headerSize + frameTableSize + footerSize; + const totalSize = actualDataOffset + totalJpegSize; + + const container = Buffer.alloc(totalSize); + let offset = 0; + + // Write signature "xV4" + this.SIGNATURE.copy(container, offset); + offset += 3; + + // Write version 0x12 + container.writeUInt8(this.VERSION, offset); + offset += 1; + + // Write unknown byte 0x48 + container.writeUInt8(this.UNKNOWN_BYTE, offset); + offset += 1; + + // Write padding (3 null bytes) + container.writeUInt8(0x00, offset); + offset += 1; + container.writeUInt8(0x00, offset); + offset += 1; + container.writeUInt8(0x00, offset); + offset += 1; + + // Write frame count (u32 LE) + container.writeUInt32LE(frameCount, offset); + offset += 4; + + // Write total size field (u32 LE) - this is frame_count * 1000 + container.writeUInt32LE(totalSizeField, offset); + offset += 4; + + // Write timing string (null-terminated) - no null byte before it! + timingBuffer.copy(container, offset); + offset += timingBuffer.length; + + // Write frame data offset (u32 LE) - this is a reference value, not actual offset in our buffer + container.writeUInt32LE(frameDataOffsetValue, offset); + offset += 4; + + // Write frame table + for (const frame of frames) { + // Write frame name with trailing dot (no null terminator) + const nameBuffer = Buffer.from(frame.name + ".", "utf-8"); + nameBuffer.copy(container, offset); + offset += nameBuffer.length; + + // Write frame size (u32 LE) + container.writeUInt32LE(frame.data.length, offset); + offset += 4; + } + + // Write footer (16 bytes) + // First frame size + container.writeUInt32LE(frames[0]?.data.length || 0, offset); + offset += 4; + + // Second frame size (or first if only one frame) + container.writeUInt32LE( + frames[1]?.data.length || frames[0]?.data.length || 0, + offset, + ); + offset += 4; + + // Unknown value (from capture: seems to be related to header size) + // In capture it was 0x0b (11) for 3 frames + container.writeUInt32LE(frameCount + 8, offset); + offset += 4; + + // Dimensions (u16 LE + u16 LE) + container.writeUInt16LE(width, offset); + offset += 2; + container.writeUInt16LE(height, offset); + offset += 2; + + // Now offset should be at actualDataOffset (where JPEG data actually starts in our buffer) + if (offset !== actualDataOffset) { + throw new Error( + `Frame data offset calculation error: expected ${actualDataOffset}, got ${offset}`, + ); + } + + // Write JPEG data + for (const frame of frames) { + frame.data.copy(container, offset); + offset += frame.data.length; + } + + return container; + } + + /** + * Validate that a buffer is a proper xV4 container + * @param buffer Buffer to validate + * @returns True if valid xV4 container + */ + static validate(buffer: Buffer): boolean { + if (buffer.length < 20) { + return false; + } + + // Check signature + if (!buffer.subarray(0, 3).equals(this.SIGNATURE)) { + return false; + } + + // Check version + if (buffer.readUInt8(3) !== this.VERSION) { + return false; + } + + // Check unknown byte + if (buffer.readUInt8(4) !== this.UNKNOWN_BYTE) { + return false; + } + + return true; + } +} diff --git a/src/lib/protocol/index.ts b/src/lib/protocol/index.ts index d656937..0aff341 100644 --- a/src/lib/protocol/index.ts +++ b/src/lib/protocol/index.ts @@ -11,6 +11,7 @@ export * from "./interfaces/index.ts"; // Builders export { PayloadBuilder } from "./builders/payload-builder.ts"; export { IMBHeaderBuilder } from "./builders/imb-header.ts"; +export { XV4HeaderBuilder, type XV4Frame } from "./builders/xv4-header.ts"; // Parsers export { ResponseParser } from "./parsers/response-parser.ts";