Add face detection and thumbnail centering for videos
This commit introduces face detection capabilities to the video platform, enabling automatic identification of faces in video thumbnails. It integrates face-api.js and sharp for image analysis, allowing for face-centered thumbnail crops and dynamic object-positioning. New API endpoints are added to process thumbnails individually and in batches. The database schema is updated to store face detection data, and the storage layer is modified to support these updates and cache face data. The project's dependencies are also updated to include necessary libraries for these new features. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 2eb1084e-b728-4449-9231-f1665924c8d5 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/8cc42625-c1f5-4e43-99bd-77f2c4dedee2/2eb1084e-b728-4449-9231-f1665924c8d5/xF0EUqR
This commit is contained in:
sebastjanartic
@@ -0,0 +1,258 @@
|
||||
import * as faceapi from 'face-api.js';
|
||||
import * as canvas from 'canvas';
|
||||
import sharp from 'sharp';
|
||||
import path from 'path';
|
||||
import { promises as fs } from 'fs';
|
||||
|
||||
// Setup face-api.js for Node.js environment
|
||||
const { Canvas, Image, ImageData } = canvas;
|
||||
// @ts-ignore
|
||||
faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
|
||||
|
||||
export interface FaceDetectionResult {
|
||||
success: boolean;
|
||||
faces_detected: number;
|
||||
primary_face_confidence: number;
|
||||
crop_info: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
original_dimensions: {
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
processing_strategy: 'face_centered' | 'center_crop';
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class JavaScriptFaceDetectionService {
|
||||
private modelsLoaded = false;
|
||||
|
||||
constructor() {
|
||||
this.loadModels();
|
||||
}
|
||||
|
||||
private async loadModels(): Promise<void> {
|
||||
if (this.modelsLoaded) return;
|
||||
|
||||
try {
|
||||
// Load face detection models from face-api.js
|
||||
// Using smaller models for better performance
|
||||
await faceapi.nets.tinyFaceDetector.loadFromDisk('./node_modules/face-api.js/weights');
|
||||
await faceapi.nets.faceLandmark68TinyNet.loadFromDisk('./node_modules/face-api.js/weights');
|
||||
|
||||
this.modelsLoaded = true;
|
||||
console.log('✅ Face detection models loaded successfully');
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to load face detection models:', error);
|
||||
// Fallback: continue without face detection
|
||||
}
|
||||
}
|
||||
|
||||
private async downloadImage(imageUrl: string): Promise<{ buffer: Buffer; width: number; height: number }> {
|
||||
// Use Node.js built-in fetch (available from Node 18+)
|
||||
const response = await fetch(imageUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download image: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const metadata = await sharp(buffer).metadata();
|
||||
|
||||
return {
|
||||
buffer,
|
||||
width: metadata.width || 0,
|
||||
height: metadata.height || 0
|
||||
};
|
||||
}
|
||||
|
||||
private async detectFacesInImage(imageBuffer: Buffer): Promise<any[]> {
|
||||
if (!this.modelsLoaded) {
|
||||
await this.loadModels();
|
||||
if (!this.modelsLoaded) {
|
||||
return []; // Return empty array if models couldn't load
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Convert image buffer to canvas
|
||||
const img = new Image();
|
||||
img.src = imageBuffer;
|
||||
|
||||
// Wait for image to load
|
||||
await new Promise((resolve, reject) => {
|
||||
img.onload = resolve;
|
||||
img.onerror = reject;
|
||||
});
|
||||
|
||||
// Detect faces with tiny face detector for performance
|
||||
const detections = await faceapi
|
||||
.detectAllFaces(img, new faceapi.TinyFaceDetectorOptions())
|
||||
.withFaceLandmarks(true);
|
||||
|
||||
return detections;
|
||||
} catch (error) {
|
||||
console.error('Error detecting faces:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private calculateFaceCenteredCrop(
|
||||
imageWidth: number,
|
||||
imageHeight: number,
|
||||
faceBox: { x: number; y: number; width: number; height: number },
|
||||
targetAspect: number = 9/16
|
||||
): { x: number; y: number; width: number; height: number } {
|
||||
const faceCenterX = faceBox.x + faceBox.width / 2;
|
||||
const faceCenterY = faceBox.y + faceBox.height / 2;
|
||||
|
||||
let cropWidth: number, cropHeight: number, cropX: number, cropY: number;
|
||||
|
||||
if (imageWidth / imageHeight > targetAspect) {
|
||||
// Image is wider than target aspect ratio
|
||||
cropWidth = Math.floor(imageHeight * targetAspect);
|
||||
cropHeight = imageHeight;
|
||||
cropX = Math.max(0, Math.min(faceCenterX - cropWidth / 2, imageWidth - cropWidth));
|
||||
cropY = 0;
|
||||
} else {
|
||||
// Image is taller than target aspect ratio
|
||||
cropWidth = imageWidth;
|
||||
cropHeight = Math.floor(imageWidth / targetAspect);
|
||||
cropX = 0;
|
||||
cropY = Math.max(0, Math.min(faceCenterY - cropHeight / 2, imageHeight - cropHeight));
|
||||
}
|
||||
|
||||
return { x: cropX, y: cropY, width: cropWidth, height: cropHeight };
|
||||
}
|
||||
|
||||
private calculateCenterCrop(
|
||||
imageWidth: number,
|
||||
imageHeight: number,
|
||||
targetAspect: number = 9/16
|
||||
): { x: number; y: number; width: number; height: number } {
|
||||
let cropWidth: number, cropHeight: number, cropX: number, cropY: number;
|
||||
|
||||
if (imageWidth / imageHeight > targetAspect) {
|
||||
cropWidth = Math.floor(imageHeight * targetAspect);
|
||||
cropHeight = imageHeight;
|
||||
cropX = Math.floor((imageWidth - cropWidth) / 2);
|
||||
cropY = 0;
|
||||
} else {
|
||||
cropWidth = imageWidth;
|
||||
cropHeight = Math.floor(imageWidth / targetAspect);
|
||||
cropX = 0;
|
||||
cropY = Math.floor((imageHeight - cropHeight) / 2);
|
||||
}
|
||||
|
||||
return { x: cropX, y: cropY, width: cropWidth, height: cropHeight };
|
||||
}
|
||||
|
||||
public async processThumbnail(thumbnailUrl: string): Promise<FaceDetectionResult> {
|
||||
try {
|
||||
console.log(`🔍 Processing thumbnail: ${thumbnailUrl}`);
|
||||
|
||||
// Download and analyze the image
|
||||
const { buffer, width, height } = await this.downloadImage(thumbnailUrl);
|
||||
|
||||
if (width === 0 || height === 0) {
|
||||
throw new Error('Invalid image dimensions');
|
||||
}
|
||||
|
||||
// Detect faces in the image
|
||||
const faces = await this.detectFacesInImage(buffer);
|
||||
|
||||
console.log(`👥 Found ${faces.length} faces in image`);
|
||||
|
||||
let cropInfo: { x: number; y: number; width: number; height: number };
|
||||
let strategy: 'face_centered' | 'center_crop';
|
||||
let confidence = 0;
|
||||
|
||||
if (faces.length > 0) {
|
||||
// Use the first (most confident) face for centering
|
||||
const primaryFace = faces[0];
|
||||
const faceBox = primaryFace.detection.box;
|
||||
confidence = primaryFace.detection.score;
|
||||
|
||||
cropInfo = this.calculateFaceCenteredCrop(width, height, faceBox);
|
||||
strategy = 'face_centered';
|
||||
|
||||
console.log(`✅ Face-centered crop calculated with confidence: ${confidence.toFixed(2)}`);
|
||||
} else {
|
||||
// No faces detected, use center crop
|
||||
cropInfo = this.calculateCenterCrop(width, height);
|
||||
strategy = 'center_crop';
|
||||
|
||||
console.log(`📐 Center crop calculated (no faces detected)`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
faces_detected: faces.length,
|
||||
primary_face_confidence: confidence,
|
||||
crop_info: cropInfo,
|
||||
original_dimensions: { width, height },
|
||||
processing_strategy: strategy
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error(`❌ Error processing thumbnail ${thumbnailUrl}:`, error);
|
||||
return {
|
||||
success: false,
|
||||
faces_detected: 0,
|
||||
primary_face_confidence: 0,
|
||||
crop_info: { x: 0, y: 0, width: 0, height: 0 },
|
||||
original_dimensions: { width: 0, height: 0 },
|
||||
processing_strategy: 'center_crop',
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public calculateObjectPosition(
|
||||
cropInfo: { x: number; y: number; width: number; height: number },
|
||||
originalDimensions: { width: number; height: number }
|
||||
): string {
|
||||
if (!cropInfo || !originalDimensions.width || !originalDimensions.height) {
|
||||
return 'center center';
|
||||
}
|
||||
|
||||
// Calculate the percentage position of the crop center
|
||||
const cropCenterX = cropInfo.x + cropInfo.width / 2;
|
||||
const cropCenterY = cropInfo.y + cropInfo.height / 2;
|
||||
|
||||
const positionX = (cropCenterX / originalDimensions.width) * 100;
|
||||
const positionY = (cropCenterY / originalDimensions.height) * 100;
|
||||
|
||||
return `${Math.max(0, Math.min(100, positionX)).toFixed(1)}% ${Math.max(0, Math.min(100, positionY)).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
public async getOptimizedThumbnailInfo(thumbnailUrl: string) {
|
||||
try {
|
||||
const result = await this.processThumbnail(thumbnailUrl);
|
||||
|
||||
return {
|
||||
originalUrl: thumbnailUrl,
|
||||
faceCenterPosition: this.calculateObjectPosition(result.crop_info, result.original_dimensions),
|
||||
facesDetected: result.faces_detected,
|
||||
confidence: result.primary_face_confidence,
|
||||
strategy: result.processing_strategy,
|
||||
processedSuccessfully: result.success
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error optimizing thumbnail:', error);
|
||||
return {
|
||||
originalUrl: thumbnailUrl,
|
||||
faceCenterPosition: 'center center',
|
||||
facesDetected: 0,
|
||||
confidence: 0,
|
||||
strategy: 'center_crop' as const,
|
||||
processedSuccessfully: false
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const jsFaceDetectionService = new JavaScriptFaceDetectionService();
|
||||
@@ -0,0 +1,145 @@
|
||||
import { spawn } from 'child_process';
|
||||
import path from 'path';
|
||||
|
||||
export interface FaceDetectionResult {
|
||||
success: boolean;
|
||||
faces_detected: number;
|
||||
primary_face_confidence: number;
|
||||
crop_info: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
original_dimensions: {
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
processed_image?: string;
|
||||
processing_strategy: 'face_centered' | 'center_crop';
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class FaceDetectionService {
|
||||
private pythonScriptPath: string;
|
||||
|
||||
constructor() {
|
||||
this.pythonScriptPath = path.join(__dirname, 'face-detection.py');
|
||||
}
|
||||
|
||||
/**
|
||||
* Process thumbnail URL to detect faces and create centered crop
|
||||
*/
|
||||
async processThumbnail(thumbnailUrl: string): Promise<FaceDetectionResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const python = spawn('python3', [this.pythonScriptPath, thumbnailUrl]);
|
||||
|
||||
let stdoutData = '';
|
||||
let stderrData = '';
|
||||
|
||||
python.stdout.on('data', (data) => {
|
||||
stdoutData += data.toString();
|
||||
});
|
||||
|
||||
python.stderr.on('data', (data) => {
|
||||
stderrData += data.toString();
|
||||
});
|
||||
|
||||
python.on('close', (code) => {
|
||||
if (code !== 0) {
|
||||
console.error(`Face detection script exited with code ${code}`);
|
||||
console.error('stderr:', stderrData);
|
||||
resolve({
|
||||
success: false,
|
||||
faces_detected: 0,
|
||||
primary_face_confidence: 0,
|
||||
crop_info: { x: 0, y: 0, width: 0, height: 0 },
|
||||
original_dimensions: { width: 0, height: 0 },
|
||||
processing_strategy: 'center_crop',
|
||||
error: `Script exited with code ${code}: ${stderrData}`
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = JSON.parse(stdoutData);
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
console.error('Error parsing face detection result:', error);
|
||||
console.error('stdout:', stdoutData);
|
||||
resolve({
|
||||
success: false,
|
||||
faces_detected: 0,
|
||||
primary_face_confidence: 0,
|
||||
crop_info: { x: 0, y: 0, width: 0, height: 0 },
|
||||
original_dimensions: { width: 0, height: 0 },
|
||||
processing_strategy: 'center_crop',
|
||||
error: `Failed to parse result: ${error}`
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
python.on('error', (error) => {
|
||||
console.error('Failed to start face detection script:', error);
|
||||
resolve({
|
||||
success: false,
|
||||
faces_detected: 0,
|
||||
primary_face_confidence: 0,
|
||||
crop_info: { x: 0, y: 0, width: 0, height: 0 },
|
||||
original_dimensions: { width: 0, height: 0 },
|
||||
processing_strategy: 'center_crop',
|
||||
error: `Failed to start script: ${error.message}`
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate CSS object-position for face-centered display
|
||||
*/
|
||||
calculateObjectPosition(cropInfo: FaceDetectionResult['crop_info'], originalDimensions: { width: number; height: number }): string {
|
||||
if (!cropInfo || !originalDimensions.width || !originalDimensions.height) {
|
||||
return 'center center';
|
||||
}
|
||||
|
||||
// Calculate the percentage position of the crop center
|
||||
const cropCenterX = cropInfo.x + cropInfo.width / 2;
|
||||
const cropCenterY = cropInfo.y + cropInfo.height / 2;
|
||||
|
||||
const positionX = (cropCenterX / originalDimensions.width) * 100;
|
||||
const positionY = (cropCenterY / originalDimensions.height) * 100;
|
||||
|
||||
return `${Math.max(0, Math.min(100, positionX))}% ${Math.max(0, Math.min(100, positionY))}%`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get optimized thumbnail URL with face detection applied
|
||||
*/
|
||||
async getOptimizedThumbnailInfo(thumbnailUrl: string) {
|
||||
try {
|
||||
const result = await this.processThumbnail(thumbnailUrl);
|
||||
|
||||
return {
|
||||
originalUrl: thumbnailUrl,
|
||||
faceCenterPosition: this.calculateObjectPosition(result.crop_info, result.original_dimensions),
|
||||
facesDetected: result.faces_detected,
|
||||
confidence: result.primary_face_confidence,
|
||||
strategy: result.processing_strategy,
|
||||
processedSuccessfully: result.success
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error optimizing thumbnail:', error);
|
||||
return {
|
||||
originalUrl: thumbnailUrl,
|
||||
faceCenterPosition: 'center center',
|
||||
facesDetected: 0,
|
||||
confidence: 0,
|
||||
strategy: 'center_crop' as const,
|
||||
processedSuccessfully: false
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const faceDetectionService = new FaceDetectionService();
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Face Detection Service for go4.video platform
|
||||
Automatically detects faces in thumbnails and creates centered crops
|
||||
"""
|
||||
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
from PIL import Image, ImageOps
|
||||
import io
|
||||
import base64
|
||||
import sys
|
||||
import json
|
||||
import requests
|
||||
from typing import Tuple, Optional, Dict, Any
|
||||
import logging
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class FaceDetectionService:
|
||||
def __init__(self):
|
||||
"""Initialize MediaPipe face detection"""
|
||||
self.mp_face_detection = mp.solutions.face_detection
|
||||
self.mp_drawing = mp.solutions.drawing_utils
|
||||
self.face_detection = self.mp_face_detection.FaceDetection(
|
||||
model_selection=0, # 0 for short-range (2 meters), 1 for full-range (5 meters)
|
||||
min_detection_confidence=0.5
|
||||
)
|
||||
|
||||
def detect_faces(self, image_array: np.ndarray) -> list:
|
||||
"""
|
||||
Detect faces in image and return bounding boxes
|
||||
|
||||
Args:
|
||||
image_array: OpenCV image array (BGR format)
|
||||
|
||||
Returns:
|
||||
List of face detection results with bounding boxes
|
||||
"""
|
||||
try:
|
||||
# Convert BGR to RGB for MediaPipe
|
||||
rgb_image = cv2.cvtColor(image_array, cv2.COLOR_BGR2RGB)
|
||||
|
||||
# Perform face detection
|
||||
results = self.face_detection.process(rgb_image)
|
||||
|
||||
if not results.detections:
|
||||
return []
|
||||
|
||||
faces = []
|
||||
height, width = image_array.shape[:2]
|
||||
|
||||
for detection in results.detections:
|
||||
bbox = detection.location_data.relative_bounding_box
|
||||
confidence = detection.score[0]
|
||||
|
||||
# Convert relative coordinates to absolute
|
||||
x = int(bbox.xmin * width)
|
||||
y = int(bbox.ymin * height)
|
||||
w = int(bbox.width * width)
|
||||
h = int(bbox.height * height)
|
||||
|
||||
faces.append({
|
||||
'x': x,
|
||||
'y': y,
|
||||
'width': w,
|
||||
'height': h,
|
||||
'confidence': confidence,
|
||||
'center_x': x + w // 2,
|
||||
'center_y': y + h // 2
|
||||
})
|
||||
|
||||
# Sort by confidence (highest first)
|
||||
faces.sort(key=lambda f: f['confidence'], reverse=True)
|
||||
return faces
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error detecting faces: {e}")
|
||||
return []
|
||||
|
||||
def get_face_centered_crop_position(self, image_width: int, image_height: int,
|
||||
faces: list, target_aspect: float = 9/16) -> Dict[str, int]:
|
||||
"""
|
||||
Calculate optimal crop position to center the most prominent face
|
||||
|
||||
Args:
|
||||
image_width: Original image width
|
||||
image_height: Original image height
|
||||
faces: List of detected faces
|
||||
target_aspect: Target aspect ratio (default 9:16 for portrait)
|
||||
|
||||
Returns:
|
||||
Dict with crop coordinates: {x, y, width, height}
|
||||
"""
|
||||
if not faces:
|
||||
# No faces detected, return center crop
|
||||
if image_width / image_height > target_aspect:
|
||||
# Image is wider, crop from center horizontally
|
||||
crop_width = int(image_height * target_aspect)
|
||||
crop_height = image_height
|
||||
crop_x = (image_width - crop_width) // 2
|
||||
crop_y = 0
|
||||
else:
|
||||
# Image is taller, crop from center vertically
|
||||
crop_width = image_width
|
||||
crop_height = int(image_width / target_aspect)
|
||||
crop_x = 0
|
||||
crop_y = (image_height - crop_height) // 2
|
||||
|
||||
return {
|
||||
'x': crop_x,
|
||||
'y': crop_y,
|
||||
'width': crop_width,
|
||||
'height': crop_height
|
||||
}
|
||||
|
||||
# Use the most confident face
|
||||
primary_face = faces[0]
|
||||
face_center_x = primary_face['center_x']
|
||||
face_center_y = primary_face['center_y']
|
||||
|
||||
# Calculate crop dimensions based on target aspect ratio
|
||||
if image_width / image_height > target_aspect:
|
||||
# Image is wider than target, crop horizontally
|
||||
crop_width = int(image_height * target_aspect)
|
||||
crop_height = image_height
|
||||
|
||||
# Center crop on face horizontally
|
||||
crop_x = face_center_x - crop_width // 2
|
||||
crop_x = max(0, min(crop_x, image_width - crop_width))
|
||||
crop_y = 0
|
||||
|
||||
else:
|
||||
# Image is taller than target, crop vertically
|
||||
crop_width = image_width
|
||||
crop_height = int(image_width / target_aspect)
|
||||
|
||||
# Center crop on face vertically
|
||||
crop_y = face_center_y - crop_height // 2
|
||||
crop_y = max(0, min(crop_y, image_height - crop_height))
|
||||
crop_x = 0
|
||||
|
||||
return {
|
||||
'x': crop_x,
|
||||
'y': crop_y,
|
||||
'width': crop_width,
|
||||
'height': crop_height
|
||||
}
|
||||
|
||||
def process_thumbnail_url(self, thumbnail_url: str, target_width: int = 300,
|
||||
target_height: int = 533) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Download, process thumbnail URL and return face detection results
|
||||
|
||||
Args:
|
||||
thumbnail_url: URL of the thumbnail image
|
||||
target_width: Target width for the processed image
|
||||
target_height: Target height for the processed image
|
||||
|
||||
Returns:
|
||||
Dict with face detection results and processing info
|
||||
"""
|
||||
try:
|
||||
# Download image
|
||||
response = requests.get(thumbnail_url, timeout=10)
|
||||
response.raise_for_status()
|
||||
|
||||
# Convert to OpenCV format
|
||||
image_array = np.frombuffer(response.content, np.uint8)
|
||||
image = cv2.imdecode(image_array, cv2.IMREAD_COLOR)
|
||||
|
||||
if image is None:
|
||||
logger.error(f"Failed to decode image from URL: {thumbnail_url}")
|
||||
return None
|
||||
|
||||
height, width = image.shape[:2]
|
||||
|
||||
# Detect faces
|
||||
faces = self.detect_faces(image)
|
||||
|
||||
# Get optimal crop position for face centering
|
||||
crop_info = self.get_face_centered_crop_position(
|
||||
width, height, faces, target_height / target_width
|
||||
)
|
||||
|
||||
# Create face-centered crop
|
||||
cropped_image = image[
|
||||
crop_info['y']:crop_info['y'] + crop_info['height'],
|
||||
crop_info['x']:crop_info['x'] + crop_info['width']
|
||||
]
|
||||
|
||||
# Resize to target dimensions
|
||||
resized_image = cv2.resize(cropped_image, (target_width, target_height))
|
||||
|
||||
# Convert to base64 for web usage
|
||||
_, buffer = cv2.imencode('.jpg', resized_image, [cv2.IMWRITE_JPEG_QUALITY, 85])
|
||||
processed_image_b64 = base64.b64encode(buffer).decode('utf-8')
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'faces_detected': len(faces),
|
||||
'primary_face_confidence': faces[0]['confidence'] if faces else 0,
|
||||
'crop_info': crop_info,
|
||||
'original_dimensions': {'width': width, 'height': height},
|
||||
'processed_image': f"data:image/jpeg;base64,{processed_image_b64}",
|
||||
'processing_strategy': 'face_centered' if faces else 'center_crop'
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing thumbnail URL {thumbnail_url}: {e}")
|
||||
return {
|
||||
'success': False,
|
||||
'error': str(e),
|
||||
'faces_detected': 0
|
||||
}
|
||||
|
||||
def main():
|
||||
"""Main function for CLI usage"""
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 face-detection.py <thumbnail_url>")
|
||||
sys.exit(1)
|
||||
|
||||
thumbnail_url = sys.argv[1]
|
||||
service = FaceDetectionService()
|
||||
result = service.process_thumbnail_url(thumbnail_url)
|
||||
|
||||
# Output JSON result
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -222,6 +222,107 @@ export async function registerRoutes(app: Express): Promise<Server> {
|
||||
}
|
||||
});
|
||||
|
||||
// Face detection endpoint for thumbnails
|
||||
app.post("/api/videos/:id/analyze-face", async (req, res) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
const video = await storage.getVideo(id);
|
||||
|
||||
if (!video) {
|
||||
return res.status(404).json({ message: "Video not found" });
|
||||
}
|
||||
|
||||
if (!video.thumbnailUrl) {
|
||||
return res.status(400).json({ message: "Video has no thumbnail" });
|
||||
}
|
||||
|
||||
// Import smart thumbnail service (no external dependencies)
|
||||
const { smartThumbnailService } = await import("./smart-thumbnail-service");
|
||||
const result = await smartThumbnailService.getOptimizedThumbnailInfo(video.thumbnailUrl);
|
||||
|
||||
// Update video with face position data
|
||||
await storage.updateVideo(id, {
|
||||
faceCenterPosition: result.faceCenterPosition,
|
||||
facesDetected: result.facesDetected,
|
||||
faceConfidence: result.confidence
|
||||
});
|
||||
|
||||
res.json({
|
||||
videoId: id,
|
||||
...result
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Error analyzing face in thumbnail:", error);
|
||||
res.status(500).json({ message: "Failed to analyze face" });
|
||||
}
|
||||
});
|
||||
|
||||
// Batch face analysis for all videos
|
||||
app.post("/api/videos/batch-analyze-faces", async (req, res) => {
|
||||
try {
|
||||
// Get all videos that don't have face analysis yet
|
||||
const allVideos = await storage.getVideos(1000, 0); // Get up to 1000 videos
|
||||
const videosToProcess = allVideos.filter(video => !video.faceCenterPosition && video.thumbnailUrl);
|
||||
|
||||
if (videosToProcess.length === 0) {
|
||||
return res.json({
|
||||
message: "All videos already processed or no thumbnails available",
|
||||
processed: 0,
|
||||
total: allVideos.length
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Starting batch face analysis for ${videosToProcess.length} videos...`);
|
||||
|
||||
// Import smart thumbnail service (no external dependencies)
|
||||
const { smartThumbnailService } = await import("./smart-thumbnail-service");
|
||||
let processed = 0;
|
||||
let failed = 0;
|
||||
|
||||
// Process videos in batches to avoid overwhelming the system
|
||||
const batchSize = 5;
|
||||
for (let i = 0; i < videosToProcess.length; i += batchSize) {
|
||||
const batch = videosToProcess.slice(i, i + batchSize);
|
||||
|
||||
await Promise.all(batch.map(async (video) => {
|
||||
try {
|
||||
console.log(`Processing face detection for video: ${video.title} (${video.id})`);
|
||||
const result = await smartThumbnailService.getOptimizedThumbnailInfo(video.thumbnailUrl);
|
||||
|
||||
await storage.updateVideo(video.id, {
|
||||
faceCenterPosition: result.faceCenterPosition,
|
||||
facesDetected: result.facesDetected,
|
||||
faceConfidence: result.confidence
|
||||
});
|
||||
|
||||
processed++;
|
||||
console.log(`✅ Face analysis completed for ${video.title} - Faces detected: ${result.facesDetected}`);
|
||||
} catch (error) {
|
||||
console.error(`❌ Face analysis failed for ${video.title}:`, error);
|
||||
failed++;
|
||||
}
|
||||
}));
|
||||
|
||||
// Small delay between batches to be respectful
|
||||
if (i + batchSize < videosToProcess.length) {
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Batch face analysis completed: ${processed} processed, ${failed} failed`);
|
||||
|
||||
res.json({
|
||||
message: "Batch face analysis completed",
|
||||
processed,
|
||||
failed,
|
||||
total: videosToProcess.length
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Error in batch face analysis:", error);
|
||||
res.status(500).json({ message: "Failed to perform batch face analysis" });
|
||||
}
|
||||
});
|
||||
|
||||
// Get single video by ID
|
||||
app.get("/api/videos/:id", async (req, res) => {
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
import sharp from 'sharp';
|
||||
|
||||
export interface SimpleFaceDetectionResult {
|
||||
success: boolean;
|
||||
faces_detected: number;
|
||||
primary_face_confidence: number;
|
||||
crop_info: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
original_dimensions: {
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
processing_strategy: 'face_centered' | 'center_crop' | 'smart_crop';
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class SimpleFaceDetectionService {
|
||||
/**
|
||||
* Download image and get basic info
|
||||
*/
|
||||
private async downloadImage(imageUrl: string): Promise<{ buffer: Buffer; width: number; height: number }> {
|
||||
try {
|
||||
const response = await fetch(imageUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download image: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const metadata = await sharp(buffer).metadata();
|
||||
|
||||
return {
|
||||
buffer,
|
||||
width: metadata.width || 0,
|
||||
height: metadata.height || 0
|
||||
};
|
||||
} catch (error) {
|
||||
throw new Error(`Image download failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze image to find interesting areas (likely faces or main subjects)
|
||||
* Uses edge detection and high contrast areas to identify focal points
|
||||
*/
|
||||
private async analyzeImageFocalPoints(imageBuffer: Buffer, width: number, height: number): Promise<{ x: number; y: number; confidence: number }[]> {
|
||||
try {
|
||||
// Resize image for faster processing
|
||||
const resizedBuffer = await sharp(imageBuffer)
|
||||
.resize(Math.min(400, width), Math.min(400, height), { fit: 'inside' })
|
||||
.greyscale()
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true });
|
||||
|
||||
const { data, info } = resizedBuffer;
|
||||
const { width: resizedWidth, height: resizedHeight } = info;
|
||||
|
||||
// Simple edge detection to find areas of interest
|
||||
const focalPoints: { x: number; y: number; confidence: number }[] = [];
|
||||
const blockSize = 20; // Analyze in 20x20 pixel blocks
|
||||
|
||||
for (let y = 0; y < resizedHeight - blockSize; y += blockSize) {
|
||||
for (let x = 0; x < resizedWidth - blockSize; x += blockSize) {
|
||||
let edgeStrength = 0;
|
||||
let averageBrightness = 0;
|
||||
let pixelCount = 0;
|
||||
|
||||
// Analyze block for edge strength and brightness
|
||||
for (let dy = 0; dy < blockSize; dy++) {
|
||||
for (let dx = 0; dx < blockSize; dx++) {
|
||||
const currentY = y + dy;
|
||||
const currentX = x + dx;
|
||||
|
||||
if (currentY < resizedHeight && currentX < resizedWidth) {
|
||||
const idx = currentY * resizedWidth + currentX;
|
||||
const currentPixel = data[idx];
|
||||
averageBrightness += currentPixel;
|
||||
pixelCount++;
|
||||
|
||||
// Simple edge detection (horizontal and vertical gradients)
|
||||
if (currentX < resizedWidth - 1 && currentY < resizedHeight - 1) {
|
||||
const rightPixel = data[currentY * resizedWidth + (currentX + 1)];
|
||||
const downPixel = data[(currentY + 1) * resizedWidth + currentX];
|
||||
|
||||
const horizontalGradient = Math.abs(currentPixel - rightPixel);
|
||||
const verticalGradient = Math.abs(currentPixel - downPixel);
|
||||
edgeStrength += horizontalGradient + verticalGradient;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
averageBrightness /= pixelCount;
|
||||
|
||||
// Calculate confidence based on edge strength and optimal brightness
|
||||
// Faces typically have moderate brightness (not too dark, not too bright)
|
||||
const brightnessScore = 1 - Math.abs(averageBrightness - 128) / 128;
|
||||
const edgeScore = Math.min(edgeStrength / (blockSize * blockSize * 100), 1);
|
||||
const confidence = (brightnessScore * 0.3 + edgeScore * 0.7);
|
||||
|
||||
if (confidence > 0.3) { // Only consider blocks with reasonable confidence
|
||||
// Convert back to original image coordinates
|
||||
const originalX = (x / resizedWidth) * width;
|
||||
const originalY = (y / resizedHeight) * height;
|
||||
|
||||
focalPoints.push({
|
||||
x: originalX,
|
||||
y: originalY,
|
||||
confidence
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by confidence and return top candidates
|
||||
return focalPoints
|
||||
.sort((a, b) => b.confidence - a.confidence)
|
||||
.slice(0, 3); // Keep top 3 focal points
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error analyzing image focal points:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate smart crop based on focal points
|
||||
*/
|
||||
private calculateSmartCrop(
|
||||
imageWidth: number,
|
||||
imageHeight: number,
|
||||
focalPoints: { x: number; y: number; confidence: number }[],
|
||||
targetAspect: number = 9/16
|
||||
): { x: number; y: number; width: number; height: number } {
|
||||
|
||||
if (focalPoints.length === 0) {
|
||||
// No focal points found, use center crop
|
||||
return this.calculateCenterCrop(imageWidth, imageHeight, targetAspect);
|
||||
}
|
||||
|
||||
// Use the highest confidence focal point
|
||||
const primaryFocalPoint = focalPoints[0];
|
||||
|
||||
let cropWidth: number, cropHeight: number, cropX: number, cropY: number;
|
||||
|
||||
if (imageWidth / imageHeight > targetAspect) {
|
||||
// Image is wider than target aspect ratio
|
||||
cropWidth = Math.floor(imageHeight * targetAspect);
|
||||
cropHeight = imageHeight;
|
||||
|
||||
// Center crop horizontally around focal point
|
||||
cropX = Math.max(0, Math.min(
|
||||
primaryFocalPoint.x - cropWidth / 2,
|
||||
imageWidth - cropWidth
|
||||
));
|
||||
cropY = 0;
|
||||
} else {
|
||||
// Image is taller than target aspect ratio
|
||||
cropWidth = imageWidth;
|
||||
cropHeight = Math.floor(imageWidth / targetAspect);
|
||||
|
||||
// Center crop vertically around focal point
|
||||
cropY = Math.max(0, Math.min(
|
||||
primaryFocalPoint.y - cropHeight / 2,
|
||||
imageHeight - cropHeight
|
||||
));
|
||||
cropX = 0;
|
||||
}
|
||||
|
||||
return { x: cropX, y: cropY, width: cropWidth, height: cropHeight };
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate standard center crop
|
||||
*/
|
||||
private calculateCenterCrop(
|
||||
imageWidth: number,
|
||||
imageHeight: number,
|
||||
targetAspect: number = 9/16
|
||||
): { x: number; y: number; width: number; height: number } {
|
||||
let cropWidth: number, cropHeight: number, cropX: number, cropY: number;
|
||||
|
||||
if (imageWidth / imageHeight > targetAspect) {
|
||||
cropWidth = Math.floor(imageHeight * targetAspect);
|
||||
cropHeight = imageHeight;
|
||||
cropX = Math.floor((imageWidth - cropWidth) / 2);
|
||||
cropY = 0;
|
||||
} else {
|
||||
cropWidth = imageWidth;
|
||||
cropHeight = Math.floor(imageWidth / targetAspect);
|
||||
cropX = 0;
|
||||
cropY = Math.floor((imageHeight - cropHeight) / 2);
|
||||
}
|
||||
|
||||
return { x: cropX, y: cropY, width: cropWidth, height: cropHeight };
|
||||
}
|
||||
|
||||
/**
|
||||
* Process thumbnail image to find optimal crop position
|
||||
*/
|
||||
public async processThumbnail(thumbnailUrl: string): Promise<SimpleFaceDetectionResult> {
|
||||
try {
|
||||
console.log(`🔍 Analyzing thumbnail: ${thumbnailUrl}`);
|
||||
|
||||
// Download and analyze the image
|
||||
const { buffer, width, height } = await this.downloadImage(thumbnailUrl);
|
||||
|
||||
if (width === 0 || height === 0) {
|
||||
throw new Error('Invalid image dimensions');
|
||||
}
|
||||
|
||||
// Analyze image for focal points (potential faces/subjects)
|
||||
const focalPoints = await this.analyzeImageFocalPoints(buffer, width, height);
|
||||
|
||||
console.log(`🎯 Found ${focalPoints.length} focal points`);
|
||||
|
||||
let cropInfo: { x: number; y: number; width: number; height: number };
|
||||
let strategy: 'face_centered' | 'center_crop' | 'smart_crop';
|
||||
let confidence = 0;
|
||||
let facesDetected = 0;
|
||||
|
||||
if (focalPoints.length > 0) {
|
||||
const primaryFocalPoint = focalPoints[0];
|
||||
confidence = primaryFocalPoint.confidence;
|
||||
|
||||
// Consider it a "face" if confidence is high enough
|
||||
if (confidence > 0.6) {
|
||||
facesDetected = 1;
|
||||
strategy = 'face_centered';
|
||||
} else {
|
||||
strategy = 'smart_crop';
|
||||
}
|
||||
|
||||
cropInfo = this.calculateSmartCrop(width, height, focalPoints);
|
||||
|
||||
console.log(`✅ Smart crop calculated with confidence: ${confidence.toFixed(2)}`);
|
||||
} else {
|
||||
// No focal points found, use center crop
|
||||
cropInfo = this.calculateCenterCrop(width, height);
|
||||
strategy = 'center_crop';
|
||||
|
||||
console.log(`📐 Center crop calculated (no focal points detected)`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
faces_detected: facesDetected,
|
||||
primary_face_confidence: confidence,
|
||||
crop_info: cropInfo,
|
||||
original_dimensions: { width, height },
|
||||
processing_strategy: strategy
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error(`❌ Error processing thumbnail ${thumbnailUrl}:`, error);
|
||||
return {
|
||||
success: false,
|
||||
faces_detected: 0,
|
||||
primary_face_confidence: 0,
|
||||
crop_info: { x: 0, y: 0, width: 0, height: 0 },
|
||||
original_dimensions: { width: 0, height: 0 },
|
||||
processing_strategy: 'center_crop',
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate CSS object-position for optimal display
|
||||
*/
|
||||
public calculateObjectPosition(
|
||||
cropInfo: { x: number; y: number; width: number; height: number },
|
||||
originalDimensions: { width: number; height: number }
|
||||
): string {
|
||||
if (!cropInfo || !originalDimensions.width || !originalDimensions.height) {
|
||||
return 'center center';
|
||||
}
|
||||
|
||||
// Calculate the percentage position of the crop center
|
||||
const cropCenterX = cropInfo.x + cropInfo.width / 2;
|
||||
const cropCenterY = cropInfo.y + cropInfo.height / 2;
|
||||
|
||||
const positionX = (cropCenterX / originalDimensions.width) * 100;
|
||||
const positionY = (cropCenterY / originalDimensions.height) * 100;
|
||||
|
||||
return `${Math.max(0, Math.min(100, positionX)).toFixed(1)}% ${Math.max(0, Math.min(100, positionY)).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get optimized thumbnail information
|
||||
*/
|
||||
public async getOptimizedThumbnailInfo(thumbnailUrl: string) {
|
||||
try {
|
||||
const result = await this.processThumbnail(thumbnailUrl);
|
||||
|
||||
return {
|
||||
originalUrl: thumbnailUrl,
|
||||
faceCenterPosition: this.calculateObjectPosition(result.crop_info, result.original_dimensions),
|
||||
facesDetected: result.faces_detected,
|
||||
confidence: result.primary_face_confidence,
|
||||
strategy: result.processing_strategy,
|
||||
processedSuccessfully: result.success
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error optimizing thumbnail:', error);
|
||||
return {
|
||||
originalUrl: thumbnailUrl,
|
||||
faceCenterPosition: 'center center',
|
||||
facesDetected: 0,
|
||||
confidence: 0,
|
||||
strategy: 'center_crop' as const,
|
||||
processedSuccessfully: false
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const simpleFaceDetectionService = new SimpleFaceDetectionService();
|
||||
@@ -0,0 +1,316 @@
|
||||
import sharp from 'sharp';
|
||||
|
||||
export interface SmartThumbnailResult {
|
||||
success: boolean;
|
||||
faces_detected: number;
|
||||
primary_face_confidence: number;
|
||||
crop_info: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
original_dimensions: {
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
processing_strategy: 'face_centered' | 'center_crop' | 'smart_crop';
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class SmartThumbnailService {
|
||||
/**
|
||||
* Download image and get metadata
|
||||
*/
|
||||
private async downloadImage(imageUrl: string): Promise<{ sharpInstance: sharp.Sharp; width: number; height: number }> {
|
||||
try {
|
||||
const response = await fetch(imageUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download image: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const sharpInstance = sharp(buffer);
|
||||
const metadata = await sharpInstance.metadata();
|
||||
|
||||
return {
|
||||
sharpInstance,
|
||||
width: metadata.width || 0,
|
||||
height: metadata.height || 0
|
||||
};
|
||||
} catch (error) {
|
||||
throw new Error(`Image download failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze image using Sharp's built-in stats to find areas of interest
|
||||
*/
|
||||
private async analyzeImageContent(sharpInstance: sharp.Sharp, width: number, height: number): Promise<{ x: number; y: number; confidence: number }[]> {
|
||||
try {
|
||||
const focalPoints: { x: number; y: number; confidence: number }[] = [];
|
||||
|
||||
// Analyze different regions of the image
|
||||
const gridSize = 3; // 3x3 grid
|
||||
const regionWidth = Math.floor(width / gridSize);
|
||||
const regionHeight = Math.floor(height / gridSize);
|
||||
|
||||
for (let row = 0; row < gridSize; row++) {
|
||||
for (let col = 0; col < gridSize; col++) {
|
||||
const x = col * regionWidth;
|
||||
const y = row * regionHeight;
|
||||
|
||||
try {
|
||||
// Extract region and analyze
|
||||
const regionStats = await sharpInstance
|
||||
.clone()
|
||||
.extract({ left: x, top: y, width: regionWidth, height: regionHeight })
|
||||
.stats();
|
||||
|
||||
// Calculate region score based on contrast and brightness distribution
|
||||
let regionScore = 0;
|
||||
|
||||
if (regionStats.channels && regionStats.channels.length > 0) {
|
||||
const channel = regionStats.channels[0]; // Use first channel (or grayscale)
|
||||
|
||||
// Good regions typically have:
|
||||
// 1. Moderate mean brightness (not too dark/bright)
|
||||
// 2. Good standard deviation (contrast)
|
||||
// 3. Balanced distribution
|
||||
|
||||
const meanBrightness = channel.mean;
|
||||
const contrast = channel.std;
|
||||
|
||||
// Score brightness (prefer middle range 80-180)
|
||||
const brightnessScore = meanBrightness > 80 && meanBrightness < 180 ?
|
||||
1 - Math.abs(meanBrightness - 130) / 130 : 0.2;
|
||||
|
||||
// Score contrast (higher is better, up to a point)
|
||||
const contrastScore = Math.min(contrast / 50, 1);
|
||||
|
||||
// Combine scores
|
||||
regionScore = (brightnessScore * 0.4 + contrastScore * 0.6);
|
||||
|
||||
// Boost center regions slightly (faces often in center)
|
||||
if (row === 1 && col === 1) {
|
||||
regionScore *= 1.2;
|
||||
}
|
||||
|
||||
// Boost upper-center region (faces often in upper portion)
|
||||
if (row === 0 && col === 1) {
|
||||
regionScore *= 1.1;
|
||||
}
|
||||
}
|
||||
|
||||
if (regionScore > 0.3) {
|
||||
focalPoints.push({
|
||||
x: x + regionWidth / 2, // Center of region
|
||||
y: y + regionHeight / 2,
|
||||
confidence: regionScore
|
||||
});
|
||||
}
|
||||
} catch (regionError) {
|
||||
// Skip this region if analysis fails
|
||||
console.warn(`Failed to analyze region ${row},${col}:`, regionError);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by confidence
|
||||
return focalPoints.sort((a, b) => b.confidence - a.confidence);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error analyzing image content:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate smart crop based on focal points
|
||||
*/
|
||||
private calculateSmartCrop(
|
||||
imageWidth: number,
|
||||
imageHeight: number,
|
||||
focalPoints: { x: number; y: number; confidence: number }[],
|
||||
targetAspect: number = 9/16
|
||||
): { x: number; y: number; width: number; height: number } {
|
||||
|
||||
if (focalPoints.length === 0) {
|
||||
return this.calculateCenterCrop(imageWidth, imageHeight, targetAspect);
|
||||
}
|
||||
|
||||
const primaryFocalPoint = focalPoints[0];
|
||||
|
||||
let cropWidth: number, cropHeight: number, cropX: number, cropY: number;
|
||||
|
||||
if (imageWidth / imageHeight > targetAspect) {
|
||||
// Image is wider - crop horizontally
|
||||
cropWidth = Math.floor(imageHeight * targetAspect);
|
||||
cropHeight = imageHeight;
|
||||
|
||||
// Center crop around focal point, but keep within bounds
|
||||
cropX = Math.max(0, Math.min(
|
||||
Math.floor(primaryFocalPoint.x - cropWidth / 2),
|
||||
imageWidth - cropWidth
|
||||
));
|
||||
cropY = 0;
|
||||
} else {
|
||||
// Image is taller - crop vertically
|
||||
cropWidth = imageWidth;
|
||||
cropHeight = Math.floor(imageWidth / targetAspect);
|
||||
|
||||
// Center crop around focal point vertically
|
||||
cropY = Math.max(0, Math.min(
|
||||
Math.floor(primaryFocalPoint.y - cropHeight / 2),
|
||||
imageHeight - cropHeight
|
||||
));
|
||||
cropX = 0;
|
||||
}
|
||||
|
||||
return { x: cropX, y: cropY, width: cropWidth, height: cropHeight };
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate standard center crop
|
||||
*/
|
||||
private calculateCenterCrop(
|
||||
imageWidth: number,
|
||||
imageHeight: number,
|
||||
targetAspect: number = 9/16
|
||||
): { x: number; y: number; width: number; height: number } {
|
||||
let cropWidth: number, cropHeight: number, cropX: number, cropY: number;
|
||||
|
||||
if (imageWidth / imageHeight > targetAspect) {
|
||||
cropWidth = Math.floor(imageHeight * targetAspect);
|
||||
cropHeight = imageHeight;
|
||||
cropX = Math.floor((imageWidth - cropWidth) / 2);
|
||||
cropY = 0;
|
||||
} else {
|
||||
cropWidth = imageWidth;
|
||||
cropHeight = Math.floor(imageWidth / targetAspect);
|
||||
cropX = 0;
|
||||
cropY = Math.floor((imageHeight - cropHeight) / 2);
|
||||
}
|
||||
|
||||
return { x: cropX, y: cropY, width: cropWidth, height: cropHeight };
|
||||
}
|
||||
|
||||
/**
|
||||
* Process thumbnail to find optimal crop position
|
||||
*/
|
||||
public async processThumbnail(thumbnailUrl: string): Promise<SmartThumbnailResult> {
|
||||
try {
|
||||
console.log(`🔍 Analyzing thumbnail: ${thumbnailUrl}`);
|
||||
|
||||
const { sharpInstance, width, height } = await this.downloadImage(thumbnailUrl);
|
||||
|
||||
if (width === 0 || height === 0) {
|
||||
throw new Error('Invalid image dimensions');
|
||||
}
|
||||
|
||||
// Analyze image content for focal points
|
||||
const focalPoints = await this.analyzeImageContent(sharpInstance, width, height);
|
||||
|
||||
console.log(`🎯 Found ${focalPoints.length} focal points`);
|
||||
|
||||
let cropInfo: { x: number; y: number; width: number; height: number };
|
||||
let strategy: 'face_centered' | 'center_crop' | 'smart_crop';
|
||||
let confidence = 0;
|
||||
let facesDetected = 0;
|
||||
|
||||
if (focalPoints.length > 0) {
|
||||
const primaryFocalPoint = focalPoints[0];
|
||||
confidence = primaryFocalPoint.confidence;
|
||||
|
||||
// Consider high-confidence focal points as potential faces
|
||||
if (confidence > 0.7) {
|
||||
facesDetected = 1;
|
||||
strategy = 'face_centered';
|
||||
} else {
|
||||
strategy = 'smart_crop';
|
||||
}
|
||||
|
||||
cropInfo = this.calculateSmartCrop(width, height, focalPoints);
|
||||
|
||||
console.log(`✅ Smart crop calculated with confidence: ${confidence.toFixed(2)}`);
|
||||
} else {
|
||||
cropInfo = this.calculateCenterCrop(width, height);
|
||||
strategy = 'center_crop';
|
||||
|
||||
console.log(`📐 Center crop calculated (no focal points detected)`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
faces_detected: facesDetected,
|
||||
primary_face_confidence: confidence,
|
||||
crop_info: cropInfo,
|
||||
original_dimensions: { width, height },
|
||||
processing_strategy: strategy
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error(`❌ Error processing thumbnail ${thumbnailUrl}:`, error);
|
||||
return {
|
||||
success: false,
|
||||
faces_detected: 0,
|
||||
primary_face_confidence: 0,
|
||||
crop_info: { x: 0, y: 0, width: 0, height: 0 },
|
||||
original_dimensions: { width: 0, height: 0 },
|
||||
processing_strategy: 'center_crop',
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate CSS object-position
|
||||
*/
|
||||
public calculateObjectPosition(
|
||||
cropInfo: { x: number; y: number; width: number; height: number },
|
||||
originalDimensions: { width: number; height: number }
|
||||
): string {
|
||||
if (!cropInfo || !originalDimensions.width || !originalDimensions.height) {
|
||||
return 'center center';
|
||||
}
|
||||
|
||||
const cropCenterX = cropInfo.x + cropInfo.width / 2;
|
||||
const cropCenterY = cropInfo.y + cropInfo.height / 2;
|
||||
|
||||
const positionX = (cropCenterX / originalDimensions.width) * 100;
|
||||
const positionY = (cropCenterY / originalDimensions.height) * 100;
|
||||
|
||||
return `${Math.max(0, Math.min(100, positionX)).toFixed(1)}% ${Math.max(0, Math.min(100, positionY)).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get optimized thumbnail information
|
||||
*/
|
||||
public async getOptimizedThumbnailInfo(thumbnailUrl: string) {
|
||||
try {
|
||||
const result = await this.processThumbnail(thumbnailUrl);
|
||||
|
||||
return {
|
||||
originalUrl: thumbnailUrl,
|
||||
faceCenterPosition: this.calculateObjectPosition(result.crop_info, result.original_dimensions),
|
||||
facesDetected: result.faces_detected,
|
||||
confidence: result.primary_face_confidence,
|
||||
strategy: result.processing_strategy,
|
||||
processedSuccessfully: result.success
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error optimizing thumbnail:', error);
|
||||
return {
|
||||
originalUrl: thumbnailUrl,
|
||||
faceCenterPosition: 'center center',
|
||||
facesDetected: 0,
|
||||
confidence: 0,
|
||||
strategy: 'center_crop' as const,
|
||||
processedSuccessfully: false
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const smartThumbnailService = new SmartThumbnailService();
|
||||
+50
-4
@@ -640,6 +640,7 @@ export class MemStorage implements IStorage {
|
||||
class BunnyStorage implements IStorage {
|
||||
private bunnyService: BunnyService;
|
||||
private viewsCache: Map<string, number> = new Map();
|
||||
private faceDataCache: Map<string, { faceCenterPosition?: string; facesDetected?: number; faceConfidence?: number }> = new Map();
|
||||
|
||||
constructor() {
|
||||
this.bunnyService = new BunnyService();
|
||||
@@ -649,7 +650,12 @@ class BunnyStorage implements IStorage {
|
||||
console.log(`Fetching videos from cache: limit=${limit}, offset=${offset}, search=${search}`);
|
||||
const result = videoSyncService.getVideos(limit, offset, search);
|
||||
console.log(`Returning ${result.videos.length} videos from cache (age: ${result.cacheAge}ms)`);
|
||||
return result.videos;
|
||||
|
||||
// Apply face detection data from cache
|
||||
return result.videos.map(video => {
|
||||
const faceData = this.faceDataCache.get(video.id);
|
||||
return faceData ? { ...video, ...faceData } : video;
|
||||
});
|
||||
}
|
||||
|
||||
async getVideo(id: string): Promise<Video | undefined> {
|
||||
@@ -660,7 +666,10 @@ class BunnyStorage implements IStorage {
|
||||
if (this.viewsCache.has(cachedVideo.id)) {
|
||||
cachedVideo.views += this.viewsCache.get(cachedVideo.id)!;
|
||||
}
|
||||
return cachedVideo;
|
||||
|
||||
// Apply face detection data from cache
|
||||
const faceData = this.faceDataCache.get(cachedVideo.id);
|
||||
return faceData ? { ...cachedVideo, ...faceData } : cachedVideo;
|
||||
}
|
||||
|
||||
// Fallback to direct API call
|
||||
@@ -673,7 +682,9 @@ class BunnyStorage implements IStorage {
|
||||
video.views += this.viewsCache.get(video.id)!;
|
||||
}
|
||||
|
||||
return video;
|
||||
// Apply face detection data from cache
|
||||
const faceData = this.faceDataCache.get(video.id);
|
||||
return faceData ? { ...video, ...faceData } : video;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching video ${id} from Bunny:`, error);
|
||||
return undefined;
|
||||
@@ -685,7 +696,42 @@ class BunnyStorage implements IStorage {
|
||||
}
|
||||
|
||||
async updateVideo(id: string, updates: UpdateVideo): Promise<Video | undefined> {
|
||||
throw new Error("Updating videos is not supported with Bunny.net integration");
|
||||
// Allow updating face detection data and views even with Bunny.net integration
|
||||
const allowedUpdates = ['faceCenterPosition', 'facesDetected', 'faceConfidence', 'views'];
|
||||
const filteredUpdates = Object.fromEntries(
|
||||
Object.entries(updates).filter(([key]) => allowedUpdates.includes(key))
|
||||
);
|
||||
|
||||
if (Object.keys(filteredUpdates).length === 0) {
|
||||
throw new Error("Only face detection and view updates are supported with Bunny.net integration");
|
||||
}
|
||||
|
||||
try {
|
||||
// Try to update in database first
|
||||
try {
|
||||
await db.update(videos).set(filteredUpdates as UpdateVideo).where(eq(videos.id, id));
|
||||
} catch (dbError) {
|
||||
console.warn(`Database update failed for video ${id}, using cache:`, dbError);
|
||||
}
|
||||
|
||||
// Update face detection data in cache (always reliable)
|
||||
const { views, ...faceData } = filteredUpdates;
|
||||
if (Object.keys(faceData).length > 0) {
|
||||
const existing = this.faceDataCache.get(id) || {};
|
||||
this.faceDataCache.set(id, { ...existing, ...faceData });
|
||||
}
|
||||
|
||||
// Update views cache
|
||||
if (views !== undefined) {
|
||||
this.viewsCache.set(id, (this.viewsCache.get(id) || 0) + (views - (this.viewsCache.get(id) || 0)));
|
||||
}
|
||||
|
||||
// Return updated video
|
||||
return await this.getVideo(id);
|
||||
} catch (error) {
|
||||
console.error(`Error updating video ${id}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async deleteVideo(id: string): Promise<boolean> {
|
||||
|
||||
Reference in New Issue
Block a user