videofolxtv/server/face-detection-js.ts

import * as faceapi from 'face-api.js';
import * as canvas from 'canvas';
import sharp from 'sharp';
import path from 'path';
import { promises as fs } from 'fs';

// Setup face-api.js for Node.js environment
const { Canvas, Image, ImageData } = canvas;
// @ts-ignore
faceapi.env.monkeyPatch({ Canvas, Image, ImageData });

export interface FaceDetectionResult {
  success: boolean;
  faces_detected: number;
  primary_face_confidence: number;
  crop_info: {
    x: number;
    y: number;
    width: number;
    height: number;
  };
  original_dimensions: {
    width: number;
    height: number;
  };
  processing_strategy: 'face_centered' | 'center_crop';
  error?: string;
}

export class JavaScriptFaceDetectionService {
  private modelsLoaded = false;

  constructor() {
    this.loadModels();
  }

  private async loadModels(): Promise<void> {
    if (this.modelsLoaded) return;

    try {
      // Load face detection models from face-api.js
      // Using smaller models for better performance
      await faceapi.nets.tinyFaceDetector.loadFromDisk('./node_modules/face-api.js/weights');
      await faceapi.nets.faceLandmark68TinyNet.loadFromDisk('./node_modules/face-api.js/weights');
      
      this.modelsLoaded = true;
      console.log('✅ Face detection models loaded successfully');
    } catch (error) {
      console.error('❌ Failed to load face detection models:', error);
      // Fallback: continue without face detection
    }
  }

  private async downloadImage(imageUrl: string): Promise<{ buffer: Buffer; width: number; height: number }> {
    // Use Node.js built-in fetch (available from Node 18+)
    const response = await fetch(imageUrl);
    if (!response.ok) {
      throw new Error(`Failed to download image: ${response.statusText}`);
    }
    
    const buffer = Buffer.from(await response.arrayBuffer());
    const metadata = await sharp(buffer).metadata();
    
    return {
      buffer,
      width: metadata.width || 0,
      height: metadata.height || 0
    };
  }

  private async detectFacesInImage(imageBuffer: Buffer): Promise<any[]> {
    if (!this.modelsLoaded) {
      await this.loadModels();
      if (!this.modelsLoaded) {
        return []; // Return empty array if models couldn't load
      }
    }

    try {
      // Convert image buffer to canvas
      const img = new Image();
      img.src = imageBuffer;
      
      // Wait for image to load
      await new Promise((resolve, reject) => {
        img.onload = resolve;
        img.onerror = reject;
      });

      // Detect faces with tiny face detector for performance
      const detections = await faceapi
        .detectAllFaces(img, new faceapi.TinyFaceDetectorOptions())
        .withFaceLandmarks(true);

      return detections;
    } catch (error) {
      console.error('Error detecting faces:', error);
      return [];
    }
  }

  private calculateFaceCenteredCrop(
    imageWidth: number,
    imageHeight: number,
    faceBox: { x: number; y: number; width: number; height: number },
    targetAspect: number = 9/16
  ): { x: number; y: number; width: number; height: number } {
    const faceCenterX = faceBox.x + faceBox.width / 2;
    const faceCenterY = faceBox.y + faceBox.height / 2;

    let cropWidth: number, cropHeight: number, cropX: number, cropY: number;

    if (imageWidth / imageHeight > targetAspect) {
      // Image is wider than target aspect ratio
      cropWidth = Math.floor(imageHeight * targetAspect);
      cropHeight = imageHeight;
      cropX = Math.max(0, Math.min(faceCenterX - cropWidth / 2, imageWidth - cropWidth));
      cropY = 0;
    } else {
      // Image is taller than target aspect ratio
      cropWidth = imageWidth;
      cropHeight = Math.floor(imageWidth / targetAspect);
      cropX = 0;
      cropY = Math.max(0, Math.min(faceCenterY - cropHeight / 2, imageHeight - cropHeight));
    }

    return { x: cropX, y: cropY, width: cropWidth, height: cropHeight };
  }

  private calculateCenterCrop(
    imageWidth: number,
    imageHeight: number,
    targetAspect: number = 9/16
  ): { x: number; y: number; width: number; height: number } {
    let cropWidth: number, cropHeight: number, cropX: number, cropY: number;

    if (imageWidth / imageHeight > targetAspect) {
      cropWidth = Math.floor(imageHeight * targetAspect);
      cropHeight = imageHeight;
      cropX = Math.floor((imageWidth - cropWidth) / 2);
      cropY = 0;
    } else {
      cropWidth = imageWidth;
      cropHeight = Math.floor(imageWidth / targetAspect);
      cropX = 0;
      cropY = Math.floor((imageHeight - cropHeight) / 2);
    }

    return { x: cropX, y: cropY, width: cropWidth, height: cropHeight };
  }

  public async processThumbnail(thumbnailUrl: string): Promise<FaceDetectionResult> {
    try {
      console.log(`🔍 Processing thumbnail: ${thumbnailUrl}`);
      
      // Download and analyze the image
      const { buffer, width, height } = await this.downloadImage(thumbnailUrl);
      
      if (width === 0 || height === 0) {
        throw new Error('Invalid image dimensions');
      }

      // Detect faces in the image
      const faces = await this.detectFacesInImage(buffer);
      
      console.log(`👥 Found ${faces.length} faces in image`);

      let cropInfo: { x: number; y: number; width: number; height: number };
      let strategy: 'face_centered' | 'center_crop';
      let confidence = 0;

      if (faces.length > 0) {
        // Use the first (most confident) face for centering
        const primaryFace = faces[0];
        const faceBox = primaryFace.detection.box;
        confidence = primaryFace.detection.score;
        
        cropInfo = this.calculateFaceCenteredCrop(width, height, faceBox);
        strategy = 'face_centered';
        
        console.log(`✅ Face-centered crop calculated with confidence: ${confidence.toFixed(2)}`);
      } else {
        // No faces detected, use center crop
        cropInfo = this.calculateCenterCrop(width, height);
        strategy = 'center_crop';
        
        console.log(`📐 Center crop calculated (no faces detected)`);
      }

      return {
        success: true,
        faces_detected: faces.length,
        primary_face_confidence: confidence,
        crop_info: cropInfo,
        original_dimensions: { width, height },
        processing_strategy: strategy
      };

    } catch (error) {
      console.error(`❌ Error processing thumbnail ${thumbnailUrl}:`, error);
      return {
        success: false,
        faces_detected: 0,
        primary_face_confidence: 0,
        crop_info: { x: 0, y: 0, width: 0, height: 0 },
        original_dimensions: { width: 0, height: 0 },
        processing_strategy: 'center_crop',
        error: error instanceof Error ? error.message : String(error)
      };
    }
  }

  public calculateObjectPosition(
    cropInfo: { x: number; y: number; width: number; height: number },
    originalDimensions: { width: number; height: number }
  ): string {
    if (!cropInfo || !originalDimensions.width || !originalDimensions.height) {
      return 'center center';
    }

    // Calculate the percentage position of the crop center
    const cropCenterX = cropInfo.x + cropInfo.width / 2;
    const cropCenterY = cropInfo.y + cropInfo.height / 2;
    
    const positionX = (cropCenterX / originalDimensions.width) * 100;
    const positionY = (cropCenterY / originalDimensions.height) * 100;
    
    return `${Math.max(0, Math.min(100, positionX)).toFixed(1)}% ${Math.max(0, Math.min(100, positionY)).toFixed(1)}%`;
  }

  public async getOptimizedThumbnailInfo(thumbnailUrl: string) {
    try {
      const result = await this.processThumbnail(thumbnailUrl);
      
      return {
        originalUrl: thumbnailUrl,
        faceCenterPosition: this.calculateObjectPosition(result.crop_info, result.original_dimensions),
        facesDetected: result.faces_detected,
        confidence: result.primary_face_confidence,
        strategy: result.processing_strategy,
        processedSuccessfully: result.success
      };
    } catch (error) {
      console.error('Error optimizing thumbnail:', error);
      return {
        originalUrl: thumbnailUrl,
        faceCenterPosition: 'center center',
        facesDetected: 0,
        confidence: 0,
        strategy: 'center_crop' as const,
        processedSuccessfully: false
      };
    }
  }
}

// Export singleton instance
export const jsFaceDetectionService = new JavaScriptFaceDetectionService();