Improve image display by intelligently focusing on subjects

Implement server-side AI analysis to detect focal points in images, ensuring faces and key subjects are not cut off.

Replit-Commit-Author: Agent
Replit-Commit-Session-Id: 517dfa7b-26ac-463d-a6e1-a58c6df97188
Replit-Commit-Checkpoint-Type: full_checkpoint
Replit-Commit-Event-Id: c1dc1745-dcd7-4247-a1e3-8dd414e4592b
Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/f209e72a-0939-48fa-84fc-57854de71967/517dfa7b-26ac-463d-a6e1-a58c6df97188/drGbo1a
Replit-Helium-Checkpoint-Created: true

client/src/pages/home.tsx

@@ -14,12 +14,20 @@ import { RecipeWidget } from "@/components/recipe-widget";
 import { NewsWidget } from "@/components/news-widget";
 import { useState, useEffect, useCallback, useRef } from "react";
 
-function SmartImage({ src, alt, className = "" }: { src: string; alt: string; className?: string }) {
+function useFocalPoints() {
+  const { data } = useQuery<Record<string, { x: number; y: number }>>({
+    queryKey: ["/api/focal-points"],
+    staleTime: Infinity,
+  });
+  return data || {};
+}
+
+function SmartImage({ src, alt, className = "", focalPoints }: { src: string; alt: string; className?: string; focalPoints?: Record<string, { x: number; y: number }> }) {
   const imgRef = useRef<HTMLImageElement>(null);
   const [isPortrait, setIsPortrait] = useState(false);
 
   useEffect(() => {
-    const img = new Image();
+    const img = new window.Image();
     img.onload = () => {
       if (img.naturalHeight > img.naturalWidth * 1.2) {
         setIsPortrait(true);
@@ -28,6 +36,11 @@ function SmartImage({ src, alt, className = "" }: { src: string; alt: string; cl
     img.src = src;
   }, [src]);
 
+  const fp = focalPoints?.[src];
+  const posStyle = fp
+    ? { objectPosition: `${fp.x}% ${fp.y}%` }
+    : { objectPosition: "center 20%" };
+
   if (isPortrait) {
     return (
       <div className="absolute inset-0 overflow-hidden">
@@ -38,7 +51,7 @@ function SmartImage({ src, alt, className = "" }: { src: string; alt: string; cl
   }
 
   return (
-    <img ref={imgRef} src={src} alt={alt} className={`w-full h-full object-cover absolute inset-0 ${className}`} style={{ objectPosition: "center 40%" }} />
+    <img ref={imgRef} src={src} alt={alt} className={`w-full h-full object-cover absolute inset-0 ${className}`} style={posStyle} />
   );
 }
 
@@ -66,13 +79,13 @@ function timeAgo(date: Date): string {
   return format(date, "d. MMM yyyy", { locale: de });
 }
 
-function HeroCard({ article }: { article: Article }) {
+function HeroCard({ article, focalPoints }: { article: Article; focalPoints?: Record<string, { x: number; y: number }> }) {
   const isVideo = article.category === "Video";
   return (
     <Link href={`/article/${article.slug}`}>
       <div className="relative group cursor-pointer rounded-lg overflow-hidden h-full" data-testid={`card-hero-${article.id}`}>
         <div className="relative h-full min-h-[280px]">
-          <SmartImage src={article.coverImage || "/images/article-1.png"} alt={article.title} className="transition-transform duration-700 group-hover:scale-105" />
+          <SmartImage src={article.coverImage || "/images/article-1.png"} alt={article.title} className="transition-transform duration-700 group-hover:scale-105" focalPoints={focalPoints} />
           {isVideo && (
             <div className="absolute inset-0 flex items-center justify-center z-10">
               <div className="w-14 h-14 rounded-full bg-primary/90 flex items-center justify-center group-hover:bg-primary transition-colors shadow-lg">
@@ -131,7 +144,7 @@ function MediumCard({ article }: { article: Article }) {
       <div className="relative group cursor-pointer rounded-lg overflow-hidden h-full bg-card border border-card-border flex flex-col" data-testid={`card-medium-${article.id}`}>
         <div className="relative flex-shrink-0">
           <div className="overflow-hidden">
-            <img src={thumbUrl(article.coverImage)} alt={article.title} className="w-full aspect-video object-cover object-center transition-transform duration-500 group-hover:scale-105" loading="lazy" />
+            <img src={thumbUrl(article.coverImage)} alt={article.title} className="w-full aspect-video object-cover transition-transform duration-500 group-hover:scale-105" style={{ objectPosition: "center 20%" }} loading="lazy" />
           </div>
           {isVideo && (
             <div className="absolute inset-0 flex items-center justify-center">
@@ -165,7 +178,7 @@ function SideCard({ article }: { article: Article }) {
       <div className="relative group cursor-pointer rounded-lg overflow-hidden bg-card border border-card-border h-full flex flex-col" data-testid={`card-side-${article.id}`}>
         <div className="relative flex-shrink-0 flex-1 min-h-0">
           <div className="overflow-hidden h-full">
-            <img src={thumbUrl(article.coverImage)} alt={article.title} className="w-full h-full object-cover object-center transition-transform duration-500 group-hover:scale-105" loading="lazy" />
+            <img src={thumbUrl(article.coverImage)} alt={article.title} className="w-full h-full object-cover transition-transform duration-500 group-hover:scale-105" style={{ objectPosition: "center 20%" }} loading="lazy" />
           </div>
           {isVideo && (
             <div className="absolute inset-0 flex items-center justify-center">
@@ -229,7 +242,7 @@ function TopStoriesList({ articles }: { articles: Article[] }) {
   );
 }
 
-function FeaturedCarousel({ articles, popular, galleryImages }: { articles: Article[]; popular?: Article[]; galleryImages?: GalleryImage[] }) {
+function FeaturedCarousel({ articles, popular, galleryImages, focalPoints }: { articles: Article[]; popular?: Article[]; galleryImages?: GalleryImage[]; focalPoints?: Record<string, { x: number; y: number }> }) {
   const hasGallery = galleryImages && galleryImages.length > 0;
   const articlePages = Math.min(5, Math.max(1, articles.length));
   const total = articlePages + (hasGallery ? 1 : 0);
@@ -271,7 +284,7 @@ function FeaturedCarousel({ articles, popular, galleryImages }: { articles: Arti
           {isGalleryPage && galleryImages ? (
             <GalleryHeroCard images={galleryImages.slice(0, 30)} />
           ) : hero ? (
-            <HeroCard article={hero} />
+            <HeroCard article={hero} focalPoints={focalPoints} />
           ) : null}
         </div>
         <div className="lg:col-span-3 grid grid-cols-1 gap-3 grid-rows-2 lg:h-[420px]">
@@ -323,6 +336,8 @@ export default function Home() {
     queryKey: ["/api/gallery"],
   });
 
+  const focalPoints = useFocalPoints();
+
   if (isLoading || !articles) {
     return (
       <div className="min-h-screen bg-background">
@@ -343,7 +358,7 @@ export default function Home() {
       <Header />
       <main className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-4 space-y-4">
 
-        <FeaturedCarousel articles={articles} popular={popular} galleryImages={galleryImages} />
+        <FeaturedCarousel articles={articles} popular={popular} galleryImages={galleryImages} focalPoints={focalPoints} />
 
         <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-4 gap-4">
           {row2Left.map((a) => (

server/focal-point.ts

@@ -0,0 +1,120 @@
+import OpenAI from "openai";
+import fs from "fs";
+import path from "path";
+
+const openai = new OpenAI({
+  apiKey: process.env.AI_INTEGRATIONS_OPENAI_API_KEY,
+  baseURL: process.env.AI_INTEGRATIONS_OPENAI_BASE_URL,
+});
+
+const cache = new Map<string, { x: number; y: number }>();
+
+export async function analyzeFocalPoint(imagePath: string): Promise<{ x: number; y: number }> {
+  const originalPath = imagePath;
+  if (cache.has(originalPath)) {
+    return cache.get(originalPath)!;
+  }
+
+  try {
+    let imageData: string;
+    let mimeType = "image/webp";
+
+    if (imagePath.startsWith("/uploads/")) {
+      const localPath = path.join(process.cwd(), "client/public", imagePath);
+      if (!fs.existsSync(localPath)) {
+        throw new Error(`File not found: ${localPath}`);
+      }
+      const buffer = fs.readFileSync(localPath);
+      imageData = buffer.toString("base64");
+      if (localPath.endsWith(".jpg") || localPath.endsWith(".jpeg")) mimeType = "image/jpeg";
+      else if (localPath.endsWith(".png")) mimeType = "image/png";
+    } else if (imagePath.startsWith("http")) {
+      const response = await openai.chat.completions.create({
+        model: "gpt-4o-mini",
+        messages: [
+          {
+            role: "system",
+            content: "You analyze images to find the main subject (person's face or group center). Return ONLY a JSON object with x and y as percentages (0-100). x=50 means horizontal center, y=20 means near top. Example: {\"x\":50,\"y\":30}"
+          },
+          {
+            role: "user",
+            content: [
+              { type: "text", text: "Where is the main subject (face/person) in this image? Return only JSON." },
+              { type: "image_url", image_url: { url: imagePath, detail: "low" } }
+            ]
+          }
+        ],
+        max_tokens: 50,
+      });
+      const text = response.choices[0]?.message?.content?.trim() || "";
+      const match = text.match(/\{[^}]+\}/);
+      if (match) {
+        const parsed = JSON.parse(match[0]);
+        const point = {
+          x: Math.max(0, Math.min(100, Number(parsed.x) || 50)),
+          y: Math.max(0, Math.min(100, Number(parsed.y) || 30)),
+        };
+        cache.set(originalPath, point);
+        return point;
+      }
+      throw new Error("Could not parse response");
+    } else {
+      throw new Error(`Unsupported path: ${imagePath}`);
+    }
+
+    const dataUrl = `data:${mimeType};base64,${imageData}`;
+
+    const response = await openai.chat.completions.create({
+      model: "gpt-4o-mini",
+      messages: [
+        {
+          role: "system",
+          content: "You analyze images to find the main subject (person's face or group center). Return ONLY a JSON object with x and y as percentages (0-100) indicating where the main subject/face is located. x=50 means horizontal center, y=20 means near the top. Example: {\"x\":45,\"y\":30}"
+        },
+        {
+          role: "user",
+          content: [
+            { type: "text", text: "Where is the main subject (face/person) in this image? Return only JSON with x,y percentages." },
+            { type: "image_url", image_url: { url: dataUrl, detail: "low" } }
+          ]
+        }
+      ],
+      max_tokens: 50,
+    });
+
+    const text = response.choices[0]?.message?.content?.trim() || "";
+    console.log(`[focal-point] ${imagePath}: AI response = ${text}`);
+    const match = text.match(/\{[^}]+\}/);
+    if (match) {
+      const parsed = JSON.parse(match[0]);
+      const point = {
+        x: Math.max(0, Math.min(100, Number(parsed.x) || 50)),
+        y: Math.max(0, Math.min(100, Number(parsed.y) || 30)),
+      };
+      cache.set(originalPath, point);
+      return point;
+    }
+  } catch (e) {
+    console.log("[focal-point] AI analysis failed for", imagePath, ":", (e as Error).message);
+  }
+
+  const fallback = { x: 50, y: 30 };
+  cache.set(originalPath, fallback);
+  return fallback;
+}
+
+export async function analyzeAllArticleImages(articles: Array<{ coverImage: string | null }>) {
+  for (const article of articles) {
+    if (!article.coverImage) continue;
+    await analyzeFocalPoint(article.coverImage);
+  }
+  console.log("[focal-point] Analyzed", cache.size, "images");
+}
+
+export function getCachedFocalPoints(): Record<string, { x: number; y: number }> {
+  const result: Record<string, { x: number; y: number }> = {};
+  for (const [key, value] of cache.entries()) {
+    result[key] = value;
+  }
+  return result;
+}

server/routes.ts

@@ -4,6 +4,7 @@ import { storage } from "./storage";
 import { insertArticleSchema } from "@shared/schema";
 import { seedDatabase } from "./seed";
 import { generateDailyHoroscopes, getHoroscopesForToday, getOrGenerateHoroscope } from "./horoscope-generator";
+import { analyzeAllArticleImages, getCachedFocalPoints } from "./focal-point";
 import multer from "multer";
 import path from "path";
 import fs from "fs";
@@ -42,6 +43,16 @@ export async function registerRoutes(
     console.error("Background horoscope generation failed:", err.message)
   );
 
+  storage.getArticles().then((articles) => {
+    analyzeAllArticleImages(articles).catch(err => 
+      console.log("[focal-point] Analysis skipped:", err.message)
+    );
+  });
+
+  app.get("/api/focal-points", (_req, res) => {
+    res.json(getCachedFocalPoints());
+  });
+
   app.get("/api/articles", async (_req, res) => {
     const articles = await storage.getArticles();
     res.json(articles);