Ensure all videos are consistently stored in the database by fetching and synchronizing them

Implement a paginated fetching mechanism for all videos from an external service and synchronize them to the PostgreSQL database, handling both new insertions and updates for existing video records.

Replit-Commit-Author: Agent
Replit-Commit-Session-Id: 2eb1084e-b728-4449-9231-f1665924c8d5
Replit-Commit-Checkpoint-Type: full_checkpoint

server/videoSync.ts

@@ -1,4 +1,7 @@
 import { BunnyService } from './bunny';
+import { db } from './db';
+import { videos } from '@shared/schema';
+import { eq, sql } from 'drizzle-orm';
 
 interface VideoSyncCache {
   videos: any[];
@@ -19,10 +22,119 @@ class VideoSyncService {
     this.bunnyService = new BunnyService();
   }
 
+  private async getAllVideos(): Promise<any[]> {
+    let allVideos: any[] = [];
+    let page = 1;
+    const itemsPerPage = 100; // Maximum per request
+    let hasMore = true;
+    
+    console.log('📥 Starting to fetch all videos with pagination...');
+    
+    while (hasMore) {
+      try {
+        console.log(`📄 Fetching page ${page}...`);
+        const result = await this.bunnyService.getVideos(page, itemsPerPage);
+        
+        allVideos = allVideos.concat(result.videos);
+        
+        // Check if there are more pages
+        const totalFetched = (page - 1) * itemsPerPage + result.videos.length;
+        hasMore = totalFetched < result.total && result.videos.length === itemsPerPage;
+        
+        console.log(`📊 Page ${page}: ${result.videos.length} videos (Total so far: ${allVideos.length}/${result.total})`);
+        page++;
+        
+        // Safety limit to prevent infinite loops
+        if (page > 100) {
+          console.log('⚠️ Reached page limit (100), stopping fetch');
+          break;
+        }
+        
+      } catch (error) {
+        console.error(`❌ Error fetching page ${page}:`, error);
+        break;
+      }
+    }
+    
+    console.log(`✅ Completed pagination fetch: ${allVideos.length} total videos`);
+    return allVideos;
+  }
+
+  private async syncVideosToDatabase() {
+    if (this.cache.videos.length === 0) {
+      console.log('⚠️ No videos in cache to sync to database');
+      return;
+    }
+
+    console.log('🔄 Syncing cached videos to PostgreSQL database...');
+    const startTime = Date.now();
+    
+    let insertedCount = 0;
+    let updatedCount = 0;
+    
+    try {
+      for (const video of this.cache.videos) {
+        try {
+          // Check if video exists in database using raw SQL
+          const existingVideo = await db.execute(sql`SELECT id FROM videos WHERE id = ${video.id} LIMIT 1`);
+          
+          const videoData = {
+            id: video.id,
+            title: video.title,
+            description: video.description || '',
+            thumbnailUrl: video.thumbnailUrl,
+            customThumbnailUrl: video.customThumbnailUrl || null,
+            videoUrl: video.videoUrl,
+            duration: video.duration,
+            views: video.views,
+            category: video.category || '',
+            tags: Array.isArray(video.tags) ? video.tags : [],
+            isPublic: video.isPublic !== false,
+            createdAt: video.createdAt ? new Date(video.createdAt) : new Date(),
+            updatedAt: new Date(),
+          };
+
+          if (existingVideo.rows.length === 0) {
+            // Insert new video using raw SQL to avoid schema issues
+            await db.execute(sql`
+              INSERT INTO videos (id, title, description, thumbnail_url, video_url, duration, views, category, custom_thumbnail_url, tags, is_public, created_at, updated_at)
+              VALUES (${videoData.id}, ${videoData.title}, ${videoData.description}, ${videoData.thumbnailUrl}, ${videoData.videoUrl}, ${videoData.duration}, ${videoData.views}, ${videoData.category}, ${videoData.customThumbnailUrl}, ${'{' + videoData.tags.join(',') + '}'}, ${videoData.isPublic}, ${videoData.createdAt.toISOString()}, ${videoData.updatedAt.toISOString()})
+            `);
+            insertedCount++;
+          } else {
+            // Update existing video using raw SQL
+            await db.execute(sql`
+              UPDATE videos 
+              SET title = ${videoData.title}, description = ${videoData.description}, thumbnail_url = ${videoData.thumbnailUrl}, 
+                  video_url = ${videoData.videoUrl}, duration = ${videoData.duration}, views = ${videoData.views}, 
+                  category = ${videoData.category}, custom_thumbnail_url = ${videoData.customThumbnailUrl}, 
+                  tags = ${'{' + videoData.tags.join(',') + '}'}, is_public = ${videoData.isPublic}, updated_at = ${videoData.updatedAt.toISOString()}
+              WHERE id = ${video.id}
+            `);
+            updatedCount++;
+          }
+        } catch (error) {
+          console.error(`❌ Failed to sync video ${video.id}:`, error);
+        }
+      }
+      
+      const duration = Date.now() - startTime;
+      console.log(`✅ Database sync completed in ${duration}ms:`);
+      console.log(`   📥 Inserted: ${insertedCount} new videos`);
+      console.log(`   🔄 Updated: ${updatedCount} existing videos`);
+      console.log(`   📊 Database total: ${insertedCount + updatedCount} videos`);
+      console.log(`   🎥 Cache total: ${this.cache.videos.length} videos`);
+      
+    } catch (error) {
+      console.error('❌ Database sync failed:', error);
+    }
+  }
+
   async initialize() {
     console.log('🔄 Initializing video sync service...');
     try {
       await this.syncVideos();
+      await this.syncVideosToDatabase();
       this.startPeriodicSync();
       console.log('✅ Video sync service initialized successfully');
     } catch (error) {
@@ -41,14 +153,14 @@ class VideoSyncService {
     const startTime = Date.now();
     
     try {
-      console.log('🔍 Fetching latest videos from Bunny.net...');
+      console.log('🔍 Fetching ALL videos from Bunny.net...');
-      const result = await this.bunnyService.getVideos(1, 100);
+      const allVideos = await this.getAllVideos();
       
-      this.cache.videos = result.videos;
+      this.cache.videos = allVideos;
       this.cache.lastUpdate = Date.now();
       
       const duration = Date.now() - startTime;
-      console.log(`✅ Video sync completed: ${result.videos.length} videos cached in ${duration}ms`);
+      console.log(`✅ Video sync completed: ${allVideos.length} videos cached in ${duration}ms`);
     } catch (error) {
       console.error('❌ Video sync failed:', error);
     } finally {