From 8a8a94b8fd8511e95f41a3b7b542f6f2ccdf65f2 Mon Sep 17 00:00:00 2001 From: sebastjanartic <45803536-sebastjanartic@users.noreply.replit.com> Date: Thu, 28 Aug 2025 17:55:46 +0000 Subject: [PATCH] Ensure all videos are consistently stored in the database by fetching and synchronizing them Implement a paginated fetching mechanism for all videos from an external service and synchronize them to the PostgreSQL database, handling both new insertions and updates for existing video records. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 2eb1084e-b728-4449-9231-f1665924c8d5 Replit-Commit-Checkpoint-Type: full_checkpoint --- server/videoSync.ts | 120 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 116 insertions(+), 4 deletions(-) diff --git a/server/videoSync.ts b/server/videoSync.ts index 65df977..3ebeee0 100644 --- a/server/videoSync.ts +++ b/server/videoSync.ts @@ -1,4 +1,7 @@ import { BunnyService } from './bunny'; +import { db } from './db'; +import { videos } from '@shared/schema'; +import { eq, sql } from 'drizzle-orm'; interface VideoSyncCache { videos: any[]; @@ -19,10 +22,119 @@ class VideoSyncService { this.bunnyService = new BunnyService(); } + private async getAllVideos(): Promise { + let allVideos: any[] = []; + let page = 1; + const itemsPerPage = 100; // Maximum per request + let hasMore = true; + + console.log('📥 Starting to fetch all videos with pagination...'); + + while (hasMore) { + try { + console.log(`📄 Fetching page ${page}...`); + const result = await this.bunnyService.getVideos(page, itemsPerPage); + + allVideos = allVideos.concat(result.videos); + + // Check if there are more pages + const totalFetched = (page - 1) * itemsPerPage + result.videos.length; + hasMore = totalFetched < result.total && result.videos.length === itemsPerPage; + + console.log(`📊 Page ${page}: ${result.videos.length} videos (Total so far: ${allVideos.length}/${result.total})`); + page++; + + // Safety limit to prevent infinite loops + if (page > 100) { + console.log('⚠️ Reached page limit (100), stopping fetch'); + break; + } + + } catch (error) { + console.error(`❌ Error fetching page ${page}:`, error); + break; + } + } + + console.log(`✅ Completed pagination fetch: ${allVideos.length} total videos`); + return allVideos; + } + + private async syncVideosToDatabase() { + if (this.cache.videos.length === 0) { + console.log('⚠️ No videos in cache to sync to database'); + return; + } + + console.log('🔄 Syncing cached videos to PostgreSQL database...'); + const startTime = Date.now(); + + let insertedCount = 0; + let updatedCount = 0; + + try { + for (const video of this.cache.videos) { + try { + // Check if video exists in database using raw SQL + const existingVideo = await db.execute(sql`SELECT id FROM videos WHERE id = ${video.id} LIMIT 1`); + + const videoData = { + id: video.id, + title: video.title, + description: video.description || '', + thumbnailUrl: video.thumbnailUrl, + customThumbnailUrl: video.customThumbnailUrl || null, + videoUrl: video.videoUrl, + duration: video.duration, + views: video.views, + category: video.category || '', + tags: Array.isArray(video.tags) ? video.tags : [], + isPublic: video.isPublic !== false, + createdAt: video.createdAt ? new Date(video.createdAt) : new Date(), + updatedAt: new Date(), + }; + + if (existingVideo.rows.length === 0) { + // Insert new video using raw SQL to avoid schema issues + await db.execute(sql` + INSERT INTO videos (id, title, description, thumbnail_url, video_url, duration, views, category, custom_thumbnail_url, tags, is_public, created_at, updated_at) + VALUES (${videoData.id}, ${videoData.title}, ${videoData.description}, ${videoData.thumbnailUrl}, ${videoData.videoUrl}, ${videoData.duration}, ${videoData.views}, ${videoData.category}, ${videoData.customThumbnailUrl}, ${'{' + videoData.tags.join(',') + '}'}, ${videoData.isPublic}, ${videoData.createdAt.toISOString()}, ${videoData.updatedAt.toISOString()}) + `); + insertedCount++; + } else { + // Update existing video using raw SQL + await db.execute(sql` + UPDATE videos + SET title = ${videoData.title}, description = ${videoData.description}, thumbnail_url = ${videoData.thumbnailUrl}, + video_url = ${videoData.videoUrl}, duration = ${videoData.duration}, views = ${videoData.views}, + category = ${videoData.category}, custom_thumbnail_url = ${videoData.customThumbnailUrl}, + tags = ${'{' + videoData.tags.join(',') + '}'}, is_public = ${videoData.isPublic}, updated_at = ${videoData.updatedAt.toISOString()} + WHERE id = ${video.id} + `); + updatedCount++; + } + } catch (error) { + console.error(`❌ Failed to sync video ${video.id}:`, error); + } + } + + const duration = Date.now() - startTime; + console.log(`✅ Database sync completed in ${duration}ms:`); + console.log(` 📥 Inserted: ${insertedCount} new videos`); + console.log(` 🔄 Updated: ${updatedCount} existing videos`); + console.log(` 📊 Database total: ${insertedCount + updatedCount} videos`); + console.log(` 🎥 Cache total: ${this.cache.videos.length} videos`); + + } catch (error) { + console.error('❌ Database sync failed:', error); + } + } + async initialize() { console.log('🔄 Initializing video sync service...'); try { await this.syncVideos(); + await this.syncVideosToDatabase(); this.startPeriodicSync(); console.log('✅ Video sync service initialized successfully'); } catch (error) { @@ -41,14 +153,14 @@ class VideoSyncService { const startTime = Date.now(); try { - console.log('🔍 Fetching latest videos from Bunny.net...'); - const result = await this.bunnyService.getVideos(1, 100); + console.log('🔍 Fetching ALL videos from Bunny.net...'); + const allVideos = await this.getAllVideos(); - this.cache.videos = result.videos; + this.cache.videos = allVideos; this.cache.lastUpdate = Date.now(); const duration = Date.now() - startTime; - console.log(`✅ Video sync completed: ${result.videos.length} videos cached in ${duration}ms`); + console.log(`✅ Video sync completed: ${allVideos.length} videos cached in ${duration}ms`); } catch (error) { console.error('❌ Video sync failed:', error); } finally {