From 2a395fd12d2f5eda6845d54de2446fb7b91fb75f Mon Sep 17 00:00:00 2001 From: sebastjanartic <45803536-sebastjanartic@users.noreply.replit.com> Date: Mon, 1 Sep 2025 17:50:12 +0000 Subject: [PATCH] Improve video synchronization reliability and error handling Update server/videoSync.ts to include database connectivity checks, batch processing for videos, and error counting to enhance the robustness of the video synchronization service. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 890577b1-c154-40a4-a177-a0c6d55320c3 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/8cc42625-c1f5-4e43-99bd-77f2c4dedee2/890577b1-c154-40a4-a177-a0c6d55320c3/heEm7CS --- server/videoSync.ts | 119 +++++++++++++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 41 deletions(-) diff --git a/server/videoSync.ts b/server/videoSync.ts index 36a7eba..d7e6742 100644 --- a/server/videoSync.ts +++ b/server/videoSync.ts @@ -76,50 +76,74 @@ class VideoSyncService { let insertedCount = 0; let updatedCount = 0; + let errorCount = 0; try { - for (const video of this.cache.videos) { - try { - // Check if video exists in database using raw SQL - const existingVideo = await db.execute(sql`SELECT id FROM videos WHERE id = ${video.id} LIMIT 1`); - - const videoData = { - id: video.id, - title: video.title, - description: video.description || '', - thumbnailUrl: video.thumbnailUrl, - customThumbnailUrl: video.customThumbnailUrl || null, - videoUrl: video.videoUrl, - duration: video.duration, - views: video.views, - category: video.category || '', - tags: Array.isArray(video.tags) ? video.tags : [], - isPublic: video.isPublic !== false, - createdAt: video.createdAt ? new Date(video.createdAt) : new Date(), - updatedAt: new Date(), - }; + // First, test database connectivity + await db.execute(sql`SELECT 1`); + console.log('📡 Database connection verified'); + + // Process videos in smaller batches to avoid overwhelming the database + const batchSize = 10; + const totalBatches = Math.ceil(this.cache.videos.length / batchSize); + + for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) { + const start = batchIndex * batchSize; + const end = Math.min(start + batchSize, this.cache.videos.length); + const batch = this.cache.videos.slice(start, end); + + console.log(`📦 Processing batch ${batchIndex + 1}/${totalBatches} (videos ${start + 1}-${end})`); + + for (const video of batch) { + try { + // Check if video exists in database using raw SQL + const existingVideo = await db.execute(sql`SELECT id FROM videos WHERE id = ${video.id} LIMIT 1`); + + const videoData = { + id: video.id, + title: video.title, + description: video.description || '', + thumbnailUrl: video.thumbnailUrl, + customThumbnailUrl: video.customThumbnailUrl || null, + videoUrl: video.videoUrl, + duration: video.duration, + views: video.views, + category: video.category || '', + tags: Array.isArray(video.tags) ? video.tags : [], + isPublic: video.isPublic !== false, + createdAt: video.createdAt ? new Date(video.createdAt) : new Date(), + updatedAt: new Date(), + }; - if (existingVideo.rows.length === 0) { - // Insert new video using raw SQL to avoid schema issues - await db.execute(sql` - INSERT INTO videos (id, title, description, thumbnail_url, video_url, duration, views, category, custom_thumbnail_url, tags, is_public, created_at, updated_at) - VALUES (${videoData.id}, ${videoData.title}, ${videoData.description}, ${videoData.thumbnailUrl}, ${videoData.videoUrl}, ${videoData.duration}, ${videoData.views}, ${videoData.category}, ${videoData.customThumbnailUrl}, ${'{' + videoData.tags.join(',') + '}'}, ${videoData.isPublic}, ${videoData.createdAt.toISOString()}, ${videoData.updatedAt.toISOString()}) - `); - insertedCount++; - } else { - // Update existing video using raw SQL - await db.execute(sql` - UPDATE videos - SET title = ${videoData.title}, description = ${videoData.description}, thumbnail_url = ${videoData.thumbnailUrl}, - video_url = ${videoData.videoUrl}, duration = ${videoData.duration}, views = ${videoData.views}, - category = ${videoData.category}, custom_thumbnail_url = ${videoData.customThumbnailUrl}, - tags = ${'{' + videoData.tags.join(',') + '}'}, is_public = ${videoData.isPublic}, updated_at = ${videoData.updatedAt.toISOString()} - WHERE id = ${video.id} - `); - updatedCount++; + if (existingVideo.rows.length === 0) { + // Insert new video using raw SQL to avoid schema issues + await db.execute(sql` + INSERT INTO videos (id, title, description, thumbnail_url, video_url, duration, views, category, custom_thumbnail_url, tags, is_public, created_at, updated_at) + VALUES (${videoData.id}, ${videoData.title}, ${videoData.description}, ${videoData.thumbnailUrl}, ${videoData.videoUrl}, ${videoData.duration}, ${videoData.views}, ${videoData.category}, ${videoData.customThumbnailUrl}, ${'{' + videoData.tags.join(',') + '}'}, ${videoData.isPublic}, ${videoData.createdAt.toISOString()}, ${videoData.updatedAt.toISOString()}) + `); + insertedCount++; + } else { + // Update existing video using raw SQL + await db.execute(sql` + UPDATE videos + SET title = ${videoData.title}, description = ${videoData.description}, thumbnail_url = ${videoData.thumbnailUrl}, + video_url = ${videoData.videoUrl}, duration = ${videoData.duration}, views = ${videoData.views}, + category = ${videoData.category}, custom_thumbnail_url = ${videoData.customThumbnailUrl}, + tags = ${'{' + videoData.tags.join(',') + '}'}, is_public = ${videoData.isPublic}, updated_at = ${videoData.updatedAt.toISOString()} + WHERE id = ${video.id} + `); + updatedCount++; + } + } catch (error) { + console.error(`❌ Failed to sync video ${video.id}:`, error); + errorCount++; + // Continue with other videos } - } catch (error) { - console.error(`❌ Failed to sync video ${video.id}:`, error); + } + + // Small delay between batches to avoid overwhelming the database + if (batchIndex < totalBatches - 1) { + await new Promise(resolve => setTimeout(resolve, 50)); } } @@ -127,11 +151,13 @@ class VideoSyncService { console.log(`✅ Database sync completed in ${duration}ms:`); console.log(` 📥 Inserted: ${insertedCount} new videos`); console.log(` 🔄 Updated: ${updatedCount} existing videos`); + console.log(` ❌ Errors: ${errorCount} videos`); console.log(` 📊 Database total: ${insertedCount + updatedCount} videos`); console.log(` 🎥 Cache total: ${this.cache.videos.length} videos`); } catch (error) { console.error('❌ Database sync failed:', error); + throw error; // Re-throw to be caught by timeout wrapper } } @@ -139,11 +165,22 @@ class VideoSyncService { console.log('🔄 Initializing video sync service...'); try { await this.syncVideos(); - await this.syncVideosToDatabase(); + + // Add timeout protection for database sync + const syncTimeout = new Promise((_, reject) => + setTimeout(() => reject(new Error('Database sync timeout after 30 seconds')), 30000) + ); + + await Promise.race([ + this.syncVideosToDatabase(), + syncTimeout + ]); + this.startPeriodicSync(); console.log('✅ Video sync service initialized successfully'); } catch (error) { console.error('❌ Failed to initialize video sync service:', error); + console.log('⚠️ Continuing without database sync - app will still work with cached data'); // Continue without crashing the server } }