import { BunnyService } from './bunny'; import { db } from './db'; import { videos } from '@shared/schema'; import { eq, sql } from 'drizzle-orm'; interface VideoSyncCache { videos: any[]; lastUpdate: number; isUpdating: boolean; } class VideoSyncService { private cache: VideoSyncCache = { videos: [], lastUpdate: 0, isUpdating: false }; private bunnyService: BunnyService; private syncInterval: NodeJS.Timeout | null = null; constructor() { this.bunnyService = new BunnyService(); } private async getAllVideos(): Promise { // Refresh list of migrated GUIDs from S3 before fetching, so generateSignedUrl picks correct URL try { await this.bunnyService.refreshMigratedGuids(); } catch {} let allVideos: any[] = []; let page = 1; const itemsPerPage = 100; // Maximum per request let hasMore = true; console.log('📥 Starting to fetch all videos with pagination...'); while (hasMore) { try { console.log(`📄 Fetching page ${page}...`); const result = await this.bunnyService.getVideos(page, itemsPerPage); allVideos = allVideos.concat(result.videos); // Check if there are more pages const totalFetched = allVideos.length; hasMore = totalFetched < result.total && result.videos.length > 0; console.log(`📊 Page ${page}: ${result.videos.length} videos (Total so far: ${allVideos.length}/${result.total})`); page++; // Add small delay to avoid rate limiting if (hasMore) { await new Promise(resolve => setTimeout(resolve, 100)); } // Safety limit to prevent infinite loops if (page > 100) { console.log('⚠️ Reached page limit (100), stopping fetch'); break; } } catch (error) { console.error(`❌ Error fetching page ${page}:`, error); break; } } console.log(`✅ Completed pagination fetch: ${allVideos.length} total videos`); return allVideos; } private async syncVideosToDatabase() { if (this.cache.videos.length === 0) { console.log('⚠️ No videos in cache to sync to database'); return; } console.log('🔄 Syncing cached videos to PostgreSQL database...'); const startTime = Date.now(); let insertedCount = 0; let updatedCount = 0; let errorCount = 0; try { // First, test database connectivity await db.execute(sql`SELECT 1`); console.log('📡 Database connection verified'); // Process videos in smaller batches to avoid overwhelming the database const batchSize = 10; const totalBatches = Math.ceil(this.cache.videos.length / batchSize); for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) { const start = batchIndex * batchSize; const end = Math.min(start + batchSize, this.cache.videos.length); const batch = this.cache.videos.slice(start, end); console.log(`📦 Processing batch ${batchIndex + 1}/${totalBatches} (videos ${start + 1}-${end})`); for (const video of batch) { try { // Check if video exists in database using Drizzle ORM const existingVideo = await db.select({ id: videos.id }).from(videos).where(eq(videos.id, video.id)).limit(1); const videoData = { id: video.id, title: video.title || '', description: video.description || '', thumbnailUrl: video.thumbnailUrl || '', customThumbnailUrl: video.customThumbnailUrl || null, videoUrl: video.videoUrl || '', duration: video.duration || 0, views: video.views || 0, category: video.category || '', tags: Array.isArray(video.tags) ? video.tags : [], isPublic: video.isPublic !== false, createdAt: video.createdAt ? new Date(video.createdAt) : new Date(), updatedAt: new Date(), }; if (existingVideo.length === 0) { // Insert new video using Drizzle ORM await db.insert(videos).values(videoData); insertedCount++; } else { // Update existing video using Drizzle ORM await db.update(videos) .set({ title: videoData.title, description: videoData.description, thumbnailUrl: videoData.thumbnailUrl, customThumbnailUrl: videoData.customThumbnailUrl, videoUrl: videoData.videoUrl, duration: videoData.duration, views: videoData.views, category: videoData.category, tags: videoData.tags, isPublic: videoData.isPublic, updatedAt: videoData.updatedAt, }) .where(eq(videos.id, video.id)); updatedCount++; } } catch (error) { console.error(`❌ Failed to sync video ${video.id}:`, error); errorCount++; // Continue with other videos } } // Small delay between batches to avoid overwhelming the database if (batchIndex < totalBatches - 1) { await new Promise(resolve => setTimeout(resolve, 50)); } } const duration = Date.now() - startTime; console.log(`✅ Database sync completed in ${duration}ms:`); console.log(` 📥 Inserted: ${insertedCount} new videos`); console.log(` 🔄 Updated: ${updatedCount} existing videos`); console.log(` ❌ Errors: ${errorCount} videos`); console.log(` 📊 Database total: ${insertedCount + updatedCount} videos`); console.log(` 🎥 Cache total: ${this.cache.videos.length} videos`); } catch (error) { console.error('❌ Database sync failed:', error); throw error; // Re-throw to be caught by timeout wrapper } } async initialize() { console.log('🔄 Initializing video sync service...'); try { await this.syncVideos(); // Add timeout protection for database sync const syncTimeout = new Promise((_, reject) => setTimeout(() => reject(new Error('Database sync timeout after 30 seconds')), 30000) ); try { await Promise.race([ this.syncVideosToDatabase(), syncTimeout ]); console.log('✅ Database sync completed successfully'); } catch (syncError) { console.error('❌ Database sync failed:', syncError); console.log('⚠️ Continuing with cached data only - videos will still be available from Bunny.net'); // Don't throw here, just log the error and continue } this.startPeriodicSync(); console.log('✅ Video sync service initialized successfully'); } catch (error) { console.error('❌ Failed to initialize video sync service:', error); console.log('⚠️ Continuing without video sync - app will work but videos might not be available'); // Continue without crashing the server - this ensures the app starts even if video sync fails } } private async syncVideos() { if (this.cache.isUpdating) { console.log('⏳ Video sync already in progress, skipping...'); return; } this.cache.isUpdating = true; const startTime = Date.now(); try { console.log('🔍 Fetching ALL videos from Bunny.net...'); const allVideos = await this.getAllVideos(); this.cache.videos = allVideos; this.cache.lastUpdate = Date.now(); const duration = Date.now() - startTime; console.log(`✅ Video sync completed: ${allVideos.length} videos cached in ${duration}ms`); } catch (error) { console.error('❌ Video sync failed:', error); } finally { this.cache.isUpdating = false; } } private startPeriodicSync() { // Sync every 5 minutes for better performance this.syncInterval = setInterval(() => { console.log('⏰ Starting scheduled video sync...'); this.syncVideos(); }, 5 * 60 * 1000); console.log('📅 Scheduled video sync every 5 minutes for optimal performance'); } getVideos(limit: number = 20, offset: number = 0, search?: string) { let filteredVideos = this.cache.videos; // Fast client-side search if (search && search.length >= 2) { const searchLower = search.toLowerCase(); filteredVideos = this.cache.videos.filter(video => video.title.toLowerCase().includes(searchLower) || video.description?.toLowerCase().includes(searchLower) ); } const paginatedVideos = filteredVideos.slice(offset, offset + limit); return { videos: paginatedVideos, total: filteredVideos.length, hasMore: offset + limit < filteredVideos.length, cacheAge: Date.now() - this.cache.lastUpdate }; } async getVideo(id: string) { // First try cache const cachedVideo = this.cache.videos.find(v => v.id === id); if (cachedVideo) { return cachedVideo; } // Fallback to direct API call return await this.bunnyService.getVideo(id); } forceVideoRefresh(videoId: string) { console.log(`🔄 Forcing refresh for video ${videoId}`); // Remove video from cache to force fresh fetch this.cache.videos = this.cache.videos.filter(v => v.id !== videoId); // Mark cache as needing update this.cache.lastUpdate = 0; } getCacheStats() { return { videosCount: this.cache.videos.length, lastUpdate: this.cache.lastUpdate, isUpdating: this.cache.isUpdating, cacheAge: Date.now() - this.cache.lastUpdate }; } stop() { if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; console.log('🛑 Video sync service stopped'); } } } export const videoSyncService = new VideoSyncService();