297 lines
9.9 KiB
TypeScript
297 lines
9.9 KiB
TypeScript
import { BunnyService } from './bunny';
|
|
import { db } from './db';
|
|
import { videos } from '@shared/schema';
|
|
import { eq, sql } from 'drizzle-orm';
|
|
|
|
interface VideoSyncCache {
|
|
videos: any[];
|
|
lastUpdate: number;
|
|
isUpdating: boolean;
|
|
}
|
|
|
|
class VideoSyncService {
|
|
private cache: VideoSyncCache = {
|
|
videos: [],
|
|
lastUpdate: 0,
|
|
isUpdating: false
|
|
};
|
|
private bunnyService: BunnyService;
|
|
private syncInterval: NodeJS.Timeout | null = null;
|
|
|
|
constructor() {
|
|
this.bunnyService = new BunnyService();
|
|
}
|
|
|
|
private async getAllVideos(): Promise<any[]> {
|
|
// Refresh list of migrated GUIDs from S3 before fetching, so generateSignedUrl picks correct URL
|
|
try { await this.bunnyService.refreshMigratedGuids(); } catch {}
|
|
|
|
let allVideos: any[] = [];
|
|
let page = 1;
|
|
const itemsPerPage = 100; // Maximum per request
|
|
let hasMore = true;
|
|
|
|
console.log('📥 Starting to fetch all videos with pagination...');
|
|
|
|
while (hasMore) {
|
|
try {
|
|
console.log(`📄 Fetching page ${page}...`);
|
|
const result = await this.bunnyService.getVideos(page, itemsPerPage);
|
|
|
|
allVideos = allVideos.concat(result.videos);
|
|
|
|
// Check if there are more pages
|
|
const totalFetched = allVideos.length;
|
|
hasMore = totalFetched < result.total && result.videos.length > 0;
|
|
|
|
console.log(`📊 Page ${page}: ${result.videos.length} videos (Total so far: ${allVideos.length}/${result.total})`);
|
|
page++;
|
|
|
|
// Add small delay to avoid rate limiting
|
|
if (hasMore) {
|
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
}
|
|
|
|
// Safety limit to prevent infinite loops
|
|
if (page > 100) {
|
|
console.log('⚠️ Reached page limit (100), stopping fetch');
|
|
break;
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error(`❌ Error fetching page ${page}:`, error);
|
|
break;
|
|
}
|
|
}
|
|
|
|
console.log(`✅ Completed pagination fetch: ${allVideos.length} total videos`);
|
|
return allVideos;
|
|
}
|
|
|
|
private async syncVideosToDatabase() {
|
|
if (this.cache.videos.length === 0) {
|
|
console.log('⚠️ No videos in cache to sync to database');
|
|
return;
|
|
}
|
|
|
|
console.log('🔄 Syncing cached videos to PostgreSQL database...');
|
|
const startTime = Date.now();
|
|
|
|
let insertedCount = 0;
|
|
let updatedCount = 0;
|
|
let errorCount = 0;
|
|
|
|
try {
|
|
// First, test database connectivity
|
|
await db.execute(sql`SELECT 1`);
|
|
console.log('📡 Database connection verified');
|
|
|
|
// Process videos in smaller batches to avoid overwhelming the database
|
|
const batchSize = 10;
|
|
const totalBatches = Math.ceil(this.cache.videos.length / batchSize);
|
|
|
|
for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) {
|
|
const start = batchIndex * batchSize;
|
|
const end = Math.min(start + batchSize, this.cache.videos.length);
|
|
const batch = this.cache.videos.slice(start, end);
|
|
|
|
console.log(`📦 Processing batch ${batchIndex + 1}/${totalBatches} (videos ${start + 1}-${end})`);
|
|
|
|
for (const video of batch) {
|
|
try {
|
|
// Check if video exists in database using Drizzle ORM
|
|
const existingVideo = await db.select({ id: videos.id }).from(videos).where(eq(videos.id, video.id)).limit(1);
|
|
|
|
const videoData = {
|
|
id: video.id,
|
|
title: video.title || '',
|
|
description: video.description || '',
|
|
thumbnailUrl: video.thumbnailUrl || '',
|
|
customThumbnailUrl: video.customThumbnailUrl || null,
|
|
videoUrl: video.videoUrl || '',
|
|
duration: video.duration || 0,
|
|
views: video.views || 0,
|
|
category: video.category || '',
|
|
tags: Array.isArray(video.tags) ? video.tags : [],
|
|
isPublic: video.isPublic !== false,
|
|
createdAt: video.createdAt ? new Date(video.createdAt) : new Date(),
|
|
updatedAt: new Date(),
|
|
};
|
|
|
|
if (existingVideo.length === 0) {
|
|
// Insert new video using Drizzle ORM
|
|
await db.insert(videos).values(videoData);
|
|
insertedCount++;
|
|
} else {
|
|
// Update existing video using Drizzle ORM
|
|
await db.update(videos)
|
|
.set({
|
|
title: videoData.title,
|
|
description: videoData.description,
|
|
thumbnailUrl: videoData.thumbnailUrl,
|
|
customThumbnailUrl: videoData.customThumbnailUrl,
|
|
videoUrl: videoData.videoUrl,
|
|
duration: videoData.duration,
|
|
views: videoData.views,
|
|
category: videoData.category,
|
|
tags: videoData.tags,
|
|
isPublic: videoData.isPublic,
|
|
updatedAt: videoData.updatedAt,
|
|
})
|
|
.where(eq(videos.id, video.id));
|
|
updatedCount++;
|
|
}
|
|
} catch (error) {
|
|
console.error(`❌ Failed to sync video ${video.id}:`, error);
|
|
errorCount++;
|
|
// Continue with other videos
|
|
}
|
|
}
|
|
|
|
// Small delay between batches to avoid overwhelming the database
|
|
if (batchIndex < totalBatches - 1) {
|
|
await new Promise(resolve => setTimeout(resolve, 50));
|
|
}
|
|
}
|
|
|
|
const duration = Date.now() - startTime;
|
|
console.log(`✅ Database sync completed in ${duration}ms:`);
|
|
console.log(` 📥 Inserted: ${insertedCount} new videos`);
|
|
console.log(` 🔄 Updated: ${updatedCount} existing videos`);
|
|
console.log(` ❌ Errors: ${errorCount} videos`);
|
|
console.log(` 📊 Database total: ${insertedCount + updatedCount} videos`);
|
|
console.log(` 🎥 Cache total: ${this.cache.videos.length} videos`);
|
|
|
|
} catch (error) {
|
|
console.error('❌ Database sync failed:', error);
|
|
throw error; // Re-throw to be caught by timeout wrapper
|
|
}
|
|
}
|
|
|
|
async initialize() {
|
|
console.log('🔄 Initializing video sync service...');
|
|
try {
|
|
await this.syncVideos();
|
|
|
|
// Add timeout protection for database sync
|
|
const syncTimeout = new Promise((_, reject) =>
|
|
setTimeout(() => reject(new Error('Database sync timeout after 30 seconds')), 30000)
|
|
);
|
|
|
|
try {
|
|
await Promise.race([
|
|
this.syncVideosToDatabase(),
|
|
syncTimeout
|
|
]);
|
|
console.log('✅ Database sync completed successfully');
|
|
} catch (syncError) {
|
|
console.error('❌ Database sync failed:', syncError);
|
|
console.log('⚠️ Continuing with cached data only - videos will still be available from Bunny.net');
|
|
// Don't throw here, just log the error and continue
|
|
}
|
|
|
|
this.startPeriodicSync();
|
|
console.log('✅ Video sync service initialized successfully');
|
|
} catch (error) {
|
|
console.error('❌ Failed to initialize video sync service:', error);
|
|
console.log('⚠️ Continuing without video sync - app will work but videos might not be available');
|
|
// Continue without crashing the server - this ensures the app starts even if video sync fails
|
|
}
|
|
}
|
|
|
|
private async syncVideos() {
|
|
if (this.cache.isUpdating) {
|
|
console.log('⏳ Video sync already in progress, skipping...');
|
|
return;
|
|
}
|
|
|
|
this.cache.isUpdating = true;
|
|
const startTime = Date.now();
|
|
|
|
try {
|
|
console.log('🔍 Fetching ALL videos from Bunny.net...');
|
|
const allVideos = await this.getAllVideos();
|
|
|
|
this.cache.videos = allVideos;
|
|
this.cache.lastUpdate = Date.now();
|
|
|
|
const duration = Date.now() - startTime;
|
|
console.log(`✅ Video sync completed: ${allVideos.length} videos cached in ${duration}ms`);
|
|
} catch (error) {
|
|
console.error('❌ Video sync failed:', error);
|
|
} finally {
|
|
this.cache.isUpdating = false;
|
|
}
|
|
}
|
|
|
|
private startPeriodicSync() {
|
|
// Sync every 5 minutes for better performance
|
|
this.syncInterval = setInterval(() => {
|
|
console.log('⏰ Starting scheduled video sync...');
|
|
this.syncVideos();
|
|
}, 5 * 60 * 1000);
|
|
|
|
console.log('📅 Scheduled video sync every 5 minutes for optimal performance');
|
|
}
|
|
|
|
getVideos(limit: number = 20, offset: number = 0, search?: string) {
|
|
let filteredVideos = this.cache.videos;
|
|
|
|
// Fast client-side search
|
|
if (search && search.length >= 2) {
|
|
const searchLower = search.toLowerCase();
|
|
filteredVideos = this.cache.videos.filter(video =>
|
|
video.title.toLowerCase().includes(searchLower) ||
|
|
video.description?.toLowerCase().includes(searchLower)
|
|
);
|
|
}
|
|
|
|
const paginatedVideos = filteredVideos.slice(offset, offset + limit);
|
|
|
|
return {
|
|
videos: paginatedVideos,
|
|
total: filteredVideos.length,
|
|
hasMore: offset + limit < filteredVideos.length,
|
|
cacheAge: Date.now() - this.cache.lastUpdate
|
|
};
|
|
}
|
|
|
|
|
|
async getVideo(id: string) {
|
|
// First try cache
|
|
const cachedVideo = this.cache.videos.find(v => v.id === id);
|
|
if (cachedVideo) {
|
|
return cachedVideo;
|
|
}
|
|
|
|
// Fallback to direct API call
|
|
return await this.bunnyService.getVideo(id);
|
|
}
|
|
|
|
forceVideoRefresh(videoId: string) {
|
|
console.log(`🔄 Forcing refresh for video ${videoId}`);
|
|
// Remove video from cache to force fresh fetch
|
|
this.cache.videos = this.cache.videos.filter(v => v.id !== videoId);
|
|
// Mark cache as needing update
|
|
this.cache.lastUpdate = 0;
|
|
}
|
|
|
|
getCacheStats() {
|
|
return {
|
|
videosCount: this.cache.videos.length,
|
|
lastUpdate: this.cache.lastUpdate,
|
|
isUpdating: this.cache.isUpdating,
|
|
cacheAge: Date.now() - this.cache.lastUpdate
|
|
};
|
|
}
|
|
|
|
stop() {
|
|
if (this.syncInterval) {
|
|
clearInterval(this.syncInterval);
|
|
this.syncInterval = null;
|
|
console.log('🛑 Video sync service stopped');
|
|
}
|
|
}
|
|
}
|
|
|
|
export const videoSyncService = new VideoSyncService(); |