Ensure all videos are consistently stored in the database by fetching and synchronizing them
Implement a paginated fetching mechanism for all videos from an external service and synchronize them to the PostgreSQL database, handling both new insertions and updates for existing video records. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 2eb1084e-b728-4449-9231-f1665924c8d5 Replit-Commit-Checkpoint-Type: full_checkpoint
This commit is contained in:
sebastjanartic
parent
42883d8409
commit
8a8a94b8fd
@ -1,4 +1,7 @@
|
||||
import { BunnyService } from './bunny';
|
||||
import { db } from './db';
|
||||
import { videos } from '@shared/schema';
|
||||
import { eq, sql } from 'drizzle-orm';
|
||||
|
||||
interface VideoSyncCache {
|
||||
videos: any[];
|
||||
@ -19,10 +22,119 @@ class VideoSyncService {
|
||||
this.bunnyService = new BunnyService();
|
||||
}
|
||||
|
||||
private async getAllVideos(): Promise<any[]> {
|
||||
let allVideos: any[] = [];
|
||||
let page = 1;
|
||||
const itemsPerPage = 100; // Maximum per request
|
||||
let hasMore = true;
|
||||
|
||||
console.log('📥 Starting to fetch all videos with pagination...');
|
||||
|
||||
while (hasMore) {
|
||||
try {
|
||||
console.log(`📄 Fetching page ${page}...`);
|
||||
const result = await this.bunnyService.getVideos(page, itemsPerPage);
|
||||
|
||||
allVideos = allVideos.concat(result.videos);
|
||||
|
||||
// Check if there are more pages
|
||||
const totalFetched = (page - 1) * itemsPerPage + result.videos.length;
|
||||
hasMore = totalFetched < result.total && result.videos.length === itemsPerPage;
|
||||
|
||||
console.log(`📊 Page ${page}: ${result.videos.length} videos (Total so far: ${allVideos.length}/${result.total})`);
|
||||
page++;
|
||||
|
||||
// Safety limit to prevent infinite loops
|
||||
if (page > 100) {
|
||||
console.log('⚠️ Reached page limit (100), stopping fetch');
|
||||
break;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`❌ Error fetching page ${page}:`, error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`✅ Completed pagination fetch: ${allVideos.length} total videos`);
|
||||
return allVideos;
|
||||
}
|
||||
|
||||
private async syncVideosToDatabase() {
|
||||
if (this.cache.videos.length === 0) {
|
||||
console.log('⚠️ No videos in cache to sync to database');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('🔄 Syncing cached videos to PostgreSQL database...');
|
||||
const startTime = Date.now();
|
||||
|
||||
let insertedCount = 0;
|
||||
let updatedCount = 0;
|
||||
|
||||
try {
|
||||
for (const video of this.cache.videos) {
|
||||
try {
|
||||
// Check if video exists in database using raw SQL
|
||||
const existingVideo = await db.execute(sql`SELECT id FROM videos WHERE id = ${video.id} LIMIT 1`);
|
||||
|
||||
const videoData = {
|
||||
id: video.id,
|
||||
title: video.title,
|
||||
description: video.description || '',
|
||||
thumbnailUrl: video.thumbnailUrl,
|
||||
customThumbnailUrl: video.customThumbnailUrl || null,
|
||||
videoUrl: video.videoUrl,
|
||||
duration: video.duration,
|
||||
views: video.views,
|
||||
category: video.category || '',
|
||||
tags: Array.isArray(video.tags) ? video.tags : [],
|
||||
isPublic: video.isPublic !== false,
|
||||
createdAt: video.createdAt ? new Date(video.createdAt) : new Date(),
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
|
||||
if (existingVideo.rows.length === 0) {
|
||||
// Insert new video using raw SQL to avoid schema issues
|
||||
await db.execute(sql`
|
||||
INSERT INTO videos (id, title, description, thumbnail_url, video_url, duration, views, category, custom_thumbnail_url, tags, is_public, created_at, updated_at)
|
||||
VALUES (${videoData.id}, ${videoData.title}, ${videoData.description}, ${videoData.thumbnailUrl}, ${videoData.videoUrl}, ${videoData.duration}, ${videoData.views}, ${videoData.category}, ${videoData.customThumbnailUrl}, ${'{' + videoData.tags.join(',') + '}'}, ${videoData.isPublic}, ${videoData.createdAt.toISOString()}, ${videoData.updatedAt.toISOString()})
|
||||
`);
|
||||
insertedCount++;
|
||||
} else {
|
||||
// Update existing video using raw SQL
|
||||
await db.execute(sql`
|
||||
UPDATE videos
|
||||
SET title = ${videoData.title}, description = ${videoData.description}, thumbnail_url = ${videoData.thumbnailUrl},
|
||||
video_url = ${videoData.videoUrl}, duration = ${videoData.duration}, views = ${videoData.views},
|
||||
category = ${videoData.category}, custom_thumbnail_url = ${videoData.customThumbnailUrl},
|
||||
tags = ${'{' + videoData.tags.join(',') + '}'}, is_public = ${videoData.isPublic}, updated_at = ${videoData.updatedAt.toISOString()}
|
||||
WHERE id = ${video.id}
|
||||
`);
|
||||
updatedCount++;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`❌ Failed to sync video ${video.id}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
console.log(`✅ Database sync completed in ${duration}ms:`);
|
||||
console.log(` 📥 Inserted: ${insertedCount} new videos`);
|
||||
console.log(` 🔄 Updated: ${updatedCount} existing videos`);
|
||||
console.log(` 📊 Database total: ${insertedCount + updatedCount} videos`);
|
||||
console.log(` 🎥 Cache total: ${this.cache.videos.length} videos`);
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Database sync failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
console.log('🔄 Initializing video sync service...');
|
||||
try {
|
||||
await this.syncVideos();
|
||||
await this.syncVideosToDatabase();
|
||||
this.startPeriodicSync();
|
||||
console.log('✅ Video sync service initialized successfully');
|
||||
} catch (error) {
|
||||
@ -41,14 +153,14 @@ class VideoSyncService {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
console.log('🔍 Fetching latest videos from Bunny.net...');
|
||||
const result = await this.bunnyService.getVideos(1, 100);
|
||||
console.log('🔍 Fetching ALL videos from Bunny.net...');
|
||||
const allVideos = await this.getAllVideos();
|
||||
|
||||
this.cache.videos = result.videos;
|
||||
this.cache.videos = allVideos;
|
||||
this.cache.lastUpdate = Date.now();
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
console.log(`✅ Video sync completed: ${result.videos.length} videos cached in ${duration}ms`);
|
||||
console.log(`✅ Video sync completed: ${allVideos.length} videos cached in ${duration}ms`);
|
||||
} catch (error) {
|
||||
console.error('❌ Video sync failed:', error);
|
||||
} finally {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user