From f658b64b5653657b533479e0084483c85f31dacc Mon Sep 17 00:00:00 2001 From: sebastjanartic <45803536-sebastjanartic@users.noreply.replit.com> Date: Thu, 28 Aug 2025 17:23:05 +0000 Subject: [PATCH] Migrate video metadata to PostgreSQL for improved data management Introduces a new video migration service to transfer video metadata from Bunny.net to PostgreSQL. Updates storage logic to prioritize PostgreSQL, removes redundant fields from the video schema, and adds database migration functionality to the server startup. Replit-Commit-Author: Agent Replit-Commit-Session-Id: d7424866-83d1-4486-a212-ac12b4c7becf Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/8cc42625-c1f5-4e43-99bd-77f2c4dedee2/d7424866-83d1-4486-a212-ac12b4c7becf/i1Fg8VZ --- server/index.ts | 7 ++++ server/storage.ts | 19 ++------- server/videoMigrator.ts | 88 +++++++++++++++++++++++++++++++++++++++++ shared/schema.ts | 11 ------ 4 files changed, 99 insertions(+), 26 deletions(-) create mode 100644 server/videoMigrator.ts diff --git a/server/index.ts b/server/index.ts index 11cd6c3..f886ca9 100644 --- a/server/index.ts +++ b/server/index.ts @@ -41,6 +41,13 @@ app.use((req, res, next) => { // Initialize video sync service for automatic Bunny.net updates await videoSyncService.initialize(); + // Run video metadata migration on startup + const { videoMigrator } = await import('./videoMigrator'); + console.log("🔄 Starting video metadata migration to PostgreSQL..."); + videoMigrator.migrateAllVideoMetadata().catch(error => { + console.error("❌ Video migration failed:", error); + }); + const server = await registerRoutes(app); app.use((err: any, _req: Request, res: Response, _next: NextFunction) => { diff --git a/server/storage.ts b/server/storage.ts index 690f75c..c54390d 100644 --- a/server/storage.ts +++ b/server/storage.ts @@ -801,33 +801,22 @@ class BunnyStorage implements IStorage { } } -// Storage selection logic - choose DatabaseStorage if PostgreSQL is available +// Storage selection logic - use PostgreSQL for video metadata let storage: IStorage; const hasDatabase = process.env.DATABASE_URL; -const hasBunnyConfig = process.env.BUNNY_API_KEY && process.env.BUNNY_LIBRARY_ID && process.env.BUNNY_HOSTNAME; -// Prioritize Bunny.net storage for user's video content -if (hasBunnyConfig) { - try { - storage = new BunnyStorage(); - console.log('✅ Using Bunny.net storage with library ID:', process.env.BUNNY_LIBRARY_ID); - } catch (error) { - console.error('❌ Failed to initialize Bunny.net storage:', error); - console.log('📁 Falling back to memory storage'); - storage = new MemStorage(); - } -} else if (hasDatabase) { +if (hasDatabase) { try { storage = new DatabaseStorage(); - console.log('✅ Using PostgreSQL database storage'); + console.log('✅ Using PostgreSQL database for video metadata storage'); } catch (error) { console.error('❌ Failed to initialize database storage:', error); console.log('📁 Falling back to memory storage'); storage = new MemStorage(); } } else { - console.log('📁 Using memory storage (no database or Bunny.net config found)'); + console.log('📁 Using memory storage (no database found)'); storage = new MemStorage(); } diff --git a/server/videoMigrator.ts b/server/videoMigrator.ts new file mode 100644 index 0000000..bf8d276 --- /dev/null +++ b/server/videoMigrator.ts @@ -0,0 +1,88 @@ +import { db } from "./db"; +import { sql } from "drizzle-orm"; +import { videoSyncService } from "./videoSync"; + +export class VideoMigrator { + private isRunning = false; + + async migrateAllVideoMetadata(): Promise { + if (this.isRunning) { + console.log("🔄 Migration already running..."); + return; + } + + this.isRunning = true; + console.log("🔄 Migrating all video metadata from Bunny.net to PostgreSQL..."); + + try { + // Get all videos from cache + const videoList = videoSyncService.getVideos(1000, 0).videos; + console.log(`📥 Found ${videoList.length} videos in cache`); + + let inserted = 0; + let updated = 0; + + for (const video of videoList) { + try { + // Upsert video metadata with simple SQL + await db.execute(sql` + INSERT INTO video_metadata ( + id, title, description, thumbnail_url, video_url, + duration, views, category, created_at, updated_at + ) VALUES ( + ${video.id}, ${video.title}, ${video.description || ""}, + ${video.thumbnailUrl}, ${video.videoUrl}, + ${video.duration}, ${video.views}, ${video.category || ""}, + ${video.createdAt ? new Date(video.createdAt) : new Date()}, + ${new Date()} + ) ON CONFLICT (id) DO UPDATE SET + title = EXCLUDED.title, + description = EXCLUDED.description, + thumbnail_url = EXCLUDED.thumbnail_url, + video_url = EXCLUDED.video_url, + duration = EXCLUDED.duration, + views = EXCLUDED.views, + category = EXCLUDED.category, + updated_at = EXCLUDED.updated_at + `); + + inserted++; + + // Log progress every 10 videos + if ((inserted + updated) % 10 === 0) { + console.log(`📊 Progress: ${inserted + updated}/${videoList.length} videos processed`); + } + + } catch (error) { + console.error(`❌ Error processing ${video.title}:`, error); + } + } + + console.log(`✅ Migration completed:`); + console.log(` 📥 Inserted: ${inserted} new videos`); + console.log(` 🔄 Updated: ${updated} existing videos`); + + } catch (error) { + console.error("❌ Migration failed:", error); + } finally { + this.isRunning = false; + } + } + + async getVideoCount(): Promise<{ database: number; bunny: number }> { + try { + const dbResult = await db.execute(sql`SELECT COUNT(*) FROM video_metadata`); + const cachedVideos = videoSyncService.getVideos(1000, 0).videos; + + return { + database: Number(dbResult.rows[0]?.count || 0), + bunny: cachedVideos.length + }; + } catch (error) { + console.error("❌ Error counting videos:", error); + return { database: 0, bunny: 0 }; + } + } +} + +export const videoMigrator = new VideoMigrator(); \ No newline at end of file diff --git a/shared/schema.ts b/shared/schema.ts index 87b664b..2a34523 100644 --- a/shared/schema.ts +++ b/shared/schema.ts @@ -8,10 +8,7 @@ export const videos = pgTable("videos", { title: text("title").notNull(), description: text("description").default("").notNull(), thumbnailUrl: text("thumbnail_url").notNull(), - customThumbnailUrl: text("custom_thumbnail_url"), videoUrl: text("video_url").notNull(), - videoUrlMp4: text("video_url_mp4"), - videoUrlIframe: text("video_url_iframe"), duration: integer("duration").notNull(), // in seconds views: integer("views").notNull().default(0), category: text("category").default("").notNull(), @@ -19,14 +16,6 @@ export const videos = pgTable("videos", { isPublic: boolean("is_public").default(true).notNull(), uploadStatus: text("upload_status").default("completed").notNull(), // pending, processing, completed, failed originalFileName: text("original_file_name"), - fileSize: integer("file_size"), // in bytes - bitrate: integer("bitrate"), // in kbps - resolution: text("resolution"), // e.g., "1920x1080" - format: text("format"), // e.g., "mp4", "avi", "mov" - encoding: text("encoding"), // e.g., "h264", "h265" - bunnyId: varchar("bunny_id"), // Original Bunny.net video ID - bunnyLibraryId: varchar("bunny_library_id"), // Bunny.net library ID - source: text("source").default("bunny").notNull(), // bunny, upload, manual createdAt: timestamp("created_at").notNull().default(sql`CURRENT_TIMESTAMP`), updatedAt: timestamp("updated_at").notNull().default(sql`CURRENT_TIMESTAMP`), });