Migrate video metadata to PostgreSQL for improved data management

Introduces a new video migration service to transfer video metadata from Bunny.net to PostgreSQL. Updates storage logic to prioritize PostgreSQL, removes redundant fields from the video schema, and adds database migration functionality to the server startup.

Replit-Commit-Author: Agent
Replit-Commit-Session-Id: d7424866-83d1-4486-a212-ac12b4c7becf
Replit-Commit-Checkpoint-Type: full_checkpoint
Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/8cc42625-c1f5-4e43-99bd-77f2c4dedee2/d7424866-83d1-4486-a212-ac12b4c7becf/i1Fg8VZ

server/index.ts

@@ -41,6 +41,13 @@ app.use((req, res, next) => {
   // Initialize video sync service for automatic Bunny.net updates
   await videoSyncService.initialize();
   
+  // Run video metadata migration on startup
+  const { videoMigrator } = await import('./videoMigrator');
+  console.log("🔄 Starting video metadata migration to PostgreSQL...");
+  videoMigrator.migrateAllVideoMetadata().catch(error => {
+    console.error("❌ Video migration failed:", error);
+  });
+  
   const server = await registerRoutes(app);
 
   app.use((err: any, _req: Request, res: Response, _next: NextFunction) => {

server/storage.ts

@@ -801,33 +801,22 @@ class BunnyStorage implements IStorage {
   }
 }
 
-// Storage selection logic - choose DatabaseStorage if PostgreSQL is available
+// Storage selection logic - use PostgreSQL for video metadata
 let storage: IStorage;
 
 const hasDatabase = process.env.DATABASE_URL;
-const hasBunnyConfig = process.env.BUNNY_API_KEY && process.env.BUNNY_LIBRARY_ID && process.env.BUNNY_HOSTNAME;
 
-// Prioritize Bunny.net storage for user's video content
-if (hasBunnyConfig) {
-  try {
-    storage = new BunnyStorage();
-    console.log('✅ Using Bunny.net storage with library ID:', process.env.BUNNY_LIBRARY_ID);
-  } catch (error) {
-    console.error('❌ Failed to initialize Bunny.net storage:', error);
-    console.log('📁 Falling back to memory storage');
-    storage = new MemStorage();
-  }
-} else if (hasDatabase) {
+if (hasDatabase) {
   try {
     storage = new DatabaseStorage();
-    console.log('✅ Using PostgreSQL database storage');
+    console.log('✅ Using PostgreSQL database for video metadata storage');
   } catch (error) {
     console.error('❌ Failed to initialize database storage:', error);
     console.log('📁 Falling back to memory storage');
     storage = new MemStorage();
   }
 } else {
-  console.log('📁 Using memory storage (no database or Bunny.net config found)');
+  console.log('📁 Using memory storage (no database found)');
   storage = new MemStorage();
 }
 

server/videoMigrator.ts

@@ -0,0 +1,88 @@
+import { db } from "./db";
+import { sql } from "drizzle-orm";
+import { videoSyncService } from "./videoSync";
+
+export class VideoMigrator {
+  private isRunning = false;
+
+  async migrateAllVideoMetadata(): Promise<void> {
+    if (this.isRunning) {
+      console.log("🔄 Migration already running...");
+      return;
+    }
+
+    this.isRunning = true;
+    console.log("🔄 Migrating all video metadata from Bunny.net to PostgreSQL...");
+
+    try {
+      // Get all videos from cache
+      const videoList = videoSyncService.getVideos(1000, 0).videos;
+      console.log(`📥 Found ${videoList.length} videos in cache`);
+
+      let inserted = 0;
+      let updated = 0;
+
+      for (const video of videoList) {
+        try {
+          // Upsert video metadata with simple SQL
+          await db.execute(sql`
+            INSERT INTO video_metadata (
+              id, title, description, thumbnail_url, video_url, 
+              duration, views, category, created_at, updated_at
+            ) VALUES (
+              ${video.id}, ${video.title}, ${video.description || ""}, 
+              ${video.thumbnailUrl}, ${video.videoUrl}, 
+              ${video.duration}, ${video.views}, ${video.category || ""}, 
+              ${video.createdAt ? new Date(video.createdAt) : new Date()}, 
+              ${new Date()}
+            ) ON CONFLICT (id) DO UPDATE SET
+              title = EXCLUDED.title,
+              description = EXCLUDED.description,
+              thumbnail_url = EXCLUDED.thumbnail_url,
+              video_url = EXCLUDED.video_url,
+              duration = EXCLUDED.duration,
+              views = EXCLUDED.views,
+              category = EXCLUDED.category,
+              updated_at = EXCLUDED.updated_at
+          `);
+          
+          inserted++;
+
+          // Log progress every 10 videos
+          if ((inserted + updated) % 10 === 0) {
+            console.log(`📊 Progress: ${inserted + updated}/${videoList.length} videos processed`);
+          }
+
+        } catch (error) {
+          console.error(`❌ Error processing ${video.title}:`, error);
+        }
+      }
+
+      console.log(`✅ Migration completed:`);
+      console.log(`   📥 Inserted: ${inserted} new videos`);
+      console.log(`   🔄 Updated: ${updated} existing videos`);
+
+    } catch (error) {
+      console.error("❌ Migration failed:", error);
+    } finally {
+      this.isRunning = false;
+    }
+  }
+
+  async getVideoCount(): Promise<{ database: number; bunny: number }> {
+    try {
+      const dbResult = await db.execute(sql`SELECT COUNT(*) FROM video_metadata`);
+      const cachedVideos = videoSyncService.getVideos(1000, 0).videos;
+
+      return {
+        database: Number(dbResult.rows[0]?.count || 0),
+        bunny: cachedVideos.length
+      };
+    } catch (error) {
+      console.error("❌ Error counting videos:", error);
+      return { database: 0, bunny: 0 };
+    }
+  }
+}
+
+export const videoMigrator = new VideoMigrator();

shared/schema.ts

@@ -8,10 +8,7 @@ export const videos = pgTable("videos", {
   title: text("title").notNull(),
   description: text("description").default("").notNull(),
   thumbnailUrl: text("thumbnail_url").notNull(),
-  customThumbnailUrl: text("custom_thumbnail_url"),
   videoUrl: text("video_url").notNull(),
-  videoUrlMp4: text("video_url_mp4"),
-  videoUrlIframe: text("video_url_iframe"),
   duration: integer("duration").notNull(), // in seconds
   views: integer("views").notNull().default(0),
   category: text("category").default("").notNull(),
@@ -19,14 +16,6 @@ export const videos = pgTable("videos", {
   isPublic: boolean("is_public").default(true).notNull(),
   uploadStatus: text("upload_status").default("completed").notNull(), // pending, processing, completed, failed
   originalFileName: text("original_file_name"),
-  fileSize: integer("file_size"), // in bytes
-  bitrate: integer("bitrate"), // in kbps
-  resolution: text("resolution"), // e.g., "1920x1080"
-  format: text("format"), // e.g., "mp4", "avi", "mov"
-  encoding: text("encoding"), // e.g., "h264", "h265"
-  bunnyId: varchar("bunny_id"), // Original Bunny.net video ID
-  bunnyLibraryId: varchar("bunny_library_id"), // Bunny.net library ID
-  source: text("source").default("bunny").notNull(), // bunny, upload, manual
   createdAt: timestamp("created_at").notNull().default(sql`CURRENT_TIMESTAMP`),
   updatedAt: timestamp("updated_at").notNull().default(sql`CURRENT_TIMESTAMP`),
 });