Direct-to-bucket video upload + transcode pipeline
- server/videoUpload.ts: S3 multipart presign helpers, bucket inventory listing
- routes: /api/admin/uploads/{init,part-url,complete,abort}, /api/admin/inventory,
/api/internal/transcode-done (worker callback, bearer INTERNAL_TOKEN)
- admin UI: browser-to-bucket uploader with progress, bucket-vs-database panel
Originals land in folx-tv/_incoming/{guid}/, the worker on the remotion box
packages them to HLS and reports back.
This commit is contained in:
Claude Agent
@@ -19,6 +19,14 @@ import fetch from "node-fetch";
|
||||
import { setupAuth, isAuthenticated, isAdmin } from "./replitAuth";
|
||||
import { ObjectStorageService, ObjectNotFoundError } from "./objectStorage";
|
||||
import { generateVideoDescription, generateBulkDescriptions } from "./aiService";
|
||||
import { db } from "./db";
|
||||
import { videos } from "@shared/schema";
|
||||
import { eq } from "drizzle-orm";
|
||||
import {
|
||||
initUpload, partUrl, completeUpload, abortUpload, writeJobFile,
|
||||
listPackagedGuids, listPendingJobs, publicUrlFor,
|
||||
FOLX_PREFIX, INCOMING_PREFIX,
|
||||
} from "./videoUpload";
|
||||
|
||||
// Extract unique artist names from video titles
|
||||
function extractArtists(videos: any[]): { name: string; videoCount: number; videos: any[] }[] {
|
||||
@@ -2134,6 +2142,174 @@ Sitemap: ${baseUrl}/sitemap.xml
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
// DIRECT-TO-BUCKET VIDEO UPLOAD (browser → Hetzner S3, multipart)
|
||||
// Flow: init → part-url (per chunk) → complete → worker transcodes
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
|
||||
app.post('/api/admin/uploads/init', isAdmin, async (req, res) => {
|
||||
try {
|
||||
const { fileName, fileSize, contentType } = req.body || {};
|
||||
if (!fileName || !fileSize) {
|
||||
return res.status(400).json({ message: "fileName and fileSize are required" });
|
||||
}
|
||||
const size = parseInt(String(fileSize), 10);
|
||||
if (!Number.isFinite(size) || size <= 0) {
|
||||
return res.status(400).json({ message: "fileSize must be a positive number" });
|
||||
}
|
||||
const guid = randomUUID();
|
||||
const result = await initUpload(guid, fileName, size, contentType);
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
console.error("upload init failed:", error);
|
||||
res.status(500).json({ message: "Failed to start upload" });
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/api/admin/uploads/part-url', isAdmin, async (req, res) => {
|
||||
try {
|
||||
const { key, uploadId, partNumber } = req.body || {};
|
||||
if (!key || !uploadId || !partNumber) {
|
||||
return res.status(400).json({ message: "key, uploadId and partNumber are required" });
|
||||
}
|
||||
if (!String(key).startsWith(INCOMING_PREFIX)) {
|
||||
return res.status(400).json({ message: "Invalid key" });
|
||||
}
|
||||
const url = await partUrl(String(key), String(uploadId), parseInt(String(partNumber), 10));
|
||||
res.json({ url });
|
||||
} catch (error) {
|
||||
console.error("part url failed:", error);
|
||||
res.status(500).json({ message: "Failed to sign part" });
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/api/admin/uploads/complete', isAdmin, async (req, res) => {
|
||||
try {
|
||||
const { guid, key, uploadId, parts, title, artist, description, category, contentType, genre, isPublic, fileSize, originalFileName } = req.body || {};
|
||||
if (!guid || !key || !uploadId || !Array.isArray(parts) || parts.length === 0) {
|
||||
return res.status(400).json({ message: "guid, key, uploadId and parts are required" });
|
||||
}
|
||||
if (!String(key).startsWith(INCOMING_PREFIX)) {
|
||||
return res.status(400).json({ message: "Invalid key" });
|
||||
}
|
||||
|
||||
await completeUpload(String(key), String(uploadId), parts);
|
||||
|
||||
const cleanTitle = (title && String(title).trim()) || String(originalFileName || "Neues Video").replace(/\.[^.]+$/, "");
|
||||
|
||||
await writeJobFile(String(guid), {
|
||||
guid,
|
||||
sourceKey: key,
|
||||
title: cleanTitle,
|
||||
originalFileName: originalFileName || null,
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Row appears in admin immediately with status "processing"
|
||||
await db.insert(videos).values({
|
||||
id: String(guid),
|
||||
title: cleanTitle,
|
||||
artist: artist || null,
|
||||
description: description || "",
|
||||
thumbnailUrl: `${process.env.S3_PUBLIC_BASE || "https://folxvideos.b-cdn.net"}/${FOLX_PREFIX}${guid}/thumbnail.jpg`,
|
||||
videoUrl: publicUrlFor(String(guid)),
|
||||
duration: 0,
|
||||
category: category || "",
|
||||
contentType: contentType || 'video',
|
||||
genre: genre || 'other',
|
||||
isPublic: isPublic === undefined ? false : !!isPublic,
|
||||
uploadStatus: 'processing',
|
||||
originalFileName: originalFileName || null,
|
||||
fileSize: fileSize ? parseInt(String(fileSize), 10) : null,
|
||||
} as any);
|
||||
|
||||
res.json({ ok: true, guid, status: 'processing', videoUrl: publicUrlFor(String(guid)) });
|
||||
} catch (error) {
|
||||
console.error("upload complete failed:", error);
|
||||
res.status(500).json({ message: "Failed to finalise upload" });
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/api/admin/uploads/abort', isAdmin, async (req, res) => {
|
||||
try {
|
||||
const { key, uploadId } = req.body || {};
|
||||
if (key && uploadId) await abortUpload(String(key), String(uploadId));
|
||||
res.json({ ok: true });
|
||||
} catch {
|
||||
res.json({ ok: true });
|
||||
}
|
||||
});
|
||||
|
||||
// Called by the transcode worker on the remotion box when a package is ready.
|
||||
app.post('/api/internal/transcode-done', async (req, res) => {
|
||||
const expected = process.env.INTERNAL_TOKEN;
|
||||
const given = (req.headers.authorization || "").replace(/^Bearer\s+/i, "");
|
||||
if (!expected || given !== expected) {
|
||||
return res.status(401).json({ message: "Unauthorized" });
|
||||
}
|
||||
try {
|
||||
const { guid, status, duration, resolution, bitrate, format, encoding, errorMessage } = req.body || {};
|
||||
if (!guid) return res.status(400).json({ message: "guid required" });
|
||||
|
||||
if (status === 'failed') {
|
||||
await db.update(videos)
|
||||
.set({ uploadStatus: 'failed', description: errorMessage ? String(errorMessage).slice(0, 500) : undefined, updatedAt: new Date() } as any)
|
||||
.where(eq(videos.id, String(guid)));
|
||||
console.error(`transcode failed for ${guid}: ${errorMessage}`);
|
||||
return res.json({ ok: true });
|
||||
}
|
||||
|
||||
await db.update(videos)
|
||||
.set({
|
||||
uploadStatus: 'completed',
|
||||
duration: duration ? Math.round(Number(duration)) : 0,
|
||||
resolution: resolution || null,
|
||||
bitrate: bitrate ? Math.round(Number(bitrate)) : null,
|
||||
format: format || 'hls',
|
||||
encoding: encoding || 'h264',
|
||||
updatedAt: new Date(),
|
||||
} as any)
|
||||
.where(eq(videos.id, String(guid)));
|
||||
|
||||
console.log(`✅ transcode done: ${guid}`);
|
||||
res.json({ ok: true });
|
||||
} catch (error) {
|
||||
console.error("transcode-done failed:", error);
|
||||
res.status(500).json({ message: "Failed" });
|
||||
}
|
||||
});
|
||||
|
||||
// Inventory: what is in the bucket vs. what is in the database
|
||||
app.get('/api/admin/inventory', isAdmin, async (_req, res) => {
|
||||
try {
|
||||
const [packaged, pending, rows] = await Promise.all([
|
||||
listPackagedGuids(),
|
||||
listPendingJobs(),
|
||||
db.select({ id: videos.id, title: videos.title, uploadStatus: videos.uploadStatus, createdAt: videos.createdAt }).from(videos),
|
||||
]);
|
||||
|
||||
const dbIds = new Set(rows.map(r => r.id));
|
||||
const missingInBucket = rows
|
||||
.filter(r => !packaged.has(r.id))
|
||||
.map(r => ({ id: r.id, title: r.title, uploadStatus: r.uploadStatus }));
|
||||
const orphansInBucket = Array.from(packaged).filter(g => !dbIds.has(g));
|
||||
|
||||
res.json({
|
||||
databaseCount: rows.length,
|
||||
bucketCount: packaged.size,
|
||||
playable: rows.length - missingInBucket.length,
|
||||
processingCount: pending.length,
|
||||
missingInBucket,
|
||||
orphansInBucket,
|
||||
pendingJobs: pending,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("inventory failed:", error);
|
||||
res.status(500).json({ message: "Failed to build inventory" });
|
||||
}
|
||||
});
|
||||
|
||||
const httpServer = createServer(app);
|
||||
return httpServer;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user