Direct-to-bucket video upload + transcode pipeline
- server/videoUpload.ts: S3 multipart presign helpers, bucket inventory listing
- routes: /api/admin/uploads/{init,part-url,complete,abort}, /api/admin/inventory,
/api/internal/transcode-done (worker callback, bearer INTERNAL_TOKEN)
- admin UI: browser-to-bucket uploader with progress, bucket-vs-database panel
Originals land in folx-tv/_incoming/{guid}/, the worker on the remotion box
packages them to HLS and reports back.
This commit is contained in:
Claude Agent
parent
f6c73f543c
commit
49d9c83201
106
client/src/components/inventory-panel.tsx
Normal file
106
client/src/components/inventory-panel.tsx
Normal file
@ -0,0 +1,106 @@
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { apiRequest } from "@/lib/queryClient";
|
||||
import { Database, RefreshCw, AlertTriangle, Loader2 } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
|
||||
interface Inventory {
|
||||
databaseCount: number;
|
||||
bucketCount: number;
|
||||
playable: number;
|
||||
processingCount: number;
|
||||
missingInBucket: Array<{ id: string; title: string; uploadStatus: string }>;
|
||||
orphansInBucket: string[];
|
||||
pendingJobs: string[];
|
||||
}
|
||||
|
||||
/** Side-by-side view of what the database claims and what the bucket actually holds. */
|
||||
export default function InventoryPanel() {
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
|
||||
const { data, isLoading, refetch, isFetching } = useQuery<Inventory>({
|
||||
queryKey: ["/api/admin/inventory"],
|
||||
queryFn: async () => {
|
||||
const res = await apiRequest("GET", "/api/admin/inventory");
|
||||
return res.json();
|
||||
},
|
||||
refetchInterval: 60_000,
|
||||
});
|
||||
|
||||
return (
|
||||
<Card className="bg-white/10 border-white/20 text-white mb-6">
|
||||
<CardHeader className="flex flex-row items-center justify-between space-y-0">
|
||||
<CardTitle className="flex items-center space-x-2">
|
||||
<Database className="w-5 h-5" />
|
||||
<span>Bestand</span>
|
||||
</CardTitle>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => refetch()}
|
||||
disabled={isFetching}
|
||||
className="text-white border-white/20 hover:bg-white/10"
|
||||
>
|
||||
{isFetching ? <Loader2 className="w-4 h-4 animate-spin" /> : <RefreshCw className="w-4 h-4" />}
|
||||
</Button>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
{isLoading || !data ? (
|
||||
<p className="text-white/70 text-sm">Wird geladen…</p>
|
||||
) : (
|
||||
<>
|
||||
<div className="grid grid-cols-2 md:grid-cols-4 gap-4 mb-4">
|
||||
<Stat label="In der Datenbank" value={data.databaseCount} />
|
||||
<Stat label="Im Bucket" value={data.bucketCount} />
|
||||
<Stat label="Abspielbar" value={data.playable} />
|
||||
<Stat label="In Umwandlung" value={data.processingCount} />
|
||||
</div>
|
||||
|
||||
{data.missingInBucket.length > 0 && (
|
||||
<div className="rounded border border-amber-400/40 bg-amber-400/10 p-3">
|
||||
<p className="flex items-center gap-2 text-sm text-amber-200">
|
||||
<AlertTriangle className="w-4 h-4" />
|
||||
{data.missingInBucket.length} Einträge ohne Dateien im Bucket — diese lassen sich nicht abspielen.
|
||||
</p>
|
||||
<Button
|
||||
variant="link"
|
||||
className="text-amber-200 px-0 h-auto"
|
||||
onClick={() => setExpanded(!expanded)}
|
||||
>
|
||||
{expanded ? "Liste ausblenden" : "Liste anzeigen"}
|
||||
</Button>
|
||||
{expanded && (
|
||||
<ul className="mt-2 max-h-64 overflow-auto text-xs space-y-1">
|
||||
{data.missingInBucket.map((v) => (
|
||||
<li key={v.id} className="flex justify-between gap-2">
|
||||
<span className="truncate">{v.title}</span>
|
||||
<Badge variant="secondary" className="shrink-0">{v.uploadStatus}</Badge>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{data.orphansInBucket.length > 0 && (
|
||||
<p className="mt-3 text-xs text-white/60">
|
||||
{data.orphansInBucket.length} Pakete liegen im Bucket ohne Datenbankeintrag.
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
function Stat({ label, value }: { label: string; value: number }) {
|
||||
return (
|
||||
<div className="rounded bg-white/5 p-3">
|
||||
<p className="text-2xl font-semibold">{value}</p>
|
||||
<p className="text-xs text-white/70">{label}</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
266
client/src/components/video-upload.tsx
Normal file
266
client/src/components/video-upload.tsx
Normal file
@ -0,0 +1,266 @@
|
||||
import { useRef, useState } from "react";
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { useToast } from "@/hooks/use-toast";
|
||||
import { apiRequest } from "@/lib/queryClient";
|
||||
import { UploadCloud, Loader2, CheckCircle2 } from "lucide-react";
|
||||
|
||||
type Phase = "idle" | "starting" | "uploading" | "finishing" | "done" | "error";
|
||||
|
||||
interface PartResult {
|
||||
PartNumber: number;
|
||||
ETag: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads the master file straight from the browser into the Hetzner bucket
|
||||
* using presigned multipart URLs, then hands the GUID to the transcode worker.
|
||||
* Nothing large ever passes through the application server.
|
||||
*/
|
||||
export default function VideoUpload() {
|
||||
const { toast } = useToast();
|
||||
const queryClient = useQueryClient();
|
||||
const fileRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
const [file, setFile] = useState<File | null>(null);
|
||||
const [title, setTitle] = useState("");
|
||||
const [artist, setArtist] = useState("");
|
||||
const [description, setDescription] = useState("");
|
||||
const [genre, setGenre] = useState("other");
|
||||
const [contentType, setContentType] = useState("video");
|
||||
const [phase, setPhase] = useState<Phase>("idle");
|
||||
const [progress, setProgress] = useState(0);
|
||||
const [errorMsg, setErrorMsg] = useState("");
|
||||
|
||||
const busy = phase === "starting" || phase === "uploading" || phase === "finishing";
|
||||
|
||||
function pickFile(f: File | null) {
|
||||
setFile(f);
|
||||
setPhase("idle");
|
||||
setProgress(0);
|
||||
setErrorMsg("");
|
||||
if (f && !title) setTitle(f.name.replace(/\.[^.]+$/, ""));
|
||||
}
|
||||
|
||||
async function uploadPart(url: string, blob: Blob, partNumber: number): Promise<PartResult> {
|
||||
// XHR (not fetch) so we could extend this with per-part progress later.
|
||||
const etag: string = await new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("PUT", url, true);
|
||||
xhr.onload = () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
const tag = xhr.getResponseHeader("ETag") || xhr.getResponseHeader("etag") || "";
|
||||
if (!tag) reject(new Error(`Part ${partNumber}: no ETag returned`));
|
||||
else resolve(tag);
|
||||
} else {
|
||||
reject(new Error(`Part ${partNumber} failed with HTTP ${xhr.status}`));
|
||||
}
|
||||
};
|
||||
xhr.onerror = () => reject(new Error(`Part ${partNumber}: network error`));
|
||||
xhr.send(blob);
|
||||
});
|
||||
return { PartNumber: partNumber, ETag: etag };
|
||||
}
|
||||
|
||||
async function start() {
|
||||
if (!file) return;
|
||||
setErrorMsg("");
|
||||
setPhase("starting");
|
||||
setProgress(0);
|
||||
|
||||
let init: any = null;
|
||||
try {
|
||||
const initRes = await apiRequest("POST", "/api/admin/uploads/init", {
|
||||
fileName: file.name,
|
||||
fileSize: file.size,
|
||||
contentType: file.type || "video/mp4",
|
||||
});
|
||||
init = await initRes.json();
|
||||
|
||||
setPhase("uploading");
|
||||
const parts: PartResult[] = [];
|
||||
const total: number = init.partCount;
|
||||
|
||||
for (let i = 0; i < total; i++) {
|
||||
const from = i * init.partSize;
|
||||
const to = Math.min(from + init.partSize, file.size);
|
||||
const blob = file.slice(from, to);
|
||||
|
||||
const urlRes = await apiRequest("POST", "/api/admin/uploads/part-url", {
|
||||
key: init.key,
|
||||
uploadId: init.uploadId,
|
||||
partNumber: i + 1,
|
||||
});
|
||||
const { url } = await urlRes.json();
|
||||
|
||||
parts.push(await uploadPart(url, blob, i + 1));
|
||||
setProgress(Math.round(((i + 1) / total) * 100));
|
||||
}
|
||||
|
||||
setPhase("finishing");
|
||||
await apiRequest("POST", "/api/admin/uploads/complete", {
|
||||
guid: init.guid,
|
||||
key: init.key,
|
||||
uploadId: init.uploadId,
|
||||
parts,
|
||||
title: title.trim(),
|
||||
artist: artist.trim() || null,
|
||||
description: description.trim(),
|
||||
genre,
|
||||
contentType,
|
||||
isPublic: false,
|
||||
fileSize: file.size,
|
||||
originalFileName: file.name,
|
||||
});
|
||||
|
||||
setPhase("done");
|
||||
toast({
|
||||
title: "Upload abgeschlossen",
|
||||
description: "Das Video wird jetzt umgewandelt und erscheint in Kürze in der Liste.",
|
||||
});
|
||||
queryClient.invalidateQueries({ queryKey: ["/api/admin/videos"] });
|
||||
queryClient.invalidateQueries({ queryKey: ["/api/admin/inventory"] });
|
||||
|
||||
setFile(null);
|
||||
setTitle("");
|
||||
setArtist("");
|
||||
setDescription("");
|
||||
if (fileRef.current) fileRef.current.value = "";
|
||||
} catch (err: any) {
|
||||
console.error(err);
|
||||
setPhase("error");
|
||||
setErrorMsg(err?.message || "Upload fehlgeschlagen");
|
||||
if (init?.key && init?.uploadId) {
|
||||
apiRequest("POST", "/api/admin/uploads/abort", { key: init.key, uploadId: init.uploadId }).catch(() => {});
|
||||
}
|
||||
toast({
|
||||
title: "Upload fehlgeschlagen",
|
||||
description: err?.message || "Unbekannter Fehler",
|
||||
variant: "destructive",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<Card className="bg-white/10 border-white/20 text-white mb-6">
|
||||
<CardHeader>
|
||||
<CardTitle className="flex items-center space-x-2">
|
||||
<UploadCloud className="w-5 h-5" />
|
||||
<span>Video hochladen</span>
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<div>
|
||||
<Label className="text-white/90">Datei</Label>
|
||||
<Input
|
||||
ref={fileRef}
|
||||
type="file"
|
||||
accept="video/*,.mkv,.mxf,.mov"
|
||||
disabled={busy}
|
||||
onChange={(e) => pickFile(e.target.files?.[0] || null)}
|
||||
className="bg-white/10 border-white/20 text-white file:text-white"
|
||||
/>
|
||||
{file && (
|
||||
<p className="text-xs text-white/70 mt-1">
|
||||
{file.name} — {(file.size / 1024 / 1024 / 1024).toFixed(2)} GB
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<Label className="text-white/90">Titel</Label>
|
||||
<Input
|
||||
value={title}
|
||||
disabled={busy}
|
||||
onChange={(e) => setTitle(e.target.value)}
|
||||
className="bg-white/10 border-white/20 text-white"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Label className="text-white/90">Interpret</Label>
|
||||
<Input
|
||||
value={artist}
|
||||
disabled={busy}
|
||||
onChange={(e) => setArtist(e.target.value)}
|
||||
className="bg-white/10 border-white/20 text-white"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<Label className="text-white/90">Genre</Label>
|
||||
<Select value={genre} onValueChange={setGenre} disabled={busy}>
|
||||
<SelectTrigger className="bg-white/10 border-white/20 text-white">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{["volksmusik", "schlager", "pop", "rock", "country", "instrumental", "dance", "other"].map((g) => (
|
||||
<SelectItem key={g} value={g}>{g}</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div>
|
||||
<Label className="text-white/90">Typ</Label>
|
||||
<Select value={contentType} onValueChange={setContentType} disabled={busy}>
|
||||
<SelectTrigger className="bg-white/10 border-white/20 text-white">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{["video", "oddaja", "music_video", "documentary", "live"].map((c) => (
|
||||
<SelectItem key={c} value={c}>{c}</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Label className="text-white/90">Beschreibung</Label>
|
||||
<Textarea
|
||||
value={description}
|
||||
disabled={busy}
|
||||
onChange={(e) => setDescription(e.target.value)}
|
||||
className="bg-white/10 border-white/20 text-white"
|
||||
rows={3}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{phase === "uploading" && (
|
||||
<div>
|
||||
<div className="h-2 w-full bg-white/20 rounded overflow-hidden">
|
||||
<div className="h-full bg-emerald-400 transition-all" style={{ width: `${progress}%` }} />
|
||||
</div>
|
||||
<p className="text-xs text-white/70 mt-1">{progress}% in den Bucket übertragen</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{phase === "finishing" && (
|
||||
<p className="text-sm text-white/80 flex items-center gap-2">
|
||||
<Loader2 className="w-4 h-4 animate-spin" /> Upload wird abgeschlossen…
|
||||
</p>
|
||||
)}
|
||||
|
||||
{phase === "done" && (
|
||||
<p className="text-sm text-emerald-300 flex items-center gap-2">
|
||||
<CheckCircle2 className="w-4 h-4" /> Fertig — die Umwandlung läuft im Hintergrund.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{phase === "error" && <p className="text-sm text-red-300">{errorMsg}</p>}
|
||||
|
||||
<Button onClick={start} disabled={!file || busy} className="w-full">
|
||||
{busy ? <Loader2 className="w-4 h-4 mr-2 animate-spin" /> : <UploadCloud className="w-4 h-4 mr-2" />}
|
||||
{busy ? "Läuft…" : "Hochladen"}
|
||||
</Button>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@ -15,6 +15,8 @@ import { apiRequest } from "@/lib/queryClient";
|
||||
import type { Video } from "@shared/schema";
|
||||
import { Shield, Edit, Upload, Search, Filter, Save, X, Sparkles, Loader2 } from "lucide-react";
|
||||
import HeaderAd from "@/components/HeaderAd";
|
||||
import VideoUpload from "@/components/video-upload";
|
||||
import InventoryPanel from "@/components/inventory-panel";
|
||||
|
||||
export default function AdminPage() {
|
||||
const { user, isLoading: authLoading, isAuthenticated, isAdmin } = useAuth();
|
||||
@ -98,6 +100,8 @@ export default function AdminPage() {
|
||||
|
||||
{/* Main Content */}
|
||||
<div className="lg:col-span-3">
|
||||
<VideoUpload />
|
||||
<InventoryPanel />
|
||||
<VideoManagement search={search} onEditVideo={setSelectedVideo} onOpenDialog={setEditDialogOpen} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
176
server/routes.ts
176
server/routes.ts
@ -19,6 +19,14 @@ import fetch from "node-fetch";
|
||||
import { setupAuth, isAuthenticated, isAdmin } from "./replitAuth";
|
||||
import { ObjectStorageService, ObjectNotFoundError } from "./objectStorage";
|
||||
import { generateVideoDescription, generateBulkDescriptions } from "./aiService";
|
||||
import { db } from "./db";
|
||||
import { videos } from "@shared/schema";
|
||||
import { eq } from "drizzle-orm";
|
||||
import {
|
||||
initUpload, partUrl, completeUpload, abortUpload, writeJobFile,
|
||||
listPackagedGuids, listPendingJobs, publicUrlFor,
|
||||
FOLX_PREFIX, INCOMING_PREFIX,
|
||||
} from "./videoUpload";
|
||||
|
||||
// Extract unique artist names from video titles
|
||||
function extractArtists(videos: any[]): { name: string; videoCount: number; videos: any[] }[] {
|
||||
@ -2134,6 +2142,174 @@ Sitemap: ${baseUrl}/sitemap.xml
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
// DIRECT-TO-BUCKET VIDEO UPLOAD (browser → Hetzner S3, multipart)
|
||||
// Flow: init → part-url (per chunk) → complete → worker transcodes
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
|
||||
app.post('/api/admin/uploads/init', isAdmin, async (req, res) => {
|
||||
try {
|
||||
const { fileName, fileSize, contentType } = req.body || {};
|
||||
if (!fileName || !fileSize) {
|
||||
return res.status(400).json({ message: "fileName and fileSize are required" });
|
||||
}
|
||||
const size = parseInt(String(fileSize), 10);
|
||||
if (!Number.isFinite(size) || size <= 0) {
|
||||
return res.status(400).json({ message: "fileSize must be a positive number" });
|
||||
}
|
||||
const guid = randomUUID();
|
||||
const result = await initUpload(guid, fileName, size, contentType);
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
console.error("upload init failed:", error);
|
||||
res.status(500).json({ message: "Failed to start upload" });
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/api/admin/uploads/part-url', isAdmin, async (req, res) => {
|
||||
try {
|
||||
const { key, uploadId, partNumber } = req.body || {};
|
||||
if (!key || !uploadId || !partNumber) {
|
||||
return res.status(400).json({ message: "key, uploadId and partNumber are required" });
|
||||
}
|
||||
if (!String(key).startsWith(INCOMING_PREFIX)) {
|
||||
return res.status(400).json({ message: "Invalid key" });
|
||||
}
|
||||
const url = await partUrl(String(key), String(uploadId), parseInt(String(partNumber), 10));
|
||||
res.json({ url });
|
||||
} catch (error) {
|
||||
console.error("part url failed:", error);
|
||||
res.status(500).json({ message: "Failed to sign part" });
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/api/admin/uploads/complete', isAdmin, async (req, res) => {
|
||||
try {
|
||||
const { guid, key, uploadId, parts, title, artist, description, category, contentType, genre, isPublic, fileSize, originalFileName } = req.body || {};
|
||||
if (!guid || !key || !uploadId || !Array.isArray(parts) || parts.length === 0) {
|
||||
return res.status(400).json({ message: "guid, key, uploadId and parts are required" });
|
||||
}
|
||||
if (!String(key).startsWith(INCOMING_PREFIX)) {
|
||||
return res.status(400).json({ message: "Invalid key" });
|
||||
}
|
||||
|
||||
await completeUpload(String(key), String(uploadId), parts);
|
||||
|
||||
const cleanTitle = (title && String(title).trim()) || String(originalFileName || "Neues Video").replace(/\.[^.]+$/, "");
|
||||
|
||||
await writeJobFile(String(guid), {
|
||||
guid,
|
||||
sourceKey: key,
|
||||
title: cleanTitle,
|
||||
originalFileName: originalFileName || null,
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Row appears in admin immediately with status "processing"
|
||||
await db.insert(videos).values({
|
||||
id: String(guid),
|
||||
title: cleanTitle,
|
||||
artist: artist || null,
|
||||
description: description || "",
|
||||
thumbnailUrl: `${process.env.S3_PUBLIC_BASE || "https://folxvideos.b-cdn.net"}/${FOLX_PREFIX}${guid}/thumbnail.jpg`,
|
||||
videoUrl: publicUrlFor(String(guid)),
|
||||
duration: 0,
|
||||
category: category || "",
|
||||
contentType: contentType || 'video',
|
||||
genre: genre || 'other',
|
||||
isPublic: isPublic === undefined ? false : !!isPublic,
|
||||
uploadStatus: 'processing',
|
||||
originalFileName: originalFileName || null,
|
||||
fileSize: fileSize ? parseInt(String(fileSize), 10) : null,
|
||||
} as any);
|
||||
|
||||
res.json({ ok: true, guid, status: 'processing', videoUrl: publicUrlFor(String(guid)) });
|
||||
} catch (error) {
|
||||
console.error("upload complete failed:", error);
|
||||
res.status(500).json({ message: "Failed to finalise upload" });
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/api/admin/uploads/abort', isAdmin, async (req, res) => {
|
||||
try {
|
||||
const { key, uploadId } = req.body || {};
|
||||
if (key && uploadId) await abortUpload(String(key), String(uploadId));
|
||||
res.json({ ok: true });
|
||||
} catch {
|
||||
res.json({ ok: true });
|
||||
}
|
||||
});
|
||||
|
||||
// Called by the transcode worker on the remotion box when a package is ready.
|
||||
app.post('/api/internal/transcode-done', async (req, res) => {
|
||||
const expected = process.env.INTERNAL_TOKEN;
|
||||
const given = (req.headers.authorization || "").replace(/^Bearer\s+/i, "");
|
||||
if (!expected || given !== expected) {
|
||||
return res.status(401).json({ message: "Unauthorized" });
|
||||
}
|
||||
try {
|
||||
const { guid, status, duration, resolution, bitrate, format, encoding, errorMessage } = req.body || {};
|
||||
if (!guid) return res.status(400).json({ message: "guid required" });
|
||||
|
||||
if (status === 'failed') {
|
||||
await db.update(videos)
|
||||
.set({ uploadStatus: 'failed', description: errorMessage ? String(errorMessage).slice(0, 500) : undefined, updatedAt: new Date() } as any)
|
||||
.where(eq(videos.id, String(guid)));
|
||||
console.error(`transcode failed for ${guid}: ${errorMessage}`);
|
||||
return res.json({ ok: true });
|
||||
}
|
||||
|
||||
await db.update(videos)
|
||||
.set({
|
||||
uploadStatus: 'completed',
|
||||
duration: duration ? Math.round(Number(duration)) : 0,
|
||||
resolution: resolution || null,
|
||||
bitrate: bitrate ? Math.round(Number(bitrate)) : null,
|
||||
format: format || 'hls',
|
||||
encoding: encoding || 'h264',
|
||||
updatedAt: new Date(),
|
||||
} as any)
|
||||
.where(eq(videos.id, String(guid)));
|
||||
|
||||
console.log(`✅ transcode done: ${guid}`);
|
||||
res.json({ ok: true });
|
||||
} catch (error) {
|
||||
console.error("transcode-done failed:", error);
|
||||
res.status(500).json({ message: "Failed" });
|
||||
}
|
||||
});
|
||||
|
||||
// Inventory: what is in the bucket vs. what is in the database
|
||||
app.get('/api/admin/inventory', isAdmin, async (_req, res) => {
|
||||
try {
|
||||
const [packaged, pending, rows] = await Promise.all([
|
||||
listPackagedGuids(),
|
||||
listPendingJobs(),
|
||||
db.select({ id: videos.id, title: videos.title, uploadStatus: videos.uploadStatus, createdAt: videos.createdAt }).from(videos),
|
||||
]);
|
||||
|
||||
const dbIds = new Set(rows.map(r => r.id));
|
||||
const missingInBucket = rows
|
||||
.filter(r => !packaged.has(r.id))
|
||||
.map(r => ({ id: r.id, title: r.title, uploadStatus: r.uploadStatus }));
|
||||
const orphansInBucket = Array.from(packaged).filter(g => !dbIds.has(g));
|
||||
|
||||
res.json({
|
||||
databaseCount: rows.length,
|
||||
bucketCount: packaged.size,
|
||||
playable: rows.length - missingInBucket.length,
|
||||
processingCount: pending.length,
|
||||
missingInBucket,
|
||||
orphansInBucket,
|
||||
pendingJobs: pending,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("inventory failed:", error);
|
||||
res.status(500).json({ message: "Failed to build inventory" });
|
||||
}
|
||||
});
|
||||
|
||||
const httpServer = createServer(app);
|
||||
return httpServer;
|
||||
}
|
||||
|
||||
195
server/videoUpload.ts
Normal file
195
server/videoUpload.ts
Normal file
@ -0,0 +1,195 @@
|
||||
import {
|
||||
S3Client,
|
||||
CreateMultipartUploadCommand,
|
||||
UploadPartCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
AbortMultipartUploadCommand,
|
||||
ListObjectsV2Command,
|
||||
HeadObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
PutObjectCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||
|
||||
const S3_ENDPOINT = process.env.S3_ENDPOINT || "https://fsn1.your-objectstorage.com";
|
||||
const S3_BUCKET = process.env.S3_BUCKET || "folxvideos-pub";
|
||||
const S3_PUBLIC_BASE = process.env.S3_PUBLIC_BASE || "https://folxvideos.b-cdn.net";
|
||||
|
||||
// Prefix where finished HLS packages live: folx-tv/{guid}/master.m3u8
|
||||
export const FOLX_PREFIX = "folx-tv/";
|
||||
// Staging prefix the transcode worker polls: folx-tv/_incoming/{guid}/original.<ext>
|
||||
export const INCOMING_PREFIX = "folx-tv/_incoming/";
|
||||
|
||||
// 64 MB parts -> 10 000 parts max = 640 GB ceiling, plenty for our masters.
|
||||
export const PART_SIZE = 64 * 1024 * 1024;
|
||||
|
||||
const s3 = new S3Client({
|
||||
region: process.env.S3_REGION || "fsn1",
|
||||
endpoint: S3_ENDPOINT,
|
||||
forcePathStyle: true,
|
||||
credentials: {
|
||||
accessKeyId: process.env.S3_ACCESS_KEY_ID || "",
|
||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || "",
|
||||
},
|
||||
});
|
||||
|
||||
export interface InitResult {
|
||||
guid: string;
|
||||
key: string;
|
||||
uploadId: string;
|
||||
partSize: number;
|
||||
partCount: number;
|
||||
}
|
||||
|
||||
function safeExt(fileName: string): string {
|
||||
const m = /\.([a-zA-Z0-9]{1,5})$/.exec(fileName || "");
|
||||
const ext = (m ? m[1] : "mp4").toLowerCase();
|
||||
const allowed = ["mp4", "mov", "mkv", "m4v", "avi", "mxf", "mpg", "mpeg", "ts", "webm"];
|
||||
return allowed.includes(ext) ? ext : "mp4";
|
||||
}
|
||||
|
||||
export function publicUrlFor(guid: string): string {
|
||||
return `${S3_PUBLIC_BASE}/${FOLX_PREFIX}${guid}/master.m3u8`;
|
||||
}
|
||||
|
||||
/** Start a multipart upload for the original master file. */
|
||||
export async function initUpload(
|
||||
guid: string,
|
||||
fileName: string,
|
||||
fileSize: number,
|
||||
contentType?: string,
|
||||
): Promise<InitResult> {
|
||||
const ext = safeExt(fileName);
|
||||
const key = `${INCOMING_PREFIX}${guid}/original.${ext}`;
|
||||
const cmd = new CreateMultipartUploadCommand({
|
||||
Bucket: S3_BUCKET,
|
||||
Key: key,
|
||||
ContentType: contentType || "video/mp4",
|
||||
ACL: "public-read",
|
||||
});
|
||||
const res = await s3.send(cmd);
|
||||
if (!res.UploadId) throw new Error("S3 did not return an UploadId");
|
||||
return {
|
||||
guid,
|
||||
key,
|
||||
uploadId: res.UploadId,
|
||||
partSize: PART_SIZE,
|
||||
partCount: Math.max(1, Math.ceil(fileSize / PART_SIZE)),
|
||||
};
|
||||
}
|
||||
|
||||
/** Presigned PUT URL for one part. Valid 6 h so slow lines can finish. */
|
||||
export async function partUrl(key: string, uploadId: string, partNumber: number): Promise<string> {
|
||||
const cmd = new UploadPartCommand({
|
||||
Bucket: S3_BUCKET,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
PartNumber: partNumber,
|
||||
});
|
||||
return getSignedUrl(s3, cmd, { expiresIn: 6 * 3600 });
|
||||
}
|
||||
|
||||
export async function completeUpload(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
parts: Array<{ PartNumber: number; ETag: string }>,
|
||||
): Promise<void> {
|
||||
const ordered = [...parts].sort((a, b) => a.PartNumber - b.PartNumber);
|
||||
await s3.send(
|
||||
new CompleteMultipartUploadCommand({
|
||||
Bucket: S3_BUCKET,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: { Parts: ordered },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function abortUpload(key: string, uploadId: string): Promise<void> {
|
||||
try {
|
||||
await s3.send(new AbortMultipartUploadCommand({ Bucket: S3_BUCKET, Key: key, UploadId: uploadId }));
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
}
|
||||
|
||||
/** Drop a small JSON sidecar next to the original so the worker knows the metadata. */
|
||||
export async function writeJobFile(guid: string, job: Record<string, any>): Promise<void> {
|
||||
await s3.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: S3_BUCKET,
|
||||
Key: `${INCOMING_PREFIX}${guid}/job.json`,
|
||||
Body: JSON.stringify(job, null, 2),
|
||||
ContentType: "application/json",
|
||||
ACL: "public-read",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function objectExists(key: string): Promise<boolean> {
|
||||
try {
|
||||
await s3.send(new HeadObjectCommand({ Bucket: S3_BUCKET, Key: key }));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteObject(key: string): Promise<void> {
|
||||
try {
|
||||
await s3.send(new DeleteObjectCommand({ Bucket: S3_BUCKET, Key: key }));
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Every GUID that has a finished master.m3u8 in the bucket.
|
||||
* Used by the admin inventory page to diff bucket against database.
|
||||
*/
|
||||
export async function listPackagedGuids(): Promise<Set<string>> {
|
||||
const found = new Set<string>();
|
||||
let token: string | undefined = undefined;
|
||||
do {
|
||||
const res: any = await s3.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: S3_BUCKET,
|
||||
Prefix: FOLX_PREFIX,
|
||||
MaxKeys: 1000,
|
||||
ContinuationToken: token,
|
||||
}),
|
||||
);
|
||||
for (const obj of res.Contents || []) {
|
||||
const k: string = obj.Key || "";
|
||||
if (k.endsWith("/master.m3u8") && !k.startsWith(INCOMING_PREFIX)) {
|
||||
const guid = k.slice(FOLX_PREFIX.length, -"/master.m3u8".length);
|
||||
if (guid && !guid.includes("/")) found.add(guid);
|
||||
}
|
||||
}
|
||||
token = res.IsTruncated ? res.NextContinuationToken : undefined;
|
||||
} while (token);
|
||||
return found;
|
||||
}
|
||||
|
||||
/** Jobs still sitting in the staging prefix (uploaded but not yet packaged). */
|
||||
export async function listPendingJobs(): Promise<string[]> {
|
||||
const guids = new Set<string>();
|
||||
let token: string | undefined = undefined;
|
||||
do {
|
||||
const res: any = await s3.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: S3_BUCKET,
|
||||
Prefix: INCOMING_PREFIX,
|
||||
MaxKeys: 1000,
|
||||
ContinuationToken: token,
|
||||
}),
|
||||
);
|
||||
for (const obj of res.Contents || []) {
|
||||
const rest = (obj.Key || "").slice(INCOMING_PREFIX.length);
|
||||
const guid = rest.split("/")[0];
|
||||
if (guid) guids.add(guid);
|
||||
}
|
||||
token = res.IsTruncated ? res.NextContinuationToken : undefined;
|
||||
} while (token);
|
||||
return Array.from(guids);
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user