Direct-to-bucket video upload + transcode pipeline
- server/videoUpload.ts: S3 multipart presign helpers, bucket inventory listing
- routes: /api/admin/uploads/{init,part-url,complete,abort}, /api/admin/inventory,
/api/internal/transcode-done (worker callback, bearer INTERNAL_TOKEN)
- admin UI: browser-to-bucket uploader with progress, bucket-vs-database panel
Originals land in folx-tv/_incoming/{guid}/, the worker on the remotion box
packages them to HLS and reports back.
This commit is contained in:
Claude Agent
parent
f6c73f543c
commit
49d9c83201
106
client/src/components/inventory-panel.tsx
Normal file
106
client/src/components/inventory-panel.tsx
Normal file
@ -0,0 +1,106 @@
|
|||||||
|
import { useQuery } from "@tanstack/react-query";
|
||||||
|
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||||
|
import { Badge } from "@/components/ui/badge";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import { apiRequest } from "@/lib/queryClient";
|
||||||
|
import { Database, RefreshCw, AlertTriangle, Loader2 } from "lucide-react";
|
||||||
|
import { useState } from "react";
|
||||||
|
|
||||||
|
interface Inventory {
|
||||||
|
databaseCount: number;
|
||||||
|
bucketCount: number;
|
||||||
|
playable: number;
|
||||||
|
processingCount: number;
|
||||||
|
missingInBucket: Array<{ id: string; title: string; uploadStatus: string }>;
|
||||||
|
orphansInBucket: string[];
|
||||||
|
pendingJobs: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Side-by-side view of what the database claims and what the bucket actually holds. */
|
||||||
|
export default function InventoryPanel() {
|
||||||
|
const [expanded, setExpanded] = useState(false);
|
||||||
|
|
||||||
|
const { data, isLoading, refetch, isFetching } = useQuery<Inventory>({
|
||||||
|
queryKey: ["/api/admin/inventory"],
|
||||||
|
queryFn: async () => {
|
||||||
|
const res = await apiRequest("GET", "/api/admin/inventory");
|
||||||
|
return res.json();
|
||||||
|
},
|
||||||
|
refetchInterval: 60_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Card className="bg-white/10 border-white/20 text-white mb-6">
|
||||||
|
<CardHeader className="flex flex-row items-center justify-between space-y-0">
|
||||||
|
<CardTitle className="flex items-center space-x-2">
|
||||||
|
<Database className="w-5 h-5" />
|
||||||
|
<span>Bestand</span>
|
||||||
|
</CardTitle>
|
||||||
|
<Button
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
onClick={() => refetch()}
|
||||||
|
disabled={isFetching}
|
||||||
|
className="text-white border-white/20 hover:bg-white/10"
|
||||||
|
>
|
||||||
|
{isFetching ? <Loader2 className="w-4 h-4 animate-spin" /> : <RefreshCw className="w-4 h-4" />}
|
||||||
|
</Button>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent>
|
||||||
|
{isLoading || !data ? (
|
||||||
|
<p className="text-white/70 text-sm">Wird geladen…</p>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<div className="grid grid-cols-2 md:grid-cols-4 gap-4 mb-4">
|
||||||
|
<Stat label="In der Datenbank" value={data.databaseCount} />
|
||||||
|
<Stat label="Im Bucket" value={data.bucketCount} />
|
||||||
|
<Stat label="Abspielbar" value={data.playable} />
|
||||||
|
<Stat label="In Umwandlung" value={data.processingCount} />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{data.missingInBucket.length > 0 && (
|
||||||
|
<div className="rounded border border-amber-400/40 bg-amber-400/10 p-3">
|
||||||
|
<p className="flex items-center gap-2 text-sm text-amber-200">
|
||||||
|
<AlertTriangle className="w-4 h-4" />
|
||||||
|
{data.missingInBucket.length} Einträge ohne Dateien im Bucket — diese lassen sich nicht abspielen.
|
||||||
|
</p>
|
||||||
|
<Button
|
||||||
|
variant="link"
|
||||||
|
className="text-amber-200 px-0 h-auto"
|
||||||
|
onClick={() => setExpanded(!expanded)}
|
||||||
|
>
|
||||||
|
{expanded ? "Liste ausblenden" : "Liste anzeigen"}
|
||||||
|
</Button>
|
||||||
|
{expanded && (
|
||||||
|
<ul className="mt-2 max-h-64 overflow-auto text-xs space-y-1">
|
||||||
|
{data.missingInBucket.map((v) => (
|
||||||
|
<li key={v.id} className="flex justify-between gap-2">
|
||||||
|
<span className="truncate">{v.title}</span>
|
||||||
|
<Badge variant="secondary" className="shrink-0">{v.uploadStatus}</Badge>
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{data.orphansInBucket.length > 0 && (
|
||||||
|
<p className="mt-3 text-xs text-white/60">
|
||||||
|
{data.orphansInBucket.length} Pakete liegen im Bucket ohne Datenbankeintrag.
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function Stat({ label, value }: { label: string; value: number }) {
|
||||||
|
return (
|
||||||
|
<div className="rounded bg-white/5 p-3">
|
||||||
|
<p className="text-2xl font-semibold">{value}</p>
|
||||||
|
<p className="text-xs text-white/70">{label}</p>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
266
client/src/components/video-upload.tsx
Normal file
266
client/src/components/video-upload.tsx
Normal file
@ -0,0 +1,266 @@
|
|||||||
|
import { useRef, useState } from "react";
|
||||||
|
import { useQueryClient } from "@tanstack/react-query";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import { Input } from "@/components/ui/input";
|
||||||
|
import { Label } from "@/components/ui/label";
|
||||||
|
import { Textarea } from "@/components/ui/textarea";
|
||||||
|
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||||
|
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||||
|
import { useToast } from "@/hooks/use-toast";
|
||||||
|
import { apiRequest } from "@/lib/queryClient";
|
||||||
|
import { UploadCloud, Loader2, CheckCircle2 } from "lucide-react";
|
||||||
|
|
||||||
|
type Phase = "idle" | "starting" | "uploading" | "finishing" | "done" | "error";
|
||||||
|
|
||||||
|
interface PartResult {
|
||||||
|
PartNumber: number;
|
||||||
|
ETag: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Uploads the master file straight from the browser into the Hetzner bucket
|
||||||
|
* using presigned multipart URLs, then hands the GUID to the transcode worker.
|
||||||
|
* Nothing large ever passes through the application server.
|
||||||
|
*/
|
||||||
|
export default function VideoUpload() {
|
||||||
|
const { toast } = useToast();
|
||||||
|
const queryClient = useQueryClient();
|
||||||
|
const fileRef = useRef<HTMLInputElement>(null);
|
||||||
|
|
||||||
|
const [file, setFile] = useState<File | null>(null);
|
||||||
|
const [title, setTitle] = useState("");
|
||||||
|
const [artist, setArtist] = useState("");
|
||||||
|
const [description, setDescription] = useState("");
|
||||||
|
const [genre, setGenre] = useState("other");
|
||||||
|
const [contentType, setContentType] = useState("video");
|
||||||
|
const [phase, setPhase] = useState<Phase>("idle");
|
||||||
|
const [progress, setProgress] = useState(0);
|
||||||
|
const [errorMsg, setErrorMsg] = useState("");
|
||||||
|
|
||||||
|
const busy = phase === "starting" || phase === "uploading" || phase === "finishing";
|
||||||
|
|
||||||
|
function pickFile(f: File | null) {
|
||||||
|
setFile(f);
|
||||||
|
setPhase("idle");
|
||||||
|
setProgress(0);
|
||||||
|
setErrorMsg("");
|
||||||
|
if (f && !title) setTitle(f.name.replace(/\.[^.]+$/, ""));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uploadPart(url: string, blob: Blob, partNumber: number): Promise<PartResult> {
|
||||||
|
// XHR (not fetch) so we could extend this with per-part progress later.
|
||||||
|
const etag: string = await new Promise((resolve, reject) => {
|
||||||
|
const xhr = new XMLHttpRequest();
|
||||||
|
xhr.open("PUT", url, true);
|
||||||
|
xhr.onload = () => {
|
||||||
|
if (xhr.status >= 200 && xhr.status < 300) {
|
||||||
|
const tag = xhr.getResponseHeader("ETag") || xhr.getResponseHeader("etag") || "";
|
||||||
|
if (!tag) reject(new Error(`Part ${partNumber}: no ETag returned`));
|
||||||
|
else resolve(tag);
|
||||||
|
} else {
|
||||||
|
reject(new Error(`Part ${partNumber} failed with HTTP ${xhr.status}`));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
xhr.onerror = () => reject(new Error(`Part ${partNumber}: network error`));
|
||||||
|
xhr.send(blob);
|
||||||
|
});
|
||||||
|
return { PartNumber: partNumber, ETag: etag };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function start() {
|
||||||
|
if (!file) return;
|
||||||
|
setErrorMsg("");
|
||||||
|
setPhase("starting");
|
||||||
|
setProgress(0);
|
||||||
|
|
||||||
|
let init: any = null;
|
||||||
|
try {
|
||||||
|
const initRes = await apiRequest("POST", "/api/admin/uploads/init", {
|
||||||
|
fileName: file.name,
|
||||||
|
fileSize: file.size,
|
||||||
|
contentType: file.type || "video/mp4",
|
||||||
|
});
|
||||||
|
init = await initRes.json();
|
||||||
|
|
||||||
|
setPhase("uploading");
|
||||||
|
const parts: PartResult[] = [];
|
||||||
|
const total: number = init.partCount;
|
||||||
|
|
||||||
|
for (let i = 0; i < total; i++) {
|
||||||
|
const from = i * init.partSize;
|
||||||
|
const to = Math.min(from + init.partSize, file.size);
|
||||||
|
const blob = file.slice(from, to);
|
||||||
|
|
||||||
|
const urlRes = await apiRequest("POST", "/api/admin/uploads/part-url", {
|
||||||
|
key: init.key,
|
||||||
|
uploadId: init.uploadId,
|
||||||
|
partNumber: i + 1,
|
||||||
|
});
|
||||||
|
const { url } = await urlRes.json();
|
||||||
|
|
||||||
|
parts.push(await uploadPart(url, blob, i + 1));
|
||||||
|
setProgress(Math.round(((i + 1) / total) * 100));
|
||||||
|
}
|
||||||
|
|
||||||
|
setPhase("finishing");
|
||||||
|
await apiRequest("POST", "/api/admin/uploads/complete", {
|
||||||
|
guid: init.guid,
|
||||||
|
key: init.key,
|
||||||
|
uploadId: init.uploadId,
|
||||||
|
parts,
|
||||||
|
title: title.trim(),
|
||||||
|
artist: artist.trim() || null,
|
||||||
|
description: description.trim(),
|
||||||
|
genre,
|
||||||
|
contentType,
|
||||||
|
isPublic: false,
|
||||||
|
fileSize: file.size,
|
||||||
|
originalFileName: file.name,
|
||||||
|
});
|
||||||
|
|
||||||
|
setPhase("done");
|
||||||
|
toast({
|
||||||
|
title: "Upload abgeschlossen",
|
||||||
|
description: "Das Video wird jetzt umgewandelt und erscheint in Kürze in der Liste.",
|
||||||
|
});
|
||||||
|
queryClient.invalidateQueries({ queryKey: ["/api/admin/videos"] });
|
||||||
|
queryClient.invalidateQueries({ queryKey: ["/api/admin/inventory"] });
|
||||||
|
|
||||||
|
setFile(null);
|
||||||
|
setTitle("");
|
||||||
|
setArtist("");
|
||||||
|
setDescription("");
|
||||||
|
if (fileRef.current) fileRef.current.value = "";
|
||||||
|
} catch (err: any) {
|
||||||
|
console.error(err);
|
||||||
|
setPhase("error");
|
||||||
|
setErrorMsg(err?.message || "Upload fehlgeschlagen");
|
||||||
|
if (init?.key && init?.uploadId) {
|
||||||
|
apiRequest("POST", "/api/admin/uploads/abort", { key: init.key, uploadId: init.uploadId }).catch(() => {});
|
||||||
|
}
|
||||||
|
toast({
|
||||||
|
title: "Upload fehlgeschlagen",
|
||||||
|
description: err?.message || "Unbekannter Fehler",
|
||||||
|
variant: "destructive",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Card className="bg-white/10 border-white/20 text-white mb-6">
|
||||||
|
<CardHeader>
|
||||||
|
<CardTitle className="flex items-center space-x-2">
|
||||||
|
<UploadCloud className="w-5 h-5" />
|
||||||
|
<span>Video hochladen</span>
|
||||||
|
</CardTitle>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className="space-y-4">
|
||||||
|
<div>
|
||||||
|
<Label className="text-white/90">Datei</Label>
|
||||||
|
<Input
|
||||||
|
ref={fileRef}
|
||||||
|
type="file"
|
||||||
|
accept="video/*,.mkv,.mxf,.mov"
|
||||||
|
disabled={busy}
|
||||||
|
onChange={(e) => pickFile(e.target.files?.[0] || null)}
|
||||||
|
className="bg-white/10 border-white/20 text-white file:text-white"
|
||||||
|
/>
|
||||||
|
{file && (
|
||||||
|
<p className="text-xs text-white/70 mt-1">
|
||||||
|
{file.name} — {(file.size / 1024 / 1024 / 1024).toFixed(2)} GB
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||||
|
<div>
|
||||||
|
<Label className="text-white/90">Titel</Label>
|
||||||
|
<Input
|
||||||
|
value={title}
|
||||||
|
disabled={busy}
|
||||||
|
onChange={(e) => setTitle(e.target.value)}
|
||||||
|
className="bg-white/10 border-white/20 text-white"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<Label className="text-white/90">Interpret</Label>
|
||||||
|
<Input
|
||||||
|
value={artist}
|
||||||
|
disabled={busy}
|
||||||
|
onChange={(e) => setArtist(e.target.value)}
|
||||||
|
className="bg-white/10 border-white/20 text-white"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||||
|
<div>
|
||||||
|
<Label className="text-white/90">Genre</Label>
|
||||||
|
<Select value={genre} onValueChange={setGenre} disabled={busy}>
|
||||||
|
<SelectTrigger className="bg-white/10 border-white/20 text-white">
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
{["volksmusik", "schlager", "pop", "rock", "country", "instrumental", "dance", "other"].map((g) => (
|
||||||
|
<SelectItem key={g} value={g}>{g}</SelectItem>
|
||||||
|
))}
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<Label className="text-white/90">Typ</Label>
|
||||||
|
<Select value={contentType} onValueChange={setContentType} disabled={busy}>
|
||||||
|
<SelectTrigger className="bg-white/10 border-white/20 text-white">
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
{["video", "oddaja", "music_video", "documentary", "live"].map((c) => (
|
||||||
|
<SelectItem key={c} value={c}>{c}</SelectItem>
|
||||||
|
))}
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<Label className="text-white/90">Beschreibung</Label>
|
||||||
|
<Textarea
|
||||||
|
value={description}
|
||||||
|
disabled={busy}
|
||||||
|
onChange={(e) => setDescription(e.target.value)}
|
||||||
|
className="bg-white/10 border-white/20 text-white"
|
||||||
|
rows={3}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{phase === "uploading" && (
|
||||||
|
<div>
|
||||||
|
<div className="h-2 w-full bg-white/20 rounded overflow-hidden">
|
||||||
|
<div className="h-full bg-emerald-400 transition-all" style={{ width: `${progress}%` }} />
|
||||||
|
</div>
|
||||||
|
<p className="text-xs text-white/70 mt-1">{progress}% in den Bucket übertragen</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{phase === "finishing" && (
|
||||||
|
<p className="text-sm text-white/80 flex items-center gap-2">
|
||||||
|
<Loader2 className="w-4 h-4 animate-spin" /> Upload wird abgeschlossen…
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{phase === "done" && (
|
||||||
|
<p className="text-sm text-emerald-300 flex items-center gap-2">
|
||||||
|
<CheckCircle2 className="w-4 h-4" /> Fertig — die Umwandlung läuft im Hintergrund.
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{phase === "error" && <p className="text-sm text-red-300">{errorMsg}</p>}
|
||||||
|
|
||||||
|
<Button onClick={start} disabled={!file || busy} className="w-full">
|
||||||
|
{busy ? <Loader2 className="w-4 h-4 mr-2 animate-spin" /> : <UploadCloud className="w-4 h-4 mr-2" />}
|
||||||
|
{busy ? "Läuft…" : "Hochladen"}
|
||||||
|
</Button>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
);
|
||||||
|
}
|
||||||
@ -15,6 +15,8 @@ import { apiRequest } from "@/lib/queryClient";
|
|||||||
import type { Video } from "@shared/schema";
|
import type { Video } from "@shared/schema";
|
||||||
import { Shield, Edit, Upload, Search, Filter, Save, X, Sparkles, Loader2 } from "lucide-react";
|
import { Shield, Edit, Upload, Search, Filter, Save, X, Sparkles, Loader2 } from "lucide-react";
|
||||||
import HeaderAd from "@/components/HeaderAd";
|
import HeaderAd from "@/components/HeaderAd";
|
||||||
|
import VideoUpload from "@/components/video-upload";
|
||||||
|
import InventoryPanel from "@/components/inventory-panel";
|
||||||
|
|
||||||
export default function AdminPage() {
|
export default function AdminPage() {
|
||||||
const { user, isLoading: authLoading, isAuthenticated, isAdmin } = useAuth();
|
const { user, isLoading: authLoading, isAuthenticated, isAdmin } = useAuth();
|
||||||
@ -98,6 +100,8 @@ export default function AdminPage() {
|
|||||||
|
|
||||||
{/* Main Content */}
|
{/* Main Content */}
|
||||||
<div className="lg:col-span-3">
|
<div className="lg:col-span-3">
|
||||||
|
<VideoUpload />
|
||||||
|
<InventoryPanel />
|
||||||
<VideoManagement search={search} onEditVideo={setSelectedVideo} onOpenDialog={setEditDialogOpen} />
|
<VideoManagement search={search} onEditVideo={setSelectedVideo} onOpenDialog={setEditDialogOpen} />
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
176
server/routes.ts
176
server/routes.ts
@ -19,6 +19,14 @@ import fetch from "node-fetch";
|
|||||||
import { setupAuth, isAuthenticated, isAdmin } from "./replitAuth";
|
import { setupAuth, isAuthenticated, isAdmin } from "./replitAuth";
|
||||||
import { ObjectStorageService, ObjectNotFoundError } from "./objectStorage";
|
import { ObjectStorageService, ObjectNotFoundError } from "./objectStorage";
|
||||||
import { generateVideoDescription, generateBulkDescriptions } from "./aiService";
|
import { generateVideoDescription, generateBulkDescriptions } from "./aiService";
|
||||||
|
import { db } from "./db";
|
||||||
|
import { videos } from "@shared/schema";
|
||||||
|
import { eq } from "drizzle-orm";
|
||||||
|
import {
|
||||||
|
initUpload, partUrl, completeUpload, abortUpload, writeJobFile,
|
||||||
|
listPackagedGuids, listPendingJobs, publicUrlFor,
|
||||||
|
FOLX_PREFIX, INCOMING_PREFIX,
|
||||||
|
} from "./videoUpload";
|
||||||
|
|
||||||
// Extract unique artist names from video titles
|
// Extract unique artist names from video titles
|
||||||
function extractArtists(videos: any[]): { name: string; videoCount: number; videos: any[] }[] {
|
function extractArtists(videos: any[]): { name: string; videoCount: number; videos: any[] }[] {
|
||||||
@ -2134,6 +2142,174 @@ Sitemap: ${baseUrl}/sitemap.xml
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────
|
||||||
|
// DIRECT-TO-BUCKET VIDEO UPLOAD (browser → Hetzner S3, multipart)
|
||||||
|
// Flow: init → part-url (per chunk) → complete → worker transcodes
|
||||||
|
// ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
app.post('/api/admin/uploads/init', isAdmin, async (req, res) => {
|
||||||
|
try {
|
||||||
|
const { fileName, fileSize, contentType } = req.body || {};
|
||||||
|
if (!fileName || !fileSize) {
|
||||||
|
return res.status(400).json({ message: "fileName and fileSize are required" });
|
||||||
|
}
|
||||||
|
const size = parseInt(String(fileSize), 10);
|
||||||
|
if (!Number.isFinite(size) || size <= 0) {
|
||||||
|
return res.status(400).json({ message: "fileSize must be a positive number" });
|
||||||
|
}
|
||||||
|
const guid = randomUUID();
|
||||||
|
const result = await initUpload(guid, fileName, size, contentType);
|
||||||
|
res.json(result);
|
||||||
|
} catch (error) {
|
||||||
|
console.error("upload init failed:", error);
|
||||||
|
res.status(500).json({ message: "Failed to start upload" });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.post('/api/admin/uploads/part-url', isAdmin, async (req, res) => {
|
||||||
|
try {
|
||||||
|
const { key, uploadId, partNumber } = req.body || {};
|
||||||
|
if (!key || !uploadId || !partNumber) {
|
||||||
|
return res.status(400).json({ message: "key, uploadId and partNumber are required" });
|
||||||
|
}
|
||||||
|
if (!String(key).startsWith(INCOMING_PREFIX)) {
|
||||||
|
return res.status(400).json({ message: "Invalid key" });
|
||||||
|
}
|
||||||
|
const url = await partUrl(String(key), String(uploadId), parseInt(String(partNumber), 10));
|
||||||
|
res.json({ url });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("part url failed:", error);
|
||||||
|
res.status(500).json({ message: "Failed to sign part" });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.post('/api/admin/uploads/complete', isAdmin, async (req, res) => {
|
||||||
|
try {
|
||||||
|
const { guid, key, uploadId, parts, title, artist, description, category, contentType, genre, isPublic, fileSize, originalFileName } = req.body || {};
|
||||||
|
if (!guid || !key || !uploadId || !Array.isArray(parts) || parts.length === 0) {
|
||||||
|
return res.status(400).json({ message: "guid, key, uploadId and parts are required" });
|
||||||
|
}
|
||||||
|
if (!String(key).startsWith(INCOMING_PREFIX)) {
|
||||||
|
return res.status(400).json({ message: "Invalid key" });
|
||||||
|
}
|
||||||
|
|
||||||
|
await completeUpload(String(key), String(uploadId), parts);
|
||||||
|
|
||||||
|
const cleanTitle = (title && String(title).trim()) || String(originalFileName || "Neues Video").replace(/\.[^.]+$/, "");
|
||||||
|
|
||||||
|
await writeJobFile(String(guid), {
|
||||||
|
guid,
|
||||||
|
sourceKey: key,
|
||||||
|
title: cleanTitle,
|
||||||
|
originalFileName: originalFileName || null,
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Row appears in admin immediately with status "processing"
|
||||||
|
await db.insert(videos).values({
|
||||||
|
id: String(guid),
|
||||||
|
title: cleanTitle,
|
||||||
|
artist: artist || null,
|
||||||
|
description: description || "",
|
||||||
|
thumbnailUrl: `${process.env.S3_PUBLIC_BASE || "https://folxvideos.b-cdn.net"}/${FOLX_PREFIX}${guid}/thumbnail.jpg`,
|
||||||
|
videoUrl: publicUrlFor(String(guid)),
|
||||||
|
duration: 0,
|
||||||
|
category: category || "",
|
||||||
|
contentType: contentType || 'video',
|
||||||
|
genre: genre || 'other',
|
||||||
|
isPublic: isPublic === undefined ? false : !!isPublic,
|
||||||
|
uploadStatus: 'processing',
|
||||||
|
originalFileName: originalFileName || null,
|
||||||
|
fileSize: fileSize ? parseInt(String(fileSize), 10) : null,
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
res.json({ ok: true, guid, status: 'processing', videoUrl: publicUrlFor(String(guid)) });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("upload complete failed:", error);
|
||||||
|
res.status(500).json({ message: "Failed to finalise upload" });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.post('/api/admin/uploads/abort', isAdmin, async (req, res) => {
|
||||||
|
try {
|
||||||
|
const { key, uploadId } = req.body || {};
|
||||||
|
if (key && uploadId) await abortUpload(String(key), String(uploadId));
|
||||||
|
res.json({ ok: true });
|
||||||
|
} catch {
|
||||||
|
res.json({ ok: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Called by the transcode worker on the remotion box when a package is ready.
|
||||||
|
app.post('/api/internal/transcode-done', async (req, res) => {
|
||||||
|
const expected = process.env.INTERNAL_TOKEN;
|
||||||
|
const given = (req.headers.authorization || "").replace(/^Bearer\s+/i, "");
|
||||||
|
if (!expected || given !== expected) {
|
||||||
|
return res.status(401).json({ message: "Unauthorized" });
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const { guid, status, duration, resolution, bitrate, format, encoding, errorMessage } = req.body || {};
|
||||||
|
if (!guid) return res.status(400).json({ message: "guid required" });
|
||||||
|
|
||||||
|
if (status === 'failed') {
|
||||||
|
await db.update(videos)
|
||||||
|
.set({ uploadStatus: 'failed', description: errorMessage ? String(errorMessage).slice(0, 500) : undefined, updatedAt: new Date() } as any)
|
||||||
|
.where(eq(videos.id, String(guid)));
|
||||||
|
console.error(`transcode failed for ${guid}: ${errorMessage}`);
|
||||||
|
return res.json({ ok: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
await db.update(videos)
|
||||||
|
.set({
|
||||||
|
uploadStatus: 'completed',
|
||||||
|
duration: duration ? Math.round(Number(duration)) : 0,
|
||||||
|
resolution: resolution || null,
|
||||||
|
bitrate: bitrate ? Math.round(Number(bitrate)) : null,
|
||||||
|
format: format || 'hls',
|
||||||
|
encoding: encoding || 'h264',
|
||||||
|
updatedAt: new Date(),
|
||||||
|
} as any)
|
||||||
|
.where(eq(videos.id, String(guid)));
|
||||||
|
|
||||||
|
console.log(`✅ transcode done: ${guid}`);
|
||||||
|
res.json({ ok: true });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("transcode-done failed:", error);
|
||||||
|
res.status(500).json({ message: "Failed" });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Inventory: what is in the bucket vs. what is in the database
|
||||||
|
app.get('/api/admin/inventory', isAdmin, async (_req, res) => {
|
||||||
|
try {
|
||||||
|
const [packaged, pending, rows] = await Promise.all([
|
||||||
|
listPackagedGuids(),
|
||||||
|
listPendingJobs(),
|
||||||
|
db.select({ id: videos.id, title: videos.title, uploadStatus: videos.uploadStatus, createdAt: videos.createdAt }).from(videos),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const dbIds = new Set(rows.map(r => r.id));
|
||||||
|
const missingInBucket = rows
|
||||||
|
.filter(r => !packaged.has(r.id))
|
||||||
|
.map(r => ({ id: r.id, title: r.title, uploadStatus: r.uploadStatus }));
|
||||||
|
const orphansInBucket = Array.from(packaged).filter(g => !dbIds.has(g));
|
||||||
|
|
||||||
|
res.json({
|
||||||
|
databaseCount: rows.length,
|
||||||
|
bucketCount: packaged.size,
|
||||||
|
playable: rows.length - missingInBucket.length,
|
||||||
|
processingCount: pending.length,
|
||||||
|
missingInBucket,
|
||||||
|
orphansInBucket,
|
||||||
|
pendingJobs: pending,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error("inventory failed:", error);
|
||||||
|
res.status(500).json({ message: "Failed to build inventory" });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
const httpServer = createServer(app);
|
const httpServer = createServer(app);
|
||||||
return httpServer;
|
return httpServer;
|
||||||
}
|
}
|
||||||
|
|||||||
195
server/videoUpload.ts
Normal file
195
server/videoUpload.ts
Normal file
@ -0,0 +1,195 @@
|
|||||||
|
import {
|
||||||
|
S3Client,
|
||||||
|
CreateMultipartUploadCommand,
|
||||||
|
UploadPartCommand,
|
||||||
|
CompleteMultipartUploadCommand,
|
||||||
|
AbortMultipartUploadCommand,
|
||||||
|
ListObjectsV2Command,
|
||||||
|
HeadObjectCommand,
|
||||||
|
DeleteObjectCommand,
|
||||||
|
PutObjectCommand,
|
||||||
|
} from "@aws-sdk/client-s3";
|
||||||
|
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||||
|
|
||||||
|
const S3_ENDPOINT = process.env.S3_ENDPOINT || "https://fsn1.your-objectstorage.com";
|
||||||
|
const S3_BUCKET = process.env.S3_BUCKET || "folxvideos-pub";
|
||||||
|
const S3_PUBLIC_BASE = process.env.S3_PUBLIC_BASE || "https://folxvideos.b-cdn.net";
|
||||||
|
|
||||||
|
// Prefix where finished HLS packages live: folx-tv/{guid}/master.m3u8
|
||||||
|
export const FOLX_PREFIX = "folx-tv/";
|
||||||
|
// Staging prefix the transcode worker polls: folx-tv/_incoming/{guid}/original.<ext>
|
||||||
|
export const INCOMING_PREFIX = "folx-tv/_incoming/";
|
||||||
|
|
||||||
|
// 64 MB parts -> 10 000 parts max = 640 GB ceiling, plenty for our masters.
|
||||||
|
export const PART_SIZE = 64 * 1024 * 1024;
|
||||||
|
|
||||||
|
const s3 = new S3Client({
|
||||||
|
region: process.env.S3_REGION || "fsn1",
|
||||||
|
endpoint: S3_ENDPOINT,
|
||||||
|
forcePathStyle: true,
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: process.env.S3_ACCESS_KEY_ID || "",
|
||||||
|
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || "",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
export interface InitResult {
|
||||||
|
guid: string;
|
||||||
|
key: string;
|
||||||
|
uploadId: string;
|
||||||
|
partSize: number;
|
||||||
|
partCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function safeExt(fileName: string): string {
|
||||||
|
const m = /\.([a-zA-Z0-9]{1,5})$/.exec(fileName || "");
|
||||||
|
const ext = (m ? m[1] : "mp4").toLowerCase();
|
||||||
|
const allowed = ["mp4", "mov", "mkv", "m4v", "avi", "mxf", "mpg", "mpeg", "ts", "webm"];
|
||||||
|
return allowed.includes(ext) ? ext : "mp4";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function publicUrlFor(guid: string): string {
|
||||||
|
return `${S3_PUBLIC_BASE}/${FOLX_PREFIX}${guid}/master.m3u8`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Start a multipart upload for the original master file. */
|
||||||
|
export async function initUpload(
|
||||||
|
guid: string,
|
||||||
|
fileName: string,
|
||||||
|
fileSize: number,
|
||||||
|
contentType?: string,
|
||||||
|
): Promise<InitResult> {
|
||||||
|
const ext = safeExt(fileName);
|
||||||
|
const key = `${INCOMING_PREFIX}${guid}/original.${ext}`;
|
||||||
|
const cmd = new CreateMultipartUploadCommand({
|
||||||
|
Bucket: S3_BUCKET,
|
||||||
|
Key: key,
|
||||||
|
ContentType: contentType || "video/mp4",
|
||||||
|
ACL: "public-read",
|
||||||
|
});
|
||||||
|
const res = await s3.send(cmd);
|
||||||
|
if (!res.UploadId) throw new Error("S3 did not return an UploadId");
|
||||||
|
return {
|
||||||
|
guid,
|
||||||
|
key,
|
||||||
|
uploadId: res.UploadId,
|
||||||
|
partSize: PART_SIZE,
|
||||||
|
partCount: Math.max(1, Math.ceil(fileSize / PART_SIZE)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Presigned PUT URL for one part. Valid 6 h so slow lines can finish. */
|
||||||
|
export async function partUrl(key: string, uploadId: string, partNumber: number): Promise<string> {
|
||||||
|
const cmd = new UploadPartCommand({
|
||||||
|
Bucket: S3_BUCKET,
|
||||||
|
Key: key,
|
||||||
|
UploadId: uploadId,
|
||||||
|
PartNumber: partNumber,
|
||||||
|
});
|
||||||
|
return getSignedUrl(s3, cmd, { expiresIn: 6 * 3600 });
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function completeUpload(
|
||||||
|
key: string,
|
||||||
|
uploadId: string,
|
||||||
|
parts: Array<{ PartNumber: number; ETag: string }>,
|
||||||
|
): Promise<void> {
|
||||||
|
const ordered = [...parts].sort((a, b) => a.PartNumber - b.PartNumber);
|
||||||
|
await s3.send(
|
||||||
|
new CompleteMultipartUploadCommand({
|
||||||
|
Bucket: S3_BUCKET,
|
||||||
|
Key: key,
|
||||||
|
UploadId: uploadId,
|
||||||
|
MultipartUpload: { Parts: ordered },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function abortUpload(key: string, uploadId: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
await s3.send(new AbortMultipartUploadCommand({ Bucket: S3_BUCKET, Key: key, UploadId: uploadId }));
|
||||||
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Drop a small JSON sidecar next to the original so the worker knows the metadata. */
|
||||||
|
export async function writeJobFile(guid: string, job: Record<string, any>): Promise<void> {
|
||||||
|
await s3.send(
|
||||||
|
new PutObjectCommand({
|
||||||
|
Bucket: S3_BUCKET,
|
||||||
|
Key: `${INCOMING_PREFIX}${guid}/job.json`,
|
||||||
|
Body: JSON.stringify(job, null, 2),
|
||||||
|
ContentType: "application/json",
|
||||||
|
ACL: "public-read",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function objectExists(key: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
await s3.send(new HeadObjectCommand({ Bucket: S3_BUCKET, Key: key }));
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteObject(key: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
await s3.send(new DeleteObjectCommand({ Bucket: S3_BUCKET, Key: key }));
|
||||||
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every GUID that has a finished master.m3u8 in the bucket.
|
||||||
|
* Used by the admin inventory page to diff bucket against database.
|
||||||
|
*/
|
||||||
|
export async function listPackagedGuids(): Promise<Set<string>> {
|
||||||
|
const found = new Set<string>();
|
||||||
|
let token: string | undefined = undefined;
|
||||||
|
do {
|
||||||
|
const res: any = await s3.send(
|
||||||
|
new ListObjectsV2Command({
|
||||||
|
Bucket: S3_BUCKET,
|
||||||
|
Prefix: FOLX_PREFIX,
|
||||||
|
MaxKeys: 1000,
|
||||||
|
ContinuationToken: token,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
for (const obj of res.Contents || []) {
|
||||||
|
const k: string = obj.Key || "";
|
||||||
|
if (k.endsWith("/master.m3u8") && !k.startsWith(INCOMING_PREFIX)) {
|
||||||
|
const guid = k.slice(FOLX_PREFIX.length, -"/master.m3u8".length);
|
||||||
|
if (guid && !guid.includes("/")) found.add(guid);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
token = res.IsTruncated ? res.NextContinuationToken : undefined;
|
||||||
|
} while (token);
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Jobs still sitting in the staging prefix (uploaded but not yet packaged). */
|
||||||
|
export async function listPendingJobs(): Promise<string[]> {
|
||||||
|
const guids = new Set<string>();
|
||||||
|
let token: string | undefined = undefined;
|
||||||
|
do {
|
||||||
|
const res: any = await s3.send(
|
||||||
|
new ListObjectsV2Command({
|
||||||
|
Bucket: S3_BUCKET,
|
||||||
|
Prefix: INCOMING_PREFIX,
|
||||||
|
MaxKeys: 1000,
|
||||||
|
ContinuationToken: token,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
for (const obj of res.Contents || []) {
|
||||||
|
const rest = (obj.Key || "").slice(INCOMING_PREFIX.length);
|
||||||
|
const guid = rest.split("/")[0];
|
||||||
|
if (guid) guids.add(guid);
|
||||||
|
}
|
||||||
|
token = res.IsTruncated ? res.NextContinuationToken : undefined;
|
||||||
|
} while (token);
|
||||||
|
return Array.from(guids);
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user