Dedup: SQLite baza za že obdelane komade

User feedback: 'dodaj da če čekira in shranjuje že obdelani komadi v SQL bazo,
da če nalagamo komad ki smo ga že naložili da ga ne naloži'

NEW: SQLite dedup database at /data/processed.db
Schema: processed_videos
  - normalized_name (PK part 1)
  - tv_station (PK part 2) — isti komad lahko obstaja na različnih postajah
  - filename_orig
  - job_id
  - nextcloud_url
  - file_size_mb
  - uploaded_at

Filename normalization removes noise:
  'BRAJDE (Official Video).mp4' → 'brajde'
  'Brajde (HD).mxf' → 'brajde'
  'BRAJDE - LIVE 2024.mp4' → 'brajde'
(strips parentheses, suffixes like Official/HD/4K/Live, extension, lowercase)

NEW endpoints:
- POST /api/dedup/check — preveri katera imena so že obdelana
- POST /api/dedup/remove — pobriše dedup zapis (Re-process)
- GET /api/dedup/list — seznam vseh obdelanih (opt. filter po tv_station)

Integration:
- Nextcloud upload (manual + auto): zabeleži v dedup po uspešnem PUT
- File queue (frontend): pred dodajanjem preveri dedup
  → prikaže rdeč warning '⚠ Že naložen na ONE DE (29.4.2026) — Re-process'
  → opacity 0.6 (vizualno blediji)
  → submit jih SKIP-a (osim če 'Re-process' kliknil)
This commit is contained in:
2026-04-30 15:00:10 +00:00
parent 16c332b490
commit f2034f9970
2 changed files with 231 additions and 5 deletions
+73 -5
View File
@@ -547,11 +547,35 @@
return [null, null];
}
function addFilesToQueue(files) {
async function addFilesToQueue(files) {
const newItems = [];
for (const f of files) {
const [artist, title] = parseArtistTitle(f.name);
pendingFiles.push({ file: f, artist, title });
newItems.push({ file: f, artist, title, dedup: null });
}
// Dedup check pred dodanjem v queue
const tvStation = $("#tv-station-input").value || "FOLX SLOVENIJA";
try {
const r = await fetch("/api/dedup/check", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ filenames: newItems.map(i => i.file.name), tv_station: tvStation }),
});
if (r.ok) {
const data = await r.json();
newItems.forEach(item => {
const match = data.results[item.file.name];
if (match) {
item.dedup = match; // {normalized_name, tv_station, filename_orig, job_id, nextcloud_url, file_size_mb, uploaded_at}
}
});
}
} catch (e) {
console.warn("Dedup check failed:", e);
}
pendingFiles.push(...newItems);
renderFileQueue();
}
@@ -560,6 +584,26 @@
renderFileQueue();
}
// Uporabnik želi vseeno re-process komada ki je bil že naložen
window.forceReprocess = async function(idx) {
const item = pendingFiles[idx];
if (!item || !item.dedup) return;
const tvStation = item.dedup.tv_station;
// Izbriši dedup zapis
try {
await fetch("/api/dedup/remove", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ filenames: [item.file.name], tv_station: tvStation }),
});
} catch (e) {
console.warn("Dedup remove failed:", e);
}
item.dedup = null;
item.forceReprocess = true;
renderFileQueue();
};
function renderFileQueue() {
const q = $("#file-queue");
if (!q) return;
@@ -589,11 +633,23 @@
nameHtml = `${escapeHtml(item.file.name)}` +
`<div class="warn">⚠ Brez razvidnega imena — ACR bo poskusil prepoznati</div>`;
}
// Dedup warning
if (item.dedup) {
const date = new Date(item.dedup.uploaded_at * 1000).toLocaleDateString("sl-SI");
nameHtml += `<div style="margin-top:4px; padding:4px 6px; background:rgba(239,68,68,0.15); border-left:3px solid #ef4444; border-radius:3px; font-size:11px; color:#fca5a5;">
⚠ <b>Že naložen na ${escapeHtml(item.dedup.tv_station)}</b> (${date}) — <a href="#" onclick="forceReprocess(${idx}); return false;" style="color:#ffd700; text-decoration:underline;">Re-process</a>
</div>`;
}
div.innerHTML = `
<div class="name">${nameHtml}</div>
<div class="size">${sizeMB} MB</div>
<button class="remove" data-idx="${idx}" title="Odstrani">×</button>
`;
if (item.dedup && !item.forceReprocess) {
div.style.opacity = "0.6";
}
q.appendChild(div);
});
q.appendChild(div);
});
@@ -769,11 +825,23 @@
// Generate batch ID za skupinsko sledenje (Telegram summary)
const batchId = "batch-" + Date.now().toString(36) + "-" + Math.random().toString(36).slice(2, 8);
const totalFiles = pendingFiles.length;
// Filtriraj ven dedup-ed items (uporabnik mora kliknili Re-process)
const filesToProcess = pendingFiles.filter(item => !item.dedup);
if (filesToProcess.length === 0) {
alert("Vsi izbrani komadi so že naloženi. Klikni 'Re-process' za ponovno obdelavo.");
$("#submit-btn").disabled = false;
return;
}
if (filesToProcess.length < pendingFiles.length) {
const skipped = pendingFiles.length - filesToProcess.length;
console.log(`Preskočil ${skipped} že obdelanih komadov`);
}
const totalFiles = filesToProcess.length;
// Upload + queue all files SEQUENTIALLY (1 hkrati za stabilnost)
for (let i = 0; i < pendingFiles.length; i++) {
const item = pendingFiles[i];
for (let i = 0; i < filesToProcess.length; i++) {
const item = filesToProcess[i];
const f = item.file;
const sizeMB = (f.size / 1024 / 1024).toFixed(1);