Compare commits
101
Commits
3ffa9740f0
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
08ca5893c7 | ||
|
|
d2c9a48cc2 | ||
|
|
12e8edba93 | ||
|
|
79f611ba73 | ||
|
|
2abd9daae1 | ||
|
|
576cc807b5 | ||
|
|
cd872d8bea | ||
|
|
2fcc4b8075 | ||
|
|
0d72d70f5d | ||
|
|
ec1d109e3b | ||
|
|
48bf0cf050 | ||
|
|
6a9e20da19 | ||
|
|
a960b7157f | ||
|
|
968eba7205 | ||
|
|
bc73fd8dd3 | ||
|
|
77075795ce | ||
|
|
7a7d7ea20d | ||
|
|
c1e00b7b73 | ||
|
|
6279b0ec03 | ||
|
|
46ec0cec9c | ||
|
|
0fe1a47295 | ||
|
|
6270c92b44 | ||
|
|
1f8565413a | ||
|
|
24e1b53aa8 | ||
|
|
b938d1e4d8 | ||
|
|
6f79aaea8d | ||
|
|
4febf0b844 | ||
|
|
2aec7f7a29 | ||
|
|
4e2c690bc5 | ||
|
|
376bb4db09 | ||
|
|
f5ef136bf4 | ||
|
|
40151f8f57 | ||
|
|
bdc1d14498 | ||
|
|
129f7d5f33 | ||
|
|
c58875c072 | ||
|
|
2de58ca7a5 | ||
|
|
d0e78cca02 | ||
|
|
c2a593de78 | ||
|
|
b9c8c066ec | ||
|
|
5f339e91b5 | ||
|
|
f2034f9970 | ||
|
|
16c332b490 | ||
|
|
1c11dfe630 | ||
|
|
8284181fb3 | ||
|
|
d03beddd0d | ||
|
|
dbb8ab3059 | ||
|
|
faf002d4f8 | ||
|
|
de094b76a5 | ||
|
|
02fbae7c4f | ||
|
|
5d817c586b | ||
|
|
4376f7529f | ||
|
|
1d6af29a23 | ||
|
|
47a114ce6a | ||
|
|
facfd6bd39 | ||
|
|
c94e6214ca | ||
|
|
63da3ad2e2 | ||
|
|
69062205fd | ||
|
|
0513768466 | ||
|
|
274ff80b34 | ||
|
|
ce827e939a | ||
|
|
4f7d8df659 | ||
|
|
47d313f39c | ||
|
|
8d7397699c | ||
|
|
a3550a444a | ||
|
|
446769b68f | ||
|
|
9dba2a1185 | ||
|
|
b4294e7113 | ||
|
|
7cb4302dcd | ||
|
|
f99574daff | ||
|
|
6cf52e1918 | ||
|
|
d8c2aae9c1 | ||
|
|
22bb3cfe02 | ||
|
|
dc1cb1ad27 | ||
|
|
865e21fe1a | ||
|
|
ab5424d37b | ||
|
|
64b0ed1edc | ||
|
|
4788a55643 | ||
|
|
0dd33c16f3 | ||
|
|
df6011c3cf | ||
|
|
d3b71942d2 | ||
|
|
d73453fe50 | ||
|
|
823eb3e91e | ||
|
|
108f6b1ad3 | ||
|
|
e06c3efb8e | ||
|
|
9df58212b2 | ||
|
|
91cc03658d | ||
|
|
157e6b781e | ||
|
|
a5097c5acc | ||
|
|
1cc8e8be35 | ||
|
|
b543057cee | ||
|
|
3877b822ff | ||
|
|
a30137f1f2 | ||
|
|
671b512917 | ||
|
|
90cdad516b | ||
|
|
389c26d012 | ||
|
|
4efd726176 | ||
|
|
81bae81401 | ||
|
|
7d00730051 | ||
|
|
40acad26f3 | ||
|
|
5f90085981 | ||
|
|
68247bb84c |
+1
-1
@@ -30,7 +30,7 @@ COPY templates/ ./templates/
|
||||
RUN mkdir -p ./static
|
||||
|
||||
# Data direktorij (Coolify bo prek Persistent Storage UI bind-al volume)
|
||||
RUN mkdir -p /data/uploads /data/outputs /data/jobs /data/cookies
|
||||
RUN mkdir -p /data/uploads /data/outputs /data/jobs /data/cookies /data/qnet
|
||||
|
||||
ENV DATA_DIR=/data
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
"""
|
||||
Cleanup module: removes local files older than N hours, *only if they exist in S3*.
|
||||
|
||||
Safe by design:
|
||||
- Never deletes a local file unless its S3 mirror is verified
|
||||
- Never deletes job metadata (jobs/*.json) — those are tiny
|
||||
- Default age threshold is conservative (48 h)
|
||||
- Dry-run mode for verification
|
||||
|
||||
Usage (cron):
|
||||
python3 -m app.cleanup --apply # actual delete
|
||||
python3 -m app.cleanup --dry-run # preview only (default)
|
||||
python3 -m app.cleanup --apply --hours 72 # custom age threshold
|
||||
|
||||
Suggested cron (every night 03:30):
|
||||
30 3 * * * cd /app && python3 -m app.cleanup --apply >> /data/cleanup.log 2>&1
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# Konfiguracija — ujema main.py
|
||||
DATA_DIR = Path(os.environ.get("DATA_DIR", "/data"))
|
||||
UPLOAD_DIR = DATA_DIR / "uploads"
|
||||
OUTPUT_DIR = DATA_DIR / "outputs"
|
||||
|
||||
DEFAULT_AGE_HOURS = 48
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [cleanup] %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _file_age_hours(p: Path) -> float:
|
||||
"""Return file age in hours based on mtime."""
|
||||
try:
|
||||
return (time.time() - p.stat().st_mtime) / 3600
|
||||
except OSError:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _scan_dir(d: Path, kind: str, min_age_h: float):
|
||||
"""Yield (local_path, kind, age_hours) for files older than threshold."""
|
||||
if not d.exists():
|
||||
return
|
||||
for p in d.iterdir():
|
||||
if not p.is_file():
|
||||
continue
|
||||
# Skip very small files (probably config/state) — only target real workfiles
|
||||
if p.stat().st_size < 1024 * 100: # <100 KB
|
||||
continue
|
||||
age = _file_age_hours(p)
|
||||
if age >= min_age_h:
|
||||
yield p, kind, age
|
||||
|
||||
|
||||
def cleanup(min_age_hours: float, apply: bool) -> dict:
|
||||
"""Run cleanup pass. Returns stats dict.
|
||||
|
||||
Logic:
|
||||
For each file older than min_age_hours in uploads/ and outputs/:
|
||||
- Verify S3 mirror exists (s3.exists())
|
||||
- If verified: delete local
|
||||
- If not verified: skip (warn) — never delete unverified
|
||||
"""
|
||||
from app import s3_storage
|
||||
|
||||
if not s3_storage.is_enabled():
|
||||
log.error("S3 not configured — refusing to run cleanup. Aborting.")
|
||||
return {"error": "s3_not_configured"}
|
||||
|
||||
stats = {
|
||||
"scanned": 0,
|
||||
"would_delete": 0,
|
||||
"deleted": 0,
|
||||
"skipped_no_s3_mirror": 0,
|
||||
"freed_mb": 0.0,
|
||||
"errors": 0,
|
||||
}
|
||||
deleted_files = []
|
||||
skipped_files = []
|
||||
|
||||
for d, kind in [(UPLOAD_DIR, "upload"), (OUTPUT_DIR, "output")]:
|
||||
for p, _, age_h in _scan_dir(d, kind, min_age_hours):
|
||||
stats["scanned"] += 1
|
||||
size_mb = p.stat().st_size / 1024 / 1024
|
||||
folder = "uploads" if kind == "upload" else "outputs"
|
||||
s3_key = f"{folder}/{p.name}"
|
||||
|
||||
# Verify S3 mirror
|
||||
try:
|
||||
s3_size = s3_storage.get_object_size(s3_key)
|
||||
except Exception as e:
|
||||
log.warning("S3 check failed for %s: %s", s3_key, e)
|
||||
stats["errors"] += 1
|
||||
continue
|
||||
|
||||
if s3_size is None:
|
||||
log.warning("SKIP — no S3 mirror: %s (age %.1fh, %.1f MB)",
|
||||
p.name, age_h, size_mb)
|
||||
stats["skipped_no_s3_mirror"] += 1
|
||||
skipped_files.append(p.name)
|
||||
continue
|
||||
|
||||
# Check size match (sanity)
|
||||
local_size = p.stat().st_size
|
||||
if abs(s3_size - local_size) > 1024: # >1 KB delta — suspicious
|
||||
log.warning("SKIP — size mismatch %s: local=%d s3=%d",
|
||||
p.name, local_size, s3_size)
|
||||
stats["skipped_no_s3_mirror"] += 1
|
||||
skipped_files.append(p.name)
|
||||
continue
|
||||
|
||||
stats["would_delete"] += 1
|
||||
stats["freed_mb"] += size_mb
|
||||
|
||||
if apply:
|
||||
try:
|
||||
p.unlink()
|
||||
stats["deleted"] += 1
|
||||
deleted_files.append(p.name)
|
||||
log.info("DEL %s (%.1f MB, age %.1fh, S3 verified)",
|
||||
p.name, size_mb, age_h)
|
||||
except OSError as e:
|
||||
log.error("Delete failed %s: %s", p, e)
|
||||
stats["errors"] += 1
|
||||
else:
|
||||
log.info("DRY %s (%.1f MB, age %.1fh, S3 verified)",
|
||||
p.name, size_mb, age_h)
|
||||
|
||||
log.info("=" * 60)
|
||||
log.info("Cleanup pass: %s", "APPLY" if apply else "DRY-RUN")
|
||||
log.info(" scanned: %d", stats["scanned"])
|
||||
log.info(" would-delete: %d", stats["would_delete"])
|
||||
log.info(" deleted: %d", stats["deleted"])
|
||||
log.info(" skipped (no S3): %d", stats["skipped_no_s3_mirror"])
|
||||
log.info(" errors: %d", stats["errors"])
|
||||
log.info(" freed: %.1f MB", stats["freed_mb"])
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true",
|
||||
help="Actually delete files (default is dry-run)")
|
||||
ap.add_argument("--dry-run", action="store_true",
|
||||
help="Preview only, do not delete (default)")
|
||||
ap.add_argument("--hours", type=float, default=DEFAULT_AGE_HOURS,
|
||||
help=f"Min file age in hours (default {DEFAULT_AGE_HOURS})")
|
||||
args = ap.parse_args()
|
||||
|
||||
apply = args.apply and not args.dry_run
|
||||
stats = cleanup(min_age_hours=args.hours, apply=apply)
|
||||
if stats.get("error"):
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+2187
-33
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,247 @@
|
||||
"""
|
||||
Qnet match modul — iz uploaded filename najde matching song v Qnet bazah.
|
||||
|
||||
Vrne (artist, title, station, confidence, matched_file).
|
||||
|
||||
Strategija:
|
||||
1. **Exact filename match** (case-insensitive, with normalized punctuation)
|
||||
— stem only, brez extension. Confidence: 1.0.
|
||||
2. **Normalized basename match** — odstrani noise besede (Official, HD, 4K,
|
||||
letnice, številke v oklepajih), normaliziraj presledke, primerjaj.
|
||||
Confidence: 0.9.
|
||||
3. **Artist+Title fuzzy match** — najprej parsa filename z obstoječim
|
||||
parse_artist_title, potem išče v bazi po normalized (artist|title)
|
||||
bigram-keywords. Confidence: 0.6–0.85 odvisno od ratio.
|
||||
4. Če ni zadetka → None.
|
||||
|
||||
Vrne lookup result dict:
|
||||
{
|
||||
"matched": bool,
|
||||
"method": "exact" | "normalized" | "fuzzy" | None,
|
||||
"confidence": float, # 0.0–1.0
|
||||
"artist": str,
|
||||
"title": str,
|
||||
"station": str,
|
||||
"file": str,
|
||||
}
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
import time
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
LOOKUP_PATH = Path(os.environ.get("QNET_LOOKUP_PATH", "/data/qnet/songs_lookup.json"))
|
||||
|
||||
# In-memory cache — re-load if file is newer than cache
|
||||
_cache: dict = {"mtime": 0.0, "songs": [], "by_norm_file": {}, "by_norm_stem": {}}
|
||||
|
||||
|
||||
def _norm(s: str) -> str:
|
||||
"""Aggressive normalize: lowercase, strip diacritics, kill punctuation."""
|
||||
if not s:
|
||||
return ""
|
||||
s = unicodedata.normalize("NFKD", s)
|
||||
s = "".join(c for c in s if not unicodedata.combining(c))
|
||||
s = s.lower()
|
||||
# Replace en-dash / em-dash / various separators with hyphen
|
||||
s = s.replace("–", "-").replace("—", "-").replace("|", "-")
|
||||
# Drop everything except alnum and space
|
||||
s = re.sub(r"[^a-z0-9\s]+", " ", s)
|
||||
s = re.sub(r"\s+", " ", s).strip()
|
||||
return s
|
||||
|
||||
|
||||
# Noise words pogosto v "Official Video" / YouTube ipd.
|
||||
_NOISE_WORDS = {
|
||||
"official", "officiel", "video", "musikvideo", "musicvideo", "music",
|
||||
"lyric", "lyrics", "audio", "hd", "4k", "8k", "uhd", "live", "remix",
|
||||
"remaster", "remastered", "version", "ver", "extended", "edit",
|
||||
"videoclip", "clip", "premiere", "premiera", "tv", "final", "promo",
|
||||
"1080p", "720p", "2160p",
|
||||
}
|
||||
|
||||
|
||||
def _norm_minus_noise(s: str) -> str:
|
||||
"""Norm + remove noise words + drop standalone digits 2/3/4-digit (years/versions)."""
|
||||
n = _norm(s)
|
||||
toks = []
|
||||
for t in n.split():
|
||||
if t in _NOISE_WORDS:
|
||||
continue
|
||||
if t.isdigit() and len(t) <= 4:
|
||||
continue
|
||||
toks.append(t)
|
||||
return " ".join(toks)
|
||||
|
||||
|
||||
def _load_db():
|
||||
"""Re-load lookup file if changed. Uses mtime check."""
|
||||
global _cache
|
||||
try:
|
||||
mtime = LOOKUP_PATH.stat().st_mtime
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
if mtime <= _cache["mtime"]:
|
||||
return True
|
||||
songs = json.loads(LOOKUP_PATH.read_text(encoding="utf-8"))
|
||||
by_norm_file = {} # normalized whole filename → song
|
||||
by_norm_stem = {} # normalized stem (no ext) → song
|
||||
by_norm_minus = {} # normalized minus noise → song
|
||||
for s in songs:
|
||||
f = s.get("file") or ""
|
||||
if not f:
|
||||
continue
|
||||
nf = _norm(f)
|
||||
ns = _norm(Path(f).stem)
|
||||
nm = _norm_minus_noise(Path(f).stem)
|
||||
# Prvi zadetek wins (po prvi station ki ima ta file — kasneje lahko
|
||||
# menjamo na "all stations match")
|
||||
by_norm_file.setdefault(nf, s)
|
||||
by_norm_stem.setdefault(ns, s)
|
||||
if nm:
|
||||
by_norm_minus.setdefault(nm, s)
|
||||
_cache = {
|
||||
"mtime": mtime,
|
||||
"songs": songs,
|
||||
"by_norm_file": by_norm_file,
|
||||
"by_norm_stem": by_norm_stem,
|
||||
"by_norm_minus": by_norm_minus,
|
||||
}
|
||||
print(f"[qnet_match] loaded {len(songs)} songs from {LOOKUP_PATH} (mtime={mtime})", flush=True)
|
||||
return True
|
||||
|
||||
|
||||
def _result(song: dict, method: str, confidence: float) -> dict:
|
||||
return {
|
||||
"matched": True,
|
||||
"method": method,
|
||||
"confidence": round(confidence, 3),
|
||||
"artist": song.get("artist", ""),
|
||||
"title": song.get("title", ""),
|
||||
"station": song.get("station", ""),
|
||||
"file": song.get("file", ""),
|
||||
}
|
||||
|
||||
|
||||
def _no_match() -> dict:
|
||||
return {
|
||||
"matched": False,
|
||||
"method": None,
|
||||
"confidence": 0.0,
|
||||
"artist": "",
|
||||
"title": "",
|
||||
"station": "",
|
||||
"file": "",
|
||||
}
|
||||
|
||||
|
||||
def match_filename(filename: str) -> dict:
|
||||
"""Najdi match za uploaded filename. Glavna API funkcija."""
|
||||
if not filename:
|
||||
return _no_match()
|
||||
if not _load_db():
|
||||
return _no_match() # baza ni naložena
|
||||
|
||||
stem = Path(filename).stem
|
||||
nf_full = _norm(filename)
|
||||
nf_stem = _norm(stem)
|
||||
nm_stem = _norm_minus_noise(stem)
|
||||
|
||||
# 1) Exact normalized match z extensionom
|
||||
if nf_full in _cache["by_norm_file"]:
|
||||
return _result(_cache["by_norm_file"][nf_full], "exact", 1.0)
|
||||
|
||||
# 2) Stem match (brez extension)
|
||||
if nf_stem in _cache["by_norm_stem"]:
|
||||
return _result(_cache["by_norm_stem"][nf_stem], "exact_stem", 0.95)
|
||||
|
||||
# 3) Stem match brez noise
|
||||
if nm_stem and nm_stem in _cache["by_norm_minus"]:
|
||||
return _result(_cache["by_norm_minus"][nm_stem], "normalized", 0.9)
|
||||
|
||||
# 4) Fuzzy substring — če se input vsebuje v song.file ali obratno (samo če dovolj dolgo)
|
||||
# Najprej probaj parse "Artist - Title" iz filename
|
||||
parsed_a, parsed_t = _parse_artist_title_simple(stem)
|
||||
if parsed_a and parsed_t:
|
||||
cand = _fuzzy_artist_title(parsed_a, parsed_t)
|
||||
if cand:
|
||||
return cand
|
||||
|
||||
# 5) Last-resort fuzzy — sequence ratio na stem proti vsem stem-om (slow but bounded)
|
||||
# Samo če stem >= 8 znakov, da ne dobimo random match-ev
|
||||
if len(nm_stem) >= 8:
|
||||
best = None
|
||||
best_ratio = 0.0
|
||||
for cand_stem, song in _cache["by_norm_minus"].items():
|
||||
ratio = SequenceMatcher(None, nm_stem, cand_stem).ratio()
|
||||
if ratio > best_ratio:
|
||||
best_ratio = ratio
|
||||
best = song
|
||||
if best and best_ratio >= 0.85:
|
||||
return _result(best, "fuzzy", best_ratio * 0.85)
|
||||
|
||||
return _no_match()
|
||||
|
||||
|
||||
def _parse_artist_title_simple(name: str):
|
||||
"""Lightweight parser za 'Artist - Title' / 'Artist – Title'."""
|
||||
if not name:
|
||||
return (None, None)
|
||||
# Probaj najprej en/em-dash (najbolj distinctive), potem hyphen
|
||||
for sep in [" – ", " — ", " - ", "_-_"]:
|
||||
if sep in name:
|
||||
parts = name.split(sep, 1)
|
||||
if len(parts) == 2:
|
||||
a = parts[0].strip()
|
||||
t = parts[1].strip()
|
||||
if a and t and len(a) <= 80 and len(t) <= 100:
|
||||
return (a, t)
|
||||
return (None, None)
|
||||
|
||||
|
||||
def _fuzzy_artist_title(artist: str, title: str) -> Optional[dict]:
|
||||
"""Najdi pesem v bazi po normalized artist+title fuzzy match."""
|
||||
na = _norm_minus_noise(artist)
|
||||
nt = _norm_minus_noise(title)
|
||||
if not na or not nt:
|
||||
return None
|
||||
|
||||
best = None
|
||||
best_score = 0.0
|
||||
for song in _cache["songs"]:
|
||||
sa = _norm_minus_noise(song.get("artist", ""))
|
||||
st = _norm_minus_noise(song.get("title", ""))
|
||||
if not sa or not st:
|
||||
continue
|
||||
ar = SequenceMatcher(None, na, sa).ratio()
|
||||
tr = SequenceMatcher(None, nt, st).ratio()
|
||||
score = (ar + tr) / 2
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best = song
|
||||
|
||||
if best and best_score >= 0.82:
|
||||
return _result(best, "fuzzy_at", best_score * 0.9)
|
||||
return None
|
||||
|
||||
|
||||
def db_stats() -> dict:
|
||||
"""Vrne statistiko baze za health check / admin endpoint."""
|
||||
_load_db()
|
||||
if not _cache["songs"]:
|
||||
return {"loaded": False, "total": 0, "stations": {}}
|
||||
by_station = {}
|
||||
for s in _cache["songs"]:
|
||||
st = s.get("station", "?")
|
||||
by_station[st] = by_station.get(st, 0) + 1
|
||||
return {
|
||||
"loaded": True,
|
||||
"total": len(_cache["songs"]),
|
||||
"stations": by_station,
|
||||
"mtime": _cache["mtime"],
|
||||
"age_seconds": time.time() - _cache["mtime"],
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
"""
|
||||
S3 storage abstraction for reels-app workfiles.
|
||||
|
||||
Bucket layout:
|
||||
s3://folxspeed/reels-app/
|
||||
uploads/{job_id}_yt.mp4
|
||||
uploads/{job_id}_yt.info.json
|
||||
outputs/{job_id}.mp4
|
||||
outputs/{job_id}_source_low.mp4
|
||||
outputs/{job_id}.analysis.json
|
||||
outputs/{job_id}.subtitles.srt
|
||||
outputs/{job_id}.subtitles.ass
|
||||
outputs/{job_id}_waveform_2400x72.png
|
||||
jobs/{job_id}.json
|
||||
|
||||
REELS/ prefix (final published reels for cron pickup) is NOT touched here.
|
||||
|
||||
ENV (set via Coolify):
|
||||
S3_ENDPOINT = https://fsn1.your-objectstorage.com
|
||||
S3_BUCKET = folxspeed
|
||||
S3_ACCESS_KEY = ...
|
||||
S3_SECRET_KEY = ...
|
||||
S3_PREFIX = reels-app/ (trailing slash required)
|
||||
S3_REGION = fsn1 (optional, default fsn1)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Iterator, Optional
|
||||
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config from environment
|
||||
# ---------------------------------------------------------------------------
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://fsn1.your-objectstorage.com")
|
||||
S3_BUCKET = os.getenv("S3_BUCKET", "folxspeed")
|
||||
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
|
||||
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "")
|
||||
S3_PREFIX = os.getenv("S3_PREFIX", "reels-app/")
|
||||
S3_REGION = os.getenv("S3_REGION", "fsn1")
|
||||
|
||||
# Make sure prefix ends with slash
|
||||
if S3_PREFIX and not S3_PREFIX.endswith("/"):
|
||||
S3_PREFIX += "/"
|
||||
|
||||
# Feature flag — if creds missing, all calls become no-ops (safe fallback during rollout)
|
||||
S3_ENABLED = bool(S3_ACCESS_KEY and S3_SECRET_KEY)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lazy client
|
||||
# ---------------------------------------------------------------------------
|
||||
_client = None
|
||||
|
||||
|
||||
def get_client():
|
||||
"""Return a cached boto3 S3 client, or None if S3 is not configured."""
|
||||
global _client
|
||||
if not S3_ENABLED:
|
||||
return None
|
||||
if _client is None:
|
||||
_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
aws_access_key_id=S3_ACCESS_KEY,
|
||||
aws_secret_access_key=S3_SECRET_KEY,
|
||||
region_name=S3_REGION,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
retries={"max_attempts": 3, "mode": "standard"},
|
||||
connect_timeout=10,
|
||||
read_timeout=120,
|
||||
),
|
||||
)
|
||||
return _client
|
||||
|
||||
|
||||
def _full_key(key: str) -> str:
|
||||
"""Prepend S3_PREFIX if not already present."""
|
||||
key = key.lstrip("/")
|
||||
if S3_PREFIX and not key.startswith(S3_PREFIX):
|
||||
return S3_PREFIX + key
|
||||
return key
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
def is_enabled() -> bool:
|
||||
"""True if S3 credentials are present and client is usable."""
|
||||
return S3_ENABLED
|
||||
|
||||
|
||||
def upload(local_path: str | Path, key: str, content_type: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Upload local file to s3://{bucket}/{prefix}{key}.
|
||||
Returns True on success, False on failure (logs error, does not raise).
|
||||
"""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
log.warning("S3 not configured, skipping upload of %s", key)
|
||||
return False
|
||||
|
||||
local = Path(local_path)
|
||||
if not local.exists():
|
||||
log.error("S3 upload: local file does not exist: %s", local)
|
||||
return False
|
||||
|
||||
full_key = _full_key(key)
|
||||
extra: dict = {}
|
||||
if content_type:
|
||||
extra["ContentType"] = content_type
|
||||
|
||||
try:
|
||||
client.upload_file(str(local), S3_BUCKET, full_key, ExtraArgs=extra or None)
|
||||
log.info("S3 upload OK: %s -> s3://%s/%s (%d bytes)",
|
||||
local.name, S3_BUCKET, full_key, local.stat().st_size)
|
||||
return True
|
||||
except ClientError as e:
|
||||
log.error("S3 upload FAILED %s: %s", full_key, e)
|
||||
return False
|
||||
except Exception as e:
|
||||
log.error("S3 upload UNEXPECTED ERROR %s: %s", full_key, e)
|
||||
return False
|
||||
|
||||
|
||||
def download(key: str, local_path: str | Path) -> bool:
|
||||
"""
|
||||
Download s3://{bucket}/{prefix}{key} to local_path.
|
||||
Creates parent directories if needed.
|
||||
Returns True on success.
|
||||
"""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
log.warning("S3 not configured, cannot download %s", key)
|
||||
return False
|
||||
|
||||
local = Path(local_path)
|
||||
local.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_key = _full_key(key)
|
||||
|
||||
try:
|
||||
client.download_file(S3_BUCKET, full_key, str(local))
|
||||
log.info("S3 download OK: s3://%s/%s -> %s", S3_BUCKET, full_key, local)
|
||||
return True
|
||||
except ClientError as e:
|
||||
code = e.response.get("Error", {}).get("Code", "")
|
||||
if code in ("NoSuchKey", "404"):
|
||||
log.info("S3 download: key not found: %s", full_key)
|
||||
else:
|
||||
log.error("S3 download FAILED %s: %s", full_key, e)
|
||||
return False
|
||||
except Exception as e:
|
||||
log.error("S3 download UNEXPECTED ERROR %s: %s", full_key, e)
|
||||
return False
|
||||
|
||||
|
||||
def exists(key: str) -> bool:
|
||||
"""Return True if s3://{bucket}/{prefix}{key} exists."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return False
|
||||
|
||||
full_key = _full_key(key)
|
||||
try:
|
||||
client.head_object(Bucket=S3_BUCKET, Key=full_key)
|
||||
return True
|
||||
except ClientError as e:
|
||||
code = e.response.get("Error", {}).get("Code", "")
|
||||
if code in ("404", "NoSuchKey", "NotFound"):
|
||||
return False
|
||||
log.error("S3 exists check FAILED %s: %s", full_key, e)
|
||||
return False
|
||||
|
||||
|
||||
def delete(key: str) -> bool:
|
||||
"""Delete a single object. Returns True if the call succeeded."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return False
|
||||
|
||||
full_key = _full_key(key)
|
||||
try:
|
||||
client.delete_object(Bucket=S3_BUCKET, Key=full_key)
|
||||
log.info("S3 delete OK: %s", full_key)
|
||||
return True
|
||||
except ClientError as e:
|
||||
log.error("S3 delete FAILED %s: %s", full_key, e)
|
||||
return False
|
||||
|
||||
|
||||
def presigned_url(key: str, expires_in: int = 3600, method: str = "get_object") -> Optional[str]:
|
||||
"""
|
||||
Generate a presigned URL for reading (default) or writing.
|
||||
method: 'get_object' or 'put_object'
|
||||
"""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
full_key = _full_key(key)
|
||||
try:
|
||||
url = client.generate_presigned_url(
|
||||
method,
|
||||
Params={"Bucket": S3_BUCKET, "Key": full_key},
|
||||
ExpiresIn=expires_in,
|
||||
)
|
||||
return url
|
||||
except ClientError as e:
|
||||
log.error("S3 presigned URL FAILED %s: %s", full_key, e)
|
||||
return None
|
||||
|
||||
|
||||
def list_keys(prefix: str = "", max_keys: int = 1000) -> Iterator[str]:
|
||||
"""
|
||||
Iterator over keys under S3_PREFIX + prefix.
|
||||
Returns keys WITHOUT S3_PREFIX (relative).
|
||||
"""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return
|
||||
|
||||
full_prefix = _full_key(prefix)
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
try:
|
||||
for page in paginator.paginate(
|
||||
Bucket=S3_BUCKET, Prefix=full_prefix, PaginationConfig={"PageSize": max_keys}
|
||||
):
|
||||
for obj in page.get("Contents", []):
|
||||
full_key = obj["Key"]
|
||||
# Strip prefix to return relative key
|
||||
if S3_PREFIX and full_key.startswith(S3_PREFIX):
|
||||
yield full_key[len(S3_PREFIX):]
|
||||
else:
|
||||
yield full_key
|
||||
except ClientError as e:
|
||||
log.error("S3 list FAILED prefix=%s: %s", full_prefix, e)
|
||||
|
||||
|
||||
def get_object_size(key: str) -> Optional[int]:
|
||||
"""Return size in bytes, or None if missing/error."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
full_key = _full_key(key)
|
||||
try:
|
||||
resp = client.head_object(Bucket=S3_BUCKET, Key=full_key)
|
||||
return resp.get("ContentLength")
|
||||
except ClientError:
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Convenience helpers tailored to reels-app file layout
|
||||
# ---------------------------------------------------------------------------
|
||||
def upload_job_file(job_id: str, kind: str, local_path: str | Path) -> bool:
|
||||
"""
|
||||
Upload a workfile by kind:
|
||||
'upload' -> uploads/{filename}
|
||||
'output' -> outputs/{filename}
|
||||
'job_meta' -> jobs/{filename}
|
||||
Filename comes from local_path.name.
|
||||
"""
|
||||
local = Path(local_path)
|
||||
folder_map = {"upload": "uploads", "output": "outputs", "job_meta": "jobs"}
|
||||
folder = folder_map.get(kind)
|
||||
if folder is None:
|
||||
log.error("upload_job_file: unknown kind %r", kind)
|
||||
return False
|
||||
return upload(local, f"{folder}/{local.name}")
|
||||
|
||||
|
||||
def download_upload(filename: str, local_path: str | Path) -> bool:
|
||||
"""Download from uploads/ prefix."""
|
||||
return download(f"uploads/{filename}", local_path)
|
||||
|
||||
|
||||
def download_output(filename: str, local_path: str | Path) -> bool:
|
||||
"""Download from outputs/ prefix."""
|
||||
return download(f"outputs/{filename}", local_path)
|
||||
|
||||
|
||||
def upload_exists(filename: str) -> bool:
|
||||
return exists(f"uploads/{filename}")
|
||||
|
||||
|
||||
def output_exists(filename: str) -> bool:
|
||||
return exists(f"outputs/{filename}")
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
telegram.py — Telegram bot helper za reels-app.
|
||||
|
||||
Pošilja obvestila o končanih jobih, batch summary, napakah.
|
||||
Credentials se preberejo iz env vars (TELEGRAM_TOKEN, TELEGRAM_CHAT_ID).
|
||||
"""
|
||||
import os
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import json
|
||||
|
||||
|
||||
def send_message(text, parse_mode="Markdown", disable_notification=False):
|
||||
"""Pošlji sporočilo na Telegram. Vrne True ob uspehu, False ob napaki.
|
||||
|
||||
text: ne sme biti daljši od 4096 znakov.
|
||||
parse_mode: 'Markdown' ali 'HTML' ali None.
|
||||
disable_notification: True za tiho obvestilo (brez zvonca).
|
||||
"""
|
||||
token = os.environ.get("TELEGRAM_TOKEN")
|
||||
chat_id = os.environ.get("TELEGRAM_CHAT_ID")
|
||||
if not token or not chat_id:
|
||||
return False
|
||||
|
||||
# Telegram limit: 4096 chars
|
||||
if len(text) > 4090:
|
||||
text = text[:4087] + "..."
|
||||
|
||||
data_dict = {
|
||||
"chat_id": chat_id,
|
||||
"text": text,
|
||||
}
|
||||
if parse_mode:
|
||||
data_dict["parse_mode"] = parse_mode
|
||||
if disable_notification:
|
||||
data_dict["disable_notification"] = "true"
|
||||
|
||||
data = urllib.parse.urlencode(data_dict).encode()
|
||||
req = urllib.request.Request(
|
||||
f"https://api.telegram.org/bot{token}/sendMessage",
|
||||
data=data,
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
result = json.loads(resp.read().decode())
|
||||
return result.get("ok", False)
|
||||
except Exception as e:
|
||||
print(f"⚠️ Telegram send failed: {e}", flush=True)
|
||||
return False
|
||||
|
||||
|
||||
def notify_job_done(job, base_url="https://reels.biba.live"):
|
||||
"""Obvesti o končanem reelu."""
|
||||
artist = job.get("parsed_artist", "")
|
||||
title = job.get("parsed_title", "")
|
||||
job_id = job.get("id", "")
|
||||
|
||||
if artist and title:
|
||||
name = f"*{artist} — {title}*"
|
||||
elif title:
|
||||
name = f"*{title}*"
|
||||
else:
|
||||
name = f"`{job_id}`"
|
||||
|
||||
duration = job.get("duration", 0)
|
||||
output_size_mb = job.get("output_size_mb") or round((job.get("output_size", 0) / 1024 / 1024), 1)
|
||||
|
||||
text = (
|
||||
f"✅ Reel pripravljen\n\n"
|
||||
f"{name}\n"
|
||||
f"⏱ {duration:.0f}s · 📦 {output_size_mb} MB\n\n"
|
||||
f"[Predogled & Download]({base_url})"
|
||||
)
|
||||
return send_message(text)
|
||||
|
||||
|
||||
def notify_job_failed(job, error_msg=""):
|
||||
"""Obvesti o neuspehu joba."""
|
||||
artist = job.get("parsed_artist", "")
|
||||
title = job.get("parsed_title", "")
|
||||
fname = job.get("filename", "")
|
||||
|
||||
if artist and title:
|
||||
name = f"*{artist} — {title}*"
|
||||
elif fname:
|
||||
name = f"`{fname}`"
|
||||
else:
|
||||
name = f"`{job.get('id', '?')}`"
|
||||
|
||||
err_short = (error_msg or "neznana napaka")[:200]
|
||||
text = f"⚠️ Reel ni uspel\n\n{name}\n\n```\n{err_short}\n```"
|
||||
return send_message(text)
|
||||
|
||||
|
||||
def notify_batch_complete(batch_id, total, succeeded, failed, base_url="https://reels.biba.live"):
|
||||
"""Obvesti o končanem batch-u (več jobov hkrati)."""
|
||||
if total <= 1:
|
||||
return False # ne pošiljaj batch summary za en sam job
|
||||
|
||||
if failed == 0:
|
||||
emoji = "🎉"
|
||||
head = f"Batch končan: vsi {total} reelov pripravljeni"
|
||||
else:
|
||||
emoji = "✅"
|
||||
head = f"Batch končan: {succeeded}/{total} uspelih"
|
||||
if failed > 0:
|
||||
head += f", {failed} neuspelih"
|
||||
|
||||
text = f"{emoji} {head}\n\n[Vsi reels →]({base_url})"
|
||||
return send_message(text)
|
||||
|
||||
|
||||
def notify_batch_started(batch_id, total):
|
||||
"""Obvesti o začetku batch-a (samo če > 1 job)."""
|
||||
if total <= 1:
|
||||
return False
|
||||
text = f"🎬 Batch za {total} datotek dodan v vrsto…"
|
||||
return send_message(text, disable_notification=True)
|
||||
@@ -6,3 +6,6 @@ faster-whisper==1.0.3
|
||||
opencv-python-headless==4.10.0.84
|
||||
numpy==1.26.4
|
||||
yt-dlp>=2025.10.0
|
||||
pyacrcloud==1.0.11
|
||||
requests==2.32.3
|
||||
boto3==1.35.49
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
"""
|
||||
acr_recognize.py — Audio recognition prek ACRCloud Audio Fingerprinting API.
|
||||
|
||||
Uporabi native binary za fingerprinting (3KB sample namesto cel audio).
|
||||
Vrne (artist, title) ali (None, None) če pesem ni prepoznana.
|
||||
|
||||
Credentials se preberejo iz env vars (ACR_ACCESS_KEY, ACR_SECRET_KEY, ACR_HOST).
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import hmac
|
||||
import base64
|
||||
import time
|
||||
import subprocess
|
||||
import tempfile
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import uuid
|
||||
|
||||
|
||||
def _sign(string_to_sign, secret):
|
||||
h = hmac.new(secret.encode('ascii'), string_to_sign.encode('ascii'), digestmod='sha1')
|
||||
return base64.b64encode(h.digest()).decode('ascii')
|
||||
|
||||
|
||||
def _build_multipart(fields, files):
|
||||
"""Sestavi multipart/form-data body."""
|
||||
boundary = uuid.uuid4().hex
|
||||
parts = []
|
||||
for k, v in fields.items():
|
||||
parts.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n".encode())
|
||||
for k, (fname, content, ctype) in files.items():
|
||||
parts.append(
|
||||
f"--{boundary}\r\nContent-Disposition: form-data; name=\"{k}\"; filename=\"{fname}\"\r\n"
|
||||
f"Content-Type: {ctype}\r\n\r\n".encode() + content + b"\r\n"
|
||||
)
|
||||
parts.append(f"--{boundary}--\r\n".encode())
|
||||
body = b"".join(parts)
|
||||
return body, boundary
|
||||
|
||||
|
||||
def recognize_audio_file(audio_path, timeout=30):
|
||||
"""Pošlji audio file (ali fingerprint) ACRCloud-u in vrni raw response.
|
||||
|
||||
audio_path: pot do MP3/WAV/M4A datoteke.
|
||||
Vrne dict z odgovorom ali None ob napaki.
|
||||
"""
|
||||
host = os.environ.get("ACR_HOST", "identify-eu-west-1.acrcloud.com")
|
||||
access_key = os.environ.get("ACR_ACCESS_KEY")
|
||||
secret_key = os.environ.get("ACR_SECRET_KEY")
|
||||
|
||||
if not access_key or not secret_key:
|
||||
print("⚠️ ACR_ACCESS_KEY/SECRET_KEY nista nastavljena", file=sys.stderr)
|
||||
return None
|
||||
|
||||
if not os.path.exists(audio_path):
|
||||
print(f"⚠️ Audio file ne obstaja: {audio_path}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
# Probaj native binary za fingerprint (manjši payload)
|
||||
fingerprint = None
|
||||
try:
|
||||
from acrcloud import acrcloud_extr_tool as acr
|
||||
fingerprint = acr.create_fingerprint_by_file(audio_path, 0, 30, 0)
|
||||
except (ImportError, Exception) as e:
|
||||
print(f" ℹ️ Fingerprint binary ni na voljo ({e}), uporabljam audio direktno", file=sys.stderr)
|
||||
|
||||
timestamp = str(int(time.time()))
|
||||
http_method = "POST"
|
||||
http_uri = "/v1/identify"
|
||||
signature_version = "1"
|
||||
|
||||
if fingerprint:
|
||||
data_type = "fingerprint"
|
||||
sample_data = fingerprint
|
||||
sample_name = "sample.fp"
|
||||
sample_ctype = "application/octet-stream"
|
||||
else:
|
||||
data_type = "audio"
|
||||
with open(audio_path, "rb") as f:
|
||||
sample_data = f.read()
|
||||
sample_name = "sample.mp3"
|
||||
sample_ctype = "audio/mpeg"
|
||||
|
||||
string_to_sign = f"{http_method}\n{http_uri}\n{access_key}\n{data_type}\n{signature_version}\n{timestamp}"
|
||||
signature = _sign(string_to_sign, secret_key)
|
||||
|
||||
fields = {
|
||||
"access_key": access_key,
|
||||
"sample_bytes": str(len(sample_data)),
|
||||
"timestamp": timestamp,
|
||||
"signature": signature,
|
||||
"data_type": data_type,
|
||||
"signature_version": signature_version,
|
||||
}
|
||||
files = {"sample": (sample_name, sample_data, sample_ctype)}
|
||||
body, boundary = _build_multipart(fields, files)
|
||||
|
||||
url = f"https://{host}/v1/identify"
|
||||
req = urllib.request.Request(
|
||||
url, data=body,
|
||||
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
except Exception as e:
|
||||
print(f"⚠️ ACR API napaka: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def extract_short_audio(video_path, duration=20, start_offset=15):
|
||||
"""Izloči kratek audio iz videa za ACR fingerprint.
|
||||
|
||||
Začnemo 15s v video (preskoči intro) in vzamemo 20s.
|
||||
Za MXF/multichannel: izberemo prvi audio stream, downmix v stereo.
|
||||
Vrne pot do tmp MP3 ali None.
|
||||
"""
|
||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".mp3")
|
||||
os.close(tmp_fd)
|
||||
cmd = [
|
||||
"ffmpeg", "-i", video_path,
|
||||
"-ss", str(start_offset),
|
||||
"-t", str(duration),
|
||||
"-map", "0:a:0", # samo prvi audio stream (varno za MXF z več streami)
|
||||
"-vn",
|
||||
"-ac", "2", # downmix v stereo (če multichannel)
|
||||
"-ar", "44100",
|
||||
"-b:a", "128k",
|
||||
"-f", "mp3", tmp_path,
|
||||
"-y", "-loglevel", "error"
|
||||
]
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
os.unlink(tmp_path)
|
||||
print(f"⚠️ ffmpeg napaka: {proc.stderr[:200]}", file=sys.stderr)
|
||||
return None
|
||||
return tmp_path
|
||||
|
||||
|
||||
def recognize_video(video_path):
|
||||
"""Glavni vstop: prepoznaj pesem v videu, vrni (artist, title) ali (None, None).
|
||||
|
||||
Probavamo 2 točki v videu (15s in 60s) za večjo robustnost — če ni intro/instrumental.
|
||||
"""
|
||||
if not os.environ.get("ACR_ACCESS_KEY"):
|
||||
return (None, None)
|
||||
|
||||
for start_offset in [15, 60]:
|
||||
audio = extract_short_audio(video_path, duration=20, start_offset=start_offset)
|
||||
if not audio:
|
||||
continue
|
||||
try:
|
||||
result = recognize_audio_file(audio)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(audio)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if not result:
|
||||
continue
|
||||
|
||||
status = result.get("status", {}).get("code")
|
||||
if status != 0:
|
||||
# 1001 = no result, drugi = napaka
|
||||
msg = result.get("status", {}).get("msg", "")
|
||||
print(f" ℹ️ ACR @{start_offset}s: status={status} ({msg})", file=sys.stderr)
|
||||
continue
|
||||
|
||||
# Uspeh — vzemi prvi music match
|
||||
music = result.get("metadata", {}).get("music", [])
|
||||
if not music:
|
||||
continue
|
||||
|
||||
first = music[0]
|
||||
title = first.get("title", "").strip()
|
||||
artists = first.get("artists", [])
|
||||
artist = artists[0].get("name", "").strip() if artists else ""
|
||||
|
||||
if artist and title:
|
||||
print(f" ✅ ACR @{start_offset}s prepoznal: {artist} - {title}", file=sys.stderr)
|
||||
return (artist, title)
|
||||
|
||||
print(" ⚠️ ACR pesem ni prepoznana", file=sys.stderr)
|
||||
return (None, None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print("Uporaba: python3 acr_recognize.py <video.mp4>")
|
||||
sys.exit(1)
|
||||
|
||||
artist, title = recognize_video(sys.argv[1])
|
||||
if artist and title:
|
||||
print(json.dumps({"artist": artist, "title": title}, ensure_ascii=False))
|
||||
sys.exit(0)
|
||||
else:
|
||||
print(json.dumps({"artist": None, "title": None}))
|
||||
sys.exit(1)
|
||||
+1416
-98
File diff suppressed because it is too large
Load Diff
+181
-14
@@ -25,7 +25,12 @@ import numpy as np
|
||||
|
||||
|
||||
def get_video_info(path):
|
||||
"""Vrni dict z width, height, fps, duration."""
|
||||
"""Vrni dict z width, height, fps, duration.
|
||||
|
||||
width/height so SQUARE-PIXEL dimenzije (popravljeno za anamorphic SAR).
|
||||
Pri broadcast 720x576 SAR 64:45 (PAL DV) bo vrnil 1024x576 (square pixel).
|
||||
Pri standardnem source z SAR 1:1 ostane nespremenjeno.
|
||||
"""
|
||||
cmd = [
|
||||
"ffprobe", "-v", "quiet", "-print_format", "json",
|
||||
"-show_streams", "-show_format", str(path)
|
||||
@@ -35,14 +40,105 @@ def get_video_info(path):
|
||||
fps_str = vstream["r_frame_rate"]
|
||||
num, den = fps_str.split("/")
|
||||
fps = float(num) / float(den)
|
||||
|
||||
raw_w = int(vstream["width"])
|
||||
raw_h = int(vstream["height"])
|
||||
|
||||
# SAR (sample aspect ratio) — razmerje stranic pikslja
|
||||
sar = vstream.get("sample_aspect_ratio", "1:1")
|
||||
try:
|
||||
sar_n, sar_d = sar.split(":")
|
||||
sar_n, sar_d = int(sar_n), int(sar_d)
|
||||
if sar_n == 0 or sar_d == 0:
|
||||
sar_n, sar_d = 1, 1
|
||||
except (ValueError, AttributeError):
|
||||
sar_n, sar_d = 1, 1
|
||||
|
||||
# Square-pixel dimenzije: če SAR != 1:1, popravi širino
|
||||
if sar_n != sar_d:
|
||||
sq_w = int(round(raw_w * sar_n / sar_d))
|
||||
# Zaokroži na sodo (libx264 ima rad sode dimenzije)
|
||||
sq_w = sq_w + (sq_w % 2)
|
||||
sq_h = raw_h
|
||||
print(f"📐 Anamorphic source: {raw_w}x{raw_h} SAR {sar_n}:{sar_d} → {sq_w}x{sq_h} (square pixel)", file=sys.stderr)
|
||||
else:
|
||||
sq_w = raw_w
|
||||
sq_h = raw_h
|
||||
|
||||
return {
|
||||
"width": int(vstream["width"]),
|
||||
"height": int(vstream["height"]),
|
||||
"width": sq_w,
|
||||
"height": sq_h,
|
||||
"raw_width": raw_w,
|
||||
"raw_height": raw_h,
|
||||
"sar_n": sar_n,
|
||||
"sar_d": sar_d,
|
||||
"fps": fps,
|
||||
"duration": float(data["format"]["duration"]),
|
||||
}
|
||||
|
||||
|
||||
def get_audio_streams(path):
|
||||
"""Vrni seznam audio streamov z njihovimi metadati.
|
||||
|
||||
MXF datoteke imajo pogosto 4-8 audio streamov ali en stream z 8 kanali.
|
||||
Vrne: [{'index': 1, 'channels': 2, 'codec': 'pcm_s24le', 'language': 'eng'}, ...]
|
||||
"""
|
||||
cmd = [
|
||||
"ffprobe", "-v", "quiet", "-print_format", "json",
|
||||
"-show_streams", "-select_streams", "a", str(path)
|
||||
]
|
||||
try:
|
||||
data = json.loads(subprocess.check_output(cmd))
|
||||
streams = []
|
||||
for s in data.get("streams", []):
|
||||
streams.append({
|
||||
"index": s.get("index"),
|
||||
"channels": int(s.get("channels", 2)),
|
||||
"codec": s.get("codec_name", ""),
|
||||
"sample_rate": int(s.get("sample_rate", 48000)),
|
||||
"language": s.get("tags", {}).get("language", ""),
|
||||
"channel_layout": s.get("channel_layout", ""),
|
||||
})
|
||||
return streams
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def build_audio_args(audio_streams):
|
||||
"""Sestavi FFmpeg argumente za audio: izberi pravi stream + downmix v stereo.
|
||||
|
||||
Strategija:
|
||||
- Če je samo 1 stream: pretvori ga v stereo (downmix iz multichannel)
|
||||
- Če je več streamov (MXF z več jezikovnimi kanali):
|
||||
* preferiraj prvi 2-kanalni stereo stream (običajno glavna audio mix)
|
||||
* sicer prvi stream
|
||||
- Vedno: output 2 channels @ 48kHz, codec AAC 192k (boljša kvaliteta za broadcast)
|
||||
"""
|
||||
if not audio_streams:
|
||||
# Brez audia: prazen output
|
||||
return ["-an"]
|
||||
|
||||
# Najdi najboljši stream
|
||||
chosen = None
|
||||
# 1. Stereo (2-kanalni) ima prednost
|
||||
for s in audio_streams:
|
||||
if s["channels"] == 2:
|
||||
chosen = s
|
||||
break
|
||||
# 2. Sicer prvi stream
|
||||
if chosen is None:
|
||||
chosen = audio_streams[0]
|
||||
|
||||
args = [
|
||||
"-map", f"0:{chosen['index']}", # samo izbrani stream
|
||||
"-ac", "2", # downmix v stereo (če je multichannel)
|
||||
"-ar", "48000",
|
||||
"-c:a", "aac",
|
||||
"-b:a", "192k", # 192k = boljša kvaliteta za glasbo (prej 128k)
|
||||
]
|
||||
return args
|
||||
|
||||
|
||||
def detect_face_centers(video_path, sample_fps=5):
|
||||
"""
|
||||
Vzorči video pri sample_fps in vrni seznam (timestamp, x_center_normalized).
|
||||
@@ -188,7 +284,7 @@ def build_center_filter(info, target_w, target_h):
|
||||
return f"scale={scale_w}:{scale_h},crop={target_w}:{target_h}:(in_w-{target_w})/2:0"
|
||||
|
||||
|
||||
def build_blur_filter(info, target_w, target_h):
|
||||
def build_blur_filter(info, target_w, target_h, anamorphic_prefix=""):
|
||||
"""
|
||||
9:16 platno: spodaj/zgoraj blur kopija, v sredini originalni 16:9.
|
||||
"""
|
||||
@@ -196,10 +292,11 @@ def build_blur_filter(info, target_w, target_h):
|
||||
src_w = info["width"]
|
||||
src_h = info["height"]
|
||||
fg_h = int(target_w * src_h / src_w)
|
||||
pre = (anamorphic_prefix + ",") if anamorphic_prefix else ""
|
||||
return (
|
||||
f"[0:v]scale={target_w}:{target_h}:force_original_aspect_ratio=increase,"
|
||||
f"[0:v]{pre}scale={target_w}:{target_h}:force_original_aspect_ratio=increase,"
|
||||
f"crop={target_w}:{target_h},gblur=sigma=30[bg];"
|
||||
f"[0:v]scale={target_w}:{fg_h}[fg];"
|
||||
f"[0:v]{pre}scale={target_w}:{fg_h}[fg];"
|
||||
f"[bg][fg]overlay=0:(H-h)/2"
|
||||
)
|
||||
|
||||
@@ -227,6 +324,23 @@ def main():
|
||||
# Če imamo --start/--duration, najprej trim z FFmpeg v temp file (hitreje)
|
||||
work_input = src
|
||||
tmp = None
|
||||
|
||||
# Probe audio streams iz originala
|
||||
audio_streams = get_audio_streams(src)
|
||||
src_ext = src.suffix.lower()
|
||||
is_broadcast = src_ext in (".mxf", ".mpg", ".mpeg", ".ts", ".m2ts", ".mts")
|
||||
has_complex_audio = (
|
||||
len(audio_streams) > 1 or
|
||||
(audio_streams and audio_streams[0].get("channels", 2) > 2) or
|
||||
is_broadcast
|
||||
)
|
||||
|
||||
if has_complex_audio:
|
||||
print(f"🎚 Broadcast format ({src_ext}) — {len(audio_streams)} audio stream(s):", file=sys.stderr)
|
||||
for s in audio_streams:
|
||||
print(f" #{s['index']}: {s['codec']} {s['channels']}ch "
|
||||
f"{s.get('channel_layout', '')} lang={s.get('language', '?')}", file=sys.stderr)
|
||||
|
||||
if args.start is not None or args.duration is not None:
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
|
||||
tmp.close()
|
||||
@@ -236,36 +350,83 @@ def main():
|
||||
cmd += ["-i", str(src)]
|
||||
if args.duration is not None:
|
||||
cmd += ["-t", str(args.duration)]
|
||||
cmd += ["-c", "copy", tmp.name]
|
||||
|
||||
if has_complex_audio:
|
||||
# Broadcast format ali multichannel: ne kopiraj, transkodiraj v stereo MP4
|
||||
audio_args = build_audio_args(audio_streams)
|
||||
cmd += [
|
||||
"-map", "0:v:0", # samo prvi video stream
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
|
||||
"-pix_fmt", "yuv420p", # web/mobile compat (broadcast .mpg = yuv422p)
|
||||
]
|
||||
cmd += audio_args
|
||||
else:
|
||||
# MP4/MOV s standardnim audiom — stream copy je OK in hitrejši
|
||||
cmd += ["-c", "copy"]
|
||||
cmd += [tmp.name]
|
||||
|
||||
print(f"🔧 TRIM CMD: {' '.join(cmd)}", file=sys.stderr)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
print(f"❌ TRIM FAILED: {result.stderr[-500:]}", file=sys.stderr)
|
||||
print(f"❌ TRIM FAILED: {result.stderr[-1000:]}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
work_input = Path(tmp.name)
|
||||
print(f"✂ Trim → {work_input}")
|
||||
# Verify trim output duration
|
||||
verify = subprocess.run(
|
||||
["ffprobe", "-v", "error", "-show_entries", "format=duration",
|
||||
"-of", "default=nw=1:nokey=1", str(work_input)],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
print(f"🔍 TRIMMED FILE DURATION: {verify.stdout.strip()}s (expected ~{args.duration}s)", file=sys.stderr)
|
||||
elif has_complex_audio:
|
||||
# Brez trim-a, ampak MXF/MPG → še vedno transkodiraj v MP4 z 2-channel audio
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
|
||||
tmp.close()
|
||||
audio_args = build_audio_args(audio_streams)
|
||||
cmd = ["ffmpeg", "-y", "-i", str(src),
|
||||
"-map", "0:v:0",
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
|
||||
"-pix_fmt", "yuv420p"] # web/mobile compat
|
||||
cmd += audio_args + [tmp.name]
|
||||
print(f"🔧 PRE-CONVERT CMD: {' '.join(cmd)}", file=sys.stderr)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
print(f"❌ PRE-CONVERT FAILED: {result.stderr[-1000:]}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
work_input = Path(tmp.name)
|
||||
print(f"🔄 Pre-converted → {work_input}")
|
||||
|
||||
info = get_video_info(work_input)
|
||||
print(f"📹 Vhod: {info['width']}x{info['height']} @ {info['fps']:.2f}fps, {info['duration']:.1f}s")
|
||||
|
||||
# Anamorphic correction prefix: če je SAR != 1:1, najprej scale-amo na square-pixel
|
||||
# in resetiramo SAR=1, šele potem track/crop filter dela na pravilnih dimenzijah.
|
||||
# Brez tega: 720x576 SAR 64:45 (DAR 16:9) PAL DV se pri scale na 1920 razteguje
|
||||
# nepravilno (ker filter misli da je 720 širina, dejansko prikazana širina je 1024).
|
||||
if info.get("sar_n", 1) != info.get("sar_d", 1):
|
||||
anamorphic_prefix = f"scale={info['width']}:{info['height']}:flags=lanczos,setsar=1,"
|
||||
print(f"🔧 Anamorphic prefix: {anamorphic_prefix.rstrip(',')}", file=sys.stderr)
|
||||
else:
|
||||
anamorphic_prefix = ""
|
||||
|
||||
if args.mode == "track":
|
||||
print("🔍 Detektiram obraze (OpenCV)...")
|
||||
samples, _, _, _, _ = detect_face_centers(work_input, sample_fps=5)
|
||||
n_with_face = sum(1 for _, x in samples if x is not None)
|
||||
print(f" {n_with_face}/{len(samples)} vzorcev z obrazom")
|
||||
x_at = smooth_track(samples, info["duration"], smoothing_window=4.0)
|
||||
vfilter = build_track_filter(info, x_at, args.target_width, args.target_height, info["fps"])
|
||||
vfilter = anamorphic_prefix + build_track_filter(info, x_at, args.target_width, args.target_height, info["fps"])
|
||||
elif args.mode == "center":
|
||||
vfilter = build_center_filter(info, args.target_width, args.target_height)
|
||||
vfilter = anamorphic_prefix + build_center_filter(info, args.target_width, args.target_height)
|
||||
elif args.mode == "blur":
|
||||
vfilter = build_blur_filter(info, args.target_width, args.target_height)
|
||||
# blur uporablja filter_complex z [0:v] referenco — anamorphic prefix gre v posebni veji
|
||||
vfilter = build_blur_filter(info, args.target_width, args.target_height,
|
||||
anamorphic_prefix=anamorphic_prefix.rstrip(","))
|
||||
|
||||
# KONČNI setsar=1: zagotovi 1:1 piksel v output-u (kompenzira morebitne rounding errore
|
||||
# iz scale/crop filtrov, ki lahko dajo SAR npr. 10240:10239)
|
||||
if args.mode != "blur":
|
||||
vfilter = vfilter + ",setsar=1"
|
||||
|
||||
preset = {"fast": "veryfast", "medium": "medium", "high": "slow"}[args.quality]
|
||||
crf = {"fast": "26", "medium": "21", "high": "18"}[args.quality]
|
||||
@@ -286,7 +447,10 @@ def main():
|
||||
"ffmpeg", "-y", "-i", str(work_input),
|
||||
"-filter_complex", vfilter,
|
||||
"-c:v", "libx264", "-preset", preset, "-crf", crf,
|
||||
"-c:a", "aac", "-b:a", "128k",
|
||||
"-pix_fmt", "yuv420p", # web/mobile compat (Instagram/FB/web players)
|
||||
"-map", "0:a:0?", # samo prvi audio stream (če obstaja)
|
||||
"-ac", "2", # force stereo
|
||||
"-c:a", "aac", "-b:a", "192k",
|
||||
]
|
||||
if audio_filter_str:
|
||||
cmd += ["-af", audio_filter_str]
|
||||
@@ -296,7 +460,10 @@ def main():
|
||||
"ffmpeg", "-y", "-i", str(work_input),
|
||||
"-vf", vfilter,
|
||||
"-c:v", "libx264", "-preset", preset, "-crf", crf,
|
||||
"-c:a", "aac", "-b:a", "128k",
|
||||
"-pix_fmt", "yuv420p", # web/mobile compat (Instagram/FB/web players)
|
||||
"-map", "0:v:0", "-map", "0:a:0?", # video + samo prvi audio
|
||||
"-ac", "2", # force stereo
|
||||
"-c:a", "aac", "-b:a", "192k",
|
||||
]
|
||||
if audio_filter_str:
|
||||
cmd += ["-af", audio_filter_str]
|
||||
|
||||
+3
-1
@@ -255,12 +255,14 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
|
||||
|
||||
# Burn-in z ass filtrom (boljši kot subtitles za naš primer)
|
||||
ass_escaped = ass_path.replace("\\", "\\\\").replace(":", "\\:").replace("'", r"\'")
|
||||
vf = f"ass='{ass_escaped}'"
|
||||
# setsar=1 + setdar=9/16: zagotovi 1:1 piksel + 9:16 display ratio v finalnem output-u
|
||||
vf = f"ass='{ass_escaped}',setsar=1"
|
||||
|
||||
cmd = [
|
||||
"ffmpeg", "-y", "-i", str(video),
|
||||
"-vf", vf,
|
||||
"-c:v", "libx264", "-preset", "medium", "-crf", "21",
|
||||
"-pix_fmt", "yuv420p", # web/mobile compat (Instagram/FB/web players)
|
||||
"-c:a", "copy",
|
||||
"-movflags", "+faststart",
|
||||
str(output),
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Qnet baz fetcher za reels-app.
|
||||
|
||||
Fetcha Songs.txt iz 5 Qnet instalacij na MB playerjih (preko ssh-api proxy-ja
|
||||
na openclaw → SSH na Windows playerje), pretvori iz Windows-1252 v UTF-8,
|
||||
parsa TSV in shrani enotno JSON bazo v /data/qnet/songs.json.
|
||||
|
||||
Cron-friendly: poženi enkrat na uro.
|
||||
|
||||
Output struktura:
|
||||
{
|
||||
"synced_at": 1746198000.0,
|
||||
"stations": {
|
||||
"FOLX DE": {"count": 4038, "fetched_at": 1746198000.0},
|
||||
...
|
||||
},
|
||||
"songs": [
|
||||
{
|
||||
"station": "FOLX DE",
|
||||
"artist": "Sašo Avsenik und seine Oberkrainer",
|
||||
"title": "Na Golici",
|
||||
"file": "Sašo Avsenik und seine Oberkrainer - Na Golici.mp4",
|
||||
"type": "DGL",
|
||||
"length": "2:32.277",
|
||||
"comments": "",
|
||||
"last_played": "17/4/2026"
|
||||
},
|
||||
...
|
||||
]
|
||||
}
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import base64
|
||||
import requests
|
||||
from pathlib import Path
|
||||
|
||||
SSH_API = os.environ.get("PTC_SSH_API", "https://mail.folx.tv/ssh-api/v2")
|
||||
SSH_TOKEN = os.environ.get("PTC_SSH_TOKEN") or "ptc-ssh-2026-a7b3c9d4e5f6012389abcdef01234567890abcdef01234567890abcdef012345"
|
||||
|
||||
OUT_PATH = Path(os.environ.get("QNET_DB_PATH", "/data/qnet/songs.json"))
|
||||
OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# (station_label, player_ip, qnet_subdir_on_C)
|
||||
STATIONS = [
|
||||
("FOLX DE", "100.64.0.2", "qnet"),
|
||||
("ZWEI", "100.64.0.2", "qnetzwei"),
|
||||
("ONE", "100.64.0.3", "QnetONE"),
|
||||
("ADRIA", "100.64.0.4", "Qnet"),
|
||||
("FOLX SLO", "100.64.0.4", "QnetFOLXSLO"),
|
||||
]
|
||||
|
||||
SSH_KEY = "/root/.ssh/players/folx_players"
|
||||
|
||||
|
||||
def ssh_exec(cmd: str, timeout: int = 60) -> dict:
|
||||
"""Pošlji ukaz preko ssh-api na openclaw."""
|
||||
r = requests.post(
|
||||
SSH_API,
|
||||
headers={
|
||||
"Authorization": f"Bearer {SSH_TOKEN}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={"host": "openclaw", "cmd": cmd, "timeout": timeout},
|
||||
timeout=timeout + 30,
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
def fetch_one(station: str, ip: str, subdir: str) -> str:
|
||||
"""Fetcha Songs.txt z windows playerja, vrne UTF-8 string.
|
||||
|
||||
Songs.txt je v CP1250 encoding (Windows Slovenian/CE), NE 1252 (Western).
|
||||
1252 bi 'Č' (0xC8) interpretiral kot 'È', 'Š' kot 'Š' OK ampak 'Ž' (0xDE) kot 'Þ' itd.
|
||||
"""
|
||||
# 1) scp z playerja na openclaw, iconv v utf8, base64 nazaj
|
||||
cmd = (
|
||||
f"set -e; "
|
||||
f"TMP=$(mktemp); "
|
||||
f"scp -i {SSH_KEY} -o StrictHostKeyChecking=no "
|
||||
f'"folxadmin@{ip}:c:/{subdir}/Data/Songs.txt" "$TMP"; '
|
||||
f'iconv -f WINDOWS-1250 -t UTF-8 "$TMP" | base64 -w 0; '
|
||||
f'rm -f "$TMP"'
|
||||
)
|
||||
res = ssh_exec(cmd, timeout=90)
|
||||
if res.get("exit_code") != 0:
|
||||
raise RuntimeError(f"{station}: ssh-api error: {res}")
|
||||
b64 = res.get("output", "").strip()
|
||||
if not b64:
|
||||
raise RuntimeError(f"{station}: empty response")
|
||||
return base64.b64decode(b64).decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def parse_songs_tsv(text: str, station: str) -> list[dict]:
|
||||
"""Parse TSV → list of clean dicts. Drop incomplete rows."""
|
||||
out = []
|
||||
reader = csv.DictReader(io.StringIO(text), delimiter="\t")
|
||||
for row in reader:
|
||||
artist = (row.get("Artist") or "").strip()
|
||||
title = (row.get("Title") or "").strip()
|
||||
file_ = (row.get("File") or "").strip()
|
||||
# Skip popolnoma prazne vrstice
|
||||
if not (artist or title or file_):
|
||||
continue
|
||||
out.append({
|
||||
"station": station,
|
||||
"artist": artist,
|
||||
"title": title,
|
||||
"file": file_,
|
||||
"type": (row.get("Type") or "").strip(),
|
||||
"length": (row.get("Length") or "").strip(),
|
||||
"comments": (row.get("Comments") or "").strip(),
|
||||
"language": (row.get("Language") or "").strip(),
|
||||
"genre": (row.get("Genre") or "").strip(),
|
||||
"last_played": (row.get("Last date played") or "").strip(),
|
||||
"display_artist": (row.get("Display artist") or "").strip(),
|
||||
"display_title": (row.get("Display title") or "").strip(),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
t0 = time.time()
|
||||
all_songs = []
|
||||
stations_meta = {}
|
||||
errors = []
|
||||
|
||||
for station, ip, subdir in STATIONS:
|
||||
try:
|
||||
print(f"→ {station} ({ip}:c:/{subdir}/Data/Songs.txt)", flush=True)
|
||||
text = fetch_one(station, ip, subdir)
|
||||
songs = parse_songs_tsv(text, station)
|
||||
all_songs.extend(songs)
|
||||
stations_meta[station] = {
|
||||
"count": len(songs),
|
||||
"fetched_at": time.time(),
|
||||
"ok": True,
|
||||
}
|
||||
print(f" ✓ {len(songs)} songov", flush=True)
|
||||
except Exception as e:
|
||||
err = f"{station}: {type(e).__name__}: {e}"
|
||||
print(f" ✗ {err}", flush=True)
|
||||
errors.append(err)
|
||||
stations_meta[station] = {"count": 0, "ok": False, "error": str(e)}
|
||||
|
||||
# Zapiši na disk (atomic preko temp + rename)
|
||||
payload = {
|
||||
"synced_at": time.time(),
|
||||
"duration_seconds": round(time.time() - t0, 1),
|
||||
"total_songs": len(all_songs),
|
||||
"stations": stations_meta,
|
||||
"errors": errors,
|
||||
"songs": all_songs,
|
||||
}
|
||||
|
||||
tmp = OUT_PATH.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
tmp.replace(OUT_PATH)
|
||||
|
||||
# Tudi ločen "lookup index" — manjši fajl samo za matching
|
||||
lookup = []
|
||||
for s in all_songs:
|
||||
if s["artist"] and s["title"]:
|
||||
lookup.append({
|
||||
"station": s["station"],
|
||||
"artist": s["artist"],
|
||||
"title": s["title"],
|
||||
"file": s["file"],
|
||||
})
|
||||
lookup_path = OUT_PATH.parent / "songs_lookup.json"
|
||||
tmp2 = lookup_path.with_suffix(".json.tmp")
|
||||
tmp2.write_text(json.dumps(lookup, ensure_ascii=False), encoding="utf-8")
|
||||
tmp2.replace(lookup_path)
|
||||
|
||||
print(f"\n✓ Done: {len(all_songs)} songov v {OUT_PATH} ({round(time.time()-t0,1)}s)")
|
||||
if errors:
|
||||
print(f"⚠ {len(errors)} napak:")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
sys.exit(1 if len(errors) == len(STATIONS) else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+199
-19
@@ -9,10 +9,75 @@ import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
|
||||
YT_API_URL = os.environ.get("YT_API_URL", "https://yt.biba.live")
|
||||
YT_API_TOKEN = os.environ.get("YT_API_TOKEN", "")
|
||||
|
||||
|
||||
def fetch_cookies_from_api(cache_path="/tmp/yt_cookies_api.txt", max_age_seconds=300):
|
||||
"""Pull sveže YouTube cookies iz yt.biba.live API.
|
||||
|
||||
Cache za 5 min (TTL) — če imamo nedavno fetched, uporabi local copy.
|
||||
Vrne path do cookies file-a ali None če API ne deluje.
|
||||
"""
|
||||
if not YT_API_TOKEN:
|
||||
return None # ni configured
|
||||
|
||||
cache = Path(cache_path)
|
||||
if cache.exists():
|
||||
age = time.time() - cache.stat().st_mtime
|
||||
if age < max_age_seconds and cache.stat().st_size > 100:
|
||||
return str(cache)
|
||||
|
||||
try:
|
||||
import urllib.request
|
||||
req = urllib.request.Request(
|
||||
f"{YT_API_URL}/cookies/raw",
|
||||
headers={"Authorization": f"Bearer {YT_API_TOKEN}"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
content = resp.read().decode("utf-8")
|
||||
if len(content) < 100:
|
||||
print(f"⚠ yt.biba.live vrnil prekratek cookies file ({len(content)} bytes)", file=sys.stderr)
|
||||
return None
|
||||
cache.write_text(content, encoding="utf-8")
|
||||
print(f"🍪 Cookies fetched iz yt.biba.live ({len(content)} bytes)", file=sys.stderr)
|
||||
return str(cache)
|
||||
except Exception as e:
|
||||
print(f"⚠ yt.biba.live cookies fetch failed: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def get_cookies_file(cookies_file=None):
|
||||
"""Vrne path do cookies file-a — prioriteta:
|
||||
1. eksplicitno podano (cookies_file param)
|
||||
2. yt.biba.live API (sveže, hourly refreshed)
|
||||
3. /data/cookies/youtube.txt (lokalni fallback)
|
||||
4. YT_COOKIES_FILE env
|
||||
"""
|
||||
if cookies_file and Path(cookies_file).exists():
|
||||
return cookies_file
|
||||
|
||||
# Try API
|
||||
api_cookies = fetch_cookies_from_api()
|
||||
if api_cookies:
|
||||
return api_cookies
|
||||
|
||||
# Fallback to local
|
||||
for candidate in [
|
||||
"/data/cookies/youtube.txt",
|
||||
os.environ.get("YT_COOKIES_FILE", ""),
|
||||
]:
|
||||
if candidate and Path(candidate).exists():
|
||||
return candidate
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def download(url, output, max_height=1080, format_str=None, cookies_file=None):
|
||||
"""
|
||||
Download YT video. Privzeto: best mp4 ≤1080p z audiotrackom.
|
||||
@@ -33,17 +98,8 @@ def download(url, output, max_height=1080, format_str=None, cookies_file=None):
|
||||
"-o", str(output),
|
||||
]
|
||||
|
||||
# Auto-detect cookies file če ni eksplicitno podan
|
||||
if cookies_file is None:
|
||||
for candidate in [
|
||||
"/data/cookies/youtube.txt",
|
||||
os.environ.get("YT_COOKIES_FILE", ""),
|
||||
]:
|
||||
if candidate and Path(candidate).exists():
|
||||
cookies_file = candidate
|
||||
break
|
||||
|
||||
if cookies_file and Path(cookies_file).exists():
|
||||
cookies_file = get_cookies_file(cookies_file)
|
||||
if cookies_file:
|
||||
cmd += ["--cookies", str(cookies_file)]
|
||||
print(f"🍪 Using cookies: {cookies_file}", file=sys.stderr)
|
||||
|
||||
@@ -58,14 +114,36 @@ def download(url, output, max_height=1080, format_str=None, cookies_file=None):
|
||||
|
||||
|
||||
def get_info(url, cookies_file=None):
|
||||
"""Vrni metadata brez prenosa."""
|
||||
"""Vrni metadata brez prenosa.
|
||||
|
||||
Strategija:
|
||||
1. Najprej probaj yt.biba.live API /download/info (residential IP,
|
||||
sveži cookies, signature solver — manj možnosti za bot detection).
|
||||
2. Fallback na lokalni yt-dlp --dump-json.
|
||||
"""
|
||||
# ─── 1. Try yt.biba.live API ───
|
||||
if YT_API_TOKEN:
|
||||
try:
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
params = urllib.parse.urlencode({"url": url})
|
||||
req = urllib.request.Request(
|
||||
f"{YT_API_URL}/download/info?{params}",
|
||||
headers={"Authorization": f"Bearer {YT_API_TOKEN}"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
api_data = json.loads(resp.read().decode("utf-8"))
|
||||
if api_data.get("ok"):
|
||||
print(f"ℹ Info via yt.biba.live API: {api_data.get('title','')[:60]}", file=sys.stderr)
|
||||
# API že vrne flat dict — preimenujmo v yt-dlp obliko za kompatibilnost
|
||||
return api_data
|
||||
except Exception as e:
|
||||
print(f"⚠ yt.biba.live /download/info failed: {e}, fallback to local", file=sys.stderr)
|
||||
|
||||
# ─── 2. Fallback: lokalni yt-dlp ───
|
||||
cmd = ["yt-dlp", "--dump-json", "--no-playlist"]
|
||||
if cookies_file is None:
|
||||
for candidate in ["/data/cookies/youtube.txt", os.environ.get("YT_COOKIES_FILE", "")]:
|
||||
if candidate and Path(candidate).exists():
|
||||
cookies_file = candidate
|
||||
break
|
||||
if cookies_file and Path(cookies_file).exists():
|
||||
cookies_file = get_cookies_file(cookies_file)
|
||||
if cookies_file:
|
||||
cmd += ["--cookies", str(cookies_file)]
|
||||
cmd.append(url)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
@@ -74,6 +152,95 @@ def get_info(url, cookies_file=None):
|
||||
return json.loads(result.stdout.strip().split("\n")[0])
|
||||
|
||||
|
||||
def get_playlist(url, cookies_file=None):
|
||||
"""Vrni listo videov v playlistu (samo metadata, ne pobere).
|
||||
|
||||
Strategija:
|
||||
1. Najprej probaj yt.biba.live API /download/playlist (centralizirano,
|
||||
vedno sveže cookies + signature solver na strežniku).
|
||||
2. Fallback na lokalni yt-dlp --flat-playlist.
|
||||
|
||||
Returns:
|
||||
{
|
||||
"is_playlist": bool,
|
||||
"playlist_title": str,
|
||||
"items": [
|
||||
{"id": "VIDEO_ID", "title": "...", "url": "https://...", "duration": 234},
|
||||
...
|
||||
]
|
||||
}
|
||||
"""
|
||||
# ─── 1. Try yt.biba.live API ───
|
||||
if YT_API_TOKEN:
|
||||
try:
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
params = urllib.parse.urlencode({"url": url, "max_items": 200})
|
||||
req = urllib.request.Request(
|
||||
f"{YT_API_URL}/download/playlist?{params}",
|
||||
headers={"Authorization": f"Bearer {YT_API_TOKEN}"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
api_data = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
if api_data.get("ok"):
|
||||
print(f"📋 Playlist resolved via yt.biba.live API: {api_data.get('count', 0)} items", file=sys.stderr)
|
||||
return {
|
||||
"is_playlist": api_data.get("is_playlist", False),
|
||||
"playlist_title": api_data.get("playlist_title", ""),
|
||||
"items": api_data.get("items", []),
|
||||
}
|
||||
else:
|
||||
print(f"⚠ yt.biba.live API error: {api_data.get('error', 'unknown')[:200]}", file=sys.stderr)
|
||||
# Fall through to local
|
||||
except Exception as e:
|
||||
print(f"⚠ yt.biba.live API failed: {e}, fallback to local yt-dlp", file=sys.stderr)
|
||||
|
||||
# ─── 2. Fallback: lokalni yt-dlp ───
|
||||
cmd = ["yt-dlp", "--flat-playlist", "--dump-json"]
|
||||
cookies_file = get_cookies_file(cookies_file)
|
||||
if cookies_file:
|
||||
cmd += ["--cookies", str(cookies_file)]
|
||||
cmd.append(url)
|
||||
print(f"📋 Resolving playlist (lokalno): {url}", file=sys.stderr)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
if result.returncode != 0:
|
||||
print(f"❌ yt-dlp playlist napaka:\n{result.stderr[-1500:]}", file=sys.stderr)
|
||||
return {"is_playlist": False, "playlist_title": "", "items": [], "error": result.stderr[-500:]}
|
||||
|
||||
items = []
|
||||
playlist_title = ""
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
# Playlist header (typ=playlist)
|
||||
if entry.get("_type") == "playlist":
|
||||
playlist_title = entry.get("title", "")
|
||||
continue
|
||||
# Video entry
|
||||
vid = entry.get("id")
|
||||
if not vid:
|
||||
continue
|
||||
items.append({
|
||||
"id": vid,
|
||||
"title": entry.get("title", "") or "",
|
||||
"url": entry.get("url") or f"https://www.youtube.com/watch?v={vid}",
|
||||
"duration": entry.get("duration"),
|
||||
"uploader": entry.get("uploader") or entry.get("channel") or "",
|
||||
})
|
||||
|
||||
is_playlist = len(items) > 1 or ("list=" in url)
|
||||
return {
|
||||
"is_playlist": is_playlist,
|
||||
"playlist_title": playlist_title or (items[0]["title"] if items else ""),
|
||||
"items": items,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("url")
|
||||
@@ -89,11 +256,24 @@ def main():
|
||||
info = get_info(args.url, cookies_file=args.cookies)
|
||||
if info:
|
||||
print(json.dumps({
|
||||
"id": info.get("id"),
|
||||
"title": info.get("title"),
|
||||
"duration": info.get("duration"),
|
||||
"uploader": info.get("uploader"),
|
||||
"uploader_id": info.get("uploader_id"),
|
||||
"channel": info.get("channel"),
|
||||
"channel_id": info.get("channel_id"),
|
||||
"thumbnail": info.get("thumbnail"),
|
||||
}, indent=2))
|
||||
"description": info.get("description"),
|
||||
"upload_date": info.get("upload_date"),
|
||||
"view_count": info.get("view_count"),
|
||||
"like_count": info.get("like_count"),
|
||||
"categories": info.get("categories"),
|
||||
"tags": info.get("tags"),
|
||||
"webpage_url": info.get("webpage_url"),
|
||||
"age_limit": info.get("age_limit"),
|
||||
"is_live": info.get("is_live"),
|
||||
}, indent=2, ensure_ascii=False))
|
||||
else:
|
||||
print("❌ Ne morem dobiti info", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
+1542
-177
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user