feat(backend): add queue system; add retry system

This commit is contained in:
2026-05-16 17:53:37 +03:00
parent 5649fc17dd
commit decc4c675d
7 changed files with 91 additions and 4 deletions
+6
View File
@@ -18,6 +18,9 @@ DEFAULT_CONFIG: Dict[str, Any] = {
"max_download_quality": "1080p",
"save_path": "./downloads",
"ffmpeg_path": _default_ffmpeg_path(),
"max_parallel_downloads": 2,
"retry_max_retries": 2,
"retry_delay_seconds": 1.5,
}
@@ -63,6 +66,9 @@ class Settings:
"max_download_quality": self.data.get("max_download_quality", DEFAULT_CONFIG["max_download_quality"]),
"save_path": self.data.get("save_path", DEFAULT_CONFIG["save_path"]),
"ffmpeg_path": self.data.get("ffmpeg_path", DEFAULT_CONFIG["ffmpeg_path"]),
"max_parallel_downloads": self.data.get("max_parallel_downloads", DEFAULT_CONFIG["max_parallel_downloads"]),
"retry_max_retries": self.data.get("retry_max_retries", DEFAULT_CONFIG["retry_max_retries"]),
"retry_delay_seconds": self.data.get("retry_delay_seconds", DEFAULT_CONFIG["retry_delay_seconds"]),
}
results: List[Dict[str, Any]] = []
+3
View File
@@ -59,3 +59,6 @@ class QueueManager:
async def enqueue(self, job: DownloadJob):
await self._queue.put(job)
async def join(self) -> None:
await self._queue.join()
+1 -1
View File
@@ -8,7 +8,7 @@ from ..models import FilesystemEntry, PlaylistItem, SyncAction, SyncActionType
class DiffEngine:
"""
Compares remote playlist items, database state, and filesystem to
produce a list of actions. Initial MVP computes DOWNLOAD/RENAME/REORDER
produce a list of actions. Initial MVP computes DOWNLOAD/RENAME/DELETE
based on simple filename scheme "0001 - Title.ext".
"""
+19 -3
View File
@@ -128,12 +128,28 @@ class ActionExecutor:
async def _apply_downloads(self, actions: Iterable[SyncAction], mode: str, audio_root: Path, video_root: Path, playlist_cfg: dict) -> None:
playlist_id = extract_playlist_id(playlist_cfg.get("url", "")) or playlist_cfg.get("url", "")
queue = QueueManager(concurrency=self.concurrency)
concurrency_cfg = playlist_cfg.get("max_parallel_downloads", self.concurrency)
try:
concurrency = int(concurrency_cfg) if concurrency_cfg is not None else self.concurrency
except Exception:
concurrency = self.concurrency
queue = QueueManager(concurrency=concurrency)
retry_max_cfg = playlist_cfg.get("retry_max_retries", 2)
retry_delay_cfg = playlist_cfg.get("retry_delay_seconds", 1.5)
try:
retry_max_retries = int(retry_max_cfg) if retry_max_cfg is not None else 2
except Exception:
retry_max_retries = 2
try:
retry_delay_seconds = float(retry_delay_cfg) if retry_delay_cfg is not None else 1.5
except Exception:
retry_delay_seconds = 1.5
async def worker(job: DownloadJob):
if self.bus and job.item:
await self.bus.publish("DownloadStarted", {"playlist_id": playlist_id, "video_id": job.item.video_id, "target": str(job.output_path)})
await default_worker(job)
await default_worker(job, max_retries=retry_max_retries, delay_seconds=retry_delay_seconds)
await queue.start(worker)
try:
@@ -223,7 +239,7 @@ class ActionExecutor:
jobs.append(job)
await queue.enqueue(job)
finally:
await queue._queue.join() # wait for all jobs
await queue.join() # wait for all jobs
await queue.stop()
# Persist DB updates for completed jobs