From d19b33a7f5350d24fb182d8f328de15f651a7d98 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 08:28:55 +0100 Subject: [PATCH 01/75] Add Dockerfile --- Dockerfile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d9a84b6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.13-slim + +LABEL description="Ace-Pace - One Pace Library Manager" + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* \ + && git clone https://github.com/timothe/Ace-Pace.git . + +RUN pip install --no-cache-dir -r requirements.txt + +RUN mkdir -p /media /data + +ENV PYTHONUNBUFFERED=1 +ENV ACEPACE_FOLDER=/media +ENV ACEPACE_URL="" +ENV ACEPACE_DB="" +ENV ACEPACE_DOWNLOAD="" + +CMD python acepace.py \ + ${ACEPACE_FOLDER:+--folder "$ACEPACE_FOLDER"} \ + ${ACEPACE_URL:+--url "$ACEPACE_URL"} \ + ${ACEPACE_DB:+--db} \ + ${ACEPACE_DOWNLOAD:+--download "$ACEPACE_DOWNLOAD"} From 389fb303c70d312b27b0f3e5b061b90add914d5f Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 08:33:59 +0100 Subject: [PATCH 02/75] Add docker-compose --- docker-compose.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5910576 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,12 @@ +services: + ace-pace: + build: . + container_name: ace-pace + + volumes: + - /volume42/media/One Pace:/media:rw + + environment: + # - ACEPACE_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc + # - ACEPACE_DB=true + # - ACEPACE_DOWNLOAD=transmission From d488e8bc43e1a3be41809950bd26c56580a32984 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 08:35:22 +0100 Subject: [PATCH 03/75] Refactor Dockerfile to simplify directory creation --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d9a84b6..4b82b27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,16 +10,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends git \ RUN pip install --no-cache-dir -r requirements.txt -RUN mkdir -p /media /data +RUN mkdir -p /media ENV PYTHONUNBUFFERED=1 -ENV ACEPACE_FOLDER=/media ENV ACEPACE_URL="" ENV ACEPACE_DB="" ENV ACEPACE_DOWNLOAD="" CMD python acepace.py \ - ${ACEPACE_FOLDER:+--folder "$ACEPACE_FOLDER"} \ + --folder /media \ ${ACEPACE_URL:+--url "$ACEPACE_URL"} \ ${ACEPACE_DB:+--db} \ ${ACEPACE_DOWNLOAD:+--download "$ACEPACE_DOWNLOAD"} From eb060a16301971fe32460e25b647fe4f4b8ea64f Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 08:44:38 +0100 Subject: [PATCH 04/75] Update docker-compose.yml with image and network settings --- docker-compose.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5910576..9651fd2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,16 @@ services: ace-pace: build: . + image: ace-pace container_name: ace-pace - volumes: - - /volume42/media/One Pace:/media:rw - - environment: - # - ACEPACE_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc - # - ACEPACE_DB=true - # - ACEPACE_DOWNLOAD=transmission + - /path/to/OnePaceLibrary:/media:ro + networks: + - "proxy" + # environment: + # - ACEPACE_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc + # - ACEPACE_DB=true + # - ACEPACE_DOWNLOAD=transmission +networks: + proxy: + driver: bridge From 375efd8df8f58e2cba7f0c7347f682528bbc44da Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 09:15:08 +0100 Subject: [PATCH 05/75] Update environment variables in docker-compose.yml --- docker-compose.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9651fd2..784f118 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,10 +7,14 @@ services: - /path/to/OnePaceLibrary:/media:ro networks: - "proxy" - # environment: - # - ACEPACE_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc - # - ACEPACE_DB=true - # - ACEPACE_DOWNLOAD=transmission + environment: + - TRANSMISSION_HOST=192.168.178.45 + - TRANSMISSION_PORT=9091 + - ACEPACE_DOWNLOAD=transmission + #- ACEPACE_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc + #- ACEPACE_DB=true + #- TRANSMISSION_USER=admin + #- TRANSMISSION_PASS=password networks: proxy: driver: bridge From 64172d4f374186a1dc12b6bcb3977b75c6069566 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 09:17:40 +0100 Subject: [PATCH 06/75] Modify Dockerfile for data directory and Transmission Updated Dockerfile to change media directory to data and added environment variables for Transmission configuration. --- Dockerfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b82b27..669d04b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,15 +10,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends git \ RUN pip install --no-cache-dir -r requirements.txt -RUN mkdir -p /media +RUN mkdir -p /data +WORKDIR /data ENV PYTHONUNBUFFERED=1 ENV ACEPACE_URL="" ENV ACEPACE_DB="" ENV ACEPACE_DOWNLOAD="" +ENV TRANSMISSION_HOST="localhost" +ENV TRANSMISSION_PORT="9091" +ENV TRANSMISSION_USER="" +ENV TRANSMISSION_PASS="" -CMD python acepace.py \ +# outwit interactive questions in acepace.py - this is a dirty workaround and definitely needs to be changed, e.g., by adding an interactive_flag in acepace.py to only take default values and/or the environment variables from docker files +CMD /bin/sh -c 'printf "y\n${ACEPACE_DOWNLOAD:-transmission}\n${TRANSMISSION_HOST}\n${TRANSMISSION_PORT}\n${TRANSMISSION_USER}\n${TRANSMISSION_PASS}\n" | python /app/acepace.py \ --folder /media \ ${ACEPACE_URL:+--url "$ACEPACE_URL"} \ ${ACEPACE_DB:+--db} \ - ${ACEPACE_DOWNLOAD:+--download "$ACEPACE_DOWNLOAD"} + ${ACEPACE_DOWNLOAD:+--download "$ACEPACE_DOWNLOAD"}' From 52158e995995b0c857bc73edb9ab76da032a3b1d Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 11:51:15 +0100 Subject: [PATCH 07/75] Create Ace-Pace_Missing.csv in /data directory --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 669d04b..907af6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends git \ RUN pip install --no-cache-dir -r requirements.txt -RUN mkdir -p /data +RUN mkdir -p /data && touch /data/Ace-Pace_Missing.csv WORKDIR /data ENV PYTHONUNBUFFERED=1 From e8e2fbd625beba3b2001bf46a727e243b6024cbe Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 11:51:31 +0100 Subject: [PATCH 08/75] Change volume mount to read-write access --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 784f118..79593b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: image: ace-pace container_name: ace-pace volumes: - - /path/to/OnePaceLibrary:/media:ro + - /path/to/OnePaceLibrary:/media:rw networks: - "proxy" environment: From d8ab08eb2191f0cbc43a471216654e8ccd556387 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 11:57:50 +0100 Subject: [PATCH 09/75] Refactor Dockerfile to streamline setup process Removed unnecessary directory creation and modified git clone command to create a missing CSV file. --- Dockerfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 907af6c..8725b2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,13 +6,11 @@ WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends git \ && rm -rf /var/lib/apt/lists/* \ - && git clone https://github.com/timothe/Ace-Pace.git . - + && git clone https://github.com/timothe/Ace-Pace.git . \ + && touch /app/Ace-Pace_Missing.csv + RUN pip install --no-cache-dir -r requirements.txt -RUN mkdir -p /data && touch /data/Ace-Pace_Missing.csv -WORKDIR /data - ENV PYTHONUNBUFFERED=1 ENV ACEPACE_URL="" ENV ACEPACE_DB="" From 4ba2be2fcbcbbcce54c94783ffeed2ce332f5c94 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 14:06:35 +0100 Subject: [PATCH 10/75] Refactor Dockerfile for improved environment setup --- Dockerfile | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8725b2c..ac1c3af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,27 +2,27 @@ FROM python:3.13-slim LABEL description="Ace-Pace - One Pace Library Manager" +COPY . /app WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends git \ && rm -rf /var/lib/apt/lists/* \ - && git clone https://github.com/timothe/Ace-Pace.git . \ && touch /app/Ace-Pace_Missing.csv - -RUN pip install --no-cache-dir -r requirements.txt -ENV PYTHONUNBUFFERED=1 -ENV ACEPACE_URL="" -ENV ACEPACE_DB="" -ENV ACEPACE_DOWNLOAD="" -ENV TRANSMISSION_HOST="localhost" -ENV TRANSMISSION_PORT="9091" -ENV TRANSMISSION_USER="" -ENV TRANSMISSION_PASS="" +RUN pip install --no-cache-dir -r /app/requirements.txt + +ENV PYTHONUNBUFFERED=1 \ + ACEPACE_URL="https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" \ + ACEPACE_DB="true" \ + ACEPACE_DOWNLOAD="transmission" \ + TRANSMISSION_HOST="localhost" \ + TRANSMISSION_PORT="9091" \ + TRANSMISSION_USER="" \ + TRANSMISSION_PASS="" \ # outwit interactive questions in acepace.py - this is a dirty workaround and definitely needs to be changed, e.g., by adding an interactive_flag in acepace.py to only take default values and/or the environment variables from docker files -CMD /bin/sh -c 'printf "y\n${ACEPACE_DOWNLOAD:-transmission}\n${TRANSMISSION_HOST}\n${TRANSMISSION_PORT}\n${TRANSMISSION_USER}\n${TRANSMISSION_PASS}\n" | python /app/acepace.py \ - --folder /media \ +CMD /bin/sh -c 'printf "y\n${ACEPACE_DOWNLOAD:-transmission}\n${TRANSMISSION_HOST}\n${TRANSMISSION_PORT}\n${TRANSMISSION_USER}\n${TRANSMISSION_PA> + --folder "/media" \ ${ACEPACE_URL:+--url "$ACEPACE_URL"} \ ${ACEPACE_DB:+--db} \ ${ACEPACE_DOWNLOAD:+--download "$ACEPACE_DOWNLOAD"}' From d86162bf72e27db2f4edbb3df4f62141e673cb1c Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 14:07:46 +0100 Subject: [PATCH 11/75] Update CMD to use TRANSMISSION_PASS variable Fix environment variable reference for TRANSMISSION_PASS in CMD. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ac1c3af..aabe8ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ ENV PYTHONUNBUFFERED=1 \ TRANSMISSION_PASS="" \ # outwit interactive questions in acepace.py - this is a dirty workaround and definitely needs to be changed, e.g., by adding an interactive_flag in acepace.py to only take default values and/or the environment variables from docker files -CMD /bin/sh -c 'printf "y\n${ACEPACE_DOWNLOAD:-transmission}\n${TRANSMISSION_HOST}\n${TRANSMISSION_PORT}\n${TRANSMISSION_USER}\n${TRANSMISSION_PA> +CMD /bin/sh -c 'printf "y\n${ACEPACE_DOWNLOAD:-transmission}\n${TRANSMISSION_HOST}\n${TRANSMISSION_PORT}\n${TRANSMISSION_USER}\n${TRANSMISSION_PASS}\n" | python /app/acepace.py \ --folder "/media" \ ${ACEPACE_URL:+--url "$ACEPACE_URL"} \ ${ACEPACE_DB:+--db} \ From eb42e98d735b28bedd6dd108ed4da39b36fc2224 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 14:14:49 +0100 Subject: [PATCH 12/75] Fix Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aabe8ca..c0d652a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ LABEL description="Ace-Pace - One Pace Library Manager" COPY . /app WORKDIR /app -RUN apt-get update && apt-get install -y --no-install-recommends git \ +RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && touch /app/Ace-Pace_Missing.csv From e3d51ced7273241ed8a8cf3645fb911932a5be51 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 19:44:21 +0100 Subject: [PATCH 13/75] Add RUN_DOCKER env to check if script runs inside container --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c0d652a..a0847f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,18 +7,18 @@ WORKDIR /app RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ + && pip install --no-cache-dir -r /app/requirements.txt \ && touch /app/Ace-Pace_Missing.csv -RUN pip install --no-cache-dir -r /app/requirements.txt - -ENV PYTHONUNBUFFERED=1 \ +ENV RUN_DOCKER="true" \ + PYTHONUNBUFFERED=1 \ ACEPACE_URL="https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" \ ACEPACE_DB="true" \ ACEPACE_DOWNLOAD="transmission" \ TRANSMISSION_HOST="localhost" \ TRANSMISSION_PORT="9091" \ TRANSMISSION_USER="" \ - TRANSMISSION_PASS="" \ + TRANSMISSION_PASS="" # outwit interactive questions in acepace.py - this is a dirty workaround and definitely needs to be changed, e.g., by adding an interactive_flag in acepace.py to only take default values and/or the environment variables from docker files CMD /bin/sh -c 'printf "y\n${ACEPACE_DOWNLOAD:-transmission}\n${TRANSMISSION_HOST}\n${TRANSMISSION_PORT}\n${TRANSMISSION_USER}\n${TRANSMISSION_PASS}\n" | python /app/acepace.py \ From 63d5fd6c283b48440aa5436b610d88dc97206d35 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 20:21:04 +0100 Subject: [PATCH 14/75] Implement Docker support for non-interactive mode Added support for Docker mode, allowing for non-interactive execution --- acepace.py | 105 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 67 insertions(+), 38 deletions(-) diff --git a/acepace.py b/acepace.py index f56aa0e..326bfa5 100644 --- a/acepace.py +++ b/acepace.py @@ -10,6 +10,9 @@ import time import getpass +# Check if running in Docker (non-interactive mode) +IS_DOCKER = "RUN_DOCKER" in os.environ + # Define regex to extract CRC32 from filename text (commonly in [xxxxx]) CRC32_REGEX = re.compile(r"\[([A-Fa-f0-9]{8})\]") @@ -440,7 +443,10 @@ def rename_local_files(conn, folder): print(f"{os.path.basename(old)} -> {os.path.basename(new)}") print(f"{len(rename_plan)}/{total} files will be renamed.") - confirm = input("Proceed with renaming? (y/n): ").strip().lower() + if IS_DOCKER: + confirm = "y" + else: + confirm = input("Proceed with renaming? (y/n): ").strip().lower() if confirm != "y": print("Renaming aborted.") return @@ -492,23 +498,30 @@ def download_with_transmission(): print("No magnet links found in 'Ace-Pace_Missing.csv'.") return - print("The details below are not stored.") - host = input("Enter Transmission host (default: localhost): ").strip() - if not host: - host = "localhost" - port_input = input("Enter Transmission port (default: 9091): ").strip() - if port_input: - try: - port = int(port_input) - except ValueError: - print("Invalid port number. Using default 9091.") - port = 9091 + if IS_DOCKER: + print("Running in Docker mode - using environment variables for Transmission config.") + host = os.getenv("TRANSMISSION_HOST", "localhost") + port = int(os.getenv("TRANSMISSION_PORT", "9091")) + rpc_username = os.getenv("TRANSMISSION_USER", "") + rpc_password = os.getenv("TRANSMISSION_PASS", "") else: - port = 9091 - rpc_username = input("Enter Transmission username (leave blank if none): ").strip() - rpc_password = getpass.getpass( - "Enter Transmission password (leave blank if none): " - ).strip() + print("The details below are not stored.") + host = input("Enter Transmission host (default: localhost): ").strip() + if not host: + host = "localhost" + port_input = input("Enter Transmission port (default: 9091): ").strip() + if port_input: + try: + port = int(port_input) + except ValueError: + print("Invalid port number. Using default 9091.") + port = 9091 + else: + port = 9091 + rpc_username = input("Enter Transmission username (leave blank if none): ").strip() + rpc_password = getpass.getpass( + "Enter Transmission password (leave blank if none): " + ).strip() base_url = f"http://{host}:{port}/transmission/rpc" session_id = None @@ -545,17 +558,23 @@ def download_with_transmission(): except Exception: default_download_dir = "" - if default_download_dir: - prompt_text = f"Enter target folder for downloads (current default: {default_download_dir}): " + if IS_DOCKER: + target_folder = os.getenv("TRANSMISSION_DOWNLOAD_DIR", default_download_dir) else: - prompt_text = "Enter target folder for downloads (leave blank for default): " - target_folder = input(prompt_text).strip() + if default_download_dir: + prompt_text = f"Enter target folder for downloads (current default: {default_download_dir}): " + else: + prompt_text = "Enter target folder for downloads (leave blank for default): " + target_folder = input(prompt_text).strip() - confirm = ( - input(f"Do you want to add {len(magnets)} torrents to Transmission? (y/n): ") - .strip() - .lower() - ) + if IS_DOCKER: + confirm = "y" + else: + confirm = ( + input(f"Do you want to add {len(magnets)} torrents to Transmission? (y/n): ") + .strip() + .lower() + ) if confirm != "y": print("Abort! Abort!") return @@ -628,6 +647,9 @@ def main(): ) args = parser.parse_args() + if IS_DOCKER: + print("Running in Docker mode (non-interactive)") + # Check if the URL points to a valid Nyaa domain if not args.url.startswith(("https://nyaa.si", "https://nyaa.land")): print( @@ -653,7 +675,10 @@ def main(): # Folder selection logic: Always prompt if folder is required but not given folder = args.folder needs_folder = not args.download # All commands except --download need folder - if needs_folder and not folder: + if IS_DOCKER: + last_folder="/media" + folder="/media" + elif needs_folder and not folder: # Try to load last_folder from metadata last_folder = get_metadata(conn, "last_folder") if last_folder: @@ -804,24 +829,28 @@ def main(): set_metadata(conn, "last_missing_export", now_str) if missing: - prompt = ( - input( - "Do you want to add missing episodes to a BitTorrent client now? (y/n): " - ) - .strip() - .lower() - ) - if prompt == "y": - client = ( - input("Enter client name (currently supported: transmission): ") + if IS_DOCKER: + prompt = "y" + client = os.getenv("ACEPACE_DOWNLOAD", "transmission") + else: + prompt = ( + input( + "Do you want to add missing episodes to a BitTorrent client now? (y/n): " + ) .strip() .lower() ) + if prompt == "y": + if not IS_DOCKER: + client = ( + input("Enter client name (currently supported: transmission): ") + .strip() + .lower() + ) if client: download_missing_to_client(client) else: print("No client specified. Skipping download.") - if __name__ == "__main__": main() From 4c164f30d3d878da178ae0d1c521111c6b6d5735 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 20:39:39 +0100 Subject: [PATCH 15/75] Update Dockerfile --- Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index a0847f5..1070638 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,9 +20,7 @@ ENV RUN_DOCKER="true" \ TRANSMISSION_USER="" \ TRANSMISSION_PASS="" -# outwit interactive questions in acepace.py - this is a dirty workaround and definitely needs to be changed, e.g., by adding an interactive_flag in acepace.py to only take default values and/or the environment variables from docker files -CMD /bin/sh -c 'printf "y\n${ACEPACE_DOWNLOAD:-transmission}\n${TRANSMISSION_HOST}\n${TRANSMISSION_PORT}\n${TRANSMISSION_USER}\n${TRANSMISSION_PASS}\n" | python /app/acepace.py \ - --folder "/media" \ +CMD python /app/acepace.py \ ${ACEPACE_URL:+--url "$ACEPACE_URL"} \ ${ACEPACE_DB:+--db} \ - ${ACEPACE_DOWNLOAD:+--download "$ACEPACE_DOWNLOAD"}' + ${ACEPACE_DOWNLOAD:+--download "$ACEPACE_DOWNLOAD"} From 8d34f522cb4099404a2b42da5a2b01a9f80b5e47 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 22:41:02 +0100 Subject: [PATCH 16/75] Rename env variables --- Dockerfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1070638..e5548cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,15 +12,15 @@ RUN apt-get update \ ENV RUN_DOCKER="true" \ PYTHONUNBUFFERED=1 \ - ACEPACE_URL="https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" \ - ACEPACE_DB="true" \ - ACEPACE_DOWNLOAD="transmission" \ - TRANSMISSION_HOST="localhost" \ - TRANSMISSION_PORT="9091" \ - TRANSMISSION_USER="" \ - TRANSMISSION_PASS="" + NYAA_URL="https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" \ + DB="true" \ + TORRENT_CLIENT="transmission" \ + TORRENT_HOST="localhost" \ + TORRENT_PORT="9091" \ + TORRENT_USER="" \ + TORRENT_PASSWORD="" CMD python /app/acepace.py \ - ${ACEPACE_URL:+--url "$ACEPACE_URL"} \ - ${ACEPACE_DB:+--db} \ - ${ACEPACE_DOWNLOAD:+--download "$ACEPACE_DOWNLOAD"} + ${NYAA_URL:+--url "$NYAA_URL"} \ + ${DB:+--db} \ + ${TORRENT_CLIENT:+--download "$TORRENT_CLIENT"} From 3c7271d4222485802fc7dc4aef9949be43110840 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 22:42:11 +0100 Subject: [PATCH 17/75] Rename environment variables --- docker-compose.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 79593b3..423802c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,13 +8,13 @@ services: networks: - "proxy" environment: - - TRANSMISSION_HOST=192.168.178.45 - - TRANSMISSION_PORT=9091 - - ACEPACE_DOWNLOAD=transmission - #- ACEPACE_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc - #- ACEPACE_DB=true - #- TRANSMISSION_USER=admin - #- TRANSMISSION_PASS=password + - TORRENT_HOST=192.168.178.45 + - TORRENT_PORT=9091 + - TORRENT_CLIENT=transmission + #- NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc + #- DB=true + #- TORRENT_USER=admin + #- TORRENT_PASSWORD=password networks: proxy: driver: bridge From e38a8a7e82fd63c37476dfcad589766960cd6b19 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Fri, 21 Nov 2025 22:44:33 +0100 Subject: [PATCH 18/75] Update environment variable --- acepace.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/acepace.py b/acepace.py index 326bfa5..91d6c07 100644 --- a/acepace.py +++ b/acepace.py @@ -500,10 +500,10 @@ def download_with_transmission(): if IS_DOCKER: print("Running in Docker mode - using environment variables for Transmission config.") - host = os.getenv("TRANSMISSION_HOST", "localhost") - port = int(os.getenv("TRANSMISSION_PORT", "9091")) - rpc_username = os.getenv("TRANSMISSION_USER", "") - rpc_password = os.getenv("TRANSMISSION_PASS", "") + host = os.getenv("TORRENT_HOST", "localhost") + port = int(os.getenv("TORREN_PORT", "9091")) + rpc_username = os.getenv("TORRENT_USER", "") + rpc_password = os.getenv("TORRENT_PASSWORD", "") else: print("The details below are not stored.") host = input("Enter Transmission host (default: localhost): ").strip() @@ -831,7 +831,7 @@ def main(): if missing: if IS_DOCKER: prompt = "y" - client = os.getenv("ACEPACE_DOWNLOAD", "transmission") + client = os.getenv("TORRENT_CLIENT", "transmission") else: prompt = ( input( From 04ccb327aa15f6abb7ba9f073126196f0b27f07d Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Sat, 22 Nov 2025 13:04:04 +0100 Subject: [PATCH 19/75] Change target folder for downloads in Docker to /media This is the default one inside the docker container --- acepace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acepace.py b/acepace.py index 91d6c07..f3d9827 100644 --- a/acepace.py +++ b/acepace.py @@ -559,7 +559,7 @@ def download_with_transmission(): default_download_dir = "" if IS_DOCKER: - target_folder = os.getenv("TRANSMISSION_DOWNLOAD_DIR", default_download_dir) + target_folder = "/media" else: if default_download_dir: prompt_text = f"Enter target folder for downloads (current default: {default_download_dir}): " From 4b418ac6488da0e748883b32761d9ffb3f33110f Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Sun, 23 Nov 2025 17:38:16 +0100 Subject: [PATCH 20/75] Change TORRENT_HOST to generic localhost --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 423802c..78f1e61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: networks: - "proxy" environment: - - TORRENT_HOST=192.168.178.45 + - TORRENT_HOST=127.0.0.1 - TORRENT_PORT=9091 - TORRENT_CLIENT=transmission #- NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc From 4d768e0ba22cb07b832761ad35dd08b8f4576a2b Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Sun, 23 Nov 2025 19:16:03 +0100 Subject: [PATCH 21/75] Update Dockerfile with new CMD and environment variables Rearranged environment variables and updated CMD to use entrypoint script. --- Dockerfile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index e5548cd..f74f8a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,14 +13,12 @@ RUN apt-get update \ ENV RUN_DOCKER="true" \ PYTHONUNBUFFERED=1 \ NYAA_URL="https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" \ - DB="true" \ TORRENT_CLIENT="transmission" \ TORRENT_HOST="localhost" \ TORRENT_PORT="9091" \ TORRENT_USER="" \ - TORRENT_PASSWORD="" + TORRENT_PASSWORD="" \ + DB="true" \ + EPISODES_UPDATE="true" -CMD python /app/acepace.py \ - ${NYAA_URL:+--url "$NYAA_URL"} \ - ${DB:+--db} \ - ${TORRENT_CLIENT:+--download "$TORRENT_CLIENT"} +CMD ["/app/entrypoint.sh"] From e48aba11ed24833391d02a4601d61ffd78dc472d Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Sun, 23 Nov 2025 19:16:20 +0100 Subject: [PATCH 22/75] Fix Dockerfile to add executable permission to entrypoint --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index f74f8a5..6c27105 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ WORKDIR /app RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && pip install --no-cache-dir -r /app/requirements.txt \ + && chmod +x /app/entrypoint.sh \ && touch /app/Ace-Pace_Missing.csv ENV RUN_DOCKER="true" \ From 670465efbc68f1a21dab540775ca6c5e0e388203 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Sun, 23 Nov 2025 19:21:36 +0100 Subject: [PATCH 23/75] Create entrypoint.sh --- entrypoint.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 entrypoint.sh diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..885a964 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -e + +if [ "$EPISODES_UPDATE" = "true" ]; then + python /app/acepace.py --episodes_update +fi + +exec python /app/acepace.py \ + ${NYAA_URL:+--url "$NYAA_URL"} \ + ${TORRENT_CLIENT:+--download "$TORRENT_CLIENT"} \ + ${DB:+--db} From b5513ccafa02f70be860819f528975c3bbfa7316 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Sun, 23 Nov 2025 19:22:26 +0100 Subject: [PATCH 24/75] Update docker-compose.yml --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 78f1e61..81e928f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,9 +12,10 @@ services: - TORRENT_PORT=9091 - TORRENT_CLIENT=transmission #- NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc - #- DB=true #- TORRENT_USER=admin #- TORRENT_PASSWORD=password + - DB=true + - EPISODES_UPDATE=true networks: proxy: driver: bridge From 1f0f1e1b45e290570c4369faac35d50e136b22d4 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Sun, 23 Nov 2025 20:59:43 +0100 Subject: [PATCH 25/75] Update Dockerfile to install tzdata and clean up --- Dockerfile | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6c27105..456b2c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,21 +5,24 @@ LABEL description="Ace-Pace - One Pace Library Manager" COPY . /app WORKDIR /app -RUN apt-get update \ - && rm -rf /var/lib/apt/lists/* \ - && pip install --no-cache-dir -r /app/requirements.txt \ - && chmod +x /app/entrypoint.sh \ - && touch /app/Ace-Pace_Missing.csv - ENV RUN_DOCKER="true" \ PYTHONUNBUFFERED=1 \ + TZ=Europe/Berlin \ NYAA_URL="https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" \ + TORRENT_HOST="127.0.0.1" \ TORRENT_CLIENT="transmission" \ - TORRENT_HOST="localhost" \ TORRENT_PORT="9091" \ TORRENT_USER="" \ TORRENT_PASSWORD="" \ DB="true" \ EPISODES_UPDATE="true" +RUN apt-get update \ + && apt-get install -y tzdata \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && pip install --no-cache-dir -r /app/requirements.txt \ + && chmod +x /app/entrypoint.sh \ + && touch /app/Ace-Pace_Missing.csv + CMD ["/app/entrypoint.sh"] From f75f51472dba10c5793321a648e4a0ffc92c1840 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Sun, 23 Nov 2025 21:00:58 +0100 Subject: [PATCH 26/75] Update docker-compose with new volumes and environment vars --- docker-compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 81e928f..b54aba7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,13 +5,17 @@ services: container_name: ace-pace volumes: - /path/to/OnePaceLibrary:/media:rw + - ./crc32_files.db:/app/crc32_files.db:rw + - ./episodes_index.db:/app/episodes_index.db:rw + - ./Ace-Pace_Missing.csv:/app/Ace-Pace_Missing.csv:rw networks: - "proxy" environment: + - TZ=Europe/Berlin - TORRENT_HOST=127.0.0.1 - TORRENT_PORT=9091 - TORRENT_CLIENT=transmission - #- NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc + - NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc #- TORRENT_USER=admin #- TORRENT_PASSWORD=password - DB=true From a789b98af83c73718c58e28b5179bd19a99c5769 Mon Sep 17 00:00:00 2001 From: Staubgeborener Date: Mon, 24 Nov 2025 11:48:59 +0100 Subject: [PATCH 27/75] =?UTF-8?q?Add=20=E2=80=94db=20as=20standalone=20fla?= =?UTF-8?q?g=20in=20entrypoint.sh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- entrypoint.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 885a964..90ace59 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,7 +5,10 @@ if [ "$EPISODES_UPDATE" = "true" ]; then python /app/acepace.py --episodes_update fi +if [ "$DB" = "true" ]; then + python /app/acepace.py --db +fi + exec python /app/acepace.py \ ${NYAA_URL:+--url "$NYAA_URL"} \ - ${TORRENT_CLIENT:+--download "$TORRENT_CLIENT"} \ - ${DB:+--db} + ${TORRENT_CLIENT:+--download "$TORRENT_CLIENT"} \ \ No newline at end of file From bc9c54a7c2f8f272e85c9a4569a0d518f18c4275 Mon Sep 17 00:00:00 2001 From: Andrea Cervesato Date: Mon, 8 Dec 2025 20:56:42 +0100 Subject: [PATCH 28/75] Implement clients for qBittorrent and Transmission with functionality to add torrents from magnet links --- .gemini/qbittorrent-api.md | 1817 ++++++++++++++++++++++++++++++++++++ .gitignore | 2 + README.md | 57 +- acepace.py | 199 ++-- clients.py | 138 +++ requirements.txt | 1 + 6 files changed, 2064 insertions(+), 150 deletions(-) create mode 100644 .gemini/qbittorrent-api.md create mode 100644 clients.py diff --git a/.gemini/qbittorrent-api.md b/.gemini/qbittorrent-api.md new file mode 100644 index 0000000..dc9d07b --- /dev/null +++ b/.gemini/qbittorrent-api.md @@ -0,0 +1,1817 @@ +### Install qBittorrent API + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/README.md + +Installs the qBittorrent API client library using pip. + +```bash +python -m pip install qbittorrent-api +``` + +-------------------------------- + +### Install qbittorrent-api via pip + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/introduction.rst + +Installs the qbittorrent-api package from PyPI using pip. This is the standard method for installing Python packages. + +```console +python -m pip install qbittorrent-api +``` + +-------------------------------- + +### Install qbittorrent-api from main branch + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/introduction.rst + +Installs the qbittorrent-api package directly from the main branch of the GitHub repository. This is useful for accessing the latest development features or bug fixes. + +```console +pip install git+https://github.com/rmartin16/qbittorrent-api.git@main#egg=qbittorrent-api +``` + +-------------------------------- + +### Install a specific qbittorrent-api version + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/introduction.rst + +Installs a specific version of the qbittorrent-api package from PyPI. Useful for ensuring compatibility with older projects or testing specific releases. + +```console +python -m pip install qbittorrent-api==2024.3.60 +``` + +-------------------------------- + +### Full Async Example with qbittorrent-api + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/async.rst + +A complete example showing how to initialize the qbittorrentapi client and fetch application build information asynchronously using `asyncio.to_thread`. This code can be run in the Python REPL. + +```python +import asyncio +import qbittorrentapi + +qbt_client = qbittorrentapi.Client() + +async def fetch_qbt_info(): + return await asyncio.to_thread(qbt_client.app_build_info) + +print(asyncio.run(fetch_qbt_info())) +``` + +-------------------------------- + +### qBittorrent Web API Reference + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/README.md + +Provides an overview of the qBittorrent Web API, detailing its version compatibility and features. It also links to comprehensive user guides and API references. + +```APIDOC +qBittorrent Web API Client + +Project: /rmartin16/qbittorrent-api + +Description: +Python client implementation for qBittorrent Web API. +Supports qBittorrent versions up to v5.1.2 (Web API v2.11.4). +Features: +- Implements the entire qBittorrent Web API. +- Automatically handles qBittorrent version checking for endpoint support. +- Automatically requests a new authentication cookie if the current one expires. + +Resources: +- User Guide and API Reference: https://qbittorrent-api.readthedocs.io/ +- qBittorrent GitHub Wiki (Web API): https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-(qBittorrent-4.1) + +API Endpoints Overview: +(Note: Specific endpoint details are extensive and found in the linked documentation. This section summarizes the scope.) +- Application Information: Retrieve details about the qBittorrent application, including version, build info, and Web API version. +- Authentication: Log in, log out, and manage authentication cookies. +- Torrents Management: + - Add torrents (by URL or content). + - Retrieve torrent information (all, by hash, by state). + - Control torrent states (start, stop, pause, resume, delete). + - Manage torrent content (select/deselect files). + - Set torrent properties (download/upload limits, priority, category). +- Downloads Management: + - Control download limits. +- Peers Management: + - Retrieve peer information for torrents. +- Trackers Management: + - Update trackers for torrents. +- Search Management: + - Initiate and retrieve search results. +- Filters Management: + - Manage torrent filters. +- Tags Management: + - Manage tags for torrents. +- Options Management: + - Retrieve and modify qBittorrent settings. +- Web Server Management: + - Control the Web Server. +- RSS Feed Management: + - Manage RSS feeds. +- Transfer List Management: + - Control transfer list operations. + +Error Handling: +- Handles `qbittorrentapi.LoginFailed` for authentication errors. +- Automatically retries authentication on expiration. +``` + +-------------------------------- + +### SearchPluginsList + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/search.rst + +Represents a list of installed search plugins. This structure allows for managing and querying the available search plugins within qBittorrent. + +```APIDOC +qbittorrentapi.search.SearchPluginsList: + __init__(...) + Initializes the SearchPluginsList. + + # Members are typically SearchPlugin objects, each representing an installed search plugin. +``` + +-------------------------------- + +### SearchPlugin + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/search.rst + +Represents a single installed search plugin, providing details about its name, version, and status. This is used to manage and interact with individual search plugins. + +```APIDOC +qbittorrentapi.search.SearchPlugin: + __init__(...) + Initializes the SearchPlugin. + + # Attributes typically include: + # - name (str): The name of the search plugin. + # - version (str): The version of the search plugin. + # - enabled (bool): Whether the plugin is currently enabled. +``` + +-------------------------------- + +### Basic qbittorrent-api Client Usage + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/introduction.rst + +Demonstrates the basic usage of the qbittorrentapi Client, including instantiation, login, logout, and retrieving application information. + +```python +import qbittorrentapi + +# instantiate a Client using the appropriate WebUI configuration +conn_info = dict( + host="localhost", + port=8080, + username="admin", + password="adminadmin", +) +qbt_client = qbittorrentapi.Client(**conn_info) + +# the Client will automatically acquire/maintain a logged-in state +# in line with any request. therefore, this is not strictly necessary; +# however, you may want to test the provided login credentials. +try: + qbt_client.auth_log_in() +except qbittorrentapi.LoginFailed as e: + print(e) + +# if the Client will not be long-lived or many Clients may be created +# in a relatively short amount of time, be sure to log out: +qbt_client.auth_log_out() + +# or use a context manager: +with qbittorrentapi.Client(**conn_info) as qbt_client: + if qbt_client.torrents_add(urls="...") != "Ok.": + raise Exception("Failed to add torrent.") + +# display qBittorrent info +print(f"qBittorrent: {qbt_client.app.version}") +print(f"qBittorrent Web API: {qbt_client.app.web_api_version}") +for k, v in qbt_client.app.build_info.items(): + print(f"{k}: {v}") + +# retrieve and show all torrents +for torrent in qbt_client.torrents_info(): + print(f"{torrent.hash[-6:]}: {torrent.name} ({torrent.state})") + +# stop all torrents +qbt_client.torrents.stop.all() +``` + +-------------------------------- + +### Client Instantiation with Credentials + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Demonstrates how to instantiate the qbittorrentapi.client.Client with host, username, and password. + +```python +from qbittorrentapi import Client + +qbt_client = Client(host="localhost:8080", username='...', password='...') +``` + +-------------------------------- + +### qBittorrent Client Initialization and Torrent Management + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/introduction.rst + +Demonstrates how to initialize the qBittorrent client and iterate through active torrents to perform common operations like setting location, reannouncing, and adjusting upload limits. + +```python +import qbittorrentapi + +qbt_client = qbittorrentapi.Client(host='localhost:8080', username='admin', password='adminadmin') + +for torrent in qbt_client.torrents.info.active(): + torrent.set_location(location='/home/user/torrents/') + torrent.reannounce() + torrent.upload_limit = -1 +``` + +-------------------------------- + +### Configuring HTTPAdapter Arguments + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Illustrates setting arguments for the requests.Session.HTTPAdapter during client instantiation using the HTTPADAPTER_ARGS parameter. + +```python +from qbittorrentapi import Client + +qbt_client = Client(host="localhost:8080", username='...', password='...', HTTPADAPTER_ARGS={"pool_connections": 100, "pool_maxsize": 100}) +``` + +-------------------------------- + +### qbittorrentapi.client.Client API + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/client.rst + +Provides comprehensive documentation for the Client class, including its methods for authentication, torrent management, and client configuration. This serves as the main entry point for interacting with the qBittorrent Web API. + +```APIDOC +Client: + __init__(host: str, port: int = 8080, username: str = None, password: str = None, **kwargs) + Initializes the qBittorrent API client. + Parameters: + host: The hostname or IP address of the qBittorrent instance. + port: The port number for the Web UI (default is 8080). + username: The username for authentication. + password: The password for authentication. + **kwargs: Additional keyword arguments for advanced configuration. + + auth_log_in() + Logs into the qBittorrent Web API. + Returns: True if login is successful, False otherwise. + + auth_log_out() + Logs out of the qBittorrent Web API. + Returns: True if logout is successful, False otherwise. + + get_torrent_list(status_filter: str = 'all', category: str = None, tag: str = None, sort: str = 'name', reverse: bool = False) -> list + Retrieves a list of torrents. + Parameters: + status_filter: Filter torrents by status (e.g., 'downloading', 'completed', 'paused', 'all'). + category: Filter torrents by category. + tag: Filter torrents by tag. + sort: Field to sort torrents by (e.g., 'name', 'size', 'progress'). + reverse: If True, sort in descending order. + Returns: A list of torrent dictionaries. + + add_torrent(torrent_files: list, urls: list = None, save_path: str = None, category: str = None, tags: str = None, is_paused: bool = False) + Adds one or more torrents to qBittorrent. + Parameters: + torrent_files: A list of torrent file contents (bytes). + urls: A list of magnet links or URLs to .torrent files. + save_path: The directory to save the torrents. + category: The category to assign to the torrents. + tags: Comma-separated string of tags to assign. + is_paused: If True, the torrents will be added in a paused state. + + pause_torrent(torrent_hash: str) + Pauses a specific torrent. + Parameters: + torrent_hash: The hash of the torrent to pause. + + resume_torrent(torrent_hash: str) + Resumes a paused torrent. + Parameters: + torrent_hash: The hash of the torrent to resume. + + delete_torrent(torrent_hash: str, delete_files: bool = False) + Deletes a torrent. + Parameters: + torrent_hash: The hash of the torrent to delete. + delete_files: If True, also deletes the torrent's data files. + + get_app_preferences() -> dict + Retrieves the application preferences. + Returns: A dictionary containing application settings. + + set_app_preferences(prefs: dict) + Sets the application preferences. + Parameters: + prefs: A dictionary of preferences to update. + + get_connection_status() -> dict + Retrieves the connection status of the client. + Returns: A dictionary with connection status information. + + shutdown_client() + Shuts down the qBittorrent client. + + reboot_client() + Reboots the qBittorrent client. + + get_web_api_version() -> str + Retrieves the version of the Web API. + Returns: The Web API version string. +``` + +-------------------------------- + +### qbittorrentapi.app.Application Methods + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/app.rst + +Represents the qBittorrent application and exposes methods for managing its settings, preferences, and retrieving information like build details and network interfaces. + +```APIDOC +qbittorrentapi.app.Application: + Manages qBittorrent application settings and retrieves information. + Excludes methods like app, application, webapiVersion, buildInfo, setPreferences, defaultSavePath, setCookies, networkInterfaceAddressList, networkInterfaceList, sendTestEmail, getDirectoryContent. +``` + +-------------------------------- + +### qBittorrent API Client Usage + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/README.md + +Demonstrates how to instantiate a qBittorrent API client, log in, log out, and interact with various API endpoints like adding torrents, retrieving application info, and managing torrent states. + +```python +import qbittorrentapi + +# instantiate a Client using the appropriate WebUI configuration +conn_info = dict( + host="localhost", + port=8080, + username="admin", + password="adminadmin", +) +qbt_client = qbittorrentapi.Client(**conn_info) + +# the Client will automatically acquire/maintain a logged-in state +# in line with any request. therefore, this is not strictly necessary; +# however, you may want to test the provided login credentials. +try: + qbt_client.auth_log_in() +except qbittorrentapi.LoginFailed as e: + print(e) + +# if the Client will not be long-lived or many Clients may be created +# in a relatively short amount of time, be sure to log out: +qbt_client.auth_log_out() + +# or use a context manager: +with qbittorrentapi.Client(**conn_info) as qbt_client: + if qbt_client.torrents_add(urls="...") != "Ok.": + raise Exception("Failed to add torrent.") + +# display qBittorrent info +print(f"qBittorrent: {qbt_client.app.version}") +print(f"qBittorrent Web API: {qbt_client.app.web_api_version}") +for k, v in qbt_client.app.build_info.items(): + print(f"{k}: {v}") + +# retrieve and show all torrents +for torrent in qbt_client.torrents_info(): + print(f"{torrent.hash[-6:]}: {torrent.name} ({torrent.state})") + +# stop all torrents +qbt_client.torrents.stop.all() +``` + +-------------------------------- + +### qbittorrentapi.app.BuildInfoDictionary + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/app.rst + +Represents a dictionary containing build information for the qBittorrent application. + +```APIDOC +qbittorrentapi.app.BuildInfoDictionary: + Dictionary structure for qBittorrent build information. +``` + +-------------------------------- + +### Namespace-Based Interaction with Web API + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/introduction.rst + +Demonstrates a more organized and intuitive way to interact with the qBittorrent Web API using namespaces, allowing for easier management of preferences and torrent operations. + +```python +import qbittorrentapi +qbt_client = qbittorrentapi.Client(host='localhost:8080', username='admin', password='adminadmin') +# changing a preference +is_dht_enabled = qbt_client.app.preferences.dht +qbt_client.app.preferences = dict(dht=not is_dht_enabled) +# stopping all torrents +qbt_client.torrents.stop.all() +# retrieve different views of the log +qbt_client.log.main.warning() +``` + +-------------------------------- + +### Direct Method Calls for Web API Endpoints + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/introduction.rst + +Illustrates how to interact with the qBittorrent Web API by directly calling methods on the client object, corresponding to individual API endpoints. + +```python +import qbittorrentapi +qbt_client = qbittorrentapi.Client(host='localhost:8080', username='admin', password='adminadmin') +qbt_client.app_version() +qbt_client.rss_rules() +qbt_client.torrents_info() +qbt_client.torrents_resume(torrent_hashes='...') +# and so on +``` + +-------------------------------- + +### Client Authentication with auth_log_in + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Shows how to explicitly log in a client instance using username and password. Authentication happens automatically for API requests. + +```python +qbt_client.auth_log_in(username='...', password='...') +``` + +-------------------------------- + +### Instantiate Client with Simple Responses + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/performance.rst + +Demonstrates how to instantiate the qbittorrentapi client with the SIMPLE_RESPONSES flag set to True to always receive simple JSON responses, improving performance by avoiding complex object conversions. + +```python +import qbittorrentapi + +qbt_client = qbittorrentapi.Client( + host='localhost:8080', + username='admin', + password='adminadmin', + SIMPLE_RESPONSES=True, +) +``` + +-------------------------------- + +### qbittorrentapi.app.AppAPIMixIn Methods + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/app.rst + +Provides methods for interacting with the qBittorrent application's API. This class serves as a mixin for application-related functionalities. + +```APIDOC +qbittorrentapi.app.AppAPIMixIn: + Methods related to application settings and information. +``` + +-------------------------------- + +### qbittorrentapi.sync.Sync Documentation + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/sync.rst + +Documentation for the Sync class, which handles synchronization operations. It includes all members, undocumented members, and the special '__call__' member. + +```APIDOC +qbittorrentapi.sync.Sync + :members: + :undoc-members: + :special-members: __call__ +``` + +-------------------------------- + +### qbittorrentapi.app.ApplicationPreferencesDictionary + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/app.rst + +Represents a dictionary for application preferences in qBittorrent. + +```APIDOC +qbittorrentapi.app.ApplicationPreferencesDictionary: + Dictionary structure for qBittorrent application preferences. +``` + +-------------------------------- + +### qbittorrentapi.sync.SyncAPIMixIn Documentation + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/sync.rst + +Documentation for the SyncAPIMixIn class, which provides synchronization-related methods. It excludes 'sync' and 'sync_torrentPeers' members and shows inheritance. + +```APIDOC +qbittorrentapi.sync.SyncAPIMixIn + :members: + :undoc-members: + :exclude-members: sync, sync_torrentPeers + :show-inheritance: +``` + +-------------------------------- + +### SearchAPIMixIn Class + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/search.rst + +Provides search-related functionalities for interacting with qBittorrent's search engine. It includes methods for initiating searches, managing plugins, and retrieving search results. Excludes specific methods that might be handled by a base class or are internal. + +```APIDOC +qbittorrentapi.search.SearchAPIMixIn: + __init__(...) + Initializes the SearchAPIMixIn class. + + search(pattern: str, **kwargs) -> SearchResultsDictionary + Searches for torrents matching the given pattern. + Parameters: + pattern (str): The search query string. + **kwargs: Additional keyword arguments for search options (e.g., category, limit). + Returns: + SearchResultsDictionary: A dictionary containing the search results. + + search_installPlugin(url: str) + Installs a search plugin from the given URL. + Parameters: + url (str): The URL of the search plugin to install. + + search_uninstallPlugin(name: str) + Uninstalls a search plugin by its name. + Parameters: + name (str): The name of the search plugin to uninstall. + + search_enablePlugin(name: str) + Enables a search plugin by its name. + Parameters: + name (str): The name of the search plugin to enable. + + search_updatePlugins() + Updates all installed search plugins. + + search_downloadTorrent(file_url: str, save_path: str, **kwargs) + Downloads a torrent file from the given URL. + Parameters: + file_url (str): The URL of the torrent file. + save_path (str): The path where the torrent should be saved. + **kwargs: Additional keyword arguments for download options. +``` + +-------------------------------- + +### Configuring Request Timeouts + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Demonstrates setting request timeouts for all HTTP requests made by the client using the REQUESTS_ARGS parameter during instantiation. + +```python +from qbittorrentapi import Client + +qbt_client = Client(host="localhost:8080", username='...', password='...', REQUESTS_ARGS={'timeout': (3.1, 30)}) +``` + +-------------------------------- + +### Context Manager for Session Management + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Illustrates using a context manager with qbittorrentapi.Client for proper session handling, ensuring the client is logged in and managing session expiration. + +```python +import qbittorrentapi + +conn_info = { + "host": "localhost:8080", + "username": "...", + "password": "..." +} + +with qbittorrentapi.Client(**conn_info) as qbt_client: + if qbt_client.torrents_add(urls="...") != "Ok.": + raise Exception("Failed to add torrent.") +``` + +-------------------------------- + +### Handle Unsupported qBittorrent Versions + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Configure the client to raise an UnsupportedQbittorrentVersion exception for qBittorrent hosts with versions not fully supported by the client. This ensures compatibility with the client's features. + +```python +from qbittorrentapi import Client + +qbt_client = Client(..., RAISE_ERROR_FOR_UNSUPPORTED_QBITTORRENT_VERSIONS=True) +``` + +-------------------------------- + +### qbittorrentapi.auth.AuthAPIMixIn + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/auth.rst + +Provides authentication methods for interacting with the qBittorrent API. It includes methods for logging in and out, and managing authentication state. Inherits from object. + +```APIDOC +qbittorrentapi.auth.AuthAPIMixIn + __init__(self, auth_client) + Initializes the AuthAPIMixIn with an authentication client. + Parameters: + auth_client: The client responsible for authentication. + + login(self, username, password, **kwargs) + Logs into the qBittorrent Web API. + Parameters: + username (str): The username for authentication. + password (str): The password for authentication. + **kwargs: Additional keyword arguments for login. + Returns: True if login is successful, False otherwise. + + logout(self) + Logs out of the qBittorrent Web API. + Returns: True if logout is successful, False otherwise. + + is_logged_in(self) + Checks if the client is currently logged in. + Returns: True if logged in, False otherwise. + + is_logged_out(self) + Checks if the client is currently logged out. + Returns: True if logged out, False otherwise. +``` + +-------------------------------- + +### qbittorrent-api Documentation Structure + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/api.rst + +This section outlines the structure of the API documentation, indicating that detailed API documentation can be found within the 'apidoc/' directory. It uses a Sphinx toctree directive to organize the documentation. + +```APIDOC +.. toctree:: + :maxdepth: 2 + :glob: + + apidoc/* +``` + +-------------------------------- + +### Set Simple Responses for Individual Method Call + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/performance.rst + +Shows how to override the default behavior and request a simple JSON response for a specific method call by passing SIMPLE_RESPONSES=True as an argument. + +```python +qbt_client.torrents.files(torrent_hash='...', SIMPLE_RESPONSES=True) +``` + +-------------------------------- + +### qbittorrentapi.sync.SyncTorrentPeersDictionary Documentation + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/sync.rst + +Documentation for the SyncTorrentPeersDictionary class, used for representing synchronized torrent peers data. It includes all members, undocumented members, and shows inheritance. + +```APIDOC +qbittorrentapi.sync.SyncTorrentPeersDictionary + :members: + :undoc-members: + :show-inheritance: +``` + +-------------------------------- + +### Instantiating qBittorrent API Client with Sub-Path (Python) + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/CHANGELOG.md + +This snippet demonstrates how to instantiate the qBittorrent API client when the qBittorrent Web API is accessible via a sub-path (e.g., behind a reverse proxy). It shows passing the combined host and sub-path to the `host` parameter of the `Client` constructor. This ensures that all API requests are correctly prefixed with the specified path. + +```Python +Client(host='localhost/qbt') +``` + +-------------------------------- + +### qbittorrentapi.app.NetworkInterfaceList + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/app.rst + +Represents a list of network interfaces available on the qBittorrent client. + +```APIDOC +qbittorrentapi.app.NetworkInterfaceList: + List structure for network interfaces in qBittorrent. +``` + +-------------------------------- + +### qbittorrentapi.sync.SyncMainDataDictionary Documentation + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/sync.rst + +Documentation for the SyncMainDataDictionary class, used for representing synchronized main data. It lists all members, including undocumented ones, and shows inheritance. + +```APIDOC +qbittorrentapi.sync.SyncMainDataDictionary + :members: + :undoc-members: + :show-inheritance: +``` + +-------------------------------- + +### Handling Untrusted Certificates + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Explains how to instantiate the Client with VERIFY_WEBUI_CERTIFICATE=False to handle untrusted or self-signed certificates, disabling certificate verification. + +```python +from qbittorrentapi import Client + +qbt_client = Client(host="localhost:8080", username='...', password='...', VERIFY_WEBUI_CERTIFICATE=False) +``` + +-------------------------------- + +### qbittorrentapi.app.DirectoryContentList + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/app.rst + +Represents a list of directory content items returned by the qBittorrent API. + +```APIDOC +qbittorrentapi.app.DirectoryContentList: + List structure for directory content in qBittorrent. +``` + +-------------------------------- + +### Adding Custom HTTP Headers during Instantiation + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Demonstrates how to include custom HTTP headers in all requests made by an instantiated client by using the EXTRA_HEADERS parameter. + +```python +from qbittorrentapi import Client + +qbt_client = Client(host="localhost:8080", username='...', password='...', EXTRA_HEADERS={'X-My-Fav-Header': 'header value'}) +``` + +-------------------------------- + +### qbittorrentapi.app.NetworkInterface + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/app.rst + +Represents a single network interface on the qBittorrent client. + +```APIDOC +qbittorrentapi.app.NetworkInterface: + Represents a single network interface. +``` + +-------------------------------- + +### qbittorrentapi.request Module Documentation + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/request.rst + +This section details the members, private members, undocumented members, and inheritance of the qbittorrentapi.request module. It serves as a comprehensive reference for interacting with the request functionalities within the library. + +```python +.. automodule:: qbittorrentapi.request + :members: + :private-members: + :undoc-members: + :show-inheritance: +``` + +-------------------------------- + +### LogEntry Class + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/log.rst + +Represents a single entry within the qBittorrent log. This class inherits from a base class, exposing all its members and undocumented members. + +```APIDOC +qbittorrentapi.log.LogEntry + :members: + :undoc-members: + :show-inheritance: +``` + +-------------------------------- + +### AttrDict Class Documentation + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/attrdict.rst + +Provides documentation for the AttrDict class, detailing its members, undocumented members, and inheritance. AttrDict is an internal class for the qbittorrent-api library. + +```python +.. autoclass:: qbittorrentapi._attrdict.AttrDict + :members: + :undoc-members: + :show-inheritance: +``` + +-------------------------------- + +### Handle Unimplemented API Endpoints + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Configure the client to raise a NotImplementedError for API endpoints that are not supported by the host's qBittorrent version. This is useful for early detection of compatibility issues. + +```python +from qbittorrentapi import Client + +qbt_client = Client(..., RAISE_NOTIMPLEMENTEDERROR_FOR_UNIMPLEMENTED_API_ENDPOINTS=True) +``` + +-------------------------------- + +### qBittorrent API - Torrent Operations + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/introduction.rst + +Provides an overview of the methods available for managing torrents through the qBittorrent API client. This includes setting download/upload limits, reannouncing, and changing the save location. + +```APIDOC +qbt_client.torrents.info.active() + - Retrieves a list of currently active torrents. + - Returns: A list of TorrentInfo objects. + +TorrentInfo.set_location(location: str) + - Sets the save location for a specific torrent. + - Parameters: + - location: The new directory path to save the torrent content. + - Returns: None. + +TorrentInfo.reannounce() + - Forces a reannounce for the torrent. + - Returns: None. + +TorrentInfo.upload_limit + - Gets or sets the upload speed limit for the torrent. + - Type: int + - Description: Set to -1 for unlimited upload speed. +``` + +-------------------------------- + +### qbittorrentapi._version_support.Version Class + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/version.rst + +Provides details on the Version class, its members, and undocumented members related to version support in the qbittorrent-api library. This class is crucial for ensuring compatibility between the client and the qBittorrent client. + +```APIDOC +qbittorrentapi._version_support.Version: + Represents and validates qBittorrent versions. + + Methods: + __init__(self, version_string: str) + Initializes the Version object with a version string. + Parameters: + version_string (str): The qBittorrent version string (e.g., "4.4.0"). + + __str__(self) -> str + Returns the string representation of the version. + + __repr__(self) -> str + Returns the detailed representation of the Version object. + + __eq__(self, other) + Checks if this version is equal to another Version object or string. + Parameters: + other (Version | str): The version to compare against. + Returns: + bool: True if versions are equal, False otherwise. + + __ne__(self, other) + Checks if this version is not equal to another Version object or string. + Parameters: + other (Version | str): The version to compare against. + Returns: + bool: True if versions are not equal, False otherwise. + + __lt__(self, other) + Checks if this version is less than another Version object or string. + Parameters: + other (Version | str): The version to compare against. + Returns: + bool: True if this version is less than the other, False otherwise. + + __le__(self, other) + Checks if this version is less than or equal to another Version object or string. + Parameters: + other (Version | str): The version to compare against. + Returns: + bool: True if this version is less than or equal to the other, False otherwise. + + __gt__(self, other) + Checks if this version is greater than another Version object or string. + Parameters: + other (Version | str): The version to compare against. + Returns: + bool: True if this version is greater than the other, False otherwise. + + __ge__(self, other) + Checks if this version is greater than or equal to another Version object or string. + Parameters: + other (Version | str): The version to compare against. + Returns: + bool: True if this version is greater than or equal to the other, False otherwise. + + is_at_least(self, required_version: str) -> bool + Checks if the current version is at least the required version. + Parameters: + required_version (str): The minimum required version string. + Returns: + bool: True if the version meets the requirement, False otherwise. + + Undocumented Members: + _version_tuple (tuple): Internal representation of the version as a tuple of integers. +``` + +-------------------------------- + +### qbittorrentapi.transfer.TransferInfoDictionary + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/transfer.rst + +Documentation for the TransferInfoDictionary class, which represents information related to transfers. This class inherits from other classes and includes various members that provide detailed transfer status and data. + +```APIDOC +TransferInfoDictionary: + Represents transfer information. + Includes members for detailed transfer status. +``` + +-------------------------------- + +### qbittorrentapi.transfer.TransferAPIMixIn Methods + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/transfer.rst + +This section documents the methods available in the TransferAPIMixIn class, which provides core transfer management functionalities. It excludes specific methods related to speed limits and peer banning, which are detailed elsewhere. + +```APIDOC +TransferAPIMixIn: + Methods related to general transfer management. +``` + +-------------------------------- + +### Attr Class Documentation + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/attrdict.rst + +Provides documentation for the Attr class, detailing its members, undocumented members, and inheritance. Attr is an internal class for the qbittorrent-api library. + +```python +.. autoclass:: qbittorrentapi._attrdict.Attr + :members: + :undoc-members: + :show-inheritance: +``` + +-------------------------------- + +### LogMainList Class + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/log.rst + +Represents the main list structure for log entries. This class inherits from a base class, exposing all its members and undocumented members. + +```APIDOC +qbittorrentapi.log.LogMainList + :members: + :undoc-members: + :show-inheritance: +``` + +-------------------------------- + +### Adding Custom HTTP Headers for Individual Requests + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Shows how to send custom HTTP headers for specific API calls using the headers parameter. + +```python +qbt_client.torrents.add(headers={'X-My-Fav-Header': 'header value'}) +``` + +-------------------------------- + +### Search Class + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/search.rst + +Represents the core search functionality in qBittorrent. This class likely encapsulates the logic for executing searches and managing search-related operations. It excludes methods related to plugin management, which are handled by SearchAPIMixIn. + +```APIDOC +qbittorrentapi.search.Search: + __init__(...) + Initializes the Search class. + + installPlugin(url: str) + Installs a search plugin from the given URL. + Parameters: + url (str): The URL of the search plugin to install. + + uninstallPlugin(name: str) + Uninstalls a search plugin by its name. + Parameters: + name (str): The name of the search plugin to uninstall. + + enablePlugin(name: str) + Enables a search plugin by its name. + Parameters: + name (str): The name of the search plugin to enable. + + updatePlugins() + Updates all installed search plugins. + + downloadTorrent(file_url: str, save_path: str, **kwargs) + Downloads a torrent file from the given URL. + Parameters: + file_url (str): The URL of the torrent file. + save_path (str): The path where the torrent should be saved. + **kwargs: Additional keyword arguments for download options. +``` + +-------------------------------- + +### TorrentCreator Class Methods + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/torrentcreator.rst + +This section outlines the methods of the TorrentCreator class, responsible for the core logic of creating torrents. It excludes methods like addTask, torrentFile, and deleteTask, which are likely internal or handled by the mixin. + +```APIDOC +qbittorrentapi.torrentcreator.TorrentCreator + __init__(...) + Initializes the TorrentCreator. + + createTorrent(..., **kwargs) + Creates a torrent file with specified parameters. + Parameters: + ...: Parameters for torrent creation (e.g., files, trackers, name). + Returns: + The created torrent file content or path. + + getTaskStatus(..., **kwargs) + Retrieves the status of a torrent creation task. + Parameters: + task_id: The ID of the task. + Returns: + The status of the specified task. +``` + +-------------------------------- + +### TorrentCreatorAPIMixIn Methods + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/torrentcreator.rst + +This section details the methods available in the TorrentCreatorAPIMixIn class, which serves as a mixin for torrent creation functionalities. It excludes specific internal methods like torrentcreator, torrentcreator_addTask, torrentcreator_torrentFile, and torrentcreator_deleteTask. + +```APIDOC +qbittorrentapi.torrentcreator.TorrentCreatorAPIMixIn + __init__(...) + Initializes the TorrentCreatorAPIMixIn. + + addTorrent(..., **kwargs) + Adds a torrent using the creator. + Parameters: + ...: Various parameters for torrent creation. + Returns: + The result of adding the torrent. + + createTorrent(..., **kwargs) + Creates a torrent file. + Parameters: + ...: Various parameters for torrent creation. + Returns: + The result of creating the torrent file. + + deleteTorrent(..., **kwargs) + Deletes a torrent task. + Parameters: + ...: Various parameters for deleting a torrent task. + Returns: + The result of deleting the torrent task. + + getTorrentCreatorTask(..., **kwargs) + Retrieves a specific torrent creator task. + Parameters: + ...: Parameters to identify the task. + Returns: + The torrent creator task details. + + getTorrentCreatorTasks(..., **kwargs) + Retrieves a list of all torrent creator tasks. + Parameters: + ...: Optional parameters for filtering or pagination. + Returns: + A list of torrent creator tasks. + + updateTorrent(..., **kwargs) + Updates an existing torrent task. + Parameters: + ...: Parameters for updating the torrent task. + Returns: + The result of updating the torrent task. +``` + +-------------------------------- + +### RSSAPIMixIn Methods + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/rss.rst + +Provides methods for interacting with the RSS feed functionalities. This class includes methods for managing RSS feeds, items, and rules. Specific methods like rss_addFolder, rss_addFeed, rss_removeItem, etc., are excluded from this documentation block. + +```APIDOC +qbittorrentapi.rss.RSSAPIMixIn: + Methods for RSS feed management. + Excludes: rss, rss_addFolder, rss_addFeed, rss_removeItem, rss_moveItem, rss_refreshItem, rss_markAsRead, rss_setRule, rss_renameRule, rss_removeRule, rss_matchingArticles, rss_setFeedURL +``` + +-------------------------------- + +### Torrents API Reference + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/torrents.rst + +This section provides a detailed reference for the Torrents API, outlining the available classes and their methods for managing torrents, files, categories, tags, and web seeds. + +```APIDOC +qbittorrentapi.torrents.TorrentsAPIMixIn: + Manages core torrent operations. + Methods: + (See excluded members for specific functionalities) + +qbittorrentapi.torrents.Torrents: + Provides access to torrent-related functionalities. + Methods: + (See excluded members for specific functionalities) + +qbittorrentapi.torrents.TorrentDictionary: + Represents a torrent with its properties. + Methods: + (See excluded members for specific functionalities) + +qbittorrentapi.torrents.TorrentCategories: + Manages torrent categories. + Methods: + removeCategories(categories: list[str]) -> None + Removes specified categories. + editCategory(old_name: str, new_name: str) -> None + Renames a category. + createCategory(name: str, save_path: str = None) -> None + Creates a new category. + +qbittorrentapi.torrents.TorrentTags: + Manages torrent tags. + Methods: + addTags(torrent_hashes: str, tags: str) -> None + Adds tags to torrents. + removeTags(torrent_hashes: str, tags: str) -> None + Removes tags from torrents. + createTags(tags: str) -> None + Creates new tags. + deleteTags(tags: str) -> None + Deletes specified tags. + setTags(torrent_hashes: str, tags: str) -> None + Sets tags for torrents, overwriting existing ones. + +qbittorrentapi.torrents.TorrentPropertiesDictionary: + Dictionary for torrent properties. + +qbittorrentapi.torrents.TorrentLimitsDictionary: + Dictionary for torrent speed limits. + +qbittorrentapi.torrents.TorrentCategoriesDictionary: + Dictionary for torrent categories. + +qbittorrentapi.torrents.TorrentsAddPeersDictionary: + Dictionary for adding peers to torrents. + +qbittorrentapi.torrents.TorrentFilesList: + List of files within a torrent. + +qbittorrentapi.torrents.TorrentFile: + Represents a single file in a torrent. + +qbittorrentapi.torrents.WebSeedsList: + List of web seeds for a torrent. + +qbittorrentapi.torrents.WebSeed: + Represents a single web seed. + +qbittorrentapi.torrents.TrackersList: + List of trackers for a torrent. + +qbittorrentapi.torrents.Tracker: + Represents a single tracker. + +qbittorrentapi.torrents.TorrentInfoList: + List of torrent information. + +qbittorrentapi.torrents.TorrentPieceInfoList: + List of piece information for a torrent. + +qbittorrentapi.torrents.TorrentPieceData: + Data for a specific piece of a torrent. + +qbittorrentapi.torrents.TagList: + List of tags. + +qbittorrentapi.torrents.Tag: + Represents a single tag. +``` + +-------------------------------- + +### LogAPIMixIn Class + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/log.rst + +Provides methods for interacting with the qBittorrent log API. It serves as a mixin class, likely intended to be inherited by other classes that require log functionality. Specific methods are exposed through its members, excluding the 'log' attribute. + +```APIDOC +qbittorrentapi.log.LogAPIMixIn + :members: + :undoc-members: + :exclude-members: log + :show-inheritance: +``` + +-------------------------------- + +### Setting Timeouts for Individual Requests + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Shows how to specify request timeouts for individual API calls using the requests_args parameter. + +```python +qbt_client.torrents_info(requests_args={'timeout': (3.1, 30)}) +``` + +-------------------------------- + +### qbittorrentapi.transfer.Transfer Methods + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/transfer.rst + +This section documents the methods of the Transfer class, focusing on transfer operations. It excludes methods for toggling speed limits, setting limits, and banning peers, which are handled separately. + +```APIDOC +Transfer: + Methods for managing transfer operations. +``` + +-------------------------------- + +### SearchJobDictionary + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/search.rst + +Represents a dictionary structure for search jobs, likely containing information about ongoing or completed search operations. It inherits from a base dictionary type and includes specific members relevant to search jobs. + +```APIDOC +qbittorrentapi.search.SearchJobDictionary: + __init__(...) + Initializes the SearchJobDictionary. + + # Members typically include information about: + # - Job ID + # - Search query + # - Status (e.g., running, completed, failed) + # - Progress + # - Number of results found +``` + +-------------------------------- + +### qbittorrentapi Exception Hierarchy + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/exceptions.rst + +This section outlines the exception classes provided by the qbittorrentapi library, detailing their inheritance structure and available members. It helps developers understand and handle potential errors during interaction with the qBittorrent API. + +```python +import qbittorrentapi + +try: + # Attempt to interact with qBittorrent API + pass +except qbittorrentapi.LoginFailed as e: + print(f"Login failed: {e}") +except qbittorrentapi.APIConnectionError as e: + print(f"Connection error: {e}") +except qbittorrentapi.NotFoundHTTPError as e: + print(f"Resource not found: {e}") +except qbittorrentapi.ForbiddenHTTPError as e: + print(f"Forbidden access: {e}") +except qbittorrentapi.BadRequestHTTPError as e: + print(f"Bad request: {e}") +except qbittorrentapi.ServerError as e: + print(f"Server error: {e}") +except qbittorrentapi.QBittorrentError as e: + print(f"An unexpected qBittorrent API error occurred: {e}") +except Exception as e: + print(f"An unexpected error occurred: {e}") +``` + +-------------------------------- + +### Log Class + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/log.rst + +Represents the main log functionality within the qbittorrent-api. This class exposes various members for accessing and managing log data. It also supports being called directly, indicated by the special member '__call__'. + +```APIDOC +qbittorrentapi.log.Log + :members: + :undoc-members: + :special-members: __call__ +``` + +-------------------------------- + +### LogPeer Class + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/log.rst + +Represents a single log peer entry. This class inherits from a base class, exposing all its members and undocumented members. + +```APIDOC +qbittorrentapi.log.LogPeer + :members: + :undoc-members: + :show-inheritance: +``` + +-------------------------------- + +### MutableAttr Class Documentation + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/attrdict.rst + +Provides documentation for the MutableAttr class, detailing its members, undocumented members, and inheritance. MutableAttr is an internal class for the qbittorrent-api library. + +```python +.. autoclass:: qbittorrentapi._attrdict.MutableAttr + :members: + :undoc-members: + :show-inheritance: +``` + +-------------------------------- + +### qbittorrentapi.app.CookieList + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/app.rst + +Represents a list of cookies used with the qBittorrent API. + +```APIDOC +qbittorrentapi.app.CookieList: + List structure for cookies in qBittorrent. +``` + +-------------------------------- + +### TorrentCreatorTaskStatus Members + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/torrentcreator.rst + +This section describes the members of the TorrentCreatorTaskStatus class, which likely enumerates the possible states for a torrent creation task. + +```APIDOC +qbittorrentapi.torrentcreator.TorrentCreatorTaskStatus + PENDING = 'pending' + The task is waiting to be processed. + + PROCESSING = 'processing' + The task is currently being processed. + + COMPLETED = 'completed' + The task has finished successfully. + + FAILED = 'failed' + The task failed to complete. +``` + +-------------------------------- + +### Manual qBittorrent Version Introspection + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Manually check if a qBittorrent application version is supported by the client using the Version.is_app_version_supported method. This allows for custom handling of version compatibility. + +```python +from qbittorrentapi import Client, Version + +qbt_client = Client(...) + +if Version.is_app_version_supported(qbt_client.app.version): + print("qBittorrent version is supported.") +else: + print("qBittorrent version is not supported.") +``` + +-------------------------------- + +### RSS Class Methods + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/rss.rst + +Represents the core RSS functionality. It includes methods for managing RSS feeds and items. The special member __call__ is documented. Methods like rss, addFolder, addFeed, removeItem, etc., are excluded. + +```APIDOC +qbittorrentapi.rss.RSS: + Core RSS functionality. + Special Members: __call__ + Excludes: rss, addFolder, addFeed, removeItem, moveItem, refreshItem, markAsRead, setRule, renameRule, removeRule, matchingArticles, setFeedURL +``` + +-------------------------------- + +### qbittorrentapi.auth.Authorization + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/auth.rst + +Represents authorization details for API requests. This class is typically used internally by the authentication mixin. + +```APIDOC +qbittorrentapi.auth.Authorization + __init__(self, username, password) + Initializes the Authorization object with username and password. + Parameters: + username (str): The username for authorization. + password (str): The password for authorization. + + username + The username associated with this authorization. + + password + The password associated with this authorization. +``` + +-------------------------------- + +### qbittorrentapi.definitions Module Members + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/definitions.rst + +This section details the members of the qbittorrentapi.definitions module. It includes all documented members but excludes the TorrentStates enum and undocumented members. The show-inheritance flag indicates that inheritance relationships are displayed. + +```python +import qbittorrentapi + +# Accessing members of the definitions module +# Example: Accessing a specific definition class or function +# print(dir(qbittorrentapi.definitions)) + +# The following is a representation of what might be documented within the module. +# Specific members are not listed here as they are dynamically generated by the automodule directive. + +# Example of a potential class within definitions: +# class SomeDefinition: +# """A sample definition class.""" +# def __init__(self, value): +# self.value = value + +# Example of a potential function within definitions: +# def some_function(param1: str) -> int: +# """A sample function.""" +# return len(param1) + +# The automodule directive with :members:, :undoc-members:, and :show-inheritance: +# implies that the following would be generated and displayed in the documentation: +# - All public members (functions, classes, variables) +# - Undocumented members (if any) +# - Inheritance hierarchy for classes +# - Excludes 'TorrentStates' as specified. +``` + +-------------------------------- + +### TorrentCreatorTaskStatusList Members + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/torrentcreator.rst + +This section describes the members of the TorrentCreatorTaskStatusList class, which is likely a container for multiple torrent creation task statuses. + +```APIDOC +qbittorrentapi.torrentcreator.TorrentCreatorTaskStatusList + __init__(...) + Initializes a TorrentCreatorTaskStatusList. + + tasks: list[TorrentCreatorTaskDictionary] + A list containing TorrentCreatorTaskDictionary objects. +``` + +-------------------------------- + +### LogPeersList Class + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/log.rst + +Defines the structure for a list of log peers. This class inherits from a base class, indicated by 'show-inheritance', and includes all its members and undocumented members. + +```APIDOC +qbittorrentapi.log.LogPeersList + :members: + :undoc-members: + :show-inheritance: +``` + +-------------------------------- + +### Fetch Torrents Asynchronously + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/async.rst + +Demonstrates how to fetch torrent information asynchronously by running the blocking `qbt_client.torrents_info` method in a separate thread using `asyncio.to_thread`. This prevents blocking the asyncio event loop. + +```python +async def fetch_torrents() -> TorrentInfoList: + return await asyncio.to_thread(qbt_client.torrents_info, category="uploaded") +``` + +-------------------------------- + +### SearchCategoriesList + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/search.rst + +Represents a list of available search categories. This structure is used to manage and retrieve the categories that can be used when performing searches. + +```APIDOC +qbittorrentapi.search.SearchCategoriesList: + __init__(...) + Initializes the SearchCategoriesList. + + # Members are typically SearchCategory objects, each representing a searchable category. +``` + +-------------------------------- + +### SearchStatusesList + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/search.rst + +Represents a list of search statuses, likely used to track the state of multiple search operations. It inherits from a base list type and contains individual SearchStatus objects. + +```APIDOC +qbittorrentapi.search.SearchStatusesList: + __init__(...) + Initializes the SearchStatusesList. + + # Members are typically SearchStatus objects, each representing the status of a single search job. +``` + +-------------------------------- + +### SearchCategory + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/search.rst + +Represents a single search category, providing its name and potentially other relevant information. This is used to define the types of content that can be searched for. + +```APIDOC +qbittorrentapi.search.SearchCategory: + __init__(...) + Initializes the SearchCategory. + + # Attributes typically include: + # - name (str): The name of the search category (e.g., 'all', 'movies', 'music'). + # - supported_by (list): A list of plugins that support this category. +``` + +-------------------------------- + +### SearchStatus + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/search.rst + +Represents the status of a single search job, providing details about its progress and completion. It inherits from a base object type and includes specific attributes for status information. + +```APIDOC +qbittorrentapi.search.SearchStatus: + __init__(...) + Initializes the SearchStatus. + + # Attributes typically include: + # - status (str): The current status of the search (e.g., 'Running', 'Completed', 'Error'). + # - progress (int): The progress of the search in percentage. + # - total (int): The total number of items found. +``` + +-------------------------------- + +### TorrentCreatorTaskDictionary Members + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/torrentcreator.rst + +Details the members of the TorrentCreatorTaskDictionary class, used for representing torrent creation tasks. It excludes torrentFile and deleteTask, suggesting these might be handled at a different level. + +```APIDOC +qbittorrentapi.torrentcreator.TorrentCreatorTaskDictionary + __init__(...) + Initializes a TorrentCreatorTaskDictionary. + + task_id: int + The unique identifier for the torrent creation task. + + status: TorrentCreatorTaskStatus + The current status of the task. + + progress: float + The progress of the torrent creation task (0.0 to 1.0). + + created_torrent_path: str + The file path where the torrent was created. + + error_message: str + An error message if the task failed. +``` + +-------------------------------- + +### SearchResultsDictionary + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/search.rst + +Represents a dictionary structure for search results, containing a list of torrents that match a search query. It inherits from a base dictionary type and provides access to individual search results. + +```APIDOC +qbittorrentapi.search.SearchResultsDictionary: + __init__(...) + Initializes the SearchResultsDictionary. + + # Members typically include: + # - A list of SearchResults (or similar objects) + # - Total number of results + # - Pagination information (if applicable) +``` + +-------------------------------- + +### qbittorrentapi.app.NetworkInterfaceAddressList + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/app.rst + +Represents a list of IP addresses associated with a network interface. + +```APIDOC +qbittorrentapi.app.NetworkInterfaceAddressList: + List structure for network interface addresses in qBittorrent. +``` + +-------------------------------- + +### qbittorrentapi.app.Cookie + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/app.rst + +Represents a cookie used for authentication or other purposes with the qBittorrent API. + +```APIDOC +qbittorrentapi.app.Cookie: + Represents a single cookie. +``` + +-------------------------------- + +### TaskStatus Members + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/torrentcreator.rst + +Details the members of the TaskStatus class, which appears to be an alias or a more general status indicator, possibly for tasks within the torrent creator system. + +```APIDOC +qbittorrentapi.torrentcreator.TaskStatus + PENDING = 'pending' + Task is pending. + + PROCESSING = 'processing' + Task is currently processing. + + COMPLETED = 'completed' + Task has been completed. + + FAILED = 'failed' + Task has failed. +``` + +-------------------------------- + +### RSSitemsDictionary Structure + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/rss.rst + +Defines the structure for dictionaries containing RSS items. This class is used to represent collections of RSS feed items. + +```APIDOC +qbittorrentapi.rss.RSSitemsDictionary: + Dictionary structure for RSS items. + Inheritance: show-inheritance +``` + +-------------------------------- + +### RSSRulesDictionary Structure + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/apidoc/rss.rst + +Defines the structure for dictionaries containing RSS rules. This class is used to represent collections of RSS feed rules. + +```APIDOC +qbittorrentapi.rss.RSSRulesDictionary: + Dictionary structure for RSS rules. + Inheritance: show-inheritance +``` + +-------------------------------- + +### Disable Logging Debug Output + +Source: https://github.com/rmartin16/qbittorrent-api/blob/main/docs/source/behavior&configuration.rst + +Disable debug logging for the qbittorrentapi and related packages by setting the logger level to INFO. This can be done during client instantiation or by directly configuring loggers. + +```python +import logging +from qbittorrentapi import Client + +# Option 1: During client instantiation +# qbt_client = Client(..., DISABLE_LOGGING_DEBUG_OUTPUT=True) + +# Option 2: Manually configure loggers +logging.getLogger('qbittorrentapi').setLevel(logging.INFO) +logging.getLogger('requests').setLevel(logging.INFO) +logging.getLogger('urllib3').setLevel(logging.INFO) +``` + +=== COMPLETE CONTENT === This response contains all available snippets from this library. No additional content exists. Do not make further requests. \ No newline at end of file diff --git a/.gitignore b/.gitignore index d6bbd50..5320855 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ pip-log.txt *.log Ace-Pace_DB.csv Ace-Pace_Missing.csv +crc32_files.db +check.sh diff --git a/README.md b/README.md index 1ae9372..3297e01 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# Ace-Pace +# 🏴‍☠️ Ace-Pace Welcome to **Ace-Pace**, your ultimate companion for organizing and managing your One-Pace library with precision and ease! Whether you're a casual viewer who wants a neat collection or a hardcore fan aiming for the perfect sync between episodes and the official One-Pace releases, Ace-Pace is designed to make your life simpler, your library cleaner, and your watching experience smoother. One-Pace is a fantastic fan project that trims the One Piece anime down to its essential story arcs, removing filler and pacing issues to deliver a tighter, more engaging narrative. However, managing your One-Pace episodes, ensuring you have all the latest releases can be a daunting task. That's where Ace-Pace comes in — it automates the heavy lifting, letting you focus on enjoying the adventure. -## How to Install +## 🚀 How to Install To get started with Ace-Pace, you'll need to have Python installed on your system. We recommend using Python 3.6 or higher. You can download Python from the [official website](https://www.python.org/downloads/). @@ -18,35 +18,64 @@ pip install -r requirements.txt This will install all necessary packages to ensure Ace-Pace runs smoothly. -## How to Use +## 🛠️ How to Use Run the script using Python with the following command: ``` -python acepace.py [-h] [--url URL] [--folder FOLDER] [--db] [--download CLIENT] +python acepace.py [-h] [--url URL] [--folder FOLDER] [--db] [--client {transmission,qbittorrent}] [--download] [--host HOST] [--port PORT] [--username USERNAME] [--password PASSWORD] [--download-folder DOWNLOAD_FOLDER] [--tag TAG]... [--category CATEGORY] ``` -- `--folder ` (required for most cases) +### 🔭 Main commands +- `--folder ` Specify the path to your local One-Pace video library. Ace-Pace will scan this directory recursively to identify and analyze your existing episodes. -- `--url ` +- `--url ` Define the Nyaa URL used for the query to get episodes metadata and download links. Defaults to `https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc`. -- `--db` (standalone flag) +- `--db` Create a CSV file with the existing local file paths and CRC32 checksums. Useful to check what's detected and debugging. -- `--download ` (standalone flag) - Enable downloading of missing episodes using a BitTorrent client (only Transmission is supported currently). +### 📥 Download commands +- `--client ` + Specify the BitTorrent client to use for downloading missing episodes. + Supported clients: `transmission`, `qbittorrent`. -### Some examples +- `--download` + Enable downloading of missing episodes using the specified BitTorrent client. + +- `--host ` + The BitTorrent client host (default: `localhost`). + +- `--port ` + The BitTorrent client port. + +- `--username ` + The BitTorrent client username. + +- `--password ` + The BitTorrent client password. + +- `--download-folder ` + The folder to download the torrents to. + +- `--tag ...` + Tag to add to the torrent in qBittorrent (can be used multiple times). + +- `--category ` + Category to add to the torrent in qBittorrent. + + +### 📚 Some examples ``` python acepace.py --folder "/volume42/media/One Piece/" --url https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc python acepace.py --folder "/volume42/media/One Piece/" -python acepace.py --download transmission +python acepace.py --client transmission --download +python acepace.py --client qbittorrent --download --host 192.168.1.100 --port 8080 --username myuser --password mypassword --download-folder /downloads/onepace --tag onepace --tag 'one pace' --category 'anime' python acepace.py --db ``` -## Workflow Overview +## 📜 Workflow Overview 1. **Scanning:** Ace-Pace begins by scanning your specified folder, computing CRC32 checksums for each video file to build an accurate inventory of your current collection and store it locally. @@ -54,8 +83,8 @@ python acepace.py --db 3. **Reporting:** A detailed report is generated, highlighting which episodes you already have, which are missing, and any discrepancies. -4. **Optional Downloading:** After that, Ace-Pace will propose to download any missing episodes directly on your BitTorrent client (Transmission only for now). +4. **Optional Downloading:** After that, Ace-Pace will propose to download any missing episodes directly on your BitTorrent client. -## Credits +## 🙏 Credits Ace-Pace is proudly inspired by and built to support the incredible work of the [One-Pace](http://onepace.net/) team. Their dedication to crafting a seamless and engaging One Piece viewing experience has allowed me to discover and share this legendary series. I salute their passion, creativity, and commitment. diff --git a/acepace.py b/acepace.py index f56aa0e..ea6abdc 100644 --- a/acepace.py +++ b/acepace.py @@ -1,14 +1,17 @@ -import requests -from bs4 import BeautifulSoup -import os -import zlib -import argparse -import re -import sqlite3 -from datetime import datetime -import csv -import time import getpass +import time +import csv +from datetime import datetime +import sqlite3 +import re +import argparse +import zlib +import os +from bs4 import BeautifulSoup +import requests + +from clients import get_client + # Define regex to extract CRC32 from filename text (commonly in [xxxxx]) CRC32_REGEX = re.compile(r"\[([A-Fa-f0-9]{8})\]") @@ -475,129 +478,6 @@ def export_db_to_csv(conn): set_metadata(conn, "last_db_export", now_str) -def download_with_transmission(): - if not os.path.exists("Ace-Pace_Missing.csv"): - print("Missing file 'Ace-Pace_Missing.csv' not found. Run the script first!") - return - - magnets = [] - with open("Ace-Pace_Missing.csv", "r", encoding="utf-8") as f: - reader = csv.DictReader(f) - for row in reader: - magnet_link = row.get("Magnet Link", "").strip() - if magnet_link.startswith("magnet:"): - magnets.append(magnet_link) - - if not magnets: - print("No magnet links found in 'Ace-Pace_Missing.csv'.") - return - - print("The details below are not stored.") - host = input("Enter Transmission host (default: localhost): ").strip() - if not host: - host = "localhost" - port_input = input("Enter Transmission port (default: 9091): ").strip() - if port_input: - try: - port = int(port_input) - except ValueError: - print("Invalid port number. Using default 9091.") - port = 9091 - else: - port = 9091 - rpc_username = input("Enter Transmission username (leave blank if none): ").strip() - rpc_password = getpass.getpass( - "Enter Transmission password (leave blank if none): " - ).strip() - - base_url = f"http://{host}:{port}/transmission/rpc" - session_id = None - session = requests.Session() - auth = (rpc_username, rpc_password) if rpc_username else None - - # Test connection and get session ID - try: - headers = {} - if session_id: - headers["X-Transmission-Session-Id"] = session_id - resp = session.post( - base_url, auth=auth, headers=headers, json={"method": "session-get"} - ) - if resp.status_code == 409: - session_id = resp.headers.get("X-Transmission-Session-Id") - headers["X-Transmission-Session-Id"] = session_id - resp = session.post( - base_url, auth=auth, headers=headers, json={"method": "session-get"} - ) - resp.raise_for_status() - except Exception as e: - print(f"Failed to connect to Transmission RPC: {e}") - return - - print("Connection to Transmission successful!") - - # Suggest default download directory to user - try: - session_info = resp.json() - default_download_dir = "" - if "arguments" in session_info and "download-dir" in session_info["arguments"]: - default_download_dir = session_info["arguments"]["download-dir"] - except Exception: - default_download_dir = "" - - if default_download_dir: - prompt_text = f"Enter target folder for downloads (current default: {default_download_dir}): " - else: - prompt_text = "Enter target folder for downloads (leave blank for default): " - target_folder = input(prompt_text).strip() - - confirm = ( - input(f"Do you want to add {len(magnets)} torrents to Transmission? (y/n): ") - .strip() - .lower() - ) - if confirm != "y": - print("Abort! Abort!") - return - - added_count = 0 - total = len(magnets) - for idx, magnet in enumerate(magnets, 1): - truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") - print(f"Adding {idx}/{total}: {truncated}") - payload = {"method": "torrent-add", "arguments": {"filename": magnet}} - if target_folder: - payload["arguments"]["download-dir"] = target_folder - try: - headers = {"X-Transmission-Session-Id": session_id} if session_id else {} - resp = session.post(base_url, auth=auth, headers=headers, json=payload) - if resp.status_code == 409: - session_id = resp.headers.get("X-Transmission-Session-Id") - headers["X-Transmission-Session-Id"] = session_id - resp = session.post(base_url, auth=auth, headers=headers, json=payload) - resp.raise_for_status() - result = resp.json() - if result.get("result") == "success": - added_count += 1 - else: - print( - f"Failed to add torrent: {truncated} Error: {result.get('result')}" - ) - time.sleep(0.1) - except Exception as e: - print(f"Failed to add torrent: {truncated} Error: {e}") - - print(f"Added {added_count} torrents to Transmission.") - - -def download_missing_to_client(client_type): - client_type = client_type.lower() - if client_type == "transmission": - download_with_transmission() - else: - print(f"Download client '{client_type}' not supported.") - - def main(): parser = argparse.ArgumentParser( description="Find missing episodes from your personal One Pace library." @@ -611,10 +491,15 @@ def main(): parser.add_argument( "--db", action="store_true", help="Export database to CSV and exit." ) + parser.add_argument( + "--client", + choices=["transmission", "qbittorrent"], + help="The BitTorrent client to use.", + ) parser.add_argument( "--download", - metavar="CLIENT", - help="Import magnet links from missing CSV and add to specified BitTorrent client (e.g. transmission).", + action="store_true", + help="Import magnet links from missing CSV and add to the specified BitTorrent client.", ) parser.add_argument( "--rename", @@ -626,6 +511,13 @@ def main(): action="store_true", help="Update episodes metadata database from Nyaa.", ) + parser.add_argument("--host", default="localhost", help="The BitTorrent client host.") + parser.add_argument("--port", type=int, help="The BitTorrent client port.") + parser.add_argument("--username", help="The BitTorrent client username.") + parser.add_argument("--password", help="The BitTorrent client password.") + parser.add_argument("--download-folder", help="The folder to download the torrents to.") + parser.add_argument("--tag", action="append", help="Tag to add to the torrent in qBittorrent (can be used multiple times).") + parser.add_argument("--category", help="Category to add to the torrent in qBittorrent.") args = parser.parse_args() # Check if the URL points to a valid Nyaa domain @@ -675,7 +567,42 @@ def main(): set_metadata(conn, "last_folder", folder) if args.download: - download_missing_to_client(args.download) + if not args.client: + print("Error: --client is required when using --download.") + return + + if not os.path.exists("Ace-Pace_Missing.csv"): + print("Missing file 'Ace-Pace_Missing.csv' not found. Run the script first!") + return + + magnets = [] + with open("Ace-Pace_Missing.csv", "r", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + magnet_link = row.get("Magnet Link", "").strip() + if magnet_link.startswith("magnet:"): + magnets.append(magnet_link) + + if not magnets: + print("No magnet links found in 'Ace-Pace_Missing.csv'.") + return + + port = args.port + if not port: + port = 9091 if args.client == "transmission" else 8080 + + try: + client = get_client(args.client, args.host, port, args.username, args.password) + client.add_torrents( + magnets, + download_folder=args.download_folder, + tags=args.tag, + category=args.category, + ) + except (ValueError, Exception) as e: + print(f"Error: {e}") + return + return if args.rename: diff --git a/clients.py b/clients.py new file mode 100644 index 0000000..7281a55 --- /dev/null +++ b/clients.py @@ -0,0 +1,138 @@ +import abc +import getpass +import time +import requests +import qbittorrentapi +import re + +class Client(abc.ABC): + @abc.abstractmethod + def add_torrents(self, torrents, download_folder=None, tags=None, category=None): + pass + +class QBittorrentClient(Client): + def __init__(self, host, port, username, password): + self.client = qbittorrentapi.Client( + host=host, + port=port, + username=username, + password=password + ) + try: + self.client.auth_log_in() + except qbittorrentapi.LoginFailed as e: + raise Exception(f"Failed to connect to qBittorrent: {e}") from e + print("Connection to qBittorrent successful!") + + def add_torrents(self, magnets, download_folder=None, tags=None, category=None): + if tags: + self.client.torrents_create_tags(tags=",".join(tags)) + + added_count = 0 + total = len(magnets) + tags_str = ",".join(tags) if tags else None + for idx, magnet in enumerate(magnets, 1): + truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") + print(f"Processing {idx}/{total}: {truncated}") + + # Extract info hash from magnet link + match = re.search(r"xt=urn:btih:([a-fA-F0-9]{40})", magnet) + if not match: + print(f"Could not find info hash in magnet link: {truncated}") + continue + + info_hash = match.group(1).lower() + + # Check if torrent already exists + existing_torrent = self.client.torrents_info(torrent_hashes=info_hash) + + if existing_torrent: + print(f"Torrent {truncated} already exists.") + if tags: + print(f"Adding tags to existing torrent: {tags_str}") + self.client.torrents_add_tags(tags=tags_str, torrent_hashes=info_hash) + else: + print(f"Adding new torrent: {truncated}") + try: + self.client.torrents_add( + urls=magnet, + save_path=download_folder if download_folder else None, + tags=tags_str, + category=category, + ) + added_count += 1 + except Exception as e: + print(f"Failed to add torrent: {truncated} Error: {e}") + time.sleep(0.1) + print(f"Added {added_count} new torrents to qBittorrent.") + + +class TransmissionClient(Client): + def __init__(self, host, port, username, password): + self.base_url = f"http://{host}:{port}/transmission/rpc" + self.session_id = None + self.session = requests.Session() + self.auth = (username, password) if username else None + + # Test connection and get session ID + try: + headers = {} + if self.session_id: + headers["X-Transmission-Session-Id"] = self.session_id + resp = self.session.post( + self.base_url, auth=self.auth, headers=headers, json={"method": "session-get"} + ) + if resp.status_code == 409: + self.session_id = resp.headers.get("X-Transmission-Session-Id") + headers["X-Transmission-Session-Id"] = self.session_id + resp = self.session.post( + self.base_url, auth=self.auth, headers=headers, json={"method": "session-get"} + ) + resp.raise_for_status() + except Exception as e: + raise Exception(f"Failed to connect to Transmission RPC: {e}") from e + + print("Connection to Transmission successful!") + self.session_info = resp.json() + + + def add_torrents(self, magnets, download_folder=None, tags=None, category=None): + if tags or category: + print("Warning: Transmission does not support tags or categories through this script.") + added_count = 0 + total = len(magnets) + for idx, magnet in enumerate(magnets, 1): + truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") + print(f"Adding {idx}/{total}: {truncated}") + payload = {"method": "torrent-add", "arguments": {"filename": magnet}} + if download_folder: + payload["arguments"]["download-dir"] = download_folder + try: + headers = {"X-Transmission-Session-Id": self.session_id} if self.session_id else {} + resp = self.session.post(self.base_url, auth=self.auth, headers=headers, json=payload) + if resp.status_code == 409: + self.session_id = resp.headers.get("X-Transmission-Session-Id") + headers["X-Transmission-Session-Id"] = self.session_id + resp = self.session.post(self.base_url, auth=self.auth, headers=headers, json=payload) + resp.raise_for_status() + result = resp.json() + if result.get("result") == "success": + added_count += 1 + else: + print( + f"Failed to add torrent: {truncated} Error: {result.get('result')}" + ) + time.sleep(0.1) + except Exception as e: + print(f"Failed to add torrent: {truncated} Error: {e}") + + print(f"Added {added_count} torrents to Transmission.") + + +def get_client(client_name, host, port, username, password): + if client_name == 'qbittorrent': + return QBittorrentClient(host, port, username, password) + elif client_name == 'transmission': + return TransmissionClient(host, port, username, password) + else: + raise ValueError(f'Unknown client: {client_name}') diff --git a/requirements.txt b/requirements.txt index 1190bd8..86847ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests beautifulsoup4 +qbittorrent-api From 76e01d40e700aeb894a4a536425fe804e3a32a83 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 20 Jan 2026 10:55:19 +0000 Subject: [PATCH 29/75] Add specs and tests --- acepace.py | 21 +- pytest.ini | 13 + requirements.txt | 2 + spec.md | 273 +++++++++++++ tests/README.md | 100 +++++ tests/__init__.py | 1 + tests/conftest.py | 146 +++++++ tests/test_clients.py | 201 ++++++++++ tests/test_crc32.py | 154 ++++++++ tests/test_database.py | 132 +++++++ tests/test_episodes.py | 661 ++++++++++++++++++++++++++++++++ tests/test_file_operations.py | 156 ++++++++ tests/test_missing_detection.py | 204 ++++++++++ 13 files changed, 2062 insertions(+), 2 deletions(-) create mode 100644 pytest.ini create mode 100644 spec.md create mode 100644 tests/README.md create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_clients.py create mode 100644 tests/test_crc32.py create mode 100644 tests/test_database.py create mode 100644 tests/test_episodes.py create mode 100644 tests/test_file_operations.py create mode 100644 tests/test_missing_detection.py diff --git a/acepace.py b/acepace.py index ea6abdc..f2e4108 100644 --- a/acepace.py +++ b/acepace.py @@ -16,6 +16,9 @@ # Define regex to extract CRC32 from filename text (commonly in [xxxxx]) CRC32_REGEX = re.compile(r"\[([A-Fa-f0-9]{8})\]") +# Quality regex patterns - matches [1080p], [720p], etc. (case insensitive) +QUALITY_REGEX = re.compile(r"\[(\d+p)\]", re.IGNORECASE) + # Video file extensions we care about VIDEO_EXTENSIONS = {".mkv", ".mp4", ".avi"} @@ -98,11 +101,25 @@ def fetch_episodes_metadata(): Returns: List of (crc32, title, page_link) """ + def _is_valid_quality(fname_text): + """Check if filename has valid quality (1080p preferred, 720p as fallback only). + Returns True if quality is 1080p or 720p, False otherwise.""" + quality_matches = QUALITY_REGEX.findall(fname_text) + if not quality_matches: + return False # No quality marker found, exclude + # Check if quality is exactly 1080p or 720p (not higher, not lower) + for quality in quality_matches: + quality_num = int(quality.lower().replace('p', '')) + if quality_num == 1080 or quality_num == 720: + return True + return False # Quality not 1080p or 720p + def _process_fname_entry(fname_text, seen_crc32, episodes, page_link): - """Helper to extract CRC32 from fname_text and store if valid and unique.""" + """Helper to extract CRC32 from fname_text and store if valid and unique. + Only accepts episodes with 1080p or 720p quality (720p as fallback).""" m = CRC32_REGEX.findall(fname_text) found = False - if m and "[One Pace]" in fname_text: + if m and "[One Pace]" in fname_text and _is_valid_quality(fname_text): crc32 = m[-1].upper() if crc32 not in seen_crc32: # print(f"New CRC32 detected: {crc32} -> Title: {fname_text}") diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..f091a6d --- /dev/null +++ b/pytest.ini @@ -0,0 +1,13 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = + -v + --strict-markers + --tb=short +markers = + unit: Unit tests + integration: Integration tests + slow: Slow running tests diff --git a/requirements.txt b/requirements.txt index 86847ec..6be6e43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ requests beautifulsoup4 qbittorrent-api +pytest>=7.0.0 +pytest-mock>=3.10.0 diff --git a/spec.md b/spec.md new file mode 100644 index 0000000..3434873 --- /dev/null +++ b/spec.md @@ -0,0 +1,273 @@ +# Ace-Pace Project Specification + +## Project Overview + +**Ace-Pace** is a Python-based tool designed to help users manage and organize their One-Pace anime library. One-Pace is a fan project that edits the One Piece anime to remove filler content and improve pacing. Ace-Pace automates the process of: + +- Identifying which One-Pace episodes are already in the user's local library +- Detecting missing episodes +- Automatically downloading missing episodes via BitTorrent clients +- Renaming local files to match official One-Pace naming conventions +- Maintaining a database of episode metadata and file checksums + +## Core Functionality + +### 1. Episode Discovery and Indexing +- Scrapes Nyaa.si torrent tracker for One-Pace episodes +- Extracts CRC32 checksums from episode filenames or torrent file lists +- **Quality Filtering**: Only extracts episodes with 1080p quality, or 720p as fallback + - Episodes without quality markers are excluded + - Episodes with quality lower than 720p (480p, 360p, etc.) are excluded + - Episodes with quality higher than 1080p (1440p, 2160p/4K, etc.) are excluded +- Builds and maintains an episodes index database (`episodes_index.db`) +- Supports both single-file and multi-file torrent structures +- Handles pagination to fetch all available episodes + +### 2. Local Library Management +- Scans local directories recursively for video files (`.mkv`, `.mp4`, `.avi`) +- Calculates CRC32 checksums for local video files +- Caches CRC32 values in `crc32_files.db` to avoid recalculating +- Tracks file paths and their corresponding checksums + +### 3. Missing Episode Detection +- Compares local CRC32 checksums against the episodes index +- Generates a CSV report (`Ace-Pace_Missing.csv`) listing missing episodes +- Includes title, page link, and magnet link for each missing episode +- Tracks new missing episodes since last export + +### 4. Automated Downloading +- Integrates with BitTorrent clients (Transmission, qBittorrent) +- Adds missing episodes to client via magnet links +- Supports custom download folders, tags, and categories +- Prevents duplicate torrent additions by checking existing torrents + +### 5. File Renaming +- Matches local files to episodes index by CRC32 +- Renames files to match official One-Pace naming conventions +- Sanitizes filenames to remove problematic characters +- Updates database with new file paths after renaming + +## Technical Architecture + +### Databases + +#### `crc32_files.db` +- **Table: `crc32_cache`** + - `file_path` (TEXT, PRIMARY KEY): Full path to local video file + - `crc32` (TEXT, UNIQUE): CRC32 checksum of the file +- **Table: `metadata`** + - `key` (TEXT, PRIMARY KEY): Metadata key + - `value` (TEXT): Metadata value + - Stores: `last_folder`, `last_run`, `last_checked_page`, `last_db_export`, `last_missing_export` + +#### `episodes_index.db` +- **Table: `episodes_index`** + - `crc32` (TEXT, PRIMARY KEY): CRC32 checksum from episode + - `title` (TEXT): Episode title/filename + - `page_link` (TEXT): URL to Nyaa.si torrent page +- **Table: `metadata`** + - `key` (TEXT, PRIMARY KEY): Metadata key + - `value` (TEXT): Metadata value + - Stores: `episodes_db_last_update` + +### Key Algorithms + +#### CRC32 Calculation +- Reads video files in 8KB chunks +- Uses Python's `zlib.crc32()` for incremental calculation +- Formats result as uppercase 8-character hexadecimal string +- Caches results to avoid redundant calculations + +#### CRC32 Extraction from Filenames +- Uses regex pattern: `\[([A-Fa-f0-9]{8})\]` +- Extracts CRC32 from square brackets in filenames +- Takes the last match if multiple CRC32s are present +- Validates that filename contains "[One Pace]" marker + +#### Web Scraping +- Uses BeautifulSoup4 for HTML parsing +- Handles Nyaa.si pagination by detecting max page number +- Extracts torrent metadata from listing pages +- Falls back to individual torrent pages when CRC32 not in title +- Processes both folder-based and single-file torrent structures + +### File Structure + +``` +Ace-Pace/ +├── acepace.py # Main application entry point +├── clients.py # BitTorrent client abstraction layer +├── requirements.txt # Python dependencies +├── docker-compose.yml # Docker configuration (if applicable) +├── spec.md # This specification document +├── crc32_files.db # Local file checksum database (generated) +├── episodes_index.db # Episodes metadata database (generated) +├── Ace-Pace_Missing.csv # Missing episodes report (generated) +└── Ace-Pace_DB.csv # Database export (generated) +``` + +## Dependencies + +### Python Packages +- `requests`: HTTP requests for web scraping and API calls +- `beautifulsoup4`: HTML parsing for Nyaa.si scraping +- `qbittorrent-api`: qBittorrent client integration +- Standard library: `sqlite3`, `argparse`, `csv`, `datetime`, `os`, `re`, `zlib`, `getpass`, `time`, `abc` + +### External Services +- **Nyaa.si**: Torrent tracker for One-Pace episodes + - Base URL: `https://nyaa.si` + - Search endpoint: `/?f=0&c=0_0&q=one+pace&o=asc` + - Supports pagination via `&p=` +- **BitTorrent Clients**: + - Transmission (RPC API on port 9091 by default) + - qBittorrent (Web API on port 8080 by default) + +## Command-Line Interface + +### Main Arguments +- `--folder `: Local video library directory +- `--url `: Nyaa.si search URL (default: One-Pace 1080p search) +- `--db`: Export database to CSV + +### Download Arguments +- `--client {transmission,qbittorrent}`: BitTorrent client to use +- `--download`: Enable automatic downloading +- `--host `: Client host (default: localhost) +- `--port `: Client port +- `--username `: Client authentication username +- `--password `: Client authentication password +- `--download-folder `: Target download directory +- `--tag `: Tag(s) for qBittorrent (repeatable) +- `--category `: Category for qBittorrent + +### Utility Arguments +- `--rename`: Rename local files based on episodes index +- `--episodes_update`: Update episodes metadata database from Nyaa + +## Workflow + +### Standard Workflow +1. User runs script with `--folder` to scan local library +2. Script calculates/retrieves CRC32 checksums for local files +3. Script fetches episode list from Nyaa.si (or uses cached index) +4. Script compares local CRC32s against episode index +5. Script generates `Ace-Pace_Missing.csv` with missing episodes +6. User optionally runs `--download` to add missing episodes to BitTorrent client + +### Episodes Index Update Workflow +1. User runs `--episodes_update` to refresh episodes database +2. Script scrapes all pages of Nyaa.si One-Pace search results +3. For each torrent, extracts CRC32 from title or file list +4. Stores CRC32, title, and page link in `episodes_index.db` +5. Updates metadata with last update timestamp + +### File Renaming Workflow +1. User runs `--rename` with `--folder` +2. Script prompts to update episodes index if outdated +3. Script loads CRC32-to-title mapping from episodes index +4. Script matches local files by CRC32 +5. Script generates rename plan and prompts for confirmation +6. Script renames files and updates database + +## Integration Points + +### BitTorrent Client Abstraction +- Abstract base class `Client` defines interface +- Concrete implementations: `QBittorrentClient`, `TransmissionClient` +- Factory function `get_client()` instantiates appropriate client +- Methods: `add_torrents(magnets, download_folder, tags, category)` + +### Database Management +- SQLite databases for persistence +- Connection management via context or explicit close +- Metadata storage for tracking state and timestamps +- Transaction support for atomic operations + +## Error Handling + +### Network Errors +- HTTP request failures are caught and logged +- Continues processing remaining items on individual failures +- Rate limiting via `time.sleep(0.2)` between requests + +### File System Errors +- Checks for file existence before operations +- Handles permission errors gracefully +- Validates file paths and extensions + +### Database Errors +- Uses `INSERT OR REPLACE` for idempotent operations +- Handles connection failures +- Validates data before insertion + +## Configuration and State + +### Persistent State +- Last used folder path +- Last run timestamp +- Last database export timestamp +- Last missing export timestamp +- Last episodes index update timestamp +- Last checked page number + +### User Prompts +- Folder selection (with last folder suggestion) +- Episodes index update confirmation +- File renaming confirmation +- BitTorrent client selection (legacy prompt) + +## Future Considerations + +### Potential Enhancements +- Support for additional BitTorrent clients +- Configuration file for default settings +- Web UI for easier interaction +- Automatic episode index updates on schedule +- Support for additional video formats +- Integration with media server APIs (Plex, Jellyfin) +- Episode quality filtering (720p, 1080p, etc.) +- Duplicate detection and cleanup +- Episode metadata enrichment (thumbnails, descriptions) + +### Technical Improvements +- Async/await for concurrent web scraping +- Better error recovery and retry logic +- Unit tests and integration tests +- Logging framework instead of print statements +- Type hints for better code documentation +- Configuration validation +- Better handling of edge cases in filename parsing + +## Development Guidelines + +### Code Style +- Follow PEP 8 Python style guide +- Use descriptive variable names +- Add docstrings for functions +- Keep functions focused and single-purpose + +### Database Schema +- Use SQLite for simplicity +- Maintain backward compatibility when possible +- Document schema changes + +### API Compatibility +- Maintain backward compatibility with command-line arguments +- Handle missing optional arguments gracefully +- Provide clear error messages + +## Notes for AI Agents + +When working on this project: + +1. **CRC32 is the primary identifier** - All episode matching relies on CRC32 checksums +2. **Nyaa.si structure** - Understand the HTML structure of Nyaa.si pages for scraping +3. **Database state** - Always consider existing database state when making changes +4. **File paths** - Handle both absolute and relative paths correctly +5. **User interaction** - Some operations require user confirmation (renaming, downloads) +6. **Client abstraction** - New BitTorrent clients should implement the `Client` interface +7. **Error tolerance** - The tool should continue processing even if individual items fail +8. **Performance** - CRC32 calculation can be slow; caching is essential +9. **Web scraping** - Be respectful with rate limiting and error handling +10. **File naming** - Sanitize filenames to be filesystem-safe across platforms diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..2a4393d --- /dev/null +++ b/tests/README.md @@ -0,0 +1,100 @@ +# Ace-Pace Test Suite + +This directory contains unit tests for the Ace-Pace application. + +## Test Structure + +- `test_database.py` - Tests for database initialization and metadata operations +- `test_crc32.py` - Tests for CRC32 calculation and extraction +- `test_episodes.py` - Tests for episode metadata fetching from Nyaa +- `test_file_operations.py` - Tests for file operations and renaming +- `test_clients.py` - Tests for BitTorrent client integrations +- `test_missing_detection.py` - Tests for missing episode detection logic +- `conftest.py` - Shared fixtures and test utilities + +## Running Tests + +To run all tests: + +```bash +pytest tests/ +``` + +To run a specific test file: + +```bash +pytest tests/test_database.py +``` + +To run a specific test: + +```bash +pytest tests/test_database.py::TestDatabaseInitialization::test_init_db_creates_tables +``` + +To run with verbose output: + +```bash +pytest tests/ -v +``` + +To run with coverage: + +```bash +pytest tests/ --cov=acepace --cov=clients +``` + +## Test Coverage + +The test suite covers: + +1. **Database Operations** + - Database initialization + - Metadata get/set operations + - Episodes index operations + +2. **CRC32 Operations** + - CRC32 extraction from filenames + - CRC32 calculation from file content + - Caching of CRC32 values + +3. **Episode Metadata** + - Fetching episodes from Nyaa + - Handling pagination + - Extracting CRC32 from titles and file lists + - Updating episodes index database + +4. **File Operations** + - File renaming based on CRC32 matching + - Filename sanitization + - CSV export functionality + +5. **BitTorrent Clients** + - qBittorrent client initialization and operations + - Transmission client initialization and operations + - Client factory function + - Error handling + +6. **Missing Episode Detection** + - Comparing local and remote CRC32s + - Generating missing episode lists + - Fetching CRC32 links from Nyaa + +## Mocking + +Tests use mocking for: +- Network requests (requests.get) +- File system operations +- BitTorrent client APIs +- User input prompts +- Time delays + +## Fixtures + +Common fixtures are defined in `conftest.py`: +- `temp_dir` - Temporary directory for test files +- `temp_db_path` - Temporary database path +- `sample_video_content` - Sample video content for testing +- `sample_crc32` - Sample CRC32 value +- `sample_episode_data` - Sample episode data +- `mock_nyaa_html_*` - Mock HTML responses from Nyaa diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..ab0593c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# Tests package for Ace-Pace diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..c5410cb --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,146 @@ +"""Shared fixtures for Ace-Pace tests.""" +import pytest +import sqlite3 +import os +import tempfile +import shutil +from pathlib import Path + + +@pytest.fixture +def temp_dir(): + """Create a temporary directory for testing.""" + temp_path = tempfile.mkdtemp() + yield temp_path + shutil.rmtree(temp_path) + + +@pytest.fixture +def temp_db_path(temp_dir): + """Create a temporary database path.""" + return os.path.join(temp_dir, "test_crc32_files.db") + + +@pytest.fixture +def temp_episodes_db_path(temp_dir): + """Create a temporary episodes database path.""" + return os.path.join(temp_dir, "test_episodes_index.db") + + +@pytest.fixture +def sample_video_content(): + """Sample video file content for CRC32 testing.""" + return b"This is sample video content for testing CRC32 calculation" * 100 + + +@pytest.fixture +def sample_crc32(): + """Sample CRC32 value for testing.""" + return "A1B2C3D4" + + +@pytest.fixture +def sample_episode_data(): + """Sample episode data for testing.""" + return [ + ("A1B2C3D4", "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv", "https://nyaa.si/view/12345"), + ("E5F6G7H8", "[One Pace] Episode 2 [1080p][E5F6G7H8].mkv", "https://nyaa.si/view/12346"), + ("I9J0K1L2", "[One Pace] Episode 3 [1080p][I9J0K1L2].mkv", "https://nyaa.si/view/12347"), + ] + + +@pytest.fixture +def mock_nyaa_html_single_page(): + """Mock HTML for a single Nyaa.si page.""" + return """ + + + + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv + Magnet +
+ [One Pace] Episode 2 [1080p][E5F6G7H8].mkv + Magnet +
+
    +
  • 1
    • +
+ + + """ + + +@pytest.fixture +def mock_nyaa_html_multi_page(): + """Mock HTML for multi-page Nyaa.si results.""" + return """ + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv +
+
    +
  • 1
    • +
  • 2
    • +
  • 3
    • +
+ + + """ + + +@pytest.fixture +def mock_nyaa_torrent_page(): + """Mock HTML for a single torrent page with file list.""" + return """ + + +
+
    +
  • [One Pace] Episode 1 [1080p][A1B2C3D4].mkv
    • +
+
+ + + """ + + +@pytest.fixture +def mock_nyaa_torrent_page_folder(): + """Mock HTML for a torrent page with folder structure.""" + return """ + + +
+ One Pace +
    +
  • +
      +
    • [One Pace] Episode 1 [1080p][A1B2C3D4].mkv
      • +
    +
    • +
+
+ + + """ + + +@pytest.fixture +def sample_magnet_links(): + """Sample magnet links for testing.""" + return [ + "magnet:?xt=urn:btih:1234567890abcdef1234567890abcdef12345678&dn=test", + "magnet:?xt=urn:btih:abcdef1234567890abcdef1234567890abcdef12&dn=test2", + ] diff --git a/tests/test_clients.py b/tests/test_clients.py new file mode 100644 index 0000000..f5ea4ab --- /dev/null +++ b/tests/test_clients.py @@ -0,0 +1,201 @@ +"""Unit tests for BitTorrent client operations.""" +import pytest +import sys +import os +from unittest.mock import patch, MagicMock, Mock + +# Add parent directory to path to import clients +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from clients import QBittorrentClient, TransmissionClient, get_client + + +class TestQBittorrentClient: + """Tests for qBittorrent client.""" + + @patch('clients.qbittorrentapi.Client') + def test_qbittorrent_client_init_success(self, mock_client_class): + """Test successful qBittorrent client initialization.""" + mock_client = MagicMock() + mock_client.auth_log_in.return_value = None + mock_client_class.return_value = mock_client + + client = QBittorrentClient("localhost", 8080, "user", "pass") + + assert client.client == mock_client + mock_client.auth_log_in.assert_called_once() + + @patch('clients.qbittorrentapi.Client') + def test_qbittorrent_client_init_login_failed(self, mock_client_class): + """Test qBittorrent client initialization with login failure.""" + import qbittorrentapi + mock_client = MagicMock() + mock_client.auth_log_in.side_effect = qbittorrentapi.LoginFailed("Invalid credentials") + mock_client_class.return_value = mock_client + + with pytest.raises(Exception) as exc_info: + QBittorrentClient("localhost", 8080, "user", "pass") + + assert "Failed to connect to qBittorrent" in str(exc_info.value) + + @patch('clients.qbittorrentapi.Client') + @patch('clients.time.sleep') + def test_qbittorrent_add_torrents(self, mock_sleep, mock_client_class, sample_magnet_links): + """Test adding torrents to qBittorrent.""" + mock_client = MagicMock() + mock_client.auth_log_in.return_value = None + mock_client.torrents_info.return_value = [] # No existing torrents + mock_client.torrents_add.return_value = None + mock_client_class.return_value = mock_client + + client = QBittorrentClient("localhost", 8080, "user", "pass") + client.add_torrents(sample_magnet_links, download_folder="/downloads", tags=["test"], category="anime") + + assert mock_client.torrents_add.call_count == 2 + mock_client.torrents_create_tags.assert_called_once() + + @patch('clients.qbittorrentapi.Client') + @patch('clients.time.sleep') + def test_qbittorrent_add_torrents_duplicate(self, mock_sleep, mock_client_class, sample_magnet_links): + """Test adding duplicate torrents to qBittorrent.""" + mock_client = MagicMock() + mock_client.auth_log_in.return_value = None + # First torrent exists, second doesn't + mock_client.torrents_info.side_effect = [ + [{"hash": "1234567890abcdef1234567890abcdef12345678"}], # First exists + [] # Second doesn't + ] + mock_client.torrents_add.return_value = None + mock_client_class.return_value = mock_client + + client = QBittorrentClient("localhost", 8080, "user", "pass") + client.add_torrents(sample_magnet_links, tags=["test"]) + + # Should only add the second torrent + assert mock_client.torrents_add.call_count == 1 + # Should add tags to existing torrent + assert mock_client.torrents_add_tags.call_count == 1 + + @patch('clients.qbittorrentapi.Client') + @patch('clients.time.sleep') + def test_qbittorrent_add_torrents_invalid_magnet(self, mock_sleep, mock_client_class): + """Test handling invalid magnet links.""" + mock_client = MagicMock() + mock_client.auth_log_in.return_value = None + mock_client_class.return_value = mock_client + + client = QBittorrentClient("localhost", 8080, "user", "pass") + invalid_magnets = ["invalid_magnet_link"] + + client.add_torrents(invalid_magnets) + + # Should not call torrents_add for invalid magnet + mock_client.torrents_add.assert_not_called() + + +class TestTransmissionClient: + """Tests for Transmission client.""" + + @patch('clients.requests.Session') + def test_transmission_client_init_success(self, mock_session_class): + """Test successful Transmission client initialization.""" + mock_session = MagicMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": "success"} + mock_session.post.return_value = mock_response + mock_session_class.return_value = mock_session + + client = TransmissionClient("localhost", 9091, "user", "pass") + + assert client.session == mock_session + mock_session.post.assert_called() + + @patch('clients.requests.Session') + def test_transmission_client_init_409_retry(self, mock_session_class): + """Test Transmission client handles 409 status code (session ID).""" + mock_session = MagicMock() + + # First call returns 409 with session ID + mock_response_409 = MagicMock() + mock_response_409.status_code = 409 + mock_response_409.headers = {"X-Transmission-Session-Id": "test_session_id"} + + # Second call succeeds + mock_response_200 = MagicMock() + mock_response_200.status_code = 200 + mock_response_200.json.return_value = {"result": "success"} + + mock_session.post.side_effect = [mock_response_409, mock_response_200] + mock_session_class.return_value = mock_session + + client = TransmissionClient("localhost", 9091, None, None) + + assert client.session_id == "test_session_id" + + @patch('clients.requests.Session') + @patch('clients.time.sleep') + def test_transmission_add_torrents(self, mock_sleep, mock_session_class, sample_magnet_links): + """Test adding torrents to Transmission.""" + mock_session = MagicMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": "success"} + mock_session.post.return_value = mock_response + mock_session_class.return_value = mock_session + + client = TransmissionClient("localhost", 9091, None, None) + client.session_id = "test_session_id" + client.add_torrents(sample_magnet_links, download_folder="/downloads") + + assert mock_session.post.call_count >= len(sample_magnet_links) + + @patch('clients.requests.Session') + @patch('clients.time.sleep') + def test_transmission_add_torrents_handles_409(self, mock_sleep, mock_session_class, sample_magnet_links): + """Test Transmission handles 409 during torrent add.""" + mock_session = MagicMock() + + # First call returns 409, second succeeds + mock_response_409 = MagicMock() + mock_response_409.status_code = 409 + mock_response_409.headers = {"X-Transmission-Session-Id": "new_session_id"} + + mock_response_200 = MagicMock() + mock_response_200.status_code = 200 + mock_response_200.json.return_value = {"result": "success"} + + mock_session.post.side_effect = [mock_response_409, mock_response_200] + mock_session_class.return_value = mock_session + + client = TransmissionClient("localhost", 9091, None, None) + client.session_id = "old_session_id" + client.add_torrents(sample_magnet_links[:1]) # Just one to simplify + + assert client.session_id == "new_session_id" + + +class TestClientFactory: + """Tests for client factory function.""" + + def test_get_client_qbittorrent(self): + """Test getting qBittorrent client.""" + with patch('clients.QBittorrentClient') as mock_class: + mock_instance = MagicMock() + mock_class.return_value = mock_instance + client = get_client("qbittorrent", "localhost", 8080, "user", "pass") + assert client == mock_instance + + def test_get_client_transmission(self): + """Test getting Transmission client.""" + with patch('clients.TransmissionClient') as mock_class: + mock_instance = MagicMock() + mock_class.return_value = mock_instance + client = get_client("transmission", "localhost", 9091, "user", "pass") + assert client == mock_instance + + def test_get_client_unknown(self): + """Test getting unknown client raises error.""" + with pytest.raises(ValueError) as exc_info: + get_client("unknown", "localhost", 8080, "user", "pass") + assert "Unknown client" in str(exc_info.value) diff --git a/tests/test_crc32.py b/tests/test_crc32.py new file mode 100644 index 0000000..fc24f0c --- /dev/null +++ b/tests/test_crc32.py @@ -0,0 +1,154 @@ +"""Unit tests for CRC32 operations.""" +import pytest +import zlib +import os +import sys +import tempfile +from unittest.mock import patch, mock_open + +# Add parent directory to path to import acepace +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import acepace + + +class TestCRC32Extraction: + """Tests for CRC32 extraction from filenames.""" + + def test_extract_crc32_from_filename(self): + """Test extracting CRC32 from filename with brackets.""" + filename = "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv" + matches = acepace.CRC32_REGEX.findall(filename) + assert len(matches) > 0 + assert matches[-1].upper() == "A1B2C3D4" + + def test_extract_crc32_multiple_matches(self): + """Test extracting CRC32 when multiple matches exist (takes last).""" + filename = "[One Pace] Episode 1 [1080p][A1B2C3D4][E5F6G7H8].mkv" + matches = acepace.CRC32_REGEX.findall(filename) + assert len(matches) == 2 + # Should take the last match + assert matches[-1].upper() == "E5F6G7H8" + + def test_extract_crc32_no_match(self): + """Test extracting CRC32 from filename without CRC32.""" + filename = "[One Pace] Episode 1 [1080p].mkv" + matches = acepace.CRC32_REGEX.findall(filename) + assert len(matches) == 0 + + def test_extract_crc32_lowercase(self): + """Test extracting CRC32 with lowercase hex.""" + filename = "[One Pace] Episode 1 [1080p][a1b2c3d4].mkv" + matches = acepace.CRC32_REGEX.findall(filename) + assert len(matches) > 0 + assert matches[-1].upper() == "A1B2C3D4" + + def test_extract_crc32_invalid_length(self): + """Test that regex doesn't match invalid CRC32 lengths.""" + filename = "[One Pace] Episode 1 [1080p][A1B2C3].mkv" # Too short + matches = acepace.CRC32_REGEX.findall(filename) + assert len(matches) == 0 + + +class TestCRC32Calculation: + """Tests for CRC32 calculation from file content.""" + + def test_calculate_crc32_from_content(self, sample_video_content, temp_dir): + """Test calculating CRC32 from file content.""" + test_file = os.path.join(temp_dir, "test_video.mkv") + with open(test_file, "wb") as f: + f.write(sample_video_content) + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + crc32s = acepace.calculate_local_crc32(temp_dir, conn) + conn.close() + + # Calculate expected CRC32 + expected_crc = 0 + for chunk in [sample_video_content[i:i+8192] for i in range(0, len(sample_video_content), 8192)]: + expected_crc = zlib.crc32(chunk, expected_crc) + expected_crc32 = f"{expected_crc & 0xFFFFFFFF:08X}" + + assert len(crc32s) == 1 + assert expected_crc32 in crc32s + + def test_calculate_crc32_caches_result(self, sample_video_content, temp_dir): + """Test that CRC32 calculation caches results in database.""" + test_file = os.path.join(temp_dir, "test_video.mkv") + with open(test_file, "wb") as f: + f.write(sample_video_content) + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + + # First calculation + crc32s1 = acepace.calculate_local_crc32(temp_dir, conn) + + # Second calculation should use cache + crc32s2 = acepace.calculate_local_crc32(temp_dir, conn) + + # Verify cache was used (check database) + cursor = conn.cursor() + cursor.execute("SELECT crc32 FROM crc32_cache WHERE file_path = ?", (test_file,)) + row = cursor.fetchone() + assert row is not None + + assert crc32s1 == crc32s2 + conn.close() + + def test_calculate_crc32_only_video_files(self, temp_dir): + """Test that only video files are processed.""" + # Create video file + video_file = os.path.join(temp_dir, "test.mkv") + with open(video_file, "wb") as f: + f.write(b"video content") + + # Create non-video file + text_file = os.path.join(temp_dir, "test.txt") + with open(text_file, "w") as f: + f.write("text content") + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + crc32s = acepace.calculate_local_crc32(temp_dir, conn) + conn.close() + + # Should only have one CRC32 (for the video file) + assert len(crc32s) == 1 + + def test_calculate_crc32_multiple_video_formats(self, temp_dir): + """Test that multiple video formats are supported.""" + files = [ + ("test1.mkv", b"mkv content"), + ("test2.mp4", b"mp4 content"), + ("test3.avi", b"avi content"), + ] + + for filename, content in files: + filepath = os.path.join(temp_dir, filename) + with open(filepath, "wb") as f: + f.write(content) + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + crc32s = acepace.calculate_local_crc32(temp_dir, conn) + conn.close() + + assert len(crc32s) == 3 + + def test_calculate_crc32_subdirectories(self, temp_dir): + """Test that CRC32 calculation works in subdirectories.""" + subdir = os.path.join(temp_dir, "subdir") + os.makedirs(subdir) + + video_file = os.path.join(subdir, "test.mkv") + with open(video_file, "wb") as f: + f.write(b"video content") + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + crc32s = acepace.calculate_local_crc32(temp_dir, conn) + conn.close() + + assert len(crc32s) == 1 diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..1ea7a93 --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,132 @@ +"""Unit tests for database operations.""" +import pytest +import sqlite3 +import os +import sys +from unittest.mock import patch, MagicMock + +# Add parent directory to path to import acepace +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import acepace + + +class TestDatabaseInitialization: + """Tests for database initialization functions.""" + + @patch('acepace.DB_NAME', 'test_crc32_files.db') + def test_init_db_creates_tables(self, temp_dir, monkeypatch): + """Test that init_db creates required tables.""" + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test_crc32_files.db')): + conn = acepace.init_db() + cursor = conn.cursor() + + # Check crc32_cache table exists + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='crc32_cache'") + assert cursor.fetchone() is not None + + # Check metadata table exists + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='metadata'") + assert cursor.fetchone() is not None + + conn.close() + os.remove(os.path.join(temp_dir, 'test_crc32_files.db')) + + @patch('acepace.EPISODES_DB_NAME', 'test_episodes_index.db') + def test_init_episodes_db_creates_tables(self, temp_dir, monkeypatch): + """Test that init_episodes_db creates required tables.""" + with patch('acepace.EPISODES_DB_NAME', os.path.join(temp_dir, 'test_episodes_index.db')): + conn = acepace.init_episodes_db() + cursor = conn.cursor() + + # Check episodes_index table exists + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='episodes_index'") + assert cursor.fetchone() is not None + + # Check metadata table exists + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='metadata'") + assert cursor.fetchone() is not None + + conn.close() + os.remove(os.path.join(temp_dir, 'test_episodes_index.db')) + + +class TestMetadataOperations: + """Tests for metadata get/set operations.""" + + def test_get_metadata_nonexistent_key(self, temp_dir): + """Test getting metadata for non-existent key returns None.""" + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + result = acepace.get_metadata(conn, "nonexistent_key") + assert result is None + conn.close() + os.remove(os.path.join(temp_dir, 'test.db')) + + def test_set_and_get_metadata(self, temp_dir): + """Test setting and getting metadata.""" + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + acepace.set_metadata(conn, "test_key", "test_value") + result = acepace.get_metadata(conn, "test_key") + assert result == "test_value" + conn.close() + os.remove(os.path.join(temp_dir, 'test.db')) + + def test_set_metadata_overwrites_existing(self, temp_dir): + """Test that set_metadata overwrites existing values.""" + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + acepace.set_metadata(conn, "test_key", "old_value") + acepace.set_metadata(conn, "test_key", "new_value") + result = acepace.get_metadata(conn, "test_key") + assert result == "new_value" + conn.close() + os.remove(os.path.join(temp_dir, 'test.db')) + + def test_get_episodes_metadata_nonexistent_key(self, temp_dir): + """Test getting episodes metadata for non-existent key returns None.""" + with patch('acepace.EPISODES_DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_episodes_db() + result = acepace.get_episodes_metadata(conn, "nonexistent_key") + assert result is None + conn.close() + os.remove(os.path.join(temp_dir, 'test.db')) + + def test_set_and_get_episodes_metadata(self, temp_dir): + """Test setting and getting episodes metadata.""" + with patch('acepace.EPISODES_DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_episodes_db() + acepace.set_episodes_metadata(conn, "test_key", "test_value") + result = acepace.get_episodes_metadata(conn, "test_key") + assert result == "test_value" + conn.close() + os.remove(os.path.join(temp_dir, 'test.db')) + + +class TestEpisodesIndexOperations: + """Tests for episodes index database operations.""" + + def test_load_crc32_to_title_from_index(self, temp_dir, sample_episode_data): + """Test loading CRC32 to title mapping from episodes index.""" + with patch('acepace.EPISODES_DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_episodes_db() + cursor = conn.cursor() + + # Insert sample data + for crc32, title, page_link in sample_episode_data: + cursor.execute( + "INSERT INTO episodes_index (crc32, title, page_link) VALUES (?, ?, ?)", + (crc32, title, page_link) + ) + conn.commit() + conn.close() + + # Load and verify + mapping = acepace.load_crc32_to_title_from_index() + assert len(mapping) == 3 + assert mapping["A1B2C3D4"] == "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv" + assert mapping["E5F6G7H8"] == "[One Pace] Episode 2 [1080p][E5F6G7H8].mkv" + assert mapping["I9J0K1L2"] == "[One Pace] Episode 3 [1080p][I9J0K1L2].mkv" + + os.remove(os.path.join(temp_dir, 'test.db')) diff --git a/tests/test_episodes.py b/tests/test_episodes.py new file mode 100644 index 0000000..fc70ddd --- /dev/null +++ b/tests/test_episodes.py @@ -0,0 +1,661 @@ +"""Unit tests for episode metadata fetching.""" +import pytest +import os +import sys +from unittest.mock import patch, MagicMock, Mock +from bs4 import BeautifulSoup + +# Add parent directory to path to import acepace +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import acepace + + +class TestEpisodeMetadataFetching: + """Tests for fetching episode metadata from Nyaa.""" + + @patch('acepace.requests.get') + def test_fetch_episodes_metadata_single_page(self, mock_get, mock_nyaa_html_single_page): + """Test fetching episodes from a single page.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = mock_nyaa_html_single_page + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + assert len(episodes) == 2 + assert any(ep[0] == "A1B2C3D4" for ep in episodes) + assert any(ep[0] == "E5F6G7H8" for ep in episodes) + + @patch('acepace.requests.get') + @patch('acepace.time.sleep') # Mock sleep to speed up tests + def test_fetch_episodes_metadata_multi_page(self, mock_sleep, mock_get, mock_nyaa_html_multi_page): + """Test fetching episodes from multiple pages.""" + # First page response + mock_response1 = MagicMock() + mock_response1.status_code = 200 + mock_response1.text = mock_nyaa_html_multi_page + + # Second page response + mock_response2 = MagicMock() + mock_response2.status_code = 200 + mock_response2.text = """ + + + + + + +
+ [One Pace] Episode 3 [1080p][I9J0K1L2].mkv +
+ + + """ + + # Third page response (empty) + mock_response3 = MagicMock() + mock_response3.status_code = 200 + mock_response3.text = "
" + + mock_get.side_effect = [mock_response1, mock_response2, mock_response3] + + episodes = acepace.fetch_episodes_metadata() + + # Should have episodes from both pages + assert len(episodes) >= 2 + assert mock_get.call_count >= 2 + + @patch('acepace.requests.get') + def test_fetch_episodes_metadata_crc32_in_title(self, mock_get): + """Test extracting CRC32 from title directly.""" + html = """ + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv +
+
    +
  • 1
    • +
+ + + """ + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + assert len(episodes) == 1 + assert episodes[0][0] == "A1B2C3D4" + assert "[One Pace]" in episodes[0][1] + + @patch('acepace.requests.get') + def test_fetch_episodes_metadata_crc32_from_file_list(self, mock_get, mock_nyaa_torrent_page): + """Test extracting CRC32 from torrent page file list.""" + # Listing page without CRC32 in title + listing_html = """ + + + + + + +
+ [One Pace] Episode 1 +
+
    +
  • 1
    • +
+ + + """ + + # Torrent page with file list + mock_listing_response = MagicMock() + mock_listing_response.status_code = 200 + mock_listing_response.text = listing_html + + mock_torrent_response = MagicMock() + mock_torrent_response.status_code = 200 + mock_torrent_response.text = mock_nyaa_torrent_page + + mock_get.side_effect = [mock_listing_response, mock_torrent_response] + + episodes = acepace.fetch_episodes_metadata() + + assert len(episodes) == 1 + assert episodes[0][0] == "A1B2C3D4" + + @patch('acepace.requests.get') + def test_fetch_episodes_metadata_handles_http_error(self, mock_get): + """Test that HTTP errors are handled gracefully.""" + mock_response = MagicMock() + mock_response.status_code = 404 + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + assert len(episodes) == 0 + + @patch('acepace.requests.get') + def test_fetch_episodes_metadata_deduplicates_crc32(self, mock_get): + """Test that duplicate CRC32s are not added multiple times.""" + html = """ + + + + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv +
+ [One Pace] Episode 1 Alt [1080p][A1B2C3D4].mkv +
+
    +
  • 1
    • +
+ + + """ + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + # Should only have one entry despite duplicate CRC32 + crc32s = [ep[0] for ep in episodes] + assert crc32s.count("A1B2C3D4") == 1 + + +class TestUpdateEpisodesIndex: + """Tests for updating episodes index database.""" + + @patch('acepace.fetch_episodes_metadata') + @patch('acepace.EPISODES_DB_NAME', 'test_episodes_index.db') + def test_update_episodes_index_db(self, mock_fetch, temp_dir, sample_episode_data): + """Test updating episodes index database.""" + with patch('acepace.EPISODES_DB_NAME', os.path.join(temp_dir, 'test.db')): + mock_fetch.return_value = sample_episode_data + + acepace.update_episodes_index_db() + + # Verify data was inserted + conn = acepace.init_episodes_db() + cursor = conn.cursor() + cursor.execute("SELECT COUNT(*) FROM episodes_index") + count = cursor.fetchone()[0] + assert count == 3 + + # Verify metadata was updated + last_update = acepace.get_episodes_metadata(conn, "episodes_db_last_update") + assert last_update is not None + + conn.close() + os.remove(os.path.join(temp_dir, 'test.db')) + + +class TestEpisodeQualityFiltering: + """Tests for ensuring only 1080p (or 720p fallback) episodes are extracted.""" + + @patch('acepace.requests.get') + def test_fetch_episodes_prefers_1080p(self, mock_get): + """Test that 1080p episodes are extracted when available.""" + html = """ + + + + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv +
+ [One Pace] Episode 2 [1080p][E5F6G7H8].mkv +
+
    +
  • 1
    • +
+ + + """ + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + # All episodes should be 1080p + assert len(episodes) == 2 + for crc32, title, _ in episodes: + assert "[1080p]" in title.upper() or "1080P" in title.upper() + assert "[One Pace]" in title + + @patch('acepace.requests.get') + def test_fetch_episodes_accepts_720p_as_fallback(self, mock_get): + """Test that 720p episodes are accepted when 1080p is not available.""" + html = """ + + + + + + + + + +
+ [One Pace] Episode 1 [720p][A1B2C3D4].mkv +
+ [One Pace] Episode 2 [720p][E5F6G7H8].mkv +
+
    +
  • 1
    • +
+ + + """ + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + # All episodes should be 720p + assert len(episodes) == 2 + for crc32, title, _ in episodes: + assert "[720p]" in title.upper() or "720P" in title.upper() + assert "[One Pace]" in title + + @patch('acepace.requests.get') + def test_fetch_episodes_excludes_lower_quality(self, mock_get): + """Test that episodes with quality lower than 720p are excluded.""" + html = """ + + + + + + + + + + + + +
+ [One Pace] Episode 1 [480p][A1B2C3D4].mkv +
+ [One Pace] Episode 2 [360p][E5F6G7H8].mkv +
+ [One Pace] Episode 3 [240p][I9J0K1L2].mkv +
+
    +
  • 1
    • +
+ + + """ + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + # Lower quality episodes should be excluded + assert len(episodes) == 0 + + @patch('acepace.requests.get') + def test_fetch_episodes_prefers_1080p_over_720p_same_episode(self, mock_get): + """Test that when both 1080p and 720p versions exist for same episode, 1080p is preferred.""" + html = """ + + + + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv +
+ [One Pace] Episode 1 [720p][A1B2C3D4].mkv +
+
    +
  • 1
    • +
+ + + """ + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + # Should only have one entry (deduplicated by CRC32) + # But we need to verify it's the 1080p version + assert len(episodes) == 1 + crc32, title, _ = episodes[0] + assert crc32 == "A1B2C3D4" + # The first one encountered should be kept (1080p in this case) + # Since CRC32 deduplication happens, we need to check which one was kept + assert "[1080p]" in title.upper() or "1080P" in title.upper() + + @patch('acepace.requests.get') + def test_fetch_episodes_mixed_qualities_only_keeps_valid(self, mock_get): + """Test that mixed quality episodes only keeps 1080p and 720p.""" + html = """ + + + + + + + + + + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv +
+ [One Pace] Episode 2 [720p][E5F6G7H8].mkv +
+ [One Pace] Episode 3 [480p][I9J0K1L2].mkv +
+ [One Pace] Episode 4 [1080p][M3N4O5P6].mkv +
+
    +
  • 1
    • +
+ + + """ + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + # Should only have 1080p and 720p episodes (3 total, excluding 480p) + assert len(episodes) == 3 + for crc32, title, _ in episodes: + title_upper = title.upper() + has_1080p = "[1080P]" in title_upper or "1080P" in title_upper + has_720p = "[720P]" in title_upper or "720P" in title_upper + assert has_1080p or has_720p, f"Episode {title} should be 1080p or 720p" + # Verify no lower quality + assert "[480P]" not in title_upper + assert "[360P]" not in title_upper + assert "[240P]" not in title_upper + + @patch('acepace.requests.get') + def test_fetch_episodes_handles_case_insensitive_quality(self, mock_get): + """Test that quality detection is case-insensitive.""" + html = """ + + + + + + + + + +
+ [One Pace] Episode 1 [1080P][A1B2C3D4].mkv +
+ [One Pace] Episode 2 [720P][E5F6G7H8].mkv +
+
    +
  • 1
    • +
+ + + """ + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + # Should accept both uppercase and lowercase quality markers + assert len(episodes) == 2 + + @patch('acepace.requests.get') + def test_fetch_episodes_excludes_episodes_without_quality_marker(self, mock_get): + """Test that episodes without quality markers are excluded.""" + html = """ + + + + + + + + + +
+ [One Pace] Episode 1 [A1B2C3D4].mkv +
+ [One Pace] Episode 2 [1080p][E5F6G7H8].mkv +
+
    +
  • 1
    • +
+ + + """ + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + # Should only include episode with quality marker + assert len(episodes) == 1 + crc32, title, _ = episodes[0] + assert crc32 == "E5F6G7H8" + assert "[1080p]" in title.upper() or "1080P" in title.upper() + + @patch('acepace.requests.get') + @patch('acepace.time.sleep') + def test_fetch_episodes_quality_filtering_from_file_list(self, mock_sleep, mock_get): + """Test quality filtering when CRC32 is extracted from torrent file list.""" + # Listing page + listing_html = """ + + + + + + +
+ [One Pace] Episode 1 +
+
    +
  • 1
    • +
+ + + """ + + # Torrent page with 1080p file + torrent_html_1080p = """ + + +
+
    +
  • [One Pace] Episode 1 [1080p][A1B2C3D4].mkv
    • +
+
+ + + """ + + mock_listing_response = MagicMock() + mock_listing_response.status_code = 200 + mock_listing_response.text = listing_html + + mock_torrent_response = MagicMock() + mock_torrent_response.status_code = 200 + mock_torrent_response.text = torrent_html_1080p + + mock_get.side_effect = [mock_listing_response, mock_torrent_response] + + episodes = acepace.fetch_episodes_metadata() + + # Should extract 1080p episode from file list + assert len(episodes) == 1 + crc32, title, _ = episodes[0] + assert crc32 == "A1B2C3D4" + assert "[1080p]" in title.upper() or "1080P" in title.upper() + + @patch('acepace.requests.get') + @patch('acepace.time.sleep') + def test_fetch_episodes_quality_filtering_from_file_list_excludes_lower_quality(self, mock_sleep, mock_get): + """Test that lower quality episodes are excluded when extracted from file list.""" + # Listing page + listing_html = """ + + + + + + +
+ [One Pace] Episode 1 +
+
    +
  • 1
    • +
+ + + """ + + # Torrent page with 480p file (should be excluded) + torrent_html_480p = """ + + +
+
    +
  • [One Pace] Episode 1 [480p][A1B2C3D4].mkv
    • +
+
+ + + """ + + mock_listing_response = MagicMock() + mock_listing_response.status_code = 200 + mock_listing_response.text = listing_html + + mock_torrent_response = MagicMock() + mock_torrent_response.status_code = 200 + mock_torrent_response.text = torrent_html_480p + + mock_get.side_effect = [mock_listing_response, mock_torrent_response] + + episodes = acepace.fetch_episodes_metadata() + + # Should exclude 480p episode + assert len(episodes) == 0 + + +class TestQualityFilteringHelper: + """Tests for the quality filtering helper function.""" + + def test_quality_filtering_accepts_1080p(self): + """Test that 1080p quality is accepted.""" + # We need to test the internal _is_valid_quality function + # Since it's nested, we'll test it through fetch_episodes_metadata + # But we can also test the regex directly + from acepace import QUALITY_REGEX + + test_cases = [ + "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv", + "[One Pace] Episode 1 [1080P][A1B2C3D4].mkv", + "[One Pace] Episode 1 [1080P][A1B2C3D4].mkv", + ] + + for test_case in test_cases: + matches = QUALITY_REGEX.findall(test_case) + assert len(matches) > 0 + quality_num = int(matches[0].lower().replace('p', '')) + assert quality_num == 1080 + + def test_quality_filtering_accepts_720p(self): + """Test that 720p quality is accepted.""" + from acepace import QUALITY_REGEX + + test_cases = [ + "[One Pace] Episode 1 [720p][A1B2C3D4].mkv", + "[One Pace] Episode 1 [720P][A1B2C3D4].mkv", + ] + + for test_case in test_cases: + matches = QUALITY_REGEX.findall(test_case) + assert len(matches) > 0 + quality_num = int(matches[0].lower().replace('p', '')) + assert quality_num == 720 + + def test_quality_filtering_rejects_lower_quality(self): + """Test that qualities lower than 720p are rejected.""" + from acepace import QUALITY_REGEX + + test_cases = [ + "[One Pace] Episode 1 [480p][A1B2C3D4].mkv", + "[One Pace] Episode 1 [360p][A1B2C3D4].mkv", + "[One Pace] Episode 1 [240p][A1B2C3D4].mkv", + ] + + for test_case in test_cases: + matches = QUALITY_REGEX.findall(test_case) + assert len(matches) > 0 + quality_num = int(matches[0].lower().replace('p', '')) + assert quality_num < 720 + + def test_quality_filtering_rejects_higher_quality(self): + """Test that qualities higher than 1080p are rejected (4K, etc.).""" + from acepace import QUALITY_REGEX + + test_cases = [ + "[One Pace] Episode 1 [2160p][A1B2C3D4].mkv", # 4K + "[One Pace] Episode 1 [1440p][A1B2C3D4].mkv", # 1440p + ] + + for test_case in test_cases: + matches = QUALITY_REGEX.findall(test_case) + assert len(matches) > 0 + quality_num = int(matches[0].lower().replace('p', '')) + assert quality_num not in [720, 1080] diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py new file mode 100644 index 0000000..8cf7506 --- /dev/null +++ b/tests/test_file_operations.py @@ -0,0 +1,156 @@ +"""Unit tests for file operations and renaming.""" +import pytest +import os +import sys +import shutil +import re +from unittest.mock import patch, MagicMock, mock_open + +# Add parent directory to path to import acepace +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import acepace + + +class TestFileRenaming: + """Tests for file renaming functionality.""" + + def test_rename_local_files_matches_by_crc32(self, temp_dir, sample_episode_data): + """Test that files are renamed based on CRC32 matching.""" + # Create test video file + test_file = os.path.join(temp_dir, "old_name.mkv") + with open(test_file, "wb") as f: + f.write(b"test video content") + + # Calculate CRC32 for the file + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + crc32s = acepace.calculate_local_crc32(temp_dir, conn) + actual_crc32 = list(crc32s)[0] + + # Update episodes index with matching CRC32 + with patch('acepace.EPISODES_DB_NAME', os.path.join(temp_dir, 'episodes.db')): + episodes_conn = acepace.init_episodes_db() + cursor = episodes_conn.cursor() + cursor.execute( + "INSERT INTO episodes_index (crc32, title, page_link) VALUES (?, ?, ?)", + (actual_crc32, "[One Pace] Episode 1 [1080p].mkv", "https://nyaa.si/view/12345") + ) + episodes_conn.commit() + episodes_conn.close() + + # Mock load_crc32_to_title_from_index to return our mapping + with patch('acepace.load_crc32_to_title_from_index') as mock_load: + mock_load.return_value = {actual_crc32: "[One Pace] Episode 1 [1080p].mkv"} + + # Note: rename_local_files requires user input, so we'll test the logic separately + # by checking the rename plan generation + cursor = conn.cursor() + cursor.execute("SELECT file_path, crc32 FROM crc32_cache") + entries = cursor.fetchall() + + crc32_to_title = mock_load.return_value + rename_plan = [] + for file_path, crc32 in entries: + title = crc32_to_title.get(crc32) + if title: + dir_name = os.path.dirname(file_path) + ext = os.path.splitext(file_path)[1] + sanitized_title = re.sub(r'[\\/*?:"<>|]', "", title).strip() + new_filename = f"{sanitized_title}" + new_path = os.path.join(dir_name, new_filename) + if os.path.abspath(file_path) != os.path.abspath(new_path): + rename_plan.append((file_path, new_path)) + + assert len(rename_plan) > 0 + assert rename_plan[0][1].endswith("[One Pace] Episode 1 [1080p].mkv") + + conn.close() + + def test_rename_sanitizes_filename(self): + """Test that filenames are sanitized to remove problematic characters.""" + title = "[One Pace] Episode 1: Test <1080p> | Special.mkv" + sanitized = re.sub(r'[\\/*?:"<>|]', "", title).strip() + assert ":" not in sanitized + assert "<" not in sanitized + assert ">" not in sanitized + assert "|" not in sanitized + + def test_rename_skips_files_without_match(self, temp_dir): + """Test that files without CRC32 match are skipped.""" + # Create test video file + test_file = os.path.join(temp_dir, "test.mkv") + with open(test_file, "wb") as f: + f.write(b"test content") + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + crc32s = acepace.calculate_local_crc32(temp_dir, conn) + actual_crc32 = list(crc32s)[0] + + # Mock load_crc32_to_title_from_index to return empty/no match + with patch('acepace.load_crc32_to_title_from_index') as mock_load: + mock_load.return_value = {} # No matches + + cursor = conn.cursor() + cursor.execute("SELECT file_path, crc32 FROM crc32_cache") + entries = cursor.fetchall() + + crc32_to_title = mock_load.return_value + rename_plan = [] + for file_path, crc32 in entries: + title = crc32_to_title.get(crc32) + if title: + # This should not execute + rename_plan.append((file_path, "new_path")) + + assert len(rename_plan) == 0 + + conn.close() + + +class TestCSVExport: + """Tests for CSV export functionality.""" + + def test_export_db_to_csv(self, temp_dir): + """Test exporting database to CSV.""" + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + cursor = conn.cursor() + + # Add test data + cursor.execute( + "INSERT INTO crc32_cache (file_path, crc32) VALUES (?, ?)", + ("/path/to/file1.mkv", "A1B2C3D4") + ) + cursor.execute( + "INSERT INTO crc32_cache (file_path, crc32) VALUES (?, ?)", + ("/path/to/file2.mkv", "E5F6G7H8") + ) + conn.commit() + + # Export to CSV + csv_path = os.path.join(temp_dir, "export.csv") + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + acepace.export_db_to_csv(conn) + + # Check if CSV was created (it should be in current directory, but we'll check the logic) + # The actual file would be created in the working directory + conn.close() + + def test_export_db_to_csv_updates_metadata(self, temp_dir): + """Test that export updates last_db_export metadata.""" + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + cursor = conn.cursor() + cursor.execute( + "INSERT INTO crc32_cache (file_path, crc32) VALUES (?, ?)", + ("/path/to/file.mkv", "A1B2C3D4") + ) + conn.commit() + + acepace.export_db_to_csv(conn) + + last_export = acepace.get_metadata(conn, "last_db_export") + assert last_export is not None + conn.close() diff --git a/tests/test_missing_detection.py b/tests/test_missing_detection.py new file mode 100644 index 0000000..18f3f69 --- /dev/null +++ b/tests/test_missing_detection.py @@ -0,0 +1,204 @@ +"""Unit tests for missing episode detection.""" +import pytest +import os +import sys +import csv +from unittest.mock import patch, MagicMock + +# Add parent directory to path to import acepace +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import acepace + + +class TestMissingEpisodeDetection: + """Tests for missing episode detection logic.""" + + def test_detect_missing_episodes(self, temp_dir): + """Test detecting missing episodes by comparing CRC32s.""" + # Setup: Create local file with known CRC32 + test_file = os.path.join(temp_dir, "test.mkv") + with open(test_file, "wb") as f: + f.write(b"test content") + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + local_crc32s = acepace.calculate_local_crc32(temp_dir, conn) + + # Simulate episodes from Nyaa + crc32_to_link = { + list(local_crc32s)[0]: "https://nyaa.si/view/12345", # Has this one + "MISSING1": "https://nyaa.si/view/12346", # Missing + "MISSING2": "https://nyaa.si/view/12347", # Missing + } + + # Find missing + missing = [crc32 for crc32 in crc32_to_link if crc32 not in local_crc32s] + + assert len(missing) == 2 + assert "MISSING1" in missing + assert "MISSING2" in missing + + conn.close() + + @patch('acepace.requests.get') + def test_fetch_crc32_links_from_nyaa(self, mock_get): + """Test fetching CRC32 links from Nyaa.""" + html = """ + + + + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv + Magnet +
+ [One Pace] Episode 2 [1080p][E5F6G7H8].mkv + Magnet +
+ + + """ + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + base_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace" + crc32_to_link, crc32_to_text, crc32_to_magnet, last_page = acepace.fetch_crc32_links(base_url) + + assert len(crc32_to_link) == 2 + assert "A1B2C3D4" in crc32_to_link + assert "E5F6G7H8" in crc32_to_link + assert "A1B2C3D4" in crc32_to_text + assert "magnet:?xt=urn:btih:abc123" in crc32_to_magnet.values() + + @patch('acepace.requests.get') + def test_fetch_crc32_links_stops_on_empty_page(self, mock_get): + """Test that fetching stops when no matches found.""" + # First page has results + html_with_results = """ + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv +
+ + + """ + + # Second page has no results + html_empty = """ + + + +
+ + + """ + + mock_response1 = MagicMock() + mock_response1.status_code = 200 + mock_response1.text = html_with_results + + mock_response2 = MagicMock() + mock_response2.status_code = 200 + mock_response2.text = html_empty + + mock_get.side_effect = [mock_response1, mock_response2] + + base_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace" + crc32_to_link, _, _, last_page = acepace.fetch_crc32_links(base_url) + + # Should stop after first page (no results on second) + assert len(crc32_to_link) == 1 + assert last_page == 1 + + @patch('acepace.requests.get') + def test_fetch_title_by_crc32(self, mock_get): + """Test fetching title by CRC32 from Nyaa search.""" + html = """ + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv +
+ + + """ + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + title = acepace.fetch_title_by_crc32("A1B2C3D4") + + assert title == "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv" + + @patch('acepace.requests.get') + def test_fetch_title_by_crc32_no_match(self, mock_get): + """Test fetching title when CRC32 not found.""" + html = """ + + + +
+ + + """ + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + title = acepace.fetch_title_by_crc32("NONEXISTENT") + + assert title is None + + @patch('acepace.requests.get') + def test_fetch_title_by_crc32_multiple_matches(self, mock_get): + """Test fetching title when multiple matches found.""" + html = """ + + + + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv +
+ [One Pace] Episode 1 Alt [1080p][A1B2C3D4].mkv +
+ + + """ + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + title = acepace.fetch_title_by_crc32("A1B2C3D4") + + # Should return None when multiple matches + assert title is None From b61ea7c50b9367f6aa276334351c7ae5b3a68cf9 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 20 Jan 2026 11:28:11 +0000 Subject: [PATCH 30/75] Fix episode db update test loop --- tests/test_missing_detection.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/test_missing_detection.py b/tests/test_missing_detection.py index 18f3f69..dab7a37 100644 --- a/tests/test_missing_detection.py +++ b/tests/test_missing_detection.py @@ -44,7 +44,7 @@ def test_detect_missing_episodes(self, temp_dir): @patch('acepace.requests.get') def test_fetch_crc32_links_from_nyaa(self, mock_get): """Test fetching CRC32 links from Nyaa.""" - html = """ + html_with_results = """ @@ -65,10 +65,26 @@ def test_fetch_crc32_links_from_nyaa(self, mock_get): """ - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.text = html - mock_get.return_value = mock_response + # Empty page to stop the loop + html_empty = """ + + +
+
+ + + """ + + mock_response1 = MagicMock() + mock_response1.status_code = 200 + mock_response1.text = html_with_results + + mock_response2 = MagicMock() + mock_response2.status_code = 200 + mock_response2.text = html_empty + + # First page has results, second page is empty to stop the loop + mock_get.side_effect = [mock_response1, mock_response2] base_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace" crc32_to_link, crc32_to_text, crc32_to_magnet, last_page = acepace.fetch_crc32_links(base_url) From c88c1ec578d048de547438a2bb14140b9e4b17e9 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 20 Jan 2026 11:41:13 +0000 Subject: [PATCH 31/75] CRC32 test files fix --- tests/conftest.py | 6 +++--- tests/test_crc32.py | 4 ++-- tests/test_database.py | 4 ++-- tests/test_episodes.py | 24 ++++++++++++------------ tests/test_file_operations.py | 2 +- tests/test_missing_detection.py | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c5410cb..ddd15c1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,8 +44,8 @@ def sample_episode_data(): """Sample episode data for testing.""" return [ ("A1B2C3D4", "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv", "https://nyaa.si/view/12345"), - ("E5F6G7H8", "[One Pace] Episode 2 [1080p][E5F6G7H8].mkv", "https://nyaa.si/view/12346"), - ("I9J0K1L2", "[One Pace] Episode 3 [1080p][I9J0K1L2].mkv", "https://nyaa.si/view/12347"), + ("E5F6A7B8", "[One Pace] Episode 2 [1080p][E5F6A7B8].mkv", "https://nyaa.si/view/12346"), + ("A9B0C1D2", "[One Pace] Episode 3 [1080p][A9B0C1D2].mkv", "https://nyaa.si/view/12347"), ] @@ -64,7 +64,7 @@ def mock_nyaa_html_single_page(): - [One Pace] Episode 2 [1080p][E5F6G7H8].mkv + [One Pace] Episode 2 [1080p][E5F6A7B8].mkv Magnet diff --git a/tests/test_crc32.py b/tests/test_crc32.py index fc24f0c..32b37d9 100644 --- a/tests/test_crc32.py +++ b/tests/test_crc32.py @@ -24,11 +24,11 @@ def test_extract_crc32_from_filename(self): def test_extract_crc32_multiple_matches(self): """Test extracting CRC32 when multiple matches exist (takes last).""" - filename = "[One Pace] Episode 1 [1080p][A1B2C3D4][E5F6G7H8].mkv" + filename = "[One Pace] Episode 1 [1080p][A1B2C3D4][E5F6A7B8].mkv" matches = acepace.CRC32_REGEX.findall(filename) assert len(matches) == 2 # Should take the last match - assert matches[-1].upper() == "E5F6G7H8" + assert matches[-1].upper() == "E5F6A7B8" def test_extract_crc32_no_match(self): """Test extracting CRC32 from filename without CRC32.""" diff --git a/tests/test_database.py b/tests/test_database.py index 1ea7a93..2b18264 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -126,7 +126,7 @@ def test_load_crc32_to_title_from_index(self, temp_dir, sample_episode_data): mapping = acepace.load_crc32_to_title_from_index() assert len(mapping) == 3 assert mapping["A1B2C3D4"] == "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv" - assert mapping["E5F6G7H8"] == "[One Pace] Episode 2 [1080p][E5F6G7H8].mkv" - assert mapping["I9J0K1L2"] == "[One Pace] Episode 3 [1080p][I9J0K1L2].mkv" + assert mapping["E5F6A7B8"] == "[One Pace] Episode 2 [1080p][E5F6A7B8].mkv" + assert mapping["A9B0C1D2"] == "[One Pace] Episode 3 [1080p][A9B0C1D2].mkv" os.remove(os.path.join(temp_dir, 'test.db')) diff --git a/tests/test_episodes.py b/tests/test_episodes.py index fc70ddd..9db42ed 100644 --- a/tests/test_episodes.py +++ b/tests/test_episodes.py @@ -26,7 +26,7 @@ def test_fetch_episodes_metadata_single_page(self, mock_get, mock_nyaa_html_sing assert len(episodes) == 2 assert any(ep[0] == "A1B2C3D4" for ep in episodes) - assert any(ep[0] == "E5F6G7H8" for ep in episodes) + assert any(ep[0] == "E5F6A7B8" for ep in episodes) @patch('acepace.requests.get') @patch('acepace.time.sleep') # Mock sleep to speed up tests @@ -46,7 +46,7 @@ def test_fetch_episodes_metadata_multi_page(self, mock_sleep, mock_get, mock_nya
- [One Pace] Episode 3 [1080p][I9J0K1L2].mkv + [One Pace] Episode 3 [1080p][A9B0C1D2].mkv
@@ -225,7 +225,7 @@ def test_fetch_episodes_prefers_1080p(self, mock_get): - [One Pace] Episode 2 [1080p][E5F6G7H8].mkv + [One Pace] Episode 2 [1080p][E5F6A7B8].mkv @@ -262,7 +262,7 @@ def test_fetch_episodes_accepts_720p_as_fallback(self, mock_get): - [One Pace] Episode 2 [720p][E5F6G7H8].mkv + [One Pace] Episode 2 [720p][E5F6A7B8].mkv @@ -299,12 +299,12 @@ def test_fetch_episodes_excludes_lower_quality(self, mock_get): - [One Pace] Episode 2 [360p][E5F6G7H8].mkv + [One Pace] Episode 2 [360p][E5F6A7B8].mkv - [One Pace] Episode 3 [240p][I9J0K1L2].mkv + [One Pace] Episode 3 [240p][A9B0C1D2].mkv @@ -378,17 +378,17 @@ def test_fetch_episodes_mixed_qualities_only_keeps_valid(self, mock_get): - [One Pace] Episode 2 [720p][E5F6G7H8].mkv + [One Pace] Episode 2 [720p][E5F6A7B8].mkv - [One Pace] Episode 3 [480p][I9J0K1L2].mkv + [One Pace] Episode 3 [480p][A9B0C1D2].mkv - [One Pace] Episode 4 [1080p][M3N4O5P6].mkv + [One Pace] Episode 4 [1080p][A3B4C5D6].mkv @@ -431,7 +431,7 @@ def test_fetch_episodes_handles_case_insensitive_quality(self, mock_get): - [One Pace] Episode 2 [720P][E5F6G7H8].mkv + [One Pace] Episode 2 [720P][E5F6A7B8].mkv @@ -465,7 +465,7 @@ def test_fetch_episodes_excludes_episodes_without_quality_marker(self, mock_get) - [One Pace] Episode 2 [1080p][E5F6G7H8].mkv + [One Pace] Episode 2 [1080p][E5F6A7B8].mkv @@ -485,7 +485,7 @@ def test_fetch_episodes_excludes_episodes_without_quality_marker(self, mock_get) # Should only include episode with quality marker assert len(episodes) == 1 crc32, title, _ = episodes[0] - assert crc32 == "E5F6G7H8" + assert crc32 == "E5F6A7B8" assert "[1080p]" in title.upper() or "1080P" in title.upper() @patch('acepace.requests.get') diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index 8cf7506..fc6c717 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -125,7 +125,7 @@ def test_export_db_to_csv(self, temp_dir): ) cursor.execute( "INSERT INTO crc32_cache (file_path, crc32) VALUES (?, ?)", - ("/path/to/file2.mkv", "E5F6G7H8") + ("/path/to/file2.mkv", "E5F6A7B8") ) conn.commit() diff --git a/tests/test_missing_detection.py b/tests/test_missing_detection.py index dab7a37..f959eae 100644 --- a/tests/test_missing_detection.py +++ b/tests/test_missing_detection.py @@ -56,7 +56,7 @@ def test_fetch_crc32_links_from_nyaa(self, mock_get): - [One Pace] Episode 2 [1080p][E5F6G7H8].mkv + [One Pace] Episode 2 [1080p][E5F6A7B8].mkv Magnet @@ -91,7 +91,7 @@ def test_fetch_crc32_links_from_nyaa(self, mock_get): assert len(crc32_to_link) == 2 assert "A1B2C3D4" in crc32_to_link - assert "E5F6G7H8" in crc32_to_link + assert "E5F6A7B8" in crc32_to_link assert "A1B2C3D4" in crc32_to_text assert "magnet:?xt=urn:btih:abc123" in crc32_to_magnet.values() From 159ad38c29b049feb09500851d56d33c24d0b6a0 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 20 Jan 2026 14:20:57 +0000 Subject: [PATCH 32/75] Test fix --- tests/test_clients.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_clients.py b/tests/test_clients.py index f5ea4ab..2cee86d 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -3,6 +3,7 @@ import sys import os from unittest.mock import patch, MagicMock, Mock +from requests.structures import CaseInsensitiveDict # Add parent directory to path to import clients sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -119,7 +120,8 @@ def test_transmission_client_init_409_retry(self, mock_session_class): # First call returns 409 with session ID mock_response_409 = MagicMock() mock_response_409.status_code = 409 - mock_response_409.headers = {"X-Transmission-Session-Id": "test_session_id"} + # Use CaseInsensitiveDict to match requests.Response.headers behavior + mock_response_409.headers = CaseInsensitiveDict({"X-Transmission-Session-Id": "test_session_id"}) # Second call succeeds mock_response_200 = MagicMock() @@ -156,16 +158,23 @@ def test_transmission_add_torrents_handles_409(self, mock_sleep, mock_session_cl """Test Transmission handles 409 during torrent add.""" mock_session = MagicMock() - # First call returns 409, second succeeds + # Response for __init__ connection test (session-get) + mock_init_response = MagicMock() + mock_init_response.status_code = 200 + mock_init_response.json.return_value = {"result": "success"} + + # First call in add_torrents returns 409, second succeeds mock_response_409 = MagicMock() mock_response_409.status_code = 409 - mock_response_409.headers = {"X-Transmission-Session-Id": "new_session_id"} + # Use CaseInsensitiveDict to match requests.Response.headers behavior + mock_response_409.headers = CaseInsensitiveDict({"X-Transmission-Session-Id": "new_session_id"}) mock_response_200 = MagicMock() mock_response_200.status_code = 200 mock_response_200.json.return_value = {"result": "success"} - mock_session.post.side_effect = [mock_response_409, mock_response_200] + # __init__ makes one POST, add_torrents makes two POSTs (409 then retry) + mock_session.post.side_effect = [mock_init_response, mock_response_409, mock_response_200] mock_session_class.return_value = mock_session client = TransmissionClient("localhost", 9091, None, None) From 5c2c82088e5e091c4160a4f1cd16fc51e6482d87 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 20 Jan 2026 15:15:16 +0000 Subject: [PATCH 33/75] Added credits --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 3297e01..205ac4b 100644 --- a/README.md +++ b/README.md @@ -88,3 +88,5 @@ python acepace.py --db ## 🙏 Credits Ace-Pace is proudly inspired by and built to support the incredible work of the [One-Pace](http://onepace.net/) team. Their dedication to crafting a seamless and engaging One Piece viewing experience has allowed me to discover and share this legendary series. I salute their passion, creativity, and commitment. + +Since the start of this project, not unlinke Luffy, a few people joined me to build or support it, namely [@Staubgeborener](https://github.com/Staubgeborener) & [@thekoma](https://github.com/thekoma) who implemented the multi-clients functionality. Check them out! From dcb98b786caa6f9253f863b65312945a3ffeca79 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 20 Jan 2026 15:57:48 +0000 Subject: [PATCH 34/75] Refactoring --- .gitignore | 1 + NAMING_CONVENTIONS.md | 137 ++++++++ acepace.py | 784 +++++++++++++++++++++++------------------- spec.md | 54 ++- 4 files changed, 621 insertions(+), 355 deletions(-) create mode 100644 NAMING_CONVENTIONS.md diff --git a/.gitignore b/.gitignore index 5320855..c62d63e 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ Ace-Pace_DB.csv Ace-Pace_Missing.csv crc32_files.db check.sh +sonar-project.properties diff --git a/NAMING_CONVENTIONS.md b/NAMING_CONVENTIONS.md new file mode 100644 index 0000000..2b60beb --- /dev/null +++ b/NAMING_CONVENTIONS.md @@ -0,0 +1,137 @@ +# Function Naming Conventions + +This document explains the naming logic for helper functions in `acepace.py`, particularly those added during the SonarQube refactoring. + +## Naming Pattern Overview + +Helper functions use a consistent naming pattern: `___` or `__` + +The underscore prefix (`_`) indicates these are **private/internal functions** that are not part of the public API. + +## Naming Categories + +### 1. Extraction Functions: `_extract_*` + +Functions that extract data from structures (HTML, files, etc.): + +- `_extract_title_link_from_row(row)` - Extracts the title link element from an HTML table row +- `_extract_filenames_from_folder_structure(filelist_div)` - Extracts filenames from a folder-based torrent file list +- `_extract_filenames_from_torrent_page(torrent_soup)` - Extracts all filenames from a torrent page's file list +- `_extract_matching_titles_from_rows(rows, crc32)` - Extracts titles matching a specific CRC32 from table rows + +**Pattern**: `_extract__from_` + +### 2. Processing Functions: `_process_*` + +Functions that process data structures or perform transformations: + +- `_process_fname_entry(fname_text, ...)` - Processes a filename entry to extract and store CRC32 +- `_process_torrent_page(page_link, ...)` - Processes a torrent page to extract CRC32 information +- `_process_episode_row(row, ...)` - Processes a single table row to extract episode information +- `_process_crc32_row(row, ...)` - Processes a table row to extract CRC32 information for missing episodes + +**Pattern**: `_process__` + +### 3. Validation Functions: `_is_*`, `_validate_*` + +Functions that validate or check conditions: + +- `_is_valid_quality(fname_text)` - Checks if a filename has valid quality (1080p or 720p) +- `_validate_url(url)` - Validates that a URL points to a valid Nyaa domain + +**Pattern**: `_is_` or `_validate_` + +### 4. Command Handlers: `_handle_*` + +Functions that handle specific command-line operations: + +- `_handle_download_command(args)` - Handles the `--download` command +- `_handle_rename_command(conn)` - Handles the `--rename` command +- `_handle_main_commands(args, conn, folder)` - Routes and handles main command execution + +**Pattern**: `_handle__` + +### 5. Getter Functions: `_get_*` + +Functions that retrieve or compute values: + +- `_get_total_pages(soup)` - Extracts total number of pages from pagination controls +- `_get_folder_from_args(args, conn, needs_folder)` - Gets folder path from arguments or prompts user +- `_get_rename_prompt(last_ep_update)` - Gets user prompt for rename confirmation + +**Pattern**: `_get__` + +### 6. Load/Save Functions: `_load_*`, `_save_*` + +Functions that load or save data: + +- `_load_old_missing_crc32s()` - Loads CRC32s from previous missing CSV file +- `_save_missing_episodes_csv(...)` - Saves missing episodes to CSV file + +**Pattern**: `_load_` or `_save__` + +### 7. Print/Report Functions: `_print_*`, `_report_*`, `_show_*` + +Functions that display information: + +- `_print_report_header(conn, folder, args)` - Prints header information for the report +- `_report_new_missing_episodes(missing, crc32_to_text)` - Reports newly detected missing episodes +- `_show_episodes_metadata_status()` - Shows last episodes metadata update status + +**Pattern**: `_print_`, `_report_`, or `_show_` + +### 8. Workflow Functions: `_generate_*`, `_calculate_*` + +Functions that orchestrate multi-step workflows: + +- `_generate_missing_episodes_report(conn, folder, args)` - Generates and saves missing episodes report +- `_calculate_and_find_missing(folder, conn, args, last_run)` - Calculates local CRC32s and finds missing episodes + +**Pattern**: `_generate_` or `_calculate__and_` + +### 9. Utility Functions: `_parse_*`, `_count_*` + +General utility functions: + +- `_parse_arguments()` - Parses command-line arguments +- `_count_video_files(folder, conn)` - Counts total video files and files already recorded in DB + +**Pattern**: `_parse_` or `_count_` + +## Design Principles + +1. **Single Responsibility**: Each function has one clear purpose +2. **Descriptive Names**: Function names clearly describe what they do +3. **Consistent Patterns**: Similar functions follow similar naming patterns +4. **Private by Default**: All helpers are prefixed with `_` to indicate internal use +5. **Context Clarity**: Function names include enough context to understand their purpose + +## Benefits of This Naming Convention + +1. **Readability**: Easy to understand what a function does from its name +2. **Discoverability**: Functions can be found by their action prefix +3. **Maintainability**: Clear separation between public API and internal helpers +4. **Documentation**: Function names serve as inline documentation +5. **Refactoring**: Easy to identify and group related functions + +## Example Usage Flow + +When reading code, you can quickly understand the flow: + +```python +# Main entry point +main() + → _parse_arguments() # Get user input + → _validate_url() # Check URL is valid + → _show_episodes_metadata_status() # Display status + → _get_folder_from_args() # Get or prompt for folder + → _handle_main_commands() # Route to appropriate handler + → _generate_missing_episodes_report() # Main workflow + → _print_report_header() # Show header info + → _calculate_and_find_missing() # Find missing episodes + → fetch_crc32_links() # Public API function + → _report_new_missing_episodes() # Show new episodes + → _save_missing_episodes_csv() # Save results +``` + +This naming convention makes the codebase more maintainable and easier to understand. diff --git a/acepace.py b/acepace.py index f2e4108..e483f7a 100644 --- a/acepace.py +++ b/acepace.py @@ -25,6 +25,11 @@ DB_NAME = "crc32_files.db" EPISODES_DB_NAME = "episodes_index.db" +# Constants for repeated string literals +HTML_PARSER = "html.parser" +MISSING_CSV_FILENAME = "Ace-Pace_Missing.csv" +NYAA_BASE_URL = "https://nyaa.si" + def init_db(): exists = os.path.exists(DB_NAME) @@ -54,7 +59,6 @@ def init_db(): # --- New: Episodes metadata DB --- def init_episodes_db(): - exists = os.path.exists(EPISODES_DB_NAME) conn = sqlite3.connect(EPISODES_DB_NAME) c = conn.cursor() c.execute( @@ -94,6 +98,134 @@ def set_episodes_metadata(conn, key, value): # --- New: Fetch and update episodes_index table --- +def _is_valid_quality(fname_text): + """Check if filename has valid quality (1080p preferred, 720p as fallback only). + Returns True if quality is 1080p or 720p, False otherwise.""" + quality_matches = QUALITY_REGEX.findall(fname_text) + if not quality_matches: + return False # No quality marker found, exclude + # Check if quality is exactly 1080p or 720p (not higher, not lower) + for quality in quality_matches: + quality_num = int(quality.lower().replace('p', '')) + if quality_num == 1080 or quality_num == 720: + return True + return False # Quality not 1080p or 720p + + +def _process_fname_entry(fname_text, seen_crc32, episodes, page_link): + """Helper to extract CRC32 from fname_text and store if valid and unique. + Only accepts episodes with 1080p or 720p quality (720p as fallback).""" + m = CRC32_REGEX.findall(fname_text) + found = False + if m and "[One Pace]" in fname_text and _is_valid_quality(fname_text): + crc32 = m[-1].upper() + if crc32 not in seen_crc32: + # print(f"New CRC32 detected: {crc32} -> Title: {fname_text}") + episodes.append((crc32, fname_text, page_link)) + seen_crc32.add(crc32) + found = True + return found + + +def _get_total_pages(soup): + """Extract total number of pages from pagination controls.""" + total_pages = 1 + pagination = soup.find("ul", class_="pagination") + if pagination: + page_links = pagination.find_all("a", href=True) + page_numbers = [] + for a in page_links: + text = a.text.strip() + if text.isdigit(): + try: + page_numbers.append(int(text)) + except Exception: + pass + if page_numbers: + total_pages = max(page_numbers) + return total_pages + + +def _extract_title_link_from_row(row): + """Extract title link from a table row.""" + links = row.find_all("a", href=True) + for a in links: + href = a.get("href", "") + if href.startswith("/view/") and a.has_attr("title"): + return a + return None + + +def _extract_filenames_from_folder_structure(filelist_div): + """Extract filenames from folder structure in file list.""" + all_uls = filelist_div.find_all("ul") + filenames = [] + for ul in all_uls: + for file_li in ul.find_all("li"): + if not file_li.find("ul"): + direct_texts = [ + t for t in file_li.contents if isinstance(t, str) + ] + fname_text = "".join(direct_texts).strip() + if fname_text: + filenames.append(fname_text) + return filenames + + +def _extract_filenames_from_torrent_page(torrent_soup): + """Extract filenames from a torrent page's file list.""" + filelist_div = torrent_soup.find("div", class_="torrent-file-list") + if not filelist_div: + return [] + + has_folder = bool(filelist_div.find("a", class_="folder")) + + if has_folder: + return _extract_filenames_from_folder_structure(filelist_div) + else: + li = filelist_div.find("li") + if li: + direct_texts = [t for t in li.contents if isinstance(t, str)] + fname_text = "".join(direct_texts).strip() + if fname_text: + return [fname_text] + return [] + + +def _process_torrent_page(page_link, seen_crc32, episodes): + """Process a torrent page to extract CRC32 information from file list.""" + try: + torrent_resp = requests.get(page_link) + if torrent_resp.status_code != 200: + print(f"Failed to fetch torrent page {page_link}") + return False + t_soup = BeautifulSoup(torrent_resp.text, HTML_PARSER) + filenames = _extract_filenames_from_torrent_page(t_soup) + found = False + for fname in filenames: + if _process_fname_entry(str(fname), seen_crc32, episodes, page_link): + found = True + return found + except Exception: + return False + + +def _process_episode_row(row, seen_crc32, episodes): + """Process a single table row to extract episode information.""" + title_link = _extract_title_link_from_row(row) + if not title_link: + return False + + title = title_link.text.strip() + page_link = NYAA_BASE_URL + title_link["href"] + matches = CRC32_REGEX.findall(title) + + if matches: + return _process_fname_entry(title, seen_crc32, episodes, page_link) + else: + return _process_torrent_page(page_link, seen_crc32, episodes) + + def fetch_episodes_metadata(): """ Fetch all One Pace episodes from Nyaa, collecting CRC32, title, and page link. @@ -101,34 +233,7 @@ def fetch_episodes_metadata(): Returns: List of (crc32, title, page_link) """ - def _is_valid_quality(fname_text): - """Check if filename has valid quality (1080p preferred, 720p as fallback only). - Returns True if quality is 1080p or 720p, False otherwise.""" - quality_matches = QUALITY_REGEX.findall(fname_text) - if not quality_matches: - return False # No quality marker found, exclude - # Check if quality is exactly 1080p or 720p (not higher, not lower) - for quality in quality_matches: - quality_num = int(quality.lower().replace('p', '')) - if quality_num == 1080 or quality_num == 720: - return True - return False # Quality not 1080p or 720p - - def _process_fname_entry(fname_text, seen_crc32, episodes, page_link): - """Helper to extract CRC32 from fname_text and store if valid and unique. - Only accepts episodes with 1080p or 720p quality (720p as fallback).""" - m = CRC32_REGEX.findall(fname_text) - found = False - if m and "[One Pace]" in fname_text and _is_valid_quality(fname_text): - crc32 = m[-1].upper() - if crc32 not in seen_crc32: - # print(f"New CRC32 detected: {crc32} -> Title: {fname_text}") - episodes.append((crc32, fname_text, page_link)) - seen_crc32.add(crc32) - found = True - return found - - base_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace" + base_url = f"{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace" episodes = [] seen_crc32 = set() page = 1 @@ -139,22 +244,8 @@ def _process_fname_entry(fname_text, seen_crc32, episodes, page_link): if resp.status_code != 200: print(f"Failed to fetch page 1, status code: {resp.status_code}") return episodes - soup = BeautifulSoup(resp.text, "html.parser") - # Find pagination links and determine max page number - total_pages = 1 - pagination = soup.find("ul", class_="pagination") - if pagination: - page_links = pagination.find_all("a", href=True) - page_numbers = [] - for a in page_links: - text = a.text.strip() - if text.isdigit(): - try: - page_numbers.append(int(text)) - except Exception: - pass - if page_numbers: - total_pages = max(page_numbers) + soup = BeautifulSoup(resp.text, HTML_PARSER) + total_pages = _get_total_pages(soup) # Now loop from page 1 to total_pages while page <= total_pages: @@ -167,84 +258,13 @@ def _process_fname_entry(fname_text, seen_crc32, episodes, page_link): if resp.status_code != 200: print(f"Failed to fetch page {page}, status code: {resp.status_code}") break - page_soup = BeautifulSoup(resp.text, "html.parser") + page_soup = BeautifulSoup(resp.text, HTML_PARSER) table = page_soup.find("table", class_="torrent-list") if not table: break rows = table.find_all("tr") - page_has_matches = False for row in rows: - links = row.find_all("a", href=True) - title_link = None - for a in links: - href = a.get("href", "") - if href.startswith("/view/") and a.has_attr("title"): - title_link = a - break - if not title_link: - continue - title = title_link.text.strip() - page_link = "https://nyaa.si" + title_link["href"] - matches = CRC32_REGEX.findall(title) - found_in_this_row = False - if matches: - found_in_this_row = _process_fname_entry( - title, seen_crc32, episodes, page_link - ) - else: - try: - # print(f"Fetching page {page_link}...") - torrent_resp = requests.get(page_link) - if torrent_resp.status_code != 200: - print(f"Failed to fetch torrent page {page_link}") - continue - t_soup = BeautifulSoup(torrent_resp.text, "html.parser") - filelist_div = t_soup.find("div", class_="torrent-file-list") - if not filelist_div: - continue - has_folder = bool(filelist_div.find("a", class_="folder")) - filenames = [] - if has_folder: - # print("Has folder") - all_uls = filelist_div.find_all("ul") - leaf_filenames = [] - for ul in all_uls: - for file_li in ul.find_all("li"): - if not file_li.find("ul"): - direct_texts = [ - t - for t in file_li.contents - if isinstance(t, str) - ] - fname_text = "".join(direct_texts).strip() - if fname_text: - leaf_filenames.append(fname_text) - for fname in leaf_filenames: - fname = str(fname) - if _process_fname_entry( - fname, seen_crc32, episodes, page_link - ): - found_in_this_row = True - else: - # print("No folder") - li = filelist_div.find("li") - direct_texts = [t for t in li.contents if isinstance(t, str)] - fname_text = "".join(direct_texts).strip() - # print(f"Direct text: {fname_text}") - if fname_text: - if _process_fname_entry( - fname_text, seen_crc32, episodes, page_link - ): - found_in_this_row = True - except Exception: - print( - f"Error occurred while processing file list for {title} ({page_link})" - ) - if found_in_this_row: - page_has_matches = True - # If no matches on this page, break (may be redundant now that we know total_pages) - # if not page_has_matches: - # break + _process_episode_row(row, seen_crc32, episodes) page += 1 time.sleep(0.2) print(f"Fetched {len(episodes)} unique episodes with CRC32s.") @@ -294,6 +314,33 @@ def set_metadata(conn, key, value): conn.commit() +def _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): + """Process a single table row to extract CRC32 information.""" + links = row.find_all("a", href=True) + title_link = None + magnet_link = "" + for a in links: + if a.has_attr("title"): + title_link = a + href = a.get("href", "") + if href.startswith("magnet:"): + magnet_link = href + if not title_link: + return False # Skip rows without a valid title link + filename_text = title_link.text + link = NYAA_BASE_URL + title_link["href"] + matches = CRC32_REGEX.findall(filename_text) + if matches: + crc32 = matches[-1].upper() + crc32_to_link[crc32] = link + crc32_to_text[crc32] = filename_text + crc32_to_magnet[crc32] = magnet_link + return True + else: + print(f"Warning: No CRC32 found in title '{filename_text}'") + return False + + def fetch_crc32_links(base_url): crc32_to_link = {} crc32_to_text = {} @@ -307,7 +354,7 @@ def fetch_crc32_links(base_url): print(f"Failed to fetch page {page}, status code: {resp.status_code}") break - soup = BeautifulSoup(resp.text, "html.parser") + soup = BeautifulSoup(resp.text, HTML_PARSER) table = soup.find("table", class_="torrent-list") if not table: print("No table found, stopping.") @@ -320,25 +367,7 @@ def fetch_crc32_links(base_url): found_in_page = False for row in rows: - links = row.find_all("a", href=True) - title_link = None - magnet_link = "" - for a in links: - if a.has_attr("title"): - title_link = a - href = a.get("href", "") - if href.startswith("magnet:"): - magnet_link = href - if not title_link: - continue # Skip rows without a valid title link - filename_text = title_link.text - link = "https://nyaa.si" + title_link["href"] - matches = CRC32_REGEX.findall(filename_text) - if matches: - crc32 = matches[-1].upper() - crc32_to_link[crc32] = link - crc32_to_text[crc32] = filename_text - crc32_to_magnet[crc32] = magnet_link + if _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): found_in_page = True else: print(f"Warning: No CRC32 found in title '{filename_text}'") @@ -352,18 +381,8 @@ def fetch_crc32_links(base_url): return crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page -def fetch_title_by_crc32(crc32): - # Search on Nyaa for the given CRC32 - search_url = f"https://nyaa.si/?f=0&c=0_0&q={crc32}&o=asc" - resp = requests.get(search_url) - if resp.status_code != 200: - print(f"Failed to fetch search results for CRC32 {crc32}") - return None - soup = BeautifulSoup(resp.text, "html.parser") - table = soup.find("table", class_="torrent-list") - if not table: - return None - rows = table.find_all("tr") +def _extract_matching_titles_from_rows(rows, crc32): + """Extract titles matching the given CRC32 from table rows.""" matched_titles = [] for row in rows: links = row.find_all("a", href=True) @@ -374,6 +393,23 @@ def fetch_title_by_crc32(crc32): matches = CRC32_REGEX.findall(filename_text) if matches and matches[-1].upper() == crc32: matched_titles.append(filename_text) + return matched_titles + + +def fetch_title_by_crc32(crc32): + # Search on Nyaa for the given CRC32 + search_url = f"{NYAA_BASE_URL}/?f=0&c=0_0&q={crc32}&o=asc" + resp = requests.get(search_url) + if resp.status_code != 200: + print(f"Failed to fetch search results for CRC32 {crc32}") + return None + soup = BeautifulSoup(resp.text, HTML_PARSER) + table = soup.find("table", class_="torrent-list") + if not table: + return None + rows = table.find_all("tr") + matched_titles = _extract_matching_titles_from_rows(rows, crc32) + if len(matched_titles) == 1: print(f"Found {crc32} on Nyaa!") return matched_titles[0] @@ -419,7 +455,7 @@ def calculate_local_crc32(folder, conn): return local_crc32s -def rename_local_files(conn, folder): +def rename_local_files(conn): c = conn.cursor() c.execute("SELECT file_path, crc32 FROM crc32_cache") entries = c.fetchall() @@ -433,7 +469,6 @@ def rename_local_files(conn, folder): for _, crc32 in entries: local_crc32s.add(crc32) - matched = 0 total = len(local_crc32s) rename_plan = [] for file_path, crc32 in entries: @@ -441,14 +476,12 @@ def rename_local_files(conn, folder): if not title: continue # No match found in index, skip dir_name = os.path.dirname(file_path) - ext = os.path.splitext(file_path)[1] # Sanitize title for filename (remove problematic characters) sanitized_title = re.sub(r'[\\/*?:"<>|]', "", title).strip() new_filename = f"{sanitized_title}" new_path = os.path.join(dir_name, new_filename) if os.path.abspath(file_path) != os.path.abspath(new_path): rename_plan.append((file_path, new_path)) - matched += 1 if not rename_plan: print("No files to rename.") @@ -495,78 +528,14 @@ def export_db_to_csv(conn): set_metadata(conn, "last_db_export", now_str) -def main(): - parser = argparse.ArgumentParser( - description="Find missing episodes from your personal One Pace library." - ) - parser.add_argument( - "--url", - default="https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc", - help="Base URL without the page param. Example: 'https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc' ", - ) - parser.add_argument("--folder", help="Folder containing local video files.") - parser.add_argument( - "--db", action="store_true", help="Export database to CSV and exit." - ) - parser.add_argument( - "--client", - choices=["transmission", "qbittorrent"], - help="The BitTorrent client to use.", - ) - parser.add_argument( - "--download", - action="store_true", - help="Import magnet links from missing CSV and add to the specified BitTorrent client.", - ) - parser.add_argument( - "--rename", - action="store_true", - help="Rename local files based on CRC32 matching titles from Nyaa.", - ) - parser.add_argument( - "--episodes_update", - action="store_true", - help="Update episodes metadata database from Nyaa.", - ) - parser.add_argument("--host", default="localhost", help="The BitTorrent client host.") - parser.add_argument("--port", type=int, help="The BitTorrent client port.") - parser.add_argument("--username", help="The BitTorrent client username.") - parser.add_argument("--password", help="The BitTorrent client password.") - parser.add_argument("--download-folder", help="The folder to download the torrents to.") - parser.add_argument("--tag", action="append", help="Tag to add to the torrent in qBittorrent (can be used multiple times).") - parser.add_argument("--category", help="Category to add to the torrent in qBittorrent.") - args = parser.parse_args() - - # Check if the URL points to a valid Nyaa domain - if not args.url.startswith(("https://nyaa.si", "https://nyaa.land")): - print( - "Error: The --url argument must point to a valid Nyaa website (https://nyaa.si or https://nyaa.land)." - ) - return - - # --- Show last episodes metadata update --- - episodes_db_conn = init_episodes_db() - last_ep_update = get_episodes_metadata(episodes_db_conn, "episodes_db_last_update") - if last_ep_update: - print(f"Episodes metadata last updated: {last_ep_update}") - else: - print("Episodes metadata database not yet updated.") - episodes_db_conn.close() - - if args.episodes_update: - update_episodes_index_db() - return - - conn = init_db() - - # Folder selection logic: Always prompt if folder is required but not given +def _get_folder_from_args(args, conn, needs_folder): + """Get folder path from arguments or prompt user.""" folder = args.folder - needs_folder = not args.download # All commands except --download need folder if needs_folder and not folder: # Try to load last_folder from metadata last_folder = get_metadata(conn, "last_folder") if last_folder: - print(f"Previously used folder: {last_folder}") + print(f"Last used folder: {last_folder}") user_input = input( "Press Enter to use this folder, or enter a new path: " ).strip() @@ -578,94 +547,85 @@ def main(): folder = input("Enter the folder containing local video files: ").strip() if not folder: print("Error: No folder specified.") - return + return None set_metadata(conn, "last_folder", folder) elif folder: set_metadata(conn, "last_folder", folder) + return folder + + +def _handle_download_command(args): + """Handle the download command.""" + if not args.client: + print("Error: --client is required when using --download.") + return False + + if not os.path.exists(MISSING_CSV_FILENAME): + print(f"Missing file '{MISSING_CSV_FILENAME}' not found. Run the script first!") + return False + + magnets = [] + with open(MISSING_CSV_FILENAME, "r", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + magnet_link = row.get("Magnet Link", "").strip() + if magnet_link.startswith("magnet:"): + magnets.append(magnet_link) + + if not magnets: + print(f"No magnet links found in '{MISSING_CSV_FILENAME}'.") + return False + + port = args.port + if not port: + port = 9091 if args.client == "transmission" else 8080 + + try: + client = get_client(args.client, args.host, port, args.username, args.password) + client.add_torrents( + magnets, + download_folder=args.download_folder, + tags=args.tag, + category=args.category, + ) + except Exception as e: + print(f"Error: {e}") + return False - if args.download: - if not args.client: - print("Error: --client is required when using --download.") - return - - if not os.path.exists("Ace-Pace_Missing.csv"): - print("Missing file 'Ace-Pace_Missing.csv' not found. Run the script first!") - return - - magnets = [] - with open("Ace-Pace_Missing.csv", "r", encoding="utf-8") as f: - reader = csv.DictReader(f) - for row in reader: - magnet_link = row.get("Magnet Link", "").strip() - if magnet_link.startswith("magnet:"): - magnets.append(magnet_link) - - if not magnets: - print("No magnet links found in 'Ace-Pace_Missing.csv'.") - return + return True - port = args.port - if not port: - port = 9091 if args.client == "transmission" else 8080 - try: - client = get_client(args.client, args.host, port, args.username, args.password) - client.add_torrents( - magnets, - download_folder=args.download_folder, - tags=args.tag, - category=args.category, - ) - except (ValueError, Exception) as e: - print(f"Error: {e}") - return - - return +def _get_rename_prompt(last_ep_update): + """Get user prompt for updating episodes database before renaming.""" + if not last_ep_update: + print("WARNING: Episodes metadata database has not been updated yet.") + return input("Update episodes metadata database before renaming? (y/n): ").strip().lower() + else: + return input( + f"Update episodes metadata database before renaming? (last update: {last_ep_update}) (y/n): " + ).strip().lower() - if args.rename: - # Prompt to update episodes_index DB if it's old - episodes_db_conn = init_episodes_db() - last_ep_update = get_episodes_metadata( - episodes_db_conn, "episodes_db_last_update" - ) - episodes_db_conn.close() - if not last_ep_update: - print("WARNING: Episodes metadata database has not been updated yet.") - elif last_ep_update: - prompt = ( - input( - f"Update episodes metadata database before renaming? (last update: {last_ep_update}) (y/n): " - ) - .strip() - .lower() - ) - else: - prompt = ( - input("Update episodes metadata database before renaming? (y/n): ") - .strip() - .lower() - ) - if prompt == "y": - update_episodes_index_db() - print( - "Renaming local files based on matching titles from One Pace episodes index..." - ) - rename_local_files(conn, folder) - return - if not folder: - print("Error: --folder argument is required.") - return - - last_missing_export = get_metadata(conn, "last_missing_export") - if last_missing_export: - print(f"Last missing files list generated on: {last_missing_export}") +def _handle_rename_command(conn): + """Handle the rename command.""" + episodes_db_conn = init_episodes_db() + last_ep_update = get_episodes_metadata( + episodes_db_conn, "episodes_db_last_update" + ) + episodes_db_conn.close() + + prompt = _get_rename_prompt(last_ep_update) + + if prompt == "y": + update_episodes_index_db() + print( + "Renaming local files based on matching titles from One Pace episodes index..." + ) + rename_local_files(conn) - if args.db: - export_db_to_csv(conn) - return - # Count total video files and files already recorded in DB +def _count_video_files(folder, conn): + """Count total video files and files already recorded in DB.""" total_files = 0 recorded_files = 0 c = conn.cursor() @@ -678,6 +638,46 @@ def main(): c.execute("SELECT 1 FROM crc32_cache WHERE file_path = ?", (file_path,)) if c.fetchone(): recorded_files += 1 + return total_files, recorded_files + + +def _load_old_missing_crc32s(): + """Load CRC32s from previous missing CSV file.""" + old_missing_crc32s = set() + if os.path.exists(MISSING_CSV_FILENAME): + with open(MISSING_CSV_FILENAME, "r", encoding="utf-8") as f: + reader = csv.reader(f) + next(reader, None) # skip header + for row in reader: + if len(row) >= 1: + title = row[0] + # Extract CRC32 from title if possible + matches = CRC32_REGEX.findall(title) + if matches: + old_missing_crc32s.add(matches[-1].upper()) + return old_missing_crc32s + + +def _save_missing_episodes_csv(missing, crc32_to_text, crc32_to_link, crc32_to_magnet): + """Save missing episodes to CSV file.""" + with open(MISSING_CSV_FILENAME, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, quoting=csv.QUOTE_ALL) + writer.writerow(["Title", "Page Link", "Magnet Link"]) + for crc32 in missing: + title = crc32_to_text[crc32] + page_link = crc32_to_link[crc32] + magnet = crc32_to_magnet.get(crc32, "") + writer.writerow([title, page_link, magnet]) + print(f"Missing files list saved to {MISSING_CSV_FILENAME}") + + +def _print_report_header(conn, folder, args): + """Print header information for the report.""" + last_missing_export = get_metadata(conn, "last_missing_export") + if last_missing_export: + print(f"Last missing files list generated on: {last_missing_export}") + + total_files, recorded_files = _count_video_files(folder, conn) last_run = get_metadata(conn, "last_run") if last_run: @@ -689,7 +689,12 @@ def main(): print(f"Using URL: {args.url}") print(f"Total video files detected: {total_files}") print(f"Episodes already recorded in DB: {recorded_files}") + + return last_run + +def _calculate_and_find_missing(folder, conn, args, last_run): + """Calculate local CRC32s and find missing episodes.""" crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = ( fetch_crc32_links(args.url) ) @@ -711,60 +716,149 @@ def main(): print( f"\nSummary: {len(missing)} missing episodes out of {len(crc32_to_link)} total found on Nyaa.\n" ) + + return missing, crc32_to_text, crc32_to_link, crc32_to_magnet, last_checked_page + + +def _report_new_missing_episodes(missing, crc32_to_text): + """Report newly detected missing episodes.""" + old_missing_crc32s = _load_old_missing_crc32s() + new_crc32s = set(missing) - old_missing_crc32s + if new_crc32s: + print(f"New missing episodes detected since last export: {len(new_crc32s)}") + for crc32 in new_crc32s: + title = crc32_to_text.get(crc32, "(Unknown Title)") + print(f"Missing: {title}") + + +def _generate_missing_episodes_report(conn, folder, args): + """Generate and save missing episodes report.""" + last_run = _print_report_header(conn, folder, args) + + missing, crc32_to_text, crc32_to_link, crc32_to_magnet, last_checked_page = ( + _calculate_and_find_missing(folder, conn, args, last_run) + ) - # Check for new CRC32 in missing compared to old file if exists - old_missing_crc32s = set() - if os.path.exists("Ace-Pace_Missing.csv"): - with open("Ace-Pace_Missing.csv", "r", encoding="utf-8") as f: - reader = csv.reader(f) - next(reader, None) # skip header - for row in reader: - if len(row) >= 1: - title = row[0] - # Extract CRC32 from title if possible - matches = CRC32_REGEX.findall(title) - if matches: - old_missing_crc32s.add(matches[-1].upper()) - new_crc32s = set(missing) - old_missing_crc32s - if new_crc32s: - print(f"New missing episodes detected since last export: {len(new_crc32s)}") - for crc32 in new_crc32s: - title = crc32_to_text.get(crc32, "(Unknown Title)") - print(f"Missing: {title}") - - with open("Ace-Pace_Missing.csv", "w", encoding="utf-8", newline="") as f: - writer = csv.writer(f, quoting=csv.QUOTE_ALL) - writer.writerow(["Title", "Page Link", "Magnet Link"]) - for crc32 in missing: - title = crc32_to_text[crc32] - page_link = crc32_to_link[crc32] - magnet = crc32_to_magnet.get(crc32, "") - writer.writerow([title, page_link, magnet]) + _report_new_missing_episodes(missing, crc32_to_text) - print("Missing files list saved to Ace-Pace_Missing.csv") + _save_missing_episodes_csv(missing, crc32_to_text, crc32_to_link, crc32_to_magnet) set_metadata(conn, "last_checked_page", str(last_checked_page)) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") set_metadata(conn, "last_missing_export", now_str) - if missing: - prompt = ( - input( - "Do you want to add missing episodes to a BitTorrent client now? (y/n): " - ) - .strip() - .lower() + return missing, crc32_to_text + + +def _parse_arguments(): + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Find missing episodes from your personal One Pace library." + ) + parser.add_argument( + "--url", + default=f"{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace+1080p&o=asc", + help=f"Base URL without the page param. Example: '{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace&o=asc' ", + ) + parser.add_argument("--folder", help="Folder containing local video files.") + parser.add_argument( + "--db", action="store_true", help="Export database to CSV and exit." + ) + parser.add_argument( + "--client", + choices=["transmission", "qbittorrent"], + help="The BitTorrent client to use.", + ) + parser.add_argument( + "--download", + action="store_true", + help="Import magnet links from missing CSV and add to the specified BitTorrent client.", + ) + parser.add_argument( + "--rename", + action="store_true", + help="Rename local files based on CRC32 matching titles from Nyaa.", + ) + parser.add_argument( + "--episodes_update", + action="store_true", + help="Update episodes metadata database from Nyaa.", + ) + parser.add_argument("--host", default="localhost", help="The BitTorrent client host.") + parser.add_argument("--port", type=int, help="The BitTorrent client port.") + parser.add_argument("--username", help="The BitTorrent client username.") + parser.add_argument("--password", help="The BitTorrent client password.") + parser.add_argument("--download-folder", help="The folder to download the torrents to.") + parser.add_argument("--tag", action="append", help="Tag to add to the torrent in qBittorrent (can be used multiple times).") + parser.add_argument("--category", help="Category to add to the torrent in qBittorrent.") + return parser.parse_args() + + +def _validate_url(url): + """Validate that URL points to a valid Nyaa domain.""" + if not url.startswith((NYAA_BASE_URL, "https://nyaa.land")): + print( + f"Error: The --url argument must point to a valid Nyaa website ({NYAA_BASE_URL} or https://nyaa.land)." ) - if prompt == "y": - client = ( - input("Enter client name (currently supported: transmission): ") - .strip() - .lower() - ) - if client: - download_missing_to_client(client) - else: - print("No client specified. Skipping download.") + return False + return True + + +def _show_episodes_metadata_status(): + """Show last episodes metadata update status.""" + episodes_db_conn = init_episodes_db() + last_ep_update = get_episodes_metadata(episodes_db_conn, "episodes_db_last_update") + if last_ep_update: + print(f"Episodes metadata last updated: {last_ep_update}") + else: + print("Episodes metadata database not yet updated.") + episodes_db_conn.close() + + +def _handle_main_commands(args, conn, folder): + """Handle main command execution.""" + if args.download: + _handle_download_command(args) + return + + if args.rename: + _handle_rename_command(conn) + return + + if not folder: + print("Error: --folder argument is required.") + return + + if args.db: + export_db_to_csv(conn) + return + + _generate_missing_episodes_report(conn, folder, args) + + # Note: To download missing episodes, use --download flag with --client + + +def main(): + args = _parse_arguments() + + if not _validate_url(args.url): + return + + _show_episodes_metadata_status() + + if args.episodes_update: + update_episodes_index_db() + return + + conn = init_db() + + # Folder selection logic: Always prompt if folder is required but not given + needs_folder = not args.download # All commands except --download need folder + folder = _get_folder_from_args(args, conn, needs_folder) + if folder is None: + return + + _handle_main_commands(args, conn, folder) if __name__ == "__main__": diff --git a/spec.md b/spec.md index 3434873..00a580b 100644 --- a/spec.md +++ b/spec.md @@ -30,10 +30,12 @@ - Tracks file paths and their corresponding checksums ### 3. Missing Episode Detection -- Compares local CRC32 checksums against the episodes index +- Fetches episode list from Nyaa.si using the provided URL (default: One-Pace 1080p search) +- Compares local CRC32 checksums against fetched episodes - Generates a CSV report (`Ace-Pace_Missing.csv`) listing missing episodes - Includes title, page link, and magnet link for each missing episode -- Tracks new missing episodes since last export +- Tracks new missing episodes since last export by comparing with previous CSV +- Note: Uses `fetch_crc32_links()` for real-time fetching, not the cached episodes index ### 4. Automated Downloading - Integrates with BitTorrent clients (Transmission, qBittorrent) @@ -149,11 +151,13 @@ Ace-Pace/ ### Standard Workflow 1. User runs script with `--folder` to scan local library -2. Script calculates/retrieves CRC32 checksums for local files -3. Script fetches episode list from Nyaa.si (or uses cached index) -4. Script compares local CRC32s against episode index -5. Script generates `Ace-Pace_Missing.csv` with missing episodes -6. User optionally runs `--download` to add missing episodes to BitTorrent client +2. Script validates URL and shows episodes metadata status +3. Script calculates/retrieves CRC32 checksums for local files +4. Script fetches episode list from Nyaa.si using `fetch_crc32_links()` +5. Script compares local CRC32s against fetched episodes +6. Script generates `Ace-Pace_Missing.csv` with missing episodes +7. Script reports new missing episodes since last export +8. User optionally runs `--download --client ` to add missing episodes to BitTorrent client ### Episodes Index Update Workflow 1. User runs `--episodes_update` to refresh episodes database @@ -213,9 +217,8 @@ Ace-Pace/ ### User Prompts - Folder selection (with last folder suggestion) -- Episodes index update confirmation +- Episodes index update confirmation (when using `--rename`) - File renaming confirmation -- BitTorrent client selection (legacy prompt) ## Future Considerations @@ -233,11 +236,40 @@ Ace-Pace/ ### Technical Improvements - Async/await for concurrent web scraping - Better error recovery and retry logic -- Unit tests and integration tests +- Unit tests and integration tests (✅ implemented) - Logging framework instead of print statements - Type hints for better code documentation - Configuration validation - Better handling of edge cases in filename parsing +- Code refactoring for reduced cognitive complexity (✅ completed) + +## Code Architecture + +### Function Organization +The codebase follows a modular structure with clear separation of concerns: + +#### Public API Functions +- `main()`: Entry point for the application +- `fetch_episodes_metadata()`: Fetches episodes from Nyaa.si +- `update_episodes_index_db()`: Updates the episodes index database +- `fetch_crc32_links()`: Fetches CRC32 links from a Nyaa.si URL +- `fetch_title_by_crc32()`: Searches for a title by CRC32 +- `calculate_local_crc32()`: Calculates CRC32 for local files +- `rename_local_files()`: Renames local files based on episodes index +- `export_db_to_csv()`: Exports database to CSV +- `load_crc32_to_title_from_index()`: Loads CRC32-to-title mapping + +#### Private Helper Functions (prefixed with `_`) +Helper functions are prefixed with `_` to indicate they are internal implementation details: + +- **Extraction functions**: `_extract_*` - Extract data from HTML/structures +- **Processing functions**: `_process_*` - Process data structures +- **Validation functions**: `_is_*`, `_validate_*` - Validate inputs/data +- **Command handlers**: `_handle_*` - Handle specific command-line operations +- **Utility functions**: `_get_*`, `_load_*`, `_save_*`, `_print_*`, `_report_*` - Utility operations +- **Workflow functions**: `_generate_*`, `_calculate_*` - Orchestrate multi-step workflows + +This naming convention improves code readability and makes the public API clear. ## Development Guidelines @@ -246,6 +278,8 @@ Ace-Pace/ - Use descriptive variable names - Add docstrings for functions - Keep functions focused and single-purpose +- Use `_` prefix for private/internal helper functions +- Maintain cognitive complexity ≤ 15 per function ### Database Schema - Use SQLite for simplicity From d7742078506a7594c66e48cf04b85d898e632132 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 20 Jan 2026 16:20:54 +0000 Subject: [PATCH 35/75] Refactoring fix --- acepace.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/acepace.py b/acepace.py index e483f7a..a1347cb 100644 --- a/acepace.py +++ b/acepace.py @@ -315,7 +315,8 @@ def set_metadata(conn, key, value): def _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): - """Process a single table row to extract CRC32 information.""" + """Process a single table row to extract CRC32 information. + Returns tuple: (success: bool, filename_text: str or None)""" links = row.find_all("a", href=True) title_link = None magnet_link = "" @@ -326,7 +327,7 @@ def _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): if href.startswith("magnet:"): magnet_link = href if not title_link: - return False # Skip rows without a valid title link + return False, None # Skip rows without a valid title link filename_text = title_link.text link = NYAA_BASE_URL + title_link["href"] matches = CRC32_REGEX.findall(filename_text) @@ -335,10 +336,9 @@ def _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): crc32_to_link[crc32] = link crc32_to_text[crc32] = filename_text crc32_to_magnet[crc32] = magnet_link - return True + return True, filename_text else: - print(f"Warning: No CRC32 found in title '{filename_text}'") - return False + return False, filename_text def fetch_crc32_links(base_url): @@ -367,9 +367,10 @@ def fetch_crc32_links(base_url): found_in_page = False for row in rows: - if _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): + success, filename_text = _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet) + if success: found_in_page = True - else: + elif filename_text: print(f"Warning: No CRC32 found in title '{filename_text}'") if not found_in_page: From 598c28afe6bba6c23ceb1319d4696b5e039d3f8f Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 21 Jan 2026 11:22:34 +0000 Subject: [PATCH 36/75] Docker Compose fix --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b54aba7..a0fb548 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: ace-pace: build: . - image: ace-pace + image: timothe/ace-pace:latest # or timothe/ace-pace:dev for dev branch container_name: ace-pace volumes: - /path/to/OnePaceLibrary:/media:rw @@ -11,11 +11,11 @@ services: networks: - "proxy" environment: - - TZ=Europe/Berlin + - TZ=Europe/London - TORRENT_HOST=127.0.0.1 - TORRENT_PORT=9091 - TORRENT_CLIENT=transmission - - NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc + - NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc #- TORRENT_USER=admin #- TORRENT_PASSWORD=password - DB=true From cebb84fd668940cefa26fe500a0743e484631d0c Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 21 Jan 2026 15:26:55 +0000 Subject: [PATCH 37/75] Updated specs --- spec.md | 168 +++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 143 insertions(+), 25 deletions(-) diff --git a/spec.md b/spec.md index 00a580b..88c3f39 100644 --- a/spec.md +++ b/spec.md @@ -100,8 +100,20 @@ Ace-Pace/ ├── acepace.py # Main application entry point ├── clients.py # BitTorrent client abstraction layer ├── requirements.txt # Python dependencies -├── docker-compose.yml # Docker configuration (if applicable) +├── Dockerfile # Docker image definition +├── docker-compose.yml # Docker Compose configuration +├── entrypoint.sh # Docker entrypoint script ├── spec.md # This specification document +├── NAMING_CONVENTIONS.md # Function naming conventions documentation +├── pytest.ini # Pytest configuration +├── tests/ # Test suite +│ ├── conftest.py +│ ├── test_clients.py +│ ├── test_crc32.py +│ ├── test_database.py +│ ├── test_episodes.py +│ ├── test_file_operations.py +│ └── test_missing_detection.py ├── crc32_files.db # Local file checksum database (generated) ├── episodes_index.db # Episodes metadata database (generated) ├── Ace-Pace_Missing.csv # Missing episodes report (generated) @@ -114,6 +126,8 @@ Ace-Pace/ - `requests`: HTTP requests for web scraping and API calls - `beautifulsoup4`: HTML parsing for Nyaa.si scraping - `qbittorrent-api`: qBittorrent client integration +- `pytest` (>=7.0.0): Testing framework +- `pytest-mock` (>=3.10.0): Mocking utilities for tests - Standard library: `sqlite3`, `argparse`, `csv`, `datetime`, `os`, `re`, `zlib`, `getpass`, `time`, `abc` ### External Services @@ -125,6 +139,18 @@ Ace-Pace/ - Transmission (RPC API on port 9091 by default) - qBittorrent (Web API on port 8080 by default) +### Docker Support +- **Docker Mode**: Detected via `RUN_DOCKER` environment variable +- **Non-Interactive Operation**: In Docker mode, skips user prompts and uses defaults +- **Default Folder**: Uses `/media` as default folder in Docker mode +- **Environment Variables**: Supports configuration via Docker environment variables + - `TORRENT_CLIENT`: BitTorrent client type (transmission/qbittorrent) + - `TORRENT_HOST`: Client host address + - `TORRENT_PORT`: Client port number + - `TORRENT_USER`: Client authentication username + - `TORRENT_PASSWORD`: Client authentication password + - `RUN_DOCKER`: Flag to enable Docker mode (non-interactive) + ## Command-Line Interface ### Main Arguments @@ -168,19 +194,38 @@ Ace-Pace/ ### File Renaming Workflow 1. User runs `--rename` with `--folder` -2. Script prompts to update episodes index if outdated -3. Script loads CRC32-to-title mapping from episodes index -4. Script matches local files by CRC32 -5. Script generates rename plan and prompts for confirmation -6. Script renames files and updates database +2. Script checks episodes index update status +3. Script prompts to update episodes index if outdated (skipped in Docker mode) +4. Script loads CRC32-to-title mapping from episodes index +5. Script matches local files by CRC32 +6. Script generates rename plan and prompts for confirmation (auto-confirms in Docker mode) +7. Script renames files and updates database + +### Docker Workflow +1. Container starts with `RUN_DOCKER` environment variable set +2. Script operates in non-interactive mode (no user prompts) +3. Default folder is `/media` (configurable via volume mount) +4. BitTorrent client connection parameters read from environment variables +5. All user prompts automatically answered with defaults +6. Database files and CSV reports persist via volume mounts ## Integration Points ### BitTorrent Client Abstraction -- Abstract base class `Client` defines interface +- Abstract base class `Client` (in `clients.py`) defines interface using `abc.ABC` - Concrete implementations: `QBittorrentClient`, `TransmissionClient` -- Factory function `get_client()` instantiates appropriate client +- Factory function `get_client(client_name, host, port, username, password)` instantiates appropriate client - Methods: `add_torrents(magnets, download_folder, tags, category)` +- **qBittorrentClient**: + - Uses `qbittorrentapi` library for API access + - Checks for existing torrents by info hash to prevent duplicates + - Supports tags and categories + - Adds tags to existing torrents if they already exist +- **TransmissionClient**: + - Uses Transmission RPC API via HTTP requests + - Handles session ID management for authentication + - Does not support tags/categories (warns if provided) + - Uses `requests` library for API calls ### Database Management - SQLite databases for persistence @@ -220,6 +265,8 @@ Ace-Pace/ - Episodes index update confirmation (when using `--rename`) - File renaming confirmation +**Note**: In Docker mode (when `RUN_DOCKER` environment variable is set), user prompts are automatically answered with defaults to enable non-interactive operation. + ## Future Considerations ### Potential Enhancements @@ -243,6 +290,22 @@ Ace-Pace/ - Better handling of edge cases in filename parsing - Code refactoring for reduced cognitive complexity (✅ completed) +## Refactoring History + +### Code Refactoring (Completed) +The codebase underwent significant refactoring to reduce cognitive complexity and improve maintainability: + +- **Function Decomposition**: Large functions were broken down into smaller, focused helper functions +- **Naming Conventions**: Established consistent naming patterns for helper functions (see `NAMING_CONVENTIONS.md`) +- **Separation of Concerns**: Clear separation between public API and private implementation details +- **Docker Support**: Added Docker mode with non-interactive operation support +- **Environment Variable Support**: Added support for configuration via environment variables in Docker mode +- **Connection Parameter Abstraction**: Separated Docker and non-Docker connection parameter handling +- **Workflow Functions**: Created dedicated workflow functions for complex multi-step operations +- **Error Handling**: Improved error handling with better separation of concerns + +All helper functions follow the naming convention documented in `NAMING_CONVENTIONS.md`, making the codebase more maintainable and easier to understand. + ## Code Architecture ### Function Organization @@ -250,26 +313,76 @@ The codebase follows a modular structure with clear separation of concerns: #### Public API Functions - `main()`: Entry point for the application +- `init_db()`: Initializes the local CRC32 cache database +- `init_episodes_db()`: Initializes the episodes index database +- `get_metadata(conn, key)`: Retrieves metadata value from database +- `set_metadata(conn, key, value)`: Stores metadata value in database +- `get_episodes_metadata(conn, key)`: Retrieves episodes database metadata +- `set_episodes_metadata(conn, key, value)`: Stores episodes database metadata - `fetch_episodes_metadata()`: Fetches episodes from Nyaa.si - `update_episodes_index_db()`: Updates the episodes index database -- `fetch_crc32_links()`: Fetches CRC32 links from a Nyaa.si URL -- `fetch_title_by_crc32()`: Searches for a title by CRC32 -- `calculate_local_crc32()`: Calculates CRC32 for local files -- `rename_local_files()`: Renames local files based on episodes index -- `export_db_to_csv()`: Exports database to CSV +- `fetch_crc32_links(base_url)`: Fetches CRC32 links from a Nyaa.si URL +- `fetch_title_by_crc32(crc32)`: Searches for a title by CRC32 +- `calculate_local_crc32(folder, conn)`: Calculates CRC32 for local files +- `rename_local_files(conn)`: Renames local files based on episodes index +- `export_db_to_csv(conn)`: Exports database to CSV - `load_crc32_to_title_from_index()`: Loads CRC32-to-title mapping #### Private Helper Functions (prefixed with `_`) -Helper functions are prefixed with `_` to indicate they are internal implementation details: - -- **Extraction functions**: `_extract_*` - Extract data from HTML/structures -- **Processing functions**: `_process_*` - Process data structures -- **Validation functions**: `_is_*`, `_validate_*` - Validate inputs/data -- **Command handlers**: `_handle_*` - Handle specific command-line operations -- **Utility functions**: `_get_*`, `_load_*`, `_save_*`, `_print_*`, `_report_*` - Utility operations -- **Workflow functions**: `_generate_*`, `_calculate_*` - Orchestrate multi-step workflows - -This naming convention improves code readability and makes the public API clear. +Helper functions are prefixed with `_` to indicate they are internal implementation details. See `NAMING_CONVENTIONS.md` for detailed documentation. + +**Extraction functions** (`_extract_*`): Extract data from HTML/structures +- `_extract_title_link_from_row(row)`: Extracts title link from table row +- `_extract_filenames_from_folder_structure(filelist_div)`: Extracts filenames from folder structure +- `_extract_filenames_from_torrent_page(torrent_soup)`: Extracts filenames from torrent page +- `_extract_matching_titles_from_rows(rows, crc32)`: Extracts titles matching CRC32 + +**Processing functions** (`_process_*`): Process data structures +- `_process_fname_entry(fname_text, ...)`: Processes filename entry to extract CRC32 +- `_process_torrent_page(page_link, ...)`: Processes torrent page to extract CRC32 +- `_process_episode_row(row, ...)`: Processes table row to extract episode info +- `_process_crc32_row(row, ...)`: Processes row to extract CRC32 for missing episodes + +**Validation functions** (`_is_*`, `_validate_*`): Validate inputs/data +- `_is_valid_quality(fname_text)`: Checks if filename has valid quality (1080p/720p) +- `_validate_url(url)`: Validates URL points to valid Nyaa domain + +**Command handlers** (`_handle_*`): Handle specific command-line operations +- `_handle_download_command(args)`: Handles the `--download` command +- `_handle_rename_command(conn)`: Handles the `--rename` command +- `_handle_main_commands(args, conn, folder)`: Routes and handles main commands + +**Getter functions** (`_get_*`): Retrieve or compute values +- `_get_total_pages(soup)`: Extracts total pages from pagination +- `_get_folder_from_args(args, conn, needs_folder)`: Gets folder from args or prompts +- `_get_client_from_args_or_env(args)`: Gets client type from args or env vars +- `_get_default_port(client)`: Gets default port for client +- `_get_docker_connection_params(args)`: Gets connection params from Docker env vars +- `_get_non_docker_connection_params(args)`: Gets connection params from CLI args +- `_get_rename_confirmation()`: Gets user confirmation for renaming +- `_get_rename_prompt(last_ep_update)`: Gets prompt for updating episodes DB + +**Load/Save functions** (`_load_*`, `_save_*`): Load or save data +- `_load_magnet_links()`: Loads magnet links from missing CSV +- `_load_old_missing_crc32s()`: Loads CRC32s from previous missing CSV +- `_save_missing_episodes_csv(...)`: Saves missing episodes to CSV + +**Print/Report functions** (`_print_*`, `_report_*`, `_show_*`): Display information +- `_print_report_header(conn, folder, args)`: Prints report header +- `_report_new_missing_episodes(missing, crc32_to_text)`: Reports new missing episodes +- `_show_episodes_metadata_status()`: Shows episodes metadata update status + +**Workflow functions** (`_generate_*`, `_calculate_*`): Orchestrate multi-step workflows +- `_generate_missing_episodes_report(conn, folder, args)`: Generates missing episodes report +- `_calculate_and_find_missing(folder, conn, args, last_run)`: Calculates CRC32s and finds missing + +**Utility functions** (`_parse_*`, `_count_*`, `_build_*`, `_execute_*`): General utilities +- `_parse_arguments()`: Parses command-line arguments +- `_count_video_files(folder, conn)`: Counts video files and recorded files +- `_build_rename_plan(entries, crc32_to_title)`: Builds plan of files to rename +- `_execute_rename(rename_plan, conn)`: Executes rename plan and updates DB + +This naming convention improves code readability and makes the public API clear. All helper functions follow consistent patterns documented in `NAMING_CONVENTIONS.md`. ## Development Guidelines @@ -299,9 +412,14 @@ When working on this project: 2. **Nyaa.si structure** - Understand the HTML structure of Nyaa.si pages for scraping 3. **Database state** - Always consider existing database state when making changes 4. **File paths** - Handle both absolute and relative paths correctly -5. **User interaction** - Some operations require user confirmation (renaming, downloads) -6. **Client abstraction** - New BitTorrent clients should implement the `Client` interface +5. **User interaction** - Some operations require user confirmation (renaming, downloads), but are auto-confirmed in Docker mode +6. **Client abstraction** - New BitTorrent clients should implement the `Client` interface from `clients.py` 7. **Error tolerance** - The tool should continue processing even if individual items fail 8. **Performance** - CRC32 calculation can be slow; caching is essential 9. **Web scraping** - Be respectful with rate limiting and error handling 10. **File naming** - Sanitize filenames to be filesystem-safe across platforms +11. **Docker mode** - Check `IS_DOCKER` flag (from `RUN_DOCKER` env var) to determine if running in Docker +12. **Function naming** - Follow the naming conventions in `NAMING_CONVENTIONS.md` when adding new helper functions +13. **Code complexity** - Maintain cognitive complexity ≤ 15 per function (refactoring completed) +14. **Testing** - Comprehensive test suite exists in `tests/` directory +15. **Environment variables** - In Docker mode, prefer environment variables over CLI args for configuration From 731ced463e29a4f93c4818a032129deefba27035 Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 22 Jan 2026 09:11:49 +0000 Subject: [PATCH 38/75] Removed image cache tag --- .github/workflows/docker-build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index e1b203e..e111a53 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -41,5 +41,3 @@ jobs: context: . push: true tags: timothe/ace-pace:${{ steps.meta.outputs.tag }} - cache-from: type=registry,ref=timothe/ace-pace:buildcache - cache-to: type=registry,ref=timothe/ace-pace:buildcache,mode=max From 29bbf13f22d7819b819279f1acc7178171bd11b6 Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 22 Jan 2026 11:43:23 +0000 Subject: [PATCH 39/75] Minor linter fix --- acepace.py | 13 ++++++------- pyrightconfig.json | 12 ++++++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 pyrightconfig.json diff --git a/acepace.py b/acepace.py index eb64826..33ae290 100644 --- a/acepace.py +++ b/acepace.py @@ -1,4 +1,3 @@ -import getpass import time import csv from datetime import datetime @@ -7,8 +6,8 @@ import argparse import zlib import os -from bs4 import BeautifulSoup -import requests +from bs4 import BeautifulSoup # type: ignore +import requests # type: ignore from clients import get_client @@ -142,7 +141,7 @@ def _get_total_pages(soup): if text.isdigit(): try: page_numbers.append(int(text)) - except Exception: + except (ValueError, TypeError): pass if page_numbers: total_pages = max(page_numbers) @@ -209,7 +208,7 @@ def _process_torrent_page(page_link, seen_crc32, episodes): if _process_fname_entry(str(fname), seen_crc32, episodes, page_link): found = True return found - except Exception: + except (requests.RequestException, AttributeError, TypeError): return False @@ -265,7 +264,7 @@ def fetch_episodes_metadata(): table = page_soup.find("table", class_="torrent-list") if not table: break - rows = table.find_all("tr") + rows = table.find_all("tr") # type: ignore for row in rows: _process_episode_row(row, seen_crc32, episodes) page += 1 @@ -411,7 +410,7 @@ def fetch_title_by_crc32(crc32): table = soup.find("table", class_="torrent-list") if not table: return None - rows = table.find_all("tr") + rows = table.find_all("tr") # type: ignore matched_titles = _extract_matching_titles_from_rows(rows, crc32) if len(matched_titles) == 1: diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..9926555 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,12 @@ +{ + "include": [ + "." + ], + "exclude": [ + "**/__pycache__", + "**/node_modules" + ], + "reportMissingImports": "warning", + "reportMissingTypeStubs": false, + "pythonVersion": "3.8" +} From 7237412e625c915906bd85c63f828771b7768bad Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 22 Jan 2026 15:07:00 +0000 Subject: [PATCH 40/75] More refactoring --- clients.py | 132 +++++++++++++++++++------------- tests/test_file_operations.py | 5 +- tests/test_missing_detection.py | 2 +- 3 files changed, 80 insertions(+), 59 deletions(-) diff --git a/clients.py b/clients.py index 7281a55..cb6245a 100644 --- a/clients.py +++ b/clients.py @@ -1,13 +1,13 @@ import abc import getpass import time -import requests -import qbittorrentapi +import requests # type: ignore +import qbittorrentapi # type: ignore import re class Client(abc.ABC): @abc.abstractmethod - def add_torrents(self, torrents, download_folder=None, tags=None, category=None): + def add_torrents(self, magnets, download_folder=None, tags=None, category=None): pass class QBittorrentClient(Client): @@ -21,9 +21,38 @@ def __init__(self, host, port, username, password): try: self.client.auth_log_in() except qbittorrentapi.LoginFailed as e: - raise Exception(f"Failed to connect to qBittorrent: {e}") from e + raise ConnectionError(f"Failed to connect to qBittorrent: {e}") from e print("Connection to qBittorrent successful!") + def _extract_info_hash(self, magnet): + """Extract info hash from magnet link.""" + match = re.search(r"xt=urn:btih:([a-fA-F0-9]{40})", magnet) + if not match: + return None + return match.group(1).lower() + + def _handle_existing_torrent(self, info_hash, tags_str, truncated): + """Handle case when torrent already exists.""" + print(f"Torrent {truncated} already exists.") + if tags_str: + print(f"Adding tags to existing torrent: {tags_str}") + self.client.torrents_add_tags(tags=tags_str, torrent_hashes=info_hash) + + def _add_new_torrent(self, magnet, download_folder, tags_str, category, truncated): + """Add a new torrent to qBittorrent.""" + print(f"Adding new torrent: {truncated}") + try: + self.client.torrents_add( + urls=magnet, + save_path=download_folder if download_folder else None, + tags=tags_str, + category=category, + ) + return True + except Exception as e: + print(f"Failed to add torrent: {truncated} Error: {e}") + return False + def add_torrents(self, magnets, download_folder=None, tags=None, category=None): if tags: self.client.torrents_create_tags(tags=",".join(tags)) @@ -35,34 +64,17 @@ def add_torrents(self, magnets, download_folder=None, tags=None, category=None): truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") print(f"Processing {idx}/{total}: {truncated}") - # Extract info hash from magnet link - match = re.search(r"xt=urn:btih:([a-fA-F0-9]{40})", magnet) - if not match: + info_hash = self._extract_info_hash(magnet) + if not info_hash: print(f"Could not find info hash in magnet link: {truncated}") continue - info_hash = match.group(1).lower() - - # Check if torrent already exists existing_torrent = self.client.torrents_info(torrent_hashes=info_hash) - if existing_torrent: - print(f"Torrent {truncated} already exists.") - if tags: - print(f"Adding tags to existing torrent: {tags_str}") - self.client.torrents_add_tags(tags=tags_str, torrent_hashes=info_hash) + self._handle_existing_torrent(info_hash, tags_str, truncated) else: - print(f"Adding new torrent: {truncated}") - try: - self.client.torrents_add( - urls=magnet, - save_path=download_folder if download_folder else None, - tags=tags_str, - category=category, - ) + if self._add_new_torrent(magnet, download_folder, tags_str, category, truncated): added_count += 1 - except Exception as e: - print(f"Failed to add torrent: {truncated} Error: {e}") time.sleep(0.1) print(f"Added {added_count} new torrents to qBittorrent.") @@ -83,19 +95,50 @@ def __init__(self, host, port, username, password): self.base_url, auth=self.auth, headers=headers, json={"method": "session-get"} ) if resp.status_code == 409: - self.session_id = resp.headers.get("X-Transmission-Session-Id") - headers["X-Transmission-Session-Id"] = self.session_id - resp = self.session.post( - self.base_url, auth=self.auth, headers=headers, json={"method": "session-get"} - ) + new_session_id = resp.headers.get("X-Transmission-Session-Id") + if new_session_id: + self.session_id = new_session_id + headers["X-Transmission-Session-Id"] = self.session_id + resp = self.session.post( + self.base_url, auth=self.auth, headers=headers, json={"method": "session-get"} + ) resp.raise_for_status() - except Exception as e: - raise Exception(f"Failed to connect to Transmission RPC: {e}") from e + except (requests.RequestException, ValueError) as e: + raise ConnectionError(f"Failed to connect to Transmission RPC: {e}") from e print("Connection to Transmission successful!") self.session_info = resp.json() + def _make_rpc_request(self, payload): + """Make an RPC request to Transmission, handling session ID updates.""" + headers = {"X-Transmission-Session-Id": self.session_id} if self.session_id else {} + resp = self.session.post(self.base_url, auth=self.auth, headers=headers, json=payload) + if resp.status_code == 409: + new_session_id = resp.headers.get("X-Transmission-Session-Id") + if new_session_id: + self.session_id = new_session_id + headers["X-Transmission-Session-Id"] = self.session_id + resp = self.session.post(self.base_url, auth=self.auth, headers=headers, json=payload) + return resp + + def _add_single_torrent(self, magnet, download_folder, truncated): + """Add a single torrent to Transmission.""" + payload = {"method": "torrent-add", "arguments": {"filename": magnet}} + if download_folder: + payload["arguments"]["download-dir"] = download_folder + try: + resp = self._make_rpc_request(payload) + resp.raise_for_status() + result = resp.json() + if result.get("result") == "success": + return True + print(f"Failed to add torrent: {truncated} Error: {result.get('result')}") + return False + except Exception as e: + print(f"Failed to add torrent: {truncated} Error: {e}") + return False + def add_torrents(self, magnets, download_folder=None, tags=None, category=None): if tags or category: print("Warning: Transmission does not support tags or categories through this script.") @@ -104,28 +147,9 @@ def add_torrents(self, magnets, download_folder=None, tags=None, category=None): for idx, magnet in enumerate(magnets, 1): truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") print(f"Adding {idx}/{total}: {truncated}") - payload = {"method": "torrent-add", "arguments": {"filename": magnet}} - if download_folder: - payload["arguments"]["download-dir"] = download_folder - try: - headers = {"X-Transmission-Session-Id": self.session_id} if self.session_id else {} - resp = self.session.post(self.base_url, auth=self.auth, headers=headers, json=payload) - if resp.status_code == 409: - self.session_id = resp.headers.get("X-Transmission-Session-Id") - headers["X-Transmission-Session-Id"] = self.session_id - resp = self.session.post(self.base_url, auth=self.auth, headers=headers, json=payload) - resp.raise_for_status() - result = resp.json() - if result.get("result") == "success": - added_count += 1 - else: - print( - f"Failed to add torrent: {truncated} Error: {result.get('result')}" - ) - time.sleep(0.1) - except Exception as e: - print(f"Failed to add torrent: {truncated} Error: {e}") - + if self._add_single_torrent(magnet, download_folder, truncated): + added_count += 1 + time.sleep(0.1) print(f"Added {added_count} torrents to Transmission.") diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index fc6c717..10f5cf0 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -55,7 +55,6 @@ def test_rename_local_files_matches_by_crc32(self, temp_dir, sample_episode_data title = crc32_to_title.get(crc32) if title: dir_name = os.path.dirname(file_path) - ext = os.path.splitext(file_path)[1] sanitized_title = re.sub(r'[\\/*?:"<>|]', "", title).strip() new_filename = f"{sanitized_title}" new_path = os.path.join(dir_name, new_filename) @@ -85,8 +84,7 @@ def test_rename_skips_files_without_match(self, temp_dir): with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): conn = acepace.init_db() - crc32s = acepace.calculate_local_crc32(temp_dir, conn) - actual_crc32 = list(crc32s)[0] + acepace.calculate_local_crc32(temp_dir, conn) # Mock load_crc32_to_title_from_index to return empty/no match with patch('acepace.load_crc32_to_title_from_index') as mock_load: @@ -130,7 +128,6 @@ def test_export_db_to_csv(self, temp_dir): conn.commit() # Export to CSV - csv_path = os.path.join(temp_dir, "export.csv") with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): acepace.export_db_to_csv(conn) diff --git a/tests/test_missing_detection.py b/tests/test_missing_detection.py index f959eae..5d07a48 100644 --- a/tests/test_missing_detection.py +++ b/tests/test_missing_detection.py @@ -87,7 +87,7 @@ def test_fetch_crc32_links_from_nyaa(self, mock_get): mock_get.side_effect = [mock_response1, mock_response2] base_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace" - crc32_to_link, crc32_to_text, crc32_to_magnet, last_page = acepace.fetch_crc32_links(base_url) + crc32_to_link, crc32_to_text, crc32_to_magnet, _ = acepace.fetch_crc32_links(base_url) assert len(crc32_to_link) == 2 assert "A1B2C3D4" in crc32_to_link From f908997b69e89a86a49b7fa61ccd3ee1244c19df Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 22 Jan 2026 15:11:01 +0000 Subject: [PATCH 41/75] Added help command --- acepace.py | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) diff --git a/acepace.py b/acepace.py index 33ae290..371c91d 100644 --- a/acepace.py +++ b/acepace.py @@ -832,10 +832,110 @@ def _generate_missing_episodes_report(conn, folder, args): return missing, crc32_to_text +def _print_help(): + """Print detailed help information about all available commands.""" + help_text = """ +Ace-Pace - Find missing episodes from your personal One Pace library + +AVAILABLE COMMANDS: + + Main Operations: + (no flags) Generate missing episodes report + Scans local folder, calculates CRC32 hashes, and compares + with episodes available on Nyaa to find missing episodes. + Outputs results to Ace-Pace_Missing.csv + + --episodes_update Update episodes metadata database from Nyaa + Fetches all One Pace episodes from Nyaa and stores their + CRC32, title, and page link in the episodes index database. + This should be run periodically to keep the database current. + + --rename Rename local files based on CRC32 matching + Matches local video files with episodes in the database + and renames them to match the official episode titles. + Prompts to update episodes database if it's outdated. + + --db Export local CRC32 database to CSV + Exports the database of calculated CRC32 hashes for + local video files to Ace-Pace_DB.csv + + --download Download missing episodes via BitTorrent client + Reads magnet links from Ace-Pace_Missing.csv and adds + them to the specified BitTorrent client (requires --client) + + BitTorrent Client Options (for --download): + --client {transmission,qbittorrent} + Specify which BitTorrent client to use + Required when using --download + + --host HOST BitTorrent client host (default: localhost) + + --port PORT BitTorrent client port + Defaults: Transmission=9091, qBittorrent=8080 + + --username USERNAME BitTorrent client username (if required) + + --password PASSWORD BitTorrent client password (if required) + + --download-folder PATH Folder where torrents should be downloaded + Default: /media (in Docker) or client default + + --tag TAG Add tag to torrents in qBittorrent + Can be used multiple times to add multiple tags + + --category CATEGORY Add category to torrents in qBittorrent + + General Options: + --url URL Custom Nyaa search URL + Default: https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc + Must point to a valid Nyaa domain + + --folder PATH Folder containing local video files + If not specified, will prompt for input + In Docker mode, defaults to /media + +EXAMPLES: + + # Generate missing episodes report + python acepace.py --folder /path/to/videos + + # Update episodes database + python acepace.py --episodes_update + + # Rename local files to match episode titles + python acepace.py --folder /path/to/videos --rename + + # Download missing episodes to qBittorrent + python acepace.py --download --client qbittorrent --host localhost --port 8080 + + # Export database to CSV + python acepace.py --folder /path/to/videos --db + +For more information, visit: https://github.com/your-repo/ace-pace +""" + print(help_text) + + def _parse_arguments(): """Parse command-line arguments.""" parser = argparse.ArgumentParser( - description="Find missing episodes from your personal One Pace library." + description="Find missing episodes from your personal One Pace library.", + formatter_class=argparse.RawDescriptionHelpFormatter, + add_help=False, # Disable automatic help to use custom one + epilog=""" +Examples: + python acepace.py --folder /path/to/videos + python acepace.py --episodes_update + python acepace.py --rename --folder /path/to/videos + python acepace.py --download --client qbittorrent + +Use --help for detailed command descriptions. + """ + ) + parser.add_argument( + "--help", "-h", + action="store_true", + help="Show detailed help message with all available commands." ) parser.add_argument( "--url", @@ -849,7 +949,7 @@ def _parse_arguments(): parser.add_argument( "--client", choices=["transmission", "qbittorrent"], - help="The BitTorrent client to use.", + help="The BitTorrent client to use (required for --download).", ) parser.add_argument( "--download", @@ -923,6 +1023,11 @@ def _handle_main_commands(args, conn, folder): def main(): args = _parse_arguments() + # Show detailed help if requested + if args.help: + _print_help() + return + if IS_DOCKER: print("Running in Docker mode (non-interactive)") From 5f083f47f1a3b011f5e664a9ec2f29eaff0dfdf5 Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 22 Jan 2026 15:28:23 +0000 Subject: [PATCH 42/75] Coverage handling --- .coveragerc | 30 ++ .gitignore | 11 + README.md | 20 ++ coverage.xml | 764 +++++++++++++++++++++++++++++++++++++++++++++++ pytest.ini | 5 + requirements.txt | 2 + 6 files changed, 832 insertions(+) create mode 100644 .coveragerc create mode 100644 coverage.xml diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..843bd71 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,30 @@ +[run] +source = . +omit = + */tests/* + */test_*.py + */__pycache__/* + */venv/* + */.venv/* + */env/* + */site-packages/* + setup.py + conftest.py +branch = True +relative_files = True + +[report] +exclude_lines = + pragma: no cover + def __repr__ + raise AssertionError + raise NotImplementedError + if __name__ == .__main__.: + if TYPE_CHECKING: + @abc.abstractmethod + +[html] +directory = htmlcov + +[xml] +output = coverage.xml diff --git a/.gitignore b/.gitignore index c62d63e..9795fe7 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,14 @@ Ace-Pace_Missing.csv crc32_files.db check.sh sonar-project.properties + +# Coverage reports +.coverage +.coverage.* +htmlcov/ +.tox/ +*.cover +.hypothesis/ +.pytest_cache/ +# Keep coverage.xml for SonarQube (but can be regenerated) +# coverage.xml diff --git a/README.md b/README.md index 205ac4b..1705644 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,26 @@ pip install -r requirements.txt This will install all necessary packages to ensure Ace-Pace runs smoothly. +## 🧪 Running Tests + +To run the test suite with coverage: + +```bash +# Install test dependencies +pip install -r requirements.txt + +# Run tests with coverage +pytest + +# Or explicitly generate coverage report +pytest --cov=. --cov-report=xml --cov-report=html --cov-report=term-missing +``` + +This will generate: +- `coverage.xml` - Used by SonarQube for test coverage analysis +- `htmlcov/` - HTML coverage report (open `htmlcov/index.html` in a browser) +- Terminal output showing coverage summary + ## 🛠️ How to Use Run the script using Python with the following command: diff --git a/coverage.xml b/coverage.xml new file mode 100644 index 0000000..b675b65 --- /dev/null +++ b/coverage.xml @@ -0,0 +1,764 @@ + + + + + + + . + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pytest.ini b/pytest.ini index f091a6d..3676feb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,6 +7,11 @@ addopts = -v --strict-markers --tb=short + --cov=. + --cov-report=term-missing + --cov-report=xml + --cov-report=html + --cov-branch markers = unit: Unit tests integration: Integration tests diff --git a/requirements.txt b/requirements.txt index 6be6e43..ee7cdb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,5 @@ beautifulsoup4 qbittorrent-api pytest>=7.0.0 pytest-mock>=3.10.0 +pytest-cov>=4.0.0 +coverage>=7.0.0 From f27d8607a8fd32abbbdf6c65b2ddc1b6d9507786 Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 22 Jan 2026 15:56:24 +0000 Subject: [PATCH 43/75] Updated README for Docker --- README.md | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1705644..03dbaeb 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,81 @@ pip install -r requirements.txt This will install all necessary packages to ensure Ace-Pace runs smoothly. +## 🐳 Docker Usage + +Ace-Pace can also be run using Docker, which simplifies deployment and ensures consistent execution across different environments. + +### Using Docker Run + +You can run Ace-Pace using `docker run` with environment variables and volume mounts: + +```bash +docker run --rm \ + -v /path/to/OnePaceLibrary:/media:rw \ + -v $(pwd)/crc32_files.db:/app/crc32_files.db:rw \ + -v $(pwd)/episodes_index.db:/app/episodes_index.db:rw \ + -v $(pwd)/Ace-Pace_Missing.csv:/app/Ace-Pace_Missing.csv:rw \ + -e TZ=Europe/London \ + -e TORRENT_HOST=127.0.0.1 \ + -e TORRENT_PORT=9091 \ + -e TORRENT_CLIENT=transmission \ + -e NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc \ + -e DB=true \ + -e EPISODES_UPDATE=true \ + timothe/ace-pace:latest +``` + +### Using Docker Compose + +For easier management, you can use the provided `docker-compose.yml` file. First, edit the compose file to match your setup: + +1. Update the volume path for your One-Pace library: + ```yaml + volumes: + - /path/to/OnePaceLibrary:/media:rw + ``` + +2. Configure environment variables as needed (Torrent client settings, Nyaa URL, etc.) + +3. Run with: + ```bash + docker-compose up + ``` + +Or run in detached mode: +```bash +docker-compose up -d +``` + +### Docker Environment Variables + +The following environment variables can be used to configure Ace-Pace in Docker: + +- `NYAA_URL` - Nyaa.si search URL (default: `https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc`) +- `TORRENT_CLIENT` - BitTorrent client type: `transmission` or `qbittorrent` (default: `transmission`) +- `TORRENT_HOST` - BitTorrent client host address (default: `127.0.0.1`) +- `TORRENT_PORT` - BitTorrent client port (default: `9091` for Transmission, `8080` for qBittorrent) +- `TORRENT_USER` - BitTorrent client username (optional) +- `TORRENT_PASSWORD` - BitTorrent client password (optional) +- `DB` - Set to `true` to generate CSV database export (default: `true`) +- `EPISODES_UPDATE` - Set to `true` to update episodes metadata from Nyaa (default: `true`) +- `TZ` - Timezone (default: `Europe/Berlin`) + +### Docker Volume Mounts + +The following volumes should be mounted for persistent data: + +- `/media` - Mount your One-Pace library directory here (read-write) +- `/app/crc32_files.db` - Database file for CRC32 checksums (read-write) +- `/app/episodes_index.db` - Database file for episodes index (read-write) +- `/app/Ace-Pace_Missing.csv` - CSV export of missing episodes (read-write) + +### Docker Notes + +- In Docker mode, Ace-Pace automatically uses `/media` as the default folder path +- The container runs non-interactively, so all configuration must be provided via environment variables +- Make sure your BitTorrent client is accessible from within the Docker network (use host network mode or configure networking appropriately) + ## 🧪 Running Tests To run the test suite with coverage: @@ -46,21 +121,23 @@ python acepace.py [-h] [--url URL] [--folder FOLDER] [--db] [--client {transmiss ``` ### 🔭 Main commands -- `--folder ` + +- `--folder ` (required for most cases) Specify the path to your local One-Pace video library. Ace-Pace will scan this directory recursively to identify and analyze your existing episodes. - `--url ` Define the Nyaa URL used for the query to get episodes metadata and download links. Defaults to `https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc`. -- `--db` +- `--db` (standalone flag) Create a CSV file with the existing local file paths and CRC32 checksums. Useful to check what's detected and debugging. ### 📥 Download commands + - `--client ` Specify the BitTorrent client to use for downloading missing episodes. Supported clients: `transmission`, `qbittorrent`. -- `--download` +- `--download` (standalone flag) Enable downloading of missing episodes using the specified BitTorrent client. - `--host ` @@ -84,7 +161,6 @@ python acepace.py [-h] [--url URL] [--folder FOLDER] [--db] [--client {transmiss - `--category ` Category to add to the torrent in qBittorrent. - ### 📚 Some examples ``` From 8770e6659cf1a637088a645e2f3377f287b9f95e Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 23 Jan 2026 09:28:10 +0000 Subject: [PATCH 44/75] Switch to full folder for Docker mode --- Dockerfile | 3 +- acepace.py | 72 +++- coverage.xml | 839 +++++++++++++++++++++++---------------------- docker-compose.yml | 4 +- 4 files changed, 485 insertions(+), 433 deletions(-) diff --git a/Dockerfile b/Dockerfile index 456b2c3..bfb377f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,6 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && pip install --no-cache-dir -r /app/requirements.txt \ - && chmod +x /app/entrypoint.sh \ - && touch /app/Ace-Pace_Missing.csv + && chmod +x /app/entrypoint.sh CMD ["/app/entrypoint.sh"] diff --git a/acepace.py b/acepace.py index 371c91d..5003261 100644 --- a/acepace.py +++ b/acepace.py @@ -24,18 +24,49 @@ # Video file extensions we care about VIDEO_EXTENSIONS = {".mkv", ".mp4", ".avi"} -DB_NAME = "crc32_files.db" -EPISODES_DB_NAME = "episodes_index.db" - # Constants for repeated string literals HTML_PARSER = "html.parser" -MISSING_CSV_FILENAME = "Ace-Pace_Missing.csv" NYAA_BASE_URL = "https://nyaa.si" +# Config directory and file names +CONFIG_DIR_DOCKER = "/config" +CONFIG_DIR_LOCAL = "." +DB_NAME = "crc32_files.db" +EPISODES_DB_NAME = "episodes_index.db" +MISSING_CSV_FILENAME = "Ace-Pace_Missing.csv" +DB_CSV_FILENAME = "Ace-Pace_DB.csv" + + +def get_config_dir(): + """Get the config directory path based on Docker mode. + Returns the config directory path, creating it if necessary.""" + if IS_DOCKER: + config_dir = CONFIG_DIR_DOCKER + else: + config_dir = CONFIG_DIR_LOCAL + + # Ensure config directory exists + if not os.path.exists(config_dir): + os.makedirs(config_dir, exist_ok=True) + + return config_dir + + +def get_config_path(filename): + """Get the full path to a config file. + Args: + filename: The name of the config file + Returns: + Full path to the config file in the appropriate config directory + """ + config_dir = get_config_dir() + return os.path.join(config_dir, filename) + def init_db(): - exists = os.path.exists(DB_NAME) - conn = sqlite3.connect(DB_NAME) + db_path = get_config_path(DB_NAME) + exists = os.path.exists(db_path) + conn = sqlite3.connect(db_path) c = conn.cursor() c.execute( """ @@ -61,7 +92,8 @@ def init_db(): # --- New: Episodes metadata DB --- def init_episodes_db(): - conn = sqlite3.connect(EPISODES_DB_NAME) + episodes_db_path = get_config_path(EPISODES_DB_NAME) + conn = sqlite3.connect(episodes_db_path) c = conn.cursor() c.execute( """ @@ -362,7 +394,7 @@ def fetch_crc32_links(base_url): print("No table found, stopping.") break - rows = table.find_all("tr") + rows = table.find_all("tr") # type: ignore if not rows: print("No rows found, stopping.") break @@ -540,12 +572,13 @@ def export_db_to_csv(conn): c = conn.cursor() c.execute("SELECT file_path, crc32 FROM crc32_cache") rows = c.fetchall() - with open("Ace-Pace_DB.csv", "w", encoding="utf-8", newline="") as f: + export_csv_path = get_config_path(DB_CSV_FILENAME) + with open(export_csv_path, "w", encoding="utf-8", newline="") as f: writer = csv.writer(f, quoting=csv.QUOTE_ALL) writer.writerow(["File Path", "CRC32"]) for row in rows: writer.writerow(row) - print("Database exported to Ace-Pace_DB.csv") + print(f"Database exported to {export_csv_path}") now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") set_metadata(conn, "last_db_export", now_str) @@ -622,12 +655,13 @@ def _get_non_docker_connection_params(args): def _load_magnet_links(): """Load magnet links from the missing CSV file.""" - if not os.path.exists(MISSING_CSV_FILENAME): - print(f"Missing file '{MISSING_CSV_FILENAME}' not found. Run the script first!") + missing_csv_path = get_config_path(MISSING_CSV_FILENAME) + if not os.path.exists(missing_csv_path): + print(f"Missing file '{missing_csv_path}' not found. Run the script first!") return None magnets = [] - with open(MISSING_CSV_FILENAME, "r", encoding="utf-8") as f: + with open(missing_csv_path, "r", encoding="utf-8") as f: reader = csv.DictReader(f) for row in reader: magnet_link = row.get("Magnet Link", "").strip() @@ -635,7 +669,7 @@ def _load_magnet_links(): magnets.append(magnet_link) if not magnets: - print(f"No magnet links found in '{MISSING_CSV_FILENAME}'.") + print(f"No magnet links found in '{missing_csv_path}'.") return None return magnets @@ -726,8 +760,9 @@ def _count_video_files(folder, conn): def _load_old_missing_crc32s(): """Load CRC32s from previous missing CSV file.""" old_missing_crc32s = set() - if os.path.exists(MISSING_CSV_FILENAME): - with open(MISSING_CSV_FILENAME, "r", encoding="utf-8") as f: + missing_csv_path = get_config_path(MISSING_CSV_FILENAME) + if os.path.exists(missing_csv_path): + with open(missing_csv_path, "r", encoding="utf-8") as f: reader = csv.reader(f) next(reader, None) # skip header for row in reader: @@ -742,7 +777,8 @@ def _load_old_missing_crc32s(): def _save_missing_episodes_csv(missing, crc32_to_text, crc32_to_link, crc32_to_magnet): """Save missing episodes to CSV file.""" - with open(MISSING_CSV_FILENAME, "w", encoding="utf-8", newline="") as f: + missing_csv_path = get_config_path(MISSING_CSV_FILENAME) + with open(missing_csv_path, "w", encoding="utf-8", newline="") as f: writer = csv.writer(f, quoting=csv.QUOTE_ALL) writer.writerow(["Title", "Page Link", "Magnet Link"]) for crc32 in missing: @@ -750,7 +786,7 @@ def _save_missing_episodes_csv(missing, crc32_to_text, crc32_to_link, crc32_to_m page_link = crc32_to_link[crc32] magnet = crc32_to_magnet.get(crc32, "") writer.writerow([title, page_link, magnet]) - print(f"Missing files list saved to {MISSING_CSV_FILENAME}") + print(f"Missing files list saved to {missing_csv_path}") def _print_report_header(conn, folder, args): diff --git a/coverage.xml b/coverage.xml index b675b65..7edf003 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -27,612 +27,631 @@ - - + + + - - - - - - - + + + + + + + + - - - - - + + + + + + - - + + - - - - - - - - - - - - + + + + + + - + - - - - + - - - - + - - - - - - + + + + + - + - - - + + + + - - - - - - - - - - - + + + + + + + + + + + + + + - - + - - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + - - - + + + - - - - + + + + - - - + + + + + + + + + + - - + + - - + + - - - + + + - - - - - - + + + + - - - - - - - + - + - + + - - + - - + + + - - + + - + + + - - + + - - + + + + + - - - + - - + + - - - + - - - - + + + - - - - - - - + + + + - - - + + + + - + - + + - + - - - - - - + + + + + + - - - + + + + + - - - - - - - + + + + + + + + - - - + + + - - - - + + - - - - - + + + - + + + - - - - - + + + - - + + + + - + + - - - + + - + - - + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - + + + + + + - - - - - - - - - + + + + + + + - - - + + + + - - - + + - - - + - - + + + + - - - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + - + - - - + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - + + + + + + + - - - + - - - - + + + + + - - - - - - - - - + + + + + + - - - - + + - - - + + + + + - - + + + - + - - - + + + + - + + + + + - - + + - - + + - + + - - - + + + + + - + + - - - - - - - - - - - - + + + + + + + + - - - - - + + - - - - - - + + + - - - - + + - + - - - - + + + + + - - + + + - - - + + + + - + + - - + + + + + + + - + + + + - + + - - - - - - - + + + + - - - + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + - - - - - - - - - - - - - + + + + + - + + - - - + + - - - + + + + - - + + - - - - + + + + - + - + + - - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index a0fb548..a3c678b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,9 +5,7 @@ services: container_name: ace-pace volumes: - /path/to/OnePaceLibrary:/media:rw - - ./crc32_files.db:/app/crc32_files.db:rw - - ./episodes_index.db:/app/episodes_index.db:rw - - ./Ace-Pace_Missing.csv:/app/Ace-Pace_Missing.csv:rw + - /path/to/config:/config:rw networks: - "proxy" environment: From 477e6307cb07b2c17e2972f02e79430dd9ff4dc3 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 23 Jan 2026 14:44:59 +0000 Subject: [PATCH 45/75] VPN mention --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 03dbaeb..df3f3fe 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,12 @@ The following volumes should be mounted for persistent data: - The container runs non-interactively, so all configuration must be provided via environment variables - Make sure your BitTorrent client is accessible from within the Docker network (use host network mode or configure networking appropriately) +### VPN Considerations + +If you're running Ace-Pace through a VPN container (such as Gluetun), you may encounter 429 (Too Many Requests) errors when querying Nyaa.si. This is because multiple requests from the same VPN exit node can trigger rate limiting. + +**Recommendation:** It's perfectly fine to run Ace-Pace without a VPN. Instead, keep your BitTorrent client behind the VPN to protect your downloads while allowing Ace-Pace to query Nyaa.si directly without rate limiting issues. + ## 🧪 Running Tests To run the test suite with coverage: From e656515a2cb2942559eb125971cee67a850f534e Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 23 Jan 2026 16:00:46 +0000 Subject: [PATCH 46/75] Clients error handling w updated tests --- acepace.py | 9 +- clients.py | 18 +- coverage.xml | 413 ++++++++++++++++++++++-------------------- tests/test_clients.py | 63 ++++++- 4 files changed, 300 insertions(+), 203 deletions(-) diff --git a/acepace.py b/acepace.py index 5003261..7b4b20a 100644 --- a/acepace.py +++ b/acepace.py @@ -700,8 +700,15 @@ def _handle_download_command(args): tags=args.tag, category=args.category, ) + except ConnectionError as e: + print(f"Connection Error: {e}") + print(f"Please verify that {client} is running and accessible at {host}:{port}") + return False + except ValueError as e: + print(f"Configuration Error: {e}") + return False except Exception as e: - print(f"Error: {e}") + print(f"Unexpected Error: {e}") return False return True diff --git a/clients.py b/clients.py index cb6245a..f7fa711 100644 --- a/clients.py +++ b/clients.py @@ -21,7 +21,13 @@ def __init__(self, host, port, username, password): try: self.client.auth_log_in() except qbittorrentapi.LoginFailed as e: - raise ConnectionError(f"Failed to connect to qBittorrent: {e}") from e + raise ConnectionError(f"Failed to authenticate with qBittorrent at {host}:{port}: {e}") from e + except qbittorrentapi.APIConnectionError as e: + raise ConnectionError(f"Failed to connect to qBittorrent at {host}:{port}. Check if the client is running and accessible: {e}") from e + except qbittorrentapi.APIError as e: + raise ConnectionError(f"qBittorrent API error at {host}:{port}: {e}") from e + except Exception as e: + raise ConnectionError(f"Unexpected error connecting to qBittorrent at {host}:{port}: {e}") from e print("Connection to qBittorrent successful!") def _extract_info_hash(self, magnet): @@ -103,8 +109,14 @@ def __init__(self, host, port, username, password): self.base_url, auth=self.auth, headers=headers, json={"method": "session-get"} ) resp.raise_for_status() - except (requests.RequestException, ValueError) as e: - raise ConnectionError(f"Failed to connect to Transmission RPC: {e}") from e + except requests.ConnectionError as e: + raise ConnectionError(f"Failed to connect to Transmission at {host}:{port}. Check if the client is running and accessible: {e}") from e + except requests.Timeout as e: + raise ConnectionError(f"Connection to Transmission at {host}:{port} timed out: {e}") from e + except requests.RequestException as e: + raise ConnectionError(f"Failed to connect to Transmission RPC at {host}:{port}: {e}") from e + except ValueError as e: + raise ConnectionError(f"Invalid response from Transmission at {host}:{port}: {e}") from e print("Connection to Transmission successful!") self.session_info = resp.json() diff --git a/coverage.xml b/coverage.xml index 7edf003..02f2706 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -488,173 +488,180 @@ + - - + + + + + - - - - - + + + + + + - - - - - - + + + + - - - + + + - - - - + - - + + - - + + + + + - - - - - - + + + + - + - - - + + + + + - - - - + - + + + + - + - - - - - + + + + - + - + + + - - + + - - + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - + + + + + - - - - + + + + - - - - - - + + + + + + - - - - + + + - - + + - + - - - - - - - - + + + + + + + + - - - - - - + + + + + + + - - - + + + + + + + - + @@ -672,109 +679,121 @@ + - - + + + - - - - + + + + - - - + + + + + + - - - - - - - - + + + + - + - - + + - - - - - + + + + + - - + + + - - + + - - - - - - + + + + + + - - - - - - - + + + + + + + + + - - - - - + + + + - - - + + - + - + - - - - - + + + + - - + + - - - - - - - - - - + + + + + + + + + + + - + + + + + + + + + + + diff --git a/tests/test_clients.py b/tests/test_clients.py index 2cee86d..3d5b7ee 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -34,10 +34,11 @@ def test_qbittorrent_client_init_login_failed(self, mock_client_class): mock_client.auth_log_in.side_effect = qbittorrentapi.LoginFailed("Invalid credentials") mock_client_class.return_value = mock_client - with pytest.raises(Exception) as exc_info: + with pytest.raises(ConnectionError) as exc_info: QBittorrentClient("localhost", 8080, "user", "pass") - assert "Failed to connect to qBittorrent" in str(exc_info.value) + assert "Failed to authenticate with qBittorrent" in str(exc_info.value) + assert "localhost:8080" in str(exc_info.value) @patch('clients.qbittorrentapi.Client') @patch('clients.time.sleep') @@ -93,6 +94,35 @@ def test_qbittorrent_add_torrents_invalid_magnet(self, mock_sleep, mock_client_c # Should not call torrents_add for invalid magnet mock_client.torrents_add.assert_not_called() + @patch('clients.qbittorrentapi.Client') + def test_qbittorrent_client_init_api_connection_error(self, mock_client_class): + """Test qBittorrent client initialization with API connection error.""" + import qbittorrentapi + mock_client = MagicMock() + mock_client.auth_log_in.side_effect = qbittorrentapi.APIConnectionError("Connection refused") + mock_client_class.return_value = mock_client + + with pytest.raises(ConnectionError) as exc_info: + QBittorrentClient("localhost", 8080, "user", "pass") + + assert "Failed to connect to qBittorrent" in str(exc_info.value) + assert "localhost:8080" in str(exc_info.value) + assert "Check if the client is running" in str(exc_info.value) + + @patch('clients.qbittorrentapi.Client') + def test_qbittorrent_client_init_api_error(self, mock_client_class): + """Test qBittorrent client initialization with general APIError.""" + import qbittorrentapi + mock_client = MagicMock() + mock_client.auth_log_in.side_effect = qbittorrentapi.APIError("General API error") + mock_client_class.return_value = mock_client + + with pytest.raises(ConnectionError) as exc_info: + QBittorrentClient("localhost", 8080, "user", "pass") + + assert "qBittorrent API error" in str(exc_info.value) + assert "localhost:8080" in str(exc_info.value) + class TestTransmissionClient: """Tests for Transmission client.""" @@ -183,6 +213,35 @@ def test_transmission_add_torrents_handles_409(self, mock_sleep, mock_session_cl assert client.session_id == "new_session_id" + @patch('clients.requests.Session') + def test_transmission_client_init_connection_error(self, mock_session_class): + """Test Transmission client initialization with connection error.""" + import requests + mock_session = MagicMock() + mock_session.post.side_effect = requests.ConnectionError("Connection refused") + mock_session_class.return_value = mock_session + + with pytest.raises(ConnectionError) as exc_info: + TransmissionClient("localhost", 9091, "user", "pass") + + assert "Failed to connect to Transmission" in str(exc_info.value) + assert "localhost:9091" in str(exc_info.value) + assert "Check if the client is running" in str(exc_info.value) + + @patch('clients.requests.Session') + def test_transmission_client_init_timeout(self, mock_session_class): + """Test Transmission client initialization with timeout.""" + import requests + mock_session = MagicMock() + mock_session.post.side_effect = requests.Timeout("Request timed out") + mock_session_class.return_value = mock_session + + with pytest.raises(ConnectionError) as exc_info: + TransmissionClient("localhost", 9091, "user", "pass") + + assert "timed out" in str(exc_info.value) + assert "localhost:9091" in str(exc_info.value) + class TestClientFactory: """Tests for client factory function.""" From 5f393660cfd26d61bab4c3b0ed8af5c9fcc74000 Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 26 Jan 2026 08:51:28 +0000 Subject: [PATCH 47/75] Fix torrent client issue in Docker --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 90ace59..f81f722 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,4 +11,4 @@ fi exec python /app/acepace.py \ ${NYAA_URL:+--url "$NYAA_URL"} \ - ${TORRENT_CLIENT:+--download "$TORRENT_CLIENT"} \ \ No newline at end of file + ${TORRENT_CLIENT:+--download --client "$TORRENT_CLIENT"} \ \ No newline at end of file From 5d45811c0f4e5521d579c6e47a7f4d046da15244 Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 26 Jan 2026 13:33:32 +0000 Subject: [PATCH 48/75] Fixed Docker issue w/ btt client & episodes discrepancy, hardened tests --- acepace.py | 50 +- coverage.xml | 846 ++++++++++++++++--------------- docker-compose.yml | 2 +- entrypoint.sh | 2 +- tests/test_crc32.py | 5 +- tests/test_file_operations.py | 38 ++ tests/test_missing_detection.py | 58 +++ tests/test_path_normalization.py | 413 +++++++++++++++ 8 files changed, 980 insertions(+), 434 deletions(-) create mode 100644 tests/test_path_normalization.py diff --git a/acepace.py b/acepace.py index 7b4b20a..c764355 100644 --- a/acepace.py +++ b/acepace.py @@ -63,6 +63,23 @@ def get_config_path(filename): return os.path.join(config_dir, filename) +def normalize_file_path(file_path): + """Normalize a file path for consistent storage and lookup. + Resolves symlinks and converts to absolute path to ensure the same file + always maps to the same path string, regardless of OS or environment. + Args: + file_path: The file path to normalize + Returns: + Normalized absolute path + """ + try: + # Use realpath to resolve symlinks and get canonical path + return os.path.realpath(os.path.abspath(file_path)) + except (OSError, ValueError): + # Fallback to abspath if realpath fails (e.g., file doesn't exist yet) + return os.path.normpath(os.path.abspath(file_path)) + + def init_db(): db_path = get_config_path(DB_NAME) exists = os.path.exists(db_path) @@ -350,6 +367,7 @@ def set_metadata(conn, key, value): def _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): """Process a single table row to extract CRC32 information. + Only accepts episodes with 1080p or 720p quality (720p as fallback). Returns tuple: (success: bool, filename_text: str or None)""" links = row.find_all("a", href=True) title_link = None @@ -366,13 +384,14 @@ def _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): link = NYAA_BASE_URL + title_link["href"] matches = CRC32_REGEX.findall(filename_text) if matches: - crc32 = matches[-1].upper() - crc32_to_link[crc32] = link - crc32_to_text[crc32] = filename_text - crc32_to_magnet[crc32] = magnet_link - return True, filename_text - else: - return False, filename_text + # Only accept episodes with valid quality (1080p or 720p) and One Pace marker + if "[One Pace]" in filename_text and _is_valid_quality(filename_text): + crc32 = matches[-1].upper() + crc32_to_link[crc32] = link + crc32_to_text[crc32] = filename_text + crc32_to_magnet[crc32] = magnet_link + return True, filename_text + return False, filename_text def fetch_crc32_links(base_url): @@ -464,9 +483,11 @@ def calculate_local_crc32(folder, conn): ext = os.path.splitext(file)[1].lower() if ext in VIDEO_EXTENSIONS: file_path = os.path.join(root, file) - # Check if file_path already in DB + # Normalize path for consistent storage and lookup + normalized_path = normalize_file_path(file_path) + # Check if normalized_path already in DB c.execute( - "SELECT crc32 FROM crc32_cache WHERE file_path = ?", (file_path,) + "SELECT crc32 FROM crc32_cache WHERE file_path = ?", (normalized_path,) ) row = c.fetchone() if row: @@ -484,7 +505,7 @@ def calculate_local_crc32(folder, conn): local_crc32s.add(crc32) c.execute( "INSERT OR REPLACE INTO crc32_cache (file_path, crc32) VALUES (?, ?)", - (file_path, crc32), + (normalized_path, crc32), ) conn.commit() return local_crc32s @@ -524,9 +545,12 @@ def _execute_rename(rename_plan, conn): continue os.rename(old, new) print(f"Renamed {old} to {new}") + # Normalize paths for consistent database updates + normalized_old = normalize_file_path(old) + normalized_new = normalize_file_path(new) # Update DB with new file path c.execute( - "UPDATE crc32_cache SET file_path = ? WHERE file_path = ?", (new, old) + "UPDATE crc32_cache SET file_path = ? WHERE file_path = ?", (normalized_new, normalized_old) ) conn.commit() except Exception as e: @@ -758,7 +782,9 @@ def _count_video_files(folder, conn): if ext in VIDEO_EXTENSIONS: total_files += 1 file_path = os.path.join(root, file) - c.execute("SELECT 1 FROM crc32_cache WHERE file_path = ?", (file_path,)) + # Normalize path for consistent lookup + normalized_path = normalize_file_path(file_path) + c.execute("SELECT 1 FROM crc32_cache WHERE file_path = ?", (normalized_path,)) if c.fetchone(): recorded_files += 1 return total_files, recorded_files diff --git a/coverage.xml b/coverage.xml index 02f2706..b877875 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -46,619 +46,629 @@ - - - - - - + + + + + + + + - - - - - + - - + + + + + + + - - - - - - - - - - - + + + + + - + - - + - - + - - - - + + - - + + + - - - - + - - - - - + + + + - - + + + - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - + - - - - - + + + + - - + + + + - - + + + + + + + - - + + - - - - - - - + + + - - - - + + - - + + - - - + - - + + - - - + + + - - + + + + + - - + + - + - - + - + + + - + + + - - - + - + - - - - - - - - + - + + + + - + - - + + - - - - + + + + - - - + + - + + - + - - - + + + - - - - + + + + + + + - + - - - - - - - - - - - + + + + + + + + - - - + + + - + + - - - + + + - - + + - - + + + + - - + - - + - + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - + + - - - + - - - + + + - + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + - - + + + - - - - + + + + + + - + - + - + - - - + + + - - - - + - - - + + + - + + - - - - - - + + + + + - - - + + + - + + - - + + + + - + - + + - + - - + - - - - + + + + + - - - - + + + + + + + - - - - - - - - - - - - - + + + + + + + + + - - - - + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + - - - - + + + + - - + + + + + - - - - + + + + + + - + + - - - - - + + + + - - + - - + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + - - - - - - + + - - - - - - + + + + + + + - - - - + + - - - - + + + + + + - - - - + + + + + - - + + - - - - - - + + + + + + + + + + + + + + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index a3c678b..133aeb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: - TORRENT_HOST=127.0.0.1 - TORRENT_PORT=9091 - TORRENT_CLIENT=transmission - - NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc + - NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc #- TORRENT_USER=admin #- TORRENT_PASSWORD=password - DB=true diff --git a/entrypoint.sh b/entrypoint.sh index f81f722..cdeeb2d 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,4 +11,4 @@ fi exec python /app/acepace.py \ ${NYAA_URL:+--url "$NYAA_URL"} \ - ${TORRENT_CLIENT:+--download --client "$TORRENT_CLIENT"} \ \ No newline at end of file + ${TORRENT_CLIENT:+--download --client "$TORRENT_CLIENT"} \ No newline at end of file diff --git a/tests/test_crc32.py b/tests/test_crc32.py index 32b37d9..f5e105b 100644 --- a/tests/test_crc32.py +++ b/tests/test_crc32.py @@ -88,9 +88,10 @@ def test_calculate_crc32_caches_result(self, sample_video_content, temp_dir): # Second calculation should use cache crc32s2 = acepace.calculate_local_crc32(temp_dir, conn) - # Verify cache was used (check database) + # Verify cache was used (check database with normalized path) + normalized_path = acepace.normalize_file_path(test_file) cursor = conn.cursor() - cursor.execute("SELECT crc32 FROM crc32_cache WHERE file_path = ?", (test_file,)) + cursor.execute("SELECT crc32 FROM crc32_cache WHERE file_path = ?", (normalized_path,)) row = cursor.fetchone() assert row is not None diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index 10f5cf0..b5d9244 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -106,6 +106,44 @@ def test_rename_skips_files_without_match(self, temp_dir): conn.close() + def test_rename_uses_normalized_paths(self, temp_dir): + """Test that rename operations use normalized paths in database.""" + test_file = os.path.join(temp_dir, "old_name.mkv") + with open(test_file, "wb") as f: + f.write(b"test video content") + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + + # Calculate CRC32 (stores normalized path) + acepace.calculate_local_crc32(temp_dir, conn) + + # Get the stored path from database + cursor = conn.cursor() + cursor.execute("SELECT file_path FROM crc32_cache") + stored_path = cursor.fetchone()[0] + + # Verify it's normalized + assert os.path.isabs(stored_path) + assert stored_path == acepace.normalize_file_path(test_file) + + # Mock rename scenario + with patch('acepace.load_crc32_to_title_from_index') as mock_load: + crc32s = acepace.calculate_local_crc32(temp_dir, conn) + actual_crc32 = list(crc32s)[0] + mock_load.return_value = {actual_crc32: "[One Pace] Episode 1 [1080p].mkv"} + + # Simulate rename + new_path = os.path.join(temp_dir, "[One Pace] Episode 1 [1080p].mkv") + normalized_old = acepace.normalize_file_path(test_file) + normalized_new = acepace.normalize_file_path(new_path) + + # Check that paths would be normalized in database update + assert normalized_old == stored_path # Should match what's in DB + assert os.path.isabs(normalized_new) + + conn.close() + class TestCSVExport: """Tests for CSV export functionality.""" diff --git a/tests/test_missing_detection.py b/tests/test_missing_detection.py index 5d07a48..2d350f5 100644 --- a/tests/test_missing_detection.py +++ b/tests/test_missing_detection.py @@ -95,6 +95,64 @@ def test_fetch_crc32_links_from_nyaa(self, mock_get): assert "A1B2C3D4" in crc32_to_text assert "magnet:?xt=urn:btih:abc123" in crc32_to_magnet.values() + @patch('acepace.requests.get') + def test_fetch_crc32_links_filters_quality(self, mock_get): + """Test that fetch_crc32_links filters episodes by quality (1080p/720p only).""" + html_with_mixed_quality = """ + + + + + + + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv + Magnet +
+ [One Pace] Episode 2 [720p][E5F6A7B8].mkv + Magnet +
+ [One Pace] Episode 3 [480p][A9B0C1D2].mkv + Magnet +
+ + + """ + + html_empty = """ + + + +
+ + + """ + + mock_response1 = MagicMock() + mock_response1.status_code = 200 + mock_response1.text = html_with_mixed_quality + + mock_response2 = MagicMock() + mock_response2.status_code = 200 + mock_response2.text = html_empty + + mock_get.side_effect = [mock_response1, mock_response2] + + base_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace" + crc32_to_link, crc32_to_text, crc32_to_magnet, _ = acepace.fetch_crc32_links(base_url) + + # Should only have 1080p and 720p episodes, not 480p + assert len(crc32_to_link) == 2 + assert "A1B2C3D4" in crc32_to_link # 1080p - should be included + assert "E5F6A7B8" in crc32_to_link # 720p - should be included + assert "A9B0C1D2" not in crc32_to_link # 480p - should be filtered out + @patch('acepace.requests.get') def test_fetch_crc32_links_stops_on_empty_page(self, mock_get): """Test that fetching stops when no matches found.""" diff --git a/tests/test_path_normalization.py b/tests/test_path_normalization.py new file mode 100644 index 0000000..77e90a5 --- /dev/null +++ b/tests/test_path_normalization.py @@ -0,0 +1,413 @@ +"""Unit tests for path normalization and quality filtering.""" +import pytest +import os +import sys +import tempfile +from unittest.mock import patch, MagicMock +from bs4 import BeautifulSoup + +# Add parent directory to path to import acepace +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import acepace + + +class TestPathNormalization: + """Tests for file path normalization functionality.""" + + def test_normalize_file_path_absolute(self, temp_dir): + """Test that normalize_file_path converts to absolute path.""" + test_file = os.path.join(temp_dir, "test.mkv") + with open(test_file, "wb") as f: + f.write(b"test content") + + normalized = acepace.normalize_file_path(test_file) + + assert os.path.isabs(normalized) + assert normalized == os.path.realpath(os.path.abspath(test_file)) + + def test_normalize_file_path_relative(self, temp_dir): + """Test that normalize_file_path handles relative paths.""" + test_file = os.path.join(temp_dir, "test.mkv") + with open(test_file, "wb") as f: + f.write(b"test content") + + # Change to temp_dir and use relative path + original_cwd = os.getcwd() + try: + os.chdir(temp_dir) + normalized = acepace.normalize_file_path("test.mkv") + + assert os.path.isabs(normalized) + assert normalized == os.path.realpath(os.path.abspath(test_file)) + finally: + os.chdir(original_cwd) + + def test_normalize_file_path_resolves_symlinks(self, temp_dir): + """Test that normalize_file_path resolves symlinks.""" + # Create a real file + real_file = os.path.join(temp_dir, "real.mkv") + with open(real_file, "wb") as f: + f.write(b"test content") + + # Create a symlink (if supported) + try: + symlink_file = os.path.join(temp_dir, "link.mkv") + os.symlink(real_file, symlink_file) + + normalized = acepace.normalize_file_path(symlink_file) + real_normalized = acepace.normalize_file_path(real_file) + + # Both should resolve to the same path + assert normalized == real_normalized + except (OSError, AttributeError): + # Symlinks not supported on this platform, skip + pytest.skip("Symlinks not supported on this platform") + + def test_normalize_file_path_nonexistent_file(self): + """Test that normalize_file_path handles nonexistent files gracefully.""" + nonexistent = "/nonexistent/path/file.mkv" + normalized = acepace.normalize_file_path(nonexistent) + + # Should still return normalized absolute path even if file doesn't exist + assert os.path.isabs(normalized) + assert normalized == os.path.normpath(os.path.abspath(nonexistent)) + + def test_normalize_file_path_consistent(self, temp_dir): + """Test that normalize_file_path produces consistent results.""" + test_file = os.path.join(temp_dir, "test.mkv") + with open(test_file, "wb") as f: + f.write(b"test content") + + # Test with different path representations + path1 = test_file + path2 = os.path.join(temp_dir, ".", "test.mkv") + path3 = os.path.join(temp_dir, "..", os.path.basename(temp_dir), "test.mkv") + + normalized1 = acepace.normalize_file_path(path1) + normalized2 = acepace.normalize_file_path(path2) + normalized3 = acepace.normalize_file_path(path3) + + # All should normalize to the same path + assert normalized1 == normalized2 == normalized3 + + +class TestPathNormalizationInCalculateCRC32: + """Tests for path normalization in calculate_local_crc32.""" + + def test_calculate_local_crc32_stores_normalized_paths(self, temp_dir): + """Test that calculate_local_crc32 stores normalized paths in database.""" + test_file = os.path.join(temp_dir, "test.mkv") + with open(test_file, "wb") as f: + f.write(b"test content") + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + acepace.calculate_local_crc32(temp_dir, conn) + + # Check database for normalized path + cursor = conn.cursor() + cursor.execute("SELECT file_path FROM crc32_cache") + rows = cursor.fetchall() + + assert len(rows) == 1 + stored_path = rows[0][0] + + # Stored path should be normalized (absolute) + assert os.path.isabs(stored_path) + assert stored_path == acepace.normalize_file_path(test_file) + + conn.close() + + def test_calculate_local_crc32_finds_cached_by_normalized_path(self, temp_dir): + """Test that calculate_local_crc32 finds cached entries using normalized paths.""" + test_file = os.path.join(temp_dir, "test.mkv") + with open(test_file, "wb") as f: + f.write(b"test content") + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + + # First calculation + crc32s1 = acepace.calculate_local_crc32(temp_dir, conn) + + # Manually insert with different path representation + normalized_path = acepace.normalize_file_path(test_file) + relative_path = "test.mkv" # Different representation + + # Try to query with relative path - should not find it + cursor = conn.cursor() + cursor.execute("SELECT crc32 FROM crc32_cache WHERE file_path = ?", (relative_path,)) + row = cursor.fetchone() + assert row is None # Should not find with relative path + + # Query with normalized path - should find it + cursor.execute("SELECT crc32 FROM crc32_cache WHERE file_path = ?", (normalized_path,)) + row = cursor.fetchone() + assert row is not None # Should find with normalized path + + # Second calculation should use cache (even with different path representation) + original_cwd = os.getcwd() + try: + os.chdir(temp_dir) + crc32s2 = acepace.calculate_local_crc32(".", conn) + assert crc32s1 == crc32s2 + finally: + os.chdir(original_cwd) + + conn.close() + + +class TestPathNormalizationInCountFiles: + """Tests for path normalization in _count_video_files.""" + + def test_count_video_files_uses_normalized_paths(self, temp_dir): + """Test that _count_video_files uses normalized paths for lookup.""" + test_file = os.path.join(temp_dir, "test.mkv") + with open(test_file, "wb") as f: + f.write(b"test content") + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + + # Calculate CRC32 (stores normalized path) + acepace.calculate_local_crc32(temp_dir, conn) + + # Count files - should find the cached entry + total, recorded = acepace._count_video_files(temp_dir, conn) + + assert total == 1 + assert recorded == 1 # Should find the cached entry + + conn.close() + + +class TestQualityFiltering: + """Tests for quality filtering in episode processing.""" + + def test_process_crc32_row_accepts_1080p(self): + """Test that _process_crc32_row accepts 1080p episodes.""" + row_html = """ + + + [One Pace] Episode 1 [1080p][A1B2C3D4].mkv + Magnet + + + """ + soup = BeautifulSoup(row_html, "html.parser") + row = soup.find("tr") + + crc32_to_link = {} + crc32_to_text = {} + crc32_to_magnet = {} + + success, filename_text = acepace._process_crc32_row( + row, crc32_to_link, crc32_to_text, crc32_to_magnet + ) + + assert success is True + assert "A1B2C3D4" in crc32_to_link + assert "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv" in crc32_to_text.values() + + def test_process_crc32_row_accepts_720p(self): + """Test that _process_crc32_row accepts 720p episodes.""" + row_html = """ + + + [One Pace] Episode 1 [720p][A1B2C3D4].mkv + Magnet + + + """ + soup = BeautifulSoup(row_html, "html.parser") + row = soup.find("tr") + + crc32_to_link = {} + crc32_to_text = {} + crc32_to_magnet = {} + + success, filename_text = acepace._process_crc32_row( + row, crc32_to_link, crc32_to_text, crc32_to_magnet + ) + + assert success is True + assert "A1B2C3D4" in crc32_to_link + + def test_process_crc32_row_rejects_480p(self): + """Test that _process_crc32_row rejects 480p episodes.""" + row_html = """ + + + [One Pace] Episode 1 [480p][A1B2C3D4].mkv + Magnet + + + """ + soup = BeautifulSoup(row_html, "html.parser") + row = soup.find("tr") + + crc32_to_link = {} + crc32_to_text = {} + crc32_to_magnet = {} + + success, filename_text = acepace._process_crc32_row( + row, crc32_to_link, crc32_to_text, crc32_to_magnet + ) + + assert success is False + assert "A1B2C3D4" not in crc32_to_link + + def test_process_crc32_row_rejects_2160p(self): + """Test that _process_crc32_row rejects 2160p (4K) episodes.""" + row_html = """ + + + [One Pace] Episode 1 [2160p][A1B2C3D4].mkv + Magnet + + + """ + soup = BeautifulSoup(row_html, "html.parser") + row = soup.find("tr") + + crc32_to_link = {} + crc32_to_text = {} + crc32_to_magnet = {} + + success, filename_text = acepace._process_crc32_row( + row, crc32_to_link, crc32_to_text, crc32_to_magnet + ) + + assert success is False + assert "A1B2C3D4" not in crc32_to_link + + def test_process_crc32_row_rejects_no_quality(self): + """Test that _process_crc32_row rejects episodes without quality marker.""" + row_html = """ + + + [One Pace] Episode 1 [A1B2C3D4].mkv + Magnet + + + """ + soup = BeautifulSoup(row_html, "html.parser") + row = soup.find("tr") + + crc32_to_link = {} + crc32_to_text = {} + crc32_to_magnet = {} + + success, filename_text = acepace._process_crc32_row( + row, crc32_to_link, crc32_to_text, crc32_to_magnet + ) + + assert success is False + assert "A1B2C3D4" not in crc32_to_link + + def test_process_crc32_row_rejects_no_one_pace_marker(self): + """Test that _process_crc32_row rejects episodes without [One Pace] marker.""" + row_html = """ + + + Episode 1 [1080p][A1B2C3D4].mkv + Magnet + + + """ + soup = BeautifulSoup(row_html, "html.parser") + row = soup.find("tr") + + crc32_to_link = {} + crc32_to_text = {} + crc32_to_magnet = {} + + success, filename_text = acepace._process_crc32_row( + row, crc32_to_link, crc32_to_text, crc32_to_magnet + ) + + assert success is False + assert "A1B2C3D4" not in crc32_to_link + + def test_process_crc32_row_case_insensitive_quality(self): + """Test that quality filtering is case insensitive.""" + row_html = """ + + + [One Pace] Episode 1 [1080P][A1B2C3D4].mkv + Magnet + + + """ + soup = BeautifulSoup(row_html, "html.parser") + row = soup.find("tr") + + crc32_to_link = {} + crc32_to_text = {} + crc32_to_magnet = {} + + success, filename_text = acepace._process_crc32_row( + row, crc32_to_link, crc32_to_text, crc32_to_magnet + ) + + assert success is True + assert "A1B2C3D4" in crc32_to_link + + @patch('acepace.requests.get') + def test_fetch_crc32_links_filters_by_quality(self, mock_get): + """Test that fetch_crc32_links filters episodes by quality.""" + html_with_mixed_quality = """ + + + + + + + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv + Magnet +
+ [One Pace] Episode 2 [720p][E5F6A7B8].mkv + Magnet +
+ [One Pace] Episode 3 [480p][A9B0C1D2].mkv + Magnet +
+ + + """ + + html_empty = """ + + + +
+ + + """ + + mock_response1 = MagicMock() + mock_response1.status_code = 200 + mock_response1.text = html_with_mixed_quality + + mock_response2 = MagicMock() + mock_response2.status_code = 200 + mock_response2.text = html_empty + + mock_get.side_effect = [mock_response1, mock_response2] + + base_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace" + crc32_to_link, crc32_to_text, crc32_to_magnet, _ = acepace.fetch_crc32_links(base_url) + + # Should only have 1080p and 720p episodes, not 480p + assert len(crc32_to_link) == 2 + assert "A1B2C3D4" in crc32_to_link # 1080p + assert "E5F6A7B8" in crc32_to_link # 720p + assert "A9B0C1D2" not in crc32_to_link # 480p should be filtered out From 39509f839d41028a8b5cf2cb73e5773a334ab08a Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 26 Jan 2026 14:53:36 +0000 Subject: [PATCH 49/75] Enhance db init and reporting features --- acepace.py | 25 +- coverage.xml | 818 ++++++++++++++++++++++++++------------------------- 2 files changed, 430 insertions(+), 413 deletions(-) diff --git a/acepace.py b/acepace.py index c764355..8faf9e8 100644 --- a/acepace.py +++ b/acepace.py @@ -80,7 +80,11 @@ def normalize_file_path(file_path): return os.path.normpath(os.path.abspath(file_path)) -def init_db(): +def init_db(suppress_messages=False): + """Initialize the database. + Args: + suppress_messages: If True, suppress informational messages (useful for automated runs) + """ db_path = get_config_path(DB_NAME) exists = os.path.exists(db_path) conn = sqlite3.connect(db_path) @@ -102,7 +106,7 @@ def init_db(): """ ) conn.commit() - if exists: + if exists and not suppress_messages: print("Database already exists. You can export it using the --db option.") return conn @@ -837,7 +841,11 @@ def _print_report_header(conn, folder, args): now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") set_metadata(conn, "last_run", now_str) - print(f"Using URL: {args.url}") + # Show URL, but note that quality filtering (1080p/720p) is applied regardless + url_display = args.url + if "1080p" not in url_display and "720p" not in url_display: + url_display += " (quality filtering: 1080p/720p only)" + print(f"Using URL: {url_display}") print(f"Total video files detected: {total_files}") print(f"Episodes already recorded in DB: {recorded_files}") @@ -897,6 +905,9 @@ def _generate_missing_episodes_report(conn, folder, args): set_metadata(conn, "last_checked_page", str(last_checked_page)) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") set_metadata(conn, "last_missing_export", now_str) + + # Print missing count prominently + print(f"Missing episodes: {len(missing)}") return missing, crc32_to_text @@ -1009,7 +1020,7 @@ def _parse_arguments(): parser.add_argument( "--url", default=f"{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace+1080p&o=asc", - help=f"Base URL without the page param. Example: '{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace&o=asc' ", + help=f"Base URL without the page param. Default includes 1080p filter. Example: '{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace+1080p&o=asc' ", ) parser.add_argument("--folder", help="Folder containing local video files.") parser.add_argument( @@ -1097,7 +1108,8 @@ def main(): _print_help() return - if IS_DOCKER: + # Only show Docker mode message once, and not for --db or --episodes_update commands + if IS_DOCKER and not args.db and not args.episodes_update: print("Running in Docker mode (non-interactive)") if not _validate_url(args.url): @@ -1109,7 +1121,8 @@ def main(): update_episodes_index_db() return - conn = init_db() + # Suppress messages when exporting DB (since it's automated) + conn = init_db(suppress_messages=args.db) # Folder selection logic: Always prompt if folder is required but not given needs_folder = not args.download # All commands except --download need folder diff --git a/coverage.xml b/coverage.xml index b877875..fb7cea3 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -51,624 +51,628 @@ - - - - - - - - - + + + + + + + + - - - - - - + + + + + - - + + - + + - - - - - - - - + + + + + + + - + - - - - + + + - - + + + - - - - + + - - - + + + - - - - + + + + - + + + - - + - - + + + - - - - - + + + - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + + - - - - - + + + - - - - + + + + - - - - - - + + + + + + + - - - + + + - - - - - - + + + + + + + - - - + - - + + + + - - - - - - + + + + - - + + - - - + + + - + + + - - - + + + + - - - + + - - + + - - + + - + - + + - - - - + - - - - + + + + - - - + + + + - - - + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - + - - - - - - - - + + + + + + + + + - - + - - - + + + + + - - - - - + + + - - + + - + + - - + - + + + - - - + - - + + + + - - - - + + + - - + - + + + + - - - - - + + - - + + + - - - - - - - - - - + + + + + + + + + - + - + + - - + + + - - + + - - - + + + - - + - - + + - + - + - - - + + + - - - - - - + + + + + - + - + + + - - - + + - - + + + - - - - + + + + - + - - - + - - - - - - - - - - - + + + + + + + + + + + - + - - - - - + + + + + - + - - - - - - + + + + + + + - - + - - + + + - - - - - - - - + + + + + + + + - + + - + - - - - - + + + + - - - - - + + + + + - - - - + + + + + - - - + + + - - - + + - - + - - - - + + + + + - - + + + - - - - + + - - + + - - + + + - - - - - - - + + + + + + - + - - - - + + + + + + - - - + + - - + + + + - - - - - - - - + + + + + + + + + - - - - + + - - - - + + + + + - + + - - - + + - - - - - - + + + + + + + + - - - - - + + + + + - - - - - - - + + + + - - - - - + + + + - + - - + + - - - - + + + - + - - - - + + + + + + + - - - + - - + + - - - + + + - - + + + + + + + + + From 32095c192edb156d6543af519fd23a903afa9930 Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 26 Jan 2026 15:07:08 +0000 Subject: [PATCH 50/75] Hopefully fixed the quality inconsistency issue --- acepace.py | 32 +- coverage.xml | 669 +++++++++++++++++++++-------------------- tests/test_episodes.py | 159 ++++++++++ 3 files changed, 516 insertions(+), 344 deletions(-) diff --git a/acepace.py b/acepace.py index 8faf9e8..cf2695f 100644 --- a/acepace.py +++ b/acepace.py @@ -281,14 +281,18 @@ def _process_episode_row(row, seen_crc32, episodes): return _process_torrent_page(page_link, seen_crc32, episodes) -def fetch_episodes_metadata(): +def fetch_episodes_metadata(base_url=None): """ Fetch all One Pace episodes from Nyaa, collecting CRC32, title, and page link. If CRC32 not in title, fetch the torrent page and try to extract CRC32s from file list. + Args: + base_url: Base URL for Nyaa search. If None, uses default without quality filter. + Note: Quality filtering (1080p/720p) is always applied regardless of URL. Returns: List of (crc32, title, page_link) """ - - base_url = f"{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace" + if base_url is None: + base_url = f"{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace" + episodes = [] seen_crc32 = set() page = 1 @@ -326,9 +330,13 @@ def fetch_episodes_metadata(): return episodes -def update_episodes_index_db(): +def update_episodes_index_db(base_url=None): + """Update episodes index database from Nyaa. + Args: + base_url: Base URL for Nyaa search. If None, uses default. + """ conn = init_episodes_db() - episodes = fetch_episodes_metadata() + episodes = fetch_episodes_metadata(base_url) c = conn.cursor() count = 0 for crc32, title, page_link in episodes: @@ -757,8 +765,12 @@ def _get_rename_prompt(last_ep_update): ).strip().lower() -def _handle_rename_command(conn): - """Handle the rename command.""" +def _handle_rename_command(conn, base_url=None): + """Handle the rename command. + Args: + conn: Database connection + base_url: Base URL for Nyaa search (optional) + """ episodes_db_conn = init_episodes_db() last_ep_update = get_episodes_metadata( episodes_db_conn, "episodes_db_last_update" @@ -768,7 +780,7 @@ def _handle_rename_command(conn): prompt = _get_rename_prompt(last_ep_update) if prompt == "y": - update_episodes_index_db() + update_episodes_index_db(base_url) print( "Renaming local files based on matching titles from One Pace episodes index..." ) @@ -1084,7 +1096,7 @@ def _handle_main_commands(args, conn, folder): return if args.rename: - _handle_rename_command(conn) + _handle_rename_command(conn, args.url) return if not folder: @@ -1118,7 +1130,7 @@ def main(): _show_episodes_metadata_status() if args.episodes_update: - update_episodes_index_db() + update_episodes_index_db(args.url) return # Suppress messages when exporting DB (since it's automated) diff --git a/coverage.xml b/coverage.xml index fb7cea3..003524a 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -171,508 +171,509 @@ - - - + - + + - - - + - - + + + + - - - - - - + + + + - - + + - - - + + + - + + + - - - - + - + - - + - + - + + + - + + - - - - - - + - - - - + + + + - - - - + + + + + - + + - - - - - + + + + - - - - - + + + + + - - - - - - + + + + + + - - - + + + + + - + + - - - - - + + - - - + + - + + + + - - - - - + + + - - + - - + + + - - + + - - - + + + + + - - + - - + - - - - + + + + + + - - - + - - + + - - + + + + + + - - - - - - - + + + - - - + + + - - - + + + + + - - - + + + - - - - - - + + + + + + + - - - - + + - + - + - - + + + - - + - + - - - + + + - - + + - - - - - - - - - + + + + + + + + + - - - - - - - - + + + + + + + + - - - + + + + + - - - + - - - - - + + + + + + - - - - - - - - - - - + + + + + + + + - + + + + - + - - - - - + + + + + - + - - - - - - + + + + - + + - + + + - - + - - - - + + + + + - - + + + - - - - - + - - - - - - - - - + + + + + + + + + + + + + - - - - - + + - - - - - - - - - - + + + + + + - - + + + - - - - - - - - - - - + + + + + + + + + - - - + + + + + + - - - + + - - - + + + - - + + + + - - - - + + + + - - - - - + + + + + + + - + - - - + + - - - - + + + + + - - - - - - + + + + + + - - - + + - - - + + + - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - + + + - - - - + + + + - - - + + - - + + - - - + + - - + + + + + - + + - - - - + + - - + + - + + - - - + + + + + + + + + diff --git a/tests/test_episodes.py b/tests/test_episodes.py index 9db42ed..afa4521 100644 --- a/tests/test_episodes.py +++ b/tests/test_episodes.py @@ -27,6 +27,51 @@ def test_fetch_episodes_metadata_single_page(self, mock_get, mock_nyaa_html_sing assert len(episodes) == 2 assert any(ep[0] == "A1B2C3D4" for ep in episodes) assert any(ep[0] == "E5F6A7B8" for ep in episodes) + + # Verify default URL was used + assert mock_get.call_count > 0 + # Check that the default URL (without 1080p) was used + call_urls = [call[0][0] for call in mock_get.call_args_list] + assert any("q=one+pace" in url and "1080p" not in url for url in call_urls) + + @patch('acepace.requests.get') + def test_fetch_episodes_metadata_uses_custom_url(self, mock_get, mock_nyaa_html_single_page): + """Test that fetch_episodes_metadata uses the provided URL parameter.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = mock_nyaa_html_single_page + mock_get.return_value = mock_response + + custom_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + episodes = acepace.fetch_episodes_metadata(custom_url) + + assert len(episodes) == 2 + + # Verify the custom URL was used in requests + assert mock_get.call_count > 0 + call_urls = [call[0][0] for call in mock_get.call_args_list] + # All URLs should start with the custom URL (with page parameter) + for url in call_urls: + assert url.startswith(custom_url + "&p=") or url == custom_url + "&p=1" + + @patch('acepace.requests.get') + def test_fetch_episodes_metadata_default_url_when_none_provided(self, mock_get, mock_nyaa_html_single_page): + """Test that fetch_episodes_metadata uses default URL when None is provided.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = mock_nyaa_html_single_page + mock_get.return_value = mock_response + + # Call with None explicitly + episodes = acepace.fetch_episodes_metadata(None) + + assert len(episodes) == 2 + + # Verify default URL was used + assert mock_get.call_count > 0 + call_urls = [call[0][0] for call in mock_get.call_args_list] + # Should use default URL without 1080p + assert any("q=one+pace" in url and "1080p" not in url for url in call_urls) @patch('acepace.requests.get') @patch('acepace.time.sleep') # Mock sleep to speed up tests @@ -207,6 +252,23 @@ def test_update_episodes_index_db(self, mock_fetch, temp_dir, sample_episode_dat conn.close() os.remove(os.path.join(temp_dir, 'test.db')) + @patch('acepace.fetch_episodes_metadata') + @patch('acepace.EPISODES_DB_NAME', 'test_episodes_index.db') + def test_update_episodes_index_db_uses_url_parameter(self, mock_fetch, temp_dir, sample_episode_data): + """Test that update_episodes_index_db passes URL parameter to fetch_episodes_metadata.""" + with patch('acepace.EPISODES_DB_NAME', os.path.join(temp_dir, 'test.db')): + mock_fetch.return_value = sample_episode_data + + test_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + acepace.update_episodes_index_db(test_url) + + # Verify fetch_episodes_metadata was called with the URL + mock_fetch.assert_called_once_with(test_url) + + # Clean up + if os.path.exists(os.path.join(temp_dir, 'test.db')): + os.remove(os.path.join(temp_dir, 'test.db')) + class TestEpisodeQualityFiltering: """Tests for ensuring only 1080p (or 720p fallback) episodes are extracted.""" @@ -659,3 +721,100 @@ def test_quality_filtering_rejects_higher_quality(self): assert len(matches) > 0 quality_num = int(matches[0].lower().replace('p', '')) assert quality_num not in [720, 1080] + + +class TestURLParameterConsistency: + """Tests to ensure URL parameter is used consistently across functions.""" + + @patch('acepace.requests.get') + def test_fetch_episodes_metadata_and_fetch_crc32_links_use_same_url(self, mock_get): + """Test that both fetch_episodes_metadata and fetch_crc32_links use the same URL when provided.""" + html_with_results = """ + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv + Magnet +
+
    +
  • 1
    • +
+ + + """ + + html_empty = """ + + + +
+ + + """ + + mock_response1 = MagicMock() + mock_response1.status_code = 200 + mock_response1.text = html_with_results + + mock_response2 = MagicMock() + mock_response2.status_code = 200 + mock_response2.text = html_empty + + mock_get.side_effect = [mock_response1, mock_response2] + + test_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + + # Test fetch_episodes_metadata + episodes = acepace.fetch_episodes_metadata(test_url) + + # Reset mock for second test + mock_get.reset_mock() + mock_get.side_effect = [mock_response1, mock_response2] + + # Test fetch_crc32_links + crc32_to_link, _, _, _ = acepace.fetch_crc32_links(test_url) + + # Both should use the same URL + assert mock_get.call_count > 0 + + # Verify URLs used in both calls + episodes_urls = [call[0][0] for call in mock_get.call_args_list] + + # Both should have used URLs starting with test_url + for url in episodes_urls: + assert url.startswith(test_url + "&p=") or url == test_url + "&p=1" + + @patch('acepace.fetch_episodes_metadata') + def test_update_episodes_index_db_passes_url_to_fetch_episodes_metadata(self, mock_fetch, temp_dir): + """Test that update_episodes_index_db correctly passes URL to fetch_episodes_metadata.""" + with patch('acepace.EPISODES_DB_NAME', os.path.join(temp_dir, 'test.db')): + mock_fetch.return_value = [] + + test_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + acepace.update_episodes_index_db(test_url) + + # Verify fetch_episodes_metadata was called with the URL + mock_fetch.assert_called_once_with(test_url) + + conn = acepace.init_episodes_db() + conn.close() + os.remove(os.path.join(temp_dir, 'test.db')) + + @patch('acepace.fetch_episodes_metadata') + def test_update_episodes_index_db_default_url_when_none_provided(self, mock_fetch, temp_dir): + """Test that update_episodes_index_db uses default URL when None is provided.""" + with patch('acepace.EPISODES_DB_NAME', os.path.join(temp_dir, 'test.db')): + mock_fetch.return_value = [] + + acepace.update_episodes_index_db() + + # Verify fetch_episodes_metadata was called with None (which triggers default) + mock_fetch.assert_called_once_with(None) + + conn = acepace.init_episodes_db() + conn.close() + os.remove(os.path.join(temp_dir, 'test.db')) From f7ca6b0b5e849faab83b738a046a459ad3f99605 Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 26 Jan 2026 15:49:02 +0000 Subject: [PATCH 51/75] Enhance Docker support; improve quality filtering; updated spec and tests --- Dockerfile | 7 +- acepace.py | 56 +- coverage.xml | 1044 ++++++++++++++++++------------------ docker-compose.yml | 38 +- entrypoint.sh | 24 +- spec.md | 161 ++++-- tests/test_database.py | 32 ++ tests/test_main_command.py | 377 +++++++++++++ 8 files changed, 1155 insertions(+), 584 deletions(-) create mode 100644 tests/test_main_command.py diff --git a/Dockerfile b/Dockerfile index bfb377f..6a58353 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,14 +8,13 @@ WORKDIR /app ENV RUN_DOCKER="true" \ PYTHONUNBUFFERED=1 \ TZ=Europe/Berlin \ - NYAA_URL="https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" \ TORRENT_HOST="127.0.0.1" \ TORRENT_CLIENT="transmission" \ TORRENT_PORT="9091" \ TORRENT_USER="" \ - TORRENT_PASSWORD="" \ - DB="true" \ - EPISODES_UPDATE="true" + TORRENT_PASSWORD="" +# Note: NYAA_URL, DB, and EPISODES_UPDATE should be set in docker-compose.yml, not here +# This allows users to override them without rebuilding the image RUN apt-get update \ && apt-get install -y tzdata \ diff --git a/acepace.py b/acepace.py index cf2695f..24724ee 100644 --- a/acepace.py +++ b/acepace.py @@ -652,7 +652,9 @@ def _get_folder_from_args(args, conn, needs_folder): def _get_client_from_args_or_env(args): - """Get client type from args or environment variables.""" + """Get client type from args or environment variables. + In Docker mode, defaults to 'transmission' if not specified. + """ if IS_DOCKER and not args.client: return os.getenv("TORRENT_CLIENT", "transmission") return args.client @@ -664,17 +666,26 @@ def _get_default_port(client): def _get_docker_connection_params(args): - """Get connection parameters from Docker environment variables.""" + """Get connection parameters from Docker environment variables. + Uses default values: localhost, 9091, transmission if not specified. + """ + # Get client (defaults to transmission in Docker) + client = _get_client_from_args_or_env(args) + + # Get host (defaults to localhost) host = os.getenv("TORRENT_HOST", args.host or "localhost") + + # Get port (defaults to 9091 for transmission, 8080 for qbittorrent) port_env = os.getenv("TORRENT_PORT") port = int(port_env) if port_env else None if not port: - default_port = _get_default_port(args.client) + default_port = _get_default_port(client) port = args.port if args.port else default_port + username = os.getenv("TORRENT_USER", args.username or "") password = os.getenv("TORRENT_PASSWORD", args.password or "") download_folder = args.download_folder or "/media" - return host, port, username, password, download_folder + return host, port, username, password, download_folder, client def _get_non_docker_connection_params(args): @@ -713,29 +724,39 @@ def _load_magnet_links(): def _handle_download_command(args): """Handle the download command.""" - client = _get_client_from_args_or_env(args) - if not client: - print("Error: --client is required when using --download.") - return False - - magnets = _load_magnet_links() - if magnets is None: - return False - # Get connection parameters based on Docker mode if IS_DOCKER: - host, port, username, password, download_folder = _get_docker_connection_params(args) + host, port, username, password, download_folder, client = _get_docker_connection_params(args) + # Log connection parameters in Docker mode + print("Download configuration:") + print(f" Client: {client}") + print(f" Host: {host}") + print(f" Port: {port}") + if username: + print(f" Username: {username}") + if download_folder: + print(f" Download folder: {download_folder}") else: + client = _get_client_from_args_or_env(args) + if not client: + print("Error: --client is required when using --download.") + return False host, port, username, password, download_folder = _get_non_docker_connection_params(args) + magnets = _load_magnet_links() + if magnets is None: + return False + try: client_obj = get_client(client, host, port, username, password) + print(f"Adding {len(magnets)} missing episode(s) to {client}...") client_obj.add_torrents( magnets, download_folder=download_folder, tags=args.tag, category=args.category, ) + print(f"Successfully added {len(magnets)} episode(s) to {client}.") except ConnectionError as e: print(f"Connection Error: {e}") print(f"Please verify that {client} is running and accessible at {host}:{port}") @@ -1121,13 +1142,16 @@ def main(): return # Only show Docker mode message once, and not for --db or --episodes_update commands - if IS_DOCKER and not args.db and not args.episodes_update: + # Also suppress for help command + if IS_DOCKER and not args.db and not args.episodes_update and not args.help: print("Running in Docker mode (non-interactive)") if not _validate_url(args.url): return - _show_episodes_metadata_status() + # Only show episodes metadata status for main command (not for --db or --episodes_update) + if not args.db and not args.episodes_update: + _show_episodes_metadata_status() if args.episodes_update: update_episodes_index_db(args.url) diff --git a/coverage.xml b/coverage.xml index 003524a..8d2e4aa 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -46,21 +46,21 @@ - - + + - - - - - - - - + + + + + + + + - + @@ -75,50 +75,50 @@ - - - + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - + + + - - - - - + + + + + @@ -132,220 +132,220 @@ - - + + - - + + - - - - - - + + + + + + - - - + + + - - - - - - - + + + + + + + - - + + - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - + + + + + + - - - - + + + + - - - + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - + + + - - - - - - - - - - - + + + + + + + + + + + - - - + + + - - - + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -405,18 +405,18 @@ - - - - - - - - - - - - + + + + + + + + + + + + @@ -440,243 +440,255 @@ - - - - - - - - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - + + - - - - + - + + - + - + + + + - - + - - - - - - - - - - + + + + + + + + + + + + + - + - - - + + - + + + - - - + + - - - + + + + + + + + + - - - + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + - - - - + + + + + - - - - - + + + + - - + + - - - - + + + + + + - - - + + + - + + + - + - - - + - - - - - + + + + + + + + + + + - - - - + + + + - - - - - - - - - + + + + + + - + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - + - - + - + + - + - - + - - - - - - - - - + + + + + + + + + - - - - + + + + + + - - - - - - - - + + + + + + + - - - + + + + + + + + + + + + + + + + - + @@ -688,127 +700,127 @@ - - - - - - - - - + + + + + + + + + - + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - + - - - - - - - - - + + + + + + + + + - - - - - + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index 133aeb6..1545736 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,23 +1,37 @@ services: ace-pace: - build: . + # build: . image: timothe/ace-pace:latest # or timothe/ace-pace:dev for dev branch container_name: ace-pace volumes: - /path/to/OnePaceLibrary:/media:rw - /path/to/config:/config:rw - networks: - - "proxy" + # networks: + # - "proxy" environment: - TZ=Europe/London - - TORRENT_HOST=127.0.0.1 - - TORRENT_PORT=9091 - - TORRENT_CLIENT=transmission - - NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc - #- TORRENT_USER=admin - #- TORRENT_PASSWORD=password + # Export database to CSV on container start (default: false) - DB=true + # Update episodes index database on container start (default: false) - EPISODES_UPDATE=true -networks: - proxy: - driver: bridge + # Nyaa.si search URL (default: https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc) + #- NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc + + # Download missing episodes after generating report (default: false) + #- DOWNLOAD=true + + # BitTorrent client type (default: transmission) + # Options: transmission, qbittorrent + #- TORRENT_CLIENT=transmission + # BitTorrent client host (default: localhost) + #- TORRENT_HOST=127.0.0.1 + # BitTorrent client port (default: 9091 for transmission, 8080 for qbittorrent) + #- TORRENT_PORT=9091 + # BitTorrent client username (default: empty, not required) + #- TORRENT_USER=admin + # BitTorrent client password (default: empty, not required) + #- TORRENT_PASSWORD=password + +# networks: +# proxy: +# driver: bridge diff --git a/entrypoint.sh b/entrypoint.sh index cdeeb2d..08f9c0e 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,14 +1,30 @@ #!/bin/sh set -e +# Run episodes update if requested if [ "$EPISODES_UPDATE" = "true" ]; then - python /app/acepace.py --episodes_update + python /app/acepace.py --episodes_update ${NYAA_URL:+--url "$NYAA_URL"} fi +# Export database if requested if [ "$DB" = "true" ]; then python /app/acepace.py --db fi -exec python /app/acepace.py \ - ${NYAA_URL:+--url "$NYAA_URL"} \ - ${TORRENT_CLIENT:+--download --client "$TORRENT_CLIENT"} \ No newline at end of file +# Always run missing episodes report first (updates Ace-Pace_Missing.csv) +python /app/acepace.py \ + --folder /media \ + ${NYAA_URL:+--url "$NYAA_URL"} + +# If DOWNLOAD is set to true, download missing episodes after generating report +if [ "$DOWNLOAD" = "true" ]; then + exec python /app/acepace.py \ + --folder /media \ + ${NYAA_URL:+--url "$NYAA_URL"} \ + --download \ + ${TORRENT_CLIENT:+--client "$TORRENT_CLIENT"} \ + ${TORRENT_HOST:+--host "$TORRENT_HOST"} \ + ${TORRENT_PORT:+--port "$TORRENT_PORT"} \ + ${TORRENT_USER:+--username "$TORRENT_USER"} \ + ${TORRENT_PASSWORD:+--password "$TORRENT_PASSWORD"} +fi \ No newline at end of file diff --git a/spec.md b/spec.md index 88c3f39..032732b 100644 --- a/spec.md +++ b/spec.md @@ -19,6 +19,12 @@ - Episodes without quality markers are excluded - Episodes with quality lower than 720p (480p, 360p, etc.) are excluded - Episodes with quality higher than 1080p (1440p, 2160p/4K, etc.) are excluded + - Quality filtering is applied in both `fetch_episodes_metadata()` and `fetch_crc32_links()` + - Filtering is case-insensitive (accepts 1080P, 720P, etc.) +- **URL Parameter Support**: Both `fetch_episodes_metadata()` and `update_episodes_index_db()` accept a `base_url` parameter + - Allows consistent URL usage across all episode fetching functions + - Default URL includes 1080p filter, but can be overridden + - Quality filtering still applies regardless of URL parameters - Builds and maintains an episodes index database (`episodes_index.db`) - Supports both single-file and multi-file torrent structures - Handles pagination to fetch all available episodes @@ -26,14 +32,24 @@ ### 2. Local Library Management - Scans local directories recursively for video files (`.mkv`, `.mp4`, `.avi`) - Calculates CRC32 checksums for local video files +- **Path Normalization**: All file paths are normalized before storage and lookup + - Uses `normalize_file_path()` to resolve symlinks and convert to absolute paths + - Ensures consistent path representation across different OS and environments + - Prevents cache misses when same file is accessed via different path representations + - Critical for consistent behavior between Python and Docker versions - Caches CRC32 values in `crc32_files.db` to avoid recalculating -- Tracks file paths and their corresponding checksums +- Tracks file paths and their corresponding checksums (using normalized paths) ### 3. Missing Episode Detection - Fetches episode list from Nyaa.si using the provided URL (default: One-Pace 1080p search) -- Compares local CRC32 checksums against fetched episodes +- **Quality Filtering**: `fetch_crc32_links()` applies quality filtering via `_process_crc32_row()` + - Only accepts episodes with 1080p or 720p quality + - Requires "[One Pace]" marker in filename + - Ensures consistent filtering regardless of URL parameters +- Compares local CRC32 checksums against fetched episodes (using normalized paths) - Generates a CSV report (`Ace-Pace_Missing.csv`) listing missing episodes - Includes title, page link, and magnet link for each missing episode +- Displays missing episode count prominently in output - Tracks new missing episodes since last export by comparing with previous CSV - Note: Uses `fetch_crc32_links()` for real-time fetching, not the cached episodes index @@ -55,8 +71,9 @@ #### `crc32_files.db` - **Table: `crc32_cache`** - - `file_path` (TEXT, PRIMARY KEY): Full path to local video file + - `file_path` (TEXT, PRIMARY KEY): Normalized absolute path to local video file - `crc32` (TEXT, UNIQUE): CRC32 checksum of the file + - Note: File paths are normalized using `normalize_file_path()` before storage - **Table: `metadata`** - `key` (TEXT, PRIMARY KEY): Metadata key - `value` (TEXT): Metadata value @@ -79,6 +96,7 @@ - Uses Python's `zlib.crc32()` for incremental calculation - Formats result as uppercase 8-character hexadecimal string - Caches results to avoid redundant calculations +- Uses normalized file paths for cache lookups to ensure consistency #### CRC32 Extraction from Filenames - Uses regex pattern: `\[([A-Fa-f0-9]{8})\]` @@ -113,7 +131,9 @@ Ace-Pace/ │ ├── test_database.py │ ├── test_episodes.py │ ├── test_file_operations.py -│ └── test_missing_detection.py +│ ├── test_missing_detection.py +│ ├── test_path_normalization.py +│ └── test_main_command.py ├── crc32_files.db # Local file checksum database (generated) ├── episodes_index.db # Episodes metadata database (generated) ├── Ace-Pace_Missing.csv # Missing episodes report (generated) @@ -143,12 +163,31 @@ Ace-Pace/ - **Docker Mode**: Detected via `RUN_DOCKER` environment variable - **Non-Interactive Operation**: In Docker mode, skips user prompts and uses defaults - **Default Folder**: Uses `/media` as default folder in Docker mode +- **Config Directory**: Uses `/config` directory in Docker mode for databases and CSV files + - Local mode uses current directory (`.`) + - Config directory is automatically created if it doesn't exist +- **Message Suppression**: In Docker mode, suppresses informational messages for automated commands + - "Running in Docker mode" message only shown once for main command (not for `--db` or `--episodes_update`) + - Episodes metadata status only shown for main command (suppressed for `--db` and `--episodes_update`) + - Database "already exists" message suppressed when `--db` flag is used +- **Entrypoint Script**: `entrypoint.sh` orchestrates Docker execution + - Runs `--episodes_update` if `EPISODES_UPDATE=true` (with URL parameter if `NYAA_URL` is set) + - Runs `--db` if `DB=true` (exports database to CSV) + - **Always runs missing episodes report** (generates/updates `Ace-Pace_Missing.csv`) + - Runs `--download` if `DOWNLOAD=true` (downloads missing episodes after report generation) + - Always passes `--folder /media` to commands + - Passes `NYAA_URL` parameter when set - **Environment Variables**: Supports configuration via Docker environment variables - - `TORRENT_CLIENT`: BitTorrent client type (transmission/qbittorrent) - - `TORRENT_HOST`: Client host address - - `TORRENT_PORT`: Client port number - - `TORRENT_USER`: Client authentication username - - `TORRENT_PASSWORD`: Client authentication password + - `DOWNLOAD`: Set to "true" to download missing episodes after generating report (default: not set/false) + - `TORRENT_CLIENT`: BitTorrent client type (default: transmission) + - Options: transmission, qbittorrent + - `TORRENT_HOST`: Client host address (default: localhost) + - `TORRENT_PORT`: Client port number (default: 9091 for transmission, 8080 for qbittorrent) + - `TORRENT_USER`: Client authentication username (default: empty, not required) + - `TORRENT_PASSWORD`: Client authentication password (default: empty, not required) + - `NYAA_URL`: Custom Nyaa.si search URL (optional, defaults to 1080p search) + - `EPISODES_UPDATE`: Set to "true" to update episodes index on container start (default: not set/false) + - `DB`: Set to "true" to export database on container start (default: not set/false) - `RUN_DOCKER`: Flag to enable Docker mode (non-interactive) ## Command-Line Interface @@ -186,28 +225,47 @@ Ace-Pace/ 8. User optionally runs `--download --client ` to add missing episodes to BitTorrent client ### Episodes Index Update Workflow -1. User runs `--episodes_update` to refresh episodes database -2. Script scrapes all pages of Nyaa.si One-Pace search results -3. For each torrent, extracts CRC32 from title or file list -4. Stores CRC32, title, and page link in `episodes_index.db` -5. Updates metadata with last update timestamp +1. User runs `--episodes_update` (optionally with `--url` to specify search URL) +2. Script uses provided URL or defaults to One-Pace search (without quality filter in URL) +3. Script scrapes all pages of Nyaa.si search results +4. For each torrent, extracts CRC32 from title or file list +5. Applies quality filtering (1080p/720p only) before storing +6. Stores CRC32, title, and page link in `episodes_index.db` +7. Updates metadata with last update timestamp +8. Note: Quality filtering is applied regardless of URL parameters ### File Renaming Workflow -1. User runs `--rename` with `--folder` +1. User runs `--rename` with `--folder` (optionally with `--url` to specify search URL) 2. Script checks episodes index update status 3. Script prompts to update episodes index if outdated (skipped in Docker mode) -4. Script loads CRC32-to-title mapping from episodes index -5. Script matches local files by CRC32 -6. Script generates rename plan and prompts for confirmation (auto-confirms in Docker mode) -7. Script renames files and updates database +4. If update is needed, uses provided URL or default for `update_episodes_index_db()` +5. Script loads CRC32-to-title mapping from episodes index +6. Script matches local files by CRC32 (using normalized paths) +7. Script generates rename plan and prompts for confirmation (auto-confirms in Docker mode) +8. Script renames files and updates database with normalized paths ### Docker Workflow 1. Container starts with `RUN_DOCKER` environment variable set -2. Script operates in non-interactive mode (no user prompts) -3. Default folder is `/media` (configurable via volume mount) -4. BitTorrent client connection parameters read from environment variables -5. All user prompts automatically answered with defaults -6. Database files and CSV reports persist via volume mounts +2. Entrypoint script (`entrypoint.sh`) orchestrates execution: + - If `EPISODES_UPDATE=true`: Runs `--episodes_update` with URL from `NYAA_URL` (if set) + - If `DB=true`: Runs `--db` to export database (suppresses informational messages) + - **Always runs missing episodes report** (generates/updates `Ace-Pace_Missing.csv`) + - If `DOWNLOAD=true`: Runs `--download` to download missing episodes + - Uses default connection parameters if not specified: + - Client: transmission + - Host: localhost + - Port: 9091 (transmission) or 8080 (qbittorrent) + - Logs connection parameters used for download +3. Script operates in non-interactive mode (no user prompts) +4. Default folder is `/media` (always passed via `--folder` in entrypoint) +5. Config directory is `/config` (databases and CSV files stored here) +6. BitTorrent client connection parameters use defaults if not specified via environment variables +7. All user prompts automatically answered with defaults +8. Database files and CSV reports persist via volume mounts +9. Docker mode messages suppressed for automated commands (`--db`, `--episodes_update`) +10. Episodes metadata status only shown for main command +11. Missing episode count prominently displayed in output +12. Missing episodes CSV is always updated before download (if download is enabled) ## Integration Points @@ -276,9 +334,9 @@ Ace-Pace/ - Automatic episode index updates on schedule - Support for additional video formats - Integration with media server APIs (Plex, Jellyfin) -- Episode quality filtering (720p, 1080p, etc.) - Duplicate detection and cleanup - Episode metadata enrichment (thumbnails, descriptions) +- Note: Episode quality filtering (720p, 1080p) is already implemented ✅ ### Technical Improvements - Async/await for concurrent web scraping @@ -313,18 +371,30 @@ The codebase follows a modular structure with clear separation of concerns: #### Public API Functions - `main()`: Entry point for the application -- `init_db()`: Initializes the local CRC32 cache database +- `init_db(suppress_messages=False)`: Initializes the local CRC32 cache database + - `suppress_messages`: If True, suppresses informational messages (useful for automated runs) - `init_episodes_db()`: Initializes the episodes index database +- `get_config_dir()`: Gets config directory path based on Docker mode (`/config` in Docker, `.` locally) +- `get_config_path(filename)`: Gets full path to a config file in the appropriate config directory +- `normalize_file_path(file_path)`: Normalizes file path for consistent storage and lookup + - Resolves symlinks and converts to absolute path + - Ensures same file always maps to same path string regardless of OS/environment - `get_metadata(conn, key)`: Retrieves metadata value from database - `set_metadata(conn, key, value)`: Stores metadata value in database - `get_episodes_metadata(conn, key)`: Retrieves episodes database metadata - `set_episodes_metadata(conn, key, value)`: Stores episodes database metadata -- `fetch_episodes_metadata()`: Fetches episodes from Nyaa.si -- `update_episodes_index_db()`: Updates the episodes index database +- `fetch_episodes_metadata(base_url=None)`: Fetches episodes from Nyaa.si + - `base_url`: Optional Nyaa.si search URL (defaults to One-Pace search without quality filter) + - Quality filtering (1080p/720p) is always applied regardless of URL +- `update_episodes_index_db(base_url=None)`: Updates the episodes index database + - `base_url`: Optional Nyaa.si search URL (passed to `fetch_episodes_metadata()`) - `fetch_crc32_links(base_url)`: Fetches CRC32 links from a Nyaa.si URL + - Applies quality filtering (1080p/720p only) via `_process_crc32_row()` - `fetch_title_by_crc32(crc32)`: Searches for a title by CRC32 - `calculate_local_crc32(folder, conn)`: Calculates CRC32 for local files + - Uses normalized paths for database storage and lookup - `rename_local_files(conn)`: Renames local files based on episodes index + - Uses normalized paths when updating database after renaming - `export_db_to_csv(conn)`: Exports database to CSV - `load_crc32_to_title_from_index()`: Loads CRC32-to-title mapping @@ -349,7 +419,8 @@ Helper functions are prefixed with `_` to indicate they are internal implementat **Command handlers** (`_handle_*`): Handle specific command-line operations - `_handle_download_command(args)`: Handles the `--download` command -- `_handle_rename_command(conn)`: Handles the `--rename` command +- `_handle_rename_command(conn, base_url=None)`: Handles the `--rename` command + - `base_url`: Optional URL parameter passed to `update_episodes_index_db()` if update is needed - `_handle_main_commands(args, conn, folder)`: Routes and handles main commands **Getter functions** (`_get_*`): Retrieve or compute values @@ -411,15 +482,41 @@ When working on this project: 1. **CRC32 is the primary identifier** - All episode matching relies on CRC32 checksums 2. **Nyaa.si structure** - Understand the HTML structure of Nyaa.si pages for scraping 3. **Database state** - Always consider existing database state when making changes -4. **File paths** - Handle both absolute and relative paths correctly +4. **File paths** - Always use `normalize_file_path()` before storing/querying file paths in database + - Ensures consistent path representation across different OS and environments + - Critical for consistent behavior between Python and Docker versions + - Resolves symlinks and converts to absolute paths 5. **User interaction** - Some operations require user confirmation (renaming, downloads), but are auto-confirmed in Docker mode 6. **Client abstraction** - New BitTorrent clients should implement the `Client` interface from `clients.py` 7. **Error tolerance** - The tool should continue processing even if individual items fail -8. **Performance** - CRC32 calculation can be slow; caching is essential +8. **Performance** - CRC32 calculation can be slow; caching is essential (uses normalized paths for cache keys) 9. **Web scraping** - Be respectful with rate limiting and error handling 10. **File naming** - Sanitize filenames to be filesystem-safe across platforms 11. **Docker mode** - Check `IS_DOCKER` flag (from `RUN_DOCKER` env var) to determine if running in Docker + - Suppress informational messages for automated commands (`--db`, `--episodes_update`) + - Use `/config` directory for databases and CSV files in Docker mode + - Use `/media` as default folder in Docker mode 12. **Function naming** - Follow the naming conventions in `NAMING_CONVENTIONS.md` when adding new helper functions 13. **Code complexity** - Maintain cognitive complexity ≤ 15 per function (refactoring completed) -14. **Testing** - Comprehensive test suite exists in `tests/` directory +14. **Testing** - Comprehensive test suite exists in `tests/` directory (100+ tests covering all features) 15. **Environment variables** - In Docker mode, prefer environment variables over CLI args for configuration +16. **URL parameter consistency** - Both `fetch_episodes_metadata()` and `fetch_crc32_links()` accept URL parameters + - Always pass `args.url` to ensure consistent URL usage across functions + - Quality filtering is applied regardless of URL parameters +17. **Quality filtering** - Applied in both `fetch_episodes_metadata()` and `fetch_crc32_links()` via `_is_valid_quality()` + - Only accepts 1080p or 720p episodes + - Requires "[One Pace]" marker in filename + - Ensures consistent filtering regardless of URL search parameters +18. **Config directory** - Use `get_config_dir()` and `get_config_path()` for consistent file location handling + - Returns `/config` in Docker mode, `.` in local mode + - Automatically creates directory if it doesn't exist +19. **Message suppression** - Use `suppress_messages=True` in `init_db()` for automated runs + - Prevents "Database already exists" message when running `--db` in Docker +20. **Docker download logic** - Use `DOWNLOAD=true` environment variable to enable downloads (not `TORRENT_CLIENT` presence) + - Missing episodes CSV is always generated/updated before download (if download enabled) + - Download happens as a separate step after report generation +21. **Default connection values** - In Docker mode, use defaults if not specified via environment variables + - Client: transmission + - Host: localhost + - Port: 9091 (transmission) or 8080 (qbittorrent) +22. **Download logging** - Log connection parameters used for download in Docker mode for transparency diff --git a/tests/test_database.py b/tests/test_database.py index 2b18264..ca5f1ab 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -50,6 +50,38 @@ def test_init_episodes_db_creates_tables(self, temp_dir, monkeypatch): conn.close() os.remove(os.path.join(temp_dir, 'test_episodes_index.db')) + def test_init_db_suppresses_messages_when_requested(self, temp_dir): + """Test that init_db suppresses messages when suppress_messages=True.""" + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + # First call creates the database (should show message if suppress_messages=False) + conn1 = acepace.init_db(suppress_messages=False) + conn1.close() + + # Second call with suppress_messages=True should not print message + with patch('builtins.print') as mock_print: + conn2 = acepace.init_db(suppress_messages=True) + conn2.close() + + # Verify "Database already exists" message was NOT printed + print_calls = [str(c) for c in mock_print.call_args_list] + assert not any("Database already exists" in str(call) for call in print_calls) + + def test_init_db_shows_messages_when_not_suppressed(self, temp_dir): + """Test that init_db shows messages when suppress_messages=False (default).""" + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + # First call creates the database + conn1 = acepace.init_db() + conn1.close() + + # Second call should show message (default behavior) + with patch('builtins.print') as mock_print: + conn2 = acepace.init_db(suppress_messages=False) + conn2.close() + + # Verify "Database already exists" message WAS printed + print_calls = [str(c) for c in mock_print.call_args_list] + assert any("Database already exists" in str(call) for call in print_calls) + class TestMetadataOperations: """Tests for metadata get/set operations.""" diff --git a/tests/test_main_command.py b/tests/test_main_command.py new file mode 100644 index 0000000..a116d65 --- /dev/null +++ b/tests/test_main_command.py @@ -0,0 +1,377 @@ +"""Unit tests for main command execution and Docker mode behavior.""" +import pytest +import os +import sys +from unittest.mock import patch, MagicMock, call + +# Add parent directory to path to import acepace +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import acepace + + +class TestDockerModeBehavior: + """Tests for Docker mode specific behavior.""" + + @patch('acepace.IS_DOCKER', True) + @patch('acepace._validate_url') + @patch('acepace._show_episodes_metadata_status') + @patch('acepace.init_db') + @patch('acepace._get_folder_from_args') + @patch('acepace._handle_main_commands') + def test_docker_mode_message_not_shown_for_db_command(self, mock_handle, mock_folder, mock_init_db, + mock_show_status, mock_validate): + """Test that Docker mode message is not shown for --db command.""" + mock_validate.return_value = True + mock_init_db.return_value = MagicMock() + mock_folder.return_value = "/media" + + # Create mock args with --db + mock_args = MagicMock() + mock_args.help = False + mock_args.db = True + mock_args.episodes_update = False + mock_args.download = False + mock_args.rename = False + mock_args.url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + mock_args.folder = None + + with patch('acepace._parse_arguments', return_value=mock_args): + with patch('builtins.print') as mock_print: + acepace.main() + + # Verify "Running in Docker mode" was NOT printed + print_calls = [str(c) for c in mock_print.call_args_list] + assert not any("Running in Docker mode" in str(call) for call in print_calls) + + @patch('acepace.IS_DOCKER', True) + @patch('acepace._validate_url') + @patch('acepace._show_episodes_metadata_status') + @patch('acepace.update_episodes_index_db') + def test_docker_mode_message_not_shown_for_episodes_update(self, mock_update, mock_show_status, mock_validate): + """Test that Docker mode message is not shown for --episodes_update command.""" + mock_validate.return_value = True + + # Create mock args with --episodes_update + mock_args = MagicMock() + mock_args.help = False + mock_args.db = False + mock_args.episodes_update = True + mock_args.url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + + with patch('acepace._parse_arguments', return_value=mock_args): + with patch('builtins.print') as mock_print: + acepace.main() + + # Verify "Running in Docker mode" was NOT printed + print_calls = [str(c) for c in mock_print.call_args_list] + assert not any("Running in Docker mode" in str(call) for call in print_calls) + + @patch('acepace.IS_DOCKER', True) + @patch('acepace._validate_url') + @patch('acepace._show_episodes_metadata_status') + @patch('acepace.init_db') + @patch('acepace._get_folder_from_args') + @patch('acepace._handle_main_commands') + def test_docker_mode_message_shown_for_main_command(self, mock_handle, mock_folder, mock_init_db, + mock_show_status, mock_validate): + """Test that Docker mode message is shown for main command (not --db or --episodes_update).""" + mock_validate.return_value = True + mock_init_db.return_value = MagicMock() + mock_folder.return_value = "/media" + + # Create mock args for main command (no --db or --episodes_update) + mock_args = MagicMock() + mock_args.help = False + mock_args.db = False + mock_args.episodes_update = False + mock_args.download = False + mock_args.rename = False + mock_args.url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + mock_args.folder = None + + with patch('acepace._parse_arguments', return_value=mock_args): + with patch('builtins.print') as mock_print: + acepace.main() + + # Verify "Running in Docker mode" WAS printed + print_calls = [str(c) for c in mock_print.call_args_list] + assert any("Running in Docker mode" in str(call) for call in print_calls) + + @patch('acepace.IS_DOCKER', False) + @patch('acepace._validate_url') + @patch('acepace._show_episodes_metadata_status') + @patch('acepace.init_db') + @patch('acepace._get_folder_from_args') + @patch('acepace._handle_main_commands') + def test_docker_mode_message_not_shown_when_not_in_docker(self, mock_handle, mock_folder, mock_init_db, + mock_show_status, mock_validate): + """Test that Docker mode message is not shown when not running in Docker.""" + mock_validate.return_value = True + mock_init_db.return_value = MagicMock() + mock_folder.return_value = "/media" + + # Create mock args for main command + mock_args = MagicMock() + mock_args.help = False + mock_args.db = False + mock_args.episodes_update = False + mock_args.download = False + mock_args.rename = False + mock_args.url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + mock_args.folder = None + + with patch('acepace._parse_arguments', return_value=mock_args): + with patch('builtins.print') as mock_print: + acepace.main() + + # Verify "Running in Docker mode" was NOT printed + print_calls = [str(c) for c in mock_print.call_args_list] + assert not any("Running in Docker mode" in str(call) for call in print_calls) + + +class TestEpisodesMetadataStatusSuppression: + """Tests for episodes metadata status message suppression.""" + + @patch('acepace._validate_url') + @patch('acepace._show_episodes_metadata_status') + @patch('acepace.init_db') + @patch('acepace.export_db_to_csv') + def test_episodes_metadata_status_not_shown_for_db_command(self, mock_export, mock_init_db, + mock_show_status, mock_validate): + """Test that episodes metadata status is not shown for --db command.""" + mock_validate.return_value = True + mock_conn = MagicMock() + mock_init_db.return_value = mock_conn + + # Create mock args with --db + mock_args = MagicMock() + mock_args.help = False + mock_args.db = True + mock_args.episodes_update = False + mock_args.download = False + mock_args.rename = False + mock_args.url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + mock_args.folder = "/media" + + with patch('acepace._parse_arguments', return_value=mock_args): + with patch('acepace._get_folder_from_args', return_value="/media"): + acepace.main() + + # Verify _show_episodes_metadata_status was NOT called + mock_show_status.assert_not_called() + + @patch('acepace._validate_url') + @patch('acepace._show_episodes_metadata_status') + @patch('acepace.update_episodes_index_db') + def test_episodes_metadata_status_not_shown_for_episodes_update(self, mock_update, mock_show_status, mock_validate): + """Test that episodes metadata status is not shown for --episodes_update command.""" + mock_validate.return_value = True + + # Create mock args with --episodes_update + mock_args = MagicMock() + mock_args.help = False + mock_args.db = False + mock_args.episodes_update = True + mock_args.url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + + with patch('acepace._parse_arguments', return_value=mock_args): + acepace.main() + + # Verify _show_episodes_metadata_status was NOT called + mock_show_status.assert_not_called() + + @patch('acepace._validate_url') + @patch('acepace._show_episodes_metadata_status') + @patch('acepace.init_db') + @patch('acepace._get_folder_from_args') + @patch('acepace._handle_main_commands') + def test_episodes_metadata_status_shown_for_main_command(self, mock_handle, mock_folder, mock_init_db, + mock_show_status, mock_validate): + """Test that episodes metadata status is shown for main command.""" + mock_validate.return_value = True + mock_init_db.return_value = MagicMock() + mock_folder.return_value = "/media" + + # Create mock args for main command + mock_args = MagicMock() + mock_args.help = False + mock_args.db = False + mock_args.episodes_update = False + mock_args.download = False + mock_args.rename = False + mock_args.url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + mock_args.folder = None + + with patch('acepace._parse_arguments', return_value=mock_args): + acepace.main() + + # Verify _show_episodes_metadata_status WAS called + mock_show_status.assert_called_once() + + +class TestURLParameterPropagation: + """Tests for URL parameter propagation through command chain.""" + + @patch('acepace._validate_url') + @patch('acepace.update_episodes_index_db') + def test_episodes_update_receives_url_parameter(self, mock_update, mock_validate): + """Test that --episodes_update receives URL parameter from args.""" + mock_validate.return_value = True + + test_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + + # Create mock args with --episodes_update and URL + mock_args = MagicMock() + mock_args.help = False + mock_args.db = False + mock_args.episodes_update = True + mock_args.url = test_url + + with patch('acepace._parse_arguments', return_value=mock_args): + acepace.main() + + # Verify update_episodes_index_db was called with URL + mock_update.assert_called_once_with(test_url) + + @patch('acepace._validate_url') + @patch('acepace.init_db') + @patch('acepace._get_folder_from_args') + @patch('acepace._handle_rename_command') + def test_rename_receives_url_parameter(self, mock_rename, mock_folder, mock_init_db, mock_validate): + """Test that --rename receives URL parameter from args.""" + mock_validate.return_value = True + mock_init_db.return_value = MagicMock() + mock_folder.return_value = "/media" + + test_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" + + # Create mock args with --rename and URL + mock_args = MagicMock() + mock_args.help = False + mock_args.db = False + mock_args.episodes_update = False + mock_args.download = False + mock_args.rename = True + mock_args.url = test_url + mock_args.folder = None + + with patch('acepace._parse_arguments', return_value=mock_args): + acepace.main() + + # Verify _handle_rename_command was called with URL + mock_rename.assert_called_once() + # Check that URL was passed (second argument after conn) + call_args = mock_rename.call_args + assert call_args[0][1] == test_url # Second positional argument is URL + + +class TestDockerDownloadDefaults: + """Tests for Docker download default values and logging.""" + + @patch('acepace.IS_DOCKER', True) + @patch('acepace._load_magnet_links') + @patch('acepace.get_client') + def test_docker_uses_default_connection_values(self, mock_get_client, mock_load_magnets): + """Test that Docker mode uses default connection values when not specified.""" + mock_load_magnets.return_value = ["magnet:?xt=urn:btih:test123"] + mock_client_obj = MagicMock() + mock_get_client.return_value = mock_client_obj + + # Create mock args without client/host/port specified + mock_args = MagicMock() + mock_args.client = None + mock_args.host = None + mock_args.port = None + mock_args.username = None + mock_args.password = None + mock_args.download_folder = None + mock_args.tag = None + mock_args.category = None + + with patch.dict('os.environ', {}, clear=False): + # Remove any TORRENT_* env vars to test defaults + for key in ['TORRENT_CLIENT', 'TORRENT_HOST', 'TORRENT_PORT', 'TORRENT_USER', 'TORRENT_PASSWORD']: + if key in os.environ: + del os.environ[key] + + acepace._handle_download_command(mock_args) + + # Verify default values were used + mock_get_client.assert_called_once() + call_args = mock_get_client.call_args + assert call_args[0][0] == "transmission" # Default client + assert call_args[0][1] == "localhost" # Default host + assert call_args[0][2] == 9091 # Default port for transmission + + @patch('acepace.IS_DOCKER', True) + @patch('acepace._load_magnet_links') + @patch('acepace.get_client') + def test_docker_logs_connection_parameters(self, mock_get_client, mock_load_magnets): + """Test that Docker mode logs connection parameters used for download.""" + mock_load_magnets.return_value = ["magnet:?xt=urn:btih:test123"] + mock_client_obj = MagicMock() + mock_get_client.return_value = mock_client_obj + + # Create mock args + mock_args = MagicMock() + mock_args.client = None + mock_args.host = None + mock_args.port = None + mock_args.username = None + mock_args.password = None + mock_args.download_folder = None + mock_args.tag = None + mock_args.category = None + + with patch.dict('os.environ', {}, clear=False): + # Remove any TORRENT_* env vars + for key in ['TORRENT_CLIENT', 'TORRENT_HOST', 'TORRENT_PORT', 'TORRENT_USER', 'TORRENT_PASSWORD']: + if key in os.environ: + del os.environ[key] + + with patch('builtins.print') as mock_print: + acepace._handle_download_command(mock_args) + + # Verify connection parameters were logged + print_calls = [str(c) for c in mock_print.call_args_list] + assert any("Download configuration:" in str(call) for call in print_calls) + assert any("Client:" in str(call) for call in print_calls) + assert any("Host:" in str(call) for call in print_calls) + assert any("Port:" in str(call) for call in print_calls) + + @patch('acepace.IS_DOCKER', True) + @patch('acepace._load_magnet_links') + @patch('acepace.get_client') + def test_docker_uses_environment_variable_overrides(self, mock_get_client, mock_load_magnets): + """Test that Docker mode uses environment variables to override defaults.""" + mock_load_magnets.return_value = ["magnet:?xt=urn:btih:test123"] + mock_client_obj = MagicMock() + mock_get_client.return_value = mock_client_obj + + # Create mock args + mock_args = MagicMock() + mock_args.client = None + mock_args.host = None + mock_args.port = None + mock_args.username = None + mock_args.password = None + mock_args.download_folder = None + mock_args.tag = None + mock_args.category = None + + with patch.dict('os.environ', { + 'TORRENT_CLIENT': 'qbittorrent', + 'TORRENT_HOST': '192.168.1.100', + 'TORRENT_PORT': '8080', + 'TORRENT_USER': 'admin' + }): + acepace._handle_download_command(mock_args) + + # Verify environment variable values were used + mock_get_client.assert_called_once() + call_args = mock_get_client.call_args + assert call_args[0][0] == "qbittorrent" # From env var + assert call_args[0][1] == "192.168.1.100" # From env var + assert call_args[0][2] == 8080 # From env var + assert call_args[0][3] == "admin" # From env var From 64cb539abf151728eed35033424b2a2eb9e281a3 Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 26 Jan 2026 15:56:52 +0000 Subject: [PATCH 52/75] Fix quality handling in Docker mode- maybe --- acepace.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/acepace.py b/acepace.py index 24724ee..e6eef8f 100644 --- a/acepace.py +++ b/acepace.py @@ -1000,7 +1000,8 @@ def _print_help(): General Options: --url URL Custom Nyaa search URL - Default: https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc + Default: https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc + Note: Quality filtering (1080p/720p) is applied in code regardless of URL Must point to a valid Nyaa domain --folder PATH Folder containing local video files @@ -1052,8 +1053,8 @@ def _parse_arguments(): ) parser.add_argument( "--url", - default=f"{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace+1080p&o=asc", - help=f"Base URL without the page param. Default includes 1080p filter. Example: '{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace+1080p&o=asc' ", + default=f"{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace&o=asc", + help=f"Base URL without the page param. Default searches for 'one pace' without quality filter (quality filtering 1080p/720p is applied in code). Example: '{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace&o=asc' ", ) parser.add_argument("--folder", help="Folder containing local video files.") parser.add_argument( From 59028efbbf9f6c56eae9d1836908a38d759188b9 Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 26 Jan 2026 16:22:25 +0000 Subject: [PATCH 53/75] Refine quality filtering to support 1080p only; update docs & Docker config accordingly. Removed naming conventions file --- NAMING_CONVENTIONS.md | 137 ------------------------------------------ README.md | 50 +++++++++------ acepace.py | 30 ++++----- docker-compose.yml | 5 +- spec.md | 21 +++---- 5 files changed, 60 insertions(+), 183 deletions(-) delete mode 100644 NAMING_CONVENTIONS.md diff --git a/NAMING_CONVENTIONS.md b/NAMING_CONVENTIONS.md deleted file mode 100644 index 2b60beb..0000000 --- a/NAMING_CONVENTIONS.md +++ /dev/null @@ -1,137 +0,0 @@ -# Function Naming Conventions - -This document explains the naming logic for helper functions in `acepace.py`, particularly those added during the SonarQube refactoring. - -## Naming Pattern Overview - -Helper functions use a consistent naming pattern: `___` or `__` - -The underscore prefix (`_`) indicates these are **private/internal functions** that are not part of the public API. - -## Naming Categories - -### 1. Extraction Functions: `_extract_*` - -Functions that extract data from structures (HTML, files, etc.): - -- `_extract_title_link_from_row(row)` - Extracts the title link element from an HTML table row -- `_extract_filenames_from_folder_structure(filelist_div)` - Extracts filenames from a folder-based torrent file list -- `_extract_filenames_from_torrent_page(torrent_soup)` - Extracts all filenames from a torrent page's file list -- `_extract_matching_titles_from_rows(rows, crc32)` - Extracts titles matching a specific CRC32 from table rows - -**Pattern**: `_extract__from_` - -### 2. Processing Functions: `_process_*` - -Functions that process data structures or perform transformations: - -- `_process_fname_entry(fname_text, ...)` - Processes a filename entry to extract and store CRC32 -- `_process_torrent_page(page_link, ...)` - Processes a torrent page to extract CRC32 information -- `_process_episode_row(row, ...)` - Processes a single table row to extract episode information -- `_process_crc32_row(row, ...)` - Processes a table row to extract CRC32 information for missing episodes - -**Pattern**: `_process__` - -### 3. Validation Functions: `_is_*`, `_validate_*` - -Functions that validate or check conditions: - -- `_is_valid_quality(fname_text)` - Checks if a filename has valid quality (1080p or 720p) -- `_validate_url(url)` - Validates that a URL points to a valid Nyaa domain - -**Pattern**: `_is_` or `_validate_` - -### 4. Command Handlers: `_handle_*` - -Functions that handle specific command-line operations: - -- `_handle_download_command(args)` - Handles the `--download` command -- `_handle_rename_command(conn)` - Handles the `--rename` command -- `_handle_main_commands(args, conn, folder)` - Routes and handles main command execution - -**Pattern**: `_handle__` - -### 5. Getter Functions: `_get_*` - -Functions that retrieve or compute values: - -- `_get_total_pages(soup)` - Extracts total number of pages from pagination controls -- `_get_folder_from_args(args, conn, needs_folder)` - Gets folder path from arguments or prompts user -- `_get_rename_prompt(last_ep_update)` - Gets user prompt for rename confirmation - -**Pattern**: `_get__` - -### 6. Load/Save Functions: `_load_*`, `_save_*` - -Functions that load or save data: - -- `_load_old_missing_crc32s()` - Loads CRC32s from previous missing CSV file -- `_save_missing_episodes_csv(...)` - Saves missing episodes to CSV file - -**Pattern**: `_load_` or `_save__` - -### 7. Print/Report Functions: `_print_*`, `_report_*`, `_show_*` - -Functions that display information: - -- `_print_report_header(conn, folder, args)` - Prints header information for the report -- `_report_new_missing_episodes(missing, crc32_to_text)` - Reports newly detected missing episodes -- `_show_episodes_metadata_status()` - Shows last episodes metadata update status - -**Pattern**: `_print_`, `_report_`, or `_show_` - -### 8. Workflow Functions: `_generate_*`, `_calculate_*` - -Functions that orchestrate multi-step workflows: - -- `_generate_missing_episodes_report(conn, folder, args)` - Generates and saves missing episodes report -- `_calculate_and_find_missing(folder, conn, args, last_run)` - Calculates local CRC32s and finds missing episodes - -**Pattern**: `_generate_` or `_calculate__and_` - -### 9. Utility Functions: `_parse_*`, `_count_*` - -General utility functions: - -- `_parse_arguments()` - Parses command-line arguments -- `_count_video_files(folder, conn)` - Counts total video files and files already recorded in DB - -**Pattern**: `_parse_` or `_count_` - -## Design Principles - -1. **Single Responsibility**: Each function has one clear purpose -2. **Descriptive Names**: Function names clearly describe what they do -3. **Consistent Patterns**: Similar functions follow similar naming patterns -4. **Private by Default**: All helpers are prefixed with `_` to indicate internal use -5. **Context Clarity**: Function names include enough context to understand their purpose - -## Benefits of This Naming Convention - -1. **Readability**: Easy to understand what a function does from its name -2. **Discoverability**: Functions can be found by their action prefix -3. **Maintainability**: Clear separation between public API and internal helpers -4. **Documentation**: Function names serve as inline documentation -5. **Refactoring**: Easy to identify and group related functions - -## Example Usage Flow - -When reading code, you can quickly understand the flow: - -```python -# Main entry point -main() - → _parse_arguments() # Get user input - → _validate_url() # Check URL is valid - → _show_episodes_metadata_status() # Display status - → _get_folder_from_args() # Get or prompt for folder - → _handle_main_commands() # Route to appropriate handler - → _generate_missing_episodes_report() # Main workflow - → _print_report_header() # Show header info - → _calculate_and_find_missing() # Find missing episodes - → fetch_crc32_links() # Public API function - → _report_new_missing_episodes() # Show new episodes - → _save_missing_episodes_csv() # Save results -``` - -This naming convention makes the codebase more maintainable and easier to understand. diff --git a/README.md b/README.md index df3f3fe..62e21e8 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,15 @@ You can run Ace-Pace using `docker run` with environment variables and volume mo ```bash docker run --rm \ -v /path/to/OnePaceLibrary:/media:rw \ - -v $(pwd)/crc32_files.db:/app/crc32_files.db:rw \ - -v $(pwd)/episodes_index.db:/app/episodes_index.db:rw \ - -v $(pwd)/Ace-Pace_Missing.csv:/app/Ace-Pace_Missing.csv:rw \ + -v /path/to/config:/config:rw \ -e TZ=Europe/London \ - -e TORRENT_HOST=127.0.0.1 \ - -e TORRENT_PORT=9091 \ - -e TORRENT_CLIENT=transmission \ - -e NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc \ -e DB=true \ -e EPISODES_UPDATE=true \ + -e DOWNLOAD=false \ + -e TORRENT_CLIENT=transmission \ + -e TORRENT_HOST=127.0.0.1 \ + -e TORRENT_PORT=9091 \ + -e NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc \ timothe/ace-pace:latest ``` @@ -46,10 +45,11 @@ docker run --rm \ For easier management, you can use the provided `docker-compose.yml` file. First, edit the compose file to match your setup: -1. Update the volume path for your One-Pace library: +1. Update the volume paths: ```yaml volumes: - /path/to/OnePaceLibrary:/media:rw + - /path/to/config:/config:rw ``` 2. Configure environment variables as needed (Torrent client settings, Nyaa URL, etc.) @@ -68,29 +68,42 @@ docker-compose up -d The following environment variables can be used to configure Ace-Pace in Docker: -- `NYAA_URL` - Nyaa.si search URL (default: `https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc`) +- `NYAA_URL` - Nyaa.si search URL (optional, default: `https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc`) + - When not set, uses default URL without quality filter. Quality filtering (1080p only) is always applied in code. +- `DB` - Set to `true` to generate CSV database export on container start (default: `false`) +- `EPISODES_UPDATE` - Set to `true` to update episodes metadata from Nyaa on container start (default: `false`) +- `DOWNLOAD` - Set to `true` to automatically download missing episodes after generating report (default: `false`) - `TORRENT_CLIENT` - BitTorrent client type: `transmission` or `qbittorrent` (default: `transmission`) -- `TORRENT_HOST` - BitTorrent client host address (default: `127.0.0.1`) +- `TORRENT_HOST` - BitTorrent client host address (default: `localhost`) - `TORRENT_PORT` - BitTorrent client port (default: `9091` for Transmission, `8080` for qBittorrent) - `TORRENT_USER` - BitTorrent client username (optional) - `TORRENT_PASSWORD` - BitTorrent client password (optional) -- `DB` - Set to `true` to generate CSV database export (default: `true`) -- `EPISODES_UPDATE` - Set to `true` to update episodes metadata from Nyaa (default: `true`) -- `TZ` - Timezone (default: `Europe/Berlin`) +- `TZ` - Timezone (default: `Europe/London`) ### Docker Volume Mounts The following volumes should be mounted for persistent data: - `/media` - Mount your One-Pace library directory here (read-write) -- `/app/crc32_files.db` - Database file for CRC32 checksums (read-write) -- `/app/episodes_index.db` - Database file for episodes index (read-write) -- `/app/Ace-Pace_Missing.csv` - CSV export of missing episodes (read-write) +- `/config` - Mount a directory for persistent configuration and data files (read-write) + - Contains: `crc32_files.db`, `episodes_index.db`, `Ace-Pace_Missing.csv`, `Ace-Pace_DB.csv` + +### Docker Execution Flow + +When the container starts, it executes the following steps in order: + +1. **Episodes Update** (if `EPISODES_UPDATE=true`): Updates the episodes metadata database from Nyaa +2. **Database Export** (if `DB=true`): Exports the CRC32 database to CSV +3. **Missing Episodes Report**: Always runs to generate/update `Ace-Pace_Missing.csv` +4. **Download** (if `DOWNLOAD=true`): Automatically downloads missing episodes via the configured BitTorrent client ### Docker Notes - In Docker mode, Ace-Pace automatically uses `/media` as the default folder path - The container runs non-interactively, so all configuration must be provided via environment variables +- All data files (databases, CSV exports) are stored in `/config` directory +- Quality filtering (1080p only) is applied in code regardless of the URL used +- When `NYAA_URL` is not set, the default URL searches for all "one pace" episodes without quality filter, then filters for 1080p in code - Make sure your BitTorrent client is accessible from within the Docker network (use host network mode or configure networking appropriately) ### VPN Considerations @@ -132,7 +145,8 @@ python acepace.py [-h] [--url URL] [--folder FOLDER] [--db] [--client {transmiss Specify the path to your local One-Pace video library. Ace-Pace will scan this directory recursively to identify and analyze your existing episodes. - `--url ` - Define the Nyaa URL used for the query to get episodes metadata and download links. Defaults to `https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc`. + Define the Nyaa URL used for the query to get episodes metadata and download links. Defaults to `https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc`. + Note: Quality filtering (1080p only) is always applied in code. - `--db` (standalone flag) Create a CSV file with the existing local file paths and CRC32 checksums. Useful to check what's detected and debugging. @@ -170,7 +184,7 @@ python acepace.py [-h] [--url URL] [--folder FOLDER] [--db] [--client {transmiss ### 📚 Some examples ``` -python acepace.py --folder "/volume42/media/One Piece/" --url https://nyaa.si/?f=0&c=0_0&q=one+pace+720p&o=asc +python acepace.py --folder "/volume42/media/One Piece/" --url https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc python acepace.py --folder "/volume42/media/One Piece/" python acepace.py --client transmission --download python acepace.py --client qbittorrent --download --host 192.168.1.100 --port 8080 --username myuser --password mypassword --download-folder /downloads/onepace --tag onepace --tag 'one pace' --category 'anime' diff --git a/acepace.py b/acepace.py index e6eef8f..0a69902 100644 --- a/acepace.py +++ b/acepace.py @@ -18,7 +18,7 @@ # Define regex to extract CRC32 from filename text (commonly in [xxxxx]) CRC32_REGEX = re.compile(r"\[([A-Fa-f0-9]{8})\]") -# Quality regex patterns - matches [1080p], [720p], etc. (case insensitive) +# Quality regex patterns - matches [1080p], etc. (case insensitive) QUALITY_REGEX = re.compile(r"\[(\d+p)\]", re.IGNORECASE) # Video file extensions we care about @@ -154,22 +154,22 @@ def set_episodes_metadata(conn, key, value): # --- New: Fetch and update episodes_index table --- def _is_valid_quality(fname_text): - """Check if filename has valid quality (1080p preferred, 720p as fallback only). - Returns True if quality is 1080p or 720p, False otherwise.""" + """Check if filename has valid quality (1080p only). + Returns True if quality is 1080p, False otherwise.""" quality_matches = QUALITY_REGEX.findall(fname_text) if not quality_matches: return False # No quality marker found, exclude - # Check if quality is exactly 1080p or 720p (not higher, not lower) + # Check if quality is exactly 1080p (not higher, not lower) for quality in quality_matches: quality_num = int(quality.lower().replace('p', '')) - if quality_num == 1080 or quality_num == 720: + if quality_num == 1080: return True - return False # Quality not 1080p or 720p + return False # Quality not 1080p def _process_fname_entry(fname_text, seen_crc32, episodes, page_link): """Helper to extract CRC32 from fname_text and store if valid and unique. - Only accepts episodes with 1080p or 720p quality (720p as fallback).""" + Only accepts episodes with 1080p quality.""" m = CRC32_REGEX.findall(fname_text) found = False if m and "[One Pace]" in fname_text and _is_valid_quality(fname_text): @@ -287,7 +287,7 @@ def fetch_episodes_metadata(base_url=None): If CRC32 not in title, fetch the torrent page and try to extract CRC32s from file list. Args: base_url: Base URL for Nyaa search. If None, uses default without quality filter. - Note: Quality filtering (1080p/720p) is always applied regardless of URL. + Note: Quality filtering (1080p only) is always applied regardless of URL. Returns: List of (crc32, title, page_link) """ if base_url is None: @@ -379,7 +379,7 @@ def set_metadata(conn, key, value): def _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): """Process a single table row to extract CRC32 information. - Only accepts episodes with 1080p or 720p quality (720p as fallback). + Only accepts episodes with 1080p quality. Returns tuple: (success: bool, filename_text: str or None)""" links = row.find_all("a", href=True) title_link = None @@ -396,7 +396,7 @@ def _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): link = NYAA_BASE_URL + title_link["href"] matches = CRC32_REGEX.findall(filename_text) if matches: - # Only accept episodes with valid quality (1080p or 720p) and One Pace marker + # Only accept episodes with valid quality (1080p only) and One Pace marker if "[One Pace]" in filename_text and _is_valid_quality(filename_text): crc32 = matches[-1].upper() crc32_to_link[crc32] = link @@ -874,10 +874,10 @@ def _print_report_header(conn, folder, args): now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") set_metadata(conn, "last_run", now_str) - # Show URL, but note that quality filtering (1080p/720p) is applied regardless + # Show URL, but note that quality filtering (1080p only) is applied regardless url_display = args.url - if "1080p" not in url_display and "720p" not in url_display: - url_display += " (quality filtering: 1080p/720p only)" + if "1080p" not in url_display: + url_display += " (quality filtering: 1080p only)" print(f"Using URL: {url_display}") print(f"Total video files detected: {total_files}") print(f"Episodes already recorded in DB: {recorded_files}") @@ -1001,7 +1001,7 @@ def _print_help(): General Options: --url URL Custom Nyaa search URL Default: https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc - Note: Quality filtering (1080p/720p) is applied in code regardless of URL + Note: Quality filtering (1080p only) is applied in code regardless of URL Must point to a valid Nyaa domain --folder PATH Folder containing local video files @@ -1054,7 +1054,7 @@ def _parse_arguments(): parser.add_argument( "--url", default=f"{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace&o=asc", - help=f"Base URL without the page param. Default searches for 'one pace' without quality filter (quality filtering 1080p/720p is applied in code). Example: '{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace&o=asc' ", + help=f"Base URL without the page param. Default searches for 'one pace' without quality filter (quality filtering 1080p only is applied in code). Example: '{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace&o=asc' ", ) parser.add_argument("--folder", help="Folder containing local video files.") parser.add_argument( diff --git a/docker-compose.yml b/docker-compose.yml index 1545736..0a88d76 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,8 +14,9 @@ services: - DB=true # Update episodes index database on container start (default: false) - EPISODES_UPDATE=true - # Nyaa.si search URL (default: https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc) - #- NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc + # Nyaa.si search URL (optional, default: https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc) + # Quality filtering (1080p only) is always applied in code regardless of URL + #- NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc # Download missing episodes after generating report (default: false) #- DOWNLOAD=true diff --git a/spec.md b/spec.md index 032732b..7ebf9c0 100644 --- a/spec.md +++ b/spec.md @@ -15,12 +15,11 @@ ### 1. Episode Discovery and Indexing - Scrapes Nyaa.si torrent tracker for One-Pace episodes - Extracts CRC32 checksums from episode filenames or torrent file lists -- **Quality Filtering**: Only extracts episodes with 1080p quality, or 720p as fallback +- **Quality Filtering**: Only extracts episodes with 1080p quality - Episodes without quality markers are excluded - - Episodes with quality lower than 720p (480p, 360p, etc.) are excluded - - Episodes with quality higher than 1080p (1440p, 2160p/4K, etc.) are excluded + - Episodes with quality other than 1080p (720p, 480p, 360p, 1440p, 2160p/4K, etc.) are excluded - Quality filtering is applied in both `fetch_episodes_metadata()` and `fetch_crc32_links()` - - Filtering is case-insensitive (accepts 1080P, 720P, etc.) + - Filtering is case-insensitive (accepts 1080P, etc.) - **URL Parameter Support**: Both `fetch_episodes_metadata()` and `update_episodes_index_db()` accept a `base_url` parameter - Allows consistent URL usage across all episode fetching functions - Default URL includes 1080p filter, but can be overridden @@ -43,7 +42,7 @@ ### 3. Missing Episode Detection - Fetches episode list from Nyaa.si using the provided URL (default: One-Pace 1080p search) - **Quality Filtering**: `fetch_crc32_links()` applies quality filtering via `_process_crc32_row()` - - Only accepts episodes with 1080p or 720p quality + - Only accepts episodes with 1080p quality - Requires "[One Pace]" marker in filename - Ensures consistent filtering regardless of URL parameters - Compares local CRC32 checksums against fetched episodes (using normalized paths) @@ -229,7 +228,7 @@ Ace-Pace/ 2. Script uses provided URL or defaults to One-Pace search (without quality filter in URL) 3. Script scrapes all pages of Nyaa.si search results 4. For each torrent, extracts CRC32 from title or file list -5. Applies quality filtering (1080p/720p only) before storing +5. Applies quality filtering (1080p only) before storing 6. Stores CRC32, title, and page link in `episodes_index.db` 7. Updates metadata with last update timestamp 8. Note: Quality filtering is applied regardless of URL parameters @@ -336,7 +335,7 @@ Ace-Pace/ - Integration with media server APIs (Plex, Jellyfin) - Duplicate detection and cleanup - Episode metadata enrichment (thumbnails, descriptions) -- Note: Episode quality filtering (720p, 1080p) is already implemented ✅ +- Note: Episode quality filtering (1080p only) is already implemented ✅ ### Technical Improvements - Async/await for concurrent web scraping @@ -385,11 +384,11 @@ The codebase follows a modular structure with clear separation of concerns: - `set_episodes_metadata(conn, key, value)`: Stores episodes database metadata - `fetch_episodes_metadata(base_url=None)`: Fetches episodes from Nyaa.si - `base_url`: Optional Nyaa.si search URL (defaults to One-Pace search without quality filter) - - Quality filtering (1080p/720p) is always applied regardless of URL + - Quality filtering (1080p only) is always applied regardless of URL - `update_episodes_index_db(base_url=None)`: Updates the episodes index database - `base_url`: Optional Nyaa.si search URL (passed to `fetch_episodes_metadata()`) - `fetch_crc32_links(base_url)`: Fetches CRC32 links from a Nyaa.si URL - - Applies quality filtering (1080p/720p only) via `_process_crc32_row()` + - Applies quality filtering (1080p only) via `_process_crc32_row()` - `fetch_title_by_crc32(crc32)`: Searches for a title by CRC32 - `calculate_local_crc32(folder, conn)`: Calculates CRC32 for local files - Uses normalized paths for database storage and lookup @@ -414,7 +413,7 @@ Helper functions are prefixed with `_` to indicate they are internal implementat - `_process_crc32_row(row, ...)`: Processes row to extract CRC32 for missing episodes **Validation functions** (`_is_*`, `_validate_*`): Validate inputs/data -- `_is_valid_quality(fname_text)`: Checks if filename has valid quality (1080p/720p) +- `_is_valid_quality(fname_text)`: Checks if filename has valid quality (1080p only) - `_validate_url(url)`: Validates URL points to valid Nyaa domain **Command handlers** (`_handle_*`): Handle specific command-line operations @@ -504,7 +503,7 @@ When working on this project: - Always pass `args.url` to ensure consistent URL usage across functions - Quality filtering is applied regardless of URL parameters 17. **Quality filtering** - Applied in both `fetch_episodes_metadata()` and `fetch_crc32_links()` via `_is_valid_quality()` - - Only accepts 1080p or 720p episodes + - Only accepts 1080p episodes - Requires "[One Pace]" marker in filename - Ensures consistent filtering regardless of URL search parameters 18. **Config directory** - Use `get_config_dir()` and `get_config_path()` for consistent file location handling From d0f9ca7468b00b9fa00a906ddcb1da2ba74c4344 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 27 Jan 2026 10:10:51 +0000 Subject: [PATCH 54/75] Implement graceful shutdown handling, enhance episode fetching logic; restrict quality filtering to 1080p only, other misc fixes --- acepace.py | 457 +++++++--- coverage.xml | 1422 ++++++++++++++++-------------- entrypoint.sh | 38 +- tests/test_episodes.py | 329 +++---- tests/test_main_command.py | 7 +- tests/test_missing_detection.py | 12 +- tests/test_path_normalization.py | 235 ++--- 7 files changed, 1331 insertions(+), 1169 deletions(-) diff --git a/acepace.py b/acepace.py index 0a69902..a16cee6 100644 --- a/acepace.py +++ b/acepace.py @@ -6,6 +6,8 @@ import argparse import zlib import os +import signal +import sys from bs4 import BeautifulSoup # type: ignore import requests # type: ignore @@ -15,6 +17,19 @@ # Check if running in Docker (non-interactive mode) IS_DOCKER = "RUN_DOCKER" in os.environ +# Global flag for graceful shutdown +_shutdown_requested = False + +# Shutdown message constant +_SHUTDOWN_MESSAGE = "Shutdown requested, stopping fetch operation..." + + +def _signal_handler(signum, frame): + """Handle shutdown signals gracefully.""" + global _shutdown_requested + _shutdown_requested = True + print("\nShutdown signal received, finishing current operation...") + # Define regex to extract CRC32 from filename text (commonly in [xxxxx]) CRC32_REGEX = re.compile(r"\[([A-Fa-f0-9]{8})\]") @@ -27,6 +42,12 @@ # Constants for repeated string literals HTML_PARSER = "html.parser" NYAA_BASE_URL = "https://nyaa.si" +ONE_PACE_MARKER = "[One Pace]" + +# HTTP and network constants +HTTP_OK = 200 +REQUEST_DELAY_SECONDS = 0.2 +CRC32_CHUNK_SIZE = 8192 # Config directory and file names CONFIG_DIR_DOCKER = "/config" @@ -113,6 +134,9 @@ def init_db(suppress_messages=False): # --- New: Episodes metadata DB --- def init_episodes_db(): + """Initialize the episodes index database. + Creates the episodes_index and metadata tables if they don't exist. + Returns: Database connection object.""" episodes_db_path = get_config_path(EPISODES_DB_NAME) conn = sqlite3.connect(episodes_db_path) c = conn.cursor() @@ -138,6 +162,11 @@ def init_episodes_db(): def get_episodes_metadata(conn, key): + """Get metadata value from episodes database. + Args: + conn: Database connection + key: Metadata key + Returns: Metadata value or None if not found.""" c = conn.cursor() c.execute("SELECT value FROM metadata WHERE key = ?", (key,)) row = c.fetchone() @@ -145,6 +174,11 @@ def get_episodes_metadata(conn, key): def set_episodes_metadata(conn, key, value): + """Set metadata value in episodes database. + Args: + conn: Database connection + key: Metadata key + value: Metadata value""" c = conn.cursor() c.execute( "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", (key, value) @@ -172,7 +206,7 @@ def _process_fname_entry(fname_text, seen_crc32, episodes, page_link): Only accepts episodes with 1080p quality.""" m = CRC32_REGEX.findall(fname_text) found = False - if m and "[One Pace]" in fname_text and _is_valid_quality(fname_text): + if m and ONE_PACE_MARKER in fname_text and _is_valid_quality(fname_text): crc32 = m[-1].upper() if crc32 not in seen_crc32: # print(f"New CRC32 detected: {crc32} -> Title: {fname_text}") @@ -251,7 +285,7 @@ def _process_torrent_page(page_link, seen_crc32, episodes): """Process a torrent page to extract CRC32 information from file list.""" try: torrent_resp = requests.get(page_link) - if torrent_resp.status_code != 200: + if torrent_resp.status_code != HTTP_OK: print(f"Failed to fetch torrent page {page_link}") return False t_soup = BeautifulSoup(torrent_resp.text, HTML_PARSER) @@ -281,6 +315,31 @@ def _process_episode_row(row, seen_crc32, episodes): return _process_torrent_page(page_link, seen_crc32, episodes) +def _fetch_episodes_page(base_url, page, soup=None): + """Fetch a single page of episodes. + Returns tuple: (page_soup, success) where success indicates if page was fetched.""" + if page == 1 and soup is not None: + return soup, True + + resp = requests.get(f"{base_url}&p={page}") + if resp.status_code != HTTP_OK: + print(f"Failed to fetch page {page}, status code: {resp.status_code}") + return None, False + return BeautifulSoup(resp.text, HTML_PARSER), True + + +def _process_episodes_page_rows(page_soup, seen_crc32, episodes): + """Process all rows from an episodes page.""" + table = page_soup.find("table", class_="torrent-list") + if not table: + return + rows = table.find_all("tr") # type: ignore + for row in rows: + if _shutdown_requested: + break + _process_episode_row(row, seen_crc32, episodes) + + def fetch_episodes_metadata(base_url=None): """ Fetch all One Pace episodes from Nyaa, collecting CRC32, title, and page link. @@ -295,37 +354,33 @@ def fetch_episodes_metadata(base_url=None): episodes = [] seen_crc32 = set() - page = 1 print(f"Browsing {base_url}...") - # --- Get total number of pages by parsing first page's pagination controls --- - resp = requests.get(f"{base_url}&p=1") - if resp.status_code != 200: - print(f"Failed to fetch page 1, status code: {resp.status_code}") + # Get total number of pages by parsing first page's pagination controls + soup, success = _fetch_episodes_page(base_url, 1) + if not success: return episodes - soup = BeautifulSoup(resp.text, HTML_PARSER) total_pages = _get_total_pages(soup) - # Now loop from page 1 to total_pages + # Loop from page 1 to total_pages + page = 1 while page <= total_pages: + if _shutdown_requested: + print(_SHUTDOWN_MESSAGE) + break + print(f"Fetching page {page}/{total_pages}...") - if page == 1: - # We've already fetched page 1 above - page_soup = soup - else: - resp = requests.get(f"{base_url}&p={page}") - if resp.status_code != 200: - print(f"Failed to fetch page {page}, status code: {resp.status_code}") - break - page_soup = BeautifulSoup(resp.text, HTML_PARSER) - table = page_soup.find("table", class_="torrent-list") - if not table: + page_soup, success = _fetch_episodes_page(base_url, page, soup if page == 1 else None) + if not success: + break + + _process_episodes_page_rows(page_soup, seen_crc32, episodes) + + if _shutdown_requested: break - rows = table.find_all("tr") # type: ignore - for row in rows: - _process_episode_row(row, seen_crc32, episodes) page += 1 - time.sleep(0.2) + time.sleep(REQUEST_DELAY_SECONDS) + print(f"Fetched {len(episodes)} unique episodes with CRC32s.") return episodes @@ -340,6 +395,10 @@ def update_episodes_index_db(base_url=None): c = conn.cursor() count = 0 for crc32, title, page_link in episodes: + # Check for shutdown request during processing + if _shutdown_requested: + print("Shutdown requested, committing partial update...") + break c.execute( "INSERT OR REPLACE INTO episodes_index (crc32, title, page_link) VALUES (?, ?, ?)", (crc32, title, page_link), @@ -354,6 +413,8 @@ def update_episodes_index_db(base_url=None): def load_crc32_to_title_from_index(): + """Load CRC32 to title mapping from episodes index database. + Returns: Dictionary mapping CRC32 to episode title.""" conn = init_episodes_db() c = conn.cursor() c.execute("SELECT crc32, title FROM episodes_index") @@ -363,6 +424,11 @@ def load_crc32_to_title_from_index(): def get_metadata(conn, key): + """Get metadata value from database. + Args: + conn: Database connection + key: Metadata key + Returns: Metadata value or None if not found.""" c = conn.cursor() c.execute("SELECT value FROM metadata WHERE key = ?", (key,)) row = c.fetchone() @@ -370,6 +436,11 @@ def get_metadata(conn, key): def set_metadata(conn, key, value): + """Set metadata value in database. + Args: + conn: Database connection + key: Metadata key + value: Metadata value""" c = conn.cursor() c.execute( "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", (key, value) @@ -377,10 +448,9 @@ def set_metadata(conn, key, value): conn.commit() -def _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): - """Process a single table row to extract CRC32 information. - Only accepts episodes with 1080p quality. - Returns tuple: (success: bool, filename_text: str or None)""" +def _extract_links_from_row(row): + """Extract title link and magnet link from a table row. + Returns tuple: (title_link, magnet_link) or (None, "") if not found.""" links = row.find_all("a", href=True) title_link = None magnet_link = "" @@ -390,57 +460,142 @@ def _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): href = a.get("href", "") if href.startswith("magnet:"): magnet_link = href + return title_link, magnet_link + + +def _process_title_with_crc32(filename_text, link, magnet_link, crc32_to_link, crc32_to_text, crc32_to_magnet): + """Process a title that has CRC32 in it. + Returns True if successfully processed, False otherwise.""" + matches = CRC32_REGEX.findall(filename_text) + if matches: + crc32 = matches[-1].upper() + crc32_to_link[crc32] = link + crc32_to_text[crc32] = filename_text + crc32_to_magnet[crc32] = magnet_link + return True + return False + + +def _process_torrent_page_for_crc32(link, magnet_link, crc32_to_link, crc32_to_text, crc32_to_magnet): + """Fetch torrent page and extract CRC32 from file list. + Returns True if CRC32 found, False otherwise.""" + try: + torrent_resp = requests.get(link) + if torrent_resp.status_code == HTTP_OK: + t_soup = BeautifulSoup(torrent_resp.text, HTML_PARSER) + filenames = _extract_filenames_from_torrent_page(t_soup) + for fname in filenames: + fname_str = str(fname) + if ONE_PACE_MARKER in fname_str and _is_valid_quality(fname_str): + fname_matches = CRC32_REGEX.findall(fname_str) + if fname_matches: + crc32 = fname_matches[-1].upper() + crc32_to_link[crc32] = link + crc32_to_text[crc32] = fname_str + crc32_to_magnet[crc32] = magnet_link + return True + except (requests.RequestException, AttributeError, TypeError): + pass + return False + + +def _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet): + """Process a single table row to extract CRC32 information. + Only accepts episodes with 1080p quality. + If CRC32 not in title, fetches torrent page to extract from file list. + Returns tuple: (success: bool, filename_text: str or None, should_warn: bool) + where should_warn indicates if a warning should be shown (only when CRC32 is missing, not when quality is wrong).""" + title_link, magnet_link = _extract_links_from_row(row) if not title_link: - return False, None # Skip rows without a valid title link + return False, None, False + filename_text = title_link.text link = NYAA_BASE_URL + title_link["href"] - matches = CRC32_REGEX.findall(filename_text) - if matches: - # Only accept episodes with valid quality (1080p only) and One Pace marker - if "[One Pace]" in filename_text and _is_valid_quality(filename_text): - crc32 = matches[-1].upper() - crc32_to_link[crc32] = link - crc32_to_text[crc32] = filename_text - crc32_to_magnet[crc32] = magnet_link - return True, filename_text - return False, filename_text + + # Check if it's a One Pace episode first + if ONE_PACE_MARKER not in filename_text: + return False, filename_text, False + + # Check quality first - if not 1080p, silently skip (don't warn) + if not _is_valid_quality(filename_text): + return False, filename_text, False + + # Quality is valid (1080p), now check for CRC32 + if _process_title_with_crc32(filename_text, link, magnet_link, crc32_to_link, crc32_to_text, crc32_to_magnet): + return True, filename_text, False + + # CRC32 not in title, try fetching torrent page + if _process_torrent_page_for_crc32(link, magnet_link, crc32_to_link, crc32_to_text, crc32_to_magnet): + return True, filename_text, False + + # CRC32 not found in title or torrent page, but quality is valid - should warn + return False, filename_text, True + + +def _fetch_crc32_page(base_url, page): + """Fetch a single page for CRC32 links. + Returns tuple: (soup, success) where success indicates if page was fetched.""" + print(f"Fetching page {page}...") + resp = requests.get(f"{base_url}&p={page}") + if resp.status_code != HTTP_OK: + print(f"Failed to fetch page {page}, status code: {resp.status_code}") + return None, False + return BeautifulSoup(resp.text, HTML_PARSER), True + + +def _process_crc32_page_rows(soup, crc32_to_link, crc32_to_text, crc32_to_magnet): + """Process all rows from a CRC32 links page. + Returns True if any episodes were found, False otherwise.""" + table = soup.find("table", class_="torrent-list") + if not table: + print("No table found, stopping.") + return False + + rows = table.find_all("tr") # type: ignore + if not rows: + print("No rows found, stopping.") + return False + + found_in_page = False + for row in rows: + if _shutdown_requested: + print(_SHUTDOWN_MESSAGE) + break + success, filename_text, should_warn = _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet) + if success: + found_in_page = True + elif should_warn and filename_text: + print(f"Warning: No CRC32 found in title '{filename_text}'") + + return found_in_page def fetch_crc32_links(base_url): + """Fetch CRC32 links from Nyaa.si search URL. + Only accepts episodes with 1080p quality. + Args: + base_url: Nyaa.si search URL + Returns: Tuple of (crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page)""" crc32_to_link = {} crc32_to_text = {} crc32_to_magnet = {} page = 1 last_checked_page = 0 + while True: - print(f"Fetching page {page}...") - resp = requests.get(f"{base_url}&p={page}") - if resp.status_code != 200: - print(f"Failed to fetch page {page}, status code: {resp.status_code}") + if _shutdown_requested: + print(_SHUTDOWN_MESSAGE) break - - soup = BeautifulSoup(resp.text, HTML_PARSER) - table = soup.find("table", class_="torrent-list") - if not table: - print("No table found, stopping.") + + soup, success = _fetch_crc32_page(base_url, page) + if not success: break - - rows = table.find_all("tr") # type: ignore - if not rows: - print("No rows found, stopping.") + + found_in_page = _process_crc32_page_rows(soup, crc32_to_link, crc32_to_text, crc32_to_magnet) + + if _shutdown_requested or not found_in_page: break - - found_in_page = False - for row in rows: - success, filename_text = _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet) - if success: - found_in_page = True - elif filename_text: - print(f"Warning: No CRC32 found in title '{filename_text}'") - - if not found_in_page: - break # No more entries found - + last_checked_page = page page += 1 @@ -463,10 +618,14 @@ def _extract_matching_titles_from_rows(rows, crc32): def fetch_title_by_crc32(crc32): + """Search on Nyaa for the given CRC32 and return the episode title. + Args: + crc32: CRC32 checksum to search for + Returns: Episode title if exactly one match found, None otherwise.""" # Search on Nyaa for the given CRC32 search_url = f"{NYAA_BASE_URL}/?f=0&c=0_0&q={crc32}&o=asc" resp = requests.get(search_url) - if resp.status_code != 200: + if resp.status_code != HTTP_OK: print(f"Failed to fetch search results for CRC32 {crc32}") return None soup = BeautifulSoup(resp.text, HTML_PARSER) @@ -487,39 +646,72 @@ def fetch_title_by_crc32(crc32): return None +def _calculate_file_crc32(file_path): + """Calculate CRC32 for a single file. + Returns the CRC32 as a string, or None if calculation was interrupted.""" + with open(file_path, "rb") as f: + crc = 0 + while chunk := f.read(CRC32_CHUNK_SIZE): + if _shutdown_requested: + return None + crc = zlib.crc32(chunk, crc) + return f"{crc & 0xFFFFFFFF:08X}" + + +def _process_video_file(file_path, c, conn, local_crc32s): + """Process a single video file: check cache or calculate CRC32. + Returns True if file was processed successfully.""" + normalized_path = normalize_file_path(file_path) + + # Check if already in DB + c.execute("SELECT crc32 FROM crc32_cache WHERE file_path = ?", (normalized_path,)) + row = c.fetchone() + if row: + local_crc32s.add(row[0]) + return True + + # Calculate CRC32 + parent_folder = os.path.basename(os.path.dirname(file_path)) + file_name = os.path.basename(file_path) + print(f"Calculating CRC32 for {parent_folder}/{file_name}...") + + crc32 = _calculate_file_crc32(file_path) + if crc32 is None: + return False # Calculation interrupted + + local_crc32s.add(crc32) + c.execute( + "INSERT OR REPLACE INTO crc32_cache (file_path, crc32) VALUES (?, ?)", + (normalized_path, crc32), + ) + conn.commit() + return True + + def calculate_local_crc32(folder, conn): + """Calculate CRC32 checksums for all video files in the given folder. + Uses cached values from database when available. + Args: + folder: Folder path to scan for video files + conn: Database connection + Returns: Set of CRC32 checksums found in the folder.""" local_crc32s = set() c = conn.cursor() + for root, dirs, files in os.walk(folder): + if _shutdown_requested: + print("Shutdown requested, stopping file processing...") + break + for file in files: + if _shutdown_requested: + break + ext = os.path.splitext(file)[1].lower() if ext in VIDEO_EXTENSIONS: file_path = os.path.join(root, file) - # Normalize path for consistent storage and lookup - normalized_path = normalize_file_path(file_path) - # Check if normalized_path already in DB - c.execute( - "SELECT crc32 FROM crc32_cache WHERE file_path = ?", (normalized_path,) - ) - row = c.fetchone() - if row: - crc32 = row[0] - local_crc32s.add(crc32) - continue - - parent_folder = os.path.basename(root) - print(f"Calculating CRC32 for {parent_folder}/{file}...") - with open(file_path, "rb") as f: - crc = 0 - while chunk := f.read(8192): - crc = zlib.crc32(chunk, crc) - crc32 = f"{crc & 0xFFFFFFFF:08X}" - local_crc32s.add(crc32) - c.execute( - "INSERT OR REPLACE INTO crc32_cache (file_path, crc32) VALUES (?, ?)", - (normalized_path, crc32), - ) - conn.commit() + _process_video_file(file_path, c, conn, local_crc32s) + return local_crc32s @@ -570,6 +762,11 @@ def _execute_rename(rename_plan, conn): def rename_local_files(conn): + """Rename local files based on CRC32 matching titles from episodes index. + Matches local video files with episodes in the database and renames them + to match the official episode titles. + Args: + conn: Database connection""" c = conn.cursor() c.execute("SELECT file_path, crc32 FROM crc32_cache") entries = c.fetchall() @@ -605,6 +802,9 @@ def rename_local_files(conn): def export_db_to_csv(conn): + """Export local CRC32 database to CSV file. + Args: + conn: Database connection""" c = conn.cursor() c.execute("SELECT file_path, crc32 FROM crc32_cache") rows = c.fetchall() @@ -1135,39 +1335,50 @@ def _handle_main_commands(args, conn, folder): def main(): - args = _parse_arguments() - - # Show detailed help if requested - if args.help: - _print_help() - return - - # Only show Docker mode message once, and not for --db or --episodes_update commands - # Also suppress for help command - if IS_DOCKER and not args.db and not args.episodes_update and not args.help: - print("Running in Docker mode (non-interactive)") - - if not _validate_url(args.url): - return - - # Only show episodes metadata status for main command (not for --db or --episodes_update) - if not args.db and not args.episodes_update: - _show_episodes_metadata_status() - - if args.episodes_update: - update_episodes_index_db(args.url) - return - - # Suppress messages when exporting DB (since it's automated) - conn = init_db(suppress_messages=args.db) - - # Folder selection logic: Always prompt if folder is required but not given - needs_folder = not args.download # All commands except --download need folder - folder = _get_folder_from_args(args, conn, needs_folder) - if folder is None: - return - - _handle_main_commands(args, conn, folder) + # Register signal handlers for graceful shutdown + signal.signal(signal.SIGTERM, _signal_handler) + signal.signal(signal.SIGINT, _signal_handler) + + try: + args = _parse_arguments() + + # Show detailed help if requested + if args.help: + _print_help() + return + + # Only show Docker mode message once, and not for --db or --episodes_update commands + # Also suppress for help command + if IS_DOCKER and not args.db and not args.episodes_update and not args.help: + print("Running in Docker mode (non-interactive)") + + if not _validate_url(args.url): + sys.exit(1) + + # Only show episodes metadata status for main command (not for --db or --episodes_update) + if not args.db and not args.episodes_update: + _show_episodes_metadata_status() + + if args.episodes_update: + update_episodes_index_db(args.url) + return + + # Suppress messages when exporting DB (since it's automated) + conn = init_db(suppress_messages=args.db) + + # Folder selection logic: Always prompt if folder is required but not given + needs_folder = not args.download # All commands except --download need folder + folder = _get_folder_from_args(args, conn, needs_folder) + if folder is None: + sys.exit(1) + + _handle_main_commands(args, conn, folder) + except KeyboardInterrupt: + print("\nInterrupted by user, exiting gracefully...") + sys.exit(130) # Standard exit code for SIGINT + except Exception as e: + print(f"Error: {e}") + sys.exit(1) if __name__ == "__main__": main() diff --git a/coverage.xml b/coverage.xml index 8d2e4aa..02222bf 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -22,673 +22,765 @@ + - - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + - - - - + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + + + + + - - - - - - - - - - - + + + + + - - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + - + + - - - + + + + + - - + + - + + - - - - - - + + + - - - - - - - - - - - - - - - + + + + + + + + + + - - - - - - - + + + + + + - - - - + + + + - - - - - - - - + + + + + + + - - + + + + + - - - - - - - - - - - - - + + + + + + + + + - - - + + + - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + - - - + + + - - - - - - - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + + + - - - - - + + + + + + + - - - - - - - - - - - + + + + + + + - + + - + - - + + + + + + + + + + - - - - + + + + - - - - - - - - - - + + + + + + + + + - - + - - + + + - + - - - + - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - + + - + + + - - - + + + + + + + - - - - + + + + + + - - - - - - - - - + + + + + + + + + + + + + - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + - + + + + + + - - - - - - - - - + + + + + - - - - - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -700,127 +792,127 @@ - - - - - - - - - + + + + + + + + + - + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - + - - - - - - - - - + + + + + + + + + - - - - - + + + + + diff --git a/entrypoint.sh b/entrypoint.sh index 08f9c0e..1a13528 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,20 +1,45 @@ #!/bin/sh -set -e + +# Track exit code +EXIT_CODE=0 + +# Signal handler for graceful shutdown +cleanup() { + echo "Received shutdown signal, cleaning up..." + # Kill any background processes + kill 0 2>/dev/null || true + exit ${EXIT_CODE} +} + +# Set up signal handlers +trap cleanup SIGTERM SIGINT # Run episodes update if requested if [ "$EPISODES_UPDATE" = "true" ]; then - python /app/acepace.py --episodes_update ${NYAA_URL:+--url "$NYAA_URL"} + python /app/acepace.py --episodes_update ${NYAA_URL:+--url "$NYAA_URL"} || EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Episodes update failed with exit code $EXIT_CODE" + exit $EXIT_CODE + fi fi # Export database if requested if [ "$DB" = "true" ]; then - python /app/acepace.py --db + python /app/acepace.py --db || EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Database export failed with exit code $EXIT_CODE" + exit $EXIT_CODE + fi fi # Always run missing episodes report first (updates Ace-Pace_Missing.csv) python /app/acepace.py \ --folder /media \ - ${NYAA_URL:+--url "$NYAA_URL"} + ${NYAA_URL:+--url "$NYAA_URL"} || EXIT_CODE=$? +if [ $EXIT_CODE -ne 0 ]; then + echo "Missing episodes report failed with exit code $EXIT_CODE" + exit $EXIT_CODE +fi # If DOWNLOAD is set to true, download missing episodes after generating report if [ "$DOWNLOAD" = "true" ]; then @@ -27,4 +52,7 @@ if [ "$DOWNLOAD" = "true" ]; then ${TORRENT_PORT:+--port "$TORRENT_PORT"} \ ${TORRENT_USER:+--username "$TORRENT_USER"} \ ${TORRENT_PASSWORD:+--password "$TORRENT_PASSWORD"} -fi \ No newline at end of file +fi + +# Exit with success code if we reach here +exit 0 \ No newline at end of file diff --git a/tests/test_episodes.py b/tests/test_episodes.py index afa4521..17fb32b 100644 --- a/tests/test_episodes.py +++ b/tests/test_episodes.py @@ -11,6 +11,40 @@ import acepace +def _create_nyaa_html_page(episode_titles): + """Helper function to create Nyaa HTML page with given episode titles. + Args: + episode_titles: List of episode title strings + Returns: HTML string""" + rows = "\n".join([ + f' \n \n {title}\n \n ' + for i, title in enumerate(episode_titles) + ]) + return f""" + + + +{rows} +
+
    +
  • 1
    • +
+ + + """ + + +def _create_mock_response(html_content): + """Helper function to create a mock HTTP response. + Args: + html_content: HTML content string + Returns: MagicMock response object""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html_content + return mock_response + + class TestEpisodeMetadataFetching: """Tests for fetching episode metadata from Nyaa.""" @@ -271,36 +305,16 @@ def test_update_episodes_index_db_uses_url_parameter(self, mock_fetch, temp_dir, class TestEpisodeQualityFiltering: - """Tests for ensuring only 1080p (or 720p fallback) episodes are extracted.""" + """Tests for ensuring only 1080p episodes are extracted.""" @patch('acepace.requests.get') def test_fetch_episodes_prefers_1080p(self, mock_get): """Test that 1080p episodes are extracted when available.""" - html = """ - - - - - - - - - -
- [One Pace] Episode 1 [1080p][A1B2C3D4].mkv -
- [One Pace] Episode 2 [1080p][E5F6A7B8].mkv -
-
    -
  • 1
    • -
- - - """ - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.text = html - mock_get.return_value = mock_response + html = _create_nyaa_html_page([ + "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv", + "[One Pace] Episode 2 [1080p][E5F6A7B8].mkv" + ]) + mock_get.return_value = _create_mock_response(html) episodes = acepace.fetch_episodes_metadata() @@ -311,75 +325,28 @@ def test_fetch_episodes_prefers_1080p(self, mock_get): assert "[One Pace]" in title @patch('acepace.requests.get') - def test_fetch_episodes_accepts_720p_as_fallback(self, mock_get): - """Test that 720p episodes are accepted when 1080p is not available.""" - html = """ - - - - - - - - - -
- [One Pace] Episode 1 [720p][A1B2C3D4].mkv -
- [One Pace] Episode 2 [720p][E5F6A7B8].mkv -
-
    -
  • 1
    • -
- - - """ - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.text = html - mock_get.return_value = mock_response + def test_fetch_episodes_rejects_720p(self, mock_get): + """Test that 720p episodes are rejected (only 1080p accepted).""" + html = _create_nyaa_html_page([ + "[One Pace] Episode 1 [720p][A1B2C3D4].mkv", + "[One Pace] Episode 2 [720p][E5F6A7B8].mkv" + ]) + mock_get.return_value = _create_mock_response(html) episodes = acepace.fetch_episodes_metadata() - # All episodes should be 720p - assert len(episodes) == 2 - for crc32, title, _ in episodes: - assert "[720p]" in title.upper() or "720P" in title.upper() - assert "[One Pace]" in title + # All 720p episodes should be rejected + assert len(episodes) == 0 @patch('acepace.requests.get') def test_fetch_episodes_excludes_lower_quality(self, mock_get): - """Test that episodes with quality lower than 720p are excluded.""" - html = """ - - - - - - - - - - - - -
- [One Pace] Episode 1 [480p][A1B2C3D4].mkv -
- [One Pace] Episode 2 [360p][E5F6A7B8].mkv -
- [One Pace] Episode 3 [240p][A9B0C1D2].mkv -
-
    -
  • 1
    • -
- - - """ - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.text = html - mock_get.return_value = mock_response + """Test that episodes with quality other than 1080p are excluded.""" + html = _create_nyaa_html_page([ + "[One Pace] Episode 1 [480p][A1B2C3D4].mkv", + "[One Pace] Episode 2 [360p][E5F6A7B8].mkv", + "[One Pace] Episode 3 [240p][A9B0C1D2].mkv" + ]) + mock_get.return_value = _create_mock_response(html) episodes = acepace.fetch_episodes_metadata() @@ -387,131 +354,69 @@ def test_fetch_episodes_excludes_lower_quality(self, mock_get): assert len(episodes) == 0 @patch('acepace.requests.get') - def test_fetch_episodes_prefers_1080p_over_720p_same_episode(self, mock_get): - """Test that when both 1080p and 720p versions exist for same episode, 1080p is preferred.""" - html = """ - - - - - - - - - -
- [One Pace] Episode 1 [1080p][A1B2C3D4].mkv -
- [One Pace] Episode 1 [720p][A1B2C3D4].mkv -
-
    -
  • 1
    • -
- - - """ - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.text = html - mock_get.return_value = mock_response + def test_fetch_episodes_only_accepts_1080p_same_episode(self, mock_get): + """Test that when both 1080p and 720p versions exist for same episode, only 1080p is accepted.""" + html = _create_nyaa_html_page([ + "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv", + "[One Pace] Episode 1 [720p][A1B2C3D4].mkv" + ]) + mock_get.return_value = _create_mock_response(html) episodes = acepace.fetch_episodes_metadata() - # Should only have one entry (deduplicated by CRC32) - # But we need to verify it's the 1080p version + # Should only have one entry (1080p version, 720p is rejected) assert len(episodes) == 1 crc32, title, _ = episodes[0] assert crc32 == "A1B2C3D4" - # The first one encountered should be kept (1080p in this case) - # Since CRC32 deduplication happens, we need to check which one was kept + # Only 1080p should be kept assert "[1080p]" in title.upper() or "1080P" in title.upper() + assert "[720p]" not in title.upper() and "720P" not in title.upper() @patch('acepace.requests.get') - def test_fetch_episodes_mixed_qualities_only_keeps_valid(self, mock_get): - """Test that mixed quality episodes only keeps 1080p and 720p.""" - html = """ - - - - - - - - - - - - - - - -
- [One Pace] Episode 1 [1080p][A1B2C3D4].mkv -
- [One Pace] Episode 2 [720p][E5F6A7B8].mkv -
- [One Pace] Episode 3 [480p][A9B0C1D2].mkv -
- [One Pace] Episode 4 [1080p][A3B4C5D6].mkv -
-
    -
  • 1
    • -
- - - """ - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.text = html - mock_get.return_value = mock_response + def test_fetch_episodes_mixed_qualities_only_keeps_1080p(self, mock_get): + """Test that mixed quality episodes only keeps 1080p.""" + html = _create_nyaa_html_page([ + "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv", + "[One Pace] Episode 2 [720p][E5F6A7B8].mkv", + "[One Pace] Episode 3 [480p][A9B0C1D2].mkv", + "[One Pace] Episode 4 [1080p][A3B4C5D6].mkv" + ]) + mock_get.return_value = _create_mock_response(html) episodes = acepace.fetch_episodes_metadata() - # Should only have 1080p and 720p episodes (3 total, excluding 480p) - assert len(episodes) == 3 + # Should only have 1080p episodes (2 total, excluding 720p and 480p) + assert len(episodes) == 2 for crc32, title, _ in episodes: title_upper = title.upper() has_1080p = "[1080P]" in title_upper or "1080P" in title_upper - has_720p = "[720P]" in title_upper or "720P" in title_upper - assert has_1080p or has_720p, f"Episode {title} should be 1080p or 720p" - # Verify no lower quality + assert has_1080p, f"Episode {title} should be 1080p" + # Verify no other qualities + assert "[720P]" not in title_upper assert "[480P]" not in title_upper assert "[360P]" not in title_upper assert "[240P]" not in title_upper @patch('acepace.requests.get') def test_fetch_episodes_handles_case_insensitive_quality(self, mock_get): - """Test that quality detection is case-insensitive.""" - html = """ - - - - - - - - - -
- [One Pace] Episode 1 [1080P][A1B2C3D4].mkv -
- [One Pace] Episode 2 [720P][E5F6A7B8].mkv -
-
    -
  • 1
    • -
- - - """ - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.text = html - mock_get.return_value = mock_response + """Test that quality detection is case-insensitive (1080p only).""" + html = _create_nyaa_html_page([ + "[One Pace] Episode 1 [1080P][A1B2C3D4].mkv", + "[One Pace] Episode 2 [720P][E5F6A7B8].mkv" + ]) + mock_get.return_value = _create_mock_response(html) episodes = acepace.fetch_episodes_metadata() - # Should accept both uppercase and lowercase quality markers - assert len(episodes) == 2 + # Should only accept 1080P (case-insensitive), reject 720P + assert len(episodes) == 1 + crc32, title, _ = episodes[0] + assert crc32 == "A1B2C3D4" + # Verify case-insensitive quality detection works for 1080p + title_upper = title.upper() + assert "[1080P]" in title_upper + # Verify 720p is rejected + assert "[720P]" not in title_upper @patch('acepace.requests.get') def test_fetch_episodes_excludes_episodes_without_quality_marker(self, mock_get): @@ -659,10 +564,7 @@ class TestQualityFilteringHelper: def test_quality_filtering_accepts_1080p(self): """Test that 1080p quality is accepted.""" - # We need to test the internal _is_valid_quality function - # Since it's nested, we'll test it through fetch_episodes_metadata - # But we can also test the regex directly - from acepace import QUALITY_REGEX + from acepace import _is_valid_quality test_cases = [ "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv", @@ -671,14 +573,11 @@ def test_quality_filtering_accepts_1080p(self): ] for test_case in test_cases: - matches = QUALITY_REGEX.findall(test_case) - assert len(matches) > 0 - quality_num = int(matches[0].lower().replace('p', '')) - assert quality_num == 1080 + assert _is_valid_quality(test_case) is True - def test_quality_filtering_accepts_720p(self): - """Test that 720p quality is accepted.""" - from acepace import QUALITY_REGEX + def test_quality_filtering_rejects_720p(self): + """Test that 720p quality is rejected (only 1080p accepted).""" + from acepace import _is_valid_quality test_cases = [ "[One Pace] Episode 1 [720p][A1B2C3D4].mkv", @@ -686,30 +585,25 @@ def test_quality_filtering_accepts_720p(self): ] for test_case in test_cases: - matches = QUALITY_REGEX.findall(test_case) - assert len(matches) > 0 - quality_num = int(matches[0].lower().replace('p', '')) - assert quality_num == 720 + assert _is_valid_quality(test_case) is False - def test_quality_filtering_rejects_lower_quality(self): - """Test that qualities lower than 720p are rejected.""" - from acepace import QUALITY_REGEX + def test_quality_filtering_rejects_non_1080p(self): + """Test that qualities other than 1080p are rejected.""" + from acepace import _is_valid_quality test_cases = [ + "[One Pace] Episode 1 [720p][A1B2C3D4].mkv", "[One Pace] Episode 1 [480p][A1B2C3D4].mkv", "[One Pace] Episode 1 [360p][A1B2C3D4].mkv", "[One Pace] Episode 1 [240p][A1B2C3D4].mkv", ] for test_case in test_cases: - matches = QUALITY_REGEX.findall(test_case) - assert len(matches) > 0 - quality_num = int(matches[0].lower().replace('p', '')) - assert quality_num < 720 + assert _is_valid_quality(test_case) is False def test_quality_filtering_rejects_higher_quality(self): """Test that qualities higher than 1080p are rejected (4K, etc.).""" - from acepace import QUALITY_REGEX + from acepace import _is_valid_quality test_cases = [ "[One Pace] Episode 1 [2160p][A1B2C3D4].mkv", # 4K @@ -717,10 +611,7 @@ def test_quality_filtering_rejects_higher_quality(self): ] for test_case in test_cases: - matches = QUALITY_REGEX.findall(test_case) - assert len(matches) > 0 - quality_num = int(matches[0].lower().replace('p', '')) - assert quality_num not in [720, 1080] + assert _is_valid_quality(test_case) is False class TestURLParameterConsistency: @@ -769,14 +660,14 @@ def test_fetch_episodes_metadata_and_fetch_crc32_links_use_same_url(self, mock_g test_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" # Test fetch_episodes_metadata - episodes = acepace.fetch_episodes_metadata(test_url) + acepace.fetch_episodes_metadata(test_url) # Reset mock for second test mock_get.reset_mock() mock_get.side_effect = [mock_response1, mock_response2] # Test fetch_crc32_links - crc32_to_link, _, _, _ = acepace.fetch_crc32_links(test_url) + _, _, _, _ = acepace.fetch_crc32_links(test_url) # Both should use the same URL assert mock_get.call_count > 0 diff --git a/tests/test_main_command.py b/tests/test_main_command.py index a116d65..56682f7 100644 --- a/tests/test_main_command.py +++ b/tests/test_main_command.py @@ -9,6 +9,9 @@ import acepace +# Test constants +TEST_HOST_IP = "localhost" # Test host for testing environment variable handling + class TestDockerModeBehavior: """Tests for Docker mode specific behavior.""" @@ -362,7 +365,7 @@ def test_docker_uses_environment_variable_overrides(self, mock_get_client, mock_ with patch.dict('os.environ', { 'TORRENT_CLIENT': 'qbittorrent', - 'TORRENT_HOST': '192.168.1.100', + 'TORRENT_HOST': TEST_HOST_IP, 'TORRENT_PORT': '8080', 'TORRENT_USER': 'admin' }): @@ -372,6 +375,6 @@ def test_docker_uses_environment_variable_overrides(self, mock_get_client, mock_ mock_get_client.assert_called_once() call_args = mock_get_client.call_args assert call_args[0][0] == "qbittorrent" # From env var - assert call_args[0][1] == "192.168.1.100" # From env var + assert call_args[0][1] == TEST_HOST_IP # From env var assert call_args[0][2] == 8080 # From env var assert call_args[0][3] == "admin" # From env var diff --git a/tests/test_missing_detection.py b/tests/test_missing_detection.py index 2d350f5..38d071b 100644 --- a/tests/test_missing_detection.py +++ b/tests/test_missing_detection.py @@ -97,7 +97,7 @@ def test_fetch_crc32_links_from_nyaa(self, mock_get): @patch('acepace.requests.get') def test_fetch_crc32_links_filters_quality(self, mock_get): - """Test that fetch_crc32_links filters episodes by quality (1080p/720p only).""" + """Test that fetch_crc32_links filters episodes by quality (1080p only).""" html_with_mixed_quality = """ @@ -145,13 +145,13 @@ def test_fetch_crc32_links_filters_quality(self, mock_get): mock_get.side_effect = [mock_response1, mock_response2] base_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace" - crc32_to_link, crc32_to_text, crc32_to_magnet, _ = acepace.fetch_crc32_links(base_url) + crc32_to_link, _, _, _ = acepace.fetch_crc32_links(base_url) - # Should only have 1080p and 720p episodes, not 480p - assert len(crc32_to_link) == 2 + # Should only have 1080p episodes, not 720p or 480p + assert len(crc32_to_link) == 1 assert "A1B2C3D4" in crc32_to_link # 1080p - should be included - assert "E5F6A7B8" in crc32_to_link # 720p - should be included - assert "A9B0C1D2" not in crc32_to_link # 480p - should be filtered out + assert "E5F6A7B8" not in crc32_to_link # 720p - should be excluded + assert "A9B0C1D2" not in crc32_to_link # 480p - should be excluded @patch('acepace.requests.get') def test_fetch_crc32_links_stops_on_empty_page(self, mock_get): diff --git a/tests/test_path_normalization.py b/tests/test_path_normalization.py index 77e90a5..22ef82d 100644 --- a/tests/test_path_normalization.py +++ b/tests/test_path_normalization.py @@ -12,6 +12,55 @@ import acepace +def _create_test_row(title): + """Helper function to create a test HTML row with given title. + Args: + title: Episode title to use in the row + Returns: BeautifulSoup row element""" + row_html = f""" + + + {title} + Magnet + + + """ + soup = BeautifulSoup(row_html, "html.parser") + return soup.find("tr") + + +def _process_row_with_assertions(title, expected_success, expected_crc32_in_link=None, expected_text_in_values=None): + """Helper function to process a row and assert results. + Args: + title: Episode title + expected_success: Expected success value (True/False) + expected_crc32_in_link: CRC32 that should be in link dict (None to skip check) + expected_text_in_values: Text that should be in text dict values (None to skip check) + Returns: Tuple of (success, should_warn, crc32_to_link, crc32_to_text)""" + row = _create_test_row(title) + crc32_to_link = {} + crc32_to_text = {} + crc32_to_magnet = {} + + success, _, should_warn = acepace._process_crc32_row( + row, crc32_to_link, crc32_to_text, crc32_to_magnet + ) + + assert success is expected_success + assert should_warn is False + + if expected_crc32_in_link is not None: + if expected_success: + assert expected_crc32_in_link in crc32_to_link + else: + assert expected_crc32_in_link not in crc32_to_link + + if expected_text_in_values is not None: + assert expected_text_in_values in crc32_to_text.values() + + return success, should_warn, crc32_to_link, crc32_to_text + + class TestPathNormalization: """Tests for file path normalization functionality.""" @@ -187,172 +236,60 @@ class TestQualityFiltering: def test_process_crc32_row_accepts_1080p(self): """Test that _process_crc32_row accepts 1080p episodes.""" - row_html = """ - - - [One Pace] Episode 1 [1080p][A1B2C3D4].mkv - Magnet - - - """ - soup = BeautifulSoup(row_html, "html.parser") - row = soup.find("tr") - - crc32_to_link = {} - crc32_to_text = {} - crc32_to_magnet = {} - - success, filename_text = acepace._process_crc32_row( - row, crc32_to_link, crc32_to_text, crc32_to_magnet + _process_row_with_assertions( + "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv", + expected_success=True, + expected_crc32_in_link="A1B2C3D4", + expected_text_in_values="[One Pace] Episode 1 [1080p][A1B2C3D4].mkv" ) - - assert success is True - assert "A1B2C3D4" in crc32_to_link - assert "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv" in crc32_to_text.values() - def test_process_crc32_row_accepts_720p(self): - """Test that _process_crc32_row accepts 720p episodes.""" - row_html = """ - - - [One Pace] Episode 1 [720p][A1B2C3D4].mkv - Magnet - - - """ - soup = BeautifulSoup(row_html, "html.parser") - row = soup.find("tr") - - crc32_to_link = {} - crc32_to_text = {} - crc32_to_magnet = {} - - success, filename_text = acepace._process_crc32_row( - row, crc32_to_link, crc32_to_text, crc32_to_magnet + def test_process_crc32_row_rejects_720p(self): + """Test that _process_crc32_row rejects 720p episodes.""" + _process_row_with_assertions( + "[One Pace] Episode 1 [720p][A1B2C3D4].mkv", + expected_success=False, + expected_crc32_in_link="A1B2C3D4" ) - - assert success is True - assert "A1B2C3D4" in crc32_to_link def test_process_crc32_row_rejects_480p(self): """Test that _process_crc32_row rejects 480p episodes.""" - row_html = """ - - - [One Pace] Episode 1 [480p][A1B2C3D4].mkv - Magnet - - - """ - soup = BeautifulSoup(row_html, "html.parser") - row = soup.find("tr") - - crc32_to_link = {} - crc32_to_text = {} - crc32_to_magnet = {} - - success, filename_text = acepace._process_crc32_row( - row, crc32_to_link, crc32_to_text, crc32_to_magnet + _process_row_with_assertions( + "[One Pace] Episode 1 [480p][A1B2C3D4].mkv", + expected_success=False, + expected_crc32_in_link="A1B2C3D4" ) - - assert success is False - assert "A1B2C3D4" not in crc32_to_link def test_process_crc32_row_rejects_2160p(self): """Test that _process_crc32_row rejects 2160p (4K) episodes.""" - row_html = """ - - - [One Pace] Episode 1 [2160p][A1B2C3D4].mkv - Magnet - - - """ - soup = BeautifulSoup(row_html, "html.parser") - row = soup.find("tr") - - crc32_to_link = {} - crc32_to_text = {} - crc32_to_magnet = {} - - success, filename_text = acepace._process_crc32_row( - row, crc32_to_link, crc32_to_text, crc32_to_magnet + _process_row_with_assertions( + "[One Pace] Episode 1 [2160p][A1B2C3D4].mkv", + expected_success=False, + expected_crc32_in_link="A1B2C3D4" ) - - assert success is False - assert "A1B2C3D4" not in crc32_to_link def test_process_crc32_row_rejects_no_quality(self): """Test that _process_crc32_row rejects episodes without quality marker.""" - row_html = """ - - - [One Pace] Episode 1 [A1B2C3D4].mkv - Magnet - - - """ - soup = BeautifulSoup(row_html, "html.parser") - row = soup.find("tr") - - crc32_to_link = {} - crc32_to_text = {} - crc32_to_magnet = {} - - success, filename_text = acepace._process_crc32_row( - row, crc32_to_link, crc32_to_text, crc32_to_magnet + _process_row_with_assertions( + "[One Pace] Episode 1 [A1B2C3D4].mkv", + expected_success=False, + expected_crc32_in_link="A1B2C3D4" ) - - assert success is False - assert "A1B2C3D4" not in crc32_to_link def test_process_crc32_row_rejects_no_one_pace_marker(self): """Test that _process_crc32_row rejects episodes without [One Pace] marker.""" - row_html = """ - - - Episode 1 [1080p][A1B2C3D4].mkv - Magnet - - - """ - soup = BeautifulSoup(row_html, "html.parser") - row = soup.find("tr") - - crc32_to_link = {} - crc32_to_text = {} - crc32_to_magnet = {} - - success, filename_text = acepace._process_crc32_row( - row, crc32_to_link, crc32_to_text, crc32_to_magnet + _process_row_with_assertions( + "Episode 1 [1080p][A1B2C3D4].mkv", + expected_success=False, + expected_crc32_in_link="A1B2C3D4" ) - - assert success is False - assert "A1B2C3D4" not in crc32_to_link def test_process_crc32_row_case_insensitive_quality(self): """Test that quality filtering is case insensitive.""" - row_html = """ - - - [One Pace] Episode 1 [1080P][A1B2C3D4].mkv - Magnet - - - """ - soup = BeautifulSoup(row_html, "html.parser") - row = soup.find("tr") - - crc32_to_link = {} - crc32_to_text = {} - crc32_to_magnet = {} - - success, filename_text = acepace._process_crc32_row( - row, crc32_to_link, crc32_to_text, crc32_to_magnet + _process_row_with_assertions( + "[One Pace] Episode 1 [1080P][A1B2C3D4].mkv", + expected_success=True, + expected_crc32_in_link="A1B2C3D4" ) - - assert success is True - assert "A1B2C3D4" in crc32_to_link @patch('acepace.requests.get') def test_fetch_crc32_links_filters_by_quality(self, mock_get): @@ -404,10 +341,10 @@ def test_fetch_crc32_links_filters_by_quality(self, mock_get): mock_get.side_effect = [mock_response1, mock_response2] base_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace" - crc32_to_link, crc32_to_text, crc32_to_magnet, _ = acepace.fetch_crc32_links(base_url) + crc32_to_link, _, _, _ = acepace.fetch_crc32_links(base_url) - # Should only have 1080p and 720p episodes, not 480p - assert len(crc32_to_link) == 2 - assert "A1B2C3D4" in crc32_to_link # 1080p - assert "E5F6A7B8" in crc32_to_link # 720p - assert "A9B0C1D2" not in crc32_to_link # 480p should be filtered out + # Should only have 1080p episodes, not 720p or 480p + assert len(crc32_to_link) == 1 + assert "A1B2C3D4" in crc32_to_link # 1080p - should be included + assert "E5F6A7B8" not in crc32_to_link # 720p - should be excluded + assert "A9B0C1D2" not in crc32_to_link # 480p - should be excluded From b6bf891ff5415f1feef125c4253e043942f971f6 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 27 Jan 2026 11:42:08 +0000 Subject: [PATCH 55/75] Potential fix for missing episodes handling --- acepace.py | 199 +++++++++++++++++++++++- coverage.xml | 303 +++++++++++++++++++++++++------------ entrypoint.sh | 28 ++-- tests/test_main_command.py | 27 ++-- 4 files changed, 434 insertions(+), 123 deletions(-) diff --git a/acepace.py b/acepace.py index a16cee6..c5d6225 100644 --- a/acepace.py +++ b/acepace.py @@ -1048,15 +1048,30 @@ def _load_old_missing_crc32s(): def _save_missing_episodes_csv(missing, crc32_to_text, crc32_to_link, crc32_to_magnet): """Save missing episodes to CSV file.""" missing_csv_path = get_config_path(MISSING_CSV_FILENAME) + saved_count = 0 + error_count = 0 with open(missing_csv_path, "w", encoding="utf-8", newline="") as f: writer = csv.writer(f, quoting=csv.QUOTE_ALL) writer.writerow(["Title", "Page Link", "Magnet Link"]) for crc32 in missing: - title = crc32_to_text[crc32] - page_link = crc32_to_link[crc32] - magnet = crc32_to_magnet.get(crc32, "") - writer.writerow([title, page_link, magnet]) + try: + title = crc32_to_text.get(crc32, f"[CRC32: {crc32}]") + page_link = crc32_to_link.get(crc32, "") + magnet = crc32_to_magnet.get(crc32, "") + writer.writerow([title, page_link, magnet]) + saved_count += 1 + except Exception as e: + error_count += 1 + print(f"ERROR: Failed to save missing episode with CRC32 '{crc32}': {e}") + # Still write a row with available information + writer.writerow([f"[ERROR: CRC32 {crc32}]", "", ""]) + print(f"Missing files list saved to {missing_csv_path}") + if error_count > 0: + print(f"WARNING: {error_count} episodes had errors while saving to CSV") + if saved_count == 0 and len(missing) > 0: + print(f"ERROR: No episodes were successfully saved to CSV despite {len(missing)} missing episodes!") + print("This indicates a critical issue with the CRC32 mapping.") def _print_report_header(conn, folder, args): @@ -1085,6 +1100,148 @@ def _print_report_header(conn, folder, args): return last_run +def _print_troubleshooting_header(crc32_to_link, local_crc32s): + """Print initial troubleshooting information header.""" + print("\n=== TROUBLESHOOTING INFO ===") + print(f"Episodes from Nyaa (crc32_to_link keys): {len(crc32_to_link)}") + print(f"Local CRC32s: {len(local_crc32s)}") + + # Check for empty sets + if len(crc32_to_link) == 0: + print("WARNING: No episodes fetched from Nyaa! Check URL and quality filtering.") + if len(local_crc32s) == 0: + print("WARNING: No local CRC32s found! Check folder path and file extensions.") + + # Show sample CRC32s from both sources (first 5) + if crc32_to_link: + sample_nyaa = list(crc32_to_link.keys())[:5] + print(f"Sample Nyaa CRC32s (first 5): {sample_nyaa}") + print(f"Sample Nyaa CRC32 types: {[type(c).__name__ for c in sample_nyaa]}") + if local_crc32s: + sample_local = list(local_crc32s)[:5] + print(f"Sample local CRC32s (first 5): {sample_local}") + print(f"Sample local CRC32 types: {[type(c).__name__ for c in sample_local]}") + + +def _normalize_crc32_sets(crc32_to_link, local_crc32s): + """Normalize CRC32 sets to uppercase strings for comparison. + Returns tuple: (nyaa_crc32s_normalized, local_crc32s_normalized)""" + nyaa_crc32s_normalized = {str(c).strip().upper() for c in crc32_to_link.keys()} + local_crc32s_normalized = {str(c).strip().upper() for c in local_crc32s} + + print("\nAfter normalization:") + print(f"Nyaa CRC32s: {len(nyaa_crc32s_normalized)}") + print(f"Local CRC32s: {len(local_crc32s_normalized)}") + + # Check for matches using normalized sets + matches_normalized = nyaa_crc32s_normalized & local_crc32s_normalized + print(f"Matches after normalization: {len(matches_normalized)}") + if matches_normalized: + print(f"Sample matches (first 3): {list(matches_normalized)[:3]}") + + return nyaa_crc32s_normalized, local_crc32s_normalized + + +def _build_normalized_to_original_mapping(crc32_to_link, nyaa_crc32s_normalized): + """Build mapping from normalized CRC32 back to original key. + Returns tuple: (normalized_to_original dict, mapping_issues list)""" + normalized_to_original = {} + for orig_key in crc32_to_link.keys(): + norm_key = str(orig_key).strip().upper() + # If we already have this normalized key, keep the first one (shouldn't happen with CRC32s) + if norm_key not in normalized_to_original: + normalized_to_original[norm_key] = orig_key + + mapping_issues = [] + # Verify the mapping is correct + if len(normalized_to_original) != len(nyaa_crc32s_normalized): + print(f"WARNING: Mapping size mismatch! normalized_to_original: {len(normalized_to_original)}, nyaa_crc32s_normalized: {len(nyaa_crc32s_normalized)}") + print("This could indicate duplicate normalized CRC32s or mapping issues.") + # Show which normalized CRC32s are missing from the mapping + missing_from_mapping = nyaa_crc32s_normalized - set(normalized_to_original.keys()) + if missing_from_mapping: + mapping_issues = list(missing_from_mapping) + print(f"Normalized CRC32s missing from mapping (first 5): {mapping_issues[:5]}") + + return normalized_to_original, mapping_issues + + +def _build_missing_list(missing_normalized_set, normalized_to_original, crc32_to_link): + """Build missing episodes list from normalized set. + Returns tuple: (missing list, mapping_errors list)""" + missing = [] + missing_normalized = list(missing_normalized_set) + mapping_errors = [] + + for norm_crc in missing_normalized: + if norm_crc in normalized_to_original: + missing.append(normalized_to_original[norm_crc]) + else: + # Try to find the original key by searching (fallback) + found = False + for orig_key in crc32_to_link.keys(): + if str(orig_key).strip().upper() == norm_crc: + missing.append(orig_key) + found = True + break + if not found: + mapping_errors.append(norm_crc) + print(f"ERROR: Could not find original key for normalized CRC32 '{norm_crc}'") + + if mapping_errors: + print(f"WARNING: {len(mapping_errors)} missing episodes could not be mapped to original keys!") + print("This is a critical error - these episodes will not be included in the missing list.") + print(f"Affected normalized CRC32s (first 10): {mapping_errors[:10]}") + + return missing, mapping_errors + + +def _print_comparison_results(nyaa_crc32s_normalized, local_crc32s_normalized, + crc32_to_link, local_crc32s, missing, missing_normalized): + """Print comparison results and troubleshooting information.""" + # Also check the original comparison for debugging + original_missing_count = len([c for c in crc32_to_link.keys() if c not in local_crc32s]) + print(f"Missing episodes (original comparison): {original_missing_count}") + print(f"Missing episodes (normalized comparison): {len(missing)}") + print(f"Missing normalized CRC32s: {len(missing_normalized)}") + + if original_missing_count != len(missing): + print(f"WARNING: Comparison mismatch detected! Original: {original_missing_count}, Normalized: {len(missing)}") + print("This suggests a data type or format issue. Using normalized comparison.") + + # Show intersection details + intersection = nyaa_crc32s_normalized & local_crc32s_normalized + print(f"Intersection (episodes found locally): {len(intersection)}") + if intersection: + print(f"Sample found episodes (first 3): {list(intersection)[:3]}") + + # Show difference details + difference = nyaa_crc32s_normalized - local_crc32s_normalized + print(f"Difference (episodes NOT found locally): {len(difference)}") + if difference: + print(f"Sample missing episodes (first 3): {list(difference)[:3]}") + + # Check if sets are suspiciously similar (potential bug indicator) + if len(nyaa_crc32s_normalized) > 0 and len(local_crc32s_normalized) > 0: + similarity_ratio = len(intersection) / len(nyaa_crc32s_normalized) + print(f"Similarity ratio (intersection/nyaa): {similarity_ratio:.2%}") + if similarity_ratio > 0.95 and len(difference) == 0: + print("WARNING: Almost all Nyaa episodes appear to be found locally!") + print("This might indicate a comparison bug or data issue.") + print("Please verify that your local files actually contain all these episodes.") + + # Check for sets being identical (definite bug) + if nyaa_crc32s_normalized == local_crc32s_normalized: + print("ERROR: Nyaa and local CRC32 sets are IDENTICAL!") + print("This indicates a critical bug - the sets should not be the same.") + print("Possible causes:") + print(" - Local CRC32s are being populated from Nyaa data (wrong source)") + print(" - Comparison is using the same set for both sides") + print(" - Database corruption or incorrect data") + + print("=== END TROUBLESHOOTING INFO ===\n") + + def _calculate_and_find_missing(folder, conn, args, last_run): """Calculate local CRC32s and find missing episodes.""" crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = ( @@ -1103,7 +1260,32 @@ def _calculate_and_find_missing(folder, conn, args, last_run): local_crc32s = calculate_local_crc32(folder, conn) print(f"Found {len(local_crc32s)} local CRC32 hashes.") - missing = [crc32 for crc32 in crc32_to_link if crc32 not in local_crc32s] + # Print troubleshooting header + _print_troubleshooting_header(crc32_to_link, local_crc32s) + + # Normalize CRC32 sets + nyaa_crc32s_normalized, local_crc32s_normalized = _normalize_crc32_sets( + crc32_to_link, local_crc32s + ) + + # Find missing using normalized comparison + missing_normalized_set = nyaa_crc32s_normalized - local_crc32s_normalized + + # Build normalized to original mapping + normalized_to_original, _ = _build_normalized_to_original_mapping( + crc32_to_link, nyaa_crc32s_normalized + ) + + # Build missing list + missing, _ = _build_missing_list( + missing_normalized_set, normalized_to_original, crc32_to_link + ) + + # Print comparison results + _print_comparison_results( + nyaa_crc32s_normalized, local_crc32s_normalized, + crc32_to_link, local_crc32s, missing, list(missing_normalized_set) + ) print( f"\nSummary: {len(missing)} missing episodes out of {len(crc32_to_link)} total found on Nyaa.\n" @@ -1345,7 +1527,7 @@ def main(): # Show detailed help if requested if args.help: _print_help() - return + sys.exit(0) # Only show Docker mode message once, and not for --db or --episodes_update commands # Also suppress for help command @@ -1361,7 +1543,7 @@ def main(): if args.episodes_update: update_episodes_index_db(args.url) - return + sys.exit(0) # Suppress messages when exporting DB (since it's automated) conn = init_db(suppress_messages=args.db) @@ -1373,6 +1555,9 @@ def main(): sys.exit(1) _handle_main_commands(args, conn, folder) + + # Exit cleanly (code 0) even if shutdown was requested during processing + sys.exit(0) except KeyboardInterrupt: print("\nInterrupted by user, exiting gracefully...") sys.exit(130) # Standard exit code for SIGINT diff --git a/coverage.xml b/coverage.xml index 02222bf..0f25eb6 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -650,134 +650,247 @@ - + - + - + + + + - - - - - - + + + + + + + - - - - + + + - - + + + - - + + + + - - - + + + + - - - + + + + + + - - - + + + - + + + - + - + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + - + + + + + + + + + + - + - - - - - - + + + + + + - - - - - - - + + - - - - + + + + + - - - - + + - - - - + + + - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/entrypoint.sh b/entrypoint.sh index 1a13528..570ecb1 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,22 +1,23 @@ #!/bin/sh -# Track exit code -EXIT_CODE=0 - # Signal handler for graceful shutdown +# Signal numbers: 15 = SIGTERM, 2 = SIGINT +# Python processes run in foreground and will receive signals directly +# This handler ensures clean exit if signal arrives between commands cleanup() { - echo "Received shutdown signal, cleaning up..." - # Kill any background processes - kill 0 2>/dev/null || true - exit ${EXIT_CODE} + echo "Received shutdown signal, exiting gracefully..." + exit 0 } -# Set up signal handlers -trap cleanup SIGTERM SIGINT +# Set up signal handlers using signal numbers (POSIX compatible) +# Note: When Python runs in foreground, it receives signals directly +# This trap handles signals that arrive when no Python process is running +trap 'cleanup' 15 2 # Run episodes update if requested if [ "$EPISODES_UPDATE" = "true" ]; then - python /app/acepace.py --episodes_update ${NYAA_URL:+--url "$NYAA_URL"} || EXIT_CODE=$? + python /app/acepace.py --episodes_update ${NYAA_URL:+--url "$NYAA_URL"} + EXIT_CODE=$? if [ $EXIT_CODE -ne 0 ]; then echo "Episodes update failed with exit code $EXIT_CODE" exit $EXIT_CODE @@ -25,7 +26,8 @@ fi # Export database if requested if [ "$DB" = "true" ]; then - python /app/acepace.py --db || EXIT_CODE=$? + python /app/acepace.py --db + EXIT_CODE=$? if [ $EXIT_CODE -ne 0 ]; then echo "Database export failed with exit code $EXIT_CODE" exit $EXIT_CODE @@ -35,7 +37,8 @@ fi # Always run missing episodes report first (updates Ace-Pace_Missing.csv) python /app/acepace.py \ --folder /media \ - ${NYAA_URL:+--url "$NYAA_URL"} || EXIT_CODE=$? + ${NYAA_URL:+--url "$NYAA_URL"} +EXIT_CODE=$? if [ $EXIT_CODE -ne 0 ]; then echo "Missing episodes report failed with exit code $EXIT_CODE" exit $EXIT_CODE @@ -43,6 +46,7 @@ fi # If DOWNLOAD is set to true, download missing episodes after generating report if [ "$DOWNLOAD" = "true" ]; then + # Use exec to replace shell process so Python becomes PID 1 and receives signals directly exec python /app/acepace.py \ --folder /media \ ${NYAA_URL:+--url "$NYAA_URL"} \ diff --git a/tests/test_main_command.py b/tests/test_main_command.py index 56682f7..8e17bd8 100644 --- a/tests/test_main_command.py +++ b/tests/test_main_command.py @@ -41,7 +41,8 @@ def test_docker_mode_message_not_shown_for_db_command(self, mock_handle, mock_fo with patch('acepace._parse_arguments', return_value=mock_args): with patch('builtins.print') as mock_print: - acepace.main() + with pytest.raises(SystemExit): + acepace.main() # Verify "Running in Docker mode" was NOT printed print_calls = [str(c) for c in mock_print.call_args_list] @@ -64,7 +65,8 @@ def test_docker_mode_message_not_shown_for_episodes_update(self, mock_update, mo with patch('acepace._parse_arguments', return_value=mock_args): with patch('builtins.print') as mock_print: - acepace.main() + with pytest.raises(SystemExit): + acepace.main() # Verify "Running in Docker mode" was NOT printed print_calls = [str(c) for c in mock_print.call_args_list] @@ -95,7 +97,8 @@ def test_docker_mode_message_shown_for_main_command(self, mock_handle, mock_fold with patch('acepace._parse_arguments', return_value=mock_args): with patch('builtins.print') as mock_print: - acepace.main() + with pytest.raises(SystemExit): + acepace.main() # Verify "Running in Docker mode" WAS printed print_calls = [str(c) for c in mock_print.call_args_list] @@ -126,7 +129,8 @@ def test_docker_mode_message_not_shown_when_not_in_docker(self, mock_handle, moc with patch('acepace._parse_arguments', return_value=mock_args): with patch('builtins.print') as mock_print: - acepace.main() + with pytest.raises(SystemExit): + acepace.main() # Verify "Running in Docker mode" was NOT printed print_calls = [str(c) for c in mock_print.call_args_list] @@ -159,7 +163,8 @@ def test_episodes_metadata_status_not_shown_for_db_command(self, mock_export, mo with patch('acepace._parse_arguments', return_value=mock_args): with patch('acepace._get_folder_from_args', return_value="/media"): - acepace.main() + with pytest.raises(SystemExit): + acepace.main() # Verify _show_episodes_metadata_status was NOT called mock_show_status.assert_not_called() @@ -179,7 +184,8 @@ def test_episodes_metadata_status_not_shown_for_episodes_update(self, mock_updat mock_args.url = "https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc" with patch('acepace._parse_arguments', return_value=mock_args): - acepace.main() + with pytest.raises(SystemExit): + acepace.main() # Verify _show_episodes_metadata_status was NOT called mock_show_status.assert_not_called() @@ -207,7 +213,8 @@ def test_episodes_metadata_status_shown_for_main_command(self, mock_handle, mock mock_args.folder = None with patch('acepace._parse_arguments', return_value=mock_args): - acepace.main() + with pytest.raises(SystemExit): + acepace.main() # Verify _show_episodes_metadata_status WAS called mock_show_status.assert_called_once() @@ -232,7 +239,8 @@ def test_episodes_update_receives_url_parameter(self, mock_update, mock_validate mock_args.url = test_url with patch('acepace._parse_arguments', return_value=mock_args): - acepace.main() + with pytest.raises(SystemExit): + acepace.main() # Verify update_episodes_index_db was called with URL mock_update.assert_called_once_with(test_url) @@ -260,7 +268,8 @@ def test_rename_receives_url_parameter(self, mock_rename, mock_folder, mock_init mock_args.folder = None with patch('acepace._parse_arguments', return_value=mock_args): - acepace.main() + with pytest.raises(SystemExit): + acepace.main() # Verify _handle_rename_command was called with URL mock_rename.assert_called_once() From 3aee17ae7c4560ea8154dab1de47b53c77be0aab Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 27 Jan 2026 11:51:25 +0000 Subject: [PATCH 56/75] Fix CRC32 link fetching to support pagination --- acepace.py | 41 +- coverage.xml | 845 ++++++++++++++++---------------- tests/test_missing_detection.py | 15 +- 3 files changed, 464 insertions(+), 437 deletions(-) diff --git a/acepace.py b/acepace.py index c5d6225..08ce2d1 100644 --- a/acepace.py +++ b/acepace.py @@ -545,59 +545,74 @@ def _fetch_crc32_page(base_url, page): def _process_crc32_page_rows(soup, crc32_to_link, crc32_to_text, crc32_to_magnet): """Process all rows from a CRC32 links page. - Returns True if any episodes were found, False otherwise.""" + Returns the number of episodes found on this page.""" table = soup.find("table", class_="torrent-list") if not table: - print("No table found, stopping.") - return False + return 0 rows = table.find_all("tr") # type: ignore if not rows: - print("No rows found, stopping.") - return False + return 0 - found_in_page = False + found_count = 0 for row in rows: if _shutdown_requested: print(_SHUTDOWN_MESSAGE) break success, filename_text, should_warn = _process_crc32_row(row, crc32_to_link, crc32_to_text, crc32_to_magnet) if success: - found_in_page = True + found_count += 1 elif should_warn and filename_text: print(f"Warning: No CRC32 found in title '{filename_text}'") - return found_in_page + return found_count def fetch_crc32_links(base_url): """Fetch CRC32 links from Nyaa.si search URL. Only accepts episodes with 1080p quality. + Uses pagination to fetch all pages, similar to fetch_episodes_metadata. Args: base_url: Nyaa.si search URL Returns: Tuple of (crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page)""" crc32_to_link = {} crc32_to_text = {} crc32_to_magnet = {} - page = 1 + + # Get total number of pages by parsing first page's pagination controls + soup, success = _fetch_crc32_page(base_url, 1) + if not success: + return crc32_to_link, crc32_to_text, crc32_to_magnet, 0 + + total_pages = _get_total_pages(soup) last_checked_page = 0 - while True: + # Loop from page 1 to total_pages (similar to fetch_episodes_metadata) + page = 1 + while page <= total_pages: if _shutdown_requested: print(_SHUTDOWN_MESSAGE) break - soup, success = _fetch_crc32_page(base_url, page) + # Use cached soup for page 1, fetch for others + if page == 1: + page_soup = soup + success = True + else: + page_soup, success = _fetch_crc32_page(base_url, page) + if not success: break - found_in_page = _process_crc32_page_rows(soup, crc32_to_link, crc32_to_text, crc32_to_magnet) + _process_crc32_page_rows(page_soup, crc32_to_link, crc32_to_text, crc32_to_magnet) - if _shutdown_requested or not found_in_page: + if _shutdown_requested: break last_checked_page = page page += 1 + if page <= total_pages: # Don't sleep after last page + time.sleep(REQUEST_DELAY_SECONDS) return crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page diff --git a/coverage.xml b/coverage.xml index 0f25eb6..c4df69b 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -111,7 +111,7 @@ - + @@ -322,575 +322,582 @@ - - - - + + + - - - - - + + + + + + - - - - - - + + + + + - - - - - - - - - - - - + + + + + + + + + + - + + - - - - - + + + - - - + + + + + + - - - - + + + + - - - - - - - + - - + + + - - - - - - + + + + + + + + + + - - - - + + + + - - - + - - + + + + + + + + + - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - + + + - + - - - - - + + + - + - + - - + + + + + + + - - - - - - - - - + + + + - + - - + + - - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - + - + - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + - - - + - + + - + - - - - + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - + + + + - + - - - - + + + + + + + - - - - - - - + + + + - + + + - + + + + - + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + - + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - + + - - + - - - - + + + - - - + + + - + + + + - - - - - - + + + + + + + - - - - + + + - - + + + - - - + + + - + - - - - - + + + - - - - + + + + + + + - - - - + + + + - - - + + + - - - - - + + - - - - + + + + + + - + + - - - - - - + + - - + + - - + + + + + + - - - + + + - - - - - + + + - - - - - - + + + + + - + - + - + - - - + + - - + + + + + + - + + - - - - - - + + + + + + + + - - + + + + - - - + - - - - - - - - - + + + + + + + + - - + - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - + + - - - - + + + + + - - - - - - - + + + + + + - - + + - - + - + + + + - - - - - + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + diff --git a/tests/test_missing_detection.py b/tests/test_missing_detection.py index 38d071b..78ef4d4 100644 --- a/tests/test_missing_detection.py +++ b/tests/test_missing_detection.py @@ -155,11 +155,15 @@ def test_fetch_crc32_links_filters_quality(self, mock_get): @patch('acepace.requests.get') def test_fetch_crc32_links_stops_on_empty_page(self, mock_get): - """Test that fetching stops when no matches found.""" - # First page has results + """Test that fetching processes all pages based on pagination.""" + # First page has results and pagination showing 2 pages html_with_results = """ +
    +
  • 1
    • +
  • 2
    • +
@@ -171,7 +175,7 @@ def test_fetch_crc32_links_stops_on_empty_page(self, mock_get): """ - # Second page has no results + # Second page has no results (empty table) html_empty = """ @@ -189,14 +193,15 @@ def test_fetch_crc32_links_stops_on_empty_page(self, mock_get): mock_response2.status_code = 200 mock_response2.text = html_empty + # Page 1 is fetched once (for pagination), page 2 is fetched in the loop mock_get.side_effect = [mock_response1, mock_response2] base_url = "https://nyaa.si/?f=0&c=0_0&q=one+pace" crc32_to_link, _, _, last_page = acepace.fetch_crc32_links(base_url) - # Should stop after first page (no results on second) + # Should process both pages (pagination shows 2 pages) assert len(crc32_to_link) == 1 - assert last_page == 1 + assert last_page == 2 # Processed both pages @patch('acepace.requests.get') def test_fetch_title_by_crc32(self, mock_get): From 711ec40b25a3b049c2582907d4e3a16280398d17 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 27 Jan 2026 12:14:20 +0000 Subject: [PATCH 57/75] Add debug mode --- README.md | 46 ++ acepace.py | 187 ++++-- coverage.xml | 1387 ++++++++++++++++++++++--------------------- docker-compose.yml | 2 + spec.md | 27 + tests/test_debug.py | 79 +++ 6 files changed, 1005 insertions(+), 723 deletions(-) create mode 100644 tests/test_debug.py diff --git a/README.md b/README.md index 62e21e8..4e77302 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ docker run --rm \ -e TORRENT_HOST=127.0.0.1 \ -e TORRENT_PORT=9091 \ -e NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc \ + -e DEBUG=true \ timothe/ace-pace:latest ``` @@ -78,6 +79,10 @@ The following environment variables can be used to configure Ace-Pace in Docker: - `TORRENT_PORT` - BitTorrent client port (default: `9091` for Transmission, `8080` for qBittorrent) - `TORRENT_USER` - BitTorrent client username (optional) - `TORRENT_PASSWORD` - BitTorrent client password (optional) +- `DEBUG` - Enable debug output for troubleshooting (default: `false`) + - Set to `true`, `1`, `yes`, or `on` to enable detailed debug information + - When enabled, shows troubleshooting info, sample CRC32s, comparison details, and processing statistics + - Useful for diagnosing issues with missing episode detection or data processing - `TZ` - Timezone (default: `Europe/London`) ### Docker Volume Mounts @@ -132,6 +137,47 @@ This will generate: - `htmlcov/` - HTML coverage report (open `htmlcov/index.html` in a browser) - Terminal output showing coverage summary +## 🐛 Debug Mode + +Ace-Pace includes a debug mode that provides detailed troubleshooting information. This is useful when diagnosing issues with missing episode detection or data processing. + +### Enabling Debug Mode + +**In Python (local execution):** +```bash +DEBUG=true python acepace.py --folder /path/to/videos +``` + +**In Docker:** +```bash +docker run --rm \ + -v /path/to/OnePaceLibrary:/media:rw \ + -v /path/to/config:/config:rw \ + timothe/ace-pace:latest +``` + +**In Docker Compose:** +Add to your `docker-compose.yml`: +```yaml +environment: + - DEBUG=true +``` + +### Debug Output + +When debug mode is enabled, you'll see additional information including: +- Episode fetching progress and page counts +- CRC32 normalization and comparison details +- Sample CRC32s from both Nyaa and local sources +- File processing statistics (cached vs calculated) +- Mapping issues and comparison mismatches +- Intersection and difference analysis +- Processing statistics for each operation + +All debug output is prefixed with "DEBUG:" for easy filtering. + +**Note:** Debug mode defaults to `false` (disabled). Set `DEBUG` to `true`, `1`, `yes`, or `on` to enable. The value is case-insensitive. + ## 🛠️ How to Use Run the script using Python with the following command: diff --git a/acepace.py b/acepace.py index 08ce2d1..0cc6fa3 100644 --- a/acepace.py +++ b/acepace.py @@ -17,6 +17,10 @@ # Check if running in Docker (non-interactive mode) IS_DOCKER = "RUN_DOCKER" in os.environ +# Check if debug mode is enabled (via DEBUG environment variable) +# Defaults to False if not set or empty +DEBUG_MODE = os.getenv("DEBUG", "").lower() in ("true", "1", "yes", "on") + # Global flag for graceful shutdown _shutdown_requested = False @@ -24,6 +28,13 @@ _SHUTDOWN_MESSAGE = "Shutdown requested, stopping fetch operation..." +def debug_print(*args, **kwargs): + """Print debug messages only if DEBUG mode is enabled. + Works exactly like print() but only outputs when DEBUG environment variable is set.""" + if DEBUG_MODE: + print(*args, **kwargs) + + def _signal_handler(signum, frame): """Handle shutdown signals gracefully.""" global _shutdown_requested @@ -359,8 +370,10 @@ def fetch_episodes_metadata(base_url=None): # Get total number of pages by parsing first page's pagination controls soup, success = _fetch_episodes_page(base_url, 1) if not success: + debug_print("DEBUG: Failed to fetch first page for episodes metadata") return episodes total_pages = _get_total_pages(soup) + debug_print(f"DEBUG: Found {total_pages} total pages to process for episodes metadata") # Loop from page 1 to total_pages page = 1 @@ -390,8 +403,10 @@ def update_episodes_index_db(base_url=None): Args: base_url: Base URL for Nyaa search. If None, uses default. """ + debug_print(f"DEBUG: Starting update_episodes_index_db with URL: {base_url}") conn = init_episodes_db() episodes = fetch_episodes_metadata(base_url) + debug_print(f"DEBUG: Fetched {len(episodes)} episodes from Nyaa") c = conn.cursor() count = 0 for crc32, title, page_link in episodes: @@ -409,6 +424,7 @@ def update_episodes_index_db(base_url=None): set_episodes_metadata(conn, "episodes_db_last_update", now_str) print(f"Episodes index updated with {count} entries.") print(f"Last update: {now_str}") + debug_print(f"DEBUG: Updated {count} entries in episodes_index database") conn.close() @@ -563,7 +579,7 @@ def _process_crc32_page_rows(soup, crc32_to_link, crc32_to_text, crc32_to_magnet if success: found_count += 1 elif should_warn and filename_text: - print(f"Warning: No CRC32 found in title '{filename_text}'") + debug_print(f"Warning: No CRC32 found in title '{filename_text}'") return found_count @@ -579,12 +595,16 @@ def fetch_crc32_links(base_url): crc32_to_text = {} crc32_to_magnet = {} + debug_print(f"DEBUG: Starting fetch_crc32_links with URL: {base_url}") + # Get total number of pages by parsing first page's pagination controls soup, success = _fetch_crc32_page(base_url, 1) if not success: + debug_print("DEBUG: Failed to fetch first page for CRC32 links") return crc32_to_link, crc32_to_text, crc32_to_magnet, 0 total_pages = _get_total_pages(soup) + debug_print(f"DEBUG: Found {total_pages} total pages to process for CRC32 links") last_checked_page = 0 # Loop from page 1 to total_pages (similar to fetch_episodes_metadata) @@ -604,7 +624,8 @@ def fetch_crc32_links(base_url): if not success: break - _process_crc32_page_rows(page_soup, crc32_to_link, crc32_to_text, crc32_to_magnet) + episodes_found = _process_crc32_page_rows(page_soup, crc32_to_link, crc32_to_text, crc32_to_magnet) + debug_print(f"DEBUG: Page {page}/{total_pages}: Found {episodes_found} valid episodes (total so far: {len(crc32_to_link)})") if _shutdown_requested: break @@ -613,6 +634,8 @@ def fetch_crc32_links(base_url): page += 1 if page <= total_pages: # Don't sleep after last page time.sleep(REQUEST_DELAY_SECONDS) + + debug_print(f"DEBUG: Completed fetch_crc32_links: {len(crc32_to_link)} total episodes found across {last_checked_page} pages") return crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page @@ -654,10 +677,10 @@ def fetch_title_by_crc32(crc32): print(f"Found {crc32} on Nyaa!") return matched_titles[0] elif len(matched_titles) == 0: - print(f"Warning: No title found for {crc32}") + debug_print(f"Warning: No title found for {crc32}") return None else: - print(f"Warning: Multiple titles found for CRC32 {crc32}: {matched_titles}") + debug_print(f"Warning: Multiple titles found for CRC32 {crc32}: {matched_titles}") return None @@ -683,6 +706,7 @@ def _process_video_file(file_path, c, conn, local_crc32s): row = c.fetchone() if row: local_crc32s.add(row[0]) + debug_print(f"DEBUG: Using cached CRC32 for {os.path.basename(file_path)}: {row[0]}") return True # Calculate CRC32 @@ -692,8 +716,10 @@ def _process_video_file(file_path, c, conn, local_crc32s): crc32 = _calculate_file_crc32(file_path) if crc32 is None: + debug_print(f"DEBUG: CRC32 calculation interrupted for {file_path}") return False # Calculation interrupted + debug_print(f"DEBUG: Calculated CRC32 for {file_name}: {crc32}") local_crc32s.add(crc32) c.execute( "INSERT OR REPLACE INTO crc32_cache (file_path, crc32) VALUES (?, ?)", @@ -703,6 +729,40 @@ def _process_video_file(file_path, c, conn, local_crc32s): return True +def _process_single_file_for_crc32(file_path, c, conn, local_crc32s): + """Process a single file for CRC32 calculation. + Returns tuple: (success: bool, was_cached: bool)""" + normalized_path = normalize_file_path(file_path) + # Check if already in DB + c.execute("SELECT crc32 FROM crc32_cache WHERE file_path = ?", (normalized_path,)) + row = c.fetchone() + was_cached = bool(row) + + if _process_video_file(file_path, c, conn, local_crc32s): + return True, was_cached + return False, was_cached + + +def _process_files_in_directory(root, files, c, conn, local_crc32s, stats): + """Process files in a directory, updating stats. + Returns True if processing should continue, False if shutdown requested.""" + for file in files: + if _shutdown_requested: + return False + + ext = os.path.splitext(file)[1].lower() + if ext in VIDEO_EXTENSIONS: + file_path = os.path.join(root, file) + success, was_cached = _process_single_file_for_crc32(file_path, c, conn, local_crc32s) + if success: + stats['processed'] += 1 + if was_cached: + stats['cached'] += 1 + else: + stats['calculated'] += 1 + return True + + def calculate_local_crc32(folder, conn): """Calculate CRC32 checksums for all video files in the given folder. Uses cached values from database when available. @@ -712,20 +772,20 @@ def calculate_local_crc32(folder, conn): Returns: Set of CRC32 checksums found in the folder.""" local_crc32s = set() c = conn.cursor() + stats = {'processed': 0, 'cached': 0, 'calculated': 0} + + debug_print(f"DEBUG: Starting calculate_local_crc32 for folder: {folder}") for root, dirs, files in os.walk(folder): if _shutdown_requested: print("Shutdown requested, stopping file processing...") break - for file in files: - if _shutdown_requested: - break - - ext = os.path.splitext(file)[1].lower() - if ext in VIDEO_EXTENSIONS: - file_path = os.path.join(root, file) - _process_video_file(file_path, c, conn, local_crc32s) + if not _process_files_in_directory(root, files, c, conn, local_crc32s, stats): + break + + debug_print(f"DEBUG: Processed {stats['processed']} video files ({stats['cached']} from cache, {stats['calculated']} calculated)") + debug_print(f"DEBUG: Found {len(local_crc32s)} unique CRC32s") return local_crc32s @@ -1117,25 +1177,25 @@ def _print_report_header(conn, folder, args): def _print_troubleshooting_header(crc32_to_link, local_crc32s): """Print initial troubleshooting information header.""" - print("\n=== TROUBLESHOOTING INFO ===") - print(f"Episodes from Nyaa (crc32_to_link keys): {len(crc32_to_link)}") - print(f"Local CRC32s: {len(local_crc32s)}") + debug_print("\n=== DEBUG: TROUBLESHOOTING INFO ===") + debug_print(f"Episodes from Nyaa (crc32_to_link keys): {len(crc32_to_link)}") + debug_print(f"Local CRC32s: {len(local_crc32s)}") # Check for empty sets if len(crc32_to_link) == 0: - print("WARNING: No episodes fetched from Nyaa! Check URL and quality filtering.") + debug_print("WARNING: No episodes fetched from Nyaa! Check URL and quality filtering.") if len(local_crc32s) == 0: - print("WARNING: No local CRC32s found! Check folder path and file extensions.") + debug_print("WARNING: No local CRC32s found! Check folder path and file extensions.") # Show sample CRC32s from both sources (first 5) if crc32_to_link: sample_nyaa = list(crc32_to_link.keys())[:5] - print(f"Sample Nyaa CRC32s (first 5): {sample_nyaa}") - print(f"Sample Nyaa CRC32 types: {[type(c).__name__ for c in sample_nyaa]}") + debug_print(f"Sample Nyaa CRC32s (first 5): {sample_nyaa}") + debug_print(f"Sample Nyaa CRC32 types: {[type(c).__name__ for c in sample_nyaa]}") if local_crc32s: sample_local = list(local_crc32s)[:5] - print(f"Sample local CRC32s (first 5): {sample_local}") - print(f"Sample local CRC32 types: {[type(c).__name__ for c in sample_local]}") + debug_print(f"Sample local CRC32s (first 5): {sample_local}") + debug_print(f"Sample local CRC32 types: {[type(c).__name__ for c in sample_local]}") def _normalize_crc32_sets(crc32_to_link, local_crc32s): @@ -1144,15 +1204,15 @@ def _normalize_crc32_sets(crc32_to_link, local_crc32s): nyaa_crc32s_normalized = {str(c).strip().upper() for c in crc32_to_link.keys()} local_crc32s_normalized = {str(c).strip().upper() for c in local_crc32s} - print("\nAfter normalization:") - print(f"Nyaa CRC32s: {len(nyaa_crc32s_normalized)}") - print(f"Local CRC32s: {len(local_crc32s_normalized)}") + debug_print("\nAfter normalization:") + debug_print(f"Nyaa CRC32s: {len(nyaa_crc32s_normalized)}") + debug_print(f"Local CRC32s: {len(local_crc32s_normalized)}") # Check for matches using normalized sets matches_normalized = nyaa_crc32s_normalized & local_crc32s_normalized - print(f"Matches after normalization: {len(matches_normalized)}") + debug_print(f"Matches after normalization: {len(matches_normalized)}") if matches_normalized: - print(f"Sample matches (first 3): {list(matches_normalized)[:3]}") + debug_print(f"Sample matches (first 3): {list(matches_normalized)[:3]}") return nyaa_crc32s_normalized, local_crc32s_normalized @@ -1170,13 +1230,13 @@ def _build_normalized_to_original_mapping(crc32_to_link, nyaa_crc32s_normalized) mapping_issues = [] # Verify the mapping is correct if len(normalized_to_original) != len(nyaa_crc32s_normalized): - print(f"WARNING: Mapping size mismatch! normalized_to_original: {len(normalized_to_original)}, nyaa_crc32s_normalized: {len(nyaa_crc32s_normalized)}") - print("This could indicate duplicate normalized CRC32s or mapping issues.") + debug_print(f"WARNING: Mapping size mismatch! normalized_to_original: {len(normalized_to_original)}, nyaa_crc32s_normalized: {len(nyaa_crc32s_normalized)}") + debug_print("This could indicate duplicate normalized CRC32s or mapping issues.") # Show which normalized CRC32s are missing from the mapping missing_from_mapping = nyaa_crc32s_normalized - set(normalized_to_original.keys()) if missing_from_mapping: mapping_issues = list(missing_from_mapping) - print(f"Normalized CRC32s missing from mapping (first 5): {mapping_issues[:5]}") + debug_print(f"Normalized CRC32s missing from mapping (first 5): {mapping_issues[:5]}") return normalized_to_original, mapping_issues @@ -1201,12 +1261,12 @@ def _build_missing_list(missing_normalized_set, normalized_to_original, crc32_to break if not found: mapping_errors.append(norm_crc) - print(f"ERROR: Could not find original key for normalized CRC32 '{norm_crc}'") + debug_print(f"ERROR: Could not find original key for normalized CRC32 '{norm_crc}'") if mapping_errors: - print(f"WARNING: {len(mapping_errors)} missing episodes could not be mapped to original keys!") - print("This is a critical error - these episodes will not be included in the missing list.") - print(f"Affected normalized CRC32s (first 10): {mapping_errors[:10]}") + debug_print(f"WARNING: {len(mapping_errors)} missing episodes could not be mapped to original keys!") + debug_print("This is a critical error - these episodes will not be included in the missing list.") + debug_print(f"Affected normalized CRC32s (first 10): {mapping_errors[:10]}") return missing, mapping_errors @@ -1216,45 +1276,45 @@ def _print_comparison_results(nyaa_crc32s_normalized, local_crc32s_normalized, """Print comparison results and troubleshooting information.""" # Also check the original comparison for debugging original_missing_count = len([c for c in crc32_to_link.keys() if c not in local_crc32s]) - print(f"Missing episodes (original comparison): {original_missing_count}") - print(f"Missing episodes (normalized comparison): {len(missing)}") - print(f"Missing normalized CRC32s: {len(missing_normalized)}") + debug_print(f"Missing episodes (original comparison): {original_missing_count}") + debug_print(f"Missing episodes (normalized comparison): {len(missing)}") + debug_print(f"Missing normalized CRC32s: {len(missing_normalized)}") if original_missing_count != len(missing): - print(f"WARNING: Comparison mismatch detected! Original: {original_missing_count}, Normalized: {len(missing)}") - print("This suggests a data type or format issue. Using normalized comparison.") + debug_print(f"WARNING: Comparison mismatch detected! Original: {original_missing_count}, Normalized: {len(missing)}") + debug_print("This suggests a data type or format issue. Using normalized comparison.") # Show intersection details intersection = nyaa_crc32s_normalized & local_crc32s_normalized - print(f"Intersection (episodes found locally): {len(intersection)}") + debug_print(f"Intersection (episodes found locally): {len(intersection)}") if intersection: - print(f"Sample found episodes (first 3): {list(intersection)[:3]}") + debug_print(f"Sample found episodes (first 3): {list(intersection)[:3]}") # Show difference details difference = nyaa_crc32s_normalized - local_crc32s_normalized - print(f"Difference (episodes NOT found locally): {len(difference)}") + debug_print(f"Difference (episodes NOT found locally): {len(difference)}") if difference: - print(f"Sample missing episodes (first 3): {list(difference)[:3]}") + debug_print(f"Sample missing episodes (first 3): {list(difference)[:3]}") # Check if sets are suspiciously similar (potential bug indicator) if len(nyaa_crc32s_normalized) > 0 and len(local_crc32s_normalized) > 0: similarity_ratio = len(intersection) / len(nyaa_crc32s_normalized) - print(f"Similarity ratio (intersection/nyaa): {similarity_ratio:.2%}") + debug_print(f"Similarity ratio (intersection/nyaa): {similarity_ratio:.2%}") if similarity_ratio > 0.95 and len(difference) == 0: - print("WARNING: Almost all Nyaa episodes appear to be found locally!") - print("This might indicate a comparison bug or data issue.") - print("Please verify that your local files actually contain all these episodes.") + debug_print("WARNING: Almost all Nyaa episodes appear to be found locally!") + debug_print("This might indicate a comparison bug or data issue.") + debug_print("Please verify that your local files actually contain all these episodes.") # Check for sets being identical (definite bug) if nyaa_crc32s_normalized == local_crc32s_normalized: - print("ERROR: Nyaa and local CRC32 sets are IDENTICAL!") - print("This indicates a critical bug - the sets should not be the same.") - print("Possible causes:") - print(" - Local CRC32s are being populated from Nyaa data (wrong source)") - print(" - Comparison is using the same set for both sides") - print(" - Database corruption or incorrect data") + debug_print("ERROR: Nyaa and local CRC32 sets are IDENTICAL!") + debug_print("This indicates a critical bug - the sets should not be the same.") + debug_print("Possible causes:") + debug_print(" - Local CRC32s are being populated from Nyaa data (wrong source)") + debug_print(" - Comparison is using the same set for both sides") + debug_print(" - Database corruption or incorrect data") - print("=== END TROUBLESHOOTING INFO ===\n") + debug_print("=== END DEBUG: TROUBLESHOOTING INFO ===\n") def _calculate_and_find_missing(folder, conn, args, last_run): @@ -1274,6 +1334,11 @@ def _calculate_and_find_missing(folder, conn, args, last_run): local_crc32s = calculate_local_crc32(folder, conn) print(f"Found {len(local_crc32s)} local CRC32 hashes.") + + debug_print("DEBUG: Starting missing episode detection") + debug_print(f"DEBUG: Folder scanned: {folder}") + debug_print(f"DEBUG: Episodes from Nyaa: {len(crc32_to_link)}") + debug_print(f"DEBUG: Local CRC32s: {len(local_crc32s)}") # Print troubleshooting header _print_troubleshooting_header(crc32_to_link, local_crc32s) @@ -1531,6 +1596,18 @@ def _handle_main_commands(args, conn, folder): # Note: To download missing episodes, use --download flag with --client +def _print_header(): + """Print Ace-Pace header banner.""" + print("=" * 60) + print(" " * 20 + "Ace-Pace") + print(" " * 12 + "One Pace Library Manager") + print("=" * 60) + if IS_DOCKER: + print("Running in Docker mode (non-interactive)") + print("-" * 60) + print() + + def main(): # Register signal handlers for graceful shutdown signal.signal(signal.SIGTERM, _signal_handler) @@ -1544,10 +1621,10 @@ def main(): _print_help() sys.exit(0) - # Only show Docker mode message once, and not for --db or --episodes_update commands + # Print header only for main command (not for --db or --episodes_update) # Also suppress for help command if IS_DOCKER and not args.db and not args.episodes_update and not args.help: - print("Running in Docker mode (non-interactive)") + _print_header() if not _validate_url(args.url): sys.exit(1) diff --git a/coverage.xml b/coverage.xml index c4df69b..f125f45 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -26,878 +26,929 @@ - - - - - - - - - - + + + + + + + + + - - - + - - + + - - + + + - - - - - + + + + + + + + - + + - - - + - - - - + + + + - - - + + + - - + + + - - - - + + + - - + + - - + + - - - - - - + + - - - - - - - + + + + + + + - + + - - + - + - - - - - - + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - + + + + + + + + + - - - - - - + + + + + - - - - + + + + - - - - - + + - - + + + + + - - + + + - - - + + - - - + + + + + + - - - - + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - + + + + - - - - - - - - + + + + + + + + - - + + + + + + - - - - - - + + + - - - - - + + + + + + + - - + + + + + - - + + + + - - - - - - - - - - - + + + + - - + + - + + + - - - - - - - - - - + + + + + + + + - - - - - + - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + - - - - - + + + + + + + + + + - + - - - + + + - - - - - - - - - - - - + + + + + + + + + - - - - - - - + + + + + + + + - - - - - + + + - - - - - - - - - - - + + + + + + + + + + + - - - + + + - + + - - - - - - - - - - + + + + + + + + - - - + + + + + + + - - - - + + - - - - - + + + + + + - + + + + + - - - + + + - - - - - - + + + + - - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + - + + - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + - - + - - + - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + - - + + - - - + - - - + + + + + - - - - - - - + + + - + - - - - + - - - - - - - - - + + + + + + - - - + + - - - + + + + + + + - - - - + - + - - - - - - - + + + + + + + + + + - + + - - - - - + + + + + + + + + + + + - + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - + - - - + - - - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - + - + + - + + + - - - + + - + - + + + - + - + + + - - + + + - - - - - - - - + + + + + + + + + - + - + + - - - - - + + + - - + + + + - - - + + + + - - - - - - - + + + - + - - - - - - - - + + + + + + + + + - - - + + + - - - - + + + + - - - - - - - - - + + + + + + - - - - - - - + + + + + + + + + + + + - - + + - - - + + + + + + + + + + - + + + + + + + - - + + + + + + - - - - - - - + + + + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - + + + + - - - - - - - - - - + + + + + + + + + + + + + - - - - + + + + - - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index 0a88d76..cce9287 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,8 @@ services: #- TORRENT_USER=admin # BitTorrent client password (default: empty, not required) #- TORRENT_PASSWORD=password + # Enable debug output for troubleshooting (default: false) + #- DEBUG=true # networks: # proxy: diff --git a/spec.md b/spec.md index 7ebf9c0..a31df2d 100644 --- a/spec.md +++ b/spec.md @@ -188,6 +188,11 @@ Ace-Pace/ - `EPISODES_UPDATE`: Set to "true" to update episodes index on container start (default: not set/false) - `DB`: Set to "true" to export database on container start (default: not set/false) - `RUN_DOCKER`: Flag to enable Docker mode (non-interactive) + - `DEBUG`: Enable debug output for troubleshooting (default: `false`) + - Set to `true`, `1`, `yes`, or `on` to enable detailed debug information + - When enabled, shows troubleshooting info, sample CRC32s, comparison details, and processing statistics + - Useful for diagnosing issues with missing episode detection or data processing + - Works in both Docker and local Python execution ## Command-Line Interface @@ -243,6 +248,21 @@ Ace-Pace/ 7. Script generates rename plan and prompts for confirmation (auto-confirms in Docker mode) 8. Script renames files and updates database with normalized paths +### Debug Mode +- **DEBUG Environment Variable**: Controls verbose troubleshooting output + - Default: `false` (no debug output) + - Set to `true`, `1`, `yes`, or `on` to enable + - When enabled, provides detailed information about: + - Episode fetching progress and page counts + - CRC32 normalization and comparison details + - Sample CRC32s from both Nyaa and local sources + - File processing statistics (cached vs calculated) + - Mapping issues and comparison mismatches + - Intersection and difference analysis + - Useful for diagnosing issues with missing episode detection + - Works in both Docker and local Python execution + - All debug output is prefixed with "DEBUG:" for easy filtering + ### Docker Workflow 1. Container starts with `RUN_DOCKER` environment variable set 2. Entrypoint script (`entrypoint.sh`) orchestrates execution: @@ -519,3 +539,10 @@ When working on this project: - Host: localhost - Port: 9091 (transmission) or 8080 (qbittorrent) 22. **Download logging** - Log connection parameters used for download in Docker mode for transparency + +23. **Debug mode** - Use `DEBUG` environment variable to control troubleshooting output + - Defaults to `false` (no debug output) + - Set to `true`, `1`, `yes`, or `on` to enable + - All debug output uses `debug_print()` function which checks `DEBUG_MODE` flag + - Debug output includes troubleshooting info, sample data, processing statistics, and comparison details + - Useful for diagnosing issues without cluttering normal output diff --git a/tests/test_debug.py b/tests/test_debug.py new file mode 100644 index 0000000..3917859 --- /dev/null +++ b/tests/test_debug.py @@ -0,0 +1,79 @@ +"""Unit tests for DEBUG mode functionality.""" +import pytest +import os +import sys +from unittest.mock import patch, MagicMock + +# Add parent directory to path to import acepace +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +class TestDebugMode: + """Tests for DEBUG mode functionality.""" + + def test_debug_print_outputs_when_enabled(self, capsys): + """Test that debug_print outputs when DEBUG mode is enabled.""" + with patch('acepace.DEBUG_MODE', True): + import acepace + acepace.debug_print("Test debug message") + captured = capsys.readouterr() + assert "Test debug message" in captured.out + + def test_debug_print_no_output_when_disabled(self, capsys): + """Test that debug_print does not output when DEBUG mode is disabled.""" + with patch('acepace.DEBUG_MODE', False): + import acepace + acepace.debug_print("Test debug message") + captured = capsys.readouterr() + assert "Test debug message" not in captured.out + + def test_debug_mode_parsing_true(self): + """Test that DEBUG mode parsing works with 'true'.""" + with patch.dict(os.environ, {'DEBUG': 'true'}, clear=False): + debug_value = os.getenv("DEBUG", "").lower() in ("true", "1", "yes", "on") + assert debug_value is True + + def test_debug_mode_parsing_1(self): + """Test that DEBUG mode parsing works with '1'.""" + with patch.dict(os.environ, {'DEBUG': '1'}, clear=False): + debug_value = os.getenv("DEBUG", "").lower() in ("true", "1", "yes", "on") + assert debug_value is True + + def test_debug_mode_parsing_yes(self): + """Test that DEBUG mode parsing works with 'yes'.""" + with patch.dict(os.environ, {'DEBUG': 'yes'}, clear=False): + debug_value = os.getenv("DEBUG", "").lower() in ("true", "1", "yes", "on") + assert debug_value is True + + def test_debug_mode_parsing_on(self): + """Test that DEBUG mode parsing works with 'on'.""" + with patch.dict(os.environ, {'DEBUG': 'on'}, clear=False): + debug_value = os.getenv("DEBUG", "").lower() in ("true", "1", "yes", "on") + assert debug_value is True + + def test_debug_mode_parsing_case_insensitive(self): + """Test that DEBUG mode parsing is case-insensitive.""" + with patch.dict(os.environ, {'DEBUG': 'TRUE'}, clear=False): + debug_value = os.getenv("DEBUG", "").lower() in ("true", "1", "yes", "on") + assert debug_value is True + + def test_debug_mode_parsing_false(self): + """Test that DEBUG mode parsing works with 'false'.""" + with patch.dict(os.environ, {'DEBUG': 'false'}, clear=False): + debug_value = os.getenv("DEBUG", "").lower() in ("true", "1", "yes", "on") + assert debug_value is False + + def test_debug_mode_parsing_empty(self): + """Test that DEBUG mode parsing works with empty string.""" + with patch.dict(os.environ, {'DEBUG': ''}, clear=False): + debug_value = os.getenv("DEBUG", "").lower() in ("true", "1", "yes", "on") + assert debug_value is False + + def test_debug_mode_parsing_not_set(self): + """Test that DEBUG mode parsing works when not set.""" + with patch.dict(os.environ, {}, clear=True): + # Remove DEBUG if it exists + if 'DEBUG' in os.environ: + del os.environ['DEBUG'] + debug_value = os.getenv("DEBUG", "").lower() in ("true", "1", "yes", "on") + assert debug_value is False From f0cf116d2e747a03b748f3c8bd8eb4aa8e0ba6f3 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 27 Jan 2026 12:43:01 +0000 Subject: [PATCH 58/75] Moving Docker headers --- acepace.py | 5 ----- entrypoint.sh | 9 +++++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/acepace.py b/acepace.py index 0cc6fa3..8e736de 100644 --- a/acepace.py +++ b/acepace.py @@ -1621,11 +1621,6 @@ def main(): _print_help() sys.exit(0) - # Print header only for main command (not for --db or --episodes_update) - # Also suppress for help command - if IS_DOCKER and not args.db and not args.episodes_update and not args.help: - _print_header() - if not _validate_url(args.url): sys.exit(1) diff --git a/entrypoint.sh b/entrypoint.sh index 570ecb1..822e80d 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -14,6 +14,15 @@ cleanup() { # This trap handles signals that arrive when no Python process is running trap 'cleanup' 15 2 +# Print Ace-Pace header at the very beginning +echo "============================================================" +echo " Ace-Pace" +echo " One Pace Library Manager" +echo "============================================================" +echo "Running in Docker mode (non-interactive)" +echo "------------------------------------------------------------" +echo "" + # Run episodes update if requested if [ "$EPISODES_UPDATE" = "true" ]; then python /app/acepace.py --episodes_update ${NYAA_URL:+--url "$NYAA_URL"} From b906c7b41f50365cac411c2ffdde78b893e3166e Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 27 Jan 2026 12:52:38 +0000 Subject: [PATCH 59/75] Fix Docker version fetching episodes twice --- acepace.py | 150 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 147 insertions(+), 3 deletions(-) diff --git a/acepace.py b/acepace.py index 8e736de..79f6869 100644 --- a/acepace.py +++ b/acepace.py @@ -59,6 +59,7 @@ def _signal_handler(signum, frame): HTTP_OK = 200 REQUEST_DELAY_SECONDS = 0.2 CRC32_CHUNK_SIZE = 8192 +MAGNET_LINK_PREFIX = "magnet:" # Config directory and file names CONFIG_DIR_DOCKER = "/config" @@ -439,6 +440,148 @@ def load_crc32_to_title_from_index(): return d +def load_episodes_from_index(): + """Load episodes from episodes_index database. + Returns: Tuple of (crc32_to_link, crc32_to_text) dictionaries.""" + conn = init_episodes_db() + c = conn.cursor() + c.execute("SELECT crc32, title, page_link FROM episodes_index") + crc32_to_link = {} + crc32_to_text = {} + for crc32, title, page_link in c.fetchall(): + crc32_to_link[crc32] = page_link + crc32_to_text[crc32] = title + conn.close() + return crc32_to_link, crc32_to_text + + +def _extract_crc32_from_row(row, crc32_set): + """Extract CRC32 and magnet link from a table row if it matches the set. + Returns tuple: (crc32, magnet_link) or (None, None) if not found or doesn't match.""" + title_link, magnet_link = _extract_links_from_row(row) + if (not title_link or not magnet_link or + not isinstance(magnet_link, str) or + not magnet_link.startswith(MAGNET_LINK_PREFIX) or + not hasattr(title_link, 'text')): + return None, None + + filename_text = title_link.text + matches = CRC32_REGEX.findall(filename_text) + if not matches: + return None, None + + crc32 = matches[-1].upper() + if crc32 in crc32_set: + return crc32, magnet_link + return None, None + + +def _process_page_for_magnet_links(page_soup, crc32_set): + """Process a single page to extract magnet links matching CRC32s. + Returns tuple: (crc32_to_magnet dict, found_count)""" + crc32_to_magnet = {} + found_count = 0 + + if page_soup is None: + return crc32_to_magnet, found_count + + table = page_soup.find("table", class_="torrent-list") + if not table: + return crc32_to_magnet, found_count + + rows = table.find_all("tr") + for row in rows: + if _shutdown_requested: + break + crc32, magnet_link = _extract_crc32_from_row(row, crc32_set) + if crc32: + crc32_to_magnet[crc32] = magnet_link + found_count += 1 + + return crc32_to_magnet, found_count + + +def _fetch_magnet_links_from_search_results(base_url, crc32_to_link): + """Fetch magnet links from Nyaa search results pages by matching CRC32s. + This is faster than fetching individual episode pages. + Args: + base_url: Nyaa search URL + crc32_to_link: Dictionary mapping CRC32 to page_link (from database) + Returns: Dictionary mapping CRC32 to magnet_link""" + crc32_to_magnet = {} + crc32_set = set(crc32_to_link.keys()) + + # Get total number of pages + soup, success = _fetch_crc32_page(base_url, 1) + if not success: + return crc32_to_magnet + + total_pages = _get_total_pages(soup) + print(f"Fetching magnet links from {total_pages} pages...") + + # Process pages to extract magnet links + page = 1 + found_count = 0 + while page <= total_pages and found_count < len(crc32_set): + if _shutdown_requested: + break + + # Get page soup + if page == 1: + page_soup = soup + else: + page_soup_result, success = _fetch_crc32_page(base_url, page) + if not success or page_soup_result is None: + break + page_soup = page_soup_result + + # Process page for magnet links + page_magnets, page_found = _process_page_for_magnet_links(page_soup, crc32_set) + crc32_to_magnet.update(page_magnets) + found_count += page_found + + page += 1 + if page <= total_pages: + time.sleep(REQUEST_DELAY_SECONDS) + + return crc32_to_magnet + + +def load_crc32_data_from_index_or_fetch(base_url, use_index_if_recent=True, recent_threshold_minutes=10): + """Load CRC32 data from episodes_index database if recently updated, otherwise fetch from Nyaa. + Args: + base_url: Nyaa search URL (used if database is not recent) + use_index_if_recent: If True, check if database was recently updated + recent_threshold_minutes: Consider database recent if updated within this many minutes + Returns: Tuple of (crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page)""" + # Check if we should use the database + if use_index_if_recent: + conn = init_episodes_db() + last_update_str = get_episodes_metadata(conn, "episodes_db_last_update") + conn.close() + + if last_update_str: + try: + last_update = datetime.strptime(last_update_str, "%Y-%m-%d %H:%M:%S") + time_diff = datetime.now() - last_update + if time_diff.total_seconds() < recent_threshold_minutes * 60: + # Database was recently updated, use it + print("Using recently updated episodes index database (skipping full Nyaa fetch)...") + crc32_to_link, crc32_to_text = load_episodes_from_index() + print(f"Loaded {len(crc32_to_link)} episodes from database.") + print("Fetching magnet links from search results...") + crc32_to_magnet = _fetch_magnet_links_from_search_results(base_url, crc32_to_link) + print(f"Fetched {len(crc32_to_magnet)} magnet links.") + # Return 0 for last_checked_page since we didn't check pages in the traditional way + return crc32_to_link, crc32_to_text, crc32_to_magnet, 0 + except (ValueError, TypeError): + # If parsing fails, fall through to fetching + pass + + # Fall back to fetching from Nyaa + return fetch_crc32_links(base_url) + + def get_metadata(conn, key): """Get metadata value from database. Args: @@ -474,7 +617,7 @@ def _extract_links_from_row(row): if a.has_attr("title"): title_link = a href = a.get("href", "") - if href.startswith("magnet:"): + if href.startswith(MAGNET_LINK_PREFIX): magnet_link = href return title_link, magnet_link @@ -987,7 +1130,7 @@ def _load_magnet_links(): reader = csv.DictReader(f) for row in reader: magnet_link = row.get("Magnet Link", "").strip() - if magnet_link.startswith("magnet:"): + if magnet_link.startswith(MAGNET_LINK_PREFIX): magnets.append(magnet_link) if not magnets: @@ -1319,8 +1462,9 @@ def _print_comparison_results(nyaa_crc32s_normalized, local_crc32s_normalized, def _calculate_and_find_missing(folder, conn, args, last_run): """Calculate local CRC32s and find missing episodes.""" + # Use database if recently updated (especially in Docker mode after --episodes_update) crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = ( - fetch_crc32_links(args.url) + load_crc32_data_from_index_or_fetch(args.url, use_index_if_recent=True, recent_threshold_minutes=10) ) print(f"Found {len(crc32_to_link)} episodes from Nyaa.") From 37d91e35505a9de1872fa37ea754b35fe7acca84 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 27 Jan 2026 14:06:36 +0000 Subject: [PATCH 60/75] Docker version fix #5423412453 --- acepace.py | 156 +------ coverage.xml | 1212 +++++++++++++++++++++++++------------------------- 2 files changed, 618 insertions(+), 750 deletions(-) diff --git a/acepace.py b/acepace.py index 79f6869..d7f8cae 100644 --- a/acepace.py +++ b/acepace.py @@ -440,146 +440,6 @@ def load_crc32_to_title_from_index(): return d -def load_episodes_from_index(): - """Load episodes from episodes_index database. - Returns: Tuple of (crc32_to_link, crc32_to_text) dictionaries.""" - conn = init_episodes_db() - c = conn.cursor() - c.execute("SELECT crc32, title, page_link FROM episodes_index") - crc32_to_link = {} - crc32_to_text = {} - for crc32, title, page_link in c.fetchall(): - crc32_to_link[crc32] = page_link - crc32_to_text[crc32] = title - conn.close() - return crc32_to_link, crc32_to_text - - -def _extract_crc32_from_row(row, crc32_set): - """Extract CRC32 and magnet link from a table row if it matches the set. - Returns tuple: (crc32, magnet_link) or (None, None) if not found or doesn't match.""" - title_link, magnet_link = _extract_links_from_row(row) - if (not title_link or not magnet_link or - not isinstance(magnet_link, str) or - not magnet_link.startswith(MAGNET_LINK_PREFIX) or - not hasattr(title_link, 'text')): - return None, None - - filename_text = title_link.text - matches = CRC32_REGEX.findall(filename_text) - if not matches: - return None, None - - crc32 = matches[-1].upper() - if crc32 in crc32_set: - return crc32, magnet_link - return None, None - - -def _process_page_for_magnet_links(page_soup, crc32_set): - """Process a single page to extract magnet links matching CRC32s. - Returns tuple: (crc32_to_magnet dict, found_count)""" - crc32_to_magnet = {} - found_count = 0 - - if page_soup is None: - return crc32_to_magnet, found_count - - table = page_soup.find("table", class_="torrent-list") - if not table: - return crc32_to_magnet, found_count - - rows = table.find_all("tr") - for row in rows: - if _shutdown_requested: - break - crc32, magnet_link = _extract_crc32_from_row(row, crc32_set) - if crc32: - crc32_to_magnet[crc32] = magnet_link - found_count += 1 - - return crc32_to_magnet, found_count - - -def _fetch_magnet_links_from_search_results(base_url, crc32_to_link): - """Fetch magnet links from Nyaa search results pages by matching CRC32s. - This is faster than fetching individual episode pages. - Args: - base_url: Nyaa search URL - crc32_to_link: Dictionary mapping CRC32 to page_link (from database) - Returns: Dictionary mapping CRC32 to magnet_link""" - crc32_to_magnet = {} - crc32_set = set(crc32_to_link.keys()) - - # Get total number of pages - soup, success = _fetch_crc32_page(base_url, 1) - if not success: - return crc32_to_magnet - - total_pages = _get_total_pages(soup) - print(f"Fetching magnet links from {total_pages} pages...") - - # Process pages to extract magnet links - page = 1 - found_count = 0 - while page <= total_pages and found_count < len(crc32_set): - if _shutdown_requested: - break - - # Get page soup - if page == 1: - page_soup = soup - else: - page_soup_result, success = _fetch_crc32_page(base_url, page) - if not success or page_soup_result is None: - break - page_soup = page_soup_result - - # Process page for magnet links - page_magnets, page_found = _process_page_for_magnet_links(page_soup, crc32_set) - crc32_to_magnet.update(page_magnets) - found_count += page_found - - page += 1 - if page <= total_pages: - time.sleep(REQUEST_DELAY_SECONDS) - - return crc32_to_magnet - - -def load_crc32_data_from_index_or_fetch(base_url, use_index_if_recent=True, recent_threshold_minutes=10): - """Load CRC32 data from episodes_index database if recently updated, otherwise fetch from Nyaa. - Args: - base_url: Nyaa search URL (used if database is not recent) - use_index_if_recent: If True, check if database was recently updated - recent_threshold_minutes: Consider database recent if updated within this many minutes - Returns: Tuple of (crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page)""" - # Check if we should use the database - if use_index_if_recent: - conn = init_episodes_db() - last_update_str = get_episodes_metadata(conn, "episodes_db_last_update") - conn.close() - - if last_update_str: - try: - last_update = datetime.strptime(last_update_str, "%Y-%m-%d %H:%M:%S") - time_diff = datetime.now() - last_update - if time_diff.total_seconds() < recent_threshold_minutes * 60: - # Database was recently updated, use it - print("Using recently updated episodes index database (skipping full Nyaa fetch)...") - crc32_to_link, crc32_to_text = load_episodes_from_index() - print(f"Loaded {len(crc32_to_link)} episodes from database.") - print("Fetching magnet links from search results...") - crc32_to_magnet = _fetch_magnet_links_from_search_results(base_url, crc32_to_link) - print(f"Fetched {len(crc32_to_magnet)} magnet links.") - # Return 0 for last_checked_page since we didn't check pages in the traditional way - return crc32_to_link, crc32_to_text, crc32_to_magnet, 0 - except (ValueError, TypeError): - # If parsing fails, fall through to fetching - pass - - # Fall back to fetching from Nyaa - return fetch_crc32_links(base_url) def get_metadata(conn, key): @@ -1462,9 +1322,8 @@ def _print_comparison_results(nyaa_crc32s_normalized, local_crc32s_normalized, def _calculate_and_find_missing(folder, conn, args, last_run): """Calculate local CRC32s and find missing episodes.""" - # Use database if recently updated (especially in Docker mode after --episodes_update) crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = ( - load_crc32_data_from_index_or_fetch(args.url, use_index_if_recent=True, recent_threshold_minutes=10) + fetch_crc32_links(args.url) ) print(f"Found {len(crc32_to_link)} episodes from Nyaa.") @@ -1524,9 +1383,11 @@ def _report_new_missing_episodes(missing, crc32_to_text): new_crc32s = set(missing) - old_missing_crc32s if new_crc32s: print(f"New missing episodes detected since last export: {len(new_crc32s)}") - for crc32 in new_crc32s: - title = crc32_to_text.get(crc32, "(Unknown Title)") - print(f"Missing: {title}") + # Only print individual episodes in DEBUG mode + if DEBUG_MODE: + for crc32 in new_crc32s: + title = crc32_to_text.get(crc32, "(Unknown Title)") + debug_print(f"Missing: {title}") def _generate_missing_episodes_report(conn, folder, args): @@ -1765,6 +1626,11 @@ def main(): _print_help() sys.exit(0) + # Print header only for main command (not for --db or --episodes_update) + # Also suppress for help command + if IS_DOCKER and not args.db and not args.episodes_update and not args.help: + _print_header() + if not _validate_url(args.url): sys.exit(1) diff --git a/coverage.xml b/coverage.xml index f125f45..23a37e3 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -44,205 +44,205 @@ - + - - - - - - - - - + + + + + + + + + - - - - - - - + + + + + + + - - - - + + + + - - + + - - + + - - + + - - + + - - - - - - - - - + + + + + + + + + - - + + - - - - + + + + - - + + - - + + - - - - + + + + - + - - + + - - - - - - - - - + + + + + + + + + - - + + - - - - + + + + - - - - - - - - - - + + + + + + + + + + - - - - - + + + + + - - + + - + - + - + - + - - - - - + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - + + + - - - - + + + + - + - - - - - - + + + + + + - + - - + + - - - + + + - - + + @@ -250,393 +250,391 @@ - - + + - - - - + + + + - - + + - - - - + + - + + + - - - + - - - - - + + + + + - + - - - - + + + + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - - - + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - + + + - - - - - - + + + + + + - - - - - - - + + + + + + - - - - - - - - - + + + + + + + + - - + + + + - - - - - - - + + + + + - + + + - - - - - - - - - + + + + + + + - - + + + - + + - - - - - - + + + + + + - - - + + + - - + - - - - - - - - + + + + + + + + + - - - + + - - - - - - - - + + + + + + + - + - - - + + + + + - - - + + - - - - - + + + + + - - + + + - - - - - + + + + - - + - - - - - - - - - - + + + + + + + + + + - + + + - - - - - - - + + + + + + + - - - + + - + - - + + - - + + - - + + - - - - + + + - + - + + + - - + - - - - - - + + + + + - - - - - - + + + + + + - - + + - - - - - - + + + + + + + + - + - - + - + + - - - + - + - - - - + + + + + - - + - - - + + + + - + - - - - + + + + - - + - - - - - + + + + + - - - - - - - + + + + + + + - - - - + + + + @@ -644,311 +642,315 @@ - + + + - - - + + - + - - - + + + + - - + - + + - - + - - - + + - + + - + + - - - + - - - + + + - - - + + + - - - - + + + + - + - + + - - - - - + + + + - - - - - - - - - + + + + + + + + + + + - - - + + - - - - - - - - - - - - - + + + + + + + + + + + + - - - - + + + + + - + + - - - - - - - - + + + + + + + - - + + - + - - + + - - - - - - - + + + + + + + + - - - - - + + + - + + - - + + - - - - + + + - + + - + + - - + + - - - - - - + + + + - - - - - + + + + + + - - - - + + + + - - - + + + - + + - + - - - - - - - - + + + + + + - - + + - + + + - - - - + + + - + + - + - - + + - + + - - + + - - - - - - - - - - - - - + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + + + + + From 963258bc357d9d5b86232cd09374336fdacdaa24 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 27 Jan 2026 14:30:06 +0000 Subject: [PATCH 61/75] Fix double fetching, maybe --- entrypoint.sh | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 822e80d..26b853b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -43,14 +43,28 @@ if [ "$DB" = "true" ]; then fi fi -# Always run missing episodes report first (updates Ace-Pace_Missing.csv) -python /app/acepace.py \ - --folder /media \ - ${NYAA_URL:+--url "$NYAA_URL"} -EXIT_CODE=$? -if [ $EXIT_CODE -ne 0 ]; then - echo "Missing episodes report failed with exit code $EXIT_CODE" - exit $EXIT_CODE +# Run missing episodes report +# Skip only if doing a single operation (episodes_update only or db only) +# This matches non-Docker behavior where each command is separate +# Run report if: no flags (default), DOWNLOAD is set, or multiple operations +SHOULD_RUN_REPORT=true +if [ "$EPISODES_UPDATE" = "true" ] && [ "$DB" != "true" ] && [ "$DOWNLOAD" != "true" ]; then + # Only updating episodes, skip report (same as non-Docker) + SHOULD_RUN_REPORT=false +elif [ "$DB" = "true" ] && [ "$EPISODES_UPDATE" != "true" ] && [ "$DOWNLOAD" != "true" ]; then + # Only exporting DB, skip report (same as non-Docker) + SHOULD_RUN_REPORT=false +fi + +if [ "$SHOULD_RUN_REPORT" = "true" ]; then + python /app/acepace.py \ + --folder /media \ + ${NYAA_URL:+--url "$NYAA_URL"} + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Missing episodes report failed with exit code $EXIT_CODE" + exit $EXIT_CODE + fi fi # If DOWNLOAD is set to true, download missing episodes after generating report From 479e742d58f589f0c0e551b35da63942a3560dec Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 27 Jan 2026 14:49:09 +0000 Subject: [PATCH 62/75] Fix Docker logic #765432 --- acepace.py | 119 ++++- coverage.xml | 1178 ++++++++++++++++++++++++++----------------------- entrypoint.sh | 17 +- 3 files changed, 746 insertions(+), 568 deletions(-) diff --git a/acepace.py b/acepace.py index d7f8cae..b5cb259 100644 --- a/acepace.py +++ b/acepace.py @@ -440,6 +440,90 @@ def load_crc32_to_title_from_index(): return d +def load_1080p_episodes_from_index(): + """Load only 1080p episodes from episodes_index database. + Returns: Tuple of (crc32_to_link, crc32_to_text) dictionaries with only 1080p episodes.""" + conn = init_episodes_db() + c = conn.cursor() + c.execute("SELECT crc32, title, page_link FROM episodes_index") + crc32_to_link = {} + crc32_to_text = {} + for crc32, title, page_link in c.fetchall(): + # Only include 1080p episodes (same filter as fetch_crc32_links) + if _is_valid_quality(title): + crc32_to_link[crc32] = page_link + crc32_to_text[crc32] = title + conn.close() + return crc32_to_link, crc32_to_text + + +def fetch_magnet_links_for_episodes_from_search(base_url, crc32_to_link): + """Fetch magnet links from Nyaa search results for episodes already in crc32_to_link. + This is more efficient than fetching all episodes again. + Args: + base_url: Nyaa search URL + crc32_to_link: Dictionary mapping CRC32 to page_link (episodes we need magnet links for) + Returns: Dictionary mapping CRC32 to magnet_link""" + crc32_to_magnet = {} + crc32_set = set(crc32_to_link.keys()) + + if not crc32_set: + return crc32_to_magnet + + # Get total number of pages + soup, success = _fetch_crc32_page(base_url, 1) + if not success: + return crc32_to_magnet + + total_pages = _get_total_pages(soup) + print(f"Fetching magnet links from {total_pages} pages...") + + # Process pages to extract magnet links for episodes we need + page = 1 + found_count = 0 + while page <= total_pages and found_count < len(crc32_set): + if _shutdown_requested: + break + + if page == 1: + page_soup = soup + else: + page_soup_result, success = _fetch_crc32_page(base_url, page) + if not success or page_soup_result is None: + break + page_soup = page_soup_result + + if page_soup is None: + break + + # Extract magnet links from rows, matching by CRC32 + table = page_soup.find("table", class_="torrent-list") + if table: + rows = table.find_all("tr") + for row in rows: + if _shutdown_requested: + break + title_link, magnet_link = _extract_links_from_row(row) + if (title_link and magnet_link and + isinstance(magnet_link, str) and + magnet_link.startswith(MAGNET_LINK_PREFIX) and + hasattr(title_link, 'text')): + filename_text = title_link.text + # Extract CRC32 from title + matches = CRC32_REGEX.findall(filename_text) + if matches: + crc32 = matches[-1].upper() + if crc32 in crc32_set: + crc32_to_magnet[crc32] = magnet_link + found_count += 1 + + page += 1 + if page <= total_pages: + time.sleep(REQUEST_DELAY_SECONDS) + + return crc32_to_magnet + + def get_metadata(conn, key): @@ -1322,9 +1406,38 @@ def _print_comparison_results(nyaa_crc32s_normalized, local_crc32s_normalized, def _calculate_and_find_missing(folder, conn, args, last_run): """Calculate local CRC32s and find missing episodes.""" - crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = ( - fetch_crc32_links(args.url) - ) + # Check if episodes_index was recently updated (within last 2 minutes) + # If so, use database instead of fetching again to avoid double fetch + conn_episodes = init_episodes_db() + last_update_str = get_episodes_metadata(conn_episodes, "episodes_db_last_update") + conn_episodes.close() + + use_database = False + if last_update_str: + try: + last_update = datetime.strptime(last_update_str, "%Y-%m-%d %H:%M:%S") + time_diff = datetime.now() - last_update + # Use database if updated within last 2 minutes (likely from --episodes_update) + if time_diff.total_seconds() < 120: + use_database = True + except (ValueError, TypeError): + # If parsing fails, fall through to normal fetch + pass + + if use_database: + # Use database to avoid double fetch + print("Using recently updated episodes index database (avoiding duplicate fetch)...") + crc32_to_link, crc32_to_text = load_1080p_episodes_from_index() + print(f"Loaded {len(crc32_to_link)} 1080p episodes from database.") + print("Fetching magnet links from search results...") + crc32_to_magnet = fetch_magnet_links_for_episodes_from_search(args.url, crc32_to_link) + print(f"Fetched {len(crc32_to_magnet)} magnet links.") + last_checked_page = 0 # Not tracking pages when using database + else: + # Normal fetch from Nyaa + crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = ( + fetch_crc32_links(args.url) + ) print(f"Found {len(crc32_to_link)} episodes from Nyaa.") diff --git a/coverage.xml b/coverage.xml index 23a37e3..e61b6dd 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -258,699 +258,773 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - + + - - + - + - - - - - - - + + + + + + - - - - - - - + + + + + + - - - + + + + + - - - + + - + + - - + + - - - - - + - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + + + + - - - - + - - - - - + + + + - + + + + + + + - - - - - - - - - - - - + + + + - + - - - - + + + + + - - - + - - - - + + + + + + + - - - - - - - + + + + + + - - - + + + - - + - + + + + + - + + - - - - + + + - - - - + + + + - - + + - - - + + + + + + - - - - - - - - - + + + + + + + + + + - - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - + + + + - - + + + + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + + + + + + + - - - + - + - - - - - + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - + + - - - - - + + - - - - - - - - - - - - + + + + + + + + + - - + + + - + + + - + + + + + + - - - + + + + + + - - - + + + - - - + + - - - - - - + + + + + + + + - - - - - - - - - - + + + + + - + + + + - - - - - - - - + + + - - + - - - - - - - - + + + - - + - - + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - + - + - - - + + + + + + - - - + + + - - + + + - + + + + + - - - + + + - + + - - - + - - - - + + + + + + - - - + + + + - - + - - - - + - - - - - - + + + + + + + + + - + - - - + + - - - - + + + - + + - - - - - - - - + + + + + - - + - + + + - - - + + + - - - - + + + + + - + + + + + + + + + - + + + + - - - + + + + + + + + - - - - - + + + + - - - + + + + + + + + - - - - - + + + + - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/entrypoint.sh b/entrypoint.sh index 26b853b..ac7b167 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -44,19 +44,10 @@ if [ "$DB" = "true" ]; then fi # Run missing episodes report -# Skip only if doing a single operation (episodes_update only or db only) -# This matches non-Docker behavior where each command is separate -# Run report if: no flags (default), DOWNLOAD is set, or multiple operations -SHOULD_RUN_REPORT=true -if [ "$EPISODES_UPDATE" = "true" ] && [ "$DB" != "true" ] && [ "$DOWNLOAD" != "true" ]; then - # Only updating episodes, skip report (same as non-Docker) - SHOULD_RUN_REPORT=false -elif [ "$DB" = "true" ] && [ "$EPISODES_UPDATE" != "true" ] && [ "$DOWNLOAD" != "true" ]; then - # Only exporting DB, skip report (same as non-Docker) - SHOULD_RUN_REPORT=false -fi - -if [ "$SHOULD_RUN_REPORT" = "true" ]; then +# Always run unless ONLY exporting DB (same as non-Docker: main command always runs) +# When EPISODES_UPDATE=true, the Python code will use the database to avoid double fetch +# Skip report only if DB=true AND no other operations +if [ "$DB" != "true" ] || [ "$EPISODES_UPDATE" = "true" ] || [ "$DOWNLOAD" = "true" ]; then python /app/acepace.py \ --folder /media \ ${NYAA_URL:+--url "$NYAA_URL"} From 4a54ca3e009aa065bccb80c5faf13242d1df8713 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 27 Jan 2026 16:12:41 +0000 Subject: [PATCH 63/75] Enhance episode update logic with force update option, change Cursor instructions --- .cursor/rules/acepace-rules.md | 287 ++++++++ README.md | 8 + acepace.py | 269 +++++-- agent.md | 12 - coverage.xml | 1261 +++++++++++++++++--------------- spec.md | 548 -------------- tests/test_main_command.py | 4 +- 7 files changed, 1148 insertions(+), 1241 deletions(-) create mode 100644 .cursor/rules/acepace-rules.md delete mode 100644 agent.md delete mode 100644 spec.md diff --git a/.cursor/rules/acepace-rules.md b/.cursor/rules/acepace-rules.md new file mode 100644 index 0000000..9601f5c --- /dev/null +++ b/.cursor/rules/acepace-rules.md @@ -0,0 +1,287 @@ +# Ace-Pace Project Rules + +This file contains the development rules, guidelines, and technical reference for the Ace-Pace project. These rules should be followed by all AI agents working on this codebase. + +## Core Workflow Requirements + +When working on this project, you MUST: + +1. **Always update tests when making changes** + - Update existing tests if functionality changes + - Add new tests for new features + - **Always run tests and verify they pass** before completing work + - Fix any failing tests + - Run: `pytest` with coverage before completing work + +2. **Always check for linter and SonarQube problems** + - Run linter checks and fix any issues + - Check SonarQube for code quality issues + - Fix all identified problems + +3. **Update documentation when appropriate** + - Review README.md after significant changes + - Update function documentation when signatures change + - Keep technical documentation accurate + +## Git Workflow Rules + +**CRITICAL: Follow these git rules strictly:** + +- **ABSOLUTELY NEVER run destructive git operations** (e.g., `git reset --hard`, `rm`, `git checkout`/`git restore` to an older commit) unless the user gives an explicit, written instruction. Treat these commands as catastrophic; if you are even slightly unsure, stop and ask before touching them. + +- **Never use `git restore`** (or similar commands) to revert files you didn't author—coordinate with other agents instead so their in-progress work stays intact. + +- **Always double-check git status** before any commit + +- **Keep commits atomic**: commit only the files you touched and list each path explicitly + - For tracked files: `git commit -m "" -- path/to/file1 path/to/file2` + - For brand-new files: `git restore --staged :/ && git add "path/to/file1" "path/to/file2" && git commit -m "" -- path/to/file1 path/to/file2` + +- **Quote any git paths** containing brackets or parentheses (e.g., `src/app/[candidate]/**`) when staging or committing so the shell does not treat them as globs or subshells. + +- **When running `git rebase`**, avoid opening editors—export `GIT_EDITOR=:` and `GIT_SEQUENCE_EDITOR=:` (or pass `--no-edit`) so the default messages are used automatically. + +- **Never amend commits** unless you have explicit written approval in the task thread. + +- **Delete unused or obsolete files** when your changes make them irrelevant (refactors, feature removals, etc.), and revert files only when the change is yours or explicitly requested. + +- **Before attempting to delete a file** to resolve a local type/lint failure, stop and ask the user. Other agents are often editing adjacent files; deleting their work to silence an error is never acceptable without explicit approval. + +- **NEVER edit `.env`** or any environment variable files—only the user may change them. + +- **Coordinate with other agents** before removing their in-progress edits—don't revert or delete work you didn't author unless everyone agrees. + +- **Moving/renaming and restoring files** is allowed. + +## Code Quality Standards + +- Follow PEP 8 Python style guide +- Maintain cognitive complexity ≤ 15 per function +- Use descriptive variable names +- Add docstrings for functions +- Keep functions focused and single-purpose +- Use `_` prefix for private/internal helper functions +- Comprehensive test suite exists in `tests/` directory (100+ tests) +- Ensure all tests pass before completing work + +## Project Overview + +**Ace-Pace** is a Python-based tool designed to help users manage and organize their One-Pace anime library. It automates: +- Identifying which One-Pace episodes are already in the user's local library +- Detecting missing episodes +- Automatically downloading missing episodes via BitTorrent clients +- Renaming local files to match official One-Pace naming conventions +- Maintaining a database of episode metadata and file checksums + +## Core Functionality + +### Episode Discovery and Indexing +- Scrapes Nyaa.si torrent tracker for One-Pace episodes +- Extracts CRC32 checksums from episode filenames or torrent file lists +- **Quality Filtering**: Only extracts episodes with 1080p quality + - Episodes without quality markers are excluded + - Episodes with quality other than 1080p are excluded + - Quality filtering is applied in both `fetch_episodes_metadata()` and `fetch_crc32_links()` + - Filtering is case-insensitive (accepts 1080P, etc.) +- **URL Parameter Support**: Both `fetch_episodes_metadata()` and `update_episodes_index_db()` accept a `base_url` parameter + - Quality filtering still applies regardless of URL parameters +- Builds and maintains an episodes index database (`episodes_index.db`) +- Supports both single-file and multi-file torrent structures +- Handles pagination to fetch all available episodes + +### Local Library Management +- Scans local directories recursively for video files (`.mkv`, `.mp4`, `.avi`) +- Calculates CRC32 checksums for local video files +- **Path Normalization**: All file paths are normalized before storage and lookup + - Uses `normalize_file_path()` to resolve symlinks and convert to absolute paths + - Ensures consistent path representation across different OS and environments + - Prevents cache misses when same file is accessed via different path representations + - **CRITICAL**: Always use `normalize_file_path()` before storing/querying file paths in database +- Caches CRC32 values in `crc32_files.db` to avoid recalculating +- Tracks file paths and their corresponding checksums (using normalized paths) + +### Missing Episode Detection +- Fetches episode list from Nyaa.si using the provided URL (default: One-Pace 1080p search) +- **Quality Filtering**: `fetch_crc32_links()` applies quality filtering via `_process_crc32_row()` + - Only accepts episodes with 1080p quality + - Requires "[One Pace]" marker in filename +- Compares local CRC32 checksums against fetched episodes (using normalized paths) +- Generates a CSV report (`Ace-Pace_Missing.csv`) listing missing episodes +- Uses `fetch_crc32_links()` for real-time fetching, not the cached episodes index + +### Automated Downloading +- Integrates with BitTorrent clients (Transmission, qBittorrent) +- Adds missing episodes to client via magnet links +- Supports custom download folders, tags, and categories +- Prevents duplicate torrent additions by checking existing torrents + +### File Renaming +- Matches local files to episodes index by CRC32 +- Renames files to match official One-Pace naming conventions +- Sanitizes filenames to remove problematic characters +- Updates database with new file paths after renaming (using normalized paths) + +## Technical Architecture + +### Databases + +#### `crc32_files.db` +- **Table: `crc32_cache`** + - `file_path` (TEXT, PRIMARY KEY): Normalized absolute path to local video file + - `crc32` (TEXT, UNIQUE): CRC32 checksum of the file + - **Note**: File paths are normalized using `normalize_file_path()` before storage +- **Table: `metadata`** + - `key` (TEXT, PRIMARY KEY): Metadata key + - `value` (TEXT): Metadata value + - Stores: `last_folder`, `last_run`, `last_checked_page`, `last_db_export`, `last_missing_export` + +#### `episodes_index.db` +- **Table: `episodes_index`** + - `crc32` (TEXT, PRIMARY KEY): CRC32 checksum from episode + - `title` (TEXT): Episode title/filename + - `page_link` (TEXT): URL to Nyaa.si torrent page +- **Table: `metadata`** + - `key` (TEXT, PRIMARY KEY): Metadata key + - `value` (TEXT): Metadata value + - Stores: `episodes_db_last_update` + +### Key Algorithms + +#### CRC32 Calculation +- Reads video files in 8KB chunks +- Uses Python's `zlib.crc32()` for incremental calculation +- Formats result as uppercase 8-character hexadecimal string +- Caches results to avoid redundant calculations +- Uses normalized file paths for cache lookups to ensure consistency + +#### CRC32 Extraction from Filenames +- Uses regex pattern: `\[([A-Fa-f0-9]{8})\]` +- Extracts CRC32 from square brackets in filenames +- Takes the last match if multiple CRC32s are present +- Validates that filename contains "[One Pace]" marker + +## Key Public API Functions + +- `main()`: Entry point for the application +- `init_db(suppress_messages=False)`: Initializes the local CRC32 cache database +- `init_episodes_db()`: Initializes the episodes index database +- `get_config_dir()`: Gets config directory path based on Docker mode (`/config` in Docker, `.` locally) +- `get_config_path(filename)`: Gets full path to a config file in the appropriate config directory +- `normalize_file_path(file_path)`: Normalizes file path for consistent storage and lookup + - **CRITICAL**: Always use this before storing/querying file paths in database +- `get_metadata(conn, key)`: Retrieves metadata value from database +- `set_metadata(conn, key, value)`: Stores metadata value in database +- `get_episodes_metadata(conn, key)`: Retrieves episodes database metadata +- `set_episodes_metadata(conn, key, value)`: Stores episodes database metadata +- `fetch_episodes_metadata(base_url=None)`: Fetches episodes from Nyaa.si + - `base_url`: Optional Nyaa.si search URL (defaults to One-Pace search without quality filter) + - Quality filtering (1080p only) is always applied regardless of URL +- `update_episodes_index_db(base_url=None, force_update=False)`: Updates the episodes index database + - `base_url`: Optional Nyaa.si search URL (passed to `fetch_episodes_metadata()`) + - `force_update`: If True, force update even if recently updated. If False, skip if updated within last 10 minutes +- `fetch_crc32_links(base_url)`: Fetches CRC32 links from a Nyaa.si URL + - Applies quality filtering (1080p only) via `_process_crc32_row()` +- `fetch_title_by_crc32(crc32)`: Searches for a title by CRC32 +- `calculate_local_crc32(folder, conn)`: Calculates CRC32 for local files + - Uses normalized paths for database storage and lookup +- `rename_local_files(conn)`: Renames local files based on episodes index + - Uses normalized paths when updating database after renaming +- `export_db_to_csv(conn)`: Exports database to CSV +- `load_crc32_to_title_from_index()`: Loads CRC32-to-title mapping + +## Private Helper Functions + +Helper functions are prefixed with `_` to indicate they are internal implementation details. + +**Extraction functions** (`_extract_*`): Extract data from HTML/structures +- `_extract_title_link_from_row(row)`: Extracts title link from table row +- `_extract_filenames_from_folder_structure(filelist_div)`: Extracts filenames from folder structure +- `_extract_filenames_from_torrent_page(torrent_soup)`: Extracts filenames from torrent page +- `_extract_matching_titles_from_rows(rows, crc32)`: Extracts titles matching CRC32 + +**Processing functions** (`_process_*`): Process data structures +- `_process_fname_entry(fname_text, ...)`: Processes filename entry to extract CRC32 +- `_process_torrent_page(page_link, ...)`: Processes torrent page to extract CRC32 +- `_process_episode_row(row, ...)`: Processes table row to extract episode info +- `_process_crc32_row(row, ...)`: Processes row to extract CRC32 for missing episodes + +**Validation functions** (`_is_*`, `_validate_*`): Validate inputs/data +- `_is_valid_quality(fname_text)`: Checks if filename has valid quality (1080p only) +- `_validate_url(url)`: Validates URL points to valid Nyaa domain + +**Command handlers** (`_handle_*`): Handle specific command-line operations +- `_handle_download_command(args)`: Handles the `--download` command +- `_handle_rename_command(conn, base_url=None)`: Handles the `--rename` command + - `base_url`: Optional URL parameter passed to `update_episodes_index_db()` if update is needed +- `_handle_main_commands(args, conn, folder)`: Routes and handles main commands + +## Docker Support + +- **Docker Mode**: Detected via `RUN_DOCKER` environment variable +- **Non-Interactive Operation**: In Docker mode, skips user prompts and uses defaults +- **Default Folder**: Uses `/media` as default folder in Docker mode +- **Config Directory**: Uses `/config` directory in Docker mode for databases and CSV files + - Local mode uses current directory (`.`) + - Config directory is automatically created if it doesn't exist +- **Message Suppression**: In Docker mode, suppresses informational messages for automated commands +- **Environment Variables**: Supports configuration via Docker environment variables + - `DOWNLOAD`: Set to "true" to download missing episodes after generating report + - `TORRENT_CLIENT`: BitTorrent client type (default: transmission) + - `TORRENT_HOST`: Client host address (default: localhost) + - `TORRENT_PORT`: Client port number (default: 9091 for transmission, 8080 for qBittorrent) + - `TORRENT_USER`: Client authentication username (optional) + - `TORRENT_PASSWORD`: Client authentication password (optional) + - `NYAA_URL`: Custom Nyaa.si search URL (optional, defaults to 1080p search) + - `EPISODES_UPDATE`: Set to "true" to update episodes index on container start + - `DB`: Set to "true" to export database on container start + - `RUN_DOCKER`: Flag to enable Docker mode (non-interactive) + - `DEBUG`: Enable debug output (set to `true`, `1`, `yes`, or `on`) + +## Critical Implementation Notes + +1. **CRC32 is the primary identifier** - All episode matching relies on CRC32 checksums +2. **Always use `normalize_file_path()`** before storing/querying file paths in database + - Ensures consistent path representation across different OS and environments + - Critical for consistent behavior between Python and Docker versions + - Resolves symlinks and converts to absolute paths +3. **Quality filtering (1080p only)** is always applied regardless of URL parameters + - Applied in both `fetch_episodes_metadata()` and `fetch_crc32_links()` via `_is_valid_quality()` + - Requires "[One Pace]" marker in filename +4. **Config directory handling**: Use `get_config_dir()` and `get_config_path()` for consistent file location handling + - Returns `/config` in Docker mode, `.` in local mode + - Automatically creates directory if it doesn't exist +5. **URL parameter consistency**: Both `fetch_episodes_metadata()` and `fetch_crc32_links()` accept URL parameters + - Always pass `args.url` to ensure consistent URL usage across functions +6. **Docker download logic**: Use `DOWNLOAD=true` environment variable to enable downloads + - Missing episodes CSV is always generated/updated before download (if download enabled) +7. **Default connection values**: In Docker mode, use defaults if not specified via environment variables + - Client: transmission + - Host: localhost + - Port: 9091 (transmission) or 8080 (qbittorrent) +8. **Debug mode**: Use `DEBUG` environment variable to control troubleshooting output + - Defaults to `false` (no debug output) + - Set to `true`, `1`, `yes`, or `on` to enable + - All debug output uses `debug_print()` function which checks `DEBUG_MODE` flag + +## BitTorrent Client Abstraction + +- Abstract base class `Client` (in `clients.py`) defines interface using `abc.ABC` +- Concrete implementations: `QBittorrentClient`, `TransmissionClient` +- Factory function `get_client(client_name, host, port, username, password)` instantiates appropriate client +- Methods: `add_torrents(magnets, download_folder, tags, category)` +- **qBittorrentClient**: Uses `qbittorrentapi` library, checks for existing torrents by info hash, supports tags and categories +- **TransmissionClient**: Uses Transmission RPC API via HTTP requests, handles session ID management, does not support tags/categories + +## Error Handling + +- Network errors: HTTP request failures are caught and logged, continues processing remaining items +- File system errors: Checks for file existence before operations, handles permission errors gracefully +- Database errors: Uses `INSERT OR REPLACE` for idempotent operations, handles connection failures +- Rate limiting: Uses `time.sleep(0.2)` between requests + +## Testing + +- Comprehensive test suite exists in `tests/` directory (100+ tests) +- Run `pytest` with coverage before completing work +- Ensure all tests pass +- Test coverage includes: clients, CRC32, database, debug mode, episodes, file operations, main commands, missing detection, path normalization diff --git a/README.md b/README.md index 4e77302..56e4734 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,12 @@ python acepace.py [-h] [--url URL] [--folder FOLDER] [--db] [--client {transmiss - `--db` (standalone flag) Create a CSV file with the existing local file paths and CRC32 checksums. Useful to check what's detected and debugging. +- `--rename` (standalone flag) + Rename local files based on matching titles from One-Pace episodes index. Requires `--folder` to be specified. Optionally use `--url` to specify a custom Nyaa.si search URL. + +- `--episodes_update` (standalone flag) + Update the episodes metadata database from Nyaa.si. Optionally use `--url` to specify a custom Nyaa.si search URL. This command forces an update even if episodes were recently updated (within the last 10 minutes). + ### 📥 Download commands - `--client ` @@ -235,6 +241,8 @@ python acepace.py --folder "/volume42/media/One Piece/" python acepace.py --client transmission --download python acepace.py --client qbittorrent --download --host 192.168.1.100 --port 8080 --username myuser --password mypassword --download-folder /downloads/onepace --tag onepace --tag 'one pace' --category 'anime' python acepace.py --db +python acepace.py --folder "/volume42/media/One Piece/" --rename +python acepace.py --episodes_update --url https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc ``` ## 📜 Workflow Overview diff --git a/acepace.py b/acepace.py index b5cb259..ded0309 100644 --- a/acepace.py +++ b/acepace.py @@ -399,12 +399,33 @@ def fetch_episodes_metadata(base_url=None): return episodes -def update_episodes_index_db(base_url=None): +def update_episodes_index_db(base_url=None, force_update=False): """Update episodes index database from Nyaa. Args: base_url: Base URL for Nyaa search. If None, uses default. + force_update: If True, force update even if recently updated. If False, skip if updated within last 10 minutes. """ - debug_print(f"DEBUG: Starting update_episodes_index_db with URL: {base_url}") + debug_print(f"DEBUG: Starting update_episodes_index_db with URL: {base_url}, force_update: {force_update}") + + # Check if episodes were recently updated (within last 10 minutes) + if not force_update: + conn_check = init_episodes_db() + last_update_str = get_episodes_metadata(conn_check, "episodes_db_last_update") + conn_check.close() + + if last_update_str: + try: + last_update = datetime.strptime(last_update_str, "%Y-%m-%d %H:%M:%S") + time_diff = datetime.now() - last_update + # Skip update if updated within last 10 minutes to avoid unnecessary double updates + if time_diff.total_seconds() < 600: # 10 minutes = 600 seconds + print(f"Episodes were recently updated ({last_update_str}), skipping update to avoid duplicate fetch.") + print("Set EPISODES_UPDATE=true or use --episodes_update to force update.") + return + except (ValueError, TypeError): + # If parsing fails, proceed with update + pass + conn = init_episodes_db() episodes = fetch_episodes_metadata(base_url) debug_print(f"DEBUG: Fetched {len(episodes)} episodes from Nyaa") @@ -457,6 +478,66 @@ def load_1080p_episodes_from_index(): return crc32_to_link, crc32_to_text +def _extract_magnet_link_from_row(row, crc32_set): + """Extract magnet link from a table row if it matches a CRC32 in the set. + Args: + row: BeautifulSoup table row element + crc32_set: Set of CRC32 values to match against + Returns: Tuple of (crc32, magnet_link) if found, (None, None) otherwise""" + title_link, magnet_link = _extract_links_from_row(row) + if not (title_link and magnet_link and + isinstance(magnet_link, str) and + magnet_link.startswith(MAGNET_LINK_PREFIX) and + hasattr(title_link, 'text')): + return None, None + + filename_text = title_link.text + matches = CRC32_REGEX.findall(filename_text) + if not matches: + return None, None + + crc32 = matches[-1].upper() + if crc32 in crc32_set: + return crc32, magnet_link + return None, None + + +def _process_magnet_links_page(page_soup, crc32_set, crc32_to_magnet): + """Process a single page to extract magnet links matching CRC32s in the set. + Args: + page_soup: BeautifulSoup object for the page + crc32_set: Set of CRC32 values to match against + crc32_to_magnet: Dictionary to update with found magnet links + Returns: Number of new magnet links found on this page""" + found_count = 0 + table = page_soup.find("table", class_="torrent-list") + if not table: + return found_count + + rows = table.find_all("tr") + for row in rows: + if _shutdown_requested: + break + crc32, magnet_link = _extract_magnet_link_from_row(row, crc32_set) + if crc32 and magnet_link: + crc32_to_magnet[crc32] = magnet_link + found_count += 1 + + return found_count + + +def _get_page_soup_for_magnet_links(base_url, page, first_page_soup): + """Get BeautifulSoup object for a specific page when fetching magnet links. + Args: + base_url: Nyaa search URL + page: Page number (1-indexed) + first_page_soup: BeautifulSoup object for page 1 (already fetched) + Returns: Tuple of (soup, success)""" + if page == 1: + return first_page_soup, True + return _fetch_crc32_page(base_url, page) + + def fetch_magnet_links_for_episodes_from_search(base_url, crc32_to_link): """Fetch magnet links from Nyaa search results for episodes already in crc32_to_link. This is more efficient than fetching all episodes again. @@ -485,37 +566,12 @@ def fetch_magnet_links_for_episodes_from_search(base_url, crc32_to_link): if _shutdown_requested: break - if page == 1: - page_soup = soup - else: - page_soup_result, success = _fetch_crc32_page(base_url, page) - if not success or page_soup_result is None: - break - page_soup = page_soup_result - - if page_soup is None: + page_soup, success = _get_page_soup_for_magnet_links(base_url, page, soup) + if not success or page_soup is None: break - # Extract magnet links from rows, matching by CRC32 - table = page_soup.find("table", class_="torrent-list") - if table: - rows = table.find_all("tr") - for row in rows: - if _shutdown_requested: - break - title_link, magnet_link = _extract_links_from_row(row) - if (title_link and magnet_link and - isinstance(magnet_link, str) and - magnet_link.startswith(MAGNET_LINK_PREFIX) and - hasattr(title_link, 'text')): - filename_text = title_link.text - # Extract CRC32 from title - matches = CRC32_REGEX.findall(filename_text) - if matches: - crc32 = matches[-1].upper() - if crc32 in crc32_set: - crc32_to_magnet[crc32] = magnet_link - found_count += 1 + page_found = _process_magnet_links_page(page_soup, crc32_set, crc32_to_magnet) + found_count += page_found page += 1 if page <= total_pages: @@ -1404,55 +1460,77 @@ def _print_comparison_results(nyaa_crc32s_normalized, local_crc32s_normalized, debug_print("=== END DEBUG: TROUBLESHOOTING INFO ===\n") -def _calculate_and_find_missing(folder, conn, args, last_run): - """Calculate local CRC32s and find missing episodes.""" - # Check if episodes_index was recently updated (within last 2 minutes) - # If so, use database instead of fetching again to avoid double fetch - conn_episodes = init_episodes_db() - last_update_str = get_episodes_metadata(conn_episodes, "episodes_db_last_update") - conn_episodes.close() +def _should_force_episodes_update(last_update_str): + """Determine if episodes should be force updated based on last update time. + Args: + last_update_str: String timestamp of last update, or None + Returns: True if should force update, False if recently updated (within 10 minutes)""" + if not last_update_str: + return True - use_database = False + try: + last_update = datetime.strptime(last_update_str, "%Y-%m-%d %H:%M:%S") + time_diff = datetime.now() - last_update + # If updated within last 10 minutes, skip to avoid double update + if time_diff.total_seconds() < 600: # 10 minutes = 600 seconds + print("EPISODES_UPDATE=true: Episodes were recently updated, using existing database...") + return False + except (ValueError, TypeError): + # If parsing fails, proceed with update + pass + return True + + +def _handle_episodes_update_decision(episodes_update_env, last_update_str, base_url): + """Handle the decision to update episodes based on EPISODES_UPDATE environment variable. + Args: + episodes_update_env: True if EPISODES_UPDATE environment variable is set + last_update_str: String timestamp of last update, or None + base_url: Base URL for Nyaa search + Returns: True if database should be used, False if should fetch from Nyaa""" + if episodes_update_env: + # EPISODES_UPDATE=true: Force update episodes even if recently updated + if _should_force_episodes_update(last_update_str): + print("EPISODES_UPDATE=true: Forcing episodes metadata update...") + update_episodes_index_db(base_url, force_update=True) + # After update (forced or skipped), always use database + return True + + # EPISODES_UPDATE=false or not set: Use database only, never fetch from Nyaa if last_update_str: - try: - last_update = datetime.strptime(last_update_str, "%Y-%m-%d %H:%M:%S") - time_diff = datetime.now() - last_update - # Use database if updated within last 2 minutes (likely from --episodes_update) - if time_diff.total_seconds() < 120: - use_database = True - except (ValueError, TypeError): - # If parsing fails, fall through to normal fetch - pass + return True - if use_database: - # Use database to avoid double fetch - print("Using recently updated episodes index database (avoiding duplicate fetch)...") - crc32_to_link, crc32_to_text = load_1080p_episodes_from_index() - print(f"Loaded {len(crc32_to_link)} 1080p episodes from database.") - print("Fetching magnet links from search results...") - crc32_to_magnet = fetch_magnet_links_for_episodes_from_search(args.url, crc32_to_link) - print(f"Fetched {len(crc32_to_magnet)} magnet links.") - last_checked_page = 0 # Not tracking pages when using database - else: - # Normal fetch from Nyaa - crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = ( - fetch_crc32_links(args.url) - ) + # Database doesn't exist, need to fetch (but this shouldn't happen in normal operation) + print("Episodes database not found. Fetching from Nyaa...") + return False - print(f"Found {len(crc32_to_link)} episodes from Nyaa.") - if last_run: - print("Calculating new local CRC32 hashes...") +def _load_episodes_from_database(episodes_update_env, base_url): + """Load episodes from database and fetch magnet links. + Args: + episodes_update_env: True if EPISODES_UPDATE environment variable is set + base_url: Base URL for Nyaa search + Returns: Tuple of (crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page)""" + if episodes_update_env: + print("Using episodes index database (EPISODES_UPDATE=true, using updated database)...") else: - print( - "Calculating local CRC32 hashes - this will take a while on first run!..." - ) - - local_crc32s = calculate_local_crc32(folder, conn) - print(f"Found {len(local_crc32s)} local CRC32 hashes.") + print("Using episodes index database (EPISODES_UPDATE=false, checking database only)...") + crc32_to_link, crc32_to_text = load_1080p_episodes_from_index() + print(f"Loaded {len(crc32_to_link)} 1080p episodes from database.") + print("Fetching magnet links from search results...") + crc32_to_magnet = fetch_magnet_links_for_episodes_from_search(base_url, crc32_to_link) + print(f"Fetched {len(crc32_to_magnet)} magnet links.") + return crc32_to_link, crc32_to_text, crc32_to_magnet, 0 + + +def _calculate_missing_episodes(crc32_to_link, local_crc32s): + """Calculate missing episodes by comparing Nyaa episodes with local CRC32s. + Args: + crc32_to_link: Dictionary mapping CRC32 to page_link + local_crc32s: Set of local CRC32 checksums + Returns: List of missing CRC32s""" debug_print("DEBUG: Starting missing episode detection") - debug_print(f"DEBUG: Folder scanned: {folder}") debug_print(f"DEBUG: Episodes from Nyaa: {len(crc32_to_link)}") debug_print(f"DEBUG: Local CRC32s: {len(local_crc32s)}") @@ -1482,6 +1560,50 @@ def _calculate_and_find_missing(folder, conn, args, last_run): nyaa_crc32s_normalized, local_crc32s_normalized, crc32_to_link, local_crc32s, missing, list(missing_normalized_set) ) + + return missing + + +def _calculate_and_find_missing(folder, conn, args, last_run): + """Calculate local CRC32s and find missing episodes.""" + # Check EPISODES_UPDATE environment variable + episodes_update_env = os.getenv("EPISODES_UPDATE", "").lower() in ("true", "1", "yes", "on") + + # Check if episodes_index exists and has data + conn_episodes = init_episodes_db() + last_update_str = get_episodes_metadata(conn_episodes, "episodes_db_last_update") + + # Determine whether to use database or fetch from Nyaa + use_database = _handle_episodes_update_decision(episodes_update_env, last_update_str, args.url) + conn_episodes.close() + + # Load episodes (from database or fetch from Nyaa) + if use_database: + crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = _load_episodes_from_database(episodes_update_env, args.url) + else: + # Normal fetch from Nyaa (only when database doesn't exist and EPISODES_UPDATE=false) + print("Fetching episodes metadata from Nyaa...") + crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = ( + fetch_crc32_links(args.url) + ) + + print(f"Found {len(crc32_to_link)} episodes from Nyaa.") + + # Calculate local CRC32s + if last_run: + print("Calculating new local CRC32 hashes...") + else: + print( + "Calculating local CRC32 hashes - this will take a while on first run!..." + ) + + local_crc32s = calculate_local_crc32(folder, conn) + print(f"Found {len(local_crc32s)} local CRC32 hashes.") + + debug_print(f"DEBUG: Folder scanned: {folder}") + + # Calculate missing episodes + missing = _calculate_missing_episodes(crc32_to_link, local_crc32s) print( f"\nSummary: {len(missing)} missing episodes out of {len(crc32_to_link)} total found on Nyaa.\n" @@ -1752,7 +1874,8 @@ def main(): _show_episodes_metadata_status() if args.episodes_update: - update_episodes_index_db(args.url) + # When --episodes_update is used directly, force update (same behavior as EPISODES_UPDATE=true) + update_episodes_index_db(args.url, force_update=True) sys.exit(0) # Suppress messages when exporting DB (since it's automated) diff --git a/agent.md b/agent.md deleted file mode 100644 index 3b12a6a..0000000 --- a/agent.md +++ /dev/null @@ -1,12 +0,0 @@ -- Delete unused or obsolete files when your changes make them irrelevant (refactors, feature removals, etc.), and revert files only when the change is yours or explicitly requested. If a git operation leaves you unsure about other agents' in-flight work, stop and coordinate instead of deleting. -- **Before attempting to delete a file to resolve a local type/lint failure, stop and ask the user.** Other agents are often editing adjacent files; deleting their work to silence an error is never acceptable without explicit approval. -- NEVER edit `.env` or any environment variable files—only the user may change them. -- Coordinate with other agents before removing their in-progress edits—don't revert or delete work you didn't author unless everyone agrees. -- Moving/renaming and restoring files is allowed. -- ABSOLUTELY NEVER run destructive git operations (e.g., `git reset --hard`, `rm`, `git checkout`/`git restore` to an older commit) unless the user gives an explicit, written instruction in this conversation. Treat these commands as catastrophic; if you are even slightly unsure, stop and ask before touching them. *(When working within Cursor or Codex Web, these git limitations do not apply; use the tooling's capabilities as needed.)* -- Never use `git restore` (or similar commands) to revert files you didn't author—coordinate with other agents instead so their in-progress work stays intact. -- Always double-check git status before any commit -- Keep commits atomic: commit only the files you touched and list each path explicitly. For tracked files run `git commit -m "" -- path/to/file1 path/to/file2`. For brand-new files, use the one-liner `git restore --staged :/ && git add "path/to/file1" "path/to/file2" && git commit -m "" -- path/to/file1 path/to/file2`. -- Quote any git paths containing brackets or parentheses (e.g., `src/app/[candidate]/**`) when staging or committing so the shell does not treat them as globs or subshells. -- When running `git rebase`, avoid opening editors—export `GIT_EDITOR=:` and `GIT_SEQUENCE_EDITOR=:` (or pass `--no-edit`) so the default messages are used automatically. -- Never amend commits unless you have explicit written approval in the task thread. \ No newline at end of file diff --git a/coverage.xml b/coverage.xml index e61b6dd..3e92be7 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -232,799 +232,848 @@ - - - - + - - - + + + - - - - - - - - + + + + + + + + + + - - - - + + + + + - - - - - - - - - - - - + + + + + + + + + + + + + + - + + - - + + + + - - - - - - - - - - - + + + + - + + - - + + + - - - - - - + + - + - - + + - - + + + + - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + + + - - + + + + + + - - - + + + - - - - + + + + + + + + - - - - - - - - - - + + + + + - - - - - - - - + + + + + + + + + - - + - - + + - - - - - - - + + + + + + + - - - - + + + + + - + + - - - - - - + - - - + + + + + + - - - - - - - - - - - - + + + + + + + + - - - + + - + + + - - - - - + + + - + + + + - - + + - + - + - - - - - - - - - + + + + + + + + + - - + + + + + + - - - - + + - - - - - + + + + + + - - - - + - + + + + + + - - - - - - - - - - + + + + + + + + - + - - - - + + + + - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + + - - - - + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - + + + + - - + + + - + + - - + - - + + - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - + + + + + + - - - - - - - + + + - - - - - - + + + - - - - + - + - - - - - - - - - - - - + + + + + + + + + + + + + + - - + - - - - - + + + + + + + - - - - + + + + + - + + + + + + + - - + + + - - - - - + + + + - - - - - - - - - - + + + + + + + + + + - - - - - + + + - - + - - - - - - + - - - - - + - - - - - - - - - - + + + + + + + + + - - - - - - - + + + + + + + - + + + - - + + - + - - + + - + + + - + - + + + - + - - + + + + - - - + + + - - - - - - - - - + + + + + + + + + + - - - - + + + - + - - - - + + + + + - - + + + - - - - + + - - - + + - - + + - + - + - - + + - - + + + - - - - - + + - - - - + + + - - + + + - + + + + + - - - - - - - + + + - - - - + + + + - + - + - - + + + + + - - + + + + + - - + + + + + + + + + - - - + + + + - - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - + + + + + - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spec.md b/spec.md deleted file mode 100644 index a31df2d..0000000 --- a/spec.md +++ /dev/null @@ -1,548 +0,0 @@ -# Ace-Pace Project Specification - -## Project Overview - -**Ace-Pace** is a Python-based tool designed to help users manage and organize their One-Pace anime library. One-Pace is a fan project that edits the One Piece anime to remove filler content and improve pacing. Ace-Pace automates the process of: - -- Identifying which One-Pace episodes are already in the user's local library -- Detecting missing episodes -- Automatically downloading missing episodes via BitTorrent clients -- Renaming local files to match official One-Pace naming conventions -- Maintaining a database of episode metadata and file checksums - -## Core Functionality - -### 1. Episode Discovery and Indexing -- Scrapes Nyaa.si torrent tracker for One-Pace episodes -- Extracts CRC32 checksums from episode filenames or torrent file lists -- **Quality Filtering**: Only extracts episodes with 1080p quality - - Episodes without quality markers are excluded - - Episodes with quality other than 1080p (720p, 480p, 360p, 1440p, 2160p/4K, etc.) are excluded - - Quality filtering is applied in both `fetch_episodes_metadata()` and `fetch_crc32_links()` - - Filtering is case-insensitive (accepts 1080P, etc.) -- **URL Parameter Support**: Both `fetch_episodes_metadata()` and `update_episodes_index_db()` accept a `base_url` parameter - - Allows consistent URL usage across all episode fetching functions - - Default URL includes 1080p filter, but can be overridden - - Quality filtering still applies regardless of URL parameters -- Builds and maintains an episodes index database (`episodes_index.db`) -- Supports both single-file and multi-file torrent structures -- Handles pagination to fetch all available episodes - -### 2. Local Library Management -- Scans local directories recursively for video files (`.mkv`, `.mp4`, `.avi`) -- Calculates CRC32 checksums for local video files -- **Path Normalization**: All file paths are normalized before storage and lookup - - Uses `normalize_file_path()` to resolve symlinks and convert to absolute paths - - Ensures consistent path representation across different OS and environments - - Prevents cache misses when same file is accessed via different path representations - - Critical for consistent behavior between Python and Docker versions -- Caches CRC32 values in `crc32_files.db` to avoid recalculating -- Tracks file paths and their corresponding checksums (using normalized paths) - -### 3. Missing Episode Detection -- Fetches episode list from Nyaa.si using the provided URL (default: One-Pace 1080p search) -- **Quality Filtering**: `fetch_crc32_links()` applies quality filtering via `_process_crc32_row()` - - Only accepts episodes with 1080p quality - - Requires "[One Pace]" marker in filename - - Ensures consistent filtering regardless of URL parameters -- Compares local CRC32 checksums against fetched episodes (using normalized paths) -- Generates a CSV report (`Ace-Pace_Missing.csv`) listing missing episodes -- Includes title, page link, and magnet link for each missing episode -- Displays missing episode count prominently in output -- Tracks new missing episodes since last export by comparing with previous CSV -- Note: Uses `fetch_crc32_links()` for real-time fetching, not the cached episodes index - -### 4. Automated Downloading -- Integrates with BitTorrent clients (Transmission, qBittorrent) -- Adds missing episodes to client via magnet links -- Supports custom download folders, tags, and categories -- Prevents duplicate torrent additions by checking existing torrents - -### 5. File Renaming -- Matches local files to episodes index by CRC32 -- Renames files to match official One-Pace naming conventions -- Sanitizes filenames to remove problematic characters -- Updates database with new file paths after renaming - -## Technical Architecture - -### Databases - -#### `crc32_files.db` -- **Table: `crc32_cache`** - - `file_path` (TEXT, PRIMARY KEY): Normalized absolute path to local video file - - `crc32` (TEXT, UNIQUE): CRC32 checksum of the file - - Note: File paths are normalized using `normalize_file_path()` before storage -- **Table: `metadata`** - - `key` (TEXT, PRIMARY KEY): Metadata key - - `value` (TEXT): Metadata value - - Stores: `last_folder`, `last_run`, `last_checked_page`, `last_db_export`, `last_missing_export` - -#### `episodes_index.db` -- **Table: `episodes_index`** - - `crc32` (TEXT, PRIMARY KEY): CRC32 checksum from episode - - `title` (TEXT): Episode title/filename - - `page_link` (TEXT): URL to Nyaa.si torrent page -- **Table: `metadata`** - - `key` (TEXT, PRIMARY KEY): Metadata key - - `value` (TEXT): Metadata value - - Stores: `episodes_db_last_update` - -### Key Algorithms - -#### CRC32 Calculation -- Reads video files in 8KB chunks -- Uses Python's `zlib.crc32()` for incremental calculation -- Formats result as uppercase 8-character hexadecimal string -- Caches results to avoid redundant calculations -- Uses normalized file paths for cache lookups to ensure consistency - -#### CRC32 Extraction from Filenames -- Uses regex pattern: `\[([A-Fa-f0-9]{8})\]` -- Extracts CRC32 from square brackets in filenames -- Takes the last match if multiple CRC32s are present -- Validates that filename contains "[One Pace]" marker - -#### Web Scraping -- Uses BeautifulSoup4 for HTML parsing -- Handles Nyaa.si pagination by detecting max page number -- Extracts torrent metadata from listing pages -- Falls back to individual torrent pages when CRC32 not in title -- Processes both folder-based and single-file torrent structures - -### File Structure - -``` -Ace-Pace/ -├── acepace.py # Main application entry point -├── clients.py # BitTorrent client abstraction layer -├── requirements.txt # Python dependencies -├── Dockerfile # Docker image definition -├── docker-compose.yml # Docker Compose configuration -├── entrypoint.sh # Docker entrypoint script -├── spec.md # This specification document -├── NAMING_CONVENTIONS.md # Function naming conventions documentation -├── pytest.ini # Pytest configuration -├── tests/ # Test suite -│ ├── conftest.py -│ ├── test_clients.py -│ ├── test_crc32.py -│ ├── test_database.py -│ ├── test_episodes.py -│ ├── test_file_operations.py -│ ├── test_missing_detection.py -│ ├── test_path_normalization.py -│ └── test_main_command.py -├── crc32_files.db # Local file checksum database (generated) -├── episodes_index.db # Episodes metadata database (generated) -├── Ace-Pace_Missing.csv # Missing episodes report (generated) -└── Ace-Pace_DB.csv # Database export (generated) -``` - -## Dependencies - -### Python Packages -- `requests`: HTTP requests for web scraping and API calls -- `beautifulsoup4`: HTML parsing for Nyaa.si scraping -- `qbittorrent-api`: qBittorrent client integration -- `pytest` (>=7.0.0): Testing framework -- `pytest-mock` (>=3.10.0): Mocking utilities for tests -- Standard library: `sqlite3`, `argparse`, `csv`, `datetime`, `os`, `re`, `zlib`, `getpass`, `time`, `abc` - -### External Services -- **Nyaa.si**: Torrent tracker for One-Pace episodes - - Base URL: `https://nyaa.si` - - Search endpoint: `/?f=0&c=0_0&q=one+pace&o=asc` - - Supports pagination via `&p=` -- **BitTorrent Clients**: - - Transmission (RPC API on port 9091 by default) - - qBittorrent (Web API on port 8080 by default) - -### Docker Support -- **Docker Mode**: Detected via `RUN_DOCKER` environment variable -- **Non-Interactive Operation**: In Docker mode, skips user prompts and uses defaults -- **Default Folder**: Uses `/media` as default folder in Docker mode -- **Config Directory**: Uses `/config` directory in Docker mode for databases and CSV files - - Local mode uses current directory (`.`) - - Config directory is automatically created if it doesn't exist -- **Message Suppression**: In Docker mode, suppresses informational messages for automated commands - - "Running in Docker mode" message only shown once for main command (not for `--db` or `--episodes_update`) - - Episodes metadata status only shown for main command (suppressed for `--db` and `--episodes_update`) - - Database "already exists" message suppressed when `--db` flag is used -- **Entrypoint Script**: `entrypoint.sh` orchestrates Docker execution - - Runs `--episodes_update` if `EPISODES_UPDATE=true` (with URL parameter if `NYAA_URL` is set) - - Runs `--db` if `DB=true` (exports database to CSV) - - **Always runs missing episodes report** (generates/updates `Ace-Pace_Missing.csv`) - - Runs `--download` if `DOWNLOAD=true` (downloads missing episodes after report generation) - - Always passes `--folder /media` to commands - - Passes `NYAA_URL` parameter when set -- **Environment Variables**: Supports configuration via Docker environment variables - - `DOWNLOAD`: Set to "true" to download missing episodes after generating report (default: not set/false) - - `TORRENT_CLIENT`: BitTorrent client type (default: transmission) - - Options: transmission, qbittorrent - - `TORRENT_HOST`: Client host address (default: localhost) - - `TORRENT_PORT`: Client port number (default: 9091 for transmission, 8080 for qbittorrent) - - `TORRENT_USER`: Client authentication username (default: empty, not required) - - `TORRENT_PASSWORD`: Client authentication password (default: empty, not required) - - `NYAA_URL`: Custom Nyaa.si search URL (optional, defaults to 1080p search) - - `EPISODES_UPDATE`: Set to "true" to update episodes index on container start (default: not set/false) - - `DB`: Set to "true" to export database on container start (default: not set/false) - - `RUN_DOCKER`: Flag to enable Docker mode (non-interactive) - - `DEBUG`: Enable debug output for troubleshooting (default: `false`) - - Set to `true`, `1`, `yes`, or `on` to enable detailed debug information - - When enabled, shows troubleshooting info, sample CRC32s, comparison details, and processing statistics - - Useful for diagnosing issues with missing episode detection or data processing - - Works in both Docker and local Python execution - -## Command-Line Interface - -### Main Arguments -- `--folder `: Local video library directory -- `--url `: Nyaa.si search URL (default: One-Pace 1080p search) -- `--db`: Export database to CSV - -### Download Arguments -- `--client {transmission,qbittorrent}`: BitTorrent client to use -- `--download`: Enable automatic downloading -- `--host `: Client host (default: localhost) -- `--port `: Client port -- `--username `: Client authentication username -- `--password `: Client authentication password -- `--download-folder `: Target download directory -- `--tag `: Tag(s) for qBittorrent (repeatable) -- `--category `: Category for qBittorrent - -### Utility Arguments -- `--rename`: Rename local files based on episodes index -- `--episodes_update`: Update episodes metadata database from Nyaa - -## Workflow - -### Standard Workflow -1. User runs script with `--folder` to scan local library -2. Script validates URL and shows episodes metadata status -3. Script calculates/retrieves CRC32 checksums for local files -4. Script fetches episode list from Nyaa.si using `fetch_crc32_links()` -5. Script compares local CRC32s against fetched episodes -6. Script generates `Ace-Pace_Missing.csv` with missing episodes -7. Script reports new missing episodes since last export -8. User optionally runs `--download --client ` to add missing episodes to BitTorrent client - -### Episodes Index Update Workflow -1. User runs `--episodes_update` (optionally with `--url` to specify search URL) -2. Script uses provided URL or defaults to One-Pace search (without quality filter in URL) -3. Script scrapes all pages of Nyaa.si search results -4. For each torrent, extracts CRC32 from title or file list -5. Applies quality filtering (1080p only) before storing -6. Stores CRC32, title, and page link in `episodes_index.db` -7. Updates metadata with last update timestamp -8. Note: Quality filtering is applied regardless of URL parameters - -### File Renaming Workflow -1. User runs `--rename` with `--folder` (optionally with `--url` to specify search URL) -2. Script checks episodes index update status -3. Script prompts to update episodes index if outdated (skipped in Docker mode) -4. If update is needed, uses provided URL or default for `update_episodes_index_db()` -5. Script loads CRC32-to-title mapping from episodes index -6. Script matches local files by CRC32 (using normalized paths) -7. Script generates rename plan and prompts for confirmation (auto-confirms in Docker mode) -8. Script renames files and updates database with normalized paths - -### Debug Mode -- **DEBUG Environment Variable**: Controls verbose troubleshooting output - - Default: `false` (no debug output) - - Set to `true`, `1`, `yes`, or `on` to enable - - When enabled, provides detailed information about: - - Episode fetching progress and page counts - - CRC32 normalization and comparison details - - Sample CRC32s from both Nyaa and local sources - - File processing statistics (cached vs calculated) - - Mapping issues and comparison mismatches - - Intersection and difference analysis - - Useful for diagnosing issues with missing episode detection - - Works in both Docker and local Python execution - - All debug output is prefixed with "DEBUG:" for easy filtering - -### Docker Workflow -1. Container starts with `RUN_DOCKER` environment variable set -2. Entrypoint script (`entrypoint.sh`) orchestrates execution: - - If `EPISODES_UPDATE=true`: Runs `--episodes_update` with URL from `NYAA_URL` (if set) - - If `DB=true`: Runs `--db` to export database (suppresses informational messages) - - **Always runs missing episodes report** (generates/updates `Ace-Pace_Missing.csv`) - - If `DOWNLOAD=true`: Runs `--download` to download missing episodes - - Uses default connection parameters if not specified: - - Client: transmission - - Host: localhost - - Port: 9091 (transmission) or 8080 (qbittorrent) - - Logs connection parameters used for download -3. Script operates in non-interactive mode (no user prompts) -4. Default folder is `/media` (always passed via `--folder` in entrypoint) -5. Config directory is `/config` (databases and CSV files stored here) -6. BitTorrent client connection parameters use defaults if not specified via environment variables -7. All user prompts automatically answered with defaults -8. Database files and CSV reports persist via volume mounts -9. Docker mode messages suppressed for automated commands (`--db`, `--episodes_update`) -10. Episodes metadata status only shown for main command -11. Missing episode count prominently displayed in output -12. Missing episodes CSV is always updated before download (if download is enabled) - -## Integration Points - -### BitTorrent Client Abstraction -- Abstract base class `Client` (in `clients.py`) defines interface using `abc.ABC` -- Concrete implementations: `QBittorrentClient`, `TransmissionClient` -- Factory function `get_client(client_name, host, port, username, password)` instantiates appropriate client -- Methods: `add_torrents(magnets, download_folder, tags, category)` -- **qBittorrentClient**: - - Uses `qbittorrentapi` library for API access - - Checks for existing torrents by info hash to prevent duplicates - - Supports tags and categories - - Adds tags to existing torrents if they already exist -- **TransmissionClient**: - - Uses Transmission RPC API via HTTP requests - - Handles session ID management for authentication - - Does not support tags/categories (warns if provided) - - Uses `requests` library for API calls - -### Database Management -- SQLite databases for persistence -- Connection management via context or explicit close -- Metadata storage for tracking state and timestamps -- Transaction support for atomic operations - -## Error Handling - -### Network Errors -- HTTP request failures are caught and logged -- Continues processing remaining items on individual failures -- Rate limiting via `time.sleep(0.2)` between requests - -### File System Errors -- Checks for file existence before operations -- Handles permission errors gracefully -- Validates file paths and extensions - -### Database Errors -- Uses `INSERT OR REPLACE` for idempotent operations -- Handles connection failures -- Validates data before insertion - -## Configuration and State - -### Persistent State -- Last used folder path -- Last run timestamp -- Last database export timestamp -- Last missing export timestamp -- Last episodes index update timestamp -- Last checked page number - -### User Prompts -- Folder selection (with last folder suggestion) -- Episodes index update confirmation (when using `--rename`) -- File renaming confirmation - -**Note**: In Docker mode (when `RUN_DOCKER` environment variable is set), user prompts are automatically answered with defaults to enable non-interactive operation. - -## Future Considerations - -### Potential Enhancements -- Support for additional BitTorrent clients -- Configuration file for default settings -- Web UI for easier interaction -- Automatic episode index updates on schedule -- Support for additional video formats -- Integration with media server APIs (Plex, Jellyfin) -- Duplicate detection and cleanup -- Episode metadata enrichment (thumbnails, descriptions) -- Note: Episode quality filtering (1080p only) is already implemented ✅ - -### Technical Improvements -- Async/await for concurrent web scraping -- Better error recovery and retry logic -- Unit tests and integration tests (✅ implemented) -- Logging framework instead of print statements -- Type hints for better code documentation -- Configuration validation -- Better handling of edge cases in filename parsing -- Code refactoring for reduced cognitive complexity (✅ completed) - -## Refactoring History - -### Code Refactoring (Completed) -The codebase underwent significant refactoring to reduce cognitive complexity and improve maintainability: - -- **Function Decomposition**: Large functions were broken down into smaller, focused helper functions -- **Naming Conventions**: Established consistent naming patterns for helper functions (see `NAMING_CONVENTIONS.md`) -- **Separation of Concerns**: Clear separation between public API and private implementation details -- **Docker Support**: Added Docker mode with non-interactive operation support -- **Environment Variable Support**: Added support for configuration via environment variables in Docker mode -- **Connection Parameter Abstraction**: Separated Docker and non-Docker connection parameter handling -- **Workflow Functions**: Created dedicated workflow functions for complex multi-step operations -- **Error Handling**: Improved error handling with better separation of concerns - -All helper functions follow the naming convention documented in `NAMING_CONVENTIONS.md`, making the codebase more maintainable and easier to understand. - -## Code Architecture - -### Function Organization -The codebase follows a modular structure with clear separation of concerns: - -#### Public API Functions -- `main()`: Entry point for the application -- `init_db(suppress_messages=False)`: Initializes the local CRC32 cache database - - `suppress_messages`: If True, suppresses informational messages (useful for automated runs) -- `init_episodes_db()`: Initializes the episodes index database -- `get_config_dir()`: Gets config directory path based on Docker mode (`/config` in Docker, `.` locally) -- `get_config_path(filename)`: Gets full path to a config file in the appropriate config directory -- `normalize_file_path(file_path)`: Normalizes file path for consistent storage and lookup - - Resolves symlinks and converts to absolute path - - Ensures same file always maps to same path string regardless of OS/environment -- `get_metadata(conn, key)`: Retrieves metadata value from database -- `set_metadata(conn, key, value)`: Stores metadata value in database -- `get_episodes_metadata(conn, key)`: Retrieves episodes database metadata -- `set_episodes_metadata(conn, key, value)`: Stores episodes database metadata -- `fetch_episodes_metadata(base_url=None)`: Fetches episodes from Nyaa.si - - `base_url`: Optional Nyaa.si search URL (defaults to One-Pace search without quality filter) - - Quality filtering (1080p only) is always applied regardless of URL -- `update_episodes_index_db(base_url=None)`: Updates the episodes index database - - `base_url`: Optional Nyaa.si search URL (passed to `fetch_episodes_metadata()`) -- `fetch_crc32_links(base_url)`: Fetches CRC32 links from a Nyaa.si URL - - Applies quality filtering (1080p only) via `_process_crc32_row()` -- `fetch_title_by_crc32(crc32)`: Searches for a title by CRC32 -- `calculate_local_crc32(folder, conn)`: Calculates CRC32 for local files - - Uses normalized paths for database storage and lookup -- `rename_local_files(conn)`: Renames local files based on episodes index - - Uses normalized paths when updating database after renaming -- `export_db_to_csv(conn)`: Exports database to CSV -- `load_crc32_to_title_from_index()`: Loads CRC32-to-title mapping - -#### Private Helper Functions (prefixed with `_`) -Helper functions are prefixed with `_` to indicate they are internal implementation details. See `NAMING_CONVENTIONS.md` for detailed documentation. - -**Extraction functions** (`_extract_*`): Extract data from HTML/structures -- `_extract_title_link_from_row(row)`: Extracts title link from table row -- `_extract_filenames_from_folder_structure(filelist_div)`: Extracts filenames from folder structure -- `_extract_filenames_from_torrent_page(torrent_soup)`: Extracts filenames from torrent page -- `_extract_matching_titles_from_rows(rows, crc32)`: Extracts titles matching CRC32 - -**Processing functions** (`_process_*`): Process data structures -- `_process_fname_entry(fname_text, ...)`: Processes filename entry to extract CRC32 -- `_process_torrent_page(page_link, ...)`: Processes torrent page to extract CRC32 -- `_process_episode_row(row, ...)`: Processes table row to extract episode info -- `_process_crc32_row(row, ...)`: Processes row to extract CRC32 for missing episodes - -**Validation functions** (`_is_*`, `_validate_*`): Validate inputs/data -- `_is_valid_quality(fname_text)`: Checks if filename has valid quality (1080p only) -- `_validate_url(url)`: Validates URL points to valid Nyaa domain - -**Command handlers** (`_handle_*`): Handle specific command-line operations -- `_handle_download_command(args)`: Handles the `--download` command -- `_handle_rename_command(conn, base_url=None)`: Handles the `--rename` command - - `base_url`: Optional URL parameter passed to `update_episodes_index_db()` if update is needed -- `_handle_main_commands(args, conn, folder)`: Routes and handles main commands - -**Getter functions** (`_get_*`): Retrieve or compute values -- `_get_total_pages(soup)`: Extracts total pages from pagination -- `_get_folder_from_args(args, conn, needs_folder)`: Gets folder from args or prompts -- `_get_client_from_args_or_env(args)`: Gets client type from args or env vars -- `_get_default_port(client)`: Gets default port for client -- `_get_docker_connection_params(args)`: Gets connection params from Docker env vars -- `_get_non_docker_connection_params(args)`: Gets connection params from CLI args -- `_get_rename_confirmation()`: Gets user confirmation for renaming -- `_get_rename_prompt(last_ep_update)`: Gets prompt for updating episodes DB - -**Load/Save functions** (`_load_*`, `_save_*`): Load or save data -- `_load_magnet_links()`: Loads magnet links from missing CSV -- `_load_old_missing_crc32s()`: Loads CRC32s from previous missing CSV -- `_save_missing_episodes_csv(...)`: Saves missing episodes to CSV - -**Print/Report functions** (`_print_*`, `_report_*`, `_show_*`): Display information -- `_print_report_header(conn, folder, args)`: Prints report header -- `_report_new_missing_episodes(missing, crc32_to_text)`: Reports new missing episodes -- `_show_episodes_metadata_status()`: Shows episodes metadata update status - -**Workflow functions** (`_generate_*`, `_calculate_*`): Orchestrate multi-step workflows -- `_generate_missing_episodes_report(conn, folder, args)`: Generates missing episodes report -- `_calculate_and_find_missing(folder, conn, args, last_run)`: Calculates CRC32s and finds missing - -**Utility functions** (`_parse_*`, `_count_*`, `_build_*`, `_execute_*`): General utilities -- `_parse_arguments()`: Parses command-line arguments -- `_count_video_files(folder, conn)`: Counts video files and recorded files -- `_build_rename_plan(entries, crc32_to_title)`: Builds plan of files to rename -- `_execute_rename(rename_plan, conn)`: Executes rename plan and updates DB - -This naming convention improves code readability and makes the public API clear. All helper functions follow consistent patterns documented in `NAMING_CONVENTIONS.md`. - -## Development Guidelines - -### Code Style -- Follow PEP 8 Python style guide -- Use descriptive variable names -- Add docstrings for functions -- Keep functions focused and single-purpose -- Use `_` prefix for private/internal helper functions -- Maintain cognitive complexity ≤ 15 per function - -### Database Schema -- Use SQLite for simplicity -- Maintain backward compatibility when possible -- Document schema changes - -### API Compatibility -- Maintain backward compatibility with command-line arguments -- Handle missing optional arguments gracefully -- Provide clear error messages - -## Notes for AI Agents - -When working on this project: - -1. **CRC32 is the primary identifier** - All episode matching relies on CRC32 checksums -2. **Nyaa.si structure** - Understand the HTML structure of Nyaa.si pages for scraping -3. **Database state** - Always consider existing database state when making changes -4. **File paths** - Always use `normalize_file_path()` before storing/querying file paths in database - - Ensures consistent path representation across different OS and environments - - Critical for consistent behavior between Python and Docker versions - - Resolves symlinks and converts to absolute paths -5. **User interaction** - Some operations require user confirmation (renaming, downloads), but are auto-confirmed in Docker mode -6. **Client abstraction** - New BitTorrent clients should implement the `Client` interface from `clients.py` -7. **Error tolerance** - The tool should continue processing even if individual items fail -8. **Performance** - CRC32 calculation can be slow; caching is essential (uses normalized paths for cache keys) -9. **Web scraping** - Be respectful with rate limiting and error handling -10. **File naming** - Sanitize filenames to be filesystem-safe across platforms -11. **Docker mode** - Check `IS_DOCKER` flag (from `RUN_DOCKER` env var) to determine if running in Docker - - Suppress informational messages for automated commands (`--db`, `--episodes_update`) - - Use `/config` directory for databases and CSV files in Docker mode - - Use `/media` as default folder in Docker mode -12. **Function naming** - Follow the naming conventions in `NAMING_CONVENTIONS.md` when adding new helper functions -13. **Code complexity** - Maintain cognitive complexity ≤ 15 per function (refactoring completed) -14. **Testing** - Comprehensive test suite exists in `tests/` directory (100+ tests covering all features) -15. **Environment variables** - In Docker mode, prefer environment variables over CLI args for configuration -16. **URL parameter consistency** - Both `fetch_episodes_metadata()` and `fetch_crc32_links()` accept URL parameters - - Always pass `args.url` to ensure consistent URL usage across functions - - Quality filtering is applied regardless of URL parameters -17. **Quality filtering** - Applied in both `fetch_episodes_metadata()` and `fetch_crc32_links()` via `_is_valid_quality()` - - Only accepts 1080p episodes - - Requires "[One Pace]" marker in filename - - Ensures consistent filtering regardless of URL search parameters -18. **Config directory** - Use `get_config_dir()` and `get_config_path()` for consistent file location handling - - Returns `/config` in Docker mode, `.` in local mode - - Automatically creates directory if it doesn't exist -19. **Message suppression** - Use `suppress_messages=True` in `init_db()` for automated runs - - Prevents "Database already exists" message when running `--db` in Docker -20. **Docker download logic** - Use `DOWNLOAD=true` environment variable to enable downloads (not `TORRENT_CLIENT` presence) - - Missing episodes CSV is always generated/updated before download (if download enabled) - - Download happens as a separate step after report generation -21. **Default connection values** - In Docker mode, use defaults if not specified via environment variables - - Client: transmission - - Host: localhost - - Port: 9091 (transmission) or 8080 (qbittorrent) -22. **Download logging** - Log connection parameters used for download in Docker mode for transparency - -23. **Debug mode** - Use `DEBUG` environment variable to control troubleshooting output - - Defaults to `false` (no debug output) - - Set to `true`, `1`, `yes`, or `on` to enable - - All debug output uses `debug_print()` function which checks `DEBUG_MODE` flag - - Debug output includes troubleshooting info, sample data, processing statistics, and comparison details - - Useful for diagnosing issues without cluttering normal output diff --git a/tests/test_main_command.py b/tests/test_main_command.py index 8e17bd8..59cf846 100644 --- a/tests/test_main_command.py +++ b/tests/test_main_command.py @@ -242,8 +242,8 @@ def test_episodes_update_receives_url_parameter(self, mock_update, mock_validate with pytest.raises(SystemExit): acepace.main() - # Verify update_episodes_index_db was called with URL - mock_update.assert_called_once_with(test_url) + # Verify update_episodes_index_db was called with URL and force_update + mock_update.assert_called_once_with(test_url, force_update=True) @patch('acepace._validate_url') @patch('acepace.init_db') From 3d68f91cf3a07668c5e9e7d42d7c128bf04c8481 Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 28 Jan 2026 09:45:58 +0000 Subject: [PATCH 64/75] Docker: wrong fetch fix #9786302 --- acepace.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/acepace.py b/acepace.py index ded0309..23b955a 100644 --- a/acepace.py +++ b/acepace.py @@ -1521,6 +1521,12 @@ def _load_episodes_from_database(episodes_update_env, base_url): print("Fetching magnet links from search results...") crc32_to_magnet = fetch_magnet_links_for_episodes_from_search(base_url, crc32_to_link) print(f"Fetched {len(crc32_to_magnet)} magnet links.") + # Restrict to episodes we have magnet links for. The index can include episodes + # discovered via torrent-page visits (CRC32 not in search row title); we only get + # magnets from search rows. Using the full index would overcount "on Nyaa" and + # inflate the missing count. Restricting to crc32_to_magnet matches fetch_crc32_links. + crc32_to_link = {c: crc32_to_link[c] for c in crc32_to_magnet if c in crc32_to_link} + crc32_to_text = {c: crc32_to_text.get(c, f"[CRC32: {c}]") for c in crc32_to_magnet} return crc32_to_link, crc32_to_text, crc32_to_magnet, 0 From 1e6f97b94c11e057f6b549a6100777ff42618a38 Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 28 Jan 2026 11:15:03 +0000 Subject: [PATCH 65/75] Fix alternative for missing episodes --- acepace.py | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/acepace.py b/acepace.py index 23b955a..bca2fad 100644 --- a/acepace.py +++ b/acepace.py @@ -480,6 +480,7 @@ def load_1080p_episodes_from_index(): def _extract_magnet_link_from_row(row, crc32_set): """Extract magnet link from a table row if it matches a CRC32 in the set. + First checks title, then visits torrent page if CRC32 not in title. Args: row: BeautifulSoup table row element crc32_set: Set of CRC32 values to match against @@ -492,13 +493,38 @@ def _extract_magnet_link_from_row(row, crc32_set): return None, None filename_text = title_link.text - matches = CRC32_REGEX.findall(filename_text) - if not matches: + link = NYAA_BASE_URL + title_link["href"] + + # Check if it's a One Pace episode with 1080p quality + if ONE_PACE_MARKER not in filename_text: + return None, None + if not _is_valid_quality(filename_text): return None, None - crc32 = matches[-1].upper() - if crc32 in crc32_set: - return crc32, magnet_link + # Check if CRC32 is in title + matches = CRC32_REGEX.findall(filename_text) + if matches: + crc32 = matches[-1].upper() + if crc32 in crc32_set: + return crc32, magnet_link + + # CRC32 not in title, try fetching torrent page to extract from file list + try: + torrent_resp = requests.get(link) + if torrent_resp.status_code == HTTP_OK: + t_soup = BeautifulSoup(torrent_resp.text, HTML_PARSER) + filenames = _extract_filenames_from_torrent_page(t_soup) + for fname in filenames: + fname_str = str(fname) + if ONE_PACE_MARKER in fname_str and _is_valid_quality(fname_str): + fname_matches = CRC32_REGEX.findall(fname_str) + if fname_matches: + crc32 = fname_matches[-1].upper() + if crc32 in crc32_set: + return crc32, magnet_link + except (requests.RequestException, AttributeError, TypeError): + pass + return None, None From ff5850f0470282f8feedb2883873ea6a5581bf19 Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 28 Jan 2026 11:47:23 +0000 Subject: [PATCH 66/75] Download dry run implementation --- README.md | 11 + acepace.py | 46 +- clients.py | 85 ++- coverage.xml | 1423 +++++++++++++++++++----------------- docker-compose.yml | 2 + entrypoint.sh | 1 + tests/test_clients.py | 105 +++ tests/test_main_command.py | 116 +++ 8 files changed, 1114 insertions(+), 675 deletions(-) diff --git a/README.md b/README.md index 56e4734..6d8297a 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ docker run --rm \ -e DB=true \ -e EPISODES_UPDATE=true \ -e DOWNLOAD=false \ + -e DRY_RUN=false \ -e TORRENT_CLIENT=transmission \ -e TORRENT_HOST=127.0.0.1 \ -e TORRENT_PORT=9091 \ @@ -74,6 +75,10 @@ The following environment variables can be used to configure Ace-Pace in Docker: - `DB` - Set to `true` to generate CSV database export on container start (default: `false`) - `EPISODES_UPDATE` - Set to `true` to update episodes metadata from Nyaa on container start (default: `false`) - `DOWNLOAD` - Set to `true` to automatically download missing episodes after generating report (default: `false`) +- `DRY_RUN` - Set to `true` to test connection to BitTorrent client without actually adding torrents (default: `false`) + - Only effective when `DOWNLOAD=true` + - Validates magnet links and checks existing torrents but does not add any downloads + - Useful for verifying configuration before enabling actual downloads - `TORRENT_CLIENT` - BitTorrent client type: `transmission` or `qbittorrent` (default: `transmission`) - `TORRENT_HOST` - BitTorrent client host address (default: `localhost`) - `TORRENT_PORT` - BitTorrent client port (default: `9091` for Transmission, `8080` for qBittorrent) @@ -101,6 +106,7 @@ When the container starts, it executes the following steps in order: 2. **Database Export** (if `DB=true`): Exports the CRC32 database to CSV 3. **Missing Episodes Report**: Always runs to generate/update `Ace-Pace_Missing.csv` 4. **Download** (if `DOWNLOAD=true`): Automatically downloads missing episodes via the configured BitTorrent client + - If `DRY_RUN=true`, tests connection and validates magnet links without adding torrents ### Docker Notes @@ -233,6 +239,9 @@ python acepace.py [-h] [--url URL] [--folder FOLDER] [--db] [--client {transmiss - `--category ` Category to add to the torrent in qBittorrent. +- `--dry-run` (standalone flag) + Test connection to BitTorrent client without actually adding torrents. Useful for verifying configuration before downloading. When enabled, validates magnet links and checks existing torrents but does not add any new downloads. + ### 📚 Some examples ``` @@ -240,6 +249,8 @@ python acepace.py --folder "/volume42/media/One Piece/" --url https://nyaa.si/?f python acepace.py --folder "/volume42/media/One Piece/" python acepace.py --client transmission --download python acepace.py --client qbittorrent --download --host 192.168.1.100 --port 8080 --username myuser --password mypassword --download-folder /downloads/onepace --tag onepace --tag 'one pace' --category 'anime' +python acepace.py --client transmission --download --dry-run +python acepace.py --client qbittorrent --download --dry-run --host 192.168.1.100 --port 8080 python acepace.py --db python acepace.py --folder "/volume42/media/One Piece/" --rename python acepace.py --episodes_update --url https://nyaa.si/?f=0&c=0_0&q=one+pace+1080p&o=asc diff --git a/acepace.py b/acepace.py index bca2fad..2b5914a 100644 --- a/acepace.py +++ b/acepace.py @@ -1180,12 +1180,16 @@ def _handle_download_command(args): print(f" Username: {username}") if download_folder: print(f" Download folder: {download_folder}") + if args.dry_run: + print(" Mode: DRY RUN (no torrents will be added)") else: client = _get_client_from_args_or_env(args) if not client: print("Error: --client is required when using --download.") return False host, port, username, password, download_folder = _get_non_docker_connection_params(args) + if args.dry_run: + print("DRY RUN MODE: Testing connection without adding torrents...") magnets = _load_magnet_links() if magnets is None: @@ -1193,14 +1197,27 @@ def _handle_download_command(args): try: client_obj = get_client(client, host, port, username, password) - print(f"Adding {len(magnets)} missing episode(s) to {client}...") - client_obj.add_torrents( - magnets, - download_folder=download_folder, - tags=args.tag, - category=args.category, - ) - print(f"Successfully added {len(magnets)} episode(s) to {client}.") + if args.dry_run: + print(f"DRY RUN: Would add {len(magnets)} missing episode(s) to {client}...") + print("DRY RUN: Testing connection and validating magnet links...") + client_obj.add_torrents( + magnets, + download_folder=download_folder, + tags=args.tag, + category=args.category, + dry_run=True, + ) + print(f"DRY RUN: Successfully validated connection to {client}.") + print(f"DRY RUN: {len(magnets)} magnet link(s) would be added (no torrents were actually added).") + else: + print(f"Adding {len(magnets)} missing episode(s) to {client}...") + client_obj.add_torrents( + magnets, + download_folder=download_folder, + tags=args.tag, + category=args.category, + ) + print(f"Successfully added {len(magnets)} episode(s) to {client}.") except ConnectionError as e: print(f"Connection Error: {e}") print(f"Please verify that {client} is running and accessible at {host}:{port}") @@ -1710,6 +1727,11 @@ def _print_help(): Reads magnet links from Ace-Pace_Missing.csv and adds them to the specified BitTorrent client (requires --client) + --dry-run Test connection to BitTorrent client without adding torrents + Validates magnet links and checks existing torrents but + does not add any downloads. Useful for verifying configuration. + Only effective when used with --download. + BitTorrent Client Options (for --download): --client {transmission,qbittorrent} Specify which BitTorrent client to use @@ -1756,6 +1778,9 @@ def _print_help(): # Download missing episodes to qBittorrent python acepace.py --download --client qbittorrent --host localhost --port 8080 + # Test connection without downloading (dry run) + python acepace.py --download --client transmission --dry-run + # Export database to CSV python acepace.py --folder /path/to/videos --db @@ -1821,6 +1846,11 @@ def _parse_arguments(): parser.add_argument("--download-folder", help="The folder to download the torrents to.") parser.add_argument("--tag", action="append", help="Tag to add to the torrent in qBittorrent (can be used multiple times).") parser.add_argument("--category", help="Category to add to the torrent in qBittorrent.") + parser.add_argument( + "--dry-run", + action="store_true", + help="Test connection to BitTorrent client without actually adding torrents. Useful for verifying configuration.", + ) return parser.parse_args() diff --git a/clients.py b/clients.py index f7fa711..b3d1030 100644 --- a/clients.py +++ b/clients.py @@ -7,7 +7,7 @@ class Client(abc.ABC): @abc.abstractmethod - def add_torrents(self, magnets, download_folder=None, tags=None, category=None): + def add_torrents(self, magnets, download_folder=None, tags=None, category=None, dry_run=False): pass class QBittorrentClient(Client): @@ -59,7 +59,52 @@ def _add_new_torrent(self, magnet, download_folder, tags_str, category, truncate print(f"Failed to add torrent: {truncated} Error: {e}") return False - def add_torrents(self, magnets, download_folder=None, tags=None, category=None): + def add_torrents(self, magnets, download_folder=None, tags=None, category=None, dry_run=False): + if dry_run: + print("DRY RUN: Validating magnet links and checking existing torrents...") + if tags: + print(f"DRY RUN: Would create tags: {', '.join(tags)}") + + total = len(magnets) + tags_str = ",".join(tags) if tags else None + valid_count = 0 + existing_count = 0 + invalid_count = 0 + + for idx, magnet in enumerate(magnets, 1): + truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") + print(f"DRY RUN: Processing {idx}/{total}: {truncated}") + + info_hash = self._extract_info_hash(magnet) + if not info_hash: + print(f"DRY RUN: Could not find info hash in magnet link: {truncated}") + invalid_count += 1 + continue + + try: + existing_torrent = self.client.torrents_info(torrent_hashes=info_hash) + if existing_torrent: + print(f"DRY RUN: Torrent already exists: {truncated}") + existing_count += 1 + if tags_str: + print(f"DRY RUN: Would add tags to existing torrent: {tags_str}") + else: + print(f"DRY RUN: Would add new torrent: {truncated}") + if download_folder: + print(f"DRY RUN: Download folder: {download_folder}") + if tags_str: + print(f"DRY RUN: Tags: {tags_str}") + if category: + print(f"DRY RUN: Category: {category}") + valid_count += 1 + except Exception as e: + print(f"DRY RUN: Error checking torrent: {truncated} Error: {e}") + invalid_count += 1 + time.sleep(0.1) + + print(f"DRY RUN: Summary - {valid_count} would be added, {existing_count} already exist, {invalid_count} invalid") + return + if tags: self.client.torrents_create_tags(tags=",".join(tags)) @@ -151,7 +196,41 @@ def _add_single_torrent(self, magnet, download_folder, truncated): print(f"Failed to add torrent: {truncated} Error: {e}") return False - def add_torrents(self, magnets, download_folder=None, tags=None, category=None): + def add_torrents(self, magnets, download_folder=None, tags=None, category=None, dry_run=False): + if dry_run: + print("DRY RUN: Validating magnet links...") + if tags or category: + print("DRY RUN: Warning - Transmission does not support tags or categories through this script.") + + total = len(magnets) + valid_count = 0 + invalid_count = 0 + + for idx, magnet in enumerate(magnets, 1): + truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") + print(f"DRY RUN: Processing {idx}/{total}: {truncated}") + + # Validate magnet link format + if not magnet.startswith("magnet:?"): + print(f"DRY RUN: Invalid magnet link format: {truncated}") + invalid_count += 1 + continue + + # Check if we can parse the magnet link (basic validation) + if "xt=urn:btih:" not in magnet: + print(f"DRY RUN: Magnet link missing info hash: {truncated}") + invalid_count += 1 + continue + + print(f"DRY RUN: Would add torrent: {truncated}") + if download_folder: + print(f"DRY RUN: Download folder: {download_folder}") + valid_count += 1 + time.sleep(0.1) + + print(f"DRY RUN: Summary - {valid_count} valid magnet links would be added, {invalid_count} invalid") + return + if tags or category: print("Warning: Transmission does not support tags or categories through this script.") added_count = 0 diff --git a/coverage.xml b/coverage.xml index 3e92be7..193b28f 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -285,798 +285,830 @@ - - - - + + + - - - - - + + + + - + + + + + - + - - - + + + + - - - + + + + - - - + + - - + + + + + + + + + - + - - - - - + + - - - - - - - + - - + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + - - - - - - - - - - - - - - + + + + + + + + - - - - - + + + + + + + + + - - - - + + + + + + - + + - - - - - - - - - + + + + + - - + - - - + + + + - - - - - + + + + + + - - - - - - - - + + + + + - - + + + + + + - - - - + - - - + + - - + + + - + - - - - + + + + - - + + + + + - - + - - + - - + + + - - + - + + + - - + - - - - - - - - - + + + + + + + + + + - - - - - + - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + - - - - - - - + + + + + + + - + - + - - - - - + + + + + + + + + - + - + - + + - - - - - + + + + - - - - - - + - + - + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - + - - + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - + - - - - - - - - + + + + + + + + - + - - - - + + + - - - + + + + - - - - - - - - - - - + + + + + + + + + + + + + - + + - - - - - + + + + + + + + - - - + + + + - - + + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + - - - - + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - + + + - + - - + + - + + - + + + + - - - - - + + + + + + + - - - - + - - - + + + + + - - + + - + - - + + + - + - - - - - - + + + + + + - - + + + - - - - + + + + - - - + + - - - - - - + + + - + - - + + + + - + - + - - - - + + - - - - - - + + + + + + + - + - + - + + - - - - - - + + + + + + + - - - - + + + - - - - - - + + + + + + + + + + + - + + + - - - - + + + + + - + + - - - - + + + - - - - - + + + + + + + + + + + - - + + + + - - - - + - - + + + - - - - - + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + - - - - - + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + - - - - + + + + - - - - - + + + + + + - - + + + + - - - + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -1121,94 +1153,157 @@ + - - + - + - - + + + + - + - - + + + - - - - - - - + + + + + + + + - - + + - + - - - - + + + + - + + - + - + - - + - - + - - + + - - - - - - - - - - + + + + + + + - + - - - - - - - + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index cce9287..3cfaeef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,8 @@ services: # Download missing episodes after generating report (default: false) #- DOWNLOAD=true + # Test connection without adding torrents (default: false, only effective when DOWNLOAD=true) + #- DRY_RUN=true # BitTorrent client type (default: transmission) # Options: transmission, qbittorrent diff --git a/entrypoint.sh b/entrypoint.sh index ac7b167..457c7f0 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -65,6 +65,7 @@ if [ "$DOWNLOAD" = "true" ]; then --folder /media \ ${NYAA_URL:+--url "$NYAA_URL"} \ --download \ + ${DRY_RUN:+--dry-run} \ ${TORRENT_CLIENT:+--client "$TORRENT_CLIENT"} \ ${TORRENT_HOST:+--host "$TORRENT_HOST"} \ ${TORRENT_PORT:+--port "$TORRENT_PORT"} \ diff --git a/tests/test_clients.py b/tests/test_clients.py index 3d5b7ee..98c1cf5 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -267,3 +267,108 @@ def test_get_client_unknown(self): with pytest.raises(ValueError) as exc_info: get_client("unknown", "localhost", 8080, "user", "pass") assert "Unknown client" in str(exc_info.value) + + +class TestDryRunMode: + """Tests for dry run mode functionality.""" + + @patch('clients.qbittorrentapi.Client') + @patch('clients.time.sleep') + def test_qbittorrent_dry_run(self, mock_sleep, mock_client_class, sample_magnet_links): + """Test qBittorrent dry run mode validates without adding torrents.""" + mock_client = MagicMock() + mock_client.auth_log_in.return_value = None + mock_client.torrents_info.return_value = [] # No existing torrents + mock_client_class.return_value = mock_client + + client = QBittorrentClient("localhost", 8080, "user", "pass") + client.add_torrents(sample_magnet_links, download_folder="/downloads", tags=["test"], category="anime", dry_run=True) + + # Should not call torrents_add in dry run mode + mock_client.torrents_add.assert_not_called() + # Should call torrents_info to check existing torrents + assert mock_client.torrents_info.call_count == len(sample_magnet_links) + + @patch('clients.qbittorrentapi.Client') + @patch('clients.time.sleep') + def test_qbittorrent_dry_run_with_existing_torrents(self, mock_sleep, mock_client_class, sample_magnet_links): + """Test qBittorrent dry run mode handles existing torrents.""" + mock_client = MagicMock() + mock_client.auth_log_in.return_value = None + # First torrent exists, second doesn't + mock_client.torrents_info.side_effect = [ + [{"hash": "1234567890abcdef1234567890abcdef12345678"}], # First exists + [] # Second doesn't + ] + mock_client_class.return_value = mock_client + + client = QBittorrentClient("localhost", 8080, "user", "pass") + client.add_torrents(sample_magnet_links, tags=["test"], dry_run=True) + + # Should not call torrents_add in dry run mode + mock_client.torrents_add.assert_not_called() + # Should not add tags to existing torrents in dry run + mock_client.torrents_add_tags.assert_not_called() + + @patch('clients.qbittorrentapi.Client') + @patch('clients.time.sleep') + def test_qbittorrent_dry_run_invalid_magnet(self, mock_sleep, mock_client_class): + """Test qBittorrent dry run mode handles invalid magnet links.""" + mock_client = MagicMock() + mock_client.auth_log_in.return_value = None + mock_client_class.return_value = mock_client + + client = QBittorrentClient("localhost", 8080, "user", "pass") + invalid_magnets = ["invalid_magnet_link"] + + client.add_torrents(invalid_magnets, dry_run=True) + + # Should not call any API methods for invalid magnets + mock_client.torrents_add.assert_not_called() + mock_client.torrents_info.assert_not_called() + + @patch('clients.requests.Session') + @patch('clients.time.sleep') + def test_transmission_dry_run(self, mock_sleep, mock_session_class, sample_magnet_links): + """Test Transmission dry run mode validates without adding torrents.""" + mock_session = MagicMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": "success"} + mock_session.post.return_value = mock_response + mock_session_class.return_value = mock_session + + client = TransmissionClient("localhost", 9091, None, None) + client.session_id = "test_session_id" + client.add_torrents(sample_magnet_links, download_folder="/downloads", dry_run=True) + + # Should not make torrent-add requests in dry run mode + # Only the initial session-get call should be made (in __init__) + # Count calls that are torrent-add (not session-get) + torrent_add_calls = [call for call in mock_session.post.call_args_list + if len(call[1].get('json', {}).get('method', '')) > 0 + and call[1]['json'].get('method') == 'torrent-add'] + assert len(torrent_add_calls) == 0 + + @patch('clients.requests.Session') + @patch('clients.time.sleep') + def test_transmission_dry_run_invalid_magnet(self, mock_sleep, mock_session_class): + """Test Transmission dry run mode handles invalid magnet links.""" + mock_session = MagicMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": "success"} + mock_session.post.return_value = mock_response + mock_session_class.return_value = mock_session + + client = TransmissionClient("localhost", 9091, None, None) + client.session_id = "test_session_id" + invalid_magnets = ["invalid_magnet_link"] + + client.add_torrents(invalid_magnets, dry_run=True) + + # Should not make any torrent-add requests for invalid magnets + torrent_add_calls = [call for call in mock_session.post.call_args_list + if len(call[1].get('json', {}).get('method', '')) > 0 + and call[1]['json'].get('method') == 'torrent-add'] + assert len(torrent_add_calls) == 0 diff --git a/tests/test_main_command.py b/tests/test_main_command.py index 59cf846..6db92fd 100644 --- a/tests/test_main_command.py +++ b/tests/test_main_command.py @@ -387,3 +387,119 @@ def test_docker_uses_environment_variable_overrides(self, mock_get_client, mock_ assert call_args[0][1] == TEST_HOST_IP # From env var assert call_args[0][2] == 8080 # From env var assert call_args[0][3] == "admin" # From env var + + +class TestDryRunMode: + """Tests for dry run mode in download command.""" + + @patch('acepace._load_magnet_links') + @patch('acepace.get_client') + def test_dry_run_mode_calls_add_torrents_with_dry_run_flag(self, mock_get_client, mock_load_magnets): + """Test that dry run mode passes dry_run=True to add_torrents.""" + mock_load_magnets.return_value = ["magnet:?xt=urn:btih:test123"] + mock_client_obj = MagicMock() + mock_get_client.return_value = mock_client_obj + + mock_args = MagicMock() + mock_args.client = "transmission" + mock_args.host = "localhost" + mock_args.port = 9091 + mock_args.username = None + mock_args.password = None + mock_args.download_folder = None + mock_args.tag = None + mock_args.category = None + mock_args.dry_run = True + + acepace._handle_download_command(mock_args) + + # Verify add_torrents was called with dry_run=True + mock_client_obj.add_torrents.assert_called_once() + call_kwargs = mock_client_obj.add_torrents.call_args[1] + assert call_kwargs['dry_run'] is True + + @patch('acepace.IS_DOCKER', True) + @patch('acepace._load_magnet_links') + @patch('acepace.get_client') + def test_docker_dry_run_mode_logs_dry_run(self, mock_get_client, mock_load_magnets): + """Test that Docker mode logs dry run status.""" + mock_load_magnets.return_value = ["magnet:?xt=urn:btih:test123"] + mock_client_obj = MagicMock() + mock_get_client.return_value = mock_client_obj + + mock_args = MagicMock() + mock_args.client = None + mock_args.host = None + mock_args.port = None + mock_args.username = None + mock_args.password = None + mock_args.download_folder = None + mock_args.tag = None + mock_args.category = None + mock_args.dry_run = True + + with patch.dict('os.environ', {}, clear=False): + for key in ['TORRENT_CLIENT', 'TORRENT_HOST', 'TORRENT_PORT', 'TORRENT_USER', 'TORRENT_PASSWORD']: + if key in os.environ: + del os.environ[key] + + with patch('builtins.print') as mock_print: + acepace._handle_download_command(mock_args) + + # Verify dry run mode was logged + print_calls = [str(c) for c in mock_print.call_args_list] + assert any("DRY RUN" in str(call) for call in print_calls) + assert any("Mode: DRY RUN" in str(call) for call in print_calls) + + @patch('acepace.IS_DOCKER', False) + @patch('acepace._load_magnet_links') + @patch('acepace.get_client') + def test_non_docker_dry_run_mode_logs_dry_run(self, mock_get_client, mock_load_magnets): + """Test that non-Docker mode logs dry run status.""" + mock_load_magnets.return_value = ["magnet:?xt=urn:btih:test123"] + mock_client_obj = MagicMock() + mock_get_client.return_value = mock_client_obj + + mock_args = MagicMock() + mock_args.client = "transmission" + mock_args.host = "localhost" + mock_args.port = 9091 + mock_args.username = None + mock_args.password = None + mock_args.download_folder = None + mock_args.tag = None + mock_args.category = None + mock_args.dry_run = True + + with patch('builtins.print') as mock_print: + acepace._handle_download_command(mock_args) + + # Verify dry run mode was logged + print_calls = [str(c) for c in mock_print.call_args_list] + assert any("DRY RUN MODE" in str(call) for call in print_calls) + + @patch('acepace._load_magnet_links') + @patch('acepace.get_client') + def test_dry_run_mode_does_not_add_torrents(self, mock_get_client, mock_load_magnets): + """Test that dry run mode does not actually add torrents.""" + mock_load_magnets.return_value = ["magnet:?xt=urn:btih:test123"] + mock_client_obj = MagicMock() + mock_get_client.return_value = mock_client_obj + + mock_args = MagicMock() + mock_args.client = "transmission" + mock_args.host = "localhost" + mock_args.port = 9091 + mock_args.username = None + mock_args.password = None + mock_args.download_folder = None + mock_args.tag = None + mock_args.category = None + mock_args.dry_run = True + + acepace._handle_download_command(mock_args) + + # Verify add_torrents was called with dry_run=True + mock_client_obj.add_torrents.assert_called_once() + call_kwargs = mock_client_obj.add_torrents.call_args[1] + assert call_kwargs['dry_run'] is True From 0dc695303f54ee05835594a14134658fa9c17bbf Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 28 Jan 2026 12:27:11 +0000 Subject: [PATCH 67/75] Refactoring and optimisation --- acepace.py | 283 +++++--- clients.py | 192 +++--- coverage.xml | 1540 +++++++++++++++++++++++--------------------- docker-compose.yml | 7 +- 4 files changed, 1104 insertions(+), 918 deletions(-) diff --git a/acepace.py b/acepace.py index 2b5914a..d590abe 100644 --- a/acepace.py +++ b/acepace.py @@ -478,6 +478,58 @@ def load_1080p_episodes_from_index(): return crc32_to_link, crc32_to_text +def _validate_row_links(title_link, magnet_link): + """Validate that row links are valid and properly formatted.""" + return (title_link and magnet_link and + isinstance(magnet_link, str) and + magnet_link.startswith(MAGNET_LINK_PREFIX) and + hasattr(title_link, 'text')) + + +def _is_valid_one_pace_episode(filename_text): + """Check if filename is a valid One Pace episode with 1080p quality.""" + if ONE_PACE_MARKER not in filename_text: + return False + return _is_valid_quality(filename_text) + + +def _extract_crc32_from_text(text): + """Extract CRC32 from text if present.""" + matches = CRC32_REGEX.findall(text) + if matches: + return matches[-1].upper() + return None + + +def _check_crc32_in_title(filename_text, crc32_set, magnet_link): + """Check if CRC32 is in the title and matches the set.""" + crc32 = _extract_crc32_from_text(filename_text) + if crc32 and crc32 in crc32_set: + return crc32, magnet_link + return None, None + + +def _fetch_crc32_from_torrent_page(link, crc32_set, magnet_link): + """Fetch torrent page and extract CRC32 from file list.""" + try: + torrent_resp = requests.get(link) + if torrent_resp.status_code != HTTP_OK: + return None, None + + t_soup = BeautifulSoup(torrent_resp.text, HTML_PARSER) + filenames = _extract_filenames_from_torrent_page(t_soup) + for fname in filenames: + fname_str = str(fname) + if ONE_PACE_MARKER in fname_str and _is_valid_quality(fname_str): + crc32 = _extract_crc32_from_text(fname_str) + if crc32 and crc32 in crc32_set: + return crc32, magnet_link + except (requests.RequestException, AttributeError, TypeError): + pass + + return None, None + + def _extract_magnet_link_from_row(row, crc32_set): """Extract magnet link from a table row if it matches a CRC32 in the set. First checks title, then visits torrent page if CRC32 not in title. @@ -486,46 +538,24 @@ def _extract_magnet_link_from_row(row, crc32_set): crc32_set: Set of CRC32 values to match against Returns: Tuple of (crc32, magnet_link) if found, (None, None) otherwise""" title_link, magnet_link = _extract_links_from_row(row) - if not (title_link and magnet_link and - isinstance(magnet_link, str) and - magnet_link.startswith(MAGNET_LINK_PREFIX) and - hasattr(title_link, 'text')): + if not _validate_row_links(title_link, magnet_link): return None, None - filename_text = title_link.text - link = NYAA_BASE_URL + title_link["href"] + # Type guard: after validation, title_link is guaranteed to be non-None + assert title_link is not None and magnet_link is not None - # Check if it's a One Pace episode with 1080p quality - if ONE_PACE_MARKER not in filename_text: - return None, None - if not _is_valid_quality(filename_text): + filename_text = title_link.text + if not _is_valid_one_pace_episode(filename_text): return None, None # Check if CRC32 is in title - matches = CRC32_REGEX.findall(filename_text) - if matches: - crc32 = matches[-1].upper() - if crc32 in crc32_set: - return crc32, magnet_link + crc32, found_magnet = _check_crc32_in_title(filename_text, crc32_set, magnet_link) + if crc32: + return crc32, found_magnet # CRC32 not in title, try fetching torrent page to extract from file list - try: - torrent_resp = requests.get(link) - if torrent_resp.status_code == HTTP_OK: - t_soup = BeautifulSoup(torrent_resp.text, HTML_PARSER) - filenames = _extract_filenames_from_torrent_page(t_soup) - for fname in filenames: - fname_str = str(fname) - if ONE_PACE_MARKER in fname_str and _is_valid_quality(fname_str): - fname_matches = CRC32_REGEX.findall(fname_str) - if fname_matches: - crc32 = fname_matches[-1].upper() - if crc32 in crc32_set: - return crc32, magnet_link - except (requests.RequestException, AttributeError, TypeError): - pass - - return None, None + link = NYAA_BASE_URL + title_link["href"] + return _fetch_crc32_from_torrent_page(link, crc32_set, magnet_link) def _process_magnet_links_page(page_soup, crc32_set, crc32_to_magnet): @@ -1166,30 +1196,101 @@ def _load_magnet_links(): return magnets +def _load_existing_magnet_links_from_csv(): + """Load existing magnet links from the missing CSV file, mapping CRC32 to magnet link. + Returns: Dictionary mapping CRC32 to magnet_link""" + missing_csv_path = get_config_path(MISSING_CSV_FILENAME) + crc32_to_magnet = {} + + if not os.path.exists(missing_csv_path): + return crc32_to_magnet + + try: + with open(missing_csv_path, "r", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + title = row.get("Title", "").strip() + magnet_link = row.get("Magnet Link", "").strip() + + # Extract CRC32 from title + matches = CRC32_REGEX.findall(title) + if matches and magnet_link.startswith(MAGNET_LINK_PREFIX): + crc32 = matches[-1].upper() + crc32_to_magnet[crc32] = magnet_link + except (IOError, csv.Error, KeyError) as e: + debug_print(f"DEBUG: Error loading existing magnet links from CSV: {e}") + + return crc32_to_magnet + + +def _setup_docker_connection(args): + """Setup connection parameters for Docker mode.""" + host, port, username, password, download_folder, client = _get_docker_connection_params(args) + # Log connection parameters in Docker mode + print("Download configuration:") + print(f" Client: {client}") + print(f" Host: {host}") + print(f" Port: {port}") + if username: + print(f" Username: {username}") + if download_folder: + print(f" Download folder: {download_folder}") + if args.dry_run: + print(" Mode: DRY RUN (no torrents will be added)") + return host, port, username, password, download_folder, client + + +def _setup_non_docker_connection(args): + """Setup connection parameters for non-Docker mode.""" + client = _get_client_from_args_or_env(args) + if not client: + print("Error: --client is required when using --download.") + return None, None, None, None, None, None + host, port, username, password, download_folder = _get_non_docker_connection_params(args) + if args.dry_run: + print("DRY RUN MODE: Testing connection without adding torrents...") + return host, port, username, password, download_folder, client + + +def _execute_download_dry_run(client_obj, magnets, client, download_folder, tags, category): + """Execute download in dry-run mode.""" + print(f"DRY RUN: Would add {len(magnets)} missing episode(s) to {client}...") + print("DRY RUN: Testing connection and validating magnet links...") + client_obj.add_torrents( + magnets, + download_folder=download_folder, + tags=tags, + category=category, + dry_run=True, + ) + print(f"DRY RUN: Successfully validated connection to {client}.") + print(f"DRY RUN: {len(magnets)} magnet link(s) would be added (no torrents were actually added).") + + +def _execute_download(client_obj, magnets, client, download_folder, tags, category): + """Execute actual download.""" + print(f"Adding {len(magnets)} missing episode(s) to {client}...") + client_obj.add_torrents( + magnets, + download_folder=download_folder, + tags=tags, + category=category, + ) + print(f"Successfully added {len(magnets)} episode(s) to {client}.") + + def _handle_download_command(args): """Handle the download command.""" # Get connection parameters based on Docker mode if IS_DOCKER: - host, port, username, password, download_folder, client = _get_docker_connection_params(args) - # Log connection parameters in Docker mode - print("Download configuration:") - print(f" Client: {client}") - print(f" Host: {host}") - print(f" Port: {port}") - if username: - print(f" Username: {username}") - if download_folder: - print(f" Download folder: {download_folder}") - if args.dry_run: - print(" Mode: DRY RUN (no torrents will be added)") + result = _setup_docker_connection(args) else: - client = _get_client_from_args_or_env(args) - if not client: - print("Error: --client is required when using --download.") - return False - host, port, username, password, download_folder = _get_non_docker_connection_params(args) - if args.dry_run: - print("DRY RUN MODE: Testing connection without adding torrents...") + result = _setup_non_docker_connection(args) + + if result[0] is None: # Check if setup failed + return False + + host, port, username, password, download_folder, client = result magnets = _load_magnet_links() if magnets is None: @@ -1198,26 +1299,9 @@ def _handle_download_command(args): try: client_obj = get_client(client, host, port, username, password) if args.dry_run: - print(f"DRY RUN: Would add {len(magnets)} missing episode(s) to {client}...") - print("DRY RUN: Testing connection and validating magnet links...") - client_obj.add_torrents( - magnets, - download_folder=download_folder, - tags=args.tag, - category=args.category, - dry_run=True, - ) - print(f"DRY RUN: Successfully validated connection to {client}.") - print(f"DRY RUN: {len(magnets)} magnet link(s) would be added (no torrents were actually added).") + _execute_download_dry_run(client_obj, magnets, client, download_folder, args.tag, args.category) else: - print(f"Adding {len(magnets)} missing episode(s) to {client}...") - client_obj.add_torrents( - magnets, - download_folder=download_folder, - tags=args.tag, - category=args.category, - ) - print(f"Successfully added {len(magnets)} episode(s) to {client}.") + _execute_download(client_obj, magnets, client, download_folder, args.tag, args.category) except ConnectionError as e: print(f"Connection Error: {e}") print(f"Please verify that {client} is running and accessible at {host}:{port}") @@ -1548,11 +1632,12 @@ def _handle_episodes_update_decision(episodes_update_env, last_update_str, base_ return False -def _load_episodes_from_database(episodes_update_env, base_url): - """Load episodes from database and fetch magnet links. +def _load_episodes_from_database(episodes_update_env, base_url, fetch_magnets=True): + """Load episodes from database and optionally fetch magnet links. Args: episodes_update_env: True if EPISODES_UPDATE environment variable is set base_url: Base URL for Nyaa search + fetch_magnets: If True, fetch magnet links for all episodes. If False, return empty dict. Returns: Tuple of (crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page)""" if episodes_update_env: print("Using episodes index database (EPISODES_UPDATE=true, using updated database)...") @@ -1561,15 +1646,20 @@ def _load_episodes_from_database(episodes_update_env, base_url): crc32_to_link, crc32_to_text = load_1080p_episodes_from_index() print(f"Loaded {len(crc32_to_link)} 1080p episodes from database.") - print("Fetching magnet links from search results...") - crc32_to_magnet = fetch_magnet_links_for_episodes_from_search(base_url, crc32_to_link) - print(f"Fetched {len(crc32_to_magnet)} magnet links.") - # Restrict to episodes we have magnet links for. The index can include episodes - # discovered via torrent-page visits (CRC32 not in search row title); we only get - # magnets from search rows. Using the full index would overcount "on Nyaa" and - # inflate the missing count. Restricting to crc32_to_magnet matches fetch_crc32_links. - crc32_to_link = {c: crc32_to_link[c] for c in crc32_to_magnet if c in crc32_to_link} - crc32_to_text = {c: crc32_to_text.get(c, f"[CRC32: {c}]") for c in crc32_to_magnet} + + if fetch_magnets: + print("Fetching magnet links from search results...") + crc32_to_magnet = fetch_magnet_links_for_episodes_from_search(base_url, crc32_to_link) + print(f"Fetched {len(crc32_to_magnet)} magnet links.") + # Restrict to episodes we have magnet links for. The index can include episodes + # discovered via torrent-page visits (CRC32 not in search row title); we only get + # magnets from search rows. Using the full index would overcount "on Nyaa" and + # inflate the missing count. Restricting to crc32_to_magnet matches fetch_crc32_links. + crc32_to_link = {c: crc32_to_link[c] for c in crc32_to_magnet if c in crc32_to_link} + crc32_to_text = {c: crc32_to_text.get(c, f"[CRC32: {c}]") for c in crc32_to_magnet} + else: + crc32_to_magnet = {} + return crc32_to_link, crc32_to_text, crc32_to_magnet, 0 @@ -1627,7 +1717,14 @@ def _calculate_and_find_missing(folder, conn, args, last_run): conn_episodes.close() # Load episodes (from database or fetch from Nyaa) - if use_database: + # When using database without EPISODES_UPDATE, don't fetch magnet links yet + # We'll fetch them only for missing episodes after determining which are missing + if use_database and not episodes_update_env: + # Load episodes from database without fetching magnet links + crc32_to_link, crc32_to_text, _, last_checked_page = _load_episodes_from_database(episodes_update_env, args.url, fetch_magnets=False) + crc32_to_magnet = {} + elif use_database: + # EPISODES_UPDATE=true: fetch all magnet links as before crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = _load_episodes_from_database(episodes_update_env, args.url) else: # Normal fetch from Nyaa (only when database doesn't exist and EPISODES_UPDATE=false) @@ -1654,6 +1751,28 @@ def _calculate_and_find_missing(folder, conn, args, last_run): # Calculate missing episodes missing = _calculate_missing_episodes(crc32_to_link, local_crc32s) + # When using database without EPISODES_UPDATE, only fetch magnet links for missing episodes + if use_database and not episodes_update_env: + # Load existing magnet links from CSV file + existing_magnets = _load_existing_magnet_links_from_csv() + debug_print(f"DEBUG: Loaded {len(existing_magnets)} existing magnet links from CSV") + + # Determine which missing episodes need magnet links + missing_without_magnets = [crc32 for crc32 in missing if crc32 not in existing_magnets] + + if missing_without_magnets: + print(f"Fetching magnet links for {len(missing_without_magnets)} missing episodes without magnet links...") + # Create a subset of crc32_to_link for missing episodes only + missing_crc32_to_link = {crc32: crc32_to_link[crc32] for crc32 in missing_without_magnets if crc32 in crc32_to_link} + # Fetch magnet links only for missing episodes without one + fetched_magnets = fetch_magnet_links_for_episodes_from_search(args.url, missing_crc32_to_link) + # Merge with existing magnets + crc32_to_magnet = {**existing_magnets, **fetched_magnets} + print(f"Fetched {len(fetched_magnets)} new magnet links.") + else: + print("All missing episodes already have magnet links in CSV.") + crc32_to_magnet = existing_magnets + print( f"\nSummary: {len(missing)} missing episodes out of {len(crc32_to_link)} total found on Nyaa.\n" ) diff --git a/clients.py b/clients.py index b3d1030..4c1da6f 100644 --- a/clients.py +++ b/clients.py @@ -59,52 +59,60 @@ def _add_new_torrent(self, magnet, download_folder, tags_str, category, truncate print(f"Failed to add torrent: {truncated} Error: {e}") return False - def add_torrents(self, magnets, download_folder=None, tags=None, category=None, dry_run=False): - if dry_run: - print("DRY RUN: Validating magnet links and checking existing torrents...") - if tags: - print(f"DRY RUN: Would create tags: {', '.join(tags)}") - - total = len(magnets) - tags_str = ",".join(tags) if tags else None - valid_count = 0 - existing_count = 0 - invalid_count = 0 - - for idx, magnet in enumerate(magnets, 1): - truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") - print(f"DRY RUN: Processing {idx}/{total}: {truncated}") - - info_hash = self._extract_info_hash(magnet) - if not info_hash: - print(f"DRY RUN: Could not find info hash in magnet link: {truncated}") - invalid_count += 1 - continue - - try: - existing_torrent = self.client.torrents_info(torrent_hashes=info_hash) - if existing_torrent: - print(f"DRY RUN: Torrent already exists: {truncated}") - existing_count += 1 - if tags_str: - print(f"DRY RUN: Would add tags to existing torrent: {tags_str}") - else: - print(f"DRY RUN: Would add new torrent: {truncated}") - if download_folder: - print(f"DRY RUN: Download folder: {download_folder}") - if tags_str: - print(f"DRY RUN: Tags: {tags_str}") - if category: - print(f"DRY RUN: Category: {category}") - valid_count += 1 - except Exception as e: - print(f"DRY RUN: Error checking torrent: {truncated} Error: {e}") - invalid_count += 1 - time.sleep(0.1) - - print(f"DRY RUN: Summary - {valid_count} would be added, {existing_count} already exist, {invalid_count} invalid") - return + def _process_torrent_dry_run(self, magnet, idx, total, tags_str, category): + """Process a single torrent in dry-run mode.""" + truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") + print(f"DRY RUN: Processing torrent {idx}/{total}: {truncated}") + + info_hash = self._extract_info_hash(magnet) + if not info_hash: + print(f"DRY RUN: Could not find info hash in magnet link: {truncated}") + return "invalid" + + try: + existing_torrent = self.client.torrents_info(torrent_hashes=info_hash) + if existing_torrent: + print(f"DRY RUN: Torrent already exists: {truncated}") + if tags_str: + print(f"DRY RUN: Would add tags to existing torrent: {tags_str}") + return "existing" + else: + print(f"DRY RUN: Would add new torrent: {truncated}") + if tags_str: + print(f"DRY RUN: Tags: {tags_str}") + if category: + print(f"DRY RUN: Category: {category}") + return "valid" + except Exception as e: + print(f"DRY RUN: Error checking torrent: {truncated} Error: {e}") + return "invalid" + + def _add_torrents_dry_run(self, magnets, tags, category): + """Handle dry-run mode for adding torrents.""" + print("DRY RUN: Validating magnet links and checking existing torrents...") + if tags: + print(f"DRY RUN: Would create tags: {', '.join(tags)}") + total = len(magnets) + tags_str = ",".join(tags) if tags else None + valid_count = 0 + existing_count = 0 + invalid_count = 0 + + for idx, magnet in enumerate(magnets, 1): + result = self._process_torrent_dry_run(magnet, idx, total, tags_str, category) + if result == "valid": + valid_count += 1 + elif result == "existing": + existing_count += 1 + else: + invalid_count += 1 + time.sleep(0.1) + + print(f"DRY RUN: Summary - {valid_count} would be added, {existing_count} already exist, {invalid_count} invalid") + + def _add_torrents_execute(self, magnets, download_folder, tags, category): + """Execute adding torrents (non-dry-run mode).""" if tags: self.client.torrents_create_tags(tags=",".join(tags)) @@ -113,7 +121,7 @@ def add_torrents(self, magnets, download_folder=None, tags=None, category=None, tags_str = ",".join(tags) if tags else None for idx, magnet in enumerate(magnets, 1): truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") - print(f"Processing {idx}/{total}: {truncated}") + print(f"Processing torrent {idx}/{total}: {truncated}") info_hash = self._extract_info_hash(magnet) if not info_hash: @@ -129,6 +137,12 @@ def add_torrents(self, magnets, download_folder=None, tags=None, category=None, time.sleep(0.1) print(f"Added {added_count} new torrents to qBittorrent.") + def add_torrents(self, magnets, download_folder=None, tags=None, category=None, dry_run=False): + if dry_run: + self._add_torrents_dry_run(magnets, tags, category) + else: + self._add_torrents_execute(magnets, download_folder, tags, category) + class TransmissionClient(Client): def __init__(self, host, port, username, password): @@ -196,43 +210,48 @@ def _add_single_torrent(self, magnet, download_folder, truncated): print(f"Failed to add torrent: {truncated} Error: {e}") return False - def add_torrents(self, magnets, download_folder=None, tags=None, category=None, dry_run=False): - if dry_run: - print("DRY RUN: Validating magnet links...") - if tags or category: - print("DRY RUN: Warning - Transmission does not support tags or categories through this script.") - - total = len(magnets) - valid_count = 0 - invalid_count = 0 - - for idx, magnet in enumerate(magnets, 1): - truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") - print(f"DRY RUN: Processing {idx}/{total}: {truncated}") - - # Validate magnet link format - if not magnet.startswith("magnet:?"): - print(f"DRY RUN: Invalid magnet link format: {truncated}") - invalid_count += 1 - continue - - # Check if we can parse the magnet link (basic validation) - if "xt=urn:btih:" not in magnet: - print(f"DRY RUN: Magnet link missing info hash: {truncated}") - invalid_count += 1 - continue - - print(f"DRY RUN: Would add torrent: {truncated}") - if download_folder: - print(f"DRY RUN: Download folder: {download_folder}") + def _validate_magnet_link(self, magnet): + """Validate a magnet link format.""" + if not magnet.startswith("magnet:?"): + return False + if "xt=urn:btih:" not in magnet: + return False + return True + + def _process_torrent_dry_run(self, magnet, idx, total): + """Process a single torrent in dry-run mode.""" + truncated = magnet[:50] + ("..." if len(magnet) > 50 else "") + print(f"DRY RUN: Processing {idx}/{total}: {truncated}") + + if not self._validate_magnet_link(magnet): + if not magnet.startswith("magnet:?"): + print(f"DRY RUN: Invalid magnet link format: {truncated}") + else: + print(f"DRY RUN: Magnet link missing info hash: {truncated}") + return False + + print(f"DRY RUN: Would add torrent: {truncated}") + return True + + def _add_torrents_dry_run(self, magnets): + """Handle dry-run mode for adding torrents.""" + print("DRY RUN: Validating magnet links...") + + total = len(magnets) + valid_count = 0 + invalid_count = 0 + + for idx, magnet in enumerate(magnets, 1): + if self._process_torrent_dry_run(magnet, idx, total): valid_count += 1 - time.sleep(0.1) - - print(f"DRY RUN: Summary - {valid_count} valid magnet links would be added, {invalid_count} invalid") - return + else: + invalid_count += 1 + time.sleep(0.1) - if tags or category: - print("Warning: Transmission does not support tags or categories through this script.") + print(f"DRY RUN: Summary - {valid_count} valid magnet links would be added, {invalid_count} invalid") + + def _add_torrents_execute(self, magnets, download_folder): + """Execute adding torrents (non-dry-run mode).""" added_count = 0 total = len(magnets) for idx, magnet in enumerate(magnets, 1): @@ -243,6 +262,17 @@ def add_torrents(self, magnets, download_folder=None, tags=None, category=None, time.sleep(0.1) print(f"Added {added_count} torrents to Transmission.") + def add_torrents(self, magnets, download_folder=None, tags=None, category=None, dry_run=False): + if tags or category: + warning_msg = "DRY RUN: Warning - " if dry_run else "Warning: " + warning_msg += "Transmission does not support tags or categories through this script." + print(warning_msg) + + if dry_run: + self._add_torrents_dry_run(magnets) + else: + self._add_torrents_execute(magnets, download_folder) + def get_client(client_name, host, port, username, password): if client_name == 'qbittorrent': diff --git a/coverage.xml b/coverage.xml index 193b28f..e2088e1 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -285,830 +285,857 @@ - - + + + + - - + + - - - - - - + + + + + - - - + + - - - - + + + - + - + - + + - - - - - - - - - + + + + + + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - + + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - + - - - - + + + - - - - - - + + + + + - - - - - - - - - - + + + + - - + + + + + + - - + + + + - - - - - + + + + + - - - - - - - - - - - - + + + + + + + + + - - - - - + + + + + - - - - - + + + + + - - + - - - + + + + + + - - - - - + + + - - - + - - - + + + - + + - - - - - - + + + - + + + - - - - + + + + + + + - + + - - - - - - - - + + + + + + + - + + - - - - - - - + + + + + + + + + - - - - + - - + - - - - - - + + + + + + + + - - - - + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + - - - - - - + + + - - + + + - + + + - + - - - - - + + + + + + - + + + - + - - - + + - - - - - - - + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - + - - - - - - - - - + + + + + + + + - - - - - - + + + + + - - - + + + + + + - - - + + + + + + - - - - - - - - - + + + + + + + - - - - - - + + + - - - - - - - - - + + + + + + + + + + + + - + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + - - - - + - - + - - - - - - - - - + + + + + + + + - - - - - - - + + + + + + + + + - - - + + + - - - - - + + + + + - + + + - - - + + + + + - + - - - + + + + + - - - - + + + + - - - - + + + + - + + + - - - - - + + + - - - - + + + + + - - + + + - - - - - + + + - - + - - - + + - - - - + + + + + - - - + + + - - - + + + - + + - - - - - - + + + - + + - - - - + + + - - + + + + + + + - - - - - + + - - - + + + + + + + + - + + + - - + + + + + + + - - - - - + + + - - - - - - + + + + + + + + + + - - - - - + + + + + + + + + + + - - - - - - + + + + - - - - - + + + + - - - - - - - - - - - - + + + + + - - - - - + + + + + - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -1151,159 +1178,174 @@ - - - - + + + - + - + + - - + + - - - - + + + - - + - - - + + + - - + + - + - - - - - - + + + + + - - - + + + - + + - - + - + + - - - - + + + - + - + - - + + - - - - - + + + + + - - - - - - + + + + + - + - - + - - + + - - - - - - - + + + + + + - - - + + + + - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - + - - - + + - - - + - + - - - - + + + + - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index 3cfaeef..39d86db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,7 @@ services: volumes: - /path/to/OnePaceLibrary:/media:rw - /path/to/config:/config:rw - # networks: - # - "proxy" + # network_mode: host environment: - TZ=Europe/London # Export database to CSV on container start (default: false) @@ -36,7 +35,3 @@ services: #- TORRENT_PASSWORD=password # Enable debug output for troubleshooting (default: false) #- DEBUG=true - -# networks: -# proxy: -# driver: bridge From be609758b436c935c36b2c8303b4b27fd15446cb Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 28 Jan 2026 13:10:05 +0000 Subject: [PATCH 68/75] Fixing logic --- acepace.py | 29 ++++++++++++++++++++--------- clients.py | 1 - 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/acepace.py b/acepace.py index d590abe..19224ff 100644 --- a/acepace.py +++ b/acepace.py @@ -607,27 +607,31 @@ def fetch_magnet_links_for_episodes_from_search(base_url, crc32_to_link): if not crc32_set: return crc32_to_magnet - # Get total number of pages - soup, success = _fetch_crc32_page(base_url, 1) - if not success: + # Get total number of pages (fetch page 1 silently first to get total pages) + resp = requests.get(f"{base_url}&p=1") + if resp.status_code != HTTP_OK: return crc32_to_magnet - + soup = BeautifulSoup(resp.text, HTML_PARSER) total_pages = _get_total_pages(soup) print(f"Fetching magnet links from {total_pages} pages...") # Process pages to extract magnet links for episodes we need + # Continue searching until we've found all requested episodes or searched all pages page = 1 - found_count = 0 - while page <= total_pages and found_count < len(crc32_set): + while page <= total_pages and len(crc32_to_magnet) < len(crc32_set): if _shutdown_requested: break - page_soup, success = _get_page_soup_for_magnet_links(base_url, page, soup) + if page == 1: + page_soup = soup + success = True + else: + page_soup, success = _get_page_soup_for_magnet_links(base_url, page, soup) + if not success or page_soup is None: break - page_found = _process_magnet_links_page(page_soup, crc32_set, crc32_to_magnet) - found_count += page_found + _process_magnet_links_page(page_soup, crc32_set, crc32_to_magnet) page += 1 if page <= total_pages: @@ -1772,6 +1776,13 @@ def _calculate_and_find_missing(folder, conn, args, last_run): else: print("All missing episodes already have magnet links in CSV.") crc32_to_magnet = existing_magnets + + # Restrict to episodes we have magnet links for (matches previous behavior) + # This ensures we only count episodes that can actually be downloaded + crc32_to_link = {c: crc32_to_link[c] for c in crc32_to_magnet if c in crc32_to_link} + crc32_to_text = {c: crc32_to_text.get(c, f"[CRC32: {c}]") for c in crc32_to_magnet} + # Recalculate missing episodes based on episodes with magnet links + missing = [crc32 for crc32 in missing if crc32 in crc32_to_magnet] print( f"\nSummary: {len(missing)} missing episodes out of {len(crc32_to_link)} total found on Nyaa.\n" diff --git a/clients.py b/clients.py index 4c1da6f..69b16c3 100644 --- a/clients.py +++ b/clients.py @@ -230,7 +230,6 @@ def _process_torrent_dry_run(self, magnet, idx, total): print(f"DRY RUN: Magnet link missing info hash: {truncated}") return False - print(f"DRY RUN: Would add torrent: {truncated}") return True def _add_torrents_dry_run(self, magnets): From 87e974737e5b808227913499bbecbfc2726aefcf Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 28 Jan 2026 16:03:02 +0000 Subject: [PATCH 69/75] Optimising logic, might break things --- README.md | 3 +- acepace.py | 283 ++--- clients.py | 11 +- coverage.xml | 1948 +++++++++++++++++---------------- episodes_index.db | Bin 151552 -> 151552 bytes tests/conftest.py | 6 +- tests/test_database.py | 44 +- tests/test_episodes.py | 47 +- tests/test_file_operations.py | 4 +- 9 files changed, 1251 insertions(+), 1095 deletions(-) diff --git a/README.md b/README.md index 6d8297a..724e5f6 100644 --- a/README.md +++ b/README.md @@ -97,12 +97,13 @@ The following volumes should be mounted for persistent data: - `/media` - Mount your One-Pace library directory here (read-write) - `/config` - Mount a directory for persistent configuration and data files (read-write) - Contains: `crc32_files.db`, `episodes_index.db`, `Ace-Pace_Missing.csv`, `Ace-Pace_DB.csv` + - `episodes_index.db` now stores magnet links for all episodes, reducing the need to fetch them repeatedly ### Docker Execution Flow When the container starts, it executes the following steps in order: -1. **Episodes Update** (if `EPISODES_UPDATE=true`): Updates the episodes metadata database from Nyaa +1. **Episodes Update** (if `EPISODES_UPDATE=true`): Updates the episodes metadata database from Nyaa, including magnet links for all episodes 2. **Database Export** (if `DB=true`): Exports the CRC32 database to CSV 3. **Missing Episodes Report**: Always runs to generate/update `Ace-Pace_Missing.csv` 4. **Download** (if `DOWNLOAD=true`): Automatically downloads missing episodes via the configured BitTorrent client diff --git a/acepace.py b/acepace.py index 19224ff..d960f29 100644 --- a/acepace.py +++ b/acepace.py @@ -68,6 +68,9 @@ def _signal_handler(signum, frame): EPISODES_DB_NAME = "episodes_index.db" MISSING_CSV_FILENAME = "Ace-Pace_Missing.csv" DB_CSV_FILENAME = "Ace-Pace_DB.csv" +CSV_COLUMN_MAGNET_LINK = "Magnet Link" +CSV_COLUMN_TITLE = "Title" +CSV_COLUMN_PAGE_LINK = "Page Link" def get_config_dir(): @@ -157,10 +160,17 @@ def init_episodes_db(): CREATE TABLE IF NOT EXISTS episodes_index ( crc32 TEXT PRIMARY KEY, title TEXT, - page_link TEXT + page_link TEXT, + magnet_link TEXT ) """ ) + # Add magnet_link column if it doesn't exist (for existing databases) + try: + c.execute("ALTER TABLE episodes_index ADD COLUMN magnet_link TEXT") + except sqlite3.OperationalError: + # Column already exists, ignore + pass c.execute( """ CREATE TABLE IF NOT EXISTS metadata ( @@ -213,7 +223,7 @@ def _is_valid_quality(fname_text): return False # Quality not 1080p -def _process_fname_entry(fname_text, seen_crc32, episodes, page_link): +def _process_fname_entry(fname_text, seen_crc32, episodes, page_link, magnet_link=""): """Helper to extract CRC32 from fname_text and store if valid and unique. Only accepts episodes with 1080p quality.""" m = CRC32_REGEX.findall(fname_text) @@ -222,7 +232,7 @@ def _process_fname_entry(fname_text, seen_crc32, episodes, page_link): crc32 = m[-1].upper() if crc32 not in seen_crc32: # print(f"New CRC32 detected: {crc32} -> Title: {fname_text}") - episodes.append((crc32, fname_text, page_link)) + episodes.append((crc32, fname_text, page_link, magnet_link)) seen_crc32.add(crc32) found = True return found @@ -293,8 +303,9 @@ def _extract_filenames_from_torrent_page(torrent_soup): return [] -def _process_torrent_page(page_link, seen_crc32, episodes): - """Process a torrent page to extract CRC32 information from file list.""" +def _process_torrent_page(page_link, seen_crc32, episodes, magnet_link=""): + """Process a torrent page to extract CRC32 information from file list. + For grouped episodes, all episodes in the group share the same magnet_link.""" try: torrent_resp = requests.get(page_link) if torrent_resp.status_code != HTTP_OK: @@ -304,7 +315,7 @@ def _process_torrent_page(page_link, seen_crc32, episodes): filenames = _extract_filenames_from_torrent_page(t_soup) found = False for fname in filenames: - if _process_fname_entry(str(fname), seen_crc32, episodes, page_link): + if _process_fname_entry(str(fname), seen_crc32, episodes, page_link, magnet_link): found = True return found except (requests.RequestException, AttributeError, TypeError): @@ -313,7 +324,7 @@ def _process_torrent_page(page_link, seen_crc32, episodes): def _process_episode_row(row, seen_crc32, episodes): """Process a single table row to extract episode information.""" - title_link = _extract_title_link_from_row(row) + title_link, magnet_link = _extract_links_from_row(row) if not title_link: return False @@ -322,9 +333,11 @@ def _process_episode_row(row, seen_crc32, episodes): matches = CRC32_REGEX.findall(title) if matches: - return _process_fname_entry(title, seen_crc32, episodes, page_link) + return _process_fname_entry(title, seen_crc32, episodes, page_link, magnet_link or "") else: - return _process_torrent_page(page_link, seen_crc32, episodes) + # CRC32 not in title, need to visit torrent page + # The magnet_link from the row applies to all episodes in the group + return _process_torrent_page(page_link, seen_crc32, episodes, magnet_link or "") def _fetch_episodes_page(base_url, page, soup=None): @@ -354,12 +367,13 @@ def _process_episodes_page_rows(page_soup, seen_crc32, episodes): def fetch_episodes_metadata(base_url=None): """ - Fetch all One Pace episodes from Nyaa, collecting CRC32, title, and page link. + Fetch all One Pace episodes from Nyaa, collecting CRC32, title, page link, and magnet link. If CRC32 not in title, fetch the torrent page and try to extract CRC32s from file list. + For grouped episodes (multiple episodes in one torrent), all episodes share the same magnet link. Args: base_url: Base URL for Nyaa search. If None, uses default without quality filter. Note: Quality filtering (1080p only) is always applied regardless of URL. - Returns: List of (crc32, title, page_link) + Returns: List of (crc32, title, page_link, magnet_link) """ if base_url is None: base_url = f"{NYAA_BASE_URL}/?f=0&c=0_0&q=one+pace" @@ -399,6 +413,33 @@ def fetch_episodes_metadata(base_url=None): return episodes +def _should_skip_episodes_update(force_update, last_update_str): + """Check if episodes update should be skipped due to recent update. + Args: + force_update: If True, never skip + last_update_str: String timestamp of last update, or None + Returns: True if should skip, False if should proceed""" + if force_update: + return False + + if not last_update_str: + return False + + try: + last_update = datetime.strptime(last_update_str, "%Y-%m-%d %H:%M:%S") + time_diff = datetime.now() - last_update + # Skip update if updated within last 10 minutes to avoid unnecessary double updates + if time_diff.total_seconds() < 600: # 10 minutes = 600 seconds + print(f"Episodes were recently updated ({last_update_str}), skipping update to avoid duplicate fetch.") + print("Set EPISODES_UPDATE=true or use --episodes_update to force update.") + return True + except (ValueError, TypeError): + # If parsing fails, proceed with update + pass + + return False + + def update_episodes_index_db(base_url=None, force_update=False): """Update episodes index database from Nyaa. Args: @@ -408,40 +449,40 @@ def update_episodes_index_db(base_url=None, force_update=False): debug_print(f"DEBUG: Starting update_episodes_index_db with URL: {base_url}, force_update: {force_update}") # Check if episodes were recently updated (within last 10 minutes) + conn = init_episodes_db() if not force_update: - conn_check = init_episodes_db() - last_update_str = get_episodes_metadata(conn_check, "episodes_db_last_update") - conn_check.close() + last_update_str = get_episodes_metadata(conn, "episodes_db_last_update") - if last_update_str: - try: - last_update = datetime.strptime(last_update_str, "%Y-%m-%d %H:%M:%S") - time_diff = datetime.now() - last_update - # Skip update if updated within last 10 minutes to avoid unnecessary double updates - if time_diff.total_seconds() < 600: # 10 minutes = 600 seconds - print(f"Episodes were recently updated ({last_update_str}), skipping update to avoid duplicate fetch.") - print("Set EPISODES_UPDATE=true or use --episodes_update to force update.") - return - except (ValueError, TypeError): - # If parsing fails, proceed with update - pass - - conn = init_episodes_db() + if _should_skip_episodes_update(force_update, last_update_str): + conn.close() + return episodes = fetch_episodes_metadata(base_url) debug_print(f"DEBUG: Fetched {len(episodes)} episodes from Nyaa") c = conn.cursor() - count = 0 - for crc32, title, page_link in episodes: + + # Prepare data for batch insert (allowing for shutdown during processing) + episode_rows = [] + for episode_data in episodes: # Check for shutdown request during processing if _shutdown_requested: print("Shutdown requested, committing partial update...") break - c.execute( - "INSERT OR REPLACE INTO episodes_index (crc32, title, page_link) VALUES (?, ?, ?)", - (crc32, title, page_link), + # Handle both old format (3 items) and new format (4 items) for backward compatibility + if len(episode_data) == 3: + crc32, title, page_link = episode_data + magnet_link = "" + else: + crc32, title, page_link, magnet_link = episode_data + episode_rows.append((crc32, title, page_link, magnet_link or "")) + + # Batch insert for better performance + if episode_rows: + c.executemany( + "INSERT OR REPLACE INTO episodes_index (crc32, title, page_link, magnet_link) VALUES (?, ?, ?, ?)", + episode_rows ) - count += 1 conn.commit() + count = len(episode_rows) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") set_episodes_metadata(conn, "episodes_db_last_update", now_str) print(f"Episodes index updated with {count} entries.") @@ -463,19 +504,33 @@ def load_crc32_to_title_from_index(): def load_1080p_episodes_from_index(): """Load only 1080p episodes from episodes_index database. - Returns: Tuple of (crc32_to_link, crc32_to_text) dictionaries with only 1080p episodes.""" + Returns: Tuple of (crc32_to_link, crc32_to_text, crc32_to_magnet) dictionaries with only 1080p episodes.""" conn = init_episodes_db() c = conn.cursor() - c.execute("SELECT crc32, title, page_link FROM episodes_index") + # Handle both old schema (without magnet_link) and new schema (with magnet_link) + try: + c.execute("SELECT crc32, title, page_link, magnet_link FROM episodes_index") + has_magnet_column = True + except sqlite3.OperationalError: + # Old schema, magnet_link column doesn't exist yet + c.execute("SELECT crc32, title, page_link FROM episodes_index") + has_magnet_column = False crc32_to_link = {} crc32_to_text = {} - for crc32, title, page_link in c.fetchall(): + crc32_to_magnet = {} + for row in c.fetchall(): + if has_magnet_column: + crc32, title, page_link, magnet_link = row + else: + crc32, title, page_link = row + magnet_link = "" # Only include 1080p episodes (same filter as fetch_crc32_links) if _is_valid_quality(title): crc32_to_link[crc32] = page_link crc32_to_text[crc32] = title + crc32_to_magnet[crc32] = magnet_link or "" conn.close() - return crc32_to_link, crc32_to_text + return crc32_to_link, crc32_to_text, crc32_to_magnet def _validate_row_links(title_link, magnet_link): @@ -1035,7 +1090,7 @@ def _execute_rename(rename_plan, conn): "UPDATE crc32_cache SET file_path = ? WHERE file_path = ?", (normalized_new, normalized_old) ) conn.commit() - except Exception as e: + except (sqlite3.Error, OSError) as e: print(f"Failed to rename {old} to {new}: {e}") @@ -1179,52 +1234,34 @@ def _get_non_docker_connection_params(args): def _load_magnet_links(): - """Load magnet links from the missing CSV file.""" + """Load magnet links from the missing CSV file. + Deduplicates magnet links so grouped episodes (sharing same magnet) are only added once.""" missing_csv_path = get_config_path(MISSING_CSV_FILENAME) if not os.path.exists(missing_csv_path): print(f"Missing file '{missing_csv_path}' not found. Run the script first!") return None - magnets = [] + magnets_set = set() + total_magnets = 0 with open(missing_csv_path, "r", encoding="utf-8") as f: reader = csv.DictReader(f) for row in reader: - magnet_link = row.get("Magnet Link", "").strip() + magnet_link = row.get(CSV_COLUMN_MAGNET_LINK, "").strip() if magnet_link.startswith(MAGNET_LINK_PREFIX): - magnets.append(magnet_link) + total_magnets += 1 + magnets_set.add(magnet_link) - if not magnets: + if not magnets_set: print(f"No magnet links found in '{missing_csv_path}'.") return None - return magnets - - -def _load_existing_magnet_links_from_csv(): - """Load existing magnet links from the missing CSV file, mapping CRC32 to magnet link. - Returns: Dictionary mapping CRC32 to magnet_link""" - missing_csv_path = get_config_path(MISSING_CSV_FILENAME) - crc32_to_magnet = {} - - if not os.path.exists(missing_csv_path): - return crc32_to_magnet + # Convert to list and return + magnets = list(magnets_set) + duplicates = total_magnets - len(magnets_set) + if duplicates > 0: + print(f"Deduplicated {duplicates} duplicate magnet links (grouped episodes share same magnet).") - try: - with open(missing_csv_path, "r", encoding="utf-8") as f: - reader = csv.DictReader(f) - for row in reader: - title = row.get("Title", "").strip() - magnet_link = row.get("Magnet Link", "").strip() - - # Extract CRC32 from title - matches = CRC32_REGEX.findall(title) - if matches and magnet_link.startswith(MAGNET_LINK_PREFIX): - crc32 = matches[-1].upper() - crc32_to_magnet[crc32] = magnet_link - except (IOError, csv.Error, KeyError) as e: - debug_print(f"DEBUG: Error loading existing magnet links from CSV: {e}") - - return crc32_to_magnet + return magnets def _setup_docker_connection(args): @@ -1401,7 +1438,7 @@ def _save_missing_episodes_csv(missing, crc32_to_text, crc32_to_link, crc32_to_m error_count = 0 with open(missing_csv_path, "w", encoding="utf-8", newline="") as f: writer = csv.writer(f, quoting=csv.QUOTE_ALL) - writer.writerow(["Title", "Page Link", "Magnet Link"]) + writer.writerow([CSV_COLUMN_TITLE, CSV_COLUMN_PAGE_LINK, CSV_COLUMN_MAGNET_LINK]) for crc32 in missing: try: title = crc32_to_text.get(crc32, f"[CRC32: {crc32}]") @@ -1409,7 +1446,7 @@ def _save_missing_episodes_csv(missing, crc32_to_text, crc32_to_link, crc32_to_m magnet = crc32_to_magnet.get(crc32, "") writer.writerow([title, page_link, magnet]) saved_count += 1 - except Exception as e: + except (IOError, OSError, csv.Error) as e: error_count += 1 print(f"ERROR: Failed to save missing episode with CRC32 '{crc32}': {e}") # Still write a row with available information @@ -1637,32 +1674,55 @@ def _handle_episodes_update_decision(episodes_update_env, last_update_str, base_ def _load_episodes_from_database(episodes_update_env, base_url, fetch_magnets=True): - """Load episodes from database and optionally fetch magnet links. + """Load episodes from database, including magnet links stored in database. Args: episodes_update_env: True if EPISODES_UPDATE environment variable is set - base_url: Base URL for Nyaa search - fetch_magnets: If True, fetch magnet links for all episodes. If False, return empty dict. + base_url: Base URL for Nyaa search (unused now, kept for compatibility) + fetch_magnets: If True, fetch missing magnet links from Nyaa. If False, use only database. Returns: Tuple of (crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page)""" if episodes_update_env: print("Using episodes index database (EPISODES_UPDATE=true, using updated database)...") else: print("Using episodes index database (EPISODES_UPDATE=false, checking database only)...") - crc32_to_link, crc32_to_text = load_1080p_episodes_from_index() + crc32_to_link, crc32_to_text, crc32_to_magnet = load_1080p_episodes_from_index() print(f"Loaded {len(crc32_to_link)} 1080p episodes from database.") + # Count how many episodes have magnet links in database + episodes_with_magnets_count = sum(1 for m in crc32_to_magnet.values() if m) + print(f"Found {episodes_with_magnets_count} episodes with magnet links in database.") + if fetch_magnets: - print("Fetching magnet links from search results...") - crc32_to_magnet = fetch_magnet_links_for_episodes_from_search(base_url, crc32_to_link) - print(f"Fetched {len(crc32_to_magnet)} magnet links.") - # Restrict to episodes we have magnet links for. The index can include episodes - # discovered via torrent-page visits (CRC32 not in search row title); we only get - # magnets from search rows. Using the full index would overcount "on Nyaa" and - # inflate the missing count. Restricting to crc32_to_magnet matches fetch_crc32_links. - crc32_to_link = {c: crc32_to_link[c] for c in crc32_to_magnet if c in crc32_to_link} - crc32_to_text = {c: crc32_to_text.get(c, f"[CRC32: {c}]") for c in crc32_to_magnet} - else: - crc32_to_magnet = {} + # Find episodes missing magnet links + missing_magnets = {crc32: crc32_to_link[crc32] for crc32 in crc32_to_link + if not crc32_to_magnet.get(crc32)} + if missing_magnets: + print(f"Fetching {len(missing_magnets)} missing magnet links from search results...") + fetched_magnets = fetch_magnet_links_for_episodes_from_search(base_url, missing_magnets) + # Update database magnet links with newly fetched ones + crc32_to_magnet.update(fetched_magnets) + print(f"Fetched {len(fetched_magnets)} new magnet links.") + + # Update database with newly fetched magnet links (batch update for efficiency) + conn = init_episodes_db() + c = conn.cursor() + c.executemany( + "UPDATE episodes_index SET magnet_link = ? WHERE crc32 = ?", + [(magnet_link, crc32) for crc32, magnet_link in fetched_magnets.items()] + ) + conn.commit() + conn.close() + + # Restrict to episodes we have magnet links for (matches previous behavior) + # This ensures we only count episodes that can actually be downloaded + # Filter to only episodes with non-empty magnet links that exist in crc32_to_link + episodes_with_magnets = {c: m for c, m in crc32_to_magnet.items() if m and c in crc32_to_link} + # Update all dictionaries to only include episodes with magnet links + # Note: All keys in episodes_with_magnets are guaranteed to exist in crc32_to_link and crc32_to_text + # since they're loaded together from the same database query + crc32_to_link = {c: crc32_to_link[c] for c in episodes_with_magnets} + crc32_to_text = {c: crc32_to_text[c] for c in episodes_with_magnets} + crc32_to_magnet = episodes_with_magnets return crc32_to_link, crc32_to_text, crc32_to_magnet, 0 @@ -1721,15 +1781,11 @@ def _calculate_and_find_missing(folder, conn, args, last_run): conn_episodes.close() # Load episodes (from database or fetch from Nyaa) - # When using database without EPISODES_UPDATE, don't fetch magnet links yet - # We'll fetch them only for missing episodes after determining which are missing - if use_database and not episodes_update_env: - # Load episodes from database without fetching magnet links - crc32_to_link, crc32_to_text, _, last_checked_page = _load_episodes_from_database(episodes_update_env, args.url, fetch_magnets=False) - crc32_to_magnet = {} - elif use_database: - # EPISODES_UPDATE=true: fetch all magnet links as before - crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = _load_episodes_from_database(episodes_update_env, args.url) + # Magnet links are now stored in the database, so we load them directly + if use_database: + # Load episodes from database, including magnet links + # fetch_magnets=True will fetch any missing magnet links from Nyaa + crc32_to_link, crc32_to_text, crc32_to_magnet, last_checked_page = _load_episodes_from_database(episodes_update_env, args.url, fetch_magnets=True) else: # Normal fetch from Nyaa (only when database doesn't exist and EPISODES_UPDATE=false) print("Fetching episodes metadata from Nyaa...") @@ -1752,37 +1808,10 @@ def _calculate_and_find_missing(folder, conn, args, last_run): debug_print(f"DEBUG: Folder scanned: {folder}") - # Calculate missing episodes + # Calculate missing episodes (only those with magnet links can be downloaded) missing = _calculate_missing_episodes(crc32_to_link, local_crc32s) - - # When using database without EPISODES_UPDATE, only fetch magnet links for missing episodes - if use_database and not episodes_update_env: - # Load existing magnet links from CSV file - existing_magnets = _load_existing_magnet_links_from_csv() - debug_print(f"DEBUG: Loaded {len(existing_magnets)} existing magnet links from CSV") - - # Determine which missing episodes need magnet links - missing_without_magnets = [crc32 for crc32 in missing if crc32 not in existing_magnets] - - if missing_without_magnets: - print(f"Fetching magnet links for {len(missing_without_magnets)} missing episodes without magnet links...") - # Create a subset of crc32_to_link for missing episodes only - missing_crc32_to_link = {crc32: crc32_to_link[crc32] for crc32 in missing_without_magnets if crc32 in crc32_to_link} - # Fetch magnet links only for missing episodes without one - fetched_magnets = fetch_magnet_links_for_episodes_from_search(args.url, missing_crc32_to_link) - # Merge with existing magnets - crc32_to_magnet = {**existing_magnets, **fetched_magnets} - print(f"Fetched {len(fetched_magnets)} new magnet links.") - else: - print("All missing episodes already have magnet links in CSV.") - crc32_to_magnet = existing_magnets - - # Restrict to episodes we have magnet links for (matches previous behavior) - # This ensures we only count episodes that can actually be downloaded - crc32_to_link = {c: crc32_to_link[c] for c in crc32_to_magnet if c in crc32_to_link} - crc32_to_text = {c: crc32_to_text.get(c, f"[CRC32: {c}]") for c in crc32_to_magnet} - # Recalculate missing episodes based on episodes with magnet links - missing = [crc32 for crc32 in missing if crc32 in crc32_to_magnet] + # Filter to only missing episodes that have magnet links (redundant check removed) + missing = [crc32 for crc32 in missing if crc32_to_magnet.get(crc32)] print( f"\nSummary: {len(missing)} missing episodes out of {len(crc32_to_link)} total found on Nyaa.\n" diff --git a/clients.py b/clients.py index 69b16c3..0b45697 100644 --- a/clients.py +++ b/clients.py @@ -5,6 +5,9 @@ import qbittorrentapi # type: ignore import re +# Rate limiting delay between torrent operations (in seconds) +TORRENT_OPERATION_DELAY = 0.1 + class Client(abc.ABC): @abc.abstractmethod def add_torrents(self, magnets, download_folder=None, tags=None, category=None, dry_run=False): @@ -107,7 +110,7 @@ def _add_torrents_dry_run(self, magnets, tags, category): existing_count += 1 else: invalid_count += 1 - time.sleep(0.1) + time.sleep(TORRENT_OPERATION_DELAY) print(f"DRY RUN: Summary - {valid_count} would be added, {existing_count} already exist, {invalid_count} invalid") @@ -134,7 +137,7 @@ def _add_torrents_execute(self, magnets, download_folder, tags, category): else: if self._add_new_torrent(magnet, download_folder, tags_str, category, truncated): added_count += 1 - time.sleep(0.1) + time.sleep(TORRENT_OPERATION_DELAY) print(f"Added {added_count} new torrents to qBittorrent.") def add_torrents(self, magnets, download_folder=None, tags=None, category=None, dry_run=False): @@ -245,7 +248,7 @@ def _add_torrents_dry_run(self, magnets): valid_count += 1 else: invalid_count += 1 - time.sleep(0.1) + time.sleep(TORRENT_OPERATION_DELAY) print(f"DRY RUN: Summary - {valid_count} valid magnet links would be added, {invalid_count} invalid") @@ -258,7 +261,7 @@ def _add_torrents_execute(self, magnets, download_folder): print(f"Adding {idx}/{total}: {truncated}") if self._add_single_torrent(magnet, download_folder, truncated): added_count += 1 - time.sleep(0.1) + time.sleep(TORRENT_OPERATION_DELAY) print(f"Added {added_count} torrents to Transmission.") def add_torrents(self, magnets, download_folder=None, tags=None, category=None, dry_run=False): diff --git a/coverage.xml b/coverage.xml index e2088e1..8a41086 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -51,1088 +51,1140 @@ + + - - - - - - + + + + + + - - + + - - - - - - - - + + + + + + - - - - + + + + - - - - + + + + - - + + + + + + - + - - - + + + - + - - - - - + - - - + + + + - - + - - - - - - + + + + + + - - - + - - - - + + + - + + + + - - - + + + - + - - - - + + + + - - + - - - + + + + - - - - - - - - + + + + + + + - - - + + + + - - - - - - - - - + + + + + - - + + + + + - - + + + - - - + + - - - + + + + + + - - - - - - - - - - + + + + + - - + + + + + + + + + - - - - - - - - - - - - - + + + + + - - - + + + + - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - + + - - - - - - - - - - - + + + + + + + + + + - + - - - - - - - - - - + + + + + + - + + - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + - - - - - - - + + + + + + + - - - + + + + + - - - - + + + + - - - - - - - - - + + + + + + + + + + + + - - - - - - + + + + + + - + - + + - - - - - + + + + + + + + + - - - - - + + + + - - - - - - + - - - - - + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + + + - + - - - - + + + + + + + + - - - - - - + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + - - + + + + - - - - + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - + + + + + + + + + + + + - - - - - - - - - + + - - + + + + + - - - - + + - - - - - - + + + + + - - + - - + + + - - - - - + + + + + - + - + + - + + + - - - - - + + + + + - + + + - + - + - + - - - - - + + + + + + - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + - - - - - - - - + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + + - + - + + - - - - - + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + + + - - - + + + - - + + - - - - - - + + + + + + + + - + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - + - + + - - - - - - - - - - - + + + + + + - - - - - - - - + + + - - - - - + + + - - + + + + - - - + + + + + + + + + + + + - - - - - + + + + + + + - - - - + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - + + + + + + + - - - - - - + + + + - - - - - + + + - - - + + + - - - - - + + + + + + + - + - - - - - - - - + + + + + + - - - - - - - - - - - + - - - - + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - + + + + - - - - - + + + + + + - - + - + - + + + + + + - + - - - + + + + - - - - + + + + + - + - - + + - - + + - - - + + + + - - - + - - + + - - - - - - - - + + + + + + + + - - + + - - - + - - - - + + + + + - - + - - - - - - + + + + - - - + + + + + + + - - - - + + + + + - - - - - + + + + + + + - - - - + + + + + - - + + + + + + + + - - + + + + - - - + + + + - - - - + + + + + + + + + - - - - - - + + + + + + - + - + - - + + + + - - + + + + - + - - + - - - - - - - + + + + - + - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - + + + + + + + + + - - - - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1144,208 +1196,208 @@ - - - - - - - + + + + + - - + + - - - - + + + + + - - - + + + - + - - - - - - - + + + + + + + - - + + - - - + + + - - - - + + + - - - - - - - - + + + + + + + + + - - + + - - - - - + + + + + - - + + - + - - + + - - + + - - - + + + - - + + - - - + + + + - - + + - - - + - - - - + + + + - + + + - - - - - + + + - - + + + + - - - - + + - + + - - - + + - + - + - - - + + + - - + + + - - + - - + + + - - - - + + + + - + - - - - + + + + - + - - + - - + + - + + - - - + + - - + + + - - + + + diff --git a/episodes_index.db b/episodes_index.db index 47b56e3943f20a7e34b3382932367600759e75ef..b61028c44f1c48d8961b8ed2da8c00e8d0800e01 100644 GIT binary patch delta 109 zcmZozz}c{XbAq&>AOiz~DiCu6F*68H)G_84WYFvV!3(5Vcvmy00RSqDiCu6F*6V|P1G^w7hupU>*nSE!N9_o&cH9YG0&EF;)Feu m)A?3$HL5eRi)(5!HgQhg$~&J?vw1gv`)+>5?YsGzTpa)e))cD% diff --git a/tests/conftest.py b/tests/conftest.py index ddd15c1..0227402 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,9 +43,9 @@ def sample_crc32(): def sample_episode_data(): """Sample episode data for testing.""" return [ - ("A1B2C3D4", "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv", "https://nyaa.si/view/12345"), - ("E5F6A7B8", "[One Pace] Episode 2 [1080p][E5F6A7B8].mkv", "https://nyaa.si/view/12346"), - ("A9B0C1D2", "[One Pace] Episode 3 [1080p][A9B0C1D2].mkv", "https://nyaa.si/view/12347"), + ("A1B2C3D4", "[One Pace] Episode 1 [1080p][A1B2C3D4].mkv", "https://nyaa.si/view/12345", "magnet:?xt=urn:btih:abc123"), + ("E5F6A7B8", "[One Pace] Episode 2 [1080p][E5F6A7B8].mkv", "https://nyaa.si/view/12346", "magnet:?xt=urn:btih:def456"), + ("A9B0C1D2", "[One Pace] Episode 3 [1080p][A9B0C1D2].mkv", "https://nyaa.si/view/12347", "magnet:?xt=urn:btih:ghi789"), ] diff --git a/tests/test_database.py b/tests/test_database.py index ca5f1ab..e8b229d 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -145,11 +145,16 @@ def test_load_crc32_to_title_from_index(self, temp_dir, sample_episode_data): conn = acepace.init_episodes_db() cursor = conn.cursor() - # Insert sample data - for crc32, title, page_link in sample_episode_data: + # Insert sample data (handle both 3-item and 4-item formats for backward compatibility) + for episode_data in sample_episode_data: + if len(episode_data) == 3: + crc32, title, page_link = episode_data + magnet_link = "" + else: + crc32, title, page_link, magnet_link = episode_data cursor.execute( - "INSERT INTO episodes_index (crc32, title, page_link) VALUES (?, ?, ?)", - (crc32, title, page_link) + "INSERT INTO episodes_index (crc32, title, page_link, magnet_link) VALUES (?, ?, ?, ?)", + (crc32, title, page_link, magnet_link) ) conn.commit() conn.close() @@ -162,3 +167,34 @@ def test_load_crc32_to_title_from_index(self, temp_dir, sample_episode_data): assert mapping["A9B0C1D2"] == "[One Pace] Episode 3 [1080p][A9B0C1D2].mkv" os.remove(os.path.join(temp_dir, 'test.db')) + + def test_load_1080p_episodes_from_index_includes_magnet_links(self, temp_dir, sample_episode_data): + """Test loading 1080p episodes from index includes magnet links.""" + with patch('acepace.EPISODES_DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_episodes_db() + cursor = conn.cursor() + + # Insert sample data with magnet links + for episode_data in sample_episode_data: + if len(episode_data) == 3: + crc32, title, page_link = episode_data + magnet_link = "" + else: + crc32, title, page_link, magnet_link = episode_data + cursor.execute( + "INSERT INTO episodes_index (crc32, title, page_link, magnet_link) VALUES (?, ?, ?, ?)", + (crc32, title, page_link, magnet_link) + ) + conn.commit() + conn.close() + + # Load and verify + crc32_to_link, crc32_to_text, crc32_to_magnet = acepace.load_1080p_episodes_from_index() + assert len(crc32_to_link) == 3 + assert len(crc32_to_text) == 3 + assert len(crc32_to_magnet) == 3 + assert crc32_to_link["A1B2C3D4"] == "https://nyaa.si/view/12345" + assert crc32_to_magnet["A1B2C3D4"] == "magnet:?xt=urn:btih:abc123" + assert crc32_to_magnet["E5F6A7B8"] == "magnet:?xt=urn:btih:def456" + + os.remove(os.path.join(temp_dir, 'test.db')) diff --git a/tests/test_episodes.py b/tests/test_episodes.py index 17fb32b..63ca9d3 100644 --- a/tests/test_episodes.py +++ b/tests/test_episodes.py @@ -175,6 +175,41 @@ def test_fetch_episodes_metadata_crc32_in_title(self, mock_get): assert len(episodes) == 1 assert episodes[0][0] == "A1B2C3D4" assert "[One Pace]" in episodes[0][1] + # Verify magnet link is included (4th element) + assert len(episodes[0]) == 4 + assert episodes[0][3] == "" # No magnet link in this test HTML + + @patch('acepace.requests.get') + def test_fetch_episodes_metadata_extracts_magnet_links(self, mock_get): + """Test that magnet links are extracted from search rows.""" + html = """ + + + + + + +
+ [One Pace] Episode 1 [1080p][A1B2C3D4].mkv + Magnet +
+
    +
  • 1
    • +
+ + + """ + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + mock_get.return_value = mock_response + + episodes = acepace.fetch_episodes_metadata() + + assert len(episodes) == 1 + crc32, _, _, magnet_link = episodes[0] + assert crc32 == "A1B2C3D4" + assert magnet_link == "magnet:?xt=urn:btih:test123456789" @patch('acepace.requests.get') def test_fetch_episodes_metadata_crc32_from_file_list(self, mock_get, mock_nyaa_torrent_page): @@ -320,7 +355,7 @@ def test_fetch_episodes_prefers_1080p(self, mock_get): # All episodes should be 1080p assert len(episodes) == 2 - for crc32, title, _ in episodes: + for crc32, title, _, _ in episodes: assert "[1080p]" in title.upper() or "1080P" in title.upper() assert "[One Pace]" in title @@ -366,7 +401,7 @@ def test_fetch_episodes_only_accepts_1080p_same_episode(self, mock_get): # Should only have one entry (1080p version, 720p is rejected) assert len(episodes) == 1 - crc32, title, _ = episodes[0] + crc32, title, _, _ = episodes[0] assert crc32 == "A1B2C3D4" # Only 1080p should be kept assert "[1080p]" in title.upper() or "1080P" in title.upper() @@ -387,7 +422,7 @@ def test_fetch_episodes_mixed_qualities_only_keeps_1080p(self, mock_get): # Should only have 1080p episodes (2 total, excluding 720p and 480p) assert len(episodes) == 2 - for crc32, title, _ in episodes: + for crc32, title, _, _ in episodes: title_upper = title.upper() has_1080p = "[1080P]" in title_upper or "1080P" in title_upper assert has_1080p, f"Episode {title} should be 1080p" @@ -410,7 +445,7 @@ def test_fetch_episodes_handles_case_insensitive_quality(self, mock_get): # Should only accept 1080P (case-insensitive), reject 720P assert len(episodes) == 1 - crc32, title, _ = episodes[0] + crc32, title, _, _ = episodes[0] assert crc32 == "A1B2C3D4" # Verify case-insensitive quality detection works for 1080p title_upper = title.upper() @@ -451,7 +486,7 @@ def test_fetch_episodes_excludes_episodes_without_quality_marker(self, mock_get) # Should only include episode with quality marker assert len(episodes) == 1 - crc32, title, _ = episodes[0] + crc32, title, _, _ = episodes[0] assert crc32 == "E5F6A7B8" assert "[1080p]" in title.upper() or "1080P" in title.upper() @@ -504,7 +539,7 @@ def test_fetch_episodes_quality_filtering_from_file_list(self, mock_sleep, mock_ # Should extract 1080p episode from file list assert len(episodes) == 1 - crc32, title, _ = episodes[0] + crc32, title, _, _ = episodes[0] assert crc32 == "A1B2C3D4" assert "[1080p]" in title.upper() or "1080P" in title.upper() diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index b5d9244..efa9f0d 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -33,8 +33,8 @@ def test_rename_local_files_matches_by_crc32(self, temp_dir, sample_episode_data episodes_conn = acepace.init_episodes_db() cursor = episodes_conn.cursor() cursor.execute( - "INSERT INTO episodes_index (crc32, title, page_link) VALUES (?, ?, ?)", - (actual_crc32, "[One Pace] Episode 1 [1080p].mkv", "https://nyaa.si/view/12345") + "INSERT INTO episodes_index (crc32, title, page_link, magnet_link) VALUES (?, ?, ?, ?)", + (actual_crc32, "[One Pace] Episode 1 [1080p].mkv", "https://nyaa.si/view/12345", "") ) episodes_conn.commit() episodes_conn.close() From 9c758ff80e6546f8310dc5de0903ac739c8c8cf4 Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 28 Jan 2026 16:16:19 +0000 Subject: [PATCH 70/75] Double checked logic, optimising --- acepace.py | 8 +- clients.py | 14 +- coverage.xml | 1683 +++++++++++++++++++++++++------------------------- 3 files changed, 859 insertions(+), 846 deletions(-) diff --git a/acepace.py b/acepace.py index d960f29..f9e9579 100644 --- a/acepace.py +++ b/acepace.py @@ -69,8 +69,6 @@ def _signal_handler(signum, frame): MISSING_CSV_FILENAME = "Ace-Pace_Missing.csv" DB_CSV_FILENAME = "Ace-Pace_DB.csv" CSV_COLUMN_MAGNET_LINK = "Magnet Link" -CSV_COLUMN_TITLE = "Title" -CSV_COLUMN_PAGE_LINK = "Page Link" def get_config_dir(): @@ -1255,8 +1253,8 @@ def _load_magnet_links(): print(f"No magnet links found in '{missing_csv_path}'.") return None - # Convert to list and return - magnets = list(magnets_set) + # Convert to list and return (sorted for consistent ordering) + magnets = sorted(list(magnets_set)) duplicates = total_magnets - len(magnets_set) if duplicates > 0: print(f"Deduplicated {duplicates} duplicate magnet links (grouped episodes share same magnet).") @@ -1438,7 +1436,7 @@ def _save_missing_episodes_csv(missing, crc32_to_text, crc32_to_link, crc32_to_m error_count = 0 with open(missing_csv_path, "w", encoding="utf-8", newline="") as f: writer = csv.writer(f, quoting=csv.QUOTE_ALL) - writer.writerow([CSV_COLUMN_TITLE, CSV_COLUMN_PAGE_LINK, CSV_COLUMN_MAGNET_LINK]) + writer.writerow(["Title", "Page Link", CSV_COLUMN_MAGNET_LINK]) for crc32 in missing: try: title = crc32_to_text.get(crc32, f"[CRC32: {crc32}]") diff --git a/clients.py b/clients.py index 0b45697..a65c447 100644 --- a/clients.py +++ b/clients.py @@ -237,13 +237,23 @@ def _process_torrent_dry_run(self, magnet, idx, total): def _add_torrents_dry_run(self, magnets): """Handle dry-run mode for adding torrents.""" + if not magnets: + print("DRY RUN: No magnet links to process.") + return + print("DRY RUN: Validating magnet links...") - total = len(magnets) + # Filter out empty/None magnets before processing to ensure consistent indexing + valid_magnets = [m for m in magnets if m and m.strip()] + if len(valid_magnets) != len(magnets): + skipped = len(magnets) - len(valid_magnets) + print(f"DRY RUN: Warning - {skipped} empty or invalid magnet link(s) skipped.") + + total = len(valid_magnets) valid_count = 0 invalid_count = 0 - for idx, magnet in enumerate(magnets, 1): + for idx, magnet in enumerate(valid_magnets, 1): if self._process_torrent_dry_run(magnet, idx, total): valid_count += 1 else: diff --git a/coverage.xml b/coverage.xml index 8a41086..8e2c0d4 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -52,762 +52,762 @@ - - - - - - - - - - - - - + + + + + + + + + + + + - + - - + + + - - - + + + - - - - + + + + - - + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - + + - - - - - + + + + + - - + + - - + - + + - - + + - + - - - - - - - - - + + + + + + + + + - + - - - - - - - - - - + + + + + + + + + + - - - + + - - - - - - + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - - - - - - - + + + + + + + - - - + + + + - - - - - - - + + + + + + + - + - - + - - - - - - - - - - + + + + + + + + + + + - - + + - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - + + + + + + + + - - - + + + - - - - + + + + - - - - - + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - + + + - - - - + + + + - - + - - + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - + + + + - - - - - + + + + + - - + + - + - + - + - - - + + + - - + + - - + + - - - - - - + + + + + + - - - - - + + + + + - - + - - - - - - - - - - + + + + + + + + + + + - + - - - - - - - - + + + + + + + - - + + + - - - + + + - - - - - - - + + + + + + + - - - - - + + + + + - - - - + + + + - + - - - - - + + + + + - - + + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + + - - + + - + - - + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - + - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - - + + - - - - - + + + + + - - - - - + + + + + - - - - - - - - - + + + + + + + + + - - + + - + - - - - + + + + - - - - - - - - - + + + + + + + + - - + + + - + - - + + - + - - - - - - - - + + + + + + + + - + - + - - + - + + - - - + + + - + - - - - + + + + - - - - + + + + - + - - + + - - + - - + + + - + - + - - - - - + + + + + - - - + + + - - - - + + + + - + - + - - - - - - - + + + + + + - - - - - + + + + + + - - + + - - + - - - - + + + + + - - - + + + - - + + - + - + - - - - - - - + + + + + + + - - + + - + - + - - - + + - - - - + + + + + - - - + + + - - - - - - - - - - + + + + + + + + + - - - + + + - - - - + + + + + - - + + @@ -817,62 +817,62 @@ - - - - - - - - + + + + + + + - - - - + + + + - - - - - + + + + + + - - - + + + - - + - + + - - - - - + + + + + - + - - - - + + + + - - - - + + + + - + - + @@ -880,314 +880,312 @@ - + - + - - - - + + + + - + - - - + + + - + - - - + + + - - - + + + - - - + + + - + - + - - - - + + + + - - - + + + - - - + + + - - - + + - - - + + + + - - + - - + + + - - - - - - - - + + + + + + + + - + - - - - + + + + - - - + + + - - - - - - + + + + + + - - - + + + - - - - + + + + - + - - - + + + - - - - + + + + - - - + + + - - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - + + + - - - - - + + + + + - - - + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - + + - + - - - - + + + + + - - - - - - - + + + + + + - + + - - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + - + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - + - + - - + - + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - + - + @@ -1364,40 +1362,47 @@ - - - + + + - - - - - + + + + + - - + + + - - - - - - - - + + + + + + - - - - - - - + + + + + + + + + + + + + + + From 3274d766acf63f93af73ee0a8ebf71699ae622cd Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 4 Feb 2026 10:45:52 +0000 Subject: [PATCH 71/75] Add release date display in header, improve Docker mode behavior, add Cursor rules and update tests --- .../{acepace-rules.md => acepace-rules.mdc} | 11 +- acepace.py | 16 +- coverage.xml | 1657 +++++++++-------- docker-compose.yml | 1 + entrypoint.sh | 4 +- tests/test_main_command.py | 32 +- 6 files changed, 886 insertions(+), 835 deletions(-) rename .cursor/rules/{acepace-rules.md => acepace-rules.mdc} (98%) diff --git a/.cursor/rules/acepace-rules.md b/.cursor/rules/acepace-rules.mdc similarity index 98% rename from .cursor/rules/acepace-rules.md rename to .cursor/rules/acepace-rules.mdc index 9601f5c..e1f6dab 100644 --- a/.cursor/rules/acepace-rules.md +++ b/.cursor/rules/acepace-rules.mdc @@ -1,3 +1,8 @@ +--- +description: Ace-Pace development rules — tests, lint, git workflow, and project conventions +alwaysApply: true +--- + # Ace-Pace Project Rules This file contains the development rules, guidelines, and technical reference for the Ace-Pace project. These rules should be followed by all AI agents working on this codebase. @@ -35,7 +40,7 @@ When working on this project, you MUST: - **Keep commits atomic**: commit only the files you touched and list each path explicitly - For tracked files: `git commit -m "" -- path/to/file1 path/to/file2` - - For brand-new files: `git restore --staged :/ && git add "path/to/file1" "path/to/file2" && git commit -m "" -- path/to/file1 path/to/file2` + - For brand-new files: `git restore --staged :/ && git add "path/to/file1" "path/to/file2" && git commit -m "" -- path/to/file1 path/to/file2 - **Quote any git paths** containing brackets or parentheses (e.g., `src/app/[candidate]/**`) when staging or committing so the shell does not treat them as globs or subshells. @@ -58,7 +63,7 @@ When working on this project, you MUST: - Follow PEP 8 Python style guide - Maintain cognitive complexity ≤ 15 per function - Use descriptive variable names -- Add docstrings for functions +- Use docstrings for functions - Keep functions focused and single-purpose - Use `_` prefix for private/internal helper functions - Comprehensive test suite exists in `tests/` directory (100+ tests) @@ -165,7 +170,7 @@ When working on this project, you MUST: - `main()`: Entry point for the application - `init_db(suppress_messages=False)`: Initializes the local CRC32 cache database - `init_episodes_db()`: Initializes the episodes index database -- `get_config_dir()`: Gets config directory path based on Docker mode (`/config` in Docker, `.` locally) +- `get_config_dir()`: Gets config directory based on Docker mode (`/config` in Docker, `.` locally) - `get_config_path(filename)`: Gets full path to a config file in the appropriate config directory - `normalize_file_path(file_path)`: Normalizes file path for consistent storage and lookup - **CRITICAL**: Always use this before storing/querying file paths in database diff --git a/acepace.py b/acepace.py index f9e9579..846abc8 100644 --- a/acepace.py +++ b/acepace.py @@ -71,6 +71,15 @@ def _signal_handler(signum, frame): CSV_COLUMN_MAGNET_LINK = "Magnet Link" +def _get_release_date(): + """Release date from modification time of this file (no repo commits, no extra file).""" + try: + mtime = os.path.getmtime(os.path.abspath(__file__)) + return datetime.fromtimestamp(mtime).strftime("%Y-%m-%d") + except (OSError, ValueError): + return "" + + def get_config_dir(): """Get the config directory path based on Docker mode. Returns the config directory path, creating it if necessary.""" @@ -2060,6 +2069,9 @@ def _print_header(): print("=" * 60) print(" " * 20 + "Ace-Pace") print(" " * 12 + "One Pace Library Manager") + release = _get_release_date() + if release: + print(" " * (26 - len(release) // 2) + f"Release {release}") print("=" * 60) if IS_DOCKER: print("Running in Docker mode (non-interactive)") @@ -2081,8 +2093,8 @@ def main(): sys.exit(0) # Print header only for main command (not for --db or --episodes_update) - # Also suppress for help command - if IS_DOCKER and not args.db and not args.episodes_update and not args.help: + # In Docker, entrypoint.sh already prints the header once; skip here to avoid duplicate + if not IS_DOCKER and not args.db and not args.episodes_update and not args.help: _print_header() if not _validate_url(args.url): diff --git a/coverage.xml b/coverage.xml index 8e2c0d4..73c535c 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -53,1136 +53,1145 @@ - - - - - - + + + + + + + + - - - + + + + + + - - - - - - - - + + + + + + + - - - - - + + + + - - - - - - - + + + + + + + + - + - - - + + + - - - - - + + + - - - - - + + + + - - + + - - - + + - + - + - - - - + - - - + + + - + + + + - - - - - - - + + + + + + + - - - - - - + + + + + - - - - - - - - + + + + + + + + + - - - - - - + + + + + + + - - + + - - - - - + - + + + - - - - - - - + + + + + + + + + - - - + + + + - - - + - - + + - - - + - - - - - - - + + + + + + + + + - - - - - - + + + + + + - - - - - - + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - + + + + + + + + - - - - - - - + - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - + - - + + + - - - + + + + - - - - - + + - - - - + + + - + + - - + + - - - - - + + + + + + + + + - - - - - + + - - - - + + + + - - + + - + - - - - - + + + - + - - - - - - - - - + + + + + + + + + + + - - + + - + + - - - - - - - - - + + + + + + - - + + - - - - - - - - - - - + + + + + + + + + + + + + - - + + - - + + - - - + + + - + + - - - - - - + + + + + + + - + + + - - - + + - - - - - + - - - + + + + + - + - - - - - - - + + + + + + + - - - - - - + + - + - + - + - - - - - - - + + + + + + + + + + - + + - - - - - - - - - + + + + + + + + + - - - - - + + + + + - - + - - + - - - - + + + + + - - + + + + + - - - - - - + + + - - - - - - - - + + + + + + + - - - - + + + + + - - - - - + + + + - - + + + + + - - - - - - + + + + + + + + - - - - - - - - + - - + + + - - - + + + + - + + + + - - - - - + + - + + + + + - - - - - + + - - - + + - - + + + + + - + - - - - + + - - - - - + + + + + - - - - - + + + - - - - + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - - - + + + + + - - - - - - + + + + + + - - - + - - + + + + + - - + + + + + - - - - - - + - + + - - - - + + + - + - + - - - - - - - - + + + + + + + + + - - - + + - - - - - - + + + + + + + - - + - - + + + + - - - - - - + + + - - - - + + + + + + + + - + - - - - - + + + + - - - + + - + - - + + - - + - - - - - - - - - - - + + + + + + + + + + - - + + - + + + - - + + - - - - - - - + + + + + + - - - - - - - + + + + + + + + - - - - + + + + - + + + + - - - - - - - - - - - + + + + + + + + + - - - - - + + + - - + + - - - - - - + + + + - - - + + + + + + + + - - - - - - - - - - + + + + + + + + + - - - - - - - - + + + + + - + - - + + + + - - - - - - - - - + + + + + + + - + - - + + + + - + - - - - + - + + + - + - + - - - - + + + + + + - - - - + + + - - - + + + - - - + + + + - - - - - - - - - - + + + + + + + + - - - + + + + + + - - - + - - + + - - - - - - - - - + + + + + + + + + - - - + + - - + + + + + - - - - + + - - - - - - - - + + + + + + - + - - + + + + + + + - - - - - + + - - + + + - - - - - - + + + + - + - + - + - - - + + + + + + + - - - - - - + + - - - - - - - - + + + + + + + + + + + - - - - - - - - + + + + + + + - + - - + - - - - + + + + + + - - + + + + - - - - + + + + - - - - - - - - - - - + + + + + + + + + + - - - - - - - - - + + + + + + + + - - - + + + - - + + - - + + + - - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + + - - - - - - + + + + - - - + + + - - - + + + + - - - - - - - - + + + + + + - - - - - - - - + + + + + + + - - + + - - + + + + + + - - - - - - + + + - + + - - - - - - - - - - - - + + + + + + + + + + + + + - - + + + - - + - + - - - - + + + + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index 39d86db..0590291 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,5 +33,6 @@ services: #- TORRENT_USER=admin # BitTorrent client password (default: empty, not required) #- TORRENT_PASSWORD=password + # Enable debug output for troubleshooting (default: false) #- DEBUG=true diff --git a/entrypoint.sh b/entrypoint.sh index 457c7f0..f4564b3 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -14,10 +14,12 @@ cleanup() { # This trap handles signals that arrive when no Python process is running trap 'cleanup' 15 2 -# Print Ace-Pace header at the very beginning +# Print Ace-Pace header (release date = mtime of acepace.py, no repo commits) +RELEASE_DATE=$(stat -c %y /app/acepace.py 2>/dev/null | cut -d' ' -f1 || true) echo "============================================================" echo " Ace-Pace" echo " One Pace Library Manager" +if [ -n "$RELEASE_DATE" ]; then echo " Release $RELEASE_DATE"; fi echo "============================================================" echo "Running in Docker mode (non-interactive)" echo "------------------------------------------------------------" diff --git a/tests/test_main_command.py b/tests/test_main_command.py index 6db92fd..a4db580 100644 --- a/tests/test_main_command.py +++ b/tests/test_main_command.py @@ -13,6 +13,27 @@ TEST_HOST_IP = "localhost" # Test host for testing environment variable handling +class TestReleaseDateHeader: + """Tests for release date in header (from acepace.py mtime).""" + + @patch('acepace._get_release_date', return_value='2025-02-04') + def test_print_header_includes_release_date(self, mock_release_date): + """Header shows Release line when _get_release_date returns a date.""" + with patch('acepace.IS_DOCKER', False), patch('builtins.print') as mock_print: + acepace._print_header() + printed = " ".join(str(c) for c in mock_print.call_args_list) + assert "Release" in printed + assert "2025-02-04" in printed + + def test_get_release_date_returns_string_from_mtime(self): + """_get_release_date returns YYYY-MM-DD from acepace.py mtime.""" + result = acepace._get_release_date() + assert isinstance(result, str) + if result: + assert len(result) == 10 + assert result[4] == "-" and result[7] == "-" + + class TestDockerModeBehavior: """Tests for Docker mode specific behavior.""" @@ -78,9 +99,10 @@ def test_docker_mode_message_not_shown_for_episodes_update(self, mock_update, mo @patch('acepace.init_db') @patch('acepace._get_folder_from_args') @patch('acepace._handle_main_commands') - def test_docker_mode_message_shown_for_main_command(self, mock_handle, mock_folder, mock_init_db, - mock_show_status, mock_validate): - """Test that Docker mode message is shown for main command (not --db or --episodes_update).""" + def test_docker_mode_message_not_printed_by_python_for_main_command(self, mock_handle, mock_folder, + mock_init_db, mock_show_status, + mock_validate): + """In Docker, entrypoint.sh prints the header once; Python must not print it again.""" mock_validate.return_value = True mock_init_db.return_value = MagicMock() mock_folder.return_value = "/media" @@ -100,9 +122,9 @@ def test_docker_mode_message_shown_for_main_command(self, mock_handle, mock_fold with pytest.raises(SystemExit): acepace.main() - # Verify "Running in Docker mode" WAS printed + # Python must NOT print header in Docker (entrypoint.sh already did) print_calls = [str(c) for c in mock_print.call_args_list] - assert any("Running in Docker mode" in str(call) for call in print_calls) + assert not any("Running in Docker mode" in str(call) for call in print_calls) @patch('acepace.IS_DOCKER', False) @patch('acepace._validate_url') From bbf9632f18b161db16b57c738861cd5884a77a30 Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 4 Feb 2026 11:22:38 +0000 Subject: [PATCH 72/75] Change episode updates logic --- acepace.py | 20 +- coverage.xml | 1310 +++++++++++++++++++++++++------------------------ entrypoint.sh | 9 +- 3 files changed, 676 insertions(+), 663 deletions(-) diff --git a/acepace.py b/acepace.py index 846abc8..1908be7 100644 --- a/acepace.py +++ b/acepace.py @@ -1875,10 +1875,12 @@ def _print_help(): with episodes available on Nyaa to find missing episodes. Outputs results to Ace-Pace_Missing.csv - --episodes_update Update episodes metadata database from Nyaa - Fetches all One Pace episodes from Nyaa and stores their - CRC32, title, and page link in the episodes index database. - This should be run periodically to keep the database current. + --episodes_update Update episodes from Nyaa and generate missing report + First fetches all One Pace episodes from Nyaa and stores + CRC32, title, page link, and magnet link in the episodes index. + Then runs the missing episodes report (same as main command): + scans local folder, compares with Nyaa, outputs Ace-Pace_Missing.csv. + In Docker mode, --folder defaults to /media if not set. --rename Rename local files based on CRC32 matching Matches local video files with episodes in the database @@ -2003,7 +2005,7 @@ def _parse_arguments(): parser.add_argument( "--episodes_update", action="store_true", - help="Update episodes metadata database from Nyaa.", + help="Update episodes from Nyaa, then run missing episodes report (like main command).", ) parser.add_argument("--host", default="localhost", help="The BitTorrent client host.") parser.add_argument("--port", type=int, help="The BitTorrent client port.") @@ -2105,8 +2107,14 @@ def main(): _show_episodes_metadata_status() if args.episodes_update: - # When --episodes_update is used directly, force update (same behavior as EPISODES_UPDATE=true) + # When --episodes_update is used: update episodes from Nyaa, then run missing episodes report (like main command) update_episodes_index_db(args.url, force_update=True) + conn = init_db(suppress_messages=False) + needs_folder = True # Missing report requires folder + folder = _get_folder_from_args(args, conn, needs_folder) + if folder is None: + sys.exit(1) + _generate_missing_episodes_report(conn, folder, args) sys.exit(0) # Suppress messages when exporting DB (since it's automated) diff --git a/coverage.xml b/coverage.xml index 73c535c..cf4e9fd 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -30,8 +30,8 @@ - - + + @@ -53,24 +53,24 @@ - - - + + + - - + + - - + + - - + + @@ -81,65 +81,65 @@ - + - - - - - - - - - - - + + + + + + + + + + + - - - - + + + + - - - + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - + + + @@ -159,94 +159,94 @@ - - + + - - + + - - - - - - + + + + + + - - - + + + - - - - - - - + + + + + + + - - + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - + + - - - + + + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - + + + - - + + - - - - + + + + - + - - + + @@ -258,66 +258,66 @@ - - - - - + + + + + - - - - - - + + + + + + - + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -408,33 +408,33 @@ - - - - + + + + - + - - - - - - - - - - + + + + + + + + + + - - - - - - - + + + + + + + @@ -456,174 +456,174 @@ - - + + - - - - - - - - + + + + + + + + - - - + + + - + - - + + - - - - - - + + + + + + - - - - + + + + - + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - + + + - - - - - - + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - + + + - - - + + + - - - - - - - - - - + + + + + + + + + + - - - - + + + + - - + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - + + + + + - - - - - - + + + + + + - - + + - - - - - - - - - - + + + + + + + + + + - - - - - - + + + + + + - + - - - + + + @@ -683,25 +683,25 @@ - - - - - - - - - - - - + + + + + + + + + + + + - - + + - + @@ -714,36 +714,36 @@ - - + + - - - + + + - + - - - - - - - - - - - + + + + + + + + + + + - - - + + + - - - - + + + + @@ -767,51 +767,51 @@ - - - - - - - - - - - - + + + + + + + + + + + + - - + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - - + + + - - - - + + + + @@ -823,7 +823,7 @@ - + @@ -841,20 +841,20 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + @@ -1104,97 +1104,103 @@ - - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - + + + + + + + + - - - + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - - - - - - - + + + + + + - - - + + + + - + - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + + - - + + + + + + + - + @@ -1207,211 +1213,211 @@ - - - - - - - - - + + + + + + + + + - + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - + + + - + - - - - - + + + + + - - + + - + - - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - + - - - + + + - - - - - + + + + + diff --git a/entrypoint.sh b/entrypoint.sh index f4564b3..afae2eb 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -45,11 +45,10 @@ if [ "$DB" = "true" ]; then fi fi -# Run missing episodes report -# Always run unless ONLY exporting DB (same as non-Docker: main command always runs) -# When EPISODES_UPDATE=true, the Python code will use the database to avoid double fetch -# Skip report only if DB=true AND no other operations -if [ "$DB" != "true" ] || [ "$EPISODES_UPDATE" = "true" ] || [ "$DOWNLOAD" = "true" ]; then +# Run missing episodes report (unless already done by --episodes_update above) +# When EPISODES_UPDATE=true, the report was already run in step 1 (--episodes_update does both) +# Skip when only exporting DB (DB=true and no other operations) +if [ "$EPISODES_UPDATE" != "true" ] && { [ "$DB" != "true" ] || [ "$DOWNLOAD" = "true" ]; }; then python /app/acepace.py \ --folder /media \ ${NYAA_URL:+--url "$NYAA_URL"} From a2a5be54bc6145cea5dd92c321acc3cd29e21289 Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 4 Feb 2026 12:55:27 +0000 Subject: [PATCH 73/75] Fix dry run issue in Docker --- entrypoint.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index afae2eb..196e5ab 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -61,12 +61,15 @@ fi # If DOWNLOAD is set to true, download missing episodes after generating report if [ "$DOWNLOAD" = "true" ]; then + # Only add --dry-run when DRY_RUN is explicitly true (not when set to "false" or empty) + DRY_RUN_ARG="" + [ "$DRY_RUN" = "true" ] || [ "$DRY_RUN" = "1" ] || [ "$DRY_RUN" = "yes" ] || [ "$DRY_RUN" = "on" ] && DRY_RUN_ARG="--dry-run" # Use exec to replace shell process so Python becomes PID 1 and receives signals directly exec python /app/acepace.py \ --folder /media \ ${NYAA_URL:+--url "$NYAA_URL"} \ --download \ - ${DRY_RUN:+--dry-run} \ + ${DRY_RUN_ARG:+$DRY_RUN_ARG} \ ${TORRENT_CLIENT:+--client "$TORRENT_CLIENT"} \ ${TORRENT_HOST:+--host "$TORRENT_HOST"} \ ${TORRENT_PORT:+--port "$TORRENT_PORT"} \ From 606c1718d9533df4f37e86823173de39102c1239 Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 5 Feb 2026 10:24:02 +0000 Subject: [PATCH 74/75] Reintroducing the RENAME feature for Docker (with dry run) --- .cursor/rules/acepace-rules.mdc | 5 +- README.md | 16 +- acepace.py | 19 +- coverage.xml | 1835 ++++++++++++++++--------------- docker-compose.yml | 6 +- entrypoint.sh | 16 + tests/test_file_operations.py | 20 + tests/test_main_command.py | 36 +- 8 files changed, 1017 insertions(+), 936 deletions(-) diff --git a/.cursor/rules/acepace-rules.mdc b/.cursor/rules/acepace-rules.mdc index e1f6dab..f590217 100644 --- a/.cursor/rules/acepace-rules.mdc +++ b/.cursor/rules/acepace-rules.mdc @@ -189,7 +189,7 @@ When working on this project, you MUST: - `fetch_title_by_crc32(crc32)`: Searches for a title by CRC32 - `calculate_local_crc32(folder, conn)`: Calculates CRC32 for local files - Uses normalized paths for database storage and lookup -- `rename_local_files(conn)`: Renames local files based on episodes index +- `rename_local_files(conn, dry_run=False)`: Renames local files based on episodes index; when dry_run=True only prints plan - Uses normalized paths when updating database after renaming - `export_db_to_csv(conn)`: Exports database to CSV - `load_crc32_to_title_from_index()`: Loads CRC32-to-title mapping @@ -216,7 +216,7 @@ Helper functions are prefixed with `_` to indicate they are internal implementat **Command handlers** (`_handle_*`): Handle specific command-line operations - `_handle_download_command(args)`: Handles the `--download` command -- `_handle_rename_command(conn, base_url=None)`: Handles the `--rename` command +- `_handle_rename_command(conn, base_url=None, dry_run=False)`: Handles the `--rename` command; dry_run only shows plan - `base_url`: Optional URL parameter passed to `update_episodes_index_db()` if update is needed - `_handle_main_commands(args, conn, folder)`: Routes and handles main commands @@ -231,6 +231,7 @@ Helper functions are prefixed with `_` to indicate they are internal implementat - **Message Suppression**: In Docker mode, suppresses informational messages for automated commands - **Environment Variables**: Supports configuration via Docker environment variables - `DOWNLOAD`: Set to "true" to download missing episodes after generating report + - `RENAME`: Set to "true" to rename local files under `/media` (non-interactive; use `DRY_RUN=true` to simulate) - `TORRENT_CLIENT`: BitTorrent client type (default: transmission) - `TORRENT_HOST`: Client host address (default: localhost) - `TORRENT_PORT`: Client port number (default: 9091 for transmission, 8080 for qBittorrent) diff --git a/README.md b/README.md index 724e5f6..5965fd7 100644 --- a/README.md +++ b/README.md @@ -75,10 +75,11 @@ The following environment variables can be used to configure Ace-Pace in Docker: - `DB` - Set to `true` to generate CSV database export on container start (default: `false`) - `EPISODES_UPDATE` - Set to `true` to update episodes metadata from Nyaa on container start (default: `false`) - `DOWNLOAD` - Set to `true` to automatically download missing episodes after generating report (default: `false`) -- `DRY_RUN` - Set to `true` to test connection to BitTorrent client without actually adding torrents (default: `false`) - - Only effective when `DOWNLOAD=true` - - Validates magnet links and checks existing torrents but does not add any downloads - - Useful for verifying configuration before enabling actual downloads +- `RENAME` - Set to `true` to rename local files under `/media` to match One-Pace episode titles from the episodes index (default: `false`) + - Non-interactive: no confirmation prompt; use `DRY_RUN=true` to simulate renaming without changing files +- `DRY_RUN` - When `DOWNLOAD=true`: test BitTorrent client without adding torrents. When `RENAME=true`: show rename plan without renaming (default: `false`) + - With download: validates magnet links and checks existing torrents but does not add any downloads + - With rename: prints which files would be renamed without modifying the filesystem - `TORRENT_CLIENT` - BitTorrent client type: `transmission` or `qbittorrent` (default: `transmission`) - `TORRENT_HOST` - BitTorrent client host address (default: `localhost`) - `TORRENT_PORT` - BitTorrent client port (default: `9091` for Transmission, `8080` for qBittorrent) @@ -105,8 +106,9 @@ When the container starts, it executes the following steps in order: 1. **Episodes Update** (if `EPISODES_UPDATE=true`): Updates the episodes metadata database from Nyaa, including magnet links for all episodes 2. **Database Export** (if `DB=true`): Exports the CRC32 database to CSV -3. **Missing Episodes Report**: Always runs to generate/update `Ace-Pace_Missing.csv` -4. **Download** (if `DOWNLOAD=true`): Automatically downloads missing episodes via the configured BitTorrent client +3. **Missing Episodes Report**: Always runs to generate/update `Ace-Pace_Missing.csv` (unless only DB export was requested) +4. **Rename** (if `RENAME=true`): Renames local files under `/media` to match One-Pace episode titles (no confirmation). Use `DRY_RUN=true` to simulate only. +5. **Download** (if `DOWNLOAD=true`): Automatically downloads missing episodes via the configured BitTorrent client - If `DRY_RUN=true`, tests connection and validates magnet links without adding torrents ### Docker Notes @@ -241,7 +243,7 @@ python acepace.py [-h] [--url URL] [--folder FOLDER] [--db] [--client {transmiss Category to add to the torrent in qBittorrent. - `--dry-run` (standalone flag) - Test connection to BitTorrent client without actually adding torrents. Useful for verifying configuration before downloading. When enabled, validates magnet links and checks existing torrents but does not add any new downloads. + With `--download`: test BitTorrent client without adding torrents. With `--rename`: show rename plan without renaming files. ### 📚 Some examples diff --git a/acepace.py b/acepace.py index 1908be7..65dedf3 100644 --- a/acepace.py +++ b/acepace.py @@ -1101,12 +1101,14 @@ def _execute_rename(rename_plan, conn): print(f"Failed to rename {old} to {new}: {e}") -def rename_local_files(conn): +def rename_local_files(conn, dry_run=False): """Rename local files based on CRC32 matching titles from episodes index. Matches local video files with episodes in the database and renames them to match the official episode titles. Args: - conn: Database connection""" + conn: Database connection + dry_run: If True, only print the rename plan and do not rename or ask for confirmation. + """ c = conn.cursor() c.execute("SELECT file_path, crc32 FROM crc32_cache") entries = c.fetchall() @@ -1133,6 +1135,10 @@ def rename_local_files(conn): print(f"{os.path.basename(old)} -> {os.path.basename(new)}") print(f"{len(rename_plan)}/{total} files will be renamed.") + if dry_run: + print("DRY RUN: would rename the above files (no changes made).") + return + confirm = _get_rename_confirmation() if confirm != "y": print("Renaming aborted.") @@ -1379,11 +1385,12 @@ def _get_rename_prompt(last_ep_update): ).strip().lower() -def _handle_rename_command(conn, base_url=None): +def _handle_rename_command(conn, base_url=None, dry_run=False): """Handle the rename command. Args: conn: Database connection base_url: Base URL for Nyaa search (optional) + dry_run: If True, only show rename plan and do not rename or ask for confirmation. """ episodes_db_conn = init_episodes_db() last_ep_update = get_episodes_metadata( @@ -1398,7 +1405,7 @@ def _handle_rename_command(conn, base_url=None): print( "Renaming local files based on matching titles from One Pace episodes index..." ) - rename_local_files(conn) + rename_local_files(conn, dry_run=dry_run) def _count_video_files(folder, conn): @@ -2017,7 +2024,7 @@ def _parse_arguments(): parser.add_argument( "--dry-run", action="store_true", - help="Test connection to BitTorrent client without actually adding torrents. Useful for verifying configuration.", + help="Download: test client without adding torrents. Rename: show rename plan without renaming.", ) return parser.parse_args() @@ -2050,7 +2057,7 @@ def _handle_main_commands(args, conn, folder): return if args.rename: - _handle_rename_command(conn, args.url) + _handle_rename_command(conn, args.url, dry_run=args.dry_run) return if not folder: diff --git a/coverage.xml b/coverage.xml index cf4e9fd..d65271a 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -30,8 +30,8 @@ - - + + @@ -53,9 +53,9 @@ - - - + + + @@ -69,8 +69,8 @@ - - + + @@ -81,65 +81,65 @@ - + - - - - - - - - - - - + + + + + + + + + + + - - - - + + + + - - - + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - + + + @@ -159,94 +159,94 @@ - - + + - - + + - - - - - - + + + + + + - - - + + + - - - - - - - + + + + + + + - - + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - + + - - - + + + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - + + + - - + + - - - - + + + + - + - - + + @@ -258,66 +258,66 @@ - - - - - + + + + + - - - - - - + + + + + + - + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -408,33 +408,33 @@ - - - - + + + + - + - - - - - - - - - - + + + + + + + + + + - - - - - - - + + + + + + + @@ -456,187 +456,187 @@ - - + + - - - - - - - - + + + + + + + + - - - + + + - + - - + + - - - - - - + + + + + + - - - - + + + + - + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - + + + - - - - - - + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - + + + - - - + + + - - - - - - - - - - + + + + + + + + + + - - - - + + + + - - + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - + + + + + - - - - - - + + + + + + - - + + - - - - - - - - - - + + + + + + + + + + - - - - - - + + + + + + - + - - - + + + - - - - + + + + - - - - - - - + + + + + + + @@ -657,550 +657,553 @@ - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - - - + + + - + - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - - - - - + + + + - + - - - + + - + + - - - - + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + + + + - - - - - - - - - + + + + + + - - - - - - - - - - + + + + + + + + + - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - + + + - - - - - - - - - - - + + + + + + + + + + - - - - - - - - + + + + + + + - + - - - + + + + + - - - - + - + + + - + - - + - - - + + + + + - - - - - + + + + - - - + + + - - + + + + - - - - + + - - + + - - - - - - + + + + + + + + - - - + + - - + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - + + + + - - - - + + + - - - + + - - - - + + + - + + - + + + + - - - - - + + + - - + - + - - - + + + - + - - - - + + + + - - + - - - - - - - + + + + + + + + - + - - - - - - - + + + + + + + + + - - - - - - - - + + + + + + + - - - + + + - - + + - - + + - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - + + + + + + + + + - - + + + - - - - - - - + + + + + + - + - - - - - - - - + + + + + - - + + + + + - - - - - + + + + - + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - + + + + - - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + + - - - - + + + + + + + - - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - + + + + + + + - - - - - + + + + - - - + + + + + + + - + @@ -1213,211 +1216,211 @@ - - - - - - - - - + + + + + + + + + - + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - + + + - + - - - - - + + + + + - - + + - + - - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - + - - - + + + - - - - - + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index 0590291..d302aab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,9 +17,13 @@ services: # Quality filtering (1080p only) is always applied in code regardless of URL #- NYAA_URL=https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc + # Rename local files under /media to match One-Pace episode titles (default: false) + # Non-interactive; use DRY_RUN=true to simulate renaming without changing files + #- RENAME=true + # Download missing episodes after generating report (default: false) #- DOWNLOAD=true - # Test connection without adding torrents (default: false, only effective when DOWNLOAD=true) + # With DOWNLOAD: test client without adding torrents. With RENAME: simulate only (default: false) #- DRY_RUN=true # BitTorrent client type (default: transmission) diff --git a/entrypoint.sh b/entrypoint.sh index 196e5ab..0963736 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -59,6 +59,22 @@ if [ "$EPISODES_UPDATE" != "true" ] && { [ "$DB" != "true" ] || [ "$DOWNLOAD" = fi fi +# Run rename if requested (non-interactive: dry-run simulates, otherwise renames without confirmation) +if [ "$RENAME" = "true" ]; then + DRY_RUN_RENAME_ARG="" + [ "$DRY_RUN" = "true" ] || [ "$DRY_RUN" = "1" ] || [ "$DRY_RUN" = "yes" ] || [ "$DRY_RUN" = "on" ] && DRY_RUN_RENAME_ARG="--dry-run" + python /app/acepace.py \ + --folder /media \ + --rename \ + ${NYAA_URL:+--url "$NYAA_URL"} \ + ${DRY_RUN_RENAME_ARG:+$DRY_RUN_RENAME_ARG} + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Rename failed with exit code $EXIT_CODE" + exit $EXIT_CODE + fi +fi + # If DOWNLOAD is set to true, download missing episodes after generating report if [ "$DOWNLOAD" = "true" ]; then # Only add --dry-run when DRY_RUN is explicitly true (not when set to "false" or empty) diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index efa9f0d..3859a91 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -144,6 +144,26 @@ def test_rename_uses_normalized_paths(self, temp_dir): conn.close() + def test_rename_dry_run_does_not_confirm_or_execute(self, temp_dir): + """Test that rename_local_files(conn, dry_run=True) does not ask for confirmation or rename files.""" + test_file = os.path.join(temp_dir, "old_name.mkv") + with open(test_file, "wb") as f: + f.write(b"test video content") + + with patch('acepace.DB_NAME', os.path.join(temp_dir, 'test.db')): + conn = acepace.init_db() + acepace.calculate_local_crc32(temp_dir, conn) + actual_crc32 = list(acepace.calculate_local_crc32(temp_dir, conn))[0] + + with patch('acepace.load_crc32_to_title_from_index') as mock_load: + mock_load.return_value = {actual_crc32: "[One Pace] Episode 1 [1080p].mkv"} + with patch('acepace._get_rename_confirmation') as mock_confirm: + with patch('acepace._execute_rename') as mock_execute: + acepace.rename_local_files(conn, dry_run=True) + mock_confirm.assert_not_called() + mock_execute.assert_not_called() + conn.close() + class TestCSVExport: """Tests for CSV export functionality.""" diff --git a/tests/test_main_command.py b/tests/test_main_command.py index a4db580..7bccf18 100644 --- a/tests/test_main_command.py +++ b/tests/test_main_command.py @@ -288,16 +288,44 @@ def test_rename_receives_url_parameter(self, mock_rename, mock_folder, mock_init mock_args.rename = True mock_args.url = test_url mock_args.folder = None - + mock_args.dry_run = False + with patch('acepace._parse_arguments', return_value=mock_args): with pytest.raises(SystemExit): acepace.main() - - # Verify _handle_rename_command was called with URL + + # Verify _handle_rename_command was called with URL and dry_run mock_rename.assert_called_once() - # Check that URL was passed (second argument after conn) call_args = mock_rename.call_args assert call_args[0][1] == test_url # Second positional argument is URL + assert call_args[1]["dry_run"] is False + + @patch('acepace._validate_url') + @patch('acepace.init_db') + @patch('acepace._get_folder_from_args') + @patch('acepace._handle_rename_command') + def test_rename_with_dry_run_passes_dry_run_true(self, mock_rename, mock_folder, mock_init_db, mock_validate): + """Test that --rename --dry-run passes dry_run=True to _handle_rename_command.""" + mock_validate.return_value = True + mock_init_db.return_value = MagicMock() + mock_folder.return_value = "/media" + + mock_args = MagicMock() + mock_args.help = False + mock_args.db = False + mock_args.episodes_update = False + mock_args.download = False + mock_args.rename = True + mock_args.url = None + mock_args.folder = None + mock_args.dry_run = True + + with patch('acepace._parse_arguments', return_value=mock_args): + with pytest.raises(SystemExit): + acepace.main() + + mock_rename.assert_called_once() + assert mock_rename.call_args[1]["dry_run"] is True class TestDockerDownloadDefaults: From 5ab3a4fe16a6b0b1a656b43f5f2b41861a0a5176 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 6 Feb 2026 11:56:51 +0000 Subject: [PATCH 75/75] Rename feature tweaking, config folder refactoring --- .cursor/rules/acepace-rules.mdc | 10 +- README.md | 15 +- acepace.py | 104 +- coverage.xml | 1738 ++++++++++++++++--------------- docker-compose.yml | 4 +- entrypoint.sh | 9 +- tests/test_file_operations.py | 27 + tests/test_main_command.py | 4 +- 8 files changed, 1010 insertions(+), 901 deletions(-) diff --git a/.cursor/rules/acepace-rules.mdc b/.cursor/rules/acepace-rules.mdc index f590217..655bf89 100644 --- a/.cursor/rules/acepace-rules.mdc +++ b/.cursor/rules/acepace-rules.mdc @@ -170,7 +170,8 @@ When working on this project, you MUST: - `main()`: Entry point for the application - `init_db(suppress_messages=False)`: Initializes the local CRC32 cache database - `init_episodes_db()`: Initializes the episodes index database -- `get_config_dir()`: Gets config directory based on Docker mode (`/config` in Docker, `.` locally) +- `get_config_dir()`: Gets config directory (Docker: `ACEPACE_CONFIG_DIR_DOCKER` default `/config`; local: `ACEPACE_CONFIG_DIR_LOCAL` default `.`) +- `_get_default_media_dir()`: Default media folder (Docker: `ACEPACE_MEDIA_DIR_DOCKER` default `/media`; local: `ACEPACE_MEDIA_DIR_LOCAL` default empty) - `get_config_path(filename)`: Gets full path to a config file in the appropriate config directory - `normalize_file_path(file_path)`: Normalizes file path for consistent storage and lookup - **CRITICAL**: Always use this before storing/querying file paths in database @@ -191,6 +192,7 @@ When working on this project, you MUST: - Uses normalized paths for database storage and lookup - `rename_local_files(conn, dry_run=False)`: Renames local files based on episodes index; when dry_run=True only prints plan - Uses normalized paths when updating database after renaming +- `_ensure_crc32_cache_complete(folder, conn)`: Ensures CRC32 cache has all video files in folder; runs `calculate_local_crc32` if any are missing (used before rename) - `export_db_to_csv(conn)`: Exports database to CSV - `load_crc32_to_title_from_index()`: Loads CRC32-to-title mapping @@ -216,15 +218,17 @@ Helper functions are prefixed with `_` to indicate they are internal implementat **Command handlers** (`_handle_*`): Handle specific command-line operations - `_handle_download_command(args)`: Handles the `--download` command -- `_handle_rename_command(conn, base_url=None, dry_run=False)`: Handles the `--rename` command; dry_run only shows plan +- `_handle_rename_command(conn, base_url=None, dry_run=False, folder=None)`: Handles the `--rename` command; calls `_ensure_crc32_cache_complete(folder, conn)` when folder is set, then renames - `base_url`: Optional URL parameter passed to `update_episodes_index_db()` if update is needed + - `folder`: Media folder for CRC32 cache check (version-specific default when run from main) - `_handle_main_commands(args, conn, folder)`: Routes and handles main commands ## Docker Support - **Docker Mode**: Detected via `RUN_DOCKER` environment variable - **Non-Interactive Operation**: In Docker mode, skips user prompts and uses defaults -- **Default Folder**: Uses `/media` as default folder in Docker mode +- **Default Folder**: Uses `ACEPACE_MEDIA_DIR_DOCKER` (default `/media`) in Docker; `ACEPACE_MEDIA_DIR_LOCAL` (default empty) locally +- **Config/Data Paths**: `ACEPACE_CONFIG_DIR_DOCKER` (default `/config`), `ACEPACE_CONFIG_DIR_LOCAL` (default `.`) - **Config Directory**: Uses `/config` directory in Docker mode for databases and CSV files - Local mode uses current directory (`.`) - Config directory is automatically created if it doesn't exist diff --git a/README.md b/README.md index 5965fd7..32e6f67 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,11 @@ The following environment variables can be used to configure Ace-Pace in Docker: - `DB` - Set to `true` to generate CSV database export on container start (default: `false`) - `EPISODES_UPDATE` - Set to `true` to update episodes metadata from Nyaa on container start (default: `false`) - `DOWNLOAD` - Set to `true` to automatically download missing episodes after generating report (default: `false`) -- `RENAME` - Set to `true` to rename local files under `/media` to match One-Pace episode titles from the episodes index (default: `false`) +- `RENAME` - Set to `true` to rename local files in the media folder to match One-Pace episode titles from the episodes index (default: `false`) - Non-interactive: no confirmation prompt; use `DRY_RUN=true` to simulate renaming without changing files + - Before renaming, ensures CRC32 cache is complete for the media folder (calculates missing CRC32s if needed) +- `ACEPACE_MEDIA_DIR_DOCKER` - Media/library folder in Docker (default: `"/media"`). Entrypoint passes this as `--folder`. +- `ACEPACE_CONFIG_DIR_DOCKER` - Config/data directory in Docker (default: `"/config"`). Not set in entrypoint; override if you mount config elsewhere. - `DRY_RUN` - When `DOWNLOAD=true`: test BitTorrent client without adding torrents. When `RENAME=true`: show rename plan without renaming (default: `false`) - With download: validates magnet links and checks existing torrents but does not add any downloads - With rename: prints which files would be renamed without modifying the filesystem @@ -95,8 +98,8 @@ The following environment variables can be used to configure Ace-Pace in Docker: The following volumes should be mounted for persistent data: -- `/media` - Mount your One-Pace library directory here (read-write) -- `/config` - Mount a directory for persistent configuration and data files (read-write) +- **Media folder** (default `/media`) - Mount your One-Pace library here (read-write). Override with `ACEPACE_MEDIA_DIR_DOCKER`. +- **Config folder** (default `/config`) - Mount a directory for persistent configuration and data files (read-write). Override with `ACEPACE_CONFIG_DIR_DOCKER`. - Contains: `crc32_files.db`, `episodes_index.db`, `Ace-Pace_Missing.csv`, `Ace-Pace_DB.csv` - `episodes_index.db` now stores magnet links for all episodes, reducing the need to fetch them repeatedly @@ -107,15 +110,15 @@ When the container starts, it executes the following steps in order: 1. **Episodes Update** (if `EPISODES_UPDATE=true`): Updates the episodes metadata database from Nyaa, including magnet links for all episodes 2. **Database Export** (if `DB=true`): Exports the CRC32 database to CSV 3. **Missing Episodes Report**: Always runs to generate/update `Ace-Pace_Missing.csv` (unless only DB export was requested) -4. **Rename** (if `RENAME=true`): Renames local files under `/media` to match One-Pace episode titles (no confirmation). Use `DRY_RUN=true` to simulate only. +4. **Rename** (if `RENAME=true`): Ensures CRC32 cache is complete for the media folder, then renames local files to match One-Pace episode titles (no confirmation). Use `DRY_RUN=true` to simulate only. 5. **Download** (if `DOWNLOAD=true`): Automatically downloads missing episodes via the configured BitTorrent client - If `DRY_RUN=true`, tests connection and validates magnet links without adding torrents ### Docker Notes -- In Docker mode, Ace-Pace automatically uses `/media` as the default folder path +- In Docker mode, the default media folder is `/media` (set `ACEPACE_MEDIA_DIR_DOCKER` to override); config/data default is `/config` (set `ACEPACE_CONFIG_DIR_DOCKER` to override) - The container runs non-interactively, so all configuration must be provided via environment variables -- All data files (databases, CSV exports) are stored in `/config` directory +- All data files (databases, CSV exports) are stored in the config directory - Quality filtering (1080p only) is applied in code regardless of the URL used - When `NYAA_URL` is not set, the default URL searches for all "one pace" episodes without quality filter, then filters for 1080p in code - Make sure your BitTorrent client is accessible from within the Docker network (use host network mode or configure networking appropriately) diff --git a/acepace.py b/acepace.py index 65dedf3..966e34c 100644 --- a/acepace.py +++ b/acepace.py @@ -61,9 +61,11 @@ def _signal_handler(signum, frame): CRC32_CHUNK_SIZE = 8192 MAGNET_LINK_PREFIX = "magnet:" -# Config directory and file names -CONFIG_DIR_DOCKER = "/config" -CONFIG_DIR_LOCAL = "." +# Config and media directory defaults (override via env: ACEPACE_CONFIG_DIR_*, ACEPACE_MEDIA_DIR_*) +CONFIG_DIR_DOCKER_DEFAULT = "/config" +CONFIG_DIR_LOCAL_DEFAULT = "." +MEDIA_DIR_DOCKER_DEFAULT = "/media" +MEDIA_DIR_LOCAL_DEFAULT = "" DB_NAME = "crc32_files.db" EPISODES_DB_NAME = "episodes_index.db" MISSING_CSV_FILENAME = "Ace-Pace_Missing.csv" @@ -82,19 +84,27 @@ def _get_release_date(): def get_config_dir(): """Get the config directory path based on Docker mode. - Returns the config directory path, creating it if necessary.""" + Returns the config directory path, creating it if necessary. + Override via ACEPACE_CONFIG_DIR_DOCKER (Docker) or ACEPACE_CONFIG_DIR_LOCAL (local). + """ if IS_DOCKER: - config_dir = CONFIG_DIR_DOCKER + config_dir = os.getenv("ACEPACE_CONFIG_DIR_DOCKER", CONFIG_DIR_DOCKER_DEFAULT) else: - config_dir = CONFIG_DIR_LOCAL - - # Ensure config directory exists + config_dir = os.getenv("ACEPACE_CONFIG_DIR_LOCAL", CONFIG_DIR_LOCAL_DEFAULT) if not os.path.exists(config_dir): os.makedirs(config_dir, exist_ok=True) - return config_dir +def _get_default_media_dir(): + """Default media/library folder for the current mode (Docker vs local). + Override via ACEPACE_MEDIA_DIR_DOCKER (Docker) or ACEPACE_MEDIA_DIR_LOCAL (local). + """ + if IS_DOCKER: + return os.getenv("ACEPACE_MEDIA_DIR_DOCKER", MEDIA_DIR_DOCKER_DEFAULT) + return os.getenv("ACEPACE_MEDIA_DIR_LOCAL", MEDIA_DIR_LOCAL_DEFAULT) + + def get_config_path(filename): """Get the full path to a config file. Args: @@ -1165,31 +1175,40 @@ def export_db_to_csv(conn): set_metadata(conn, "last_db_export", now_str) +def _prompt_folder_interactive(conn): + """Prompt user for folder using last_folder metadata or raw input. Returns folder or None.""" + last_folder = get_metadata(conn, "last_folder") + if last_folder: + print(f"Last used folder: {last_folder}") + user_input = input( + "Press Enter to use this folder, or enter a new path: " + ).strip() + folder = user_input if user_input else last_folder + else: + folder = input("Enter the folder containing local video files: ").strip() + if not folder: + print("Error: No folder specified.") + return None + return folder + + def _get_folder_from_args(args, conn, needs_folder): - """Get folder path from arguments or prompt user.""" + """Get folder path from arguments or prompt user. + In Docker uses ACEPACE_MEDIA_DIR_DOCKER (default /media); locally uses ACEPACE_MEDIA_DIR_LOCAL if set. + """ folder = args.folder if IS_DOCKER and needs_folder: - # In Docker mode, use /media as default folder - folder = "/media" + folder = _get_default_media_dir() set_metadata(conn, "last_folder", folder) return folder - if needs_folder and not folder: - # Try to load last_folder from metadata - last_folder = get_metadata(conn, "last_folder") - if last_folder: - print(f"Last used folder: {last_folder}") - user_input = input( - "Press Enter to use this folder, or enter a new path: " - ).strip() - if user_input: - folder = user_input - else: - folder = last_folder - else: - folder = input("Enter the folder containing local video files: ").strip() - if not folder: - print("Error: No folder specified.") + default_media = _get_default_media_dir() + if default_media: + folder = default_media + set_metadata(conn, "last_folder", folder) + return folder + folder = _prompt_folder_interactive(conn) + if folder is None: return None set_metadata(conn, "last_folder", folder) elif folder: @@ -1230,7 +1249,7 @@ def _get_docker_connection_params(args): username = os.getenv("TORRENT_USER", args.username or "") password = os.getenv("TORRENT_PASSWORD", args.password or "") - download_folder = args.download_folder or "/media" + download_folder = args.download_folder or _get_default_media_dir() return host, port, username, password, download_folder, client @@ -1385,12 +1404,33 @@ def _get_rename_prompt(last_ep_update): ).strip().lower() -def _handle_rename_command(conn, base_url=None, dry_run=False): +def _ensure_crc32_cache_complete(folder, conn): + """Ensure CRC32 cache includes all local video files for the folder. + If any video files in folder are not in the cache, runs calculate_local_crc32. + Respects config/data paths from get_config_dir (Docker vs local via env). + """ + total_files, recorded_files = _count_video_files(folder, conn) + if total_files == 0: + print("No video files found in folder; skipping CRC32 cache check.") + return + if recorded_files < total_files: + missing_count = total_files - recorded_files + print( + f"CRC32 cache missing {missing_count} of {total_files} files. " + "Calculating CRC32s for local files..." + ) + calculate_local_crc32(folder, conn) + else: + print("CRC32 cache is up to date for local files.") + + +def _handle_rename_command(conn, base_url=None, dry_run=False, folder=None): """Handle the rename command. Args: conn: Database connection base_url: Base URL for Nyaa search (optional) dry_run: If True, only show rename plan and do not rename or ask for confirmation. + folder: Local media folder for CRC32 cache check (uses version-specific default if not set). """ episodes_db_conn = init_episodes_db() last_ep_update = get_episodes_metadata( @@ -1402,6 +1442,8 @@ def _handle_rename_command(conn, base_url=None, dry_run=False): if prompt == "y": update_episodes_index_db(base_url) + if folder: + _ensure_crc32_cache_complete(folder, conn) print( "Renaming local files based on matching titles from One Pace episodes index..." ) @@ -2057,7 +2099,7 @@ def _handle_main_commands(args, conn, folder): return if args.rename: - _handle_rename_command(conn, args.url, dry_run=args.dry_run) + _handle_rename_command(conn, args.url, dry_run=args.dry_run, folder=folder) return if not folder: diff --git a/coverage.xml b/coverage.xml index d65271a..78a0c00 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -7,9 +7,9 @@ . - + - + @@ -52,1155 +52,1181 @@ - + + - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - + + - - - + + - - - - - - - + + + + + + + + - + - - - + + + - + - - - - - + - - - + + + + - - + - - - - - - + + + + + + - - - + - - - - + + + - - - + + + + + + - - - + + + - - - - + + + + - - + - - - + + + + - - - - - - - - + + + + + + + - - + + + + - - + + - - - - - + + + - - - - - - - - - + + + + + + + + + + + + + - - - - - - + + + - - - - + + + + - - - - + + + - - - - - - - - + + + + + + + + + - - - - - - + + + + + + - - - - + + + + - - - - - - - + + + + + + + + + + - - - - + - - - + + + - - - - - - - - + + + + + + + + - - + + - - + + - - - - - - - - - + + + + + + + + + + + - - - + - + + - - + + + - - - - + - - - + + - + - + + - - + + - - - - - + + + + + + + + + - - - - - - - - + + + + + - - - - - - + + + + + + + - - - - + - - + + - + - + + + + + + + + - - - - - - + + + - - + - - + + + + + + - - - - - - - - - + + - - - - - - - + + + + + + + + + + + + + + - - - - - - + - - - - + + + + - + + - - - - - + + + + + - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - + + + + + + - - + + - - - + - - - - + + + + + + - - - - - + - + - + - - - - - + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - + + - - + + + - - - - - - + + + - - - - - - - + + + + + + + + + + + + + - - - - - - - + + + - - - + + - - - - - - + + + + + + + - - - - - - - - + + + + + + + - + + - - - - - - - - - - + + + + + + - + + + + + + + + - - - - - - - - + - - - - + + + + + - + + + + + - - - - - - - + + + + + + + + - - - - - + - - - - - + + + + + - - + + + + + - - + + - - - - - - - - - + + + + + + + - - - - + + + + + + + + + - - - - - - - - - + + + + - - - - - + + + + + + - - + - - - - + + + + - - - + + + + + - - - - + + - - - + + + + - - - + + + + + + - - - - - - + - - - - - - + + + + + + + - - - - - - - - - + + + + + + + + + + - - - - - - + + + + + - - - + + + - - - - - - - - + + + + + + + + - + + - + - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + + - - - - - + - - - - - - - + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - + - + - - + + + + + - - + + + + + + + - - - - - + + + + + + - - + - - - - - - - - - - - - - - + + + + + + + - - - - - + + + + + - + + - + + + - - + + + + + + + + - - - - - - - + + + + + - + - - - - + - - - - - - - - - - - - + + + - - + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + + + + - - - - - + + + + + + - - + + - + - + + - + + + + - + - + - - - - - + + + + - + - + + + - + - - + + - + + - - - + + + - - + + - + - - + + + - - - - - - - + + + + + + - - - - - - + + + - - - - + + + + + + - + - - - - + + - + - + + - - - + + + - - + + - + + - - - - - - + + + + + + - - + - + + - - + - - - - - + + + + + + + + + + + - - - - - - - + + + + + + + + + + + + - - - - - - - - - + + + + + + - - - + + - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - + + + + + + + - - - - + + + + - - - + + - - - - - - - + + + - - - - - - - - - - - - + + + + + + + + - + - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - + - - - - - - - - - + + + + + + - - - + - - - - - - - + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + + + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index d302aab..e1e5831 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,9 +8,11 @@ services: - /path/to/config:/config:rw # network_mode: host environment: + # Timezone - TZ=Europe/London + # Export database to CSV on container start (default: false) - - DB=true + # - DB=true # Update episodes index database on container start (default: false) - EPISODES_UPDATE=true # Nyaa.si search URL (optional, default: https://nyaa.si/?f=0&c=0_0&q=one+pace&o=asc) diff --git a/entrypoint.sh b/entrypoint.sh index 0963736..968d969 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -25,6 +25,9 @@ echo "Running in Docker mode (non-interactive)" echo "------------------------------------------------------------" echo "" +# Media folder: ACEPACE_MEDIA_DIR_DOCKER (default /media) +MEDIA_DIR="${ACEPACE_MEDIA_DIR_DOCKER:-/media}" + # Run episodes update if requested if [ "$EPISODES_UPDATE" = "true" ]; then python /app/acepace.py --episodes_update ${NYAA_URL:+--url "$NYAA_URL"} @@ -50,7 +53,7 @@ fi # Skip when only exporting DB (DB=true and no other operations) if [ "$EPISODES_UPDATE" != "true" ] && { [ "$DB" != "true" ] || [ "$DOWNLOAD" = "true" ]; }; then python /app/acepace.py \ - --folder /media \ + --folder "$MEDIA_DIR" \ ${NYAA_URL:+--url "$NYAA_URL"} EXIT_CODE=$? if [ $EXIT_CODE -ne 0 ]; then @@ -64,7 +67,7 @@ if [ "$RENAME" = "true" ]; then DRY_RUN_RENAME_ARG="" [ "$DRY_RUN" = "true" ] || [ "$DRY_RUN" = "1" ] || [ "$DRY_RUN" = "yes" ] || [ "$DRY_RUN" = "on" ] && DRY_RUN_RENAME_ARG="--dry-run" python /app/acepace.py \ - --folder /media \ + --folder "$MEDIA_DIR" \ --rename \ ${NYAA_URL:+--url "$NYAA_URL"} \ ${DRY_RUN_RENAME_ARG:+$DRY_RUN_RENAME_ARG} @@ -82,7 +85,7 @@ if [ "$DOWNLOAD" = "true" ]; then [ "$DRY_RUN" = "true" ] || [ "$DRY_RUN" = "1" ] || [ "$DRY_RUN" = "yes" ] || [ "$DRY_RUN" = "on" ] && DRY_RUN_ARG="--dry-run" # Use exec to replace shell process so Python becomes PID 1 and receives signals directly exec python /app/acepace.py \ - --folder /media \ + --folder "$MEDIA_DIR" \ ${NYAA_URL:+--url "$NYAA_URL"} \ --download \ ${DRY_RUN_ARG:+$DRY_RUN_ARG} \ diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index 3859a91..dbf33eb 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -164,6 +164,33 @@ def test_rename_dry_run_does_not_confirm_or_execute(self, temp_dir): mock_execute.assert_not_called() conn.close() + def test_ensure_crc32_cache_complete_runs_calculation_when_missing(self, temp_dir): + """When cache is missing CRC32s for some files, calculate_local_crc32 is called.""" + conn = MagicMock() + with patch('acepace._count_video_files') as mock_count: + with patch('acepace.calculate_local_crc32') as mock_calc: + mock_count.return_value = (3, 1) + acepace._ensure_crc32_cache_complete(temp_dir, conn) + mock_calc.assert_called_once_with(temp_dir, conn) + + def test_ensure_crc32_cache_complete_skips_when_up_to_date(self, temp_dir): + """When all files are in cache, calculate_local_crc32 is not called.""" + conn = MagicMock() + with patch('acepace._count_video_files') as mock_count: + with patch('acepace.calculate_local_crc32') as mock_calc: + mock_count.return_value = (2, 2) + acepace._ensure_crc32_cache_complete(temp_dir, conn) + mock_calc.assert_not_called() + + def test_ensure_crc32_cache_complete_skips_when_no_files(self, temp_dir): + """When folder has no video files, calculate_local_crc32 is not called.""" + conn = MagicMock() + with patch('acepace._count_video_files') as mock_count: + with patch('acepace.calculate_local_crc32') as mock_calc: + mock_count.return_value = (0, 0) + acepace._ensure_crc32_cache_complete(temp_dir, conn) + mock_calc.assert_not_called() + class TestCSVExport: """Tests for CSV export functionality.""" diff --git a/tests/test_main_command.py b/tests/test_main_command.py index 7bccf18..a2a141a 100644 --- a/tests/test_main_command.py +++ b/tests/test_main_command.py @@ -294,11 +294,12 @@ def test_rename_receives_url_parameter(self, mock_rename, mock_folder, mock_init with pytest.raises(SystemExit): acepace.main() - # Verify _handle_rename_command was called with URL and dry_run + # Verify _handle_rename_command was called with URL, dry_run, and folder mock_rename.assert_called_once() call_args = mock_rename.call_args assert call_args[0][1] == test_url # Second positional argument is URL assert call_args[1]["dry_run"] is False + assert call_args[1]["folder"] == "/media" @patch('acepace._validate_url') @patch('acepace.init_db') @@ -326,6 +327,7 @@ def test_rename_with_dry_run_passes_dry_run_true(self, mock_rename, mock_folder, mock_rename.assert_called_once() assert mock_rename.call_args[1]["dry_run"] is True + assert mock_rename.call_args[1]["folder"] == "/media" class TestDockerDownloadDefaults: