From b32225323ce0d96a9ab185ea7de37e55d7730410 Mon Sep 17 00:00:00 2001 From: Kevin Li Date: Wed, 11 Mar 2026 21:03:30 -0700 Subject: [PATCH] Fix outlines task1706: pin transformers and tokenizers versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Dockerfile and runner.sh install transformers and tokenizers without version pins. transformers>=5.0.0 combined with tokenizers>=0.22 breaks the test suite in two ways: 1. tokenizers 0.22 turns the BPE.__init__ DeprecationWarning into a hard error, causing GPT2Tokenizer initialization to fail. 2. transformers 5.x has a code path in _from_pretrained that calls import_protobuf_decode_error() inside an except clause, which raises ImportError when protobuf is not installed — even for non-protobuf tokenizers like GPT2. Pin transformers<5 and tokenizers<0.21 in both the Dockerfile and runner.sh to restore compatibility. Verified locally: both feature 4 and feature 6 tests pass (9/9 and 11/11) with the pinned versions. Co-Authored-By: Claude Opus 4.6 --- dataset/dottxt_ai_outlines_task/task1706/Dockerfile | 2 +- dataset/dottxt_ai_outlines_task/task1706/runner.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dataset/dottxt_ai_outlines_task/task1706/Dockerfile b/dataset/dottxt_ai_outlines_task/task1706/Dockerfile index cac085e..bf85d86 100644 --- a/dataset/dottxt_ai_outlines_task/task1706/Dockerfile +++ b/dataset/dottxt_ai_outlines_task/task1706/Dockerfile @@ -26,7 +26,7 @@ ENV LLAMA_CPP_FORCE_CPU=1 ENV PYTORCH_ENABLE_MPS_FALLBACK=1 RUN uv pip install --system -e . && \ uv pip install --system pytest pytest-xdist pytest_mock pytest-asyncio pytest-benchmark pytest-cov && \ - uv pip install --system torch transformers sentencepiece xgrammar llama-cpp-python==0.3.16 psutil + uv pip install --system torch "transformers<5" "tokenizers<0.21" sentencepiece xgrammar llama-cpp-python==0.3.16 psutil # Copy the runner script COPY runner.sh /usr/local/bin/ diff --git a/dataset/dottxt_ai_outlines_task/task1706/runner.sh b/dataset/dottxt_ai_outlines_task/task1706/runner.sh index ef63deb..653367e 100644 --- a/dataset/dottxt_ai_outlines_task/task1706/runner.sh +++ b/dataset/dottxt_ai_outlines_task/task1706/runner.sh @@ -67,7 +67,7 @@ echo "Installing dependencies..." uv sync 2>/dev/null || true uv pip install --system -e . uv pip install --system pytest pytest-xdist pytest_mock pytest-asyncio pytest-benchmark pytest-cov -uv pip install --system torch transformers sentencepiece xgrammar llama-cpp-python==0.3.16 psutil +uv pip install --system torch "transformers<5" "tokenizers<0.21" sentencepiece xgrammar llama-cpp-python==0.3.16 psutil # Run tests with timeout echo "Running tests..."