Migrate from pyenv to uv

.github/workflows/integration-tests.yml

@@ -35,10 +35,10 @@ jobs:
     steps:
     - name: 'Check out repo'
       uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3.10
       uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         cd workflows/

.github/workflows/unit-tests.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: ["3.10", "3.13"]
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}

CONTRIBUTING.md

@@ -31,23 +31,27 @@ This library has been developed on Mac OS. To get started:
 ❯ make setup
 ```
 
-Will update `brew`, install `pyenv` and other things an required by ML libraries (e.g `libomp`, required by `xgboost`).
+Will update `brew`, install `uv` and other things required by ML libraries (e.g `libomp`, required by `xgboost`).
 
 ### Setup a virtual environment
 
-This library has been developed using [pyenv](https://github.com/pyenv/pyenv) and [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv), using the requirements that are in `requirements.txt` and `requirements-dev.txt`.
+This library uses [uv](https://github.com/astral-sh/uv) for managing virtual environments, using the requirements that are in `requirements.txt` and `requirements-dev.txt`.
 
 This project has two types of requirements files:
 * `requirements.txt` contains any dependencies that `modelstore` users must have in order to use `modelstore`. This should be as lightweight as possible. We do not require users to install every single machine learning library - just the ones that they want to use.
-* `requirements-dev[X].txt` contains all of the dependencies that `modelstore` developers must have. These files contain all of the machine learning frameworks that are supported by `modelstore` - they must be installed to enable running all of the unit tests. 
+* `requirements-dev[X].txt` contains all of the dependencies that `modelstore` developers must have. These files contain all of the machine learning frameworks that are supported by `modelstore` - they must be installed to enable running all of the unit tests.
 
-Once you have set up `pyenv` and `pyenv-virtualenv` installed, use this `Makefile` command that does the rest for you:
+Once you have `uv` installed, use this `Makefile` command that does the rest for you:
 
 ```bash
 ❯ make install
 ```
 
-This will create a Python virtual environment, using `pyenv-virtualenv`, and install all of the dependencies in the requirements files. If you want to use a different version of Python, update the [bin/_config](bin/config) file.
+This will create a `.venv` virtual environment in the project root and install all of the dependencies in the requirements files. The Python version is specified in `.python-version`. Activate the environment with:
+
+```bash
+❯ source .venv/bin/activate
+```
 
 Notes:
 * I've seen trouble with installing `prophet` and have sometimes had to install it manually

Dockerfile

@@ -1,26 +1,24 @@
-FROM ubuntu
+FROM python:3.11-slim
 WORKDIR /usr/src/app
 
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
     apt-get install -y build-essential && \
     apt-get install -y git ninja-build ccache libopenblas-dev libopencv-dev cmake && \
     apt-get install -y gcc mono-mcs g++ && \
-    apt-get install -y python3 python3-pip && \
     apt-get install -y default-jdk && \
     apt-get install -y libhdf5-dev && \
     rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install --upgrade pip setuptools wheel
-
 # Install & install requirements
 COPY requirements-dev0.txt ./requirements-dev0.txt
 COPY requirements-dev1.txt ./requirements-dev1.txt
 COPY requirements.txt ./requirements.txt
 
-RUN pip3 install -r requirements-dev0.txt
-RUN pip3 install -r requirements-dev1.txt
-RUN pip3 install -r requirements.txt
+RUN pip install --upgrade pip setuptools wheel
+RUN pip install -r requirements-dev0.txt
+RUN pip install -r requirements-dev1.txt
+RUN pip install -r requirements.txt
 
 # Copy library source
 COPY modelstore ./modelstore

Makefile

@@ -1,20 +1,18 @@
-VIRTUALENV_NAME=$(shell pwd | rev | cut -d '/' -f 1 | rev)-dev
-
 .PHONY: uninstall
 uninstall:
-	@./bin/_pyenv_uninstall $(VIRTUALENV_NAME)
+	@./bin/_uv_uninstall
 
 .PHONY: setup
 setup:
 	@./bin/_brew_install
 
 .PHONY: install
 install: uninstall
-	@./bin/_pyenv_install $(VIRTUALENV_NAME)
+	@./bin/_uv_install
 
 .PHONY: update
 update:
-	@./bin/_pyenv_update
+	@./bin/_uv_update
 
 .PHONY: build
 build:

bin/_brew_install

@@ -12,9 +12,8 @@ function install {
 echo -e "\n 💬  Running brew update..."
 brew update
 
-echo -e "\n 💬  Installing pyenv & pyenv-virtualenv..."
-install pyenv
-install pyenv-virtualenv
+echo -e "\n 💬  Installing uv..."
+install uv
 
 # To get pystan to install correctly (required by prophet)
 # https://stackoverflow.com/questions/52814868/pystan-compileerror-command-gcc-failed-with-exit-status-1-macos

bin/_uv_install

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+REPO_ROOT=$(cd $(dirname $0)/.. && pwd)
+
+echo -e "\n 💬  Installing..."
+
+echo -e "\n 💬  Creating virtual environment..."
+uv venv
+
+echo -e "\n 💬  Upgrading pip, setuptools, wheel..."
+uv pip install --upgrade pip setuptools wheel
+
+for i in "$REPO_ROOT"/requirements*txt; do
+    echo -e "\n\n 💬  Installing requirements in: $i"
+    uv pip install -r "$i"
+done
+
+uv pip install -e "$REPO_ROOT"
+echo -e "\n ✅  Done."

bin/_uv_uninstall

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+REPO_ROOT=$(cd $(dirname $0)/.. && pwd)
+
+echo -e "\n 💬  Uninstalling..."
+
+if [[ -d "$REPO_ROOT/.venv" ]]; then
+    echo -e "\n ⏱  Removing: $REPO_ROOT/.venv"
+    rm -rf "$REPO_ROOT/.venv"
+    echo -e "\n ✅  Done."
+else
+    echo -e "\n ✅  Nothing to do."
+fi

bin/_uv_update

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+REPO_ROOT=$(cd $(dirname $0)/.. && pwd)
+
+echo -e "\n 💬  Updating..."
+
+uv pip install --upgrade pip setuptools wheel
+for i in "$REPO_ROOT"/requirements*txt; do
+    echo -e "\n\n 💬  Updating requirements in: $i"
+    uv pip install --upgrade -r "$i"
+done
+
+uv pip install -e "$REPO_ROOT"
+echo -e "\n ✅  Done."

modelstore/models/pytorch.py

@@ -114,7 +114,7 @@ def load(self, model_path: str, meta_data: metadata.Summary) -> Any:
         import torch
 
         file_path = _get_model_path(model_path)
-        return torch.load(file_path)
+        return torch.load(file_path, weights_only=False)
 
 
 def _get_model_path(parent_dir: str) -> str:

modelstore/models/transformers.py

@@ -80,7 +80,7 @@ def matches_with(self, **kwargs) -> bool:
 
             if isinstance(kwargs.get("model"), TFPreTrainedModel):
                 return True
-        except RuntimeError:
+        except (RuntimeError, ImportError):
             # Cannot import tensorflow things
             pass
 
@@ -143,7 +143,7 @@ def load(self, model_path: str, meta_data: metadata.Summary) -> Any:
 
         # Infer whether we're loading a PyTorch or Tensorflow model
         # @TODO: this does not appear to hold with more recent versions of transformers
-        is_pytorch = "pytorch_model.bin" in model_files
+        is_pytorch = "pytorch_model.bin" in model_files or "model.safetensors" in model_files
         logger.debug("Loading transformers model with pytorch=%s", is_pytorch)
 
         if is_pytorch:

requirements-dev0.txt

@@ -1,7 +1,7 @@
 # Code & testing
 black>=22.12.0
 flake8>=5.0.4
-isort==5.11.3  # Note: this version is asserted in unit tests
+isort==5.11.3
 moto>=4.0.11
 pylint>=2.15.8
 pytest>=7.2.0
@@ -11,12 +11,11 @@ twine>=4.0.2
 
 # Data / dependencies for ML libraries
 numba>=0.58.1
-numpy==1.23.5
+numpy
 Cython>=3.0.8
 python-Levenshtein>=0.24.0
-pandas>=1.3.5; python_version < '3.8'
-pandas>=1.4.1; python_version > '3.7'
-scipy==1.10.1 # More recent versions were not compatible with Gensim releases https://github.com/piskvorky/gensim/issues/3525
+pandas>=1.4.1
+scipy
 
 # ML Dependencies
 # pydoop<=2.0.0; sys_platform == 'darwin'

requirements-dev1.txt

@@ -13,6 +13,7 @@ annoy
 catboost
 causalml
 fastai  # Note: 1.0.61 has different import paths!
+ipython  # Required by fastprogress (fastai dependency)
 gensim
 Keras-Preprocessing
 lightgbm

setup.py

@@ -18,13 +18,14 @@
     author="Neal Lathia",
     classifiers=[
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
         "License :: OSI Approved :: Apache Software License",
     ],
     license="Please refer to the readme",
-    python_requires=">=3.6",
+    python_requires=">=3.10",
     install_requires=requirements,
     entry_points={
         'console_scripts': ['modelstore=modelstore.__main__:cli']

tests/metadata/code/test_dependencies.py

@@ -13,6 +13,7 @@
 #    limitations under the License.
 import sys
 
+import isort
 import pytest
 
 from modelstore.metadata.code import dependencies
@@ -27,7 +28,7 @@ def test_get_version():
     if "isort" in sys.modules:
         # Force import
         del sys.modules["isort"]
-    assert dependencies._get_version("isort") == "5.11.3"
+    assert dependencies._get_version("isort") == isort.__version__
 
 
 def test_get_dependency_versions():

tests/models/test_pytorch.py

@@ -114,7 +114,7 @@ def test_save_model(pytorch_model, tmp_path):
     file_path = _save_model(tmp_path, pytorch_model)
     assert exp == file_path
 
-    model = torch.load(file_path)
+    model = torch.load(file_path, weights_only=False)
     assert_models_equal(pytorch_model, model)
 
 
@@ -123,7 +123,7 @@ def test_save_state_dict(pytorch_model, pytorch_optim, tmp_path):
     file_path = _save_state_dict(tmp_path, pytorch_model, pytorch_optim)
     assert file_path == exp
 
-    state_dict = torch.load(file_path)
+    state_dict = torch.load(file_path, weights_only=False)
     model = ExampleNet()
 
     model.load_state_dict(state_dict["model_state_dict"])
@@ -135,7 +135,7 @@ def test_save_state_dict_without_optimizer(pytorch_model, tmp_path):
     file_path = _save_state_dict(tmp_path, pytorch_model)
     assert file_path == exp
 
-    state_dict = torch.load(file_path)
+    state_dict = torch.load(file_path, weights_only=False)
     model = ExampleNet()
 
     model.load_state_dict(state_dict["model_state_dict"])

tests/models/test_transformers.py

@@ -19,8 +19,8 @@
     AutoModelForSequenceClassification,
     AutoTokenizer,
     DistilBertForSequenceClassification,
-    TFDistilBertModel,
-    DistilBertTokenizerFast,
+    PreTrainedModel,
+    PreTrainedTokenizerBase,
     PreTrainedTokenizerFast
 )
 
@@ -156,6 +156,6 @@ def test_load_model(tmp_path, tr_manager, tr_model, tr_config, tr_tokenizer):
     loaded_model, loaded_tokenizer, loaded_config = tr_manager.load(tmp_path, meta_data)
 
     # Expect the two to be the same
-    assert isinstance(loaded_model, TFDistilBertModel)
+    assert isinstance(loaded_model, PreTrainedModel)
     assert isinstance(loaded_config, type(tr_config))
-    assert isinstance(loaded_tokenizer, DistilBertTokenizerFast)
+    assert isinstance(loaded_tokenizer, PreTrainedTokenizerBase)