diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e707d1e5..9467dc98 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -35,10 +35,10 @@ jobs: steps: - name: 'Check out repo' uses: actions/checkout@v3 - - name: Set up Python 3.8 + - name: Set up Python 3.10 uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: "3.10" - name: Install dependencies run: | cd workflows/ diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index de03b39b..d3a0ee0d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9] + python-version: ["3.10", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0337a97a..ad623446 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,23 +31,27 @@ This library has been developed on Mac OS. To get started: ❯ make setup ``` -Will update `brew`, install `pyenv` and other things an required by ML libraries (e.g `libomp`, required by `xgboost`). +Will update `brew`, install `uv` and other things required by ML libraries (e.g `libomp`, required by `xgboost`). ### Setup a virtual environment -This library has been developed using [pyenv](https://github.com/pyenv/pyenv) and [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv), using the requirements that are in `requirements.txt` and `requirements-dev.txt`. +This library uses [uv](https://github.com/astral-sh/uv) for managing virtual environments, using the requirements that are in `requirements.txt` and `requirements-dev.txt`. This project has two types of requirements files: * `requirements.txt` contains any dependencies that `modelstore` users must have in order to use `modelstore`. This should be as lightweight as possible. We do not require users to install every single machine learning library - just the ones that they want to use. -* `requirements-dev[X].txt` contains all of the dependencies that `modelstore` developers must have. These files contain all of the machine learning frameworks that are supported by `modelstore` - they must be installed to enable running all of the unit tests. +* `requirements-dev[X].txt` contains all of the dependencies that `modelstore` developers must have. These files contain all of the machine learning frameworks that are supported by `modelstore` - they must be installed to enable running all of the unit tests. -Once you have set up `pyenv` and `pyenv-virtualenv` installed, use this `Makefile` command that does the rest for you: +Once you have `uv` installed, use this `Makefile` command that does the rest for you: ```bash ❯ make install ``` -This will create a Python virtual environment, using `pyenv-virtualenv`, and install all of the dependencies in the requirements files. If you want to use a different version of Python, update the [bin/_config](bin/config) file. +This will create a `.venv` virtual environment in the project root and install all of the dependencies in the requirements files. The Python version is specified in `.python-version`. Activate the environment with: + +```bash +❯ source .venv/bin/activate +``` Notes: * I've seen trouble with installing `prophet` and have sometimes had to install it manually diff --git a/Dockerfile b/Dockerfile index 021b021a..47d1d9ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu +FROM python:3.11-slim WORKDIR /usr/src/app ARG DEBIAN_FRONTEND=noninteractive @@ -6,21 +6,19 @@ RUN apt-get update && \ apt-get install -y build-essential && \ apt-get install -y git ninja-build ccache libopenblas-dev libopencv-dev cmake && \ apt-get install -y gcc mono-mcs g++ && \ - apt-get install -y python3 python3-pip && \ apt-get install -y default-jdk && \ apt-get install -y libhdf5-dev && \ rm -rf /var/lib/apt/lists/* -RUN pip3 install --upgrade pip setuptools wheel - # Install & install requirements COPY requirements-dev0.txt ./requirements-dev0.txt COPY requirements-dev1.txt ./requirements-dev1.txt COPY requirements.txt ./requirements.txt -RUN pip3 install -r requirements-dev0.txt -RUN pip3 install -r requirements-dev1.txt -RUN pip3 install -r requirements.txt +RUN pip install --upgrade pip setuptools wheel +RUN pip install -r requirements-dev0.txt +RUN pip install -r requirements-dev1.txt +RUN pip install -r requirements.txt # Copy library source COPY modelstore ./modelstore diff --git a/Makefile b/Makefile index 552759fd..270abff9 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,6 @@ -VIRTUALENV_NAME=$(shell pwd | rev | cut -d '/' -f 1 | rev)-dev - .PHONY: uninstall uninstall: - @./bin/_pyenv_uninstall $(VIRTUALENV_NAME) + @./bin/_uv_uninstall .PHONY: setup setup: @@ -10,11 +8,11 @@ setup: .PHONY: install install: uninstall - @./bin/_pyenv_install $(VIRTUALENV_NAME) + @./bin/_uv_install .PHONY: update update: - @./bin/_pyenv_update + @./bin/_uv_update .PHONY: build build: diff --git a/bin/_brew_install b/bin/_brew_install index 0d4a4981..082b5791 100755 --- a/bin/_brew_install +++ b/bin/_brew_install @@ -12,9 +12,8 @@ function install { echo -e "\n 💬 Running brew update..." brew update -echo -e "\n 💬 Installing pyenv & pyenv-virtualenv..." -install pyenv -install pyenv-virtualenv +echo -e "\n 💬 Installing uv..." +install uv # To get pystan to install correctly (required by prophet) # https://stackoverflow.com/questions/52814868/pystan-compileerror-command-gcc-failed-with-exit-status-1-macos diff --git a/bin/_pyenv_config b/bin/_pyenv_config deleted file mode 100755 index 1ac89ad7..00000000 --- a/bin/_pyenv_config +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# export PYTHON_VERSION=3.7.15 -export PYTHON_VERSION=3.8.12 -# export PYTHON_VERSION=3.9.16 - -export VIRTUALENV_NAME="$1-${PYTHON_VERSION//./-}" -export REPO_ROOT=$(cd $(dirname $0)/.. && pwd) - -echo -e "\n 💬 Using a venv called: ${VIRTUALENV_NAME}" - -eval "$(pyenv init --path)" -eval "$(pyenv virtualenv-init -)" diff --git a/bin/_pyenv_install b/bin/_pyenv_install deleted file mode 100755 index 34a69a12..00000000 --- a/bin/_pyenv_install +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -e - -echo -e "\n 💬 Installing..." - -source $(dirname $0)/_pyenv_config "$@" - -if [[ $(pyenv versions | grep -L $PYTHON_VERSION) ]]; then - echo -e "\n 💬 Installing Python $PYTHON_VERSION" - pyenv install $PYTHON_VERSION -fi - -echo -e "\n 💬 Creating a $PYTHON_VERSION environment: $VIRTUALENV_NAME" -env PYTHON_CONFIGURE_OPTS="--enable-framework CC=clang" \ - pyenv virtualenv \ - --force $PYTHON_VERSION \ - "$VIRTUALENV_NAME" - -echo -e "\n 💬 Setting local: $VIRTUALENV_NAME" -pyenv local $VIRTUALENV_NAME - -echo -e "\n 💬 Upgrading pip" -pip install --upgrade pip setuptools wheel - -for i in ./requirements*txt; do - echo -e "\n\n 💬 Installing requirements in: $i" - pip install -r $i -done - -pip install -e $REPO_ROOT -echo -e "\n ✅ Done." diff --git a/bin/_pyenv_uninstall b/bin/_pyenv_uninstall deleted file mode 100755 index 76cb6eb0..00000000 --- a/bin/_pyenv_uninstall +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -e - -echo -e "\n 💬 Uninstalling..." - -source $(dirname $0)/_pyenv_config "$@" - -if [[ -f ".python-version" ]]; then - # Keep pyenv-virtualenvs for other versions of Python - if [ "${VIRTUALENV_NAME}" == "$(cat .python-version)" ] ;then - echo -e "\n ⏱ Force removing: $VIRTUALENV_NAME" - pyenv uninstall -f $VIRTUALENV_NAME - fi - rm .python-version - echo -e "\n ✅ Done." -else - echo -e "\n ✅ Nothing to do." -fi - diff --git a/bin/_pyenv_update b/bin/_pyenv_update deleted file mode 100755 index 2abcbab6..00000000 --- a/bin/_pyenv_update +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -echo -e "\n 💬 Updating..." - -pip install --upgrade pip setuptools wheel -for i in ./requirements*txt; do - echo -e "\n\n 💬 Updating requirements in: $i" - pip install --upgrade -r $i -done - -pip install -e $REPO_ROOT -echo -e "\n ✅ Done." diff --git a/bin/_uv_install b/bin/_uv_install new file mode 100755 index 00000000..220624fa --- /dev/null +++ b/bin/_uv_install @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +REPO_ROOT=$(cd $(dirname $0)/.. && pwd) + +echo -e "\n 💬 Installing..." + +echo -e "\n 💬 Creating virtual environment..." +uv venv + +echo -e "\n 💬 Upgrading pip, setuptools, wheel..." +uv pip install --upgrade pip setuptools wheel + +for i in "$REPO_ROOT"/requirements*txt; do + echo -e "\n\n 💬 Installing requirements in: $i" + uv pip install -r "$i" +done + +uv pip install -e "$REPO_ROOT" +echo -e "\n ✅ Done." diff --git a/bin/_uv_uninstall b/bin/_uv_uninstall new file mode 100755 index 00000000..dbfcd217 --- /dev/null +++ b/bin/_uv_uninstall @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +REPO_ROOT=$(cd $(dirname $0)/.. && pwd) + +echo -e "\n 💬 Uninstalling..." + +if [[ -d "$REPO_ROOT/.venv" ]]; then + echo -e "\n ⏱ Removing: $REPO_ROOT/.venv" + rm -rf "$REPO_ROOT/.venv" + echo -e "\n ✅ Done." +else + echo -e "\n ✅ Nothing to do." +fi diff --git a/bin/_uv_update b/bin/_uv_update new file mode 100755 index 00000000..e5ac6961 --- /dev/null +++ b/bin/_uv_update @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +REPO_ROOT=$(cd $(dirname $0)/.. && pwd) + +echo -e "\n 💬 Updating..." + +uv pip install --upgrade pip setuptools wheel +for i in "$REPO_ROOT"/requirements*txt; do + echo -e "\n\n 💬 Updating requirements in: $i" + uv pip install --upgrade -r "$i" +done + +uv pip install -e "$REPO_ROOT" +echo -e "\n ✅ Done." diff --git a/modelstore/models/pytorch.py b/modelstore/models/pytorch.py index d8f79978..4d836bbc 100644 --- a/modelstore/models/pytorch.py +++ b/modelstore/models/pytorch.py @@ -114,7 +114,7 @@ def load(self, model_path: str, meta_data: metadata.Summary) -> Any: import torch file_path = _get_model_path(model_path) - return torch.load(file_path) + return torch.load(file_path, weights_only=False) def _get_model_path(parent_dir: str) -> str: diff --git a/modelstore/models/transformers.py b/modelstore/models/transformers.py index 7f9a2df9..272b36b3 100644 --- a/modelstore/models/transformers.py +++ b/modelstore/models/transformers.py @@ -80,7 +80,7 @@ def matches_with(self, **kwargs) -> bool: if isinstance(kwargs.get("model"), TFPreTrainedModel): return True - except RuntimeError: + except (RuntimeError, ImportError): # Cannot import tensorflow things pass @@ -143,7 +143,7 @@ def load(self, model_path: str, meta_data: metadata.Summary) -> Any: # Infer whether we're loading a PyTorch or Tensorflow model # @TODO: this does not appear to hold with more recent versions of transformers - is_pytorch = "pytorch_model.bin" in model_files + is_pytorch = "pytorch_model.bin" in model_files or "model.safetensors" in model_files logger.debug("Loading transformers model with pytorch=%s", is_pytorch) if is_pytorch: diff --git a/requirements-dev0.txt b/requirements-dev0.txt index 3a8c73ee..21870214 100644 --- a/requirements-dev0.txt +++ b/requirements-dev0.txt @@ -1,7 +1,7 @@ # Code & testing black>=22.12.0 flake8>=5.0.4 -isort==5.11.3 # Note: this version is asserted in unit tests +isort==5.11.3 moto>=4.0.11 pylint>=2.15.8 pytest>=7.2.0 @@ -11,12 +11,11 @@ twine>=4.0.2 # Data / dependencies for ML libraries numba>=0.58.1 -numpy==1.23.5 +numpy Cython>=3.0.8 python-Levenshtein>=0.24.0 -pandas>=1.3.5; python_version < '3.8' -pandas>=1.4.1; python_version > '3.7' -scipy==1.10.1 # More recent versions were not compatible with Gensim releases https://github.com/piskvorky/gensim/issues/3525 +pandas>=1.4.1 +scipy # ML Dependencies # pydoop<=2.0.0; sys_platform == 'darwin' diff --git a/requirements-dev1.txt b/requirements-dev1.txt index 08524da0..20d5f894 100644 --- a/requirements-dev1.txt +++ b/requirements-dev1.txt @@ -13,6 +13,7 @@ annoy catboost causalml fastai # Note: 1.0.61 has different import paths! +ipython # Required by fastprogress (fastai dependency) gensim Keras-Preprocessing lightgbm diff --git a/setup.py b/setup.py index 74a0ca79..5fff72f7 100644 --- a/setup.py +++ b/setup.py @@ -18,13 +18,14 @@ author="Neal Lathia", classifiers=[ "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "License :: OSI Approved :: Apache Software License", ], license="Please refer to the readme", - python_requires=">=3.6", + python_requires=">=3.10", install_requires=requirements, entry_points={ 'console_scripts': ['modelstore=modelstore.__main__:cli'] diff --git a/tests/metadata/code/test_dependencies.py b/tests/metadata/code/test_dependencies.py index 89a2b799..71639132 100644 --- a/tests/metadata/code/test_dependencies.py +++ b/tests/metadata/code/test_dependencies.py @@ -13,6 +13,7 @@ # limitations under the License. import sys +import isort import pytest from modelstore.metadata.code import dependencies @@ -27,7 +28,7 @@ def test_get_version(): if "isort" in sys.modules: # Force import del sys.modules["isort"] - assert dependencies._get_version("isort") == "5.11.3" + assert dependencies._get_version("isort") == isort.__version__ def test_get_dependency_versions(): diff --git a/tests/models/test_pytorch.py b/tests/models/test_pytorch.py index 7c364d84..13729f59 100644 --- a/tests/models/test_pytorch.py +++ b/tests/models/test_pytorch.py @@ -114,7 +114,7 @@ def test_save_model(pytorch_model, tmp_path): file_path = _save_model(tmp_path, pytorch_model) assert exp == file_path - model = torch.load(file_path) + model = torch.load(file_path, weights_only=False) assert_models_equal(pytorch_model, model) @@ -123,7 +123,7 @@ def test_save_state_dict(pytorch_model, pytorch_optim, tmp_path): file_path = _save_state_dict(tmp_path, pytorch_model, pytorch_optim) assert file_path == exp - state_dict = torch.load(file_path) + state_dict = torch.load(file_path, weights_only=False) model = ExampleNet() model.load_state_dict(state_dict["model_state_dict"]) @@ -135,7 +135,7 @@ def test_save_state_dict_without_optimizer(pytorch_model, tmp_path): file_path = _save_state_dict(tmp_path, pytorch_model) assert file_path == exp - state_dict = torch.load(file_path) + state_dict = torch.load(file_path, weights_only=False) model = ExampleNet() model.load_state_dict(state_dict["model_state_dict"]) diff --git a/tests/models/test_transformers.py b/tests/models/test_transformers.py index f68b1fad..34b4335d 100644 --- a/tests/models/test_transformers.py +++ b/tests/models/test_transformers.py @@ -19,8 +19,8 @@ AutoModelForSequenceClassification, AutoTokenizer, DistilBertForSequenceClassification, - TFDistilBertModel, - DistilBertTokenizerFast, + PreTrainedModel, + PreTrainedTokenizerBase, PreTrainedTokenizerFast ) @@ -156,6 +156,6 @@ def test_load_model(tmp_path, tr_manager, tr_model, tr_config, tr_tokenizer): loaded_model, loaded_tokenizer, loaded_config = tr_manager.load(tmp_path, meta_data) # Expect the two to be the same - assert isinstance(loaded_model, TFDistilBertModel) + assert isinstance(loaded_model, PreTrainedModel) assert isinstance(loaded_config, type(tr_config)) - assert isinstance(loaded_tokenizer, DistilBertTokenizerFast) + assert isinstance(loaded_tokenizer, PreTrainedTokenizerBase) diff --git a/workflows/Makefile b/workflows/Makefile index 3fc026a0..ed7d5971 100644 --- a/workflows/Makefile +++ b/workflows/Makefile @@ -1,21 +1,20 @@ -VIRTUALENV_NAME=modelstore.$(shell pwd | rev | cut -d '/' -f 1 | rev) REPO_ROOT=$(shell cd ../ && pwd) -.PHONY: setup pyenv pyenv-uninstall refresh +.PHONY: setup install uninstall refresh -pyenv: pyenv-uninstall - @$(REPO_ROOT)/bin/_pyenv_install $(VIRTUALENV_NAME) - find requirements/ -name "*.txt" -type f -exec pip install -r '{}' ';' +install: uninstall + @$(REPO_ROOT)/bin/_uv_install + find requirements/ -name "*.txt" -type f -exec uv pip install -r '{}' ';' + +uninstall: + @$(REPO_ROOT)/bin/_uv_uninstall refresh: @echo "\n 🔵 Refreshing installation of modelstore" - pip install --upgrade pip setuptools wheel - pip uninstall -y modelstore - pip install --no-cache-dir -e $(REPO_ROOT) - -pyenv-uninstall: - @$(REPO_ROOT)/bin/_pyenv_uninstall $(VIRTUALENV_NAME) + uv pip install --upgrade pip setuptools wheel + uv pip uninstall -y modelstore + uv pip install --no-cache-dir -e $(REPO_ROOT) setup: - pip install --upgrade pip setuptools wheel + pip install --upgrade pip setuptools wheel pip install -r requirements.txt diff --git a/workflows/actions/actions.py b/workflows/actions/actions.py index bb73fecf..eea8bc3b 100644 --- a/workflows/actions/actions.py +++ b/workflows/actions/actions.py @@ -15,7 +15,7 @@ from modelstore import ModelStore # pylint: disable=import-error -from workflows.actions import cli, models, storage +from actions import cli, models, storage MODEL_DOMAIN = "diabetes-boosting-demo"