Tests and improvements

Author: luismedel

.github/workflows/cd.yml

@@ -26,14 +26,6 @@ jobs:
     - name: Update pip
       run: |
         python -m pip install --upgrade pip
-    - name: Mypy check
-      run: |
-        pip install mypy
-        mypy --python-version 3.7 src/
-    - name: Test
-      run: |
-        pip install pytest
-        pytest src/tsidpy/*.py --doctest-modules
     - name: Build package
       run: |
         pip install build

.github/workflows/ci.yml

@@ -9,10 +9,8 @@ permissions:
   contents: read
 
 jobs:
-  deploy:
-
+  mypy:
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python
@@ -26,7 +24,26 @@ jobs:
       run: |
         pip install mypy
         mypy --python-version 3.7 src/
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Update pip
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install package and test dependencies
+      run: |
+        pip install -e .[test]
     - name: Test
       run: |
-        pip install pytest
-        pytest src/tsidpy/*.py --doctest-modules
+        pytest --doctest-modules src/tsidpy tests -m "not perf"

CHANGELOG.md

@@ -6,6 +6,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [Unreleased]
+
+### Added
+
+- Automated `pytest` coverage for the public API, deterministic generator behavior, doctests, and opt-in performance benchmarks.
+
+### Changed
+
+- Reduced `TSIDGenerator` hot-path overhead by shrinking the locked section and precomputing the fixed node bit segment.
+- Optimized generic base decoding by replacing repeated `ALPHABET.index()` lookups with a precomputed lookup table.
+- Added fast hexadecimal paths for `TSID.to_string('X'/'x')` and `TSID.from_string('X'/'x')`.
+
+### Performance
+
+- Local `perf` benchmark results on `macOS-15.7.3-arm64-arm-64bit`:
+  - `tsid_from_string_X` improved from about `0.66M ops/s` to about `3.43M ops/s`.
+  - `tsid_to_string_X` improved from about `0.82M ops/s` to about `4.18M ops/s`.
+  - Shared-generator throughput improved from about `1.58M ops/s` to about `1.68M ops/s` in the 1-thread case, with similar throughput retained under multi-thread contention.
+
 ## [1.1.5] - 2023-11-28
 
 ### Added

Makefile

@@ -2,6 +2,12 @@
 clean:
 	rm -rf dist build tsidpy.egg-info
 
+tests:
+	python3 -m pytest --doctest-modules src/tsidpy tests -m "not perf"
+
+bench:
+	python3 -m pytest tests -m perf --run-perf -s
+
 build:
 	python3 -m build
 

pyproject.toml

@@ -19,8 +19,23 @@ classifiers = [
     "Typing :: Typed",
 ]
 
+[project.optional-dependencies]
+test = [
+  "pytest>=7.0",
+]
+
 [project.urls]
 "Homepage" = "https://github.com/luismedel/tsid-python"
 
 [tool.setuptools.package-data]
 tsidpy = ["py.typed"]
+
+[tool.pytest.ini_options]
+addopts = "--strict-markers"
+doctest_optionflags = ["ELLIPSIS"]
+markers = [
+  "perf: opt-in performance and benchmark tests",
+]
+testpaths = [
+  "tests",
+]

src/tsidpy/basen.py

@@ -3,6 +3,7 @@
 
 # Base 62
 ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+ALPHABET_INDEX = {char: index for index, char in enumerate(ALPHABET)}
 
 
 def encode(
@@ -29,6 +30,6 @@ def decode(value: str, base: int) -> int:
 
     for c in value:
         result *= base
-        result += ALPHABET.index(c)
+        result += ALPHABET_INDEX[c]
 
     return result

src/tsidpy/tsid.py

@@ -247,10 +247,9 @@ def to_string(self, fmt: str = 'S') -> str:
         elif fmt == 's':  # canonical string in lower case
             result = self._to_canonical_string().lower()
         elif fmt == 'X':  # hexadecimal in upper case
-            result = encode(self.number, 16, min_length=TSID_BYTES*2)
+            result = format(self.number, '016X')
         elif fmt == 'x':  # hexadecimal in lower case
-            result = encode(self.number, 16, min_length=TSID_BYTES*2)
-            result = result.lower()
+            result = format(self.number, '016x')
         elif fmt == 'd':  # base-10
             result = encode(self.number, 10)
         elif fmt == 'z':  # base-62
@@ -339,13 +338,13 @@ def from_string(value: str, fmt: str = 'S') -> 'TSID':
                 raise ValueError(f'Invalid TSID string: '
                                     f'(len={len(value)} chars, '
                                     f'but expected {TSID_HEXCHARS})')
-            number = decode(value, 16)
+            number = int(value, 16)
         elif fmt == 'x':  # hexadecimal in lower case
             if len(value) != TSID_HEXCHARS:
                 raise ValueError(f'Invalid TSID string: '
                                     f'(len={len(value)} chars, '
                                     f'but expected {TSID_HEXCHARS})')
-            number = decode(value.upper(), 16)
+            number = int(value, 16)
         elif fmt == 'd':  # base-10
             number = decode(value, 10)
         elif fmt == 'z':  # base-62
@@ -425,16 +424,22 @@ def __init__(
                              f'node_bits=={node_bits}')
 
         self._epoch: float = epoch
+        self._epoch_millis: int = int(epoch)
         self._node_bits: int = node_bits
         self._counter_bits: int = RANDOM_BITS - node_bits
         self.counter: int = self.random_fn(self._counter_bits)
 
-        self._millis: float = time.time() * 1000
+        self._millis: int = self._current_millis()
         self._counter_mask: int = RANDOM_MASK >> node_bits
         self._node_mask: int = RANDOM_MASK >> self._counter_bits
+        self._node_value: int = (self.node & self._node_mask) << self._counter_bits
 
         self._lock = threading.Lock()
 
+    @staticmethod
+    def _current_millis() -> int:
+        return int(time.time() * 1000)
+
     def create(self) -> TSID:
         """
         >>> ### Test node extraction ------------------------------
@@ -485,10 +490,10 @@ def create(self) -> TSID:
         True
         """
         with self._lock:
-            current_millis: float = time.time() * 1000
+            current_millis: int = self._current_millis()
 
             # If same millisecond, increment counter
-            if current_millis - self._millis < 1:
+            if current_millis - self._millis <= 1:
                 self.counter += 1
 
                 # If the counter overflows, go to the next millisecond
@@ -500,13 +505,13 @@ def create(self) -> TSID:
                 rnd: int = self.random_fn(self._counter_bits)
                 self.counter = rnd & self._counter_mask
 
-            millis = int(self._millis - self._epoch) << RANDOM_BITS
-            node = (self.node & self._node_mask) << self._counter_bits
+            millis = self._millis
             counter = self.counter & self._counter_mask
 
-            result = TSID(millis + node + counter)
-            result._epoch = self._epoch
-            return result
+        millis_value = (millis - self._epoch_millis) << RANDOM_BITS
+        result = TSID(millis_value + self._node_value + counter)
+        result._epoch = self._epoch
+        return result
 
 
 _default_generator = TSIDGenerator(node_bits=0)

tests/collision_test.py

@@ -0,0 +1,44 @@
+import sys
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SRC = ROOT / "src"
+
+if str(SRC) not in sys.path:
+    sys.path.insert(0, str(SRC))
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--run-perf",
+        action="store_true",
+        default=False,
+        help="run performance benchmarks marked with 'perf'",
+    )
+    parser.addoption(
+        "--perf-loops",
+        action="store",
+        type=int,
+        default=50000,
+        help="iterations used by performance benchmarks",
+    )
+    parser.addoption(
+        "--perf-repeats",
+        action="store",
+        type=int,
+        default=5,
+        help="number of repeated runs per performance scenario",
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--run-perf"):
+        return
+
+    skip_perf = pytest.mark.skip(reason="use --run-perf to run performance tests")
+    for item in items:
+        if "perf" in item.keywords:
+            item.add_marker(skip_perf)