diff --git a/README.md b/README.md index 28b437d..24bc0d2 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,10 @@ Fixtures are tested against the [libipld](https://github.com/ipld/libipld) stack ### Python Fixtures are tested against the [ipld-dag-pb](https://github.com/storacha/py-ipld-dag-pb) -library: +and [ipld-car](https://github.com/storacha/py-ipld-car) library: * DAG-PB: [ipld-dag-pb](https://pypi.org/project/ipld-dag-pb/) +* CAR: [ipld-car](https://pypi.org/project/ipld-car) ## Running tests diff --git a/fixtures.car b/fixtures.car index 08e36bc..2edf1d7 100644 Binary files a/fixtures.car and b/fixtures.car differ diff --git a/js/make-car.js b/js/make-car.js index f56d12f..2ad5d65 100644 --- a/js/make-car.js +++ b/js/make-car.js @@ -10,13 +10,20 @@ const outStream = createWriteStream(outFile) const { writer, out } = await CarWriter.create([]) const pipe = pipeline(out, outStream) +const blocks = [] for (const { name, url } of fixtureDirectories()) { const data = await loadFixture(url) for (const { cid, bytes } of Object.values(data)) { - await writer.put({ cid: CID.parse(cid), bytes }) + blocks.push({ cid, bytes }) } } +blocks.sort((a, b) => a.cid < b.cid ? -1 : a.cid > b.cid ? 1 : 0) + +for (const { cid, bytes } of blocks) { + await writer.put({ cid: CID.parse(cid), bytes }) +} + await writer.close() await pipe -console.log(`Wrote fixtures to ${outFile}`) \ No newline at end of file +console.log(`Wrote fixtures to ${outFile}`) diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 0000000..82f9275 --- /dev/null +++ b/python/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/python/pyproject.toml b/python/pyproject.toml index 52db23d..035f736 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -3,7 +3,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "py-ipld-dag-pb-test" +name = "python" version = "0.0.1" requires-python = ">=3.10" classifiers = [ @@ -13,5 +13,8 @@ classifiers = [ dependencies = [ "multiformats", "ipld-dag-pb", - "pytest" + "pytest", + "ipld-car", ] +[tool.hatch.build.targets.wheel] +packages = ["python"] diff --git a/python/requirements.txt b/python/requirements.txt index 83414b0..de8faea 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -8,6 +8,8 @@ bases==0.3.0 # via multiformats iniconfig==2.1.0 # via pytest +ipld-car==0.0.1 + # via py-ipld-dag-pb-test (pyproject.toml) ipld-dag-pb==0.0.1 # via py-ipld-dag-pb-test (pyproject.toml) multiformats==0.3.1.post4 diff --git a/python/tests/test_car.py b/python/tests/test_car.py new file mode 100644 index 0000000..003488b --- /dev/null +++ b/python/tests/test_car.py @@ -0,0 +1,47 @@ +import pytest +from pathlib import Path +from multiformats import CID +import ipld_car + +FIXTURES_DIR = Path(__file__).parents[2] / "fixtures" +REPO_ROOT = FIXTURES_DIR.parent + + +def load_all_fixtures() -> list[ipld_car.Block]: + """Load all non-negative fixtures CID and data""" + fixture_blocks: list[ipld_car.Block] = [] + + for dir in sorted(FIXTURES_DIR.iterdir()): + if dir.is_dir(): # skip .gitattributes file + for file in dir.iterdir(): + fixture_cid = file.stem + with file.open(mode="rb") as file_obj: + fixture_data = file_obj.read() + fixture_blocks.append((CID.decode(fixture_cid), fixture_data,)) + + return fixture_blocks + + +@pytest.fixture +def car_fixture_data() -> bytes: + """Load CAR fixture data from CAR file: `fixtures.car`""" + with open(REPO_ROOT / "fixtures.car", mode="rb") as f: + return f.read() + + +def test_car_decode(car_fixture_data): + decoded_roots, decoded_blocks = ipld_car.decode(car_fixture_data) + assert decoded_roots == [] + assert ( + sorted([(block[0].encode(), block[1],) for block in load_all_fixtures()]) == + sorted([(block[0].encode(), block[1],) for block in decoded_blocks]) + ) + + +def test_car_encode(car_fixture_data): + # encoded blocks from the fixtures dir into CAR + fixture_blocks = sorted(load_all_fixtures(), key=lambda block: block[0].encode()) + encoded_fixture_blocks_car = ipld_car.encode(roots=[], blocks=fixture_blocks) + assert encoded_fixture_blocks_car == car_fixture_data + # verify same blocks are present in encoded-then-decoded-fixture-blocks + # and decoded car fixture file