From a56b424cbdab71331c39fbfd70bdce43f4c74bad Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 25 Mar 2026 13:33:20 +0100 Subject: [PATCH 1/2] Add llms.txt and llms-full.txt generation --- .github/workflows/main.yml | 5 +++ .pre-commit-config.yaml | 5 +++ .readthedocs.yml | 18 ++++------ docs/Makefile | 21 ------------ docs/conf.py | 8 +---- docs/make.bat | 36 -------------------- docs/quickstart.rst | 2 +- docs/requirements.in | 2 ++ docs/requirements.txt | 70 ++++++++++++++++++++++++++++++++++++-- pyproject.toml | 3 ++ tox.ini | 10 ++---- 11 files changed, 94 insertions(+), 86 deletions(-) create mode 100644 .pre-commit-config.yaml delete mode 100644 docs/Makefile delete mode 100644 docs/make.bat create mode 100644 docs/requirements.in create mode 100644 pyproject.toml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f4f2a6c4..c4b86e6b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,6 +43,11 @@ jobs: - python-version: "3.13" env: TOXENV: "json" + # Keep in sync with pyproject.toml's + # tool.sphinx-scrapy.python-version. + - python-version: "3.14" + env: + TOXENV: "docs" # https://github.com/kevin1024/vcrpy/issues/885 # - python-version: "pypy3.11-v7.3.20" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..d41921eb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: +- repo: https://github.com/scrapy/sphinx-scrapy + rev: 999017c29b8bc213843e86e0e350a85c47493e34 + hooks: + - id: sphinx-scrapy diff --git a/.readthedocs.yml b/.readthedocs.yml index f9a5741e..3ea6996b 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,15 +1,9 @@ version: 2 -formats: all -sphinx: - configuration: docs/conf.py - fail_on_warning: true build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.10" # Keep in sync with .github/workflows/main.yml -python: - install: - - requirements: docs/requirements.txt - - path: . + python: "3.14" + commands: + - pip install tox + - tox -e docs + - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/ diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index c44acee0..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SPHINXPROJ = python-scrapinghub -SPHINXAPIDOCS = sphinx-apidoc -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py index c3b9b3f6..61181de9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -39,7 +39,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc'] +extensions = ['sphinx_scrapy'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -160,9 +160,3 @@ author, 'python-scrapinghub', 'One line description of project.', 'Miscellaneous'), ] - -# Set Sphinx Read The Docs theme -import sphinx_rtd_theme - -html_theme = 'sphinx_rtd_theme' -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index f7dfb3e2..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,36 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build -set SPHINXPROJ=python-scrapinghub - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 45f9807e..8512bc20 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -4,7 +4,7 @@ Quickstart Requirements ------------ -* Python 3.8+ +* Python 3.9+ Installation diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 00000000..fd365b3c --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,2 @@ +sphinx-rtd-theme +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34 diff --git a/docs/requirements.txt b/docs/requirements.txt index d5476d84..600984e2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,68 @@ -sphinx==7.2.6 -sphinx-rtd-theme==2.0.0 +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in +alabaster==1.0.0 + # via sphinx +babel==2.18.0 + # via sphinx +certifi==2026.2.25 + # via requests +charset-normalizer==3.4.6 + # via requests +docutils==0.22.4 + # via + # sphinx + # sphinx-markdown-builder + # sphinx-rtd-theme +idna==3.11 + # via requests +imagesize==2.0.0 + # via sphinx +jinja2==3.1.6 + # via sphinx +markupsafe==3.0.3 + # via jinja2 +packaging==26.0 + # via + # sphinx + # sphinx-scrapy +pygments==2.19.2 + # via sphinx +requests==2.32.5 + # via sphinx +roman-numerals==4.1.0 + # via sphinx +snowballstemmer==3.0.1 + # via sphinx +sphinx==9.1.0 + # via + # sphinx-llms-txt + # sphinx-markdown-builder + # sphinx-rtd-theme + # sphinx-scrapy + # sphinxcontrib-jquery +sphinx-llms-txt==0.7.1 + # via sphinx-scrapy +sphinx-markdown-builder @ git+https://github.com/AdrianAtZyte/sphinx-markdown-builder.git@2050ffaed43fcc523e03cc7a677cbcb5f5f6d30b + # via sphinx-scrapy +sphinx-rtd-theme==3.1.0 + # via -r requirements.in +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34 + # via -r requirements.in +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +tabulate==0.10.0 + # via sphinx-markdown-builder +urllib3==2.6.3 + # via requests diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..e9210475 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.sphinx-scrapy] +# Keep in sync with .github/workflows/main.yml +python-version = "3.14" diff --git a/tox.ini b/tox.ini index 6498dd88..c3232e66 100644 --- a/tox.ini +++ b/tox.ini @@ -4,8 +4,10 @@ # and then run "tox" from this directory. [tox] +requires = + sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34 # pypy3 excluded due to https://github.com/kevin1024/vcrpy/issues/885 -envlist = py{39,310,311,312}-{json,msgpack} +envlist = py{39,310,311,312}-{json,msgpack},docs [testenv] deps = @@ -13,9 +15,3 @@ deps = msgpack: -r tests/requirements-msgpack.txt commands = py.test --cov=scrapinghub --cov-report=xml {posargs: scrapinghub tests} -[testenv:docs] -changedir = docs -deps = - -r docs/requirements.txt -commands = - sphinx-build -W -b html . {envtmpdir}/html From 4629565ff1db0ea53fde9fe1e24912043d8fc1d5 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 25 Mar 2026 13:38:00 +0100 Subject: [PATCH 2/2] Drop Python 3.9 support --- .github/workflows/main.yml | 6 ------ README.rst | 2 +- docs/quickstart.rst | 2 +- setup.py | 3 +-- tox.ini | 2 +- 5 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c4b86e6b..5bd464b0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,12 +13,6 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.9" - env: - TOXENV: "msgpack" - - python-version: "3.9" - env: - TOXENV: "json" - python-version: "3.10" env: TOXENV: "msgpack" diff --git a/README.rst b/README.rst index c2d58795..92c4f12c 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,7 @@ Client interface for Scrapinghub API .. image:: https://codecov.io/gh/scrapinghub/python-scrapinghub/branch/master/graph/badge.svg :target: https://app.codecov.io/gh/scrapinghub/python-scrapinghub -The ``scrapinghub`` is a Python 3.9+ library for communicating with the +The ``scrapinghub`` is a Python 3.10+ library for communicating with the `Scrapinghub API`_. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 8512bc20..278161e9 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -4,7 +4,7 @@ Quickstart Requirements ------------ -* Python 3.9+ +* Python 3.10+ Installation diff --git a/setup.py b/setup.py index 51ee111b..8f73b841 100644 --- a/setup.py +++ b/setup.py @@ -28,13 +28,12 @@ package_data={'scrapinghub': ['VERSION']}, install_requires=['requests>=1.0', 'retrying>=1.3.3', 'six>=1.10.0'], extras_require={'msgpack': mpack_required}, - python_requires='>=3.9', + python_requires='>=3.10', classifiers=[ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', diff --git a/tox.ini b/tox.ini index c3232e66..d68666c8 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ requires = sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34 # pypy3 excluded due to https://github.com/kevin1024/vcrpy/issues/885 -envlist = py{39,310,311,312}-{json,msgpack},docs +envlist = py{310,311,312}-{json,msgpack},docs [testenv] deps =