Add llms.txt and llms-full.txt generation

.github/workflows/main.yml

@@ -13,12 +13,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python-version: "3.9"
-            env:
-              TOXENV: "msgpack"
-          - python-version: "3.9"
-            env:
-              TOXENV: "json"
           - python-version: "3.10"
             env:
               TOXENV: "msgpack"
@@ -43,6 +37,11 @@ jobs:
           - python-version: "3.13"
             env:
               TOXENV: "json"
+          # Keep in sync with pyproject.toml's
+          # tool.sphinx-scrapy.python-version.
+          - python-version: "3.14"
+            env:
+              TOXENV: "docs"
 
           # https://github.com/kevin1024/vcrpy/issues/885
           # - python-version: "pypy3.11-v7.3.20"

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+repos:
+- repo: https://github.com/scrapy/sphinx-scrapy
+  rev: 999017c29b8bc213843e86e0e350a85c47493e34
+  hooks:
+    - id: sphinx-scrapy

.readthedocs.yml

@@ -1,15 +1,9 @@
 version: 2
-formats: all
-sphinx:
-  configuration: docs/conf.py
-  fail_on_warning: true
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    # For available versions, see:
-    # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
-    python: "3.10"  # Keep in sync with .github/workflows/main.yml
-python:
-  install:
-    - requirements: docs/requirements.txt
-    - path: .
+    python: "3.14"
+  commands:
+    - pip install tox
+    - tox -e docs
+    - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/

README.rst

@@ -14,7 +14,7 @@ Client interface for Scrapinghub API
 .. image:: https://codecov.io/gh/scrapinghub/python-scrapinghub/branch/master/graph/badge.svg
    :target: https://app.codecov.io/gh/scrapinghub/python-scrapinghub
 
-The ``scrapinghub`` is a Python 3.9+ library for communicating with the
+The ``scrapinghub`` is a Python 3.10+ library for communicating with the
 `Scrapinghub API`_.
 
 

docs/conf.py

@@ -39,7 +39,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.autodoc']
+extensions = ['sphinx_scrapy']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -160,9 +160,3 @@
      author, 'python-scrapinghub', 'One line description of project.',
      'Miscellaneous'),
 ]
-
-# Set Sphinx Read The Docs theme
-import sphinx_rtd_theme
-
-html_theme = 'sphinx_rtd_theme'
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

docs/quickstart.rst

@@ -4,7 +4,7 @@ Quickstart
 Requirements
 ------------
 
-* Python 3.8+
+* Python 3.10+
 
 
 Installation

docs/requirements.in

@@ -0,0 +1,2 @@
+sphinx-rtd-theme
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34

docs/requirements.txt

@@ -1,2 +1,68 @@
-sphinx==7.2.6
-sphinx-rtd-theme==2.0.0
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements.in
+alabaster==1.0.0
+    # via sphinx
+babel==2.18.0
+    # via sphinx
+certifi==2026.2.25
+    # via requests
+charset-normalizer==3.4.6
+    # via requests
+docutils==0.22.4
+    # via
+    #   sphinx
+    #   sphinx-markdown-builder
+    #   sphinx-rtd-theme
+idna==3.11
+    # via requests
+imagesize==2.0.0
+    # via sphinx
+jinja2==3.1.6
+    # via sphinx
+markupsafe==3.0.3
+    # via jinja2
+packaging==26.0
+    # via
+    #   sphinx
+    #   sphinx-scrapy
+pygments==2.19.2
+    # via sphinx
+requests==2.32.5
+    # via sphinx
+roman-numerals==4.1.0
+    # via sphinx
+snowballstemmer==3.0.1
+    # via sphinx
+sphinx==9.1.0
+    # via
+    #   sphinx-llms-txt
+    #   sphinx-markdown-builder
+    #   sphinx-rtd-theme
+    #   sphinx-scrapy
+    #   sphinxcontrib-jquery
+sphinx-llms-txt==0.7.1
+    # via sphinx-scrapy
+sphinx-markdown-builder @ git+https://github.com/AdrianAtZyte/sphinx-markdown-builder.git@2050ffaed43fcc523e03cc7a677cbcb5f5f6d30b
+    # via sphinx-scrapy
+sphinx-rtd-theme==3.1.0
+    # via -r requirements.in
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34
+    # via -r requirements.in
+sphinxcontrib-applehelp==2.0.0
+    # via sphinx
+sphinxcontrib-devhelp==2.0.0
+    # via sphinx
+sphinxcontrib-htmlhelp==2.1.0
+    # via sphinx
+sphinxcontrib-jquery==4.1
+    # via sphinx-rtd-theme
+sphinxcontrib-jsmath==1.0.1
+    # via sphinx
+sphinxcontrib-qthelp==2.0.0
+    # via sphinx
+sphinxcontrib-serializinghtml==2.0.0
+    # via sphinx
+tabulate==0.10.0
+    # via sphinx-markdown-builder
+urllib3==2.6.3
+    # via requests

pyproject.toml

@@ -0,0 +1,3 @@
+[tool.sphinx-scrapy]
+# Keep in sync with .github/workflows/main.yml
+python-version = "3.14"

setup.py

@@ -28,13 +28,12 @@
     package_data={'scrapinghub': ['VERSION']},
     install_requires=['requests>=1.0', 'retrying>=1.3.3', 'six>=1.10.0'],
     extras_require={'msgpack': mpack_required},
-    python_requires='>=3.9',
+    python_requires='>=3.10',
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'License :: OSI Approved :: BSD License',
         'Operating System :: OS Independent',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: 3.12',

tox.ini

@@ -4,18 +4,14 @@
 # and then run "tox" from this directory.
 
 [tox]
+requires =
+    sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34
 # pypy3 excluded due to https://github.com/kevin1024/vcrpy/issues/885
-envlist = py{39,310,311,312}-{json,msgpack}
+envlist = py{310,311,312}-{json,msgpack},docs
 
 [testenv]
 deps =
     -r tests/requirements.txt
     msgpack: -r tests/requirements-msgpack.txt
 commands = py.test --cov=scrapinghub --cov-report=xml {posargs: scrapinghub tests}
 
-[testenv:docs]
-changedir = docs
-deps =
-    -r docs/requirements.txt
-commands =
-    sphinx-build -W -b html . {envtmpdir}/html