diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 1cb27e6d..ebbd6529 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,7 +8,7 @@ version: 2 build: os: ubuntu-24.04 tools: - python: "3.13" + python: "3.11" rust: "latest" apt_packages: - build-essential @@ -23,10 +23,17 @@ build: - pip install maturin - maturin build --out dist - pip install --find-links=dist wreq - -# Build documentation with Mkdocs -mkdocs: - configuration: docs/mkdocs.yml + # We recommend using a requirements file for reproducible builds. + # This is just a quick example to get started. + # https://docs.readthedocs.io/page/guides/reproducible-builds.html + install: + - pip install zensical + build: + html: + - zensical build -f docs/mkdocs.yml + post_build: + - mkdir -p $READTHEDOCS_OUTPUT/html/ + - cp --recursive site/* $READTHEDOCS_OUTPUT/html/ # Optionally, but recommended, # declare the Python requirements required to build your documentation # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html diff --git a/Cargo.toml b/Cargo.toml index 915bd77b..fa75514d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "wreq-python" version = "0.10.0" -description = "An ergonomic Python HTTP client with TLS fingerprint" +description = "An ergonomic Python HTTP Client with TLS fingerprint" license = "Apache-2.0" readme = "README.md" repository = "https://github.com/0x676e67/wreq-python" diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 68da495a..ff3386aa 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: wreq +site_name: wreq-python site_description: Ergonomic Python HTTP Client with TLS Fingerprint site_author: "0x676e67" site_url: https://0x676e67.github.io/wreq-python @@ -14,6 +14,8 @@ extra: theme: name: material + font: + code: Roboto Mono palette: - media: "(prefers-color-scheme)" toggle: @@ -34,16 +36,21 @@ theme: icon: material/toggle-switch-off name: Switch to system preference features: + - search.highlight + - search.share + - content.code.copy + - content.code.annotate + - header.autohide + - toc.follow + - toc.integrate + - navigation.path - navigation.tabs - navigation.sections - navigation.top - navigation.tracking - navigation.indexes - - toc.integrate - - search.highlight - - search.share - - content.code.copy - - content.code.annotate + - navigation.instant + - navigation.instant.progress plugins: - search @@ -52,7 +59,10 @@ plugins: python: paths: [../python] options: - docstring_style: google + extensions: + - unpack_typeddict + - dataclasses + docstring_style: auto show_source: true show_root_heading: true show_symbol_type_heading: true @@ -76,15 +86,29 @@ markdown_extensions: - pymdownx.details - attr_list - md_in_html + - pymdownx.emoji + - pymdownx.tasklist + - pymdownx.caret + - pymdownx.mark + - pymdownx.tilde nav: - - Home: index.md - Getting Started: + - Introduction: getting-started/introduction.md - Installation: getting-started/installation.md - Quick Start: getting-started/quickstart.md + - Guides: + - Basic Usage: guide/basic.md + - Emulation: guide/emulation.md + - Authentication: guide/auth.md + - WebSocket: guide/websocket.md + - Proxy: guide/proxy.md + - Redirect & Errors: guide/redirect-errors.md + - Advanced Features: guide/advanced.md + - Blocking/Sync API: guide/blocking.md - API Reference: - - Core Module: api/core.md - Modules: + - wreq: api/wreq.md - wreq.blocking: api/blocking.md - wreq.header: api/header.md - wreq.cookie: api/cookie.md @@ -96,13 +120,4 @@ nav: - wreq.dns: api/dns.md - wreq.proxy: api/proxy.md - wreq.redirect: api/redirect.md - - Examples: - - Basic Usage: examples/basic.md - - Browser Emulation: examples/emulation.md - - Authentication: examples/auth.md - - WebSocket: examples/websocket.md - - Proxy: examples/proxy.md - - Redirect & Errors: examples/redirect-errors.md - - Advanced Features: examples/advanced.md - - Blocking/Sync API: examples/blocking.md - Sponsors: sponsors.md diff --git a/docs/requirements.txt b/docs/requirements.txt index eaa778b8..417ebcc5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -mkdocs>=1.5.0 +zensical mkdocs-material>=9.5.0 -mkdocstrings[python]>=0.24.0 +mkdocstrings[python]>=1.0.3 mike>=2.0.0 \ No newline at end of file diff --git a/docs/source/api/core.md b/docs/source/api/wreq.md similarity index 96% rename from docs/source/api/core.md rename to docs/source/api/wreq.md index e310e463..3d157f79 100644 --- a/docs/source/api/core.md +++ b/docs/source/api/wreq.md @@ -1,9 +1,7 @@ -# wreq (Main Module) +# wreq The main `wreq` module contains core classes and types used throughout the library. -## Core Classes - ::: wreq.Client options: show_root_heading: true diff --git a/docs/source/getting-started/installation.md b/docs/source/getting-started/installation.md index c21ad72b..82aa432e 100644 --- a/docs/source/getting-started/installation.md +++ b/docs/source/getting-started/installation.md @@ -1,43 +1,57 @@ -# Installation +# :package: Installation -## Requirements +!!! info "Supported Platforms" + - **Python 3.11+ is required** + - Linux (glibc/musl): `x86_64`, `aarch64`, `armv7`, `i686` + - macOS: `x86_64`, `aarch64` + - Windows: `x86_64`, `i686`, `aarch64` + - Android: `aarch64`, `x86_64` -- Python 3.11 or higher -- pip or uv package manager +--- ## Install from PyPI -The simplest way to install wreq is from PyPI: +The easiest way to install wreq is via PyPI: ```bash pip install wreq ``` -Or using uv: +Or with [uv](https://github.com/astral-sh/uv): ```bash uv pip install wreq ``` -## Install from Source +--- -To install the latest development version: +## Build from Source + +To build from source, first set up the BoringSSL build environment. See the [boringssl build guide](https://github.com/google/boringssl/blob/main/BUILDING.md) for details. + +Example (Ubuntu/Debian): ```bash -git clone https://github.com/0x676e67/wreq.git -cd wreq -pip install -e . -``` +sudo apt install -y build-essential cmake perl pkg-config libclang-dev musl-tools git +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +pip install uv maturin -## Verify Installation +uv venv +source .venv/bin/activate -Verify that wreq is installed correctly: +# Development install +maturin develop --uv -```python -import wreq -print(wreq.__version__) +# Build wheel +maturin build --release + +# Install from local wheel +pip install target/wheels/wreq-*.whl ``` +--- + ## Next Steps -Continue to the [Quick Start](quickstart.md) guide to learn how to use wreq. +- See the [Guides](../guide/basic.md) for usage examples +- Browse the [API Reference](../api/wreq.md) for full documentation \ No newline at end of file diff --git a/docs/source/getting-started/introduction.md b/docs/source/getting-started/introduction.md new file mode 100644 index 00000000..69d20a31 --- /dev/null +++ b/docs/source/getting-started/introduction.md @@ -0,0 +1,103 @@ +# Introduction + +[![CI](https://github.com/0x676e67/wreq-python/actions/workflows/ci.yml/badge.svg)](https://github.com/0x676e67/wreq-python/actions/workflows/ci.yml) +![GitHub License](https://img.shields.io/github/license/0x676e67/wreq-python?color=blue) +![Python Version from PEP 621 TOML](https://img.shields.io/python/required-version-toml?tomlFilePath=https%3A%2F%2Fraw.githubusercontent.com%2F0x676e67%2Fwreq-python%2Fmain%2Fpyproject.toml&logo=python) +[![PyPI](https://img.shields.io/pypi/v/wreq?logo=python)](https://pypi.org/project/wreq/) +[![Discord chat][discord-badge]][discord-url] + +[discord-badge]: https://img.shields.io/discord/1486741856397164788.svg?logo=discord +[discord-url]: https://discord.gg/rfbvyFkgq3 + +> 🚀 Help me work seamlessly with open source sharing by [sponsoring me on GitHub](https://github.com/0x676e67/0x676e67/blob/main/SPONSOR.md) + + +An ergonomic and modular **Python** HTTP Client for advanced and low-level emulation, featuring customizable **TLS**, **JA3/JA4**, and **HTTP/2** fingerprinting capabilities, powered by [wreq]. + +## Features + +- Async and Blocking `Client`s +- Plain bodies, JSON, urlencoded, multipart +- HTTP Trailer +- Cookie Store +- Redirect Policy +- Original Header +- Rotating Proxies +- Connection Pooling +- Streaming Transfers +- Zero-Copy Transfers +- WebSocket Upgrade +- HTTPS via BoringSSL +- Free-Threaded Safety +- Automatic Decompression +- Certificate Store (CAs & mTLS) + + +## Why wreq? + + +1. When your **HTTP** requests succeed in a browser but get blocked in Python due to network fingerprint issues, this tool bridges the gap. [wreq-python] allows you to customize your **TLS**, **JA3/JA4**, and **HTTP/2** fingerprints to mimic real browsers, making it ideal for web scraping, penetration testing, and security research. + +2. The standard **HTTP Client**, such as [requests] and [httpx], have different network fingerprints from browsers. The main differences lie in **TLS handshake**, **HTTP/2 frame characteristics**, and **JA3/JA4** fingerprints. Browsers use specific encryption suites and extensions in the TLS handshake, while standard HTTP clients may use different default settings, causing servers to recognize and block these requests. + +3. [wreq-python] uses the **BoringSSL** library, which is fully sufficient to set TLS fingerprints that match mainstream browsers while maintaining native performance. + +4. In addition, the basic functions of [wreq-python] are similar to those of the standard **HTTP Client**, offering a wide range of features such as connection pooling, redirection policies, Cookie storage, and streaming transmission, which can meet various complex HTTP request requirements. + + +## Behavior + +1. **HTTP/2 over TLS** + +Due to the complexity of **TLS** encryption and the widespread adoption of **HTTP/2**, browser fingerprints such as **JA3**, **JA4**, and **Akamai** cannot be reliably emulated using simple fingerprint strings. Instead of parsing and emulating these string-based fingerprints, [wreq-python] provides fine-grained control over **TLS** and **HTTP/2** extensions and settings for precise browser behavior emulation. + +2. **Device Emulation** + +**TLS** and **HTTP/2** fingerprints are often identical across various browser models because these underlying protocols evolve slower than browser release cycles. **100+ browser device emulation profiles** are maintained in **[wreq-python]**. + +??? note "Available OS emulations" + + | **OS** | **Description** | + | ----------- | ------------------------------ | + | **Windows** | Windows (any version) | + | **MacOS** | macOS (any version) | + | **Linux** | Linux (any distribution) | + | **Android** | Android (mobile) | + | **iOS** | iOS (iPhone/iPad) | + +??? note "Available browser emulations" + + | **Browser** | **Versions** | + | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | + | **Chrome** | `Chrome100`, `Chrome101`, `Chrome104`, `Chrome105`, `Chrome106`, `Chrome107`, `Chrome108`, `Chrome109`, `Chrome110`, `Chrome114`, `Chrome116`, `Chrome117`, `Chrome118`, `Chrome119`, `Chrome120`, `Chrome123`, `Chrome124`, `Chrome126`, `Chrome127`, `Chrome128`, `Chrome129`, `Chrome130`, `Chrome131`, `Chrome132`, `Chrome133`, `Chrome134`, `Chrome135`, `Chrome136`, `Chrome137`, `Chrome138`, `Chrome139`, `Chrome140`, `Chrome141`, `Chrome142`, `Chrome143`, `Chrome144`, `Chrome145` | + | **Safari** | `SafariIos17_2`, `SafariIos17_4_1`, `SafariIos16_5`, `Safari15_3`, `Safari15_5`, `Safari15_6_1`, `Safari16`, `Safari16_5`, `Safari17_0`, `Safari17_2_1`, `Safari17_4_1`, `Safari17_5`, `Safari18`, `SafariIPad18`, `Safari18_2`, `SafariIos18_1_1`, `Safari18_3`, `Safari18_3_1`, `Safari18_5`, `Safari26`, `Safari26_1`, `Safari26_2`, `SafariIos26`, `SafariIos26_2`, `SafariIPad26`, `SafariIPad26_2` | + | **Firefox** | `Firefox109`, `Firefox117`, `Firefox128`, `Firefox133`, `Firefox135`, `FirefoxPrivate135`, `FirefoxAndroid135`, `Firefox136`, `FirefoxPrivate136`, `Firefox139`, `Firefox142`, `Firefox143`, `Firefox144`, `Firefox145`, `Firefox146`, `Firefox147` | + | **OkHttp** | `OkHttp3_9`, `OkHttp3_11`, `OkHttp3_13`, `OkHttp3_14`, `OkHttp4_9`, `OkHttp4_10`, `OkHttp4_12`, `OkHttp5` | + | **Edge** | `Edge101`, `Edge122`, `Edge127`, `Edge131`, `Edge134`, `Edge135`, `Edge136`, `Edge137`, `Edge138`, `Edge139`, `Edge140`, `Edge141`, `Edge142`, `Edge143`, `Edge144`, `Edge145` | + | **Opera** | `Opera116`, `Opera117`, `Opera118`, `Opera119` + + +## Performance + +1. [wreq-python] This is designed to achieve high performance, leveraging the efficiency of the **BoringSSL** library in **TLS** operations and optimized **HTTP/2** processing. Although its running speed may not be comparable to that of low-level languages like **Rust** or **C++**, it offers significant performance improvements compared to traditional Python **HTTP** clients. + +2. In terms of API module design, [wreq-python] adopts dual support for both asynchronous and blocking clients, allowing developers to choose the appropriate programming model according to their needs. + +3. API calls have made every effort to release the [GIL], which means that performance can be maximally exploited. In terms of data transmission, [wreq-python] implements Python's [Buffer] Protocol, supporting zero-copy transmission, further enhancing performance. + + +## License + +Licensed under either of Apache License, Version 2.0 ([LICENSE](./LICENSE) or http://www.apache.org/licenses/LICENSE-2.0). + +## Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the [Apache-2.0](./LICENSE) license, shall be licensed as above, without any additional terms or conditions. + + +[wreq]: https://github.com/0x676e67/wreq +[wreq-python]: https://github.com/0x676e67/wreq-python +[requests]: https://github.com/psf/requests +[httpx]: https://github.com/encode/httpx +[Buffer]: https://docs.python.org/3/c-api/buffer.html +[GIL]: https://docs.python.org/3/c-api/init.html#thread-and-gil-management \ No newline at end of file diff --git a/docs/source/getting-started/quickstart.md b/docs/source/getting-started/quickstart.md index 1c0b997e..715c59c7 100644 --- a/docs/source/getting-started/quickstart.md +++ b/docs/source/getting-started/quickstart.md @@ -1,34 +1,35 @@ -# Quick Start +# :zap: Quick Start -This guide will help you get started with wreq quickly. +This page will help you get up and running with wreq in minutes. + +--- ## Basic GET Request -### Async +=== "Async" + ```python + import asyncio + from wreq import Client -```python -import asyncio -from wreq import Client + async def main(): + client = Client() + resp = await client.get("https://httpbin.org/get") + print(resp.status_code) + print(await resp.text()) + + asyncio.run(main()) + ``` +=== "Blocking" + ```python + from wreq.blocking import Client -async def main(): client = Client() - resp = await client.get("https://httpbin.org/get") + resp = client.get("https://httpbin.org/get") print(resp.status_code) - print(await resp.text()) - -asyncio.run(main()) -``` - -### Blocking + print(resp.text()) + ``` -```python -from wreq.blocking import Client - -client = Client() -resp = client.get("https://httpbin.org/get") -print(resp.status_code) -print(resp.text()) -``` +--- ## POST with JSON @@ -46,7 +47,9 @@ async def main(): asyncio.run(main()) ``` -## Browser Emulation +--- + +## Emulation Emulate different browsers to bypass detection: @@ -55,14 +58,15 @@ import asyncio from wreq import Client, Emulation async def main(): - # Emulate Chrome 120 - client = Client(emulation=Emulation.Chrome120) - resp = await client.get("https://httpbin.org/get") + client = Client(emulation=Emulation.Safari26) + resp = await client.get("https://tls.peet.ws/api/all") print(await resp.text()) asyncio.run(main()) ``` +--- + ## Using Proxies ```python @@ -71,14 +75,15 @@ from wreq import Client from wreq.proxy import Proxy async def main(): - proxy = Proxy.http("http://proxy.example.com:8080") - client = Client(proxy=proxy) + client = Client(proxy=Proxy.all("http://proxy.example.com:8080")) resp = await client.get("https://httpbin.org/ip") print(await resp.text()) asyncio.run(main()) ``` +--- + ## Custom Headers ```python @@ -91,15 +96,15 @@ async def main(): headers = HeaderMap() headers["User-Agent"] = "MyApp/1.0" headers["Custom-Header"] = "value" - resp = await client.get("https://httpbin.org/headers", headers=headers) print(await resp.text()) asyncio.run(main()) ``` +--- + ## Next Steps -- Explore the [API Reference](../api/core.md) for detailed documentation -- Check out [Examples](../examples/basic.md) for more code samples -- Learn about [Browser Emulation](../examples/emulation.md) for advanced use cases +- See the [Examples](../guide/basic.md) for more code samples +- Explore the [API Reference](../api/wreq.md) for detailed documentation \ No newline at end of file diff --git a/docs/source/examples/advanced.md b/docs/source/guide/advanced.md similarity index 93% rename from docs/source/examples/advanced.md rename to docs/source/guide/advanced.md index a34e697a..a7bb3987 100644 --- a/docs/source/examples/advanced.md +++ b/docs/source/guide/advanced.md @@ -1,6 +1,10 @@ -# Advanced Examples +# :star2: Advanced Features -## Streaming Request Body +!!! info "On this page" + - Header order + - Other advanced usage + +### Streaming Request Body Send data using async generators for streaming uploads: @@ -33,7 +37,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## Multipart File Upload +### Multipart File Upload Upload multiple files and data parts: @@ -83,7 +87,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## TLS Key Logging +### TLS Key Logging Capture TLS keys for debugging with tools like Wireshark: @@ -104,7 +108,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## Original Header Order Preservation +### Original Header Order Preservation Preserve header case and order for specific sites: diff --git a/docs/source/examples/auth.md b/docs/source/guide/auth.md similarity index 78% rename from docs/source/examples/auth.md rename to docs/source/guide/auth.md index 67fc6bc6..77a8473f 100644 --- a/docs/source/examples/auth.md +++ b/docs/source/guide/auth.md @@ -1,6 +1,11 @@ -# Authentication Examples +# :lock: Authentication Guide -## Basic Authentication +!!! tip "Supported authentication methods" + - Basic Auth + - Bearer Token + - Simple Token + +### Basic Authentication ```python import asyncio @@ -19,7 +24,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## Bearer Token Authentication +### Bearer Token Authentication ```python import asyncio @@ -38,7 +43,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## Simple Auth Token +### Simple Auth Token ```python import asyncio diff --git a/docs/source/examples/basic.md b/docs/source/guide/basic.md similarity index 92% rename from docs/source/examples/basic.md rename to docs/source/guide/basic.md index 17de6121..b6744866 100644 --- a/docs/source/examples/basic.md +++ b/docs/source/guide/basic.md @@ -1,6 +1,12 @@ -# Basic Usage Examples +# :rocket: Basic Usage -## Simple GET Request +!!! info "On this page" + - GET/POST requests + - Form and JSON + - Custom headers + - Query parameters & streaming + +### Simple GET Request ```python import asyncio @@ -23,7 +29,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## POST Request with JSON +### POST Request with JSON ```python import asyncio @@ -42,7 +48,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## Form Data +### Form Data ```python import asyncio @@ -83,7 +89,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## Custom Headers +### Custom Headers ```python from wreq.header import HeaderMap @@ -103,7 +109,7 @@ if __name__ == "__main__": print("Content-Type:", headers.get("Content-Type")) ``` -## Query Parameters +### Query Parameters ```python import asyncio @@ -142,7 +148,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## Streaming Response +### Streaming Response ```python import asyncio diff --git a/docs/source/examples/blocking.md b/docs/source/guide/blocking.md similarity index 92% rename from docs/source/examples/blocking.md rename to docs/source/guide/blocking.md index d7f0defb..c8e4901f 100644 --- a/docs/source/examples/blocking.md +++ b/docs/source/guide/blocking.md @@ -1,8 +1,13 @@ -# Blocking/Synchronous API Examples +# :hourglass: Blocking/Sync API + +!!! info "On this page" + - Blocking GET + - Configuration + - Cookie/Auth/Streaming The blocking API provides synchronous methods for environments where async/await is not needed. -## Simple GET Request +### Simple GET Request ```python import datetime @@ -24,7 +29,7 @@ if __name__ == "__main__": main() ``` -## Client Configuration +### Client Configuration ```python from wreq import Proxy @@ -60,7 +65,7 @@ if __name__ == "__main__": main() ``` -## Cookies +### Cookies ```python from wreq.blocking import Client, Method @@ -77,7 +82,7 @@ if __name__ == "__main__": main() ``` -## Authentication +### Authentication ```python from wreq.blocking import Client @@ -103,7 +108,7 @@ if __name__ == "__main__": main() ``` -## JSON and Form Data +### JSON and Form Data ```python from wreq.blocking import Client @@ -135,7 +140,7 @@ if __name__ == "__main__": main() ``` -## Query Parameters +### Query Parameters ```python from wreq.blocking import Client @@ -158,7 +163,7 @@ if __name__ == "__main__": main() ``` -## Streaming Response +### Streaming Response ```python from wreq.blocking import Client diff --git a/docs/source/examples/emulation.md b/docs/source/guide/emulation.md similarity index 93% rename from docs/source/examples/emulation.md rename to docs/source/guide/emulation.md index 92f94442..017b2494 100644 --- a/docs/source/examples/emulation.md +++ b/docs/source/guide/emulation.md @@ -1,6 +1,10 @@ -# Browser Emulation Examples +# :busts_in_silhouette: Emulation -## Simple Firefox Emulation +!!! info "Emulation topics" + - Firefox/Chrome/Android + - Custom TLS/HTTP2 + +### Simple Firefox Emulation ```python import asyncio @@ -21,7 +25,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## Chrome on Android Emulation +### Chrome on Android Emulation ```python import asyncio @@ -48,7 +52,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## Advanced Configuration with Custom TLS and HTTP/2 +### Advanced Configuration with Custom TLS and HTTP/2 ```python import asyncio diff --git a/docs/source/examples/proxy.md b/docs/source/guide/proxy.md similarity index 89% rename from docs/source/examples/proxy.md rename to docs/source/guide/proxy.md index c241c022..cec9ad19 100644 --- a/docs/source/examples/proxy.md +++ b/docs/source/guide/proxy.md @@ -1,6 +1,10 @@ -# Proxy Examples +# :globe_with_meridians: Proxy Usage -## HTTP/HTTPS Proxy +!!! info "On this page" + - HTTP/HTTPS proxy + - Unix Socket proxy + +### HTTP/HTTPS Proxy Using proxies with authentication: @@ -40,7 +44,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## Unix Socket Proxy +### Unix Socket Proxy Using Unix sockets for local services like Docker: diff --git a/docs/source/examples/redirect-errors.md b/docs/source/guide/redirect-errors.md similarity index 93% rename from docs/source/examples/redirect-errors.md rename to docs/source/guide/redirect-errors.md index 14fa4739..43f27056 100644 --- a/docs/source/examples/redirect-errors.md +++ b/docs/source/guide/redirect-errors.md @@ -1,6 +1,10 @@ -# Redirect and Error Handling Examples +# :repeat: Redirects & Error Handling -## Custom Redirect Policy +!!! info "On this page" + - Custom redirect policy + - Error handling + +### Custom Redirect Policy Control redirect behavior with custom policies: @@ -43,7 +47,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## Error Handling +### Error Handling Handle various request exceptions: diff --git a/docs/source/examples/websocket.md b/docs/source/guide/websocket.md similarity index 95% rename from docs/source/examples/websocket.md rename to docs/source/guide/websocket.md index 0fba6912..264fd092 100644 --- a/docs/source/examples/websocket.md +++ b/docs/source/guide/websocket.md @@ -1,6 +1,10 @@ -# WebSocket Examples +# :satellite: WebSocket -## HTTP/1.1 WebSocket Connection +!!! info "On this page" + - HTTP/1.1 WebSocket + - HTTP/2 WebSocket + +### HTTP/1.1 WebSocket Connection ```python import asyncio @@ -55,7 +59,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## HTTP/2 WebSocket Connection +### HTTP/2 WebSocket Connection ```python import asyncio diff --git a/docs/source/index.md b/docs/source/index.md index f254f970..330e2b09 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -1,143 +1,2 @@ -# wreq-python - -[![CI](https://github.com/0x676e67/wreq-python/actions/workflows/ci.yml/badge.svg)](https://github.com/0x676e67/wreq-python/actions/workflows/ci.yml) -![GitHub License](https://img.shields.io/github/license/0x676e67/wreq-python?color=blue) -![Python Version from PEP 621 TOML](https://img.shields.io/python/required-version-toml?tomlFilePath=https%3A%2F%2Fraw.githubusercontent.com%2F0x676e67%2Frnet%2Fmain%2Fpyproject.toml&logo=python) -[![PyPI](https://img.shields.io/pypi/v/wreq?logo=python)](https://pypi.org/project/wreq/) -[![Discord chat][discord-badge]][discord-url] - -[discord-badge]: https://img.shields.io/discord/1486741856397164788.svg?logo=discord -[discord-url]: https://discord.gg/rfbvyFkgq3 - -> 🚀 Help me work seamlessly with open source sharing by [sponsoring me on GitHub](https://github.com/0x676e67/0x676e67/blob/main/SPONSOR.md) - -An ergonomic and modular Python HTTP client for advanced and low-level emulation, featuring customizable TLS, JA3/JA4, and HTTP/2 fingerprinting capabilities, powered by [wreq](https://github.com/0x676e67/wreq). - -## Features - -- Async and Blocking `Client`s -- Plain bodies, JSON, urlencoded, multipart -- HTTP Trailer -- Cookie Store -- Redirect Policy -- Original Header -- Rotating Proxies -- Connection Pooling -- Streaming Transfers -- Zero-Copy Transfers -- WebSocket Upgrade -- HTTPS via BoringSSL -- Free-Threaded Safety -- Automatic Decompression -- Certificate Store (CAs & mTLS) - -## Example - -The following example uses the `asyncio` runtime with `wreq` installed via pip: - -```bash -pip install wreq --upgrade -``` - -And then the code: - -```python -import asyncio -from wreq import Client, Emulation - - -async def main(): - # Build a client - client = Client(emulation=Emulation.Safari26) - - # Use the API you're already familiar with - resp = await client.get("https://tls.peet.ws/api/all") - print(await resp.text()) - - -if __name__ == "__main__": - asyncio.run(main()) - -``` - -Additional learning resources include: - -- [DeepWiki](https://deepwiki.com/0x676e67/wreq-python) -- [Examples](https://github.com/0x676e67/wreq-python/tree/main/examples) -- [Documentation](https://wreq.readthedocs.io/) - -## Behavior - -1. **HTTP/2 over TLS** - -Due to the complexity of TLS encryption and the widespread adoption of HTTP/2, browser fingerprints such as **JA3**, **JA4**, and **Akamai** cannot be reliably emulated using simple fingerprint strings. Instead of parsing and emulating these string-based fingerprints, `wreq` provides fine-grained control over TLS and HTTP/2 extensions and settings for precise browser behavior emulation. - -2. **Device Emulation** - -Most browser device models share identical TLS and HTTP/2 configurations, differing only in the `User-Agent` string. - -??? note "Available OS emulations" - - | **OS** | **Description** | - | ----------- | ------------------------------ | - | **Windows** | Windows (any version) | - | **MacOS** | macOS (any version) | - | **Linux** | Linux (any distribution) | - | **Android** | Android (mobile) | - | **iOS** | iOS (iPhone/iPad) | - -??? note "Available browser emulations" - - | **Browser** | **Versions** | - | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | - | **Chrome** | `Chrome100`, `Chrome101`, `Chrome104`, `Chrome105`, `Chrome106`, `Chrome107`, `Chrome108`, `Chrome109`, `Chrome110`, `Chrome114`, `Chrome116`, `Chrome117`, `Chrome118`, `Chrome119`, `Chrome120`, `Chrome123`, `Chrome124`, `Chrome126`, `Chrome127`, `Chrome128`, `Chrome129`, `Chrome130`, `Chrome131`, `Chrome132`, `Chrome133`, `Chrome134`, `Chrome135`, `Chrome136`, `Chrome137`, `Chrome138`, `Chrome139`, `Chrome140`, `Chrome141`, `Chrome142`, `Chrome143`, `Chrome144`, `Chrome145` | - | **Safari** | `SafariIos17_2`, `SafariIos17_4_1`, `SafariIos16_5`, `Safari15_3`, `Safari15_5`, `Safari15_6_1`, `Safari16`, `Safari16_5`, `Safari17_0`, `Safari17_2_1`, `Safari17_4_1`, `Safari17_5`, `Safari18`, `SafariIPad18`, `Safari18_2`, `SafariIos18_1_1`, `Safari18_3`, `Safari18_3_1`, `Safari18_5`, `Safari26`, `Safari26_1`, `Safari26_2`, `SafariIos26`, `SafariIos26_2`, `SafariIPad26`, `SafariIPad26_2` | - | **Firefox** | `Firefox109`, `Firefox117`, `Firefox128`, `Firefox133`, `Firefox135`, `FirefoxPrivate135`, `FirefoxAndroid135`, `Firefox136`, `FirefoxPrivate136`, `Firefox139`, `Firefox142`, `Firefox143`, `Firefox144`, `Firefox145`, `Firefox146`, `Firefox147` | - | **OkHttp** | `OkHttp3_9`, `OkHttp3_11`, `OkHttp3_13`, `OkHttp3_14`, `OkHttp4_9`, `OkHttp4_10`, `OkHttp4_12`, `OkHttp5` | - | **Edge** | `Edge101`, `Edge122`, `Edge127`, `Edge131`, `Edge134`, `Edge135`, `Edge136`, `Edge137`, `Edge138`, `Edge139`, `Edge140`, `Edge141`, `Edge142`, `Edge143`, `Edge144`, `Edge145` | - | **Opera** | `Opera116`, `Opera117`, `Opera118`, `Opera119` | - -## Building - -1. Platforms - -- Linux(**glibc**/**musl**): `x86_64`, `aarch64`, `armv7`, `i686` -- macOS: `x86_64`,`aarch64` -- Windows: `x86_64`,`i686`,`aarch64` -- Android: `aarch64`, `x86_64` - -2. Development - -Install the BoringSSL build environment by referring to [boringssl](https://github.com/google/boringssl/blob/main/BUILDING.md). - -```bash -# on ubuntu or debian -sudo apt install -y build-essential cmake perl pkg-config libclang-dev musl-tools git -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -pip install uv maturin - -uv venv -source .venv/bin/activate - -# development -maturin develop --uv - -# build wheels -maturin build --release -``` - -## Benchmark - -Outperforms `requests`, `httpx`, `aiohttp`, and `curl_cffi`, and you can see the [benchmark](https://github.com/0x676e67/wreq/tree/main/bench) for details — benchmark data is for reference only and actual performance may vary based on your environment and use case. - -## Services - -Help sustain the ongoing development of this open-source project by reaching out for [commercial support](mailto:gngppz@gmail.com). Receive private guidance, expert reviews, or direct access to the maintainer, with personalized technical assistance tailored to your needs. - -## License - -Licensed under either of Apache License, Version 2.0 ([LICENSE](https://github.com/0x676e67/wreq/blob/main/LICENSE) or http://www.apache.org/licenses/LICENSE-2.0). - -## Contribution - -Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the [Apache-2.0](https://github.com/0x676e67/wreq/blob/main/LICENSE) license, shall be licensed as above, without any additional terms or conditions. + +If you are not redirected, click here. \ No newline at end of file diff --git a/examples/header_map.py b/examples/header_map.py index 20675be0..05b248f5 100644 --- a/examples/header_map.py +++ b/examples/header_map.py @@ -1,6 +1,5 @@ from wreq.header import HeaderMap - if __name__ == "__main__": headers = HeaderMap() # Add Content-Type header diff --git a/python/wreq/__init__.py b/python/wreq/__init__.py index 200affbd..f92a9ae4 100644 --- a/python/wreq/__init__.py +++ b/python/wreq/__init__.py @@ -1,7 +1,6 @@ # wreq/__init__.py from .wreq import * -from .wreq import __all__ from .cookie import * from .exceptions import * diff --git a/python/wreq/blocking.py b/python/wreq/blocking.py index b3661095..3900ca4a 100644 --- a/python/wreq/blocking.py +++ b/python/wreq/blocking.py @@ -463,7 +463,7 @@ def get( ``` """ ... - + def __enter__(self) -> Any: ... def __exit__(self, _exc_type: Any, _exc_value: Any, _traceback: Any) -> None: ... diff --git a/python/wreq/__init__.pyi b/python/wreq/wreq.py similarity index 99% rename from python/wreq/__init__.pyi rename to python/wreq/wreq.py index 69cf4073..7db0b897 100644 --- a/python/wreq/__init__.pyi +++ b/python/wreq/wreq.py @@ -26,6 +26,7 @@ from .redirect import History from .tls import * + @final class Method(Enum): r""" @@ -41,6 +42,7 @@ class Method(Enum): TRACE = auto() PATCH = auto() + @final class Version(Enum): r""" @@ -53,6 +55,7 @@ class Version(Enum): HTTP_2 = auto() HTTP_3 = auto() + @final class StatusCode: r""" @@ -98,6 +101,7 @@ def is_server_error(self) -> bool: def __str__(self) -> str: ... def __richcmp__(self, other: Any, op: int) -> bool: ... + @final class SocketAddr: r""" @@ -109,11 +113,14 @@ def ip(self) -> IPv4Address | IPv6Address: r""" Returns the IP address of the socket address. """ + ... def port(self) -> int: r""" Returns the port number of the socket address. """ + ... + @final class Multipart: @@ -121,12 +128,13 @@ class Multipart: A multipart form for a request. """ - def __init__(self, *parts: Part) -> None: + def __init__(self, *parts: "Part") -> None: r""" Creates a new multipart form. """ ... + @final class Part: r""" @@ -161,6 +169,7 @@ def __init__( """ ... + class Message: r""" A WebSocket message. @@ -259,6 +268,7 @@ def from_close(code: int, reason: str | None = None) -> "Message": def __str__(self) -> str: ... + class Streamer: r""" A stream response. @@ -303,6 +313,7 @@ async def __aexit__( self, _exc_type: Any, _exc_value: Any, _traceback: Any ) -> None: ... + class Response: r""" A response from a request. @@ -390,11 +401,13 @@ def stream(self) -> Streamer: r""" Get the response into a `Streamer` of `bytes` from the body. """ + ... async def text(self, encoding: str | None = None) -> str: r""" Get the text content with the response encoding, defaulting to utf-8 when unspecified. """ + ... async def json(self) -> Any: r""" @@ -405,6 +418,7 @@ async def bytes(self) -> bytes: r""" Get the bytes content of the response. """ + ... async def close(self) -> None: r""" @@ -434,6 +448,7 @@ async def __aexit__( ) -> Any: ... def __str__(self) -> str: ... + class WebSocket: r""" A WebSocket response. @@ -497,6 +512,7 @@ def __aenter__(self) -> Any: ... def __aexit__(self, _exc_type: Any, _exc_value: Any, _traceback: Any) -> Any: ... def __str__(self) -> str: ... + class ClientConfig(TypedDict): emulation: NotRequired[Emulation | EmulationOption] """Emulation config.""" @@ -767,6 +783,7 @@ class ClientConfig(TypedDict): Enable auto zstd decompression by checking the `Content-Encoding` response header. """ + class Request(TypedDict): emulation: NotRequired[Emulation | EmulationOption] """ @@ -932,6 +949,7 @@ class Request(TypedDict): The multipart form to use for the request. """ + class WebSocketRequest(TypedDict): emulation: NotRequired[Emulation | EmulationOption] """ @@ -1072,6 +1090,7 @@ class WebSocketRequest(TypedDict): ignoring the RFC. By default this option is set to False, i.e. according to RFC6455. """ + class Client: r""" A client for making HTTP requests. @@ -1136,6 +1155,7 @@ async def main(): asyncio.run(main()) ``` """ + ... async def websocket( self, @@ -1162,6 +1182,7 @@ async def main(): asyncio.run(main()) ``` """ + ... async def trace( self, @@ -1186,6 +1207,7 @@ async def main(): asyncio.run(main()) ``` """ + ... async def options( self, @@ -1210,6 +1232,7 @@ async def main(): asyncio.run(main()) ``` """ + ... async def patch( self, @@ -1234,6 +1257,7 @@ async def main(): asyncio.run(main()) ``` """ + ... async def delete( self, @@ -1258,6 +1282,7 @@ async def main(): asyncio.run(main()) ``` """ + ... async def put( self, @@ -1282,6 +1307,7 @@ async def main(): asyncio.run(main()) ``` """ + ... async def post( self, @@ -1306,6 +1332,7 @@ async def main(): asyncio.run(main()) ``` """ + ... async def head( self, @@ -1330,6 +1357,7 @@ async def main(): asyncio.run(main()) ``` """ + ... async def get( self, @@ -1354,12 +1382,14 @@ async def main(): asyncio.run(main()) ``` """ - + ... + async def __aenter__(self) -> Any: ... async def __aexit__( self, _exc_type: Any, _exc_value: Any, _traceback: Any ) -> Any: ... + async def delete( url: str, **kwargs: Unpack[Request], @@ -1381,6 +1411,8 @@ async def run(): asyncio.run(run()) ``` """ + ... + async def get( url: str, @@ -1403,6 +1435,8 @@ async def run(): asyncio.run(run()) ``` """ + ... + async def head( url: str, @@ -1424,6 +1458,8 @@ async def run(): asyncio.run(run()) ``` """ + ... + async def options( url: str, @@ -1445,6 +1481,8 @@ async def run(): asyncio.run(run()) ``` """ + ... + async def patch( url: str, @@ -1467,6 +1505,8 @@ async def run(): asyncio.run(run()) ``` """ + ... + async def post( url: str, @@ -1489,6 +1529,8 @@ async def run(): asyncio.run(run()) ``` """ + ... + async def put( url: str, @@ -1511,6 +1553,8 @@ async def run(): asyncio.run(run()) ``` """ + ... + async def request( method: Method, @@ -1541,6 +1585,8 @@ async def run(): asyncio.run(run()) ``` """ + ... + async def trace( url: str, @@ -1562,6 +1608,8 @@ async def run(): asyncio.run(run()) ``` """ + ... + async def websocket( url: str, @@ -1587,3 +1635,4 @@ async def run(): asyncio.run(run()) ``` """ + ...