Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/deployment/code_examples/google/cloud_run_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from crawlee.storage_clients import MemoryStorageClient


@get('/')
@get('/') # type: ignore[untyped-decorator]
async def main() -> str:
"""The crawler entry point that will be called when the HTTP endpoint is accessed."""
# highlight-start
Expand Down
7 changes: 2 additions & 5 deletions docs/deployment/code_examples/google/google_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,7 @@
import functions_framework
from flask import Request, Response

from crawlee.crawlers import (
BeautifulSoupCrawler,
BeautifulSoupCrawlingContext,
)
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
from crawlee.storage_clients import MemoryStorageClient


Expand Down Expand Up @@ -51,7 +48,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
# highlight-end


@functions_framework.http
@functions_framework.http # type: ignore[untyped-decorator]
def crawlee_run(request: Request) -> Response:
# You can pass data to your crawler using `request`
function_id = request.headers['Function-Execution-Id']
Expand Down
4 changes: 2 additions & 2 deletions docs/guides/code_examples/running_in_web_server/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
app = FastAPI(lifespan=lifespan, title='Crawler app')


@app.get('/', response_class=HTMLResponse)
@app.get('/', response_class=HTMLResponse) # type: ignore[untyped-decorator]
def index() -> str:
return """
<!DOCTYPE html>
Expand All @@ -32,7 +32,7 @@ def index() -> str:
"""


@app.get('/scrape')
@app.get('/scrape') # type: ignore[untyped-decorator]
async def scrape_url(request: Request, url: str | None = None) -> dict:
if not url:
return {'url': 'missing', 'scrape result': 'no results'}
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ dev = [
"build<2.0.0", # For e2e tests.
"dycw-pytest-only<3.0.0",
"fakeredis[probabilistic,json,lua]<3.0.0",
"mypy~=1.18.0",
"mypy~=1.19.0",
"pre-commit<5.0.0",
"proxy-py<3.0.0",
"pydoc-markdown<5.0.0",
Expand Down
3 changes: 3 additions & 0 deletions tests/unit/_utils/test_system.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ def no_extra_memory_child(ready: synchronize.Barrier, measured: synchronize.Barr

def extra_memory_child(ready: synchronize.Barrier, measured: synchronize.Barrier) -> None:
memory = SharedMemory(size=extra_memory_size, create=True)
assert memory.buf is not None
memory.buf[:] = bytearray([255 for _ in range(extra_memory_size)])
print(f'Using the memory... {memory.buf[-1]}')
ready.wait()
Expand All @@ -64,6 +65,7 @@ def extra_memory_child(ready: synchronize.Barrier, measured: synchronize.Barrier
def shared_extra_memory_child(
ready: synchronize.Barrier, measured: synchronize.Barrier, memory: SharedMemory
) -> None:
assert memory.buf is not None
print(f'Using the memory... {memory.buf[-1]}')
ready.wait()
measured.wait()
Expand All @@ -79,6 +81,7 @@ def get_additional_memory_estimation_while_running_processes(

if use_shared_memory:
shared_memory = SharedMemory(size=extra_memory_size, create=True)
assert shared_memory.buf is not None
shared_memory.buf[:] = bytearray([255 for _ in range(extra_memory_size)])
extra_args = [shared_memory]
else:
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/crawlers/_basic/test_basic_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ async def request_handler(context: BasicCrawlingContext) -> None:
raise RuntimeError('Arbitrary crash for testing purposes')

# Apply one of the handlers
@getattr(crawler, handler) # type:ignore[misc] # Untyped decorator is ok to make the test concise
@getattr(crawler, handler) # type: ignore[untyped-decorator]
async def handler_implementation(context: BasicCrawlingContext, error: Exception) -> None:
await context.push_data(test_data)
await context.add_requests(requests=[test_request], rq_alias=rq_alias)
Expand Down
Loading
Loading