|
|
from __future__ import annotations |
|
|
|
|
|
import json |
|
|
from contextlib import asynccontextmanager |
|
|
from functools import partialmethod |
|
|
from typing import AsyncGenerator |
|
|
from urllib.parse import urlparse |
|
|
from curl_cffi.requests import AsyncSession, Session, Response |
|
|
from .webdriver import WebDriver, WebDriverSession, bypass_cloudflare |
|
|
|
|
|
class StreamResponse: |
|
|
def __init__(self, inner: Response) -> None: |
|
|
self.inner: Response = inner |
|
|
self.request = inner.request |
|
|
self.status_code: int = inner.status_code |
|
|
self.reason: str = inner.reason |
|
|
self.ok: bool = inner.ok |
|
|
self.headers = inner.headers |
|
|
self.cookies = inner.cookies |
|
|
|
|
|
async def text(self) -> str: |
|
|
return await self.inner.atext() |
|
|
|
|
|
def raise_for_status(self) -> None: |
|
|
self.inner.raise_for_status() |
|
|
|
|
|
async def json(self, **kwargs) -> dict: |
|
|
return json.loads(await self.inner.acontent(), **kwargs) |
|
|
|
|
|
async def iter_lines(self) -> AsyncGenerator[bytes, None]: |
|
|
async for line in self.inner.aiter_lines(): |
|
|
yield line |
|
|
|
|
|
async def iter_content(self) -> AsyncGenerator[bytes, None]: |
|
|
async for chunk in self.inner.aiter_content(): |
|
|
yield chunk |
|
|
|
|
|
class StreamSession(AsyncSession): |
|
|
@asynccontextmanager |
|
|
async def request( |
|
|
self, method: str, url: str, **kwargs |
|
|
) -> AsyncGenerator[StreamResponse]: |
|
|
response = await super().request(method, url, stream=True, **kwargs) |
|
|
try: |
|
|
yield StreamResponse(response) |
|
|
finally: |
|
|
await response.aclose() |
|
|
|
|
|
head = partialmethod(request, "HEAD") |
|
|
get = partialmethod(request, "GET") |
|
|
post = partialmethod(request, "POST") |
|
|
put = partialmethod(request, "PUT") |
|
|
patch = partialmethod(request, "PATCH") |
|
|
delete = partialmethod(request, "DELETE") |
|
|
|
|
|
def get_session_from_browser(url: str, webdriver: WebDriver = None, proxy: str = None, timeout: int = 120): |
|
|
with WebDriverSession(webdriver, "", proxy=proxy, virtual_display=True) as driver: |
|
|
bypass_cloudflare(driver, url, timeout) |
|
|
|
|
|
cookies = dict([(cookie["name"], cookie["value"]) for cookie in driver.get_cookies()]) |
|
|
user_agent = driver.execute_script("return navigator.userAgent") |
|
|
|
|
|
parse = urlparse(url) |
|
|
return Session( |
|
|
cookies=cookies, |
|
|
headers={ |
|
|
'accept': '*/*', |
|
|
'authority': parse.netloc, |
|
|
'origin': f'{parse.scheme}://{parse.netloc}', |
|
|
'referer': url, |
|
|
'sec-fetch-dest': 'empty', |
|
|
'sec-fetch-mode': 'cors', |
|
|
'sec-fetch-site': 'same-origin', |
|
|
'user-agent': user_agent |
|
|
}, |
|
|
proxies={"https": proxy, "http": proxy}, |
|
|
timeout=timeout, |
|
|
impersonate="chrome110" |
|
|
) |
|
|
|
