diff --git a/common/http_client.py b/common/http_client.py index 2ffbb3bce..51f2e7fcc 100644 --- a/common/http_client.py +++ b/common/http_client.py @@ -24,7 +24,9 @@ logger = logging.getLogger(__name__) # Default knobs; keep conservative to avoid unexpected behavioural changes. DEFAULT_TIMEOUT = float(os.environ.get("HTTP_CLIENT_TIMEOUT", "15")) # Align with requests default: follow redirects with a max of 30 unless overridden. -DEFAULT_FOLLOW_REDIRECTS = bool(int(os.environ.get("HTTP_CLIENT_FOLLOW_REDIRECTS", "1"))) +DEFAULT_FOLLOW_REDIRECTS = bool( + int(os.environ.get("HTTP_CLIENT_FOLLOW_REDIRECTS", "1")) +) DEFAULT_MAX_REDIRECTS = int(os.environ.get("HTTP_CLIENT_MAX_REDIRECTS", "30")) DEFAULT_MAX_RETRIES = int(os.environ.get("HTTP_CLIENT_MAX_RETRIES", "2")) DEFAULT_BACKOFF_FACTOR = float(os.environ.get("HTTP_CLIENT_BACKOFF_FACTOR", "0.5")) @@ -32,7 +34,9 @@ DEFAULT_PROXY = os.environ.get("HTTP_CLIENT_PROXY") DEFAULT_USER_AGENT = os.environ.get("HTTP_CLIENT_USER_AGENT", "ragflow-http-client") -def _clean_headers(headers: Optional[Dict[str, str]], auth_token: Optional[str] = None) -> Optional[Dict[str, str]]: +def _clean_headers( + headers: Optional[Dict[str, str]], auth_token: Optional[str] = None +) -> Optional[Dict[str, str]]: merged_headers: Dict[str, str] = {} if DEFAULT_USER_AGENT: merged_headers["User-Agent"] = DEFAULT_USER_AGENT @@ -59,39 +63,51 @@ async def async_request( auth_token: Optional[str] = None, retries: Optional[int] = None, backoff_factor: Optional[float] = None, - proxies: Any = None, + proxy: Any = None, **kwargs: Any, ) -> httpx.Response: """Lightweight async HTTP wrapper using httpx.AsyncClient with safe defaults.""" timeout = timeout if timeout is not None else DEFAULT_TIMEOUT - follow_redirects = DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects + follow_redirects = ( + DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects + ) max_redirects = DEFAULT_MAX_REDIRECTS if max_redirects is None else max_redirects retries = DEFAULT_MAX_RETRIES if retries is None else max(retries, 0) - backoff_factor = DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor + backoff_factor = ( + DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor + ) headers = _clean_headers(headers, auth_token=auth_token) - proxies = DEFAULT_PROXY if proxies is None else proxies + proxy = DEFAULT_PROXY if proxy is None else proxy async with httpx.AsyncClient( timeout=timeout, follow_redirects=follow_redirects, max_redirects=max_redirects, - proxies=proxies, + proxy=proxy, ) as client: last_exc: Exception | None = None for attempt in range(retries + 1): try: start = time.monotonic() - response = await client.request(method=method, url=url, headers=headers, **kwargs) + response = await client.request( + method=method, url=url, headers=headers, **kwargs + ) duration = time.monotonic() - start - logger.debug(f"async_request {method} {url} -> {response.status_code} in {duration:.3f}s") + logger.debug( + f"async_request {method} {url} -> {response.status_code} in {duration:.3f}s" + ) return response except httpx.RequestError as exc: last_exc = exc if attempt >= retries: - logger.warning(f"async_request exhausted retries for {method} {url}: {exc}") + logger.warning( + f"async_request exhausted retries for {method} {url}: {exc}" + ) raise delay = _get_delay(backoff_factor, attempt) - logger.warning(f"async_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s") + logger.warning( + f"async_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s" + ) await asyncio.sleep(delay) raise last_exc # pragma: no cover @@ -107,39 +123,51 @@ def sync_request( auth_token: Optional[str] = None, retries: Optional[int] = None, backoff_factor: Optional[float] = None, - proxies: Any = None, + proxy: Any = None, **kwargs: Any, ) -> httpx.Response: """Synchronous counterpart to async_request, for CLI/tests or sync contexts.""" timeout = timeout if timeout is not None else DEFAULT_TIMEOUT - follow_redirects = DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects + follow_redirects = ( + DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects + ) max_redirects = DEFAULT_MAX_REDIRECTS if max_redirects is None else max_redirects retries = DEFAULT_MAX_RETRIES if retries is None else max(retries, 0) - backoff_factor = DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor + backoff_factor = ( + DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor + ) headers = _clean_headers(headers, auth_token=auth_token) - proxies = DEFAULT_PROXY if proxies is None else proxies + proxy = DEFAULT_PROXY if proxy is None else proxy with httpx.Client( timeout=timeout, follow_redirects=follow_redirects, max_redirects=max_redirects, - proxies=proxies, + proxy=proxy, ) as client: last_exc: Exception | None = None for attempt in range(retries + 1): try: start = time.monotonic() - response = client.request(method=method, url=url, headers=headers, **kwargs) + response = client.request( + method=method, url=url, headers=headers, **kwargs + ) duration = time.monotonic() - start - logger.debug(f"sync_request {method} {url} -> {response.status_code} in {duration:.3f}s") + logger.debug( + f"sync_request {method} {url} -> {response.status_code} in {duration:.3f}s" + ) return response except httpx.RequestError as exc: last_exc = exc if attempt >= retries: - logger.warning(f"sync_request exhausted retries for {method} {url}: {exc}") + logger.warning( + f"sync_request exhausted retries for {method} {url}: {exc}" + ) raise delay = _get_delay(backoff_factor, attempt) - logger.warning(f"sync_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s") + logger.warning( + f"sync_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s" + ) time.sleep(delay) raise last_exc # pragma: no cover