mirror of
https://github.com/infiniflow/ragflow.git
synced 2025-12-18 11:36:44 +08:00
fix: align http client proxy kwarg (#11818)
### What problem does this PR solve?
Our HTTP wrapper still passed proxies to httpx.Client/AsyncClient, which
expect proxy. As a result, configured proxies were ignored and calls
could fail with ValueError("Failed to fetch OIDC metadata:
Client.__init__() got an unexpected keyword argument 'proxies'"). This
PR switches to the correct proxy kwarg so proxies are honored and the
runtime error is resolved.
### Type of change
- [X] Bug Fix (non-breaking change which fixes an issue)
---
Contribution during my time at RAGcon GmbH.
This commit is contained in:
@ -24,7 +24,9 @@ logger = logging.getLogger(__name__)
|
|||||||
# Default knobs; keep conservative to avoid unexpected behavioural changes.
|
# Default knobs; keep conservative to avoid unexpected behavioural changes.
|
||||||
DEFAULT_TIMEOUT = float(os.environ.get("HTTP_CLIENT_TIMEOUT", "15"))
|
DEFAULT_TIMEOUT = float(os.environ.get("HTTP_CLIENT_TIMEOUT", "15"))
|
||||||
# Align with requests default: follow redirects with a max of 30 unless overridden.
|
# Align with requests default: follow redirects with a max of 30 unless overridden.
|
||||||
DEFAULT_FOLLOW_REDIRECTS = bool(int(os.environ.get("HTTP_CLIENT_FOLLOW_REDIRECTS", "1")))
|
DEFAULT_FOLLOW_REDIRECTS = bool(
|
||||||
|
int(os.environ.get("HTTP_CLIENT_FOLLOW_REDIRECTS", "1"))
|
||||||
|
)
|
||||||
DEFAULT_MAX_REDIRECTS = int(os.environ.get("HTTP_CLIENT_MAX_REDIRECTS", "30"))
|
DEFAULT_MAX_REDIRECTS = int(os.environ.get("HTTP_CLIENT_MAX_REDIRECTS", "30"))
|
||||||
DEFAULT_MAX_RETRIES = int(os.environ.get("HTTP_CLIENT_MAX_RETRIES", "2"))
|
DEFAULT_MAX_RETRIES = int(os.environ.get("HTTP_CLIENT_MAX_RETRIES", "2"))
|
||||||
DEFAULT_BACKOFF_FACTOR = float(os.environ.get("HTTP_CLIENT_BACKOFF_FACTOR", "0.5"))
|
DEFAULT_BACKOFF_FACTOR = float(os.environ.get("HTTP_CLIENT_BACKOFF_FACTOR", "0.5"))
|
||||||
@ -32,7 +34,9 @@ DEFAULT_PROXY = os.environ.get("HTTP_CLIENT_PROXY")
|
|||||||
DEFAULT_USER_AGENT = os.environ.get("HTTP_CLIENT_USER_AGENT", "ragflow-http-client")
|
DEFAULT_USER_AGENT = os.environ.get("HTTP_CLIENT_USER_AGENT", "ragflow-http-client")
|
||||||
|
|
||||||
|
|
||||||
def _clean_headers(headers: Optional[Dict[str, str]], auth_token: Optional[str] = None) -> Optional[Dict[str, str]]:
|
def _clean_headers(
|
||||||
|
headers: Optional[Dict[str, str]], auth_token: Optional[str] = None
|
||||||
|
) -> Optional[Dict[str, str]]:
|
||||||
merged_headers: Dict[str, str] = {}
|
merged_headers: Dict[str, str] = {}
|
||||||
if DEFAULT_USER_AGENT:
|
if DEFAULT_USER_AGENT:
|
||||||
merged_headers["User-Agent"] = DEFAULT_USER_AGENT
|
merged_headers["User-Agent"] = DEFAULT_USER_AGENT
|
||||||
@ -59,39 +63,51 @@ async def async_request(
|
|||||||
auth_token: Optional[str] = None,
|
auth_token: Optional[str] = None,
|
||||||
retries: Optional[int] = None,
|
retries: Optional[int] = None,
|
||||||
backoff_factor: Optional[float] = None,
|
backoff_factor: Optional[float] = None,
|
||||||
proxies: Any = None,
|
proxy: Any = None,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> httpx.Response:
|
) -> httpx.Response:
|
||||||
"""Lightweight async HTTP wrapper using httpx.AsyncClient with safe defaults."""
|
"""Lightweight async HTTP wrapper using httpx.AsyncClient with safe defaults."""
|
||||||
timeout = timeout if timeout is not None else DEFAULT_TIMEOUT
|
timeout = timeout if timeout is not None else DEFAULT_TIMEOUT
|
||||||
follow_redirects = DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects
|
follow_redirects = (
|
||||||
|
DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects
|
||||||
|
)
|
||||||
max_redirects = DEFAULT_MAX_REDIRECTS if max_redirects is None else max_redirects
|
max_redirects = DEFAULT_MAX_REDIRECTS if max_redirects is None else max_redirects
|
||||||
retries = DEFAULT_MAX_RETRIES if retries is None else max(retries, 0)
|
retries = DEFAULT_MAX_RETRIES if retries is None else max(retries, 0)
|
||||||
backoff_factor = DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor
|
backoff_factor = (
|
||||||
|
DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor
|
||||||
|
)
|
||||||
headers = _clean_headers(headers, auth_token=auth_token)
|
headers = _clean_headers(headers, auth_token=auth_token)
|
||||||
proxies = DEFAULT_PROXY if proxies is None else proxies
|
proxy = DEFAULT_PROXY if proxy is None else proxy
|
||||||
|
|
||||||
async with httpx.AsyncClient(
|
async with httpx.AsyncClient(
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
follow_redirects=follow_redirects,
|
follow_redirects=follow_redirects,
|
||||||
max_redirects=max_redirects,
|
max_redirects=max_redirects,
|
||||||
proxies=proxies,
|
proxy=proxy,
|
||||||
) as client:
|
) as client:
|
||||||
last_exc: Exception | None = None
|
last_exc: Exception | None = None
|
||||||
for attempt in range(retries + 1):
|
for attempt in range(retries + 1):
|
||||||
try:
|
try:
|
||||||
start = time.monotonic()
|
start = time.monotonic()
|
||||||
response = await client.request(method=method, url=url, headers=headers, **kwargs)
|
response = await client.request(
|
||||||
|
method=method, url=url, headers=headers, **kwargs
|
||||||
|
)
|
||||||
duration = time.monotonic() - start
|
duration = time.monotonic() - start
|
||||||
logger.debug(f"async_request {method} {url} -> {response.status_code} in {duration:.3f}s")
|
logger.debug(
|
||||||
|
f"async_request {method} {url} -> {response.status_code} in {duration:.3f}s"
|
||||||
|
)
|
||||||
return response
|
return response
|
||||||
except httpx.RequestError as exc:
|
except httpx.RequestError as exc:
|
||||||
last_exc = exc
|
last_exc = exc
|
||||||
if attempt >= retries:
|
if attempt >= retries:
|
||||||
logger.warning(f"async_request exhausted retries for {method} {url}: {exc}")
|
logger.warning(
|
||||||
|
f"async_request exhausted retries for {method} {url}: {exc}"
|
||||||
|
)
|
||||||
raise
|
raise
|
||||||
delay = _get_delay(backoff_factor, attempt)
|
delay = _get_delay(backoff_factor, attempt)
|
||||||
logger.warning(f"async_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s")
|
logger.warning(
|
||||||
|
f"async_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s"
|
||||||
|
)
|
||||||
await asyncio.sleep(delay)
|
await asyncio.sleep(delay)
|
||||||
raise last_exc # pragma: no cover
|
raise last_exc # pragma: no cover
|
||||||
|
|
||||||
@ -107,39 +123,51 @@ def sync_request(
|
|||||||
auth_token: Optional[str] = None,
|
auth_token: Optional[str] = None,
|
||||||
retries: Optional[int] = None,
|
retries: Optional[int] = None,
|
||||||
backoff_factor: Optional[float] = None,
|
backoff_factor: Optional[float] = None,
|
||||||
proxies: Any = None,
|
proxy: Any = None,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> httpx.Response:
|
) -> httpx.Response:
|
||||||
"""Synchronous counterpart to async_request, for CLI/tests or sync contexts."""
|
"""Synchronous counterpart to async_request, for CLI/tests or sync contexts."""
|
||||||
timeout = timeout if timeout is not None else DEFAULT_TIMEOUT
|
timeout = timeout if timeout is not None else DEFAULT_TIMEOUT
|
||||||
follow_redirects = DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects
|
follow_redirects = (
|
||||||
|
DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects
|
||||||
|
)
|
||||||
max_redirects = DEFAULT_MAX_REDIRECTS if max_redirects is None else max_redirects
|
max_redirects = DEFAULT_MAX_REDIRECTS if max_redirects is None else max_redirects
|
||||||
retries = DEFAULT_MAX_RETRIES if retries is None else max(retries, 0)
|
retries = DEFAULT_MAX_RETRIES if retries is None else max(retries, 0)
|
||||||
backoff_factor = DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor
|
backoff_factor = (
|
||||||
|
DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor
|
||||||
|
)
|
||||||
headers = _clean_headers(headers, auth_token=auth_token)
|
headers = _clean_headers(headers, auth_token=auth_token)
|
||||||
proxies = DEFAULT_PROXY if proxies is None else proxies
|
proxy = DEFAULT_PROXY if proxy is None else proxy
|
||||||
|
|
||||||
with httpx.Client(
|
with httpx.Client(
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
follow_redirects=follow_redirects,
|
follow_redirects=follow_redirects,
|
||||||
max_redirects=max_redirects,
|
max_redirects=max_redirects,
|
||||||
proxies=proxies,
|
proxy=proxy,
|
||||||
) as client:
|
) as client:
|
||||||
last_exc: Exception | None = None
|
last_exc: Exception | None = None
|
||||||
for attempt in range(retries + 1):
|
for attempt in range(retries + 1):
|
||||||
try:
|
try:
|
||||||
start = time.monotonic()
|
start = time.monotonic()
|
||||||
response = client.request(method=method, url=url, headers=headers, **kwargs)
|
response = client.request(
|
||||||
|
method=method, url=url, headers=headers, **kwargs
|
||||||
|
)
|
||||||
duration = time.monotonic() - start
|
duration = time.monotonic() - start
|
||||||
logger.debug(f"sync_request {method} {url} -> {response.status_code} in {duration:.3f}s")
|
logger.debug(
|
||||||
|
f"sync_request {method} {url} -> {response.status_code} in {duration:.3f}s"
|
||||||
|
)
|
||||||
return response
|
return response
|
||||||
except httpx.RequestError as exc:
|
except httpx.RequestError as exc:
|
||||||
last_exc = exc
|
last_exc = exc
|
||||||
if attempt >= retries:
|
if attempt >= retries:
|
||||||
logger.warning(f"sync_request exhausted retries for {method} {url}: {exc}")
|
logger.warning(
|
||||||
|
f"sync_request exhausted retries for {method} {url}: {exc}"
|
||||||
|
)
|
||||||
raise
|
raise
|
||||||
delay = _get_delay(backoff_factor, attempt)
|
delay = _get_delay(backoff_factor, attempt)
|
||||||
logger.warning(f"sync_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s")
|
logger.warning(
|
||||||
|
f"sync_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s"
|
||||||
|
)
|
||||||
time.sleep(delay)
|
time.sleep(delay)
|
||||||
raise last_exc # pragma: no cover
|
raise last_exc # pragma: no cover
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user