Ref: update loggings (#11987)

### What problem does this PR solve?

Ref: update loggins

### Type of change

- [x] Refactoring
This commit is contained in:
Magicbook1108
2025-12-17 15:43:25 +08:00
committed by GitHub
parent d16643a53d
commit 82d4e5fb87
4 changed files with 28 additions and 8 deletions

View File

@ -245,7 +245,7 @@ class JiraConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPermSync
while True:
attempt += 1
jql = self._build_jql(attempt_start, end)
logger.info(f"[Jira] Executing Jira JQL attempt {attempt} (start={attempt_start}, end={end}, buffered_retry={retried_with_buffer})")
logger.info(f"[Jira] Executing Jira JQL attempt {attempt} (buffered_retry={retried_with_buffer})[start and end parameters redacted]")
try:
return (yield from self._load_from_checkpoint_internal(jql, checkpoint, start_filter=start))
except Exception as exc:

View File

@ -144,7 +144,7 @@ async def async_request(
method=method, url=url, headers=headers, **kwargs
)
duration = time.monotonic() - start
log_url = "<SENSITIVE ENDPOINT>" if _is_sensitive_url else _redact_sensitive_url_params(url)
log_url = "<SENSITIVE ENDPOINT>" if _is_sensitive_url(url) else _redact_sensitive_url_params(url)
logger.debug(
f"async_request {method} {log_url} -> {response.status_code} in {duration:.3f}s"
)
@ -152,13 +152,13 @@ async def async_request(
except httpx.RequestError as exc:
last_exc = exc
if attempt >= retries:
log_url = "<SENSITIVE ENDPOINT>" if _is_sensitive_url else _redact_sensitive_url_params(url)
log_url = "<SENSITIVE ENDPOINT>" if _is_sensitive_url(url) else _redact_sensitive_url_params(url)
logger.warning(
f"async_request exhausted retries for {method} {log_url}"
)
raise
delay = _get_delay(backoff_factor, attempt)
log_url = "<SENSITIVE ENDPOINT>" if _is_sensitive_url else _redact_sensitive_url_params(url)
log_url = "<SENSITIVE ENDPOINT>" if _is_sensitive_url(url) else _redact_sensitive_url_params(url)
logger.warning(
f"async_request attempt {attempt + 1}/{retries + 1} failed for {method} {log_url}; retrying in {delay:.2f}s"
)

View File

@ -57,8 +57,18 @@ class MinerUOcrModel(Base, MinerUParser):
self.mineru_server_url = _resolve_config("mineru_server_url", "MINERU_SERVER_URL", "")
self.mineru_delete_output = bool(int(_resolve_config("mineru_delete_output", "MINERU_DELETE_OUTPUT", 1)))
# Redact sensitive config keys before logging
redacted_config = {}
for k, v in config.items():
if any(
sensitive_word in k.lower()
for sensitive_word in ("key", "password", "token", "secret")
):
redacted_config[k] = "[REDACTED]"
else:
redacted_config[k] = v
logging.info(
f"Parsed MinerU config: backend={self.mineru_backend} api={self.mineru_api} server_url={self.mineru_server_url} output_dir={self.mineru_output_dir} delete_output={self.mineru_delete_output}"
f"Parsed MinerU config (sensitive fields redacted): {redacted_config}"
)
MinerUParser.__init__(self, mineru_api=self.mineru_api, mineru_server_url=self.mineru_server_url)

View File

@ -41,9 +41,19 @@ def get_opendal_config():
scheme = opendal_config.get("scheme")
config_data = opendal_config.get("config", {})
kwargs = {"scheme": scheme, **config_data}
safe_log_keys=['scheme', 'host', 'port', 'database', 'table']
# Only include non-sensitive keys in logs. Do NOT
# add 'password' or any key containing embedded credentials
# (like 'connection_string').
safe_log_keys = ['scheme', 'host', 'port', 'database', 'table']
loggable_kwargs = {k: v for k, v in kwargs.items() if k in safe_log_keys}
logging.info("Loaded OpenDAL configuration(non sensitive): %s", loggable_kwargs)
logging.info("Loaded OpenDAL configuration (non sensitive): %s", loggable_kwargs)
# For safety, explicitly remove sensitive keys from kwargs after use
if "password" in kwargs:
del kwargs["password"]
if "connection_string" in kwargs:
del kwargs["connection_string"]
return kwargs
except Exception as e:
logging.error("Failed to load OpenDAL configuration from yaml: %s", str(e))