pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-08 20:42:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Rework logging (#3358)

Browse Source 
Unified all log files into one.

### What problem does this PR solve?

Unified all log files into one.

### Type of change

- [x] Refactoring
This commit is contained in:
Zhichang Yu
2024-11-12 17:35:13 +08:00
committed by GitHub
parent 567a7563e7
commit a2a5631da4
75 changed files with 481 additions and 853 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
agent/component/arxiv.py
View File

@ -16,9 +16,8 @@
from abc import ABC
import arxiv
import pandas as pd
from agent.settings import DEBUG
from agent.component.base import ComponentBase, ComponentParamBase
from api.utils.log_utils import logger
class ArXivParam(ComponentParamBase):
"""
@ -65,5 +64,5 @@ class ArXiv(ComponentBase, ABC):
return ArXiv.be_output("")
df = pd.DataFrame(arxiv_res)
if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
logger.debug(f"df: {str(df)}")
return df

6
agent/component/baidu.py
View File

@ -13,14 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import random
from abc import ABC
from functools import partial
import pandas as pd
import requests
import re
from agent.settings import DEBUG
from agent.component.base import ComponentBase, ComponentParamBase
from api.utils.log_utils import logger
class BaiduParam(ComponentParamBase):
@ -64,6 +62,6 @@ class Baidu(ComponentBase, ABC):
return Baidu.be_output("")
df = pd.DataFrame(baidu_res)
if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
logger.debug(f"df: {str(df)}")
return df

14
agent/component/base.py
View File

@ -17,14 +17,14 @@ from abc import ABC
import builtins
import json
import os
from copy import deepcopy
from functools import partial
from typing import List, Dict, Tuple, Union
from typing import Tuple, Union
import pandas as pd
from agent import settings
from agent.settings import flow_logger, DEBUG
from api.utils.log_utils import logger
_FEEDED_DEPRECATED_PARAMS = "_feeded_deprecated_params"
_DEPRECATED_PARAMS = "_deprecated_params"
@ -361,13 +361,13 @@ class ComponentParamBase(ABC):
def _warn_deprecated_param(self, param_name, descr):
if self._deprecated_params_set.get(param_name):
flow_logger.warning(
logger.warning(
f"{descr} {param_name} is deprecated and ignored in this version."
)
def _warn_to_deprecate_param(self, param_name, descr, new_param):
if self._deprecated_params_set.get(param_name):
flow_logger.warning(
logger.warning(
f"{descr} {param_name} will be deprecated in future release; "
f"please use {new_param} instead."
)
@ -403,7 +403,7 @@ class ComponentBase(ABC):
return cpnts
def run(self, history, **kwargs):
flow_logger.info("{}, history: {}, kwargs: {}".format(self, json.dumps(history, ensure_ascii=False),
logger.info("{}, history: {}, kwargs: {}".format(self, json.dumps(history, ensure_ascii=False),
json.dumps(kwargs, ensure_ascii=False)))
try:
res = self._run(history, **kwargs)
@ -463,7 +463,7 @@ class ComponentBase(ABC):
reversed_cpnts.extend(self._canvas.path[-2])
reversed_cpnts.extend(self._canvas.path[-1])
if DEBUG: print(self.component_name, reversed_cpnts[::-1])
logger.debug(f"{self.component_name} {reversed_cpnts[::-1]}")
for u in reversed_cpnts[::-1]:
if self.get_component_name(u) in ["switch", "concentrator"]: continue
if self.component_name.lower() == "generate" and self.get_component_name(u) == "retrieval":

5
agent/component/bing.py
View File

@ -16,9 +16,8 @@
from abc import ABC
import requests
import pandas as pd
from agent.settings import DEBUG
from agent.component.base import ComponentBase, ComponentParamBase
from api.utils.log_utils import logger
class BingParam(ComponentParamBase):
"""
@ -81,5 +80,5 @@ class Bing(ComponentBase, ABC):
return Bing.be_output("")
df = pd.DataFrame(bing_res)
if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
logger.debug(f"df: {str(df)}")
return df

6
agent/component/categorize.py
View File

@ -17,7 +17,7 @@ from abc import ABC
from api.db import LLMType
from api.db.services.llm_service import LLMBundle
from agent.component import GenerateParam, Generate
from agent.settings import DEBUG
from api.utils.log_utils import logger
class CategorizeParam(GenerateParam):
@ -34,7 +34,7 @@ class CategorizeParam(GenerateParam):
super().check()
self.check_empty(self.category_description, "[Categorize] Category examples")
for k, v in self.category_description.items():
if not k: raise ValueError(f"[Categorize] Category name can not be empty!")
if not k: raise ValueError("[Categorize] Category name can not be empty!")
if not v.get("to"): raise ValueError(f"[Categorize] 'To' of category {k} can not be empty!")
def get_prompt(self):
@ -77,7 +77,7 @@ class Categorize(Generate, ABC):
chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
ans = chat_mdl.chat(self._param.get_prompt(), [{"role": "user", "content": input}],
self._param.gen_conf())
if DEBUG: print(ans, ":::::::::::::::::::::::::::::::::", input)
logger.debug(f"input: {input}, answer: {str(ans)}")
for c in self._param.category_description.keys():
if ans.lower().find(c.lower()) >= 0:
return Categorize.be_output(self._param.category_description[c]["to"])

4
agent/component/duckduckgo.py
View File

@ -16,8 +16,8 @@
from abc import ABC
from duckduckgo_search import DDGS
import pandas as pd
from agent.settings import DEBUG
from agent.component.base import ComponentBase, ComponentParamBase
from api.utils.log_utils import logger
class DuckDuckGoParam(ComponentParamBase):
@ -62,5 +62,5 @@ class DuckDuckGo(ComponentBase, ABC):
return DuckDuckGo.be_output("")
df = pd.DataFrame(duck_res)
if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
logger.debug("df: {df}")
return df

4
agent/component/github.py
View File

@ -16,8 +16,8 @@
from abc import ABC
import pandas as pd
import requests
from agent.settings import DEBUG
from agent.component.base import ComponentBase, ComponentParamBase
from api.utils.log_utils import logger
class GitHubParam(ComponentParamBase):
@ -57,5 +57,5 @@ class GitHub(ComponentBase, ABC):
return GitHub.be_output("")
df = pd.DataFrame(github_res)
if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
logger.debug(f"df: {df}")
return df

6
agent/component/google.py
View File

@ -16,8 +16,8 @@
from abc import ABC
from serpapi import GoogleSearch
import pandas as pd
from agent.settings import DEBUG
from agent.component.base import ComponentBase, ComponentParamBase
from api.utils.log_utils import logger
class GoogleParam(ComponentParamBase):
@ -85,12 +85,12 @@ class Google(ComponentBase, ABC):
"hl": self._param.language, "num": self._param.top_n})
google_res = [{"content": '<a href="' + i["link"] + '">' + i["title"] + '</a> ' + i["snippet"]} for i in
client.get_dict()["organic_results"]]
except Exception as e:
except Exception:
return Google.be_output("**ERROR**: Existing Unavailable Parameters!")
if not google_res:
return Google.be_output("")
df = pd.DataFrame(google_res)
if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
logger.debug(f"df: {df}")
return df

8
agent/component/googlescholar.py
View File

@ -15,9 +15,9 @@
#
from abc import ABC
import pandas as pd
from agent.settings import DEBUG
from agent.component.base import ComponentBase, ComponentParamBase
from scholarly import scholarly
from api.utils.log_utils import logger
class GoogleScholarParam(ComponentParamBase):
@ -58,13 +58,13 @@ class GoogleScholar(ComponentBase, ABC):
'pub_url'] + '"></a> ' + "\n author: " + ",".join(pub['bib']['author']) + '\n Abstract: ' + pub[
'bib'].get('abstract', 'no abstract')})
except StopIteration or Exception as e:
print("**ERROR** " + str(e))
except StopIteration or Exception:
logger.exception("GoogleScholar")
break
if not scholar_res:
return GoogleScholar.be_output("")
df = pd.DataFrame(scholar_res)
if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
logger.debug(f"df: {df}")
return df

4
agent/component/keyword.py
View File

@ -18,7 +18,7 @@ from abc import ABC
from api.db import LLMType
from api.db.services.llm_service import LLMBundle
from agent.component import GenerateParam, Generate
from agent.settings import DEBUG
from api.utils.log_utils import logger
class KeywordExtractParam(GenerateParam):
@ -58,5 +58,5 @@ class KeywordExtract(Generate, ABC):
self._param.gen_conf())
ans = re.sub(r".*keyword:", "", ans).strip()
if DEBUG: print(ans, ":::::::::::::::::::::::::::::::::")
logger.info(f"ans: {ans}")
return KeywordExtract.be_output(ans)

4
agent/component/pubmed.py
View File

@ -18,8 +18,8 @@ from Bio import Entrez
import re
import pandas as pd
import xml.etree.ElementTree as ET
from agent.settings import DEBUG
from agent.component.base import ComponentBase, ComponentParamBase
from api.utils.log_utils import logger
class PubMedParam(ComponentParamBase):
@ -65,5 +65,5 @@ class PubMed(ComponentBase, ABC):
return PubMed.be_output("")
df = pd.DataFrame(pubmed_res)
if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
logger.debug(f"df: {df}")
return df

3
agent/component/relevant.py
View File

@ -18,6 +18,7 @@ from api.db import LLMType
from api.db.services.llm_service import LLMBundle
from agent.component import GenerateParam, Generate
from rag.utils import num_tokens_from_string, encoder
from api.utils.log_utils import logger
class RelevantParam(GenerateParam):
@ -70,7 +71,7 @@ class Relevant(Generate, ABC):
ans = chat_mdl.chat(self._param.get_prompt(), [{"role": "user", "content": ans}],
self._param.gen_conf())
print(ans, ":::::::::::::::::::::::::::::::::")
logger.info(ans)
if ans.lower().find("yes") >= 0:
return Relevant.be_output(self._param.yes)
if ans.lower().find("no") >= 0:

3
agent/component/retrieval.py
View File

@ -22,6 +22,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.services.llm_service import LLMBundle
from api.settings import retrievaler
from agent.component.base import ComponentBase, ComponentParamBase
from api.utils.log_utils import logger
class RetrievalParam(ComponentParamBase):
@ -80,7 +81,7 @@ class Retrieval(ComponentBase, ABC):
df = pd.DataFrame(kbinfos["chunks"])
df["content"] = df["content_with_weight"]
del df["content_with_weight"]
print(">>>>>>>>>>>>>>>>>>>>>>>>>>\n", query, df)
logger.debug("{} {}".format(query, df))
return df

3
agent/component/rewrite.py
View File

@ -17,6 +17,7 @@ from abc import ABC
from api.db import LLMType
from api.db.services.llm_service import LLMBundle
from agent.component import GenerateParam, Generate
from api.utils.log_utils import logger
class RewriteQuestionParam(GenerateParam):
@ -104,7 +105,7 @@ class RewriteQuestion(Generate, ABC):
self._canvas.history.pop()
self._canvas.history.append(("user", ans))
print(ans, ":::::::::::::::::::::::::::::::::")
logger.info(ans)
return RewriteQuestion.be_output(ans)

6
agent/component/wikipedia.py
View File

@ -13,13 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import random
from abc import ABC
from functools import partial
import wikipedia
import pandas as pd
from agent.settings import DEBUG
from agent.component.base import ComponentBase, ComponentParamBase
from api.utils.log_utils import logger
class WikipediaParam(ComponentParamBase):
@ -65,5 +63,5 @@ class Wikipedia(ComponentBase, ABC):
return Wikipedia.be_output("")
df = pd.DataFrame(wiki_res)
if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
logger.debug(f"df: {df}")
return df

5
agent/component/yahoofinance.py
View File

@ -17,6 +17,7 @@ from abc import ABC
import pandas as pd
from agent.component.base import ComponentBase, ComponentParamBase
import yfinance as yf
from api.utils.log_utils import logger
class YahooFinanceParam(ComponentParamBase):
@ -74,8 +75,8 @@ class YahooFinance(ComponentBase, ABC):
{"content": "quarterly cash flow statement:\n" + msft.quarterly_cashflow.to_markdown() + "\n"})
if self._param.news:
yohoo_res.append({"content": "news:\n" + pd.DataFrame(msft.news).to_markdown() + "\n"})
except Exception as e:
print("**ERROR** " + str(e))
except Exception:
logger.exception("YahooFinance got exception")
if not yohoo_res:
return YahooFinance.be_output("")