diff --git a/graph/component/__init__.py b/graph/component/__init__.py index 7044cac13..dfb288dc2 100644 --- a/graph/component/__init__.py +++ b/graph/component/__init__.py @@ -8,6 +8,8 @@ from .switch import Switch, SwitchParam from .relevant import Relevant, RelevantParam from .message import Message, MessageParam from .rewrite import RewriteQuestion, RewriteQuestionParam +from .keyword import KeywordExtract, KeywordExtractParam +from .baidu import Baidu, BaiduParam def component_class(class_name): diff --git a/graph/component/baidu.py b/graph/component/baidu.py new file mode 100644 index 000000000..394963e96 --- /dev/null +++ b/graph/component/baidu.py @@ -0,0 +1,62 @@ +# +# Copyright 2024 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import random +from abc import ABC +from functools import partial +import pandas as pd +import requests +import re + +from graph.component.base import ComponentBase, ComponentParamBase + + +class BaiduParam(ComponentParamBase): + """ + Define the Baidu component parameters. + """ + + def __init__(self): + super().__init__() + self.top_n = 10 + + def check(self): + self.check_positive_integer(self.top_n, "Top N") + + +class Baidu(ComponentBase, ABC): + component_name = "Baidu" + + def _run(self, history, **kwargs): + ans = self.get_input() + ans = " - ".join(ans["content"]) if "content" in ans else "" + if not ans: + return Baidu.be_output(self._param.no) + + url = 'https://www.baidu.com/s?wd=' + ans + '&rn=' + str(self._param.top_n) + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36'} + response = requests.get(url=url, headers=headers) + + baidu_res = re.findall(r'"contentText":"(.*?)"', response.text) + url_res = re.findall(r"'url': \\\"(.*?)\\\"}", response.text) + for i in range(min(len(baidu_res), len(url_res))): + baidu_res[i] += '' + url_res[i] + '' + + del url_res + + br = pd.DataFrame(baidu_res, columns=['content']) + print(">>>>>>>>>>>>>>>>>>>>>>>>>>\n", br) + return br diff --git a/graph/component/keyword.py b/graph/component/keyword.py index 542e07a58..7a326cbfe 100644 --- a/graph/component/keyword.py +++ b/graph/component/keyword.py @@ -22,18 +22,17 @@ from graph.settings import DEBUG class KeywordExtractParam(GenerateParam): - """ Define the KeywordExtract component parameters. """ + def __init__(self): super().__init__() - self.temperature = 0.5 - self.prompt = "" - self.topn = 1 + self.top_n = 1 def check(self): super().check() + self.check_positive_integer(self.top_n, "Top N") def get_prompt(self): self.prompt = """ @@ -43,12 +42,12 @@ class KeywordExtractParam(GenerateParam): - Use comma as a delimiter to separate keywords/phrases. - Answer format: (in language of user's question) - keyword: -"""%self.topn +""" % self.top_n return self.prompt class KeywordExtract(Generate, ABC): - component_name = "RewriteQuestion" + component_name = "KeywordExtract" def _run(self, history, **kwargs): q = "" @@ -64,5 +63,3 @@ class KeywordExtract(Generate, ABC): ans = re.sub(r".*keyword:", "", ans).strip() if DEBUG: print(ans, ":::::::::::::::::::::::::::::::::") return KeywordExtract.be_output(ans) - - diff --git a/graph/test/dsl_examples/retrieval_relevant_keyword_baidu_and_generate.json b/graph/test/dsl_examples/retrieval_relevant_keyword_baidu_and_generate.json new file mode 100644 index 000000000..a34b58a36 --- /dev/null +++ b/graph/test/dsl_examples/retrieval_relevant_keyword_baidu_and_generate.json @@ -0,0 +1,103 @@ +{ + "components": { + "begin": { + "obj":{ + "component_name": "Begin", + "params": { + "prologue": "Hi there!" + } + }, + "downstream": ["answer:0"], + "upstream": [] + }, + "answer:0": { + "obj": { + "component_name": "Answer", + "params": {} + }, + "downstream": ["retrieval:0"], + "upstream": ["begin"] + }, + "retrieval:0": { + "obj": { + "component_name": "Retrieval", + "params": { + "similarity_threshold": 0.2, + "keywords_similarity_weight": 0.3, + "top_n": 6, + "top_k": 1024, + "rerank_id": "BAAI/bge-reranker-v2-m3", + "kb_ids": ["21ca4e6a2c8911ef8b1e0242ac120006"], + "empty_response": "Sorry, knowledge base has noting related information." + } + }, + "downstream": ["relevant:0"], + "upstream": ["answer:0"] + }, + "relevant:0": { + "obj": { + "component_name": "Relevant", + "params": { + "llm_id": "deepseek-chat", + "temperature": 0.02, + "yes": "generate:0", + "no": "keyword:0" + } + }, + "downstream": ["keyword:0", "generate:0"], + "upstream": ["retrieval:0"] + }, + "generate:0": { + "obj": { + "component_name": "Generate", + "params": { + "llm_id": "deepseek-chat", + "prompt": "You are an intelligent assistant. Please answer the question based on content of knowledge base. When all knowledge base content is irrelevant to the question, your answer must include the sentence \"The answer you are looking for is not found in the knowledge base!\". Answers need to consider chat history.\n Knowledge base content is as following:\n {input}\n The above is the content of knowledge base.", + "temperature": 0.2 + } + }, + "downstream": ["answer:0"], + "upstream": ["relevant:0"] + }, + "keyword:0": { + "obj": { + "component_name": "KeywordExtract", + "params": { + "llm_id": "deepseek-chat", + "prompt": "- Role: You're a question analyzer.\n - Requirements:\n - Summarize user's question, and give top %s important keyword/phrase.\n - Use comma as a delimiter to separate keywords/phrases.\n - Answer format: (in language of user's question)\n - keyword: ", + "temperature": 0.2, + "top_n": 1 + } + }, + "downstream": ["baidu:0"], + "upstream": ["relevant:0"] + }, + "baidu:0": { + "obj":{ + "component_name": "Baidu", + "params": { + "top_n": 10 + } + }, + "downstream": ["generate:1"], + "upstream": ["keyword:0"] + }, + "generate:1": { + "obj": { + "component_name": "Generate", + "params": { + "llm_id": "deepseek-chat", + "prompt": "You are an intelligent assistant. Please answer the question based on content searched from Baidu. When the answer from a Baidu search is incomplete, you need to output the URL link of the corresponding content as well. When all the content searched from Baidu is irrelevant to the question, your answer must include the sentence, \"The answer you are looking for is not found in the Baidu search!\". Answers need to consider chat history.\n The content of Baidu search is as follows:\n {input}\n The above is the content of Baidu search.", + "temperature": 0.2 + } + }, + "downstream": ["answer:0"], + "upstream": ["baidu:0"] + } + }, + "history": [], + "path": [], + "messages": [], + "reference": {}, + "answer": [] +}