search between multiple indiices for team function (#3079)

### What problem does this PR solve?

#2834 
### Type of change

- [x] New Feature (non-breaking change which adds functionality)

agent/component/__init__.py

@@ -29,6 +29,7 @@ from .jin10 import Jin10, Jin10Param
 from .tushare import TuShare, TuShareParam
 from .akshare import AkShare, AkShareParam
 from .crawler import Crawler, CrawlerParam
+from .invoke import Invoke, InvokeParam
 
 
 def component_class(class_name):

agent/component/generate.py

@@ -17,6 +17,7 @@ import re
 from functools import partial
 import pandas as pd
 from api.db import LLMType
+from api.db.services.dialog_service import message_fit_in
 from api.db.services.llm_service import LLMBundle
 from api.settings import retrievaler
 from agent.component.base import ComponentBase, ComponentParamBase
@@ -112,7 +113,7 @@ class Generate(ComponentBase):
 
         kwargs["input"] = input
         for n, v in kwargs.items():
-            prompt = re.sub(r"\{%s\}" % re.escape(n), str(v), prompt)
+            prompt = re.sub(r"\{%s\}" % re.escape(n), re.escape(str(v)), prompt)
 
         downstreams = self._canvas.get_component(self._id)["downstream"]
         if kwargs.get("stream") and len(downstreams) == 1 and self._canvas.get_component(downstreams[0])[
@@ -124,8 +125,10 @@ class Generate(ComponentBase):
                 retrieval_res["empty_response"]) else "Nothing found in knowledgebase!", "reference": []}
             return pd.DataFrame([res])
 
-        ans = chat_mdl.chat(prompt, self._canvas.get_history(self._param.message_history_window_size),
-                            self._param.gen_conf())
+        msg = self._canvas.get_history(self._param.message_history_window_size)
+        _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(chat_mdl.max_length * 0.97))
+        ans = chat_mdl.chat(msg[0]["content"], msg[1:], self._param.gen_conf())
+
         if self._param.cite and "content_ltks" in retrieval_res.columns and "vector" in retrieval_res.columns:
             res = self.set_cite(retrieval_res, ans)
             return pd.DataFrame([res])
@@ -141,9 +144,10 @@ class Generate(ComponentBase):
             self.set_output(res)
             return
 
+        msg = self._canvas.get_history(self._param.message_history_window_size)
+        _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(chat_mdl.max_length * 0.97))
         answer = ""
-        for ans in chat_mdl.chat_streamly(prompt, self._canvas.get_history(self._param.message_history_window_size),
-                                          self._param.gen_conf()):
+        for ans in chat_mdl.chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf()):
             res = {"content": ans, "reference": []}
             answer = ans
             yield res

agent/component/invoke.py

@@ -14,10 +14,10 @@
 #  limitations under the License.
 #
 import json
+import re
 from abc import ABC
-
 import requests
-
+from deepdoc.parser import HtmlParser
 from agent.component.base import ComponentBase, ComponentParamBase
 
 
@@ -34,11 +34,13 @@ class InvokeParam(ComponentParamBase):
         self.variables = []
         self.url = ""
         self.timeout = 60
+        self.clean_html = False
 
     def check(self):
         self.check_valid_value(self.method.lower(), "Type of content from the crawler", ['get', 'post', 'put'])
         self.check_empty(self.url, "End point URL")
         self.check_positive_integer(self.timeout, "Timeout time in second")
+        self.check_boolean(self.clean_html, "Clean HTML")
 
 
 class Invoke(ComponentBase, ABC):
@@ -63,7 +65,7 @@ class Invoke(ComponentBase, ABC):
         if self._param.headers:
             headers = json.loads(self._param.headers)
         proxies = None
-        if self._param.proxy:
+        if re.sub(r"https?:?/?/?", "", self._param.proxy):
             proxies = {"http": self._param.proxy, "https": self._param.proxy}
 
         if method == 'get':
@@ -72,6 +74,10 @@ class Invoke(ComponentBase, ABC):
                                     headers=headers,
                                     proxies=proxies,
                                     timeout=self._param.timeout)
+            if self._param.clean_html:
+                sections = HtmlParser()(None, response.content)
+                return Invoke.be_output("\n".join(sections))
+
             return Invoke.be_output(response.text)
 
         if method == 'put':
@@ -80,5 +86,18 @@ class Invoke(ComponentBase, ABC):
                                     headers=headers,
                                     proxies=proxies,
                                     timeout=self._param.timeout)
-
+            if self._param.clean_html:
+                sections = HtmlParser()(None, response.content)
+                return Invoke.be_output("\n".join(sections))
+            return Invoke.be_output(response.text)
+
+        if method == 'post':
+            response = requests.post(url=url,
+                                    json=args,
+                                    headers=headers,
+                                    proxies=proxies,
+                                    timeout=self._param.timeout)
+            if self._param.clean_html:
+                sections = HtmlParser()(None, response.content)
+                return Invoke.be_output("\n".join(sections))
             return Invoke.be_output(response.text)