mirror of
https://github.com/infiniflow/ragflow.git
synced 2025-12-08 20:42:30 +08:00
Compare commits
11 Commits
d5f8548200
...
a793dd2ea8
| Author | SHA1 | Date | |
|---|---|---|---|
| a793dd2ea8 | |||
| 915e385244 | |||
| 7a344a32f9 | |||
| 8c1ee3845a | |||
| 8c751d5afc | |||
| f5faf0c94f | |||
| af72e8dc33 | |||
| bcd70affb5 | |||
| 6987e9f23b | |||
| 41665b0865 | |||
| d1744aaaf3 |
@ -206,17 +206,28 @@ class Graph:
|
||||
for key in path.split('.'):
|
||||
if cur is None:
|
||||
return None
|
||||
|
||||
if isinstance(cur, str):
|
||||
try:
|
||||
cur = json.loads(cur)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
if isinstance(cur, dict):
|
||||
cur = cur.get(key)
|
||||
else:
|
||||
cur = getattr(cur, key, None)
|
||||
continue
|
||||
|
||||
if isinstance(cur, (list, tuple)):
|
||||
try:
|
||||
idx = int(key)
|
||||
cur = cur[idx]
|
||||
except Exception:
|
||||
return None
|
||||
continue
|
||||
|
||||
cur = getattr(cur, key, None)
|
||||
return cur
|
||||
|
||||
|
||||
def set_variable_value(self, exp: str,value):
|
||||
exp = exp.strip("{").strip("}").strip(" ").strip("{").strip("}")
|
||||
if exp.find("@") < 0:
|
||||
@ -440,7 +451,7 @@ class Canvas(Graph):
|
||||
|
||||
if isinstance(cpn_obj.output("attachment"), tuple):
|
||||
yield decorate("message", {"attachment": cpn_obj.output("attachment")})
|
||||
|
||||
|
||||
yield decorate("message_end", {"reference": self.get_reference() if cite else None})
|
||||
|
||||
while partials:
|
||||
@ -647,4 +658,3 @@ class Canvas(Graph):
|
||||
|
||||
def get_component_thoughts(self, cpn_id) -> str:
|
||||
return self.components.get(cpn_id)["obj"].thoughts()
|
||||
|
||||
|
||||
@ -13,16 +13,20 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import ast
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from abc import ABC
|
||||
from strenum import StrEnum
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
|
||||
from common.connection_utils import timeout
|
||||
from strenum import StrEnum
|
||||
|
||||
from agent.tools.base import ToolBase, ToolMeta, ToolParamBase
|
||||
from common import settings
|
||||
from common.connection_utils import timeout
|
||||
|
||||
|
||||
class Language(StrEnum):
|
||||
@ -62,7 +66,7 @@ class CodeExecParam(ToolParamBase):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.meta:ToolMeta = {
|
||||
self.meta: ToolMeta = {
|
||||
"name": "execute_code",
|
||||
"description": """
|
||||
This tool has a sandbox that can execute code written in 'Python'/'Javascript'. It recieves a piece of code and return a Json string.
|
||||
@ -99,16 +103,12 @@ module.exports = { main };
|
||||
"enum": ["python", "javascript"],
|
||||
"required": True,
|
||||
},
|
||||
"script": {
|
||||
"type": "string",
|
||||
"description": "A piece of code in right format. There MUST be main function.",
|
||||
"required": True
|
||||
}
|
||||
}
|
||||
"script": {"type": "string", "description": "A piece of code in right format. There MUST be main function.", "required": True},
|
||||
},
|
||||
}
|
||||
super().__init__()
|
||||
self.lang = Language.PYTHON.value
|
||||
self.script = "def main(arg1: str, arg2: str) -> dict: return {\"result\": arg1 + arg2}"
|
||||
self.script = 'def main(arg1: str, arg2: str) -> dict: return {"result": arg1 + arg2}'
|
||||
self.arguments = {}
|
||||
self.outputs = {"result": {"value": "", "type": "string"}}
|
||||
|
||||
@ -119,17 +119,14 @@ module.exports = { main };
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
res = {}
|
||||
for k, v in self.arguments.items():
|
||||
res[k] = {
|
||||
"type": "line",
|
||||
"name": k
|
||||
}
|
||||
res[k] = {"type": "line", "name": k}
|
||||
return res
|
||||
|
||||
|
||||
class CodeExec(ToolBase, ABC):
|
||||
component_name = "CodeExec"
|
||||
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
def _invoke(self, **kwargs):
|
||||
if self.check_if_canceled("CodeExec processing"):
|
||||
return
|
||||
@ -138,17 +135,12 @@ class CodeExec(ToolBase, ABC):
|
||||
script = kwargs.get("script", self._param.script)
|
||||
arguments = {}
|
||||
for k, v in self._param.arguments.items():
|
||||
|
||||
if kwargs.get(k):
|
||||
arguments[k] = kwargs[k]
|
||||
continue
|
||||
arguments[k] = self._canvas.get_variable_value(v) if v else None
|
||||
|
||||
self._execute_code(
|
||||
language=lang,
|
||||
code=script,
|
||||
arguments=arguments
|
||||
)
|
||||
self._execute_code(language=lang, code=script, arguments=arguments)
|
||||
|
||||
def _execute_code(self, language: str, code: str, arguments: dict):
|
||||
import requests
|
||||
@ -169,7 +161,7 @@ class CodeExec(ToolBase, ABC):
|
||||
if self.check_if_canceled("CodeExec execution"):
|
||||
return "Task has been canceled"
|
||||
|
||||
resp = requests.post(url=f"http://{settings.SANDBOX_HOST}:9385/run", json=code_req, timeout=int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
|
||||
resp = requests.post(url=f"http://{settings.SANDBOX_HOST}:9385/run", json=code_req, timeout=int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
logging.info(f"http://{settings.SANDBOX_HOST}:9385/run, code_req: {code_req}, resp.status_code {resp.status_code}:")
|
||||
|
||||
if self.check_if_canceled("CodeExec execution"):
|
||||
@ -183,35 +175,10 @@ class CodeExec(ToolBase, ABC):
|
||||
if stderr:
|
||||
self.set_output("_ERROR", stderr)
|
||||
return
|
||||
try:
|
||||
rt = eval(body.get("stdout", ""))
|
||||
except Exception:
|
||||
rt = body.get("stdout", "")
|
||||
logging.info(f"http://{settings.SANDBOX_HOST}:9385/run -> {rt}")
|
||||
if isinstance(rt, tuple):
|
||||
for i, (k, o) in enumerate(self._param.outputs.items()):
|
||||
if self.check_if_canceled("CodeExec execution"):
|
||||
return
|
||||
|
||||
if k.find("_") == 0:
|
||||
continue
|
||||
o["value"] = rt[i]
|
||||
elif isinstance(rt, dict):
|
||||
for i, (k, o) in enumerate(self._param.outputs.items()):
|
||||
if self.check_if_canceled("CodeExec execution"):
|
||||
return
|
||||
|
||||
if k not in rt or k.find("_") == 0:
|
||||
continue
|
||||
o["value"] = rt[k]
|
||||
else:
|
||||
for i, (k, o) in enumerate(self._param.outputs.items()):
|
||||
if self.check_if_canceled("CodeExec execution"):
|
||||
return
|
||||
|
||||
if k.find("_") == 0:
|
||||
continue
|
||||
o["value"] = rt
|
||||
raw_stdout = body.get("stdout", "")
|
||||
parsed_stdout = self._deserialize_stdout(raw_stdout)
|
||||
logging.info(f"[CodeExec]: http://{settings.SANDBOX_HOST}:9385/run -> {parsed_stdout}")
|
||||
self._populate_outputs(parsed_stdout, raw_stdout)
|
||||
else:
|
||||
self.set_output("_ERROR", "There is no response from sandbox")
|
||||
|
||||
@ -228,3 +195,149 @@ class CodeExec(ToolBase, ABC):
|
||||
|
||||
def thoughts(self) -> str:
|
||||
return "Running a short script to process data."
|
||||
|
||||
def _deserialize_stdout(self, stdout: str):
|
||||
text = str(stdout).strip()
|
||||
if not text:
|
||||
return ""
|
||||
for loader in (json.loads, ast.literal_eval):
|
||||
try:
|
||||
return loader(text)
|
||||
except Exception:
|
||||
continue
|
||||
return text
|
||||
|
||||
def _coerce_output_value(self, value, expected_type: Optional[str]):
|
||||
if expected_type is None:
|
||||
return value
|
||||
|
||||
etype = expected_type.strip().lower()
|
||||
inner_type = None
|
||||
if etype.startswith("array<") and etype.endswith(">"):
|
||||
inner_type = etype[6:-1].strip()
|
||||
etype = "array"
|
||||
|
||||
try:
|
||||
if etype == "string":
|
||||
return "" if value is None else str(value)
|
||||
|
||||
if etype == "number":
|
||||
if value is None or value == "":
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return value
|
||||
return float(value)
|
||||
|
||||
if etype == "boolean":
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
lv = value.lower()
|
||||
if lv in ("true", "1", "yes", "y", "on"):
|
||||
return True
|
||||
if lv in ("false", "0", "no", "n", "off"):
|
||||
return False
|
||||
return bool(value)
|
||||
|
||||
if etype == "array":
|
||||
candidate = value
|
||||
if isinstance(candidate, str):
|
||||
parsed = self._deserialize_stdout(candidate)
|
||||
candidate = parsed
|
||||
if isinstance(candidate, tuple):
|
||||
candidate = list(candidate)
|
||||
if not isinstance(candidate, list):
|
||||
candidate = [] if candidate is None else [candidate]
|
||||
|
||||
if inner_type == "string":
|
||||
return ["" if v is None else str(v) for v in candidate]
|
||||
if inner_type == "number":
|
||||
coerced = []
|
||||
for v in candidate:
|
||||
try:
|
||||
if v is None or v == "":
|
||||
coerced.append(None)
|
||||
elif isinstance(v, (int, float)):
|
||||
coerced.append(v)
|
||||
else:
|
||||
coerced.append(float(v))
|
||||
except Exception:
|
||||
coerced.append(v)
|
||||
return coerced
|
||||
return candidate
|
||||
|
||||
if etype == "object":
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
parsed = self._deserialize_stdout(value)
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
return value
|
||||
except Exception:
|
||||
return value
|
||||
|
||||
return value
|
||||
|
||||
def _populate_outputs(self, parsed_stdout, raw_stdout: str):
|
||||
outputs_items = list(self._param.outputs.items())
|
||||
logging.info(f"[CodeExec]: outputs schema keys: {[k for k, _ in outputs_items]}")
|
||||
if not outputs_items:
|
||||
return
|
||||
|
||||
if isinstance(parsed_stdout, dict):
|
||||
for key, meta in outputs_items:
|
||||
if key.startswith("_"):
|
||||
continue
|
||||
val = self._get_by_path(parsed_stdout, key)
|
||||
coerced = self._coerce_output_value(val, meta.get("type"))
|
||||
logging.info(f"[CodeExec]: populate dict key='{key}' raw='{val}' coerced='{coerced}'")
|
||||
self.set_output(key, coerced)
|
||||
return
|
||||
|
||||
if isinstance(parsed_stdout, (list, tuple)):
|
||||
for idx, (key, meta) in enumerate(outputs_items):
|
||||
if key.startswith("_"):
|
||||
continue
|
||||
val = parsed_stdout[idx] if idx < len(parsed_stdout) else None
|
||||
coerced = self._coerce_output_value(val, meta.get("type"))
|
||||
logging.info(f"[CodeExec]: populate list key='{key}' raw='{val}' coerced='{coerced}'")
|
||||
self.set_output(key, coerced)
|
||||
return
|
||||
|
||||
default_val = parsed_stdout if parsed_stdout is not None else raw_stdout
|
||||
for idx, (key, meta) in enumerate(outputs_items):
|
||||
if key.startswith("_"):
|
||||
continue
|
||||
val = default_val if idx == 0 else None
|
||||
coerced = self._coerce_output_value(val, meta.get("type"))
|
||||
logging.info(f"[CodeExec]: populate scalar key='{key}' raw='{val}' coerced='{coerced}'")
|
||||
self.set_output(key, coerced)
|
||||
|
||||
def _get_by_path(self, data, path: str):
|
||||
if not path:
|
||||
return None
|
||||
cur = data
|
||||
for part in path.split("."):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
return None
|
||||
if isinstance(cur, dict):
|
||||
cur = cur.get(part)
|
||||
elif isinstance(cur, list):
|
||||
try:
|
||||
idx = int(part)
|
||||
cur = cur[idx]
|
||||
except Exception:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
if cur is None:
|
||||
return None
|
||||
logging.info(f"[CodeExec]: resolve path '{path}' -> {cur}")
|
||||
return cur
|
||||
|
||||
@ -304,6 +304,8 @@ def meta_filter(metas: dict, filters: list[dict], logic: str = "and"):
|
||||
for conds in [
|
||||
(operator == "contains", str(value).lower() in str(input).lower()),
|
||||
(operator == "not contains", str(value).lower() not in str(input).lower()),
|
||||
(operator == "in", str(input).lower() in str(value).lower()),
|
||||
(operator == "not in", str(input).lower() not in str(value).lower()),
|
||||
(operator == "start with", str(input).lower().startswith(str(value).lower())),
|
||||
(operator == "end with", str(input).lower().endswith(str(value).lower())),
|
||||
(operator == "empty", not input),
|
||||
|
||||
@ -119,6 +119,7 @@ class FileSource(StrEnum):
|
||||
SLACK = "slack"
|
||||
TEAMS = "teams"
|
||||
MOODLE = "moodle"
|
||||
DROPBOX = "dropbox"
|
||||
|
||||
|
||||
class PipelineTaskType(StrEnum):
|
||||
|
||||
@ -90,7 +90,7 @@ class BlobStorageConnector(LoadConnector, PollConnector):
|
||||
elif self.bucket_type == BlobType.S3_COMPATIBLE:
|
||||
if not all(
|
||||
credentials.get(key)
|
||||
for key in ["endpoint_url", "aws_access_key_id", "aws_secret_access_key"]
|
||||
for key in ["endpoint_url", "aws_access_key_id", "aws_secret_access_key", "addressing_style"]
|
||||
):
|
||||
raise ConnectorMissingCredentialError("S3 Compatible Storage")
|
||||
|
||||
|
||||
@ -50,6 +50,7 @@ class DocumentSource(str, Enum):
|
||||
DISCORD = "discord"
|
||||
MOODLE = "moodle"
|
||||
S3_COMPATIBLE = "s3_compatible"
|
||||
DROPBOX = "dropbox"
|
||||
|
||||
|
||||
class FileOrigin(str, Enum):
|
||||
|
||||
@ -1,13 +1,24 @@
|
||||
"""Dropbox connector"""
|
||||
|
||||
import logging
|
||||
from datetime import timezone
|
||||
from typing import Any
|
||||
|
||||
from dropbox import Dropbox
|
||||
from dropbox.exceptions import ApiError, AuthError
|
||||
from dropbox.files import FileMetadata, FolderMetadata
|
||||
|
||||
from common.data_source.config import INDEX_BATCH_SIZE
|
||||
from common.data_source.exceptions import ConnectorValidationError, InsufficientPermissionsError, ConnectorMissingCredentialError
|
||||
from common.data_source.config import INDEX_BATCH_SIZE, DocumentSource
|
||||
from common.data_source.exceptions import (
|
||||
ConnectorMissingCredentialError,
|
||||
ConnectorValidationError,
|
||||
InsufficientPermissionsError,
|
||||
)
|
||||
from common.data_source.interfaces import LoadConnector, PollConnector, SecondsSinceUnixEpoch
|
||||
from common.data_source.models import Document, GenerateDocumentsOutput
|
||||
from common.data_source.utils import get_file_ext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DropboxConnector(LoadConnector, PollConnector):
|
||||
@ -19,29 +30,29 @@ class DropboxConnector(LoadConnector, PollConnector):
|
||||
|
||||
def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Load Dropbox credentials"""
|
||||
try:
|
||||
access_token = credentials.get("dropbox_access_token")
|
||||
if not access_token:
|
||||
raise ConnectorMissingCredentialError("Dropbox access token is required")
|
||||
|
||||
self.dropbox_client = Dropbox(access_token)
|
||||
return None
|
||||
except Exception as e:
|
||||
raise ConnectorMissingCredentialError(f"Dropbox: {e}")
|
||||
access_token = credentials.get("dropbox_access_token")
|
||||
if not access_token:
|
||||
raise ConnectorMissingCredentialError("Dropbox access token is required")
|
||||
|
||||
self.dropbox_client = Dropbox(access_token)
|
||||
return None
|
||||
|
||||
def validate_connector_settings(self) -> None:
|
||||
"""Validate Dropbox connector settings"""
|
||||
if not self.dropbox_client:
|
||||
if self.dropbox_client is None:
|
||||
raise ConnectorMissingCredentialError("Dropbox")
|
||||
|
||||
|
||||
try:
|
||||
# Test connection by getting current account info
|
||||
self.dropbox_client.users_get_current_account()
|
||||
except (AuthError, ApiError) as e:
|
||||
if "invalid_access_token" in str(e).lower():
|
||||
raise InsufficientPermissionsError("Invalid Dropbox access token")
|
||||
else:
|
||||
raise ConnectorValidationError(f"Dropbox validation error: {e}")
|
||||
self.dropbox_client.files_list_folder(path="", limit=1)
|
||||
except AuthError as e:
|
||||
logger.exception("[Dropbox]: Failed to validate Dropbox credentials")
|
||||
raise ConnectorValidationError(f"Dropbox credential is invalid: {e}")
|
||||
except ApiError as e:
|
||||
if e.error is not None and "insufficient_permissions" in str(e.error).lower():
|
||||
raise InsufficientPermissionsError("Your Dropbox token does not have sufficient permissions.")
|
||||
raise ConnectorValidationError(f"Unexpected Dropbox error during validation: {e.user_message_text or e}")
|
||||
except Exception as e:
|
||||
raise ConnectorValidationError(f"Unexpected error during Dropbox settings validation: {e}")
|
||||
|
||||
def _download_file(self, path: str) -> bytes:
|
||||
"""Download a single file from Dropbox."""
|
||||
@ -54,26 +65,105 @@ class DropboxConnector(LoadConnector, PollConnector):
|
||||
"""Create a shared link for a file in Dropbox."""
|
||||
if self.dropbox_client is None:
|
||||
raise ConnectorMissingCredentialError("Dropbox")
|
||||
|
||||
|
||||
try:
|
||||
# Try to get existing shared links first
|
||||
shared_links = self.dropbox_client.sharing_list_shared_links(path=path)
|
||||
if shared_links.links:
|
||||
return shared_links.links[0].url
|
||||
|
||||
# Create a new shared link
|
||||
link_settings = self.dropbox_client.sharing_create_shared_link_with_settings(path)
|
||||
return link_settings.url
|
||||
except Exception:
|
||||
# Fallback to basic link format
|
||||
return f"https://www.dropbox.com/home{path}"
|
||||
|
||||
def poll_source(self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch) -> Any:
|
||||
link_metadata = self.dropbox_client.sharing_create_shared_link_with_settings(path)
|
||||
return link_metadata.url
|
||||
except ApiError as err:
|
||||
logger.exception(f"[Dropbox]: Failed to create a shared link for {path}: {err}")
|
||||
return ""
|
||||
|
||||
def _yield_files_recursive(
|
||||
self,
|
||||
path: str,
|
||||
start: SecondsSinceUnixEpoch | None,
|
||||
end: SecondsSinceUnixEpoch | None,
|
||||
) -> GenerateDocumentsOutput:
|
||||
"""Yield files in batches from a specified Dropbox folder, including subfolders."""
|
||||
if self.dropbox_client is None:
|
||||
raise ConnectorMissingCredentialError("Dropbox")
|
||||
|
||||
result = self.dropbox_client.files_list_folder(
|
||||
path,
|
||||
limit=self.batch_size,
|
||||
recursive=False,
|
||||
include_non_downloadable_files=False,
|
||||
)
|
||||
|
||||
while True:
|
||||
batch: list[Document] = []
|
||||
for entry in result.entries:
|
||||
if isinstance(entry, FileMetadata):
|
||||
modified_time = entry.client_modified
|
||||
if modified_time.tzinfo is None:
|
||||
modified_time = modified_time.replace(tzinfo=timezone.utc)
|
||||
else:
|
||||
modified_time = modified_time.astimezone(timezone.utc)
|
||||
|
||||
time_as_seconds = modified_time.timestamp()
|
||||
if start is not None and time_as_seconds <= start:
|
||||
continue
|
||||
if end is not None and time_as_seconds > end:
|
||||
continue
|
||||
|
||||
try:
|
||||
downloaded_file = self._download_file(entry.path_display)
|
||||
except Exception:
|
||||
logger.exception(f"[Dropbox]: Error downloading file {entry.path_display}")
|
||||
continue
|
||||
|
||||
batch.append(
|
||||
Document(
|
||||
id=f"dropbox:{entry.id}",
|
||||
blob=downloaded_file,
|
||||
source=DocumentSource.DROPBOX,
|
||||
semantic_identifier=entry.name,
|
||||
extension=get_file_ext(entry.name),
|
||||
doc_updated_at=modified_time,
|
||||
size_bytes=entry.size if getattr(entry, "size", None) is not None else len(downloaded_file),
|
||||
)
|
||||
)
|
||||
|
||||
elif isinstance(entry, FolderMetadata):
|
||||
yield from self._yield_files_recursive(entry.path_lower, start, end)
|
||||
|
||||
if batch:
|
||||
yield batch
|
||||
|
||||
if not result.has_more:
|
||||
break
|
||||
|
||||
result = self.dropbox_client.files_list_folder_continue(result.cursor)
|
||||
|
||||
def poll_source(self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch) -> GenerateDocumentsOutput:
|
||||
"""Poll Dropbox for recent file changes"""
|
||||
# Simplified implementation - in production this would handle actual polling
|
||||
return []
|
||||
if self.dropbox_client is None:
|
||||
raise ConnectorMissingCredentialError("Dropbox")
|
||||
|
||||
def load_from_state(self) -> Any:
|
||||
for batch in self._yield_files_recursive("", start, end):
|
||||
yield batch
|
||||
|
||||
def load_from_state(self) -> GenerateDocumentsOutput:
|
||||
"""Load files from Dropbox state"""
|
||||
# Simplified implementation
|
||||
return []
|
||||
return self._yield_files_recursive("", None, None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import os
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
connector = DropboxConnector()
|
||||
connector.load_credentials({"dropbox_access_token": os.environ.get("DROPBOX_ACCESS_TOKEN")})
|
||||
connector.validate_connector_settings()
|
||||
document_batches = connector.load_from_state()
|
||||
try:
|
||||
first_batch = next(document_batches)
|
||||
print(f"Loaded {len(first_batch)} documents in first batch.")
|
||||
for doc in first_batch:
|
||||
print(f"- {doc.semantic_identifier} ({doc.size_bytes} bytes)")
|
||||
except StopIteration:
|
||||
print("No documents available in Dropbox.")
|
||||
|
||||
@ -312,12 +312,15 @@ def create_s3_client(bucket_type: BlobType, credentials: dict[str, Any], europea
|
||||
region_name=credentials["region"],
|
||||
)
|
||||
elif bucket_type == BlobType.S3_COMPATIBLE:
|
||||
addressing_style = credentials.get("addressing_style", "virtual")
|
||||
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=credentials["endpoint_url"],
|
||||
aws_access_key_id=credentials["aws_access_key_id"],
|
||||
aws_secret_access_key=credentials["aws_secret_access_key"],
|
||||
)
|
||||
config=Config(s3={'addressing_style': addressing_style}),
|
||||
)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unsupported bucket type: {bucket_type}")
|
||||
|
||||
@ -80,7 +80,7 @@ dependencies = [
|
||||
"pyclipper==1.3.0.post5",
|
||||
"pycryptodomex==3.20.0",
|
||||
"pymysql>=1.1.1,<2.0.0",
|
||||
"pypdf==6.0.0",
|
||||
"pypdf==6.4.0",
|
||||
"python-dotenv==1.0.1",
|
||||
"python-dateutil==2.8.2",
|
||||
"python-pptx>=1.0.2,<2.0.0",
|
||||
|
||||
@ -51,9 +51,11 @@ def chunk(
|
||||
attachment_res = []
|
||||
|
||||
if binary:
|
||||
msg = BytesParser(policy=policy.default).parse(io.BytesIO(binary))
|
||||
with io.BytesIO(binary) as buffer:
|
||||
msg = BytesParser(policy=policy.default).parse(buffer)
|
||||
else:
|
||||
msg = BytesParser(policy=policy.default).parse(open(filename, "rb"))
|
||||
with open(filename, "rb") as buffer:
|
||||
msg = BytesParser(policy=policy.default).parse(buffer)
|
||||
|
||||
text_txt, html_txt = [], []
|
||||
# get the email header info
|
||||
|
||||
@ -200,8 +200,7 @@ class GptV4(Base):
|
||||
res = self.client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=self.prompt(b64),
|
||||
extra_body=self.extra_body,
|
||||
unused=None,
|
||||
extra_body=self.extra_body
|
||||
)
|
||||
return res.choices[0].message.content.strip(), total_token_count_from_response(res)
|
||||
|
||||
@ -284,6 +283,8 @@ class QWenCV(GptV4):
|
||||
model=self.model_name,
|
||||
messages=messages,
|
||||
)
|
||||
if response.get("message"):
|
||||
raise Exception(response["message"])
|
||||
summary = response["output"]["choices"][0]["message"].content[0]["text"]
|
||||
return summary, num_tokens_from_string(summary)
|
||||
|
||||
|
||||
@ -35,7 +35,7 @@ You are a metadata filtering condition generator. Analyze the user's question an
|
||||
- Value has no match in metadata
|
||||
|
||||
5. **Example A**:
|
||||
- User query: "上市日期七月份的有哪些商品,不要蓝色的"
|
||||
- User query: "上市日期七月份的有哪些新品,不要蓝色的,只看鞋子和帽子"
|
||||
- Metadata: { "color": {...}, "listing_date": {...} }
|
||||
- Output:
|
||||
{
|
||||
@ -43,19 +43,21 @@ You are a metadata filtering condition generator. Analyze the user's question an
|
||||
"conditions": [
|
||||
{"key": "listing_date", "value": "2025-07-01", "op": "≥"},
|
||||
{"key": "listing_date", "value": "2025-08-01", "op": "<"},
|
||||
{"key": "color", "value": "blue", "op": "≠"}
|
||||
{"key": "color", "value": "blue", "op": "≠"},
|
||||
{"key": "category", "value": "shoes, hat", "op": "in"}
|
||||
]
|
||||
}
|
||||
|
||||
6. **Example B**:
|
||||
- User query: "Both blue and red are acceptable."
|
||||
- Metadata: { "color": {...}, "listing_date": {...} }
|
||||
- User query: "It must be from China or India. Otherwise, it must not be blue or red."
|
||||
- Metadata: { "color": {...}, "country": {...} }
|
||||
-
|
||||
- Output:
|
||||
{
|
||||
"logic": "or",
|
||||
"conditions": [
|
||||
{"key": "color", "value": "blue", "op": "="},
|
||||
{"key": "color", "value": "red", "op": "="}
|
||||
{"key": "color", "value": "blue, red", "op": "not in"},
|
||||
{"key": "country", "value": "china, india", "op": "in"},
|
||||
]
|
||||
}
|
||||
|
||||
@ -94,6 +96,8 @@ You are a metadata filtering condition generator. Analyze the user's question an
|
||||
"enum": [
|
||||
"contains",
|
||||
"not contains",
|
||||
"in",
|
||||
"not in",
|
||||
"start with",
|
||||
"end with",
|
||||
"empty",
|
||||
|
||||
@ -37,7 +37,7 @@ from api.db.services.connector_service import ConnectorService, SyncLogsService
|
||||
from api.db.services.knowledgebase_service import KnowledgebaseService
|
||||
from common import settings
|
||||
from common.config_utils import show_configs
|
||||
from common.data_source import BlobStorageConnector, NotionConnector, DiscordConnector, GoogleDriveConnector, MoodleConnector, JiraConnector
|
||||
from common.data_source import BlobStorageConnector, NotionConnector, DiscordConnector, GoogleDriveConnector, MoodleConnector, JiraConnector, DropboxConnector
|
||||
from common.constants import FileSource, TaskStatus
|
||||
from common.data_source.config import INDEX_BATCH_SIZE
|
||||
from common.data_source.confluence_connector import ConfluenceConnector
|
||||
@ -211,6 +211,27 @@ class Gmail(SyncBase):
|
||||
pass
|
||||
|
||||
|
||||
class Dropbox(SyncBase):
|
||||
SOURCE_NAME: str = FileSource.DROPBOX
|
||||
|
||||
async def _generate(self, task: dict):
|
||||
self.connector = DropboxConnector(batch_size=self.conf.get("batch_size", INDEX_BATCH_SIZE))
|
||||
self.connector.load_credentials(self.conf["credentials"])
|
||||
|
||||
if task["reindex"] == "1" or not task["poll_range_start"]:
|
||||
document_generator = self.connector.load_from_state()
|
||||
begin_info = "totally"
|
||||
else:
|
||||
poll_start = task["poll_range_start"]
|
||||
document_generator = self.connector.poll_source(
|
||||
poll_start.timestamp(), datetime.now(timezone.utc).timestamp()
|
||||
)
|
||||
begin_info = f"from {poll_start}"
|
||||
|
||||
logging.info(f"[Dropbox] Connect to Dropbox {begin_info}")
|
||||
return document_generator
|
||||
|
||||
|
||||
class GoogleDrive(SyncBase):
|
||||
SOURCE_NAME: str = FileSource.GOOGLE_DRIVE
|
||||
|
||||
@ -454,7 +475,8 @@ func_factory = {
|
||||
FileSource.SHAREPOINT: SharePoint,
|
||||
FileSource.SLACK: Slack,
|
||||
FileSource.TEAMS: Teams,
|
||||
FileSource.MOODLE: Moodle
|
||||
FileSource.MOODLE: Moodle,
|
||||
FileSource.DROPBOX: Dropbox,
|
||||
}
|
||||
|
||||
|
||||
|
||||
11
web/package-lock.json
generated
11
web/package-lock.json
generated
@ -86,6 +86,7 @@
|
||||
"react-infinite-scroll-component": "^6.1.0",
|
||||
"react-markdown": "^9.0.1",
|
||||
"react-pdf-highlighter": "^6.1.0",
|
||||
"react-resizable-panels": "^3.0.6",
|
||||
"react-string-replace": "^1.1.1",
|
||||
"react-syntax-highlighter": "^15.5.0",
|
||||
"react18-json-view": "^0.2.8",
|
||||
@ -30306,6 +30307,16 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/react-resizable-panels": {
|
||||
"version": "3.0.6",
|
||||
"resolved": "https://registry.npmmirror.com/react-resizable-panels/-/react-resizable-panels-3.0.6.tgz",
|
||||
"integrity": "sha512-b3qKHQ3MLqOgSS+FRYKapNkJZf5EQzuf6+RLiq1/IlTHw99YrZ2NJZLk4hQIzTnnIkRg2LUqyVinu6YWWpUYew==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"react": "^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc",
|
||||
"react-dom": "^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
|
||||
}
|
||||
},
|
||||
"node_modules/react-rnd": {
|
||||
"version": "10.4.1",
|
||||
"resolved": "https://registry.npmmirror.com/react-rnd/-/react-rnd-10.4.1.tgz",
|
||||
|
||||
@ -99,6 +99,7 @@
|
||||
"react-infinite-scroll-component": "^6.1.0",
|
||||
"react-markdown": "^9.0.1",
|
||||
"react-pdf-highlighter": "^6.1.0",
|
||||
"react-resizable-panels": "^3.0.6",
|
||||
"react-string-replace": "^1.1.1",
|
||||
"react-syntax-highlighter": "^15.5.0",
|
||||
"react18-json-view": "^0.2.8",
|
||||
|
||||
1
web/src/assets/svg/data-source/dropbox.svg
Normal file
1
web/src/assets/svg/data-source/dropbox.svg
Normal file
@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="89.9 347.3 32 32" width="64" height="64" fill="#007ee5"><path d="M99.337 348.42L89.9 354.5l6.533 5.263 9.467-5.837m-16 11l9.437 6.2 6.563-5.505-9.467-5.868m9.467 5.868l6.594 5.505 9.406-6.14-6.503-5.233m6.503-5.203l-9.406-6.14-6.594 5.505 9.497 5.837m-9.467 7.047l-6.594 5.474-2.843-1.845v2.087l9.437 5.656 9.437-5.656v-2.087l-2.843 1.845"/></svg>
|
||||
|
After Width: | Height: | Size: 396 B |
@ -28,6 +28,7 @@ import { useHandleFreedomChange } from './use-watch-change';
|
||||
interface LlmSettingFieldItemsProps {
|
||||
prefix?: string;
|
||||
options?: any[];
|
||||
llmId?: string;
|
||||
showFields?: Array<
|
||||
| 'temperature'
|
||||
| 'top_p'
|
||||
@ -73,6 +74,7 @@ export function LlmSettingFieldItems({
|
||||
'frequency_penalty',
|
||||
'max_tokens',
|
||||
],
|
||||
llmId,
|
||||
}: LlmSettingFieldItemsProps) {
|
||||
const form = useFormContext();
|
||||
const { t } = useTranslate('chat');
|
||||
@ -131,7 +133,7 @@ export function LlmSettingFieldItems({
|
||||
<div className="space-y-5">
|
||||
<LLMFormField
|
||||
options={options}
|
||||
name={getFieldWithPrefix('llm_id')}
|
||||
name={llmId ?? getFieldWithPrefix('llm_id')}
|
||||
></LLMFormField>
|
||||
<FormField
|
||||
control={form.control}
|
||||
|
||||
@ -219,7 +219,11 @@ export const SelectWithSearch = forwardRef<
|
||||
value={group.value}
|
||||
disabled={group.disabled}
|
||||
onSelect={handleSelect}
|
||||
className="min-h-10"
|
||||
className={
|
||||
value === group.value
|
||||
? 'bg-bg-card min-h-10'
|
||||
: 'min-h-10'
|
||||
}
|
||||
>
|
||||
<span className="leading-none">{group.label}</span>
|
||||
|
||||
|
||||
@ -159,7 +159,7 @@ export function SimilaritySliderFormField({
|
||||
<FormControl>
|
||||
<NumberInput
|
||||
className={cn(
|
||||
'h-6 w-10 p-0 text-center bg-bg-input border-border-default border text-text-secondary',
|
||||
'h-6 w-10 p-0 text-center bg-bg-input border-border-button border text-text-secondary',
|
||||
'[appearance:textfield] [&::-webkit-outer-spin-button]:appearance-none [&::-webkit-inner-spin-button]:appearance-none',
|
||||
numberInputClassName,
|
||||
)}
|
||||
|
||||
@ -82,7 +82,7 @@ export function SliderInputFormField({
|
||||
<FormControl>
|
||||
<NumberInput
|
||||
className={cn(
|
||||
'h-6 w-10 p-0 text-center bg-bg-input border border-border-default text-text-secondary',
|
||||
'h-6 w-10 p-0 text-center bg-bg-input border border-border-button text-text-secondary',
|
||||
'[appearance:textfield] [&::-webkit-outer-spin-button]:appearance-none [&::-webkit-inner-spin-button]:appearance-none',
|
||||
numberInputClassName,
|
||||
)}
|
||||
|
||||
@ -64,7 +64,7 @@ const Input = React.forwardRef<HTMLInputElement, InputProps>(
|
||||
className,
|
||||
)}
|
||||
style={{
|
||||
paddingLeft: !!prefix ? `${prefixWidth}px` : '',
|
||||
paddingLeft: !!prefix && prefixWidth ? `${prefixWidth}px` : '',
|
||||
paddingRight: isPasswordInput
|
||||
? '40px'
|
||||
: !!suffix
|
||||
@ -144,7 +144,9 @@ export interface ExpandedInputProps extends InputProps {}
|
||||
const ExpandedInput = Input;
|
||||
|
||||
const SearchInput = (props: InputProps) => {
|
||||
return <Input {...props} prefix={<Search className="ml-3 size-[1em]" />} />;
|
||||
return (
|
||||
<Input {...props} prefix={<Search className="ml-2 mr-1 size-[1em]" />} />
|
||||
);
|
||||
};
|
||||
|
||||
type Value = string | readonly string[] | number | undefined;
|
||||
|
||||
@ -200,7 +200,7 @@ const Modal: ModalType = ({
|
||||
<DialogPrimitive.Close asChild>
|
||||
<button
|
||||
type="button"
|
||||
className="flex h-7 w-7 items-center justify-center text-text-secondary rounded-full hover:bg-bg-card focus-visible:outline-none"
|
||||
className="flex h-7 w-7 items-center justify-center text-text-secondary rounded-full hover:text-text-primary focus-visible:outline-none"
|
||||
onClick={handleCancel}
|
||||
>
|
||||
{closeIcon}
|
||||
|
||||
54
web/src/components/ui/resizable.tsx
Normal file
54
web/src/components/ui/resizable.tsx
Normal file
@ -0,0 +1,54 @@
|
||||
import { GripVerticalIcon } from 'lucide-react';
|
||||
import * as React from 'react';
|
||||
import * as ResizablePrimitive from 'react-resizable-panels';
|
||||
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
function ResizablePanelGroup({
|
||||
className,
|
||||
...props
|
||||
}: React.ComponentProps<typeof ResizablePrimitive.PanelGroup>) {
|
||||
return (
|
||||
<ResizablePrimitive.PanelGroup
|
||||
data-slot="resizable-panel-group"
|
||||
className={cn(
|
||||
'flex h-full w-full data-[panel-group-direction=vertical]:flex-col',
|
||||
className,
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function ResizablePanel({
|
||||
...props
|
||||
}: React.ComponentProps<typeof ResizablePrimitive.Panel>) {
|
||||
return <ResizablePrimitive.Panel data-slot="resizable-panel" {...props} />;
|
||||
}
|
||||
|
||||
function ResizableHandle({
|
||||
withHandle,
|
||||
className,
|
||||
...props
|
||||
}: React.ComponentProps<typeof ResizablePrimitive.PanelResizeHandle> & {
|
||||
withHandle?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<ResizablePrimitive.PanelResizeHandle
|
||||
data-slot="resizable-handle"
|
||||
className={cn(
|
||||
'bg-border focus-visible:ring-ring relative flex w-px items-center justify-center after:absolute after:inset-y-0 after:left-1/2 after:w-1 after:-translate-x-1/2 focus-visible:ring-1 focus-visible:ring-offset-1 focus-visible:outline-hidden data-[panel-group-direction=vertical]:h-px data-[panel-group-direction=vertical]:w-full data-[panel-group-direction=vertical]:after:left-0 data-[panel-group-direction=vertical]:after:h-1 data-[panel-group-direction=vertical]:after:w-full data-[panel-group-direction=vertical]:after:translate-x-0 data-[panel-group-direction=vertical]:after:-translate-y-1/2 [&[data-panel-group-direction=vertical]>div]:rotate-90',
|
||||
className,
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
{withHandle && (
|
||||
<div className="bg-border z-10 flex h-4 w-3 items-center justify-center rounded-xs border">
|
||||
<GripVerticalIcon className="size-2.5" />
|
||||
</div>
|
||||
)}
|
||||
</ResizablePrimitive.PanelResizeHandle>
|
||||
);
|
||||
}
|
||||
|
||||
export { ResizableHandle, ResizablePanel, ResizablePanelGroup };
|
||||
@ -133,6 +133,8 @@ export enum ComparisonOperator {
|
||||
EndWith = 'end with',
|
||||
Empty = 'empty',
|
||||
NotEmpty = 'not empty',
|
||||
In = 'in',
|
||||
NotIn = 'not in',
|
||||
}
|
||||
|
||||
export const SwitchOperatorOptions = [
|
||||
@ -168,6 +170,16 @@ export const SwitchOperatorOptions = [
|
||||
label: 'notEmpty',
|
||||
icon: <CircleSlash2 className="size-4" />,
|
||||
},
|
||||
{
|
||||
value: ComparisonOperator.In,
|
||||
label: 'in',
|
||||
icon: <CircleSlash2 className="size-4" />,
|
||||
},
|
||||
{
|
||||
value: ComparisonOperator.NotIn,
|
||||
label: 'notIn',
|
||||
icon: <CircleSlash2 className="size-4" />,
|
||||
},
|
||||
];
|
||||
|
||||
export const AgentStructuredOutputField = 'structured';
|
||||
|
||||
@ -715,6 +715,8 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s
|
||||
Example: general/v2/`,
|
||||
S3CompatibleEndpointUrlTip: `Required for S3 compatible Storage Box. Specify the S3-compatible endpoint URL.
|
||||
Example: https://fsn1.your-objectstorage.com`,
|
||||
S3CompatibleAddressingStyleTip: `Required for S3 compatible Storage Box. Specify the S3-compatible addressing style.
|
||||
Example: Virtual Hosted Style`,
|
||||
addDataSourceModalTital: 'Create your {{name}} connector',
|
||||
deleteSourceModalTitle: 'Delete data source',
|
||||
deleteSourceModalContent: `
|
||||
@ -742,6 +744,10 @@ Example: https://fsn1.your-objectstorage.com`,
|
||||
'Comma-separated emails whose "My Drive" contents should be indexed (include the primary admin).',
|
||||
google_driveSharedFoldersTip:
|
||||
'Comma-separated Google Drive folder links to crawl.',
|
||||
dropboxDescription:
|
||||
'Connect your Dropbox to sync files and folders from a chosen account.',
|
||||
dropboxAccessTokenTip:
|
||||
'Generate a long-lived access token in the Dropbox App Console with files.metadata.read, files.content.read, and sharing.read scopes.',
|
||||
moodleDescription:
|
||||
'Connect to your Moodle LMS to sync course content, forums, and resources.',
|
||||
moodleUrlTip:
|
||||
@ -1395,6 +1401,8 @@ Example: https://fsn1.your-objectstorage.com`,
|
||||
endWith: 'Ends with',
|
||||
empty: 'Is empty',
|
||||
notEmpty: 'Not empty',
|
||||
in: 'In',
|
||||
notIn: 'Not in',
|
||||
},
|
||||
switchLogicOperatorOptions: {
|
||||
and: 'AND',
|
||||
|
||||
@ -715,6 +715,8 @@ export default {
|
||||
Пример: general/v2/`,
|
||||
S3CompatibleEndpointUrlTip: `Требуется для S3 совместимого Storage Box. Укажите URL конечной точки, совместимой с S3.
|
||||
Пример: https://fsn1.your-objectstorage.com`,
|
||||
S3CompatibleAddressingStyleTip: `Требуется для S3 совместимого Storage Box. Укажите стиль адресации, совместимый с S3.
|
||||
Пример: Virtual Hosted Style`,
|
||||
addDataSourceModalTital: 'Создайте ваш коннектор {{name}}',
|
||||
deleteSourceModalTitle: 'Удалить источник данных',
|
||||
deleteSourceModalContent: `
|
||||
|
||||
@ -722,6 +722,9 @@ General:实体和关系提取提示来自 GitHub - microsoft/graphrag:基于
|
||||
'需要索引其 “我的云端硬盘” 的邮箱,多个邮箱用逗号分隔(建议包含管理员)。',
|
||||
google_driveSharedFoldersTip:
|
||||
'需要同步的 Google Drive 文件夹链接,多个链接用逗号分隔。',
|
||||
dropboxDescription: '连接 Dropbox,同步指定账号下的文件与文件夹。',
|
||||
dropboxAccessTokenTip:
|
||||
'请在 Dropbox App Console 生成 Access Token,并勾选 files.metadata.read、files.content.read、sharing.read 等必要权限。',
|
||||
jiraDescription: '接入 Jira 工作区,持续同步Issues、评论与附件。',
|
||||
jiraBaseUrlTip:
|
||||
'Jira 的 Base URL,例如:https://your-domain.atlassian.net。',
|
||||
|
||||
@ -3,7 +3,10 @@ import { LlmSettingFieldItems } from '@/components/llm-setting-items/next';
|
||||
export function ChatModelSettings() {
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
<LlmSettingFieldItems prefix="llm_setting"></LlmSettingFieldItems>
|
||||
<LlmSettingFieldItems
|
||||
prefix="llm_setting"
|
||||
llmId="llm_id"
|
||||
></LlmSettingFieldItems>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@ -12,6 +12,7 @@ export enum DataSourceKey {
|
||||
MOODLE = 'moodle',
|
||||
// GMAIL = 'gmail',
|
||||
JIRA = 'jira',
|
||||
DROPBOX = 'dropbox',
|
||||
// SHAREPOINT = 'sharepoint',
|
||||
// SLACK = 'slack',
|
||||
// TEAMS = 'teams',
|
||||
@ -53,6 +54,11 @@ export const DataSourceInfo = {
|
||||
description: t(`setting.${DataSourceKey.JIRA}Description`),
|
||||
icon: <SvgIcon name={'data-source/jira'} width={38} />,
|
||||
},
|
||||
[DataSourceKey.DROPBOX]: {
|
||||
name: 'Dropbox',
|
||||
description: t(`setting.${DataSourceKey.DROPBOX}Description`),
|
||||
icon: <SvgIcon name={'data-source/dropbox'} width={38} />,
|
||||
},
|
||||
};
|
||||
|
||||
export const DataSourceFormBaseFields = [
|
||||
@ -115,6 +121,21 @@ export const DataSourceFormFields = {
|
||||
],
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
label: 'Addressing Style',
|
||||
name: 'config.credentials.addressing_style',
|
||||
type: FormFieldType.Select,
|
||||
options: [
|
||||
{ label: 'Virtual Hosted Style', value: 'virtual' },
|
||||
{ label: 'Path Style', value: 'path' },
|
||||
],
|
||||
required: false,
|
||||
placeholder: 'Virtual Hosted Style',
|
||||
tooltip: t('setting.S3CompatibleAddressingStyleTip'),
|
||||
shouldRender: (formValues: any) => {
|
||||
return formValues?.config?.bucket_type === 's3_compatible';
|
||||
},
|
||||
},
|
||||
{
|
||||
label: 'Endpoint URL',
|
||||
name: 'config.credentials.endpoint_url',
|
||||
@ -408,6 +429,22 @@ export const DataSourceFormFields = {
|
||||
tooltip: t('setting.jiraPasswordTip'),
|
||||
},
|
||||
],
|
||||
[DataSourceKey.DROPBOX]: [
|
||||
{
|
||||
label: 'Access Token',
|
||||
name: 'config.credentials.dropbox_access_token',
|
||||
type: FormFieldType.Password,
|
||||
required: true,
|
||||
tooltip: t('setting.dropboxAccessTokenTip'),
|
||||
},
|
||||
{
|
||||
label: 'Batch Size',
|
||||
name: 'config.batch_size',
|
||||
type: FormFieldType.Number,
|
||||
required: false,
|
||||
placeholder: 'Defaults to 2',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
export const DataSourceFormDefaultValues = {
|
||||
@ -422,6 +459,7 @@ export const DataSourceFormDefaultValues = {
|
||||
aws_access_key_id: '',
|
||||
aws_secret_access_key: '',
|
||||
endpoint_url: '',
|
||||
addressing_style: 'virtual',
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -508,4 +546,14 @@ export const DataSourceFormDefaultValues = {
|
||||
},
|
||||
},
|
||||
},
|
||||
[DataSourceKey.DROPBOX]: {
|
||||
name: '',
|
||||
source: DataSourceKey.DROPBOX,
|
||||
config: {
|
||||
batch_size: 2,
|
||||
credentials: {
|
||||
dropbox_access_token: '',
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@ -13,50 +13,16 @@ import { AddedSourceCard } from './component/added-source-card';
|
||||
import { DataSourceInfo, DataSourceKey } from './contant';
|
||||
import { useAddDataSource, useListDataSource } from './hooks';
|
||||
import { IDataSorceInfo } from './interface';
|
||||
const dataSourceTemplates = [
|
||||
{
|
||||
id: DataSourceKey.CONFLUENCE,
|
||||
name: DataSourceInfo[DataSourceKey.CONFLUENCE].name,
|
||||
description: DataSourceInfo[DataSourceKey.CONFLUENCE].description,
|
||||
icon: DataSourceInfo[DataSourceKey.CONFLUENCE].icon,
|
||||
},
|
||||
{
|
||||
id: DataSourceKey.S3,
|
||||
name: DataSourceInfo[DataSourceKey.S3].name,
|
||||
description: DataSourceInfo[DataSourceKey.S3].description,
|
||||
icon: DataSourceInfo[DataSourceKey.S3].icon,
|
||||
},
|
||||
{
|
||||
id: DataSourceKey.GOOGLE_DRIVE,
|
||||
name: DataSourceInfo[DataSourceKey.GOOGLE_DRIVE].name,
|
||||
description: DataSourceInfo[DataSourceKey.GOOGLE_DRIVE].description,
|
||||
icon: DataSourceInfo[DataSourceKey.GOOGLE_DRIVE].icon,
|
||||
},
|
||||
{
|
||||
id: DataSourceKey.DISCORD,
|
||||
name: DataSourceInfo[DataSourceKey.DISCORD].name,
|
||||
description: DataSourceInfo[DataSourceKey.DISCORD].description,
|
||||
icon: DataSourceInfo[DataSourceKey.DISCORD].icon,
|
||||
},
|
||||
{
|
||||
id: DataSourceKey.NOTION,
|
||||
name: DataSourceInfo[DataSourceKey.NOTION].name,
|
||||
description: DataSourceInfo[DataSourceKey.NOTION].description,
|
||||
icon: DataSourceInfo[DataSourceKey.NOTION].icon,
|
||||
},
|
||||
{
|
||||
id: DataSourceKey.MOODLE,
|
||||
name: DataSourceInfo[DataSourceKey.MOODLE].name,
|
||||
description: DataSourceInfo[DataSourceKey.MOODLE].description,
|
||||
icon: DataSourceInfo[DataSourceKey.MOODLE].icon,
|
||||
},
|
||||
{
|
||||
id: DataSourceKey.JIRA,
|
||||
name: DataSourceInfo[DataSourceKey.JIRA].name,
|
||||
description: DataSourceInfo[DataSourceKey.JIRA].description,
|
||||
icon: DataSourceInfo[DataSourceKey.JIRA].icon,
|
||||
},
|
||||
];
|
||||
|
||||
const dataSourceTemplates = Object.values(DataSourceKey).map((id) => {
|
||||
return {
|
||||
id,
|
||||
name: DataSourceInfo[id].name,
|
||||
description: DataSourceInfo[id].description,
|
||||
icon: DataSourceInfo[id].icon,
|
||||
};
|
||||
});
|
||||
|
||||
const DataSource = () => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
|
||||
@ -1,14 +1,8 @@
|
||||
import { Collapse } from '@/components/collapse';
|
||||
import { Button, ButtonLoading } from '@/components/ui/button';
|
||||
import { Card, CardContent } from '@/components/ui/card';
|
||||
import {
|
||||
Dialog,
|
||||
DialogClose,
|
||||
DialogContent,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { DialogClose, DialogFooter } from '@/components/ui/dialog';
|
||||
import { Modal } from '@/components/ui/modal/modal';
|
||||
import { useGetMcpServer, useTestMcpServer } from '@/hooks/use-mcp-request';
|
||||
import { IModalProps } from '@/interfaces/common';
|
||||
import { IMCPTool, IMCPToolObject } from '@/interfaces/database/mcp';
|
||||
@ -114,50 +108,73 @@ export function EditMcpDialog({
|
||||
const disabled = !!!tools?.length || testLoading || fieldChanged;
|
||||
|
||||
return (
|
||||
<Dialog open onOpenChange={hideModal}>
|
||||
<DialogContent>
|
||||
<DialogHeader>
|
||||
<DialogTitle>{id ? t('mcp.editMCP') : t('mcp.addMCP')}</DialogTitle>
|
||||
</DialogHeader>
|
||||
<EditMcpForm
|
||||
onOk={handleOk}
|
||||
form={form}
|
||||
setFieldChanged={setFieldChanged}
|
||||
></EditMcpForm>
|
||||
<Card className="bg-transparent">
|
||||
<CardContent className="p-3">
|
||||
<Collapse
|
||||
title={
|
||||
<div>
|
||||
{nextTools?.length || 0} {t('mcp.toolsAvailable')}
|
||||
</div>
|
||||
}
|
||||
open={collapseOpen}
|
||||
onOpenChange={setCollapseOpen}
|
||||
rightContent={
|
||||
<Button
|
||||
variant={'transparent'}
|
||||
form={FormId}
|
||||
type="submit"
|
||||
onClick={handleTest}
|
||||
className="border-none p-0 hover:bg-transparent"
|
||||
>
|
||||
<RefreshCw
|
||||
className={cn('text-text-secondary', {
|
||||
'animate-spin': testLoading,
|
||||
})}
|
||||
/>
|
||||
</Button>
|
||||
}
|
||||
>
|
||||
<div className="overflow-auto max-h-80 divide-y-[0.5px] divide-border-button bg-bg-card rounded-md px-2.5 scrollbar-auto">
|
||||
{nextTools?.map((x) => (
|
||||
<McpToolCard key={x.name} data={x}></McpToolCard>
|
||||
))}
|
||||
</div>
|
||||
</Collapse>
|
||||
</CardContent>
|
||||
</Card>
|
||||
// <Dialog open onOpenChange={hideModal}>
|
||||
// <DialogContent>
|
||||
// <DialogHeader>
|
||||
// <DialogTitle>{id ? t('mcp.editMCP') : t('mcp.addMCP')}</DialogTitle>
|
||||
// </DialogHeader>
|
||||
// <EditMcpForm
|
||||
// onOk={handleOk}
|
||||
// form={form}
|
||||
// setFieldChanged={setFieldChanged}
|
||||
// ></EditMcpForm>
|
||||
// <Card className="bg-transparent">
|
||||
// <CardContent className="p-3">
|
||||
// <Collapse
|
||||
// title={
|
||||
// <div>
|
||||
// {nextTools?.length || 0} {t('mcp.toolsAvailable')}
|
||||
// </div>
|
||||
// }
|
||||
// open={collapseOpen}
|
||||
// onOpenChange={setCollapseOpen}
|
||||
// rightContent={
|
||||
// <Button
|
||||
// variant={'transparent'}
|
||||
// form={FormId}
|
||||
// type="submit"
|
||||
// onClick={handleTest}
|
||||
// className="border-none p-0 hover:bg-transparent"
|
||||
// >
|
||||
// <RefreshCw
|
||||
// className={cn('text-text-secondary', {
|
||||
// 'animate-spin': testLoading,
|
||||
// })}
|
||||
// />
|
||||
// </Button>
|
||||
// }
|
||||
// >
|
||||
// <div className="overflow-auto max-h-80 divide-y-[0.5px] divide-border-button bg-bg-card rounded-md px-2.5 scrollbar-auto">
|
||||
// {nextTools?.map((x) => (
|
||||
// <McpToolCard key={x.name} data={x}></McpToolCard>
|
||||
// ))}
|
||||
// </div>
|
||||
// </Collapse>
|
||||
// </CardContent>
|
||||
// </Card>
|
||||
// <DialogFooter>
|
||||
// <DialogClose asChild>
|
||||
// <Button variant="outline">{t('common.cancel')}</Button>
|
||||
// </DialogClose>
|
||||
// <ButtonLoading
|
||||
// type="submit"
|
||||
// form={FormId}
|
||||
// loading={loading}
|
||||
// onClick={handleSave}
|
||||
// disabled={disabled}
|
||||
// >
|
||||
// {t('common.save')}
|
||||
// </ButtonLoading>
|
||||
// </DialogFooter>
|
||||
// </DialogContent>
|
||||
// </Dialog>
|
||||
<Modal
|
||||
title={id ? t('mcp.editMCP') : t('mcp.addMCP')}
|
||||
open={true}
|
||||
onOpenChange={hideModal}
|
||||
cancelText={t('common.cancel')}
|
||||
okText={t('common.save')}
|
||||
footer={
|
||||
<DialogFooter>
|
||||
<DialogClose asChild>
|
||||
<Button variant="outline">{t('common.cancel')}</Button>
|
||||
@ -172,7 +189,47 @@ export function EditMcpDialog({
|
||||
{t('common.save')}
|
||||
</ButtonLoading>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
}
|
||||
>
|
||||
<EditMcpForm
|
||||
onOk={handleOk}
|
||||
form={form}
|
||||
setFieldChanged={setFieldChanged}
|
||||
></EditMcpForm>
|
||||
<Card className="bg-transparent">
|
||||
<CardContent className="p-3">
|
||||
<Collapse
|
||||
title={
|
||||
<div>
|
||||
{nextTools?.length || 0} {t('mcp.toolsAvailable')}
|
||||
</div>
|
||||
}
|
||||
open={collapseOpen}
|
||||
onOpenChange={setCollapseOpen}
|
||||
rightContent={
|
||||
<Button
|
||||
variant={'transparent'}
|
||||
form={FormId}
|
||||
type="submit"
|
||||
onClick={handleTest}
|
||||
className="border-none p-0 text-text-secondary hover:bg-transparent hover:text-text-primary"
|
||||
>
|
||||
<RefreshCw
|
||||
className={cn({
|
||||
'animate-spin': testLoading,
|
||||
})}
|
||||
/>
|
||||
</Button>
|
||||
}
|
||||
>
|
||||
<div className="overflow-auto max-h-80 divide-y-[0.5px] divide-border-button bg-bg-card rounded-md px-2.5 scrollbar-auto">
|
||||
{nextTools?.map((x) => (
|
||||
<McpToolCard key={x.name} data={x}></McpToolCard>
|
||||
))}
|
||||
</div>
|
||||
</Collapse>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
@ -7,7 +7,7 @@ export type McpToolCardProps = {
|
||||
export function McpToolCard({ data }: McpToolCardProps) {
|
||||
return (
|
||||
<section className="group py-2.5">
|
||||
<h3 className="text-sm font-semibold line-clamp-1 pb-2">{data.name}</h3>
|
||||
<div className="text-sm font-normal line-clamp-1 pb-2">{data.name}</div>
|
||||
<div className="text-xs font-normal text-text-secondary">
|
||||
{data.description}
|
||||
</div>
|
||||
|
||||
@ -139,7 +139,7 @@ const SystemSetting = ({ onOk, loading }: IProps) => {
|
||||
}) => {
|
||||
return (
|
||||
<div className="flex gap-3">
|
||||
<label className="block text-sm font-medium text-text-secondary mb-1 w-1/4">
|
||||
<label className="block text-sm font-normal text-text-secondary mb-1 w-1/4">
|
||||
{isRequired && <span className="text-red-500">*</span>}
|
||||
{label}
|
||||
{tooltip && (
|
||||
|
||||
Reference in New Issue
Block a user