From ffee598cbdccc388181742485136d253ab35f0f0 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 17:23:55 +0000 Subject: [PATCH] [autofix.ci] apply automated fixes --- src/lfx/src/lfx/_assets/component_index.json | 158 +++++++++---------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/src/lfx/src/lfx/_assets/component_index.json b/src/lfx/src/lfx/_assets/component_index.json index bce86b23c4..06c52a8406 100644 --- a/src/lfx/src/lfx/_assets/component_index.json +++ b/src/lfx/src/lfx/_assets/component_index.json @@ -9883,7 +9883,7 @@ "icon": "Chroma", "legacy": false, "metadata": { - "code_hash": "ead0979e0576", + "code_hash": "59a957b32729", "dependencies": { "dependencies": [ { @@ -10084,7 +10084,7 @@ "show": true, "title_case": false, "type": "code", - "value": "from copy import deepcopy\nfrom typing import TYPE_CHECKING\n\nfrom langchain_chroma import Chroma\nfrom typing_extensions import override\n\nfrom lfx.base.vectorstores.chroma_security import chroma_langchain_collection_kwargs\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.utils import chroma_collection_to_data\nfrom lfx.inputs.inputs import BoolInput, DropdownInput, HandleInput, IntInput, StrInput\nfrom lfx.schema.data import Data\n\nif TYPE_CHECKING:\n from lfx.schema.dataframe import DataFrame\n\n\nclass ChromaVectorStoreComponent(LCVectorStoreComponent):\n \"\"\"Chroma Vector Store with search capabilities.\"\"\"\n\n display_name: str = \"Chroma DB\"\n description: str = \"Chroma Vector Store with search capabilities\"\n name = \"Chroma\"\n icon = \"Chroma\"\n\n inputs = [\n StrInput(\n name=\"collection_name\",\n display_name=\"Collection Name\",\n value=\"langflow\",\n ),\n StrInput(\n name=\"persist_directory\",\n display_name=\"Persist Directory\",\n ),\n *LCVectorStoreComponent.inputs,\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"]),\n StrInput(\n name=\"chroma_server_cors_allow_origins\",\n display_name=\"Server CORS Allow Origins\",\n advanced=True,\n ),\n StrInput(\n name=\"chroma_server_host\",\n display_name=\"Server Host\",\n advanced=True,\n ),\n IntInput(\n name=\"chroma_server_http_port\",\n display_name=\"Server HTTP Port\",\n advanced=True,\n ),\n IntInput(\n name=\"chroma_server_grpc_port\",\n display_name=\"Server gRPC Port\",\n advanced=True,\n ),\n BoolInput(\n name=\"chroma_server_ssl_enabled\",\n display_name=\"Server SSL Enabled\",\n advanced=True,\n ),\n BoolInput(\n name=\"allow_duplicates\",\n display_name=\"Allow Duplicates\",\n advanced=True,\n info=\"If false, will not add documents that are already in the Vector Store.\",\n ),\n DropdownInput(\n name=\"search_type\",\n display_name=\"Search Type\",\n options=[\"Similarity\", \"MMR\"],\n value=\"Similarity\",\n advanced=True,\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Number of Results\",\n info=\"Number of results to return.\",\n advanced=True,\n value=10,\n ),\n IntInput(\n name=\"limit\",\n display_name=\"Limit\",\n advanced=True,\n info=\"Limit the number of records to compare when Allow Duplicates is False.\",\n ),\n ]\n\n @override\n @check_cached_vector_store\n def build_vector_store(self) -> Chroma:\n \"\"\"Builds the Chroma object.\"\"\"\n try:\n from langchain_chroma import Chroma\n except ImportError as e:\n msg = \"Could not import Chroma integration package. Please install it with `pip install langchain-chroma`.\"\n raise ImportError(msg) from e\n client = None\n if self.chroma_server_host:\n try:\n from chromadb import HttpClient\n except ImportError as e:\n msg = \"Could not import chromadb. Please install it with `pip install chromadb`.\"\n raise ImportError(msg) from e\n client = HttpClient(\n host=self.chroma_server_host,\n port=self.chroma_server_http_port or 8000,\n ssl=bool(self.chroma_server_ssl_enabled),\n )\n\n # Check persist_directory and expand it if it is a relative path\n persist_directory = self.resolve_path(self.persist_directory) if self.persist_directory is not None else None\n\n from chromadb.errors import ChromaError\n\n try:\n chroma = Chroma(\n persist_directory=persist_directory,\n client=client,\n embedding_function=self.embedding,\n collection_name=self.collection_name,\n **chroma_langchain_collection_kwargs(),\n )\n except Exception as e:\n if isinstance(e, ChromaError):\n if not persist_directory and client is None:\n msg = \"Chroma DB failed to initialize. Please set a 'Persist Directory' path (e.g., './chroma_db').\"\n elif not persist_directory:\n msg = f\"Chroma DB failed to initialize in server mode: {e}.\"\n else:\n msg = (\n f\"Chroma DB failed at '{persist_directory}': {e}. \"\n \"If you deleted the database, restart the server and re-run ingestion.\"\n )\n raise RuntimeError(msg) from e # noqa: TRY004\n raise\n\n self._add_documents_to_vector_store(chroma)\n limit = int(self.limit) if self.limit is not None and str(self.limit).strip() else None\n self.status = chroma_collection_to_data(chroma.get(limit=limit))\n return chroma\n\n def _add_documents_to_vector_store(self, vector_store: \"Chroma\") -> None:\n \"\"\"Adds documents to the Vector Store.\"\"\"\n ingest_data: list | Data | DataFrame = self.ingest_data\n if not ingest_data:\n self.status = \"\"\n return\n\n # Convert DataFrame to Data if needed using parent's method\n ingest_data = self._prepare_ingest_data()\n\n stored_documents_without_id = []\n if self.allow_duplicates:\n stored_data = []\n else:\n limit = int(self.limit) if self.limit is not None and str(self.limit).strip() else None\n stored_data = chroma_collection_to_data(vector_store.get(limit=limit))\n for value in deepcopy(stored_data):\n del value.id\n stored_documents_without_id.append(value)\n\n documents = []\n for _input in ingest_data or []:\n if isinstance(_input, Data):\n if _input not in stored_documents_without_id:\n documents.append(_input.to_lc_document())\n else:\n msg = \"Vector Store Inputs must be Data objects.\"\n raise TypeError(msg)\n\n if documents and self.embedding is not None:\n self.log(f\"Adding {len(documents)} documents to the Vector Store.\")\n # Filter complex metadata to prevent ChromaDB errors\n try:\n from langchain_community.vectorstores.utils import filter_complex_metadata\n\n filtered_documents = filter_complex_metadata(documents)\n vector_store.add_documents(filtered_documents)\n except ImportError:\n self.log(\"Warning: Could not import filter_complex_metadata. Adding documents without filtering.\")\n vector_store.add_documents(documents)\n else:\n self.log(\"No documents to add to the Vector Store.\")\n" + "value": "from copy import deepcopy\nfrom typing import TYPE_CHECKING\n\nfrom langchain_chroma import Chroma\nfrom typing_extensions import override\n\nfrom lfx.base.vectorstores.chroma_security import chroma_langchain_collection_kwargs\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.utils import chroma_collection_to_data\nfrom lfx.inputs.inputs import BoolInput, DropdownInput, HandleInput, IntInput, StrInput\nfrom lfx.schema.data import Data\nfrom lfx.utils.ssrf_protection import validate_connector_url_for_ssrf\n\nif TYPE_CHECKING:\n from lfx.schema.dataframe import DataFrame\n\n\nclass ChromaVectorStoreComponent(LCVectorStoreComponent):\n \"\"\"Chroma Vector Store with search capabilities.\"\"\"\n\n display_name: str = \"Chroma DB\"\n description: str = \"Chroma Vector Store with search capabilities\"\n name = \"Chroma\"\n icon = \"Chroma\"\n\n inputs = [\n StrInput(\n name=\"collection_name\",\n display_name=\"Collection Name\",\n value=\"langflow\",\n ),\n StrInput(\n name=\"persist_directory\",\n display_name=\"Persist Directory\",\n ),\n *LCVectorStoreComponent.inputs,\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"]),\n StrInput(\n name=\"chroma_server_cors_allow_origins\",\n display_name=\"Server CORS Allow Origins\",\n advanced=True,\n ),\n StrInput(\n name=\"chroma_server_host\",\n display_name=\"Server Host\",\n advanced=True,\n ),\n IntInput(\n name=\"chroma_server_http_port\",\n display_name=\"Server HTTP Port\",\n advanced=True,\n ),\n IntInput(\n name=\"chroma_server_grpc_port\",\n display_name=\"Server gRPC Port\",\n advanced=True,\n ),\n BoolInput(\n name=\"chroma_server_ssl_enabled\",\n display_name=\"Server SSL Enabled\",\n advanced=True,\n ),\n BoolInput(\n name=\"allow_duplicates\",\n display_name=\"Allow Duplicates\",\n advanced=True,\n info=\"If false, will not add documents that are already in the Vector Store.\",\n ),\n DropdownInput(\n name=\"search_type\",\n display_name=\"Search Type\",\n options=[\"Similarity\", \"MMR\"],\n value=\"Similarity\",\n advanced=True,\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Number of Results\",\n info=\"Number of results to return.\",\n advanced=True,\n value=10,\n ),\n IntInput(\n name=\"limit\",\n display_name=\"Limit\",\n advanced=True,\n info=\"Limit the number of records to compare when Allow Duplicates is False.\",\n ),\n ]\n\n @override\n @check_cached_vector_store\n def build_vector_store(self) -> Chroma:\n \"\"\"Builds the Chroma object.\"\"\"\n try:\n from langchain_chroma import Chroma\n except ImportError as e:\n msg = \"Could not import Chroma integration package. Please install it with `pip install langchain-chroma`.\"\n raise ImportError(msg) from e\n client = None\n if self.chroma_server_host:\n try:\n from chromadb import HttpClient\n except ImportError as e:\n msg = \"Could not import chromadb. Please install it with `pip install chromadb`.\"\n raise ImportError(msg) from e\n # chroma_server_host is tenant-controlled: block SSRF to internal/metadata hosts.\n scheme = \"https\" if self.chroma_server_ssl_enabled else \"http\"\n validate_connector_url_for_ssrf(\n f\"{scheme}://{self.chroma_server_host}:{self.chroma_server_http_port or 8000}\"\n )\n client = HttpClient(\n host=self.chroma_server_host,\n port=self.chroma_server_http_port or 8000,\n ssl=bool(self.chroma_server_ssl_enabled),\n )\n\n # Check persist_directory and expand it if it is a relative path\n persist_directory = self.resolve_path(self.persist_directory) if self.persist_directory is not None else None\n\n from chromadb.errors import ChromaError\n\n try:\n chroma = Chroma(\n persist_directory=persist_directory,\n client=client,\n embedding_function=self.embedding,\n collection_name=self.collection_name,\n **chroma_langchain_collection_kwargs(),\n )\n except Exception as e:\n if isinstance(e, ChromaError):\n if not persist_directory and client is None:\n msg = \"Chroma DB failed to initialize. Please set a 'Persist Directory' path (e.g., './chroma_db').\"\n elif not persist_directory:\n msg = f\"Chroma DB failed to initialize in server mode: {e}.\"\n else:\n msg = (\n f\"Chroma DB failed at '{persist_directory}': {e}. \"\n \"If you deleted the database, restart the server and re-run ingestion.\"\n )\n raise RuntimeError(msg) from e # noqa: TRY004\n raise\n\n self._add_documents_to_vector_store(chroma)\n limit = int(self.limit) if self.limit is not None and str(self.limit).strip() else None\n self.status = chroma_collection_to_data(chroma.get(limit=limit))\n return chroma\n\n def _add_documents_to_vector_store(self, vector_store: \"Chroma\") -> None:\n \"\"\"Adds documents to the Vector Store.\"\"\"\n ingest_data: list | Data | DataFrame = self.ingest_data\n if not ingest_data:\n self.status = \"\"\n return\n\n # Convert DataFrame to Data if needed using parent's method\n ingest_data = self._prepare_ingest_data()\n\n stored_documents_without_id = []\n if self.allow_duplicates:\n stored_data = []\n else:\n limit = int(self.limit) if self.limit is not None and str(self.limit).strip() else None\n stored_data = chroma_collection_to_data(vector_store.get(limit=limit))\n for value in deepcopy(stored_data):\n del value.id\n stored_documents_without_id.append(value)\n\n documents = []\n for _input in ingest_data or []:\n if isinstance(_input, Data):\n if _input not in stored_documents_without_id:\n documents.append(_input.to_lc_document())\n else:\n msg = \"Vector Store Inputs must be Data objects.\"\n raise TypeError(msg)\n\n if documents and self.embedding is not None:\n self.log(f\"Adding {len(documents)} documents to the Vector Store.\")\n # Filter complex metadata to prevent ChromaDB errors\n try:\n from langchain_community.vectorstores.utils import filter_complex_metadata\n\n filtered_documents = filter_complex_metadata(documents)\n vector_store.add_documents(filtered_documents)\n except ImportError:\n self.log(\"Warning: Could not import filter_complex_metadata. Adding documents without filtering.\")\n vector_store.add_documents(documents)\n else:\n self.log(\"No documents to add to the Vector Store.\")\n" }, "collection_name": { "_input_type": "StrInput", @@ -11247,7 +11247,7 @@ "icon": "Clickhouse", "legacy": false, "metadata": { - "code_hash": "ab991e83da44", + "code_hash": "f7f52312f55a", "dependencies": { "dependencies": [ { @@ -11318,7 +11318,7 @@ "show": true, "title_case": false, "type": "code", - "value": "from langchain_community.vectorstores import Clickhouse, ClickhouseSettings\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.helpers.data import docs_to_data\nfrom lfx.inputs.inputs import BoolInput, FloatInput\nfrom lfx.io import (\n DictInput,\n DropdownInput,\n HandleInput,\n IntInput,\n SecretStrInput,\n StrInput,\n)\nfrom lfx.schema.data import Data\n\n\nclass ClickhouseVectorStoreComponent(LCVectorStoreComponent):\n display_name = \"ClickHouse\"\n description = \"ClickHouse Vector Store with search capabilities\"\n name = \"Clickhouse\"\n icon = \"Clickhouse\"\n\n inputs = [\n StrInput(name=\"host\", display_name=\"hostname\", required=True, value=\"localhost\"),\n IntInput(name=\"port\", display_name=\"port\", required=True, value=8123),\n StrInput(name=\"database\", display_name=\"database\", required=True),\n StrInput(name=\"table\", display_name=\"Table name\", required=True),\n StrInput(name=\"username\", display_name=\"The ClickHouse user name.\", required=True),\n SecretStrInput(name=\"password\", display_name=\"Clickhouse Password\", required=True),\n DropdownInput(\n name=\"index_type\",\n display_name=\"index_type\",\n options=[\"annoy\", \"vector_similarity\"],\n info=\"Type of the index.\",\n value=\"annoy\",\n advanced=True,\n ),\n DropdownInput(\n name=\"metric\",\n display_name=\"metric\",\n options=[\"angular\", \"euclidean\", \"manhattan\", \"hamming\", \"dot\"],\n info=\"Metric to compute distance.\",\n value=\"angular\",\n advanced=True,\n ),\n BoolInput(\n name=\"secure\",\n display_name=\"Use https/TLS. This overrides inferred values from the interface or port arguments.\",\n value=False,\n advanced=True,\n ),\n StrInput(name=\"index_param\", display_name=\"Param of the index\", value=\"100,'L2Distance'\", advanced=True),\n DictInput(name=\"index_query_params\", display_name=\"index query params\", advanced=True),\n *LCVectorStoreComponent.inputs,\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"]),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Number of Results\",\n info=\"Number of results to return.\",\n value=4,\n advanced=True,\n ),\n FloatInput(name=\"score_threshold\", display_name=\"Score threshold\", advanced=True),\n ]\n\n @check_cached_vector_store\n def build_vector_store(self) -> Clickhouse:\n try:\n import clickhouse_connect\n except ImportError as e:\n msg = (\n \"Failed to import ClickHouse dependencies. \"\n \"Install it using `uv pip install langflow[clickhouse-connect] --pre`\"\n )\n raise ImportError(msg) from e\n\n try:\n client = clickhouse_connect.get_client(\n host=self.host, port=self.port, username=self.username, password=self.password\n )\n client.command(\"SELECT 1\")\n except Exception as e:\n msg = f\"Failed to connect to Clickhouse: {e}\"\n raise ValueError(msg) from e\n\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n documents = []\n for _input in self.ingest_data or []:\n if isinstance(_input, Data):\n documents.append(_input.to_lc_document())\n else:\n documents.append(_input)\n\n kwargs = {}\n if self.index_param:\n kwargs[\"index_param\"] = self.index_param.split(\",\")\n if self.index_query_params:\n kwargs[\"index_query_params\"] = self.index_query_params\n\n settings = ClickhouseSettings(\n table=self.table,\n database=self.database,\n host=self.host,\n index_type=self.index_type,\n metric=self.metric,\n password=self.password,\n port=self.port,\n secure=self.secure,\n username=self.username,\n **kwargs,\n )\n if documents:\n clickhouse_vs = Clickhouse.from_documents(documents=documents, embedding=self.embedding, config=settings)\n\n else:\n clickhouse_vs = Clickhouse(embedding=self.embedding, config=settings)\n\n return clickhouse_vs\n\n def search_documents(self) -> list[Data]:\n vector_store = self.build_vector_store()\n\n if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():\n kwargs = {}\n if self.score_threshold:\n kwargs[\"score_threshold\"] = self.score_threshold\n\n docs = vector_store.similarity_search(query=self.search_query, k=self.number_of_results, **kwargs)\n\n data = docs_to_data(docs)\n self.status = data\n return data\n return []\n" + "value": "from langchain_community.vectorstores import Clickhouse, ClickhouseSettings\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.helpers.data import docs_to_data\nfrom lfx.inputs.inputs import BoolInput, FloatInput\nfrom lfx.io import (\n DictInput,\n DropdownInput,\n HandleInput,\n IntInput,\n SecretStrInput,\n StrInput,\n)\nfrom lfx.schema.data import Data\nfrom lfx.utils.ssrf_protection import validate_connector_url_for_ssrf\n\n\nclass ClickhouseVectorStoreComponent(LCVectorStoreComponent):\n display_name = \"ClickHouse\"\n description = \"ClickHouse Vector Store with search capabilities\"\n name = \"Clickhouse\"\n icon = \"Clickhouse\"\n\n inputs = [\n StrInput(name=\"host\", display_name=\"hostname\", required=True, value=\"localhost\"),\n IntInput(name=\"port\", display_name=\"port\", required=True, value=8123),\n StrInput(name=\"database\", display_name=\"database\", required=True),\n StrInput(name=\"table\", display_name=\"Table name\", required=True),\n StrInput(name=\"username\", display_name=\"The ClickHouse user name.\", required=True),\n SecretStrInput(name=\"password\", display_name=\"Clickhouse Password\", required=True),\n DropdownInput(\n name=\"index_type\",\n display_name=\"index_type\",\n options=[\"annoy\", \"vector_similarity\"],\n info=\"Type of the index.\",\n value=\"annoy\",\n advanced=True,\n ),\n DropdownInput(\n name=\"metric\",\n display_name=\"metric\",\n options=[\"angular\", \"euclidean\", \"manhattan\", \"hamming\", \"dot\"],\n info=\"Metric to compute distance.\",\n value=\"angular\",\n advanced=True,\n ),\n BoolInput(\n name=\"secure\",\n display_name=\"Use https/TLS. This overrides inferred values from the interface or port arguments.\",\n value=False,\n advanced=True,\n ),\n StrInput(name=\"index_param\", display_name=\"Param of the index\", value=\"100,'L2Distance'\", advanced=True),\n DictInput(name=\"index_query_params\", display_name=\"index query params\", advanced=True),\n *LCVectorStoreComponent.inputs,\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"]),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Number of Results\",\n info=\"Number of results to return.\",\n value=4,\n advanced=True,\n ),\n FloatInput(name=\"score_threshold\", display_name=\"Score threshold\", advanced=True),\n ]\n\n @check_cached_vector_store\n def build_vector_store(self) -> Clickhouse:\n try:\n import clickhouse_connect\n except ImportError as e:\n msg = (\n \"Failed to import ClickHouse dependencies. \"\n \"Install it using `uv pip install langflow[clickhouse-connect] --pre`\"\n )\n raise ImportError(msg) from e\n\n # host is tenant-controlled: block SSRF to internal/cloud-metadata hosts.\n scheme = \"https\" if getattr(self, \"secure\", False) else \"http\"\n validate_connector_url_for_ssrf(f\"{scheme}://{self.host}:{self.port}\")\n try:\n client = clickhouse_connect.get_client(\n host=self.host, port=self.port, username=self.username, password=self.password\n )\n client.command(\"SELECT 1\")\n except Exception as e:\n msg = f\"Failed to connect to Clickhouse: {e}\"\n raise ValueError(msg) from e\n\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n documents = []\n for _input in self.ingest_data or []:\n if isinstance(_input, Data):\n documents.append(_input.to_lc_document())\n else:\n documents.append(_input)\n\n kwargs = {}\n if self.index_param:\n kwargs[\"index_param\"] = self.index_param.split(\",\")\n if self.index_query_params:\n kwargs[\"index_query_params\"] = self.index_query_params\n\n settings = ClickhouseSettings(\n table=self.table,\n database=self.database,\n host=self.host,\n index_type=self.index_type,\n metric=self.metric,\n password=self.password,\n port=self.port,\n secure=self.secure,\n username=self.username,\n **kwargs,\n )\n if documents:\n clickhouse_vs = Clickhouse.from_documents(documents=documents, embedding=self.embedding, config=settings)\n\n else:\n clickhouse_vs = Clickhouse(embedding=self.embedding, config=settings)\n\n return clickhouse_vs\n\n def search_documents(self) -> list[Data]:\n vector_store = self.build_vector_store()\n\n if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():\n kwargs = {}\n if self.score_threshold:\n kwargs[\"score_threshold\"] = self.score_threshold\n\n docs = vector_store.similarity_search(query=self.search_query, k=self.number_of_results, **kwargs)\n\n data = docs_to_data(docs)\n self.status = data\n return data\n return []\n" }, "database": { "_input_type": "StrInput", @@ -57471,7 +57471,7 @@ "icon": "Globe", "legacy": false, "metadata": { - "code_hash": "1a052ebb9519", + "code_hash": "7efa8917d56c", "dependencies": { "dependencies": [ { @@ -57573,7 +57573,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import json\nimport re\nimport tempfile\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom typing import Any\nfrom urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse\n\nimport aiofiles\nimport aiofiles.os as aiofiles_os\nimport httpx\nimport validators\n\nfrom lfx.base.curl.parse import parse_context\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs.inputs import TabInput\nfrom lfx.io import (\n BoolInput,\n DataInput,\n DropdownInput,\n IntInput,\n MessageTextInput,\n MultilineInput,\n Output,\n TableInput,\n)\nfrom lfx.schema.data import Data\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.utils.component_utils import set_current_fields, set_field_advanced, set_field_display\n\n# SSRF Protection imports - for preventing Server-Side Request Forgery attacks\nfrom lfx.utils.ssrf_protection import (\n SSRFProtectionError,\n is_ssrf_protection_enabled,\n validate_and_resolve_url,\n)\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Define fields for each mode\nMODE_FIELDS = {\n \"URL\": [\n \"url_input\",\n \"method\",\n ],\n \"cURL\": [\"curl_input\"],\n}\n\n# Fields that should always be visible\nDEFAULT_FIELDS = [\"mode\"]\n\n# HTTP redirect status codes (RFC 9110).\nHTTP_MOVED_PERMANENTLY = 301\nHTTP_FOUND = 302\nHTTP_SEE_OTHER = 303\nHTTP_TEMPORARY_REDIRECT = 307\nHTTP_PERMANENT_REDIRECT = 308\n\n# Maximum number of redirects to follow when re-validating each hop (matches httpx's default).\nMAX_REDIRECTS = 20\n\n# HTTP status codes that represent a redirect carrying a Location header.\nREDIRECT_STATUS_CODES = frozenset(\n {\n HTTP_MOVED_PERMANENTLY,\n HTTP_FOUND,\n HTTP_SEE_OTHER,\n HTTP_TEMPORARY_REDIRECT,\n HTTP_PERMANENT_REDIRECT,\n }\n)\n\n\nclass APIRequestComponent(Component):\n display_name = \"API Request\"\n description = \"Make HTTP requests using URL or cURL commands.\"\n documentation: str = \"https://docs.langflow.org/api-request\"\n icon = \"Globe\"\n name = \"APIRequest\"\n\n inputs = [\n MessageTextInput(\n name=\"url_input\",\n display_name=\"URL\",\n info=\"Enter the URL for the request.\",\n advanced=False,\n tool_mode=True,\n ),\n MultilineInput(\n name=\"curl_input\",\n display_name=\"cURL\",\n info=(\n \"Paste a curl command to populate the fields. \"\n \"This will fill in the dictionary fields for headers and body.\"\n ),\n real_time_refresh=True,\n tool_mode=True,\n advanced=True,\n show=False,\n ),\n DropdownInput(\n name=\"method\",\n display_name=\"Method\",\n options=[\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"],\n value=\"GET\",\n info=\"The HTTP method to use.\",\n real_time_refresh=True,\n ),\n TabInput(\n name=\"mode\",\n display_name=\"Mode\",\n options=[\"URL\", \"cURL\"],\n value=\"URL\",\n info=\"Enable cURL mode to populate fields from a cURL command.\",\n real_time_refresh=True,\n ),\n DataInput(\n name=\"query_params\",\n display_name=\"Query Parameters\",\n info=\"The query parameters to append to the URL.\",\n advanced=True,\n ),\n TableInput(\n name=\"body\",\n display_name=\"Body\",\n info=\"The body to send with the request as a dictionary (for POST, PATCH, PUT).\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Parameter name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"description\": \"Parameter value\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n advanced=True,\n real_time_refresh=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": \"Langflow/1.0\"}],\n advanced=True,\n input_types=[\"Data\", \"JSON\"],\n real_time_refresh=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n value=30,\n info=\"The timeout to use for the request.\",\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n value=False,\n info=(\n \"Whether to follow HTTP redirects. \"\n \"WARNING: Enabling redirects may allow SSRF bypass attacks where a public URL \"\n \"redirects to internal resources. Only enable if you trust the target server. \"\n \"See OWASP SSRF Prevention Cheat Sheet for details.\"\n ),\n advanced=True,\n ),\n BoolInput(\n name=\"save_to_file\",\n display_name=\"Save to File\",\n value=False,\n info=\"Save the API response to a temporary file\",\n advanced=True,\n ),\n BoolInput(\n name=\"include_httpx_metadata\",\n display_name=\"Include HTTPx Metadata\",\n value=False,\n info=(\n \"Include properties such as headers, status_code, response_headers, \"\n \"and redirection_history in the output.\"\n ),\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"API Response\", name=\"data\", method=\"make_api_request\"),\n ]\n\n def _parse_json_value(self, value: Any) -> Any:\n \"\"\"Parse a value that might be a JSON string.\"\"\"\n if not isinstance(value, str):\n return value\n\n try:\n parsed = json.loads(value)\n except json.JSONDecodeError:\n return value\n else:\n return parsed\n\n def _process_body(self, body: Any) -> dict:\n \"\"\"Process the body input into a valid dictionary.\"\"\"\n if body is None:\n return {}\n if hasattr(body, \"data\"):\n body = body.data\n if isinstance(body, dict):\n return self._process_dict_body(body)\n if isinstance(body, str):\n return self._process_string_body(body)\n if isinstance(body, list):\n return self._process_list_body(body)\n return {}\n\n def _process_dict_body(self, body: dict) -> dict:\n \"\"\"Process dictionary body by parsing JSON values.\"\"\"\n return {k: self._parse_json_value(v) for k, v in body.items()}\n\n def _process_string_body(self, body: str) -> dict:\n \"\"\"Process string body by attempting JSON parse.\"\"\"\n try:\n return self._process_body(json.loads(body))\n except json.JSONDecodeError:\n return {\"data\": body}\n\n def _process_list_body(self, body: list) -> dict:\n \"\"\"Process list body by converting to key-value dictionary.\"\"\"\n processed_dict = {}\n try:\n for item in body:\n # Unwrap Data objects\n current_item = item\n if hasattr(item, \"data\"):\n unwrapped_data = item.data\n # If the unwrapped data is a dict but not key-value format, use it directly\n if isinstance(unwrapped_data, dict) and not self._is_valid_key_value_item(unwrapped_data):\n return unwrapped_data\n current_item = unwrapped_data\n if not self._is_valid_key_value_item(current_item):\n continue\n key = current_item[\"key\"]\n value = self._parse_json_value(current_item[\"value\"])\n processed_dict[key] = value\n except (KeyError, TypeError, ValueError) as e:\n self.log(f\"Failed to process body list: {e}\")\n return {}\n return processed_dict\n\n def _is_valid_key_value_item(self, item: Any) -> bool:\n \"\"\"Check if an item is a valid key-value dictionary.\"\"\"\n return isinstance(item, dict) and \"key\" in item and \"value\" in item\n\n def parse_curl(self, curl: str, build_config: dotdict) -> dotdict:\n \"\"\"Parse a cURL command and update build configuration.\"\"\"\n try:\n parsed = parse_context(curl)\n\n # Update basic configuration\n url = parsed.url\n # Normalize URL before setting it\n url = self._normalize_url(url)\n\n build_config[\"url_input\"][\"value\"] = url\n build_config[\"method\"][\"value\"] = parsed.method.upper()\n\n # Process headers\n headers_list = [{\"key\": k, \"value\": v} for k, v in parsed.headers.items()]\n build_config[\"headers\"][\"value\"] = headers_list\n\n # Process body data\n if not parsed.data:\n build_config[\"body\"][\"value\"] = []\n elif parsed.data:\n try:\n json_data = json.loads(parsed.data)\n if isinstance(json_data, dict):\n body_list = [\n {\"key\": k, \"value\": json.dumps(v) if isinstance(v, dict | list) else str(v)}\n for k, v in json_data.items()\n ]\n build_config[\"body\"][\"value\"] = body_list\n else:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": json.dumps(json_data)}]\n except json.JSONDecodeError:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": parsed.data}]\n\n except Exception as exc:\n msg = f\"Error parsing curl: {exc}\"\n self.log(msg)\n raise ValueError(msg) from exc\n\n return build_config\n\n def _normalize_url(self, url: str) -> str:\n \"\"\"Normalize URL by adding https:// if no protocol is specified.\"\"\"\n if not url or not isinstance(url, str):\n msg = \"URL cannot be empty\"\n raise ValueError(msg)\n\n url = url.strip()\n if url.startswith((\"http://\", \"https://\")):\n return url\n return f\"https://{url}\"\n\n async def make_request(\n self,\n client: httpx.AsyncClient,\n method: str,\n url: str,\n headers: dict | None = None,\n body: Any = None,\n timeout: int = 5,\n *,\n follow_redirects: bool = False,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n redirection_history = []\n\n try:\n # Prepare request parameters\n request_params = {\n \"method\": method,\n \"url\": url,\n \"headers\": headers,\n \"timeout\": timeout,\n \"follow_redirects\": follow_redirects,\n }\n # Only include body for methods that support it (GET must not have a body per HTTP spec)\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n response = await client.request(**request_params)\n\n redirection_history = [\n {\n \"url\": redirect.headers.get(\"Location\", str(redirect.url)),\n \"status_code\": redirect.status_code,\n }\n for redirect in response.history\n ]\n\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n async def _build_response_data(\n self,\n response: httpx.Response,\n source_url: str,\n headers: dict | None,\n redirection_history: list,\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Turn an httpx response into the component's ``Data`` output.\n\n Shared by the standard request path (``make_request``) and the redirect\n re-validation path (``_follow_redirects_with_validation``) so both produce\n identical metadata, optional file saving, and body decoding.\n \"\"\"\n is_binary, file_path = await self._response_info(response, with_file_path=save_to_file)\n response_headers = self._headers_to_dict(response.headers)\n\n # Base metadata\n metadata = {\n \"source\": source_url,\n \"status_code\": response.status_code,\n \"response_headers\": response_headers,\n }\n\n if redirection_history:\n metadata[\"redirection_history\"] = redirection_history\n\n if save_to_file:\n mode = \"wb\" if is_binary else \"w\"\n encoding = response.encoding if mode == \"w\" else None\n if file_path:\n await aiofiles_os.makedirs(file_path.parent, exist_ok=True)\n if is_binary:\n async with aiofiles.open(file_path, \"wb\") as f:\n await f.write(response.content)\n await f.flush()\n else:\n async with aiofiles.open(file_path, \"w\", encoding=encoding) as f:\n await f.write(response.text)\n await f.flush()\n metadata[\"file_path\"] = str(file_path)\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n return Data(data=metadata)\n\n # Handle response content\n if is_binary:\n result = response.content\n else:\n try:\n result = response.json()\n except json.JSONDecodeError:\n self.log(\"Failed to decode JSON response\")\n result = response.text.encode(\"utf-8\")\n\n metadata[\"result\"] = result\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n\n return Data(data=metadata)\n\n def add_query_params(self, url: str, params: dict) -> str:\n \"\"\"Add query parameters to URL efficiently.\"\"\"\n if not params:\n return url\n url_parts = list(urlparse(url))\n query = dict(parse_qsl(url_parts[4]))\n query.update(params)\n url_parts[4] = urlencode(query)\n return urlunparse(url_parts)\n\n def _headers_to_dict(self, headers: httpx.Headers) -> dict[str, str]:\n \"\"\"Convert HTTP headers to a dictionary with lowercased keys.\"\"\"\n return {k.lower(): v for k, v in headers.items()}\n\n def _process_headers(self, headers: Any) -> dict:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if headers is None:\n return {}\n if isinstance(headers, dict):\n return headers\n if isinstance(headers, list):\n return {item[\"key\"]: item[\"value\"] for item in headers if self._is_valid_key_value_item(item)}\n return {}\n\n async def make_api_request(self) -> Data:\n \"\"\"Make HTTP request with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. The protection works by:\n 1. Validating the URL and resolving DNS during security check\n 2. Pinning the validated IP address\n 3. Forcing the HTTP client to use the pinned IP for the actual request\n 4. Ignoring any subsequent DNS changes (prevents rebinding attacks)\n\n Returns:\n Data: Response data from the HTTP request\n\n Raises:\n ValueError: If URL is invalid or blocked by SSRF protection\n \"\"\"\n # Extract request parameters\n method = self.method\n url = self.url_input.strip() if isinstance(self.url_input, str) else \"\"\n headers = self.headers or {}\n body = self.body or {}\n timeout = self.timeout\n follow_redirects = self.follow_redirects\n save_to_file = self.save_to_file\n include_httpx_metadata = self.include_httpx_metadata\n\n # Security warning: HTTP redirects can bypass SSRF protection\n # A public URL could redirect to an internal resource\n if follow_redirects:\n self.log(\n \"Security Warning: HTTP redirects are enabled. This may allow SSRF bypass attacks \"\n \"where a public URL redirects to internal resources (e.g., cloud metadata endpoints). \"\n \"Only enable this if you trust the target server.\"\n )\n\n # Normalize URL (add https:// if no protocol specified)\n url = self._normalize_url(url)\n\n # Basic URL format validation\n if not validators.url(url):\n msg = f\"Invalid URL provided: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Pinning the validated IP address\n # 3. Using a custom HTTP transport that forces use of the pinned IP\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IP is pinned: \"example.com = 93.184.216.34\"\n # - HTTP request: uses pinned IP directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IP\n # ============================================================================\n\n try:\n # Validate URL and get validated IPs for DNS pinning\n _validated_url, validated_ips = validate_and_resolve_url(url)\n\n # Log DNS pinning information for security auditing\n if validated_ips:\n self.log(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s)\")\n\n except SSRFProtectionError as e:\n # SSRF protection blocked the request (private IP, internal network, etc.)\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n\n # Process query parameters (from string or Data object)\n if isinstance(self.query_params, str):\n query_params = dict(parse_qsl(self.query_params))\n else:\n query_params = self.query_params.data if self.query_params else {}\n\n # Process headers and body into proper format\n headers = self._process_headers(headers)\n body = self._process_body(body)\n url = self.add_query_params(url, query_params)\n\n # ============================================================================\n # Execute the request (re-validating any redirects when SSRF protection is on)\n # ============================================================================\n # When SSRF protection is enabled we must NOT let httpx auto-follow redirects:\n # a validated public URL can redirect to an internal address (loopback, RFC1918,\n # link-local / cloud metadata) that was never checked, bypassing both the initial\n # validation and DNS pinning. Instead we follow redirects manually so every hop\n # is re-validated with the same denylist + DNS pinning. When protection is\n # disabled, we preserve the previous behavior and let httpx handle redirects.\n if is_ssrf_protection_enabled() and follow_redirects:\n result = await self._follow_redirects_with_validation(\n method,\n url,\n headers,\n body,\n timeout,\n validated_ips,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n else:\n # No redirect re-validation needed:\n # - SSRF protection is disabled (user opted out), or\n # - redirects are disabled, so httpx makes a single request.\n # DNS pinning still applies to the single request when protection is enabled\n # and the host resolved to validated IPs.\n async with self._build_http_client(url, validated_ips) as client:\n result = await self.make_request(\n client,\n method,\n url,\n headers,\n body,\n timeout,\n follow_redirects=follow_redirects,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n self.status = result\n return result\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client, pinning DNS to validated IPs when SSRF protection applies.\n\n Args:\n url: The request URL whose hostname will be pinned.\n validated_ips: IPs validated by ``validate_and_resolve_url`` for this hop.\n\n Returns:\n httpx.AsyncClient: A client that pins DNS to ``validated_ips`` (preventing\n rebinding) when SSRF protection is enabled and the hop has validated IPs;\n otherwise a standard client (protection disabled, allowlisted host, or\n hostname extraction failure).\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n # Extract hostname from the URL so the custom transport can pin it while\n # preserving the Host header for virtual hosting / TLS SNI.\n hostname = urlparse(url).hostname\n if hostname:\n # The custom transport tries validated IPs in order (dual-stack / LB).\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _method_for_redirect(method: str, status_code: int) -> str:\n \"\"\"Return the HTTP method to use after a redirect, mirroring httpx semantics.\n\n A 303 (See Other) always becomes GET; 301/302 downgrade POST to GET for\n browser compatibility; 307/308 preserve the original method (and body).\n \"\"\"\n method = method.upper()\n if status_code == HTTP_SEE_OTHER and method != \"HEAD\":\n return \"GET\"\n if status_code in (HTTP_MOVED_PERMANENTLY, HTTP_FOUND) and method == \"POST\":\n return \"GET\"\n return method\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different host.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to a different host than the one the\n caller intended them for. Same-host redirects keep all headers.\n \"\"\"\n if not headers:\n return headers\n if urlparse(current_url).hostname == urlparse(next_url).hostname:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n async def _follow_redirects_with_validation(\n self,\n method: str,\n url: str,\n headers: dict | None,\n body: Any,\n timeout: int,\n validated_ips: list[str],\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Make the request and follow redirects manually, re-validating every hop.\n\n This closes an SSRF bypass: with ``follow_redirects`` enabled, httpx would\n otherwise auto-follow a redirect from a validated public URL to an internal\n address that was never checked. Here each redirect ``Location`` is resolved\n (relative locations included) and re-validated with ``validate_and_resolve_url``\n — the same private/loopback/link-local denylist and DNS pinning applied to the\n initial request — before any connection to it is made. A blocked hop raises\n ``ValueError``; the number of redirects is capped at ``MAX_REDIRECTS``.\n \"\"\"\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n current_url = url\n current_ips = validated_ips\n redirection_history: list[dict] = []\n\n for _ in range(MAX_REDIRECTS + 1):\n request_params: dict[str, Any] = {\n \"method\": method,\n \"url\": current_url,\n \"headers\": headers,\n \"timeout\": timeout,\n # Never let httpx follow redirects itself; each hop is validated below.\n \"follow_redirects\": False,\n }\n # Only include body for methods that support it (GET must not have a body).\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n\n try:\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.request(**request_params)\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {current_url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n location = response.headers.get(\"Location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n redirection_history.append({\"url\": location, \"status_code\": response.status_code})\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n # Non-http(s) schemes, private/loopback/link-local hosts, and hostnames that\n # resolve to blocked IPs all raise SSRFProtectionError here.\n try:\n _validated_url, current_ips = validate_and_resolve_url(next_url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n method = self._method_for_redirect(method, response.status_code)\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict:\n \"\"\"Update the build config based on the selected mode.\"\"\"\n if field_name != \"mode\":\n if field_name == \"curl_input\" and self.mode == \"cURL\" and self.curl_input:\n return self.parse_curl(self.curl_input, build_config)\n return build_config\n\n if field_value == \"cURL\":\n set_field_display(build_config, \"curl_input\", value=True)\n if build_config[\"curl_input\"][\"value\"]:\n try:\n build_config = self.parse_curl(build_config[\"curl_input\"][\"value\"], build_config)\n except ValueError as e:\n self.log(f\"Failed to parse cURL input: {e}\")\n else:\n set_field_display(build_config, \"curl_input\", value=False)\n\n return set_current_fields(\n build_config=build_config,\n action_fields=MODE_FIELDS,\n selected_action=field_value,\n default_fields=DEFAULT_FIELDS,\n func=set_field_advanced,\n default_value=True,\n )\n\n async def _response_info(\n self, response: httpx.Response, *, with_file_path: bool = False\n ) -> tuple[bool, Path | None]:\n \"\"\"Determine the file path and whether the response content is binary.\n\n Args:\n response (Response): The HTTP response object.\n with_file_path (bool): Whether to save the response content to a file.\n\n Returns:\n Tuple[bool, Path | None]:\n A tuple containing a boolean indicating if the content is binary and the full file path (if applicable).\n \"\"\"\n content_type = response.headers.get(\"Content-Type\", \"\")\n is_binary = \"application/octet-stream\" in content_type or \"application/binary\" in content_type\n\n if not with_file_path:\n return is_binary, None\n\n component_temp_dir = Path(tempfile.gettempdir()) / self.__class__.__name__\n\n # Create directory asynchronously\n await aiofiles_os.makedirs(component_temp_dir, exist_ok=True)\n\n filename = None\n if \"Content-Disposition\" in response.headers:\n content_disposition = response.headers[\"Content-Disposition\"]\n filename_match = re.search(r'filename=\"(.+?)\"', content_disposition)\n if filename_match:\n extracted_filename = filename_match.group(1)\n filename = extracted_filename\n\n # Step 3: Infer file extension or use part of the request URL if no filename\n if not filename:\n # Extract the last segment of the URL path\n url_path = urlparse(str(response.request.url) if response.request else \"\").path\n base_name = Path(url_path).name # Get the last segment of the path\n if not base_name: # If the path ends with a slash or is empty\n base_name = \"response\"\n\n # Infer file extension\n content_type_to_extension = {\n \"text/plain\": \".txt\",\n \"application/json\": \".json\",\n \"image/jpeg\": \".jpg\",\n \"image/png\": \".png\",\n \"application/octet-stream\": \".bin\",\n }\n extension = content_type_to_extension.get(content_type, \".bin\" if is_binary else \".txt\")\n filename = f\"{base_name}{extension}\"\n\n # Step 4: Define the full file path\n file_path = component_temp_dir / filename\n\n # Step 5: Check if file exists asynchronously and handle accordingly\n try:\n # Try to create the file exclusively (x mode) to check existence\n async with aiofiles.open(file_path, \"x\") as _:\n pass # File created successfully, we can use this path\n except FileExistsError:\n # If file exists, append a timestamp to the filename\n timestamp = datetime.now(timezone.utc).strftime(\"%Y%m%d%H%M%S%f\")\n file_path = component_temp_dir / f\"{timestamp}-{filename}\"\n\n return is_binary, file_path\n" + "value": "import json\nimport re\nimport tempfile\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom typing import Any\nfrom urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse\n\nimport aiofiles\nimport aiofiles.os as aiofiles_os\nimport httpx\nimport validators\n\nfrom lfx.base.curl.parse import parse_context\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs.inputs import TabInput\nfrom lfx.io import (\n BoolInput,\n DataInput,\n DropdownInput,\n IntInput,\n MessageTextInput,\n MultilineInput,\n Output,\n TableInput,\n)\nfrom lfx.schema.data import Data\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.utils.component_utils import set_current_fields, set_field_advanced, set_field_display\n\n# SSRF Protection imports - for preventing Server-Side Request Forgery attacks\nfrom lfx.utils.ssrf_protection import (\n SSRFProtectionError,\n is_ssrf_protection_enabled,\n validate_and_resolve_url,\n)\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Define fields for each mode\nMODE_FIELDS = {\n \"URL\": [\n \"url_input\",\n \"method\",\n ],\n \"cURL\": [\"curl_input\"],\n}\n\n# Fields that should always be visible\nDEFAULT_FIELDS = [\"mode\"]\n\n# HTTP redirect status codes (RFC 9110).\nHTTP_MOVED_PERMANENTLY = 301\nHTTP_FOUND = 302\nHTTP_SEE_OTHER = 303\nHTTP_TEMPORARY_REDIRECT = 307\nHTTP_PERMANENT_REDIRECT = 308\n\n# Maximum number of redirects to follow when re-validating each hop (matches httpx's default).\nMAX_REDIRECTS = 20\n\n# HTTP status codes that represent a redirect carrying a Location header.\nREDIRECT_STATUS_CODES = frozenset(\n {\n HTTP_MOVED_PERMANENTLY,\n HTTP_FOUND,\n HTTP_SEE_OTHER,\n HTTP_TEMPORARY_REDIRECT,\n HTTP_PERMANENT_REDIRECT,\n }\n)\n\n\nclass APIRequestComponent(Component):\n display_name = \"API Request\"\n description = \"Make HTTP requests using URL or cURL commands.\"\n documentation: str = \"https://docs.langflow.org/api-request\"\n icon = \"Globe\"\n name = \"APIRequest\"\n\n inputs = [\n MessageTextInput(\n name=\"url_input\",\n display_name=\"URL\",\n info=\"Enter the URL for the request.\",\n advanced=False,\n tool_mode=True,\n ),\n MultilineInput(\n name=\"curl_input\",\n display_name=\"cURL\",\n info=(\n \"Paste a curl command to populate the fields. \"\n \"This will fill in the dictionary fields for headers and body.\"\n ),\n real_time_refresh=True,\n tool_mode=True,\n advanced=True,\n show=False,\n ),\n DropdownInput(\n name=\"method\",\n display_name=\"Method\",\n options=[\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"],\n value=\"GET\",\n info=\"The HTTP method to use.\",\n real_time_refresh=True,\n ),\n TabInput(\n name=\"mode\",\n display_name=\"Mode\",\n options=[\"URL\", \"cURL\"],\n value=\"URL\",\n info=\"Enable cURL mode to populate fields from a cURL command.\",\n real_time_refresh=True,\n ),\n DataInput(\n name=\"query_params\",\n display_name=\"Query Parameters\",\n info=\"The query parameters to append to the URL.\",\n advanced=True,\n ),\n TableInput(\n name=\"body\",\n display_name=\"Body\",\n info=\"The body to send with the request as a dictionary (for POST, PATCH, PUT).\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Parameter name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"description\": \"Parameter value\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n advanced=True,\n real_time_refresh=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": \"Langflow/1.0\"}],\n advanced=True,\n input_types=[\"Data\", \"JSON\"],\n real_time_refresh=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n value=30,\n info=\"The timeout to use for the request.\",\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n value=False,\n info=(\n \"Whether to follow HTTP redirects. \"\n \"WARNING: Enabling redirects may allow SSRF bypass attacks where a public URL \"\n \"redirects to internal resources. Only enable if you trust the target server. \"\n \"See OWASP SSRF Prevention Cheat Sheet for details.\"\n ),\n advanced=True,\n ),\n BoolInput(\n name=\"save_to_file\",\n display_name=\"Save to File\",\n value=False,\n info=\"Save the API response to a temporary file\",\n advanced=True,\n ),\n BoolInput(\n name=\"include_httpx_metadata\",\n display_name=\"Include HTTPx Metadata\",\n value=False,\n info=(\n \"Include properties such as headers, status_code, response_headers, \"\n \"and redirection_history in the output.\"\n ),\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"API Response\", name=\"data\", method=\"make_api_request\"),\n ]\n\n def _parse_json_value(self, value: Any) -> Any:\n \"\"\"Parse a value that might be a JSON string.\"\"\"\n if not isinstance(value, str):\n return value\n\n try:\n parsed = json.loads(value)\n except json.JSONDecodeError:\n return value\n else:\n return parsed\n\n def _process_body(self, body: Any) -> dict:\n \"\"\"Process the body input into a valid dictionary.\"\"\"\n if body is None:\n return {}\n if hasattr(body, \"data\"):\n body = body.data\n if isinstance(body, dict):\n return self._process_dict_body(body)\n if isinstance(body, str):\n return self._process_string_body(body)\n if isinstance(body, list):\n return self._process_list_body(body)\n return {}\n\n def _process_dict_body(self, body: dict) -> dict:\n \"\"\"Process dictionary body by parsing JSON values.\"\"\"\n return {k: self._parse_json_value(v) for k, v in body.items()}\n\n def _process_string_body(self, body: str) -> dict:\n \"\"\"Process string body by attempting JSON parse.\"\"\"\n try:\n return self._process_body(json.loads(body))\n except json.JSONDecodeError:\n return {\"data\": body}\n\n def _process_list_body(self, body: list) -> dict:\n \"\"\"Process list body by converting to key-value dictionary.\"\"\"\n processed_dict = {}\n try:\n for item in body:\n # Unwrap Data objects\n current_item = item\n if hasattr(item, \"data\"):\n unwrapped_data = item.data\n # If the unwrapped data is a dict but not key-value format, use it directly\n if isinstance(unwrapped_data, dict) and not self._is_valid_key_value_item(unwrapped_data):\n return unwrapped_data\n current_item = unwrapped_data\n if not self._is_valid_key_value_item(current_item):\n continue\n key = current_item[\"key\"]\n value = self._parse_json_value(current_item[\"value\"])\n processed_dict[key] = value\n except (KeyError, TypeError, ValueError) as e:\n self.log(f\"Failed to process body list: {e}\")\n return {}\n return processed_dict\n\n def _is_valid_key_value_item(self, item: Any) -> bool:\n \"\"\"Check if an item is a valid key-value dictionary.\"\"\"\n return isinstance(item, dict) and \"key\" in item and \"value\" in item\n\n def parse_curl(self, curl: str, build_config: dotdict) -> dotdict:\n \"\"\"Parse a cURL command and update build configuration.\"\"\"\n try:\n parsed = parse_context(curl)\n\n # Update basic configuration\n url = parsed.url\n # Normalize URL before setting it\n url = self._normalize_url(url)\n\n build_config[\"url_input\"][\"value\"] = url\n build_config[\"method\"][\"value\"] = parsed.method.upper()\n\n # Process headers\n headers_list = [{\"key\": k, \"value\": v} for k, v in parsed.headers.items()]\n build_config[\"headers\"][\"value\"] = headers_list\n\n # Process body data\n if not parsed.data:\n build_config[\"body\"][\"value\"] = []\n elif parsed.data:\n try:\n json_data = json.loads(parsed.data)\n if isinstance(json_data, dict):\n body_list = [\n {\"key\": k, \"value\": json.dumps(v) if isinstance(v, dict | list) else str(v)}\n for k, v in json_data.items()\n ]\n build_config[\"body\"][\"value\"] = body_list\n else:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": json.dumps(json_data)}]\n except json.JSONDecodeError:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": parsed.data}]\n\n except Exception as exc:\n msg = f\"Error parsing curl: {exc}\"\n self.log(msg)\n raise ValueError(msg) from exc\n\n return build_config\n\n def _normalize_url(self, url: str) -> str:\n \"\"\"Normalize URL by adding https:// if no protocol is specified.\"\"\"\n if not url or not isinstance(url, str):\n msg = \"URL cannot be empty\"\n raise ValueError(msg)\n\n url = url.strip()\n if url.startswith((\"http://\", \"https://\")):\n return url\n return f\"https://{url}\"\n\n async def make_request(\n self,\n client: httpx.AsyncClient,\n method: str,\n url: str,\n headers: dict | None = None,\n body: Any = None,\n timeout: int = 5,\n *,\n follow_redirects: bool = False,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n redirection_history = []\n\n try:\n # Prepare request parameters\n request_params = {\n \"method\": method,\n \"url\": url,\n \"headers\": headers,\n \"timeout\": timeout,\n \"follow_redirects\": follow_redirects,\n }\n # Only include body for methods that support it (GET must not have a body per HTTP spec)\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n response = await client.request(**request_params)\n\n redirection_history = [\n {\n \"url\": redirect.headers.get(\"Location\", str(redirect.url)),\n \"status_code\": redirect.status_code,\n }\n for redirect in response.history\n ]\n\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n async def _build_response_data(\n self,\n response: httpx.Response,\n source_url: str,\n headers: dict | None,\n redirection_history: list,\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Turn an httpx response into the component's ``Data`` output.\n\n Shared by the standard request path (``make_request``) and the redirect\n re-validation path (``_follow_redirects_with_validation``) so both produce\n identical metadata, optional file saving, and body decoding.\n \"\"\"\n is_binary, file_path = await self._response_info(response, with_file_path=save_to_file)\n response_headers = self._headers_to_dict(response.headers)\n\n # Base metadata\n metadata = {\n \"source\": source_url,\n \"status_code\": response.status_code,\n \"response_headers\": response_headers,\n }\n\n if redirection_history:\n metadata[\"redirection_history\"] = redirection_history\n\n if save_to_file:\n mode = \"wb\" if is_binary else \"w\"\n encoding = response.encoding if mode == \"w\" else None\n if file_path:\n await aiofiles_os.makedirs(file_path.parent, exist_ok=True)\n if is_binary:\n async with aiofiles.open(file_path, \"wb\") as f:\n await f.write(response.content)\n await f.flush()\n else:\n async with aiofiles.open(file_path, \"w\", encoding=encoding) as f:\n await f.write(response.text)\n await f.flush()\n metadata[\"file_path\"] = str(file_path)\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n return Data(data=metadata)\n\n # Handle response content\n if is_binary:\n result = response.content\n else:\n try:\n result = response.json()\n except json.JSONDecodeError:\n self.log(\"Failed to decode JSON response\")\n result = response.text.encode(\"utf-8\")\n\n metadata[\"result\"] = result\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n\n return Data(data=metadata)\n\n def add_query_params(self, url: str, params: dict) -> str:\n \"\"\"Add query parameters to URL efficiently.\"\"\"\n if not params:\n return url\n url_parts = list(urlparse(url))\n query = dict(parse_qsl(url_parts[4]))\n query.update(params)\n url_parts[4] = urlencode(query)\n return urlunparse(url_parts)\n\n def _headers_to_dict(self, headers: httpx.Headers) -> dict[str, str]:\n \"\"\"Convert HTTP headers to a dictionary with lowercased keys.\"\"\"\n return {k.lower(): v for k, v in headers.items()}\n\n def _process_headers(self, headers: Any) -> dict:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if headers is None:\n return {}\n if isinstance(headers, dict):\n return headers\n if isinstance(headers, list):\n return {item[\"key\"]: item[\"value\"] for item in headers if self._is_valid_key_value_item(item)}\n return {}\n\n async def make_api_request(self) -> Data:\n \"\"\"Make HTTP request with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. The protection works by:\n 1. Validating the URL and resolving DNS during security check\n 2. Pinning the validated IP address\n 3. Forcing the HTTP client to use the pinned IP for the actual request\n 4. Ignoring any subsequent DNS changes (prevents rebinding attacks)\n\n Returns:\n Data: Response data from the HTTP request\n\n Raises:\n ValueError: If URL is invalid or blocked by SSRF protection\n \"\"\"\n # Extract request parameters\n method = self.method\n url = self.url_input.strip() if isinstance(self.url_input, str) else \"\"\n headers = self.headers or {}\n body = self.body or {}\n timeout = self.timeout\n follow_redirects = self.follow_redirects\n save_to_file = self.save_to_file\n include_httpx_metadata = self.include_httpx_metadata\n\n # Security warning: HTTP redirects can bypass SSRF protection\n # A public URL could redirect to an internal resource\n if follow_redirects:\n self.log(\n \"Security Warning: HTTP redirects are enabled. This may allow SSRF bypass attacks \"\n \"where a public URL redirects to internal resources (e.g., cloud metadata endpoints). \"\n \"Only enable this if you trust the target server.\"\n )\n\n # Normalize URL (add https:// if no protocol specified)\n url = self._normalize_url(url)\n\n # Basic URL format validation\n if not validators.url(url):\n msg = f\"Invalid URL provided: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Pinning the validated IP address\n # 3. Using a custom HTTP transport that forces use of the pinned IP\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IP is pinned: \"example.com = 93.184.216.34\"\n # - HTTP request: uses pinned IP directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IP\n # ============================================================================\n\n try:\n # Validate URL and get validated IPs for DNS pinning\n _validated_url, validated_ips = validate_and_resolve_url(url)\n\n # Log DNS pinning information for security auditing\n if validated_ips:\n self.log(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s)\")\n\n except SSRFProtectionError as e:\n # SSRF protection blocked the request (private IP, internal network, etc.)\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n\n # Process query parameters (from string or Data object)\n if isinstance(self.query_params, str):\n query_params = dict(parse_qsl(self.query_params))\n else:\n query_params = self.query_params.data if self.query_params else {}\n\n # Process headers and body into proper format\n headers = self._process_headers(headers)\n body = self._process_body(body)\n url = self.add_query_params(url, query_params)\n\n # ============================================================================\n # Execute the request (re-validating any redirects when SSRF protection is on)\n # ============================================================================\n # When SSRF protection is enabled we must NOT let httpx auto-follow redirects:\n # a validated public URL can redirect to an internal address (loopback, RFC1918,\n # link-local / cloud metadata) that was never checked, bypassing both the initial\n # validation and DNS pinning. Instead we follow redirects manually so every hop\n # is re-validated with the same denylist + DNS pinning. When protection is\n # disabled, we preserve the previous behavior and let httpx handle redirects.\n if is_ssrf_protection_enabled() and follow_redirects:\n result = await self._follow_redirects_with_validation(\n method,\n url,\n headers,\n body,\n timeout,\n validated_ips,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n else:\n # No redirect re-validation needed:\n # - SSRF protection is disabled (user opted out), or\n # - redirects are disabled, so httpx makes a single request.\n # DNS pinning still applies to the single request when protection is enabled\n # and the host resolved to validated IPs.\n async with self._build_http_client(url, validated_ips) as client:\n result = await self.make_request(\n client,\n method,\n url,\n headers,\n body,\n timeout,\n follow_redirects=follow_redirects,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n self.status = result\n return result\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client, pinning DNS to validated IPs when SSRF protection applies.\n\n Args:\n url: The request URL whose hostname will be pinned.\n validated_ips: IPs validated by ``validate_and_resolve_url`` for this hop.\n\n Returns:\n httpx.AsyncClient: A client that pins DNS to ``validated_ips`` (preventing\n rebinding) when SSRF protection is enabled and the hop has validated IPs;\n otherwise a standard client (protection disabled, allowlisted host, or\n hostname extraction failure).\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n # Extract hostname from the URL so the custom transport can pin it while\n # preserving the Host header for virtual hosting / TLS SNI.\n hostname = urlparse(url).hostname\n if hostname:\n # The custom transport tries validated IPs in order (dual-stack / LB).\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _method_for_redirect(method: str, status_code: int) -> str:\n \"\"\"Return the HTTP method to use after a redirect, mirroring httpx semantics.\n\n A 303 (See Other) always becomes GET; 301/302 downgrade POST to GET for\n browser compatibility; 307/308 preserve the original method (and body).\n \"\"\"\n method = method.upper()\n if status_code == HTTP_SEE_OTHER and method != \"HEAD\":\n return \"GET\"\n if status_code in (HTTP_MOVED_PERMANENTLY, HTTP_FOUND) and method == \"POST\":\n return \"GET\"\n return method\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different host.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to a different host than the one the\n caller intended them for. Same-host redirects keep all headers.\n \"\"\"\n if not headers:\n return headers\n if urlparse(current_url).hostname == urlparse(next_url).hostname:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n async def _follow_redirects_with_validation(\n self,\n method: str,\n url: str,\n headers: dict | None,\n body: Any,\n timeout: int,\n validated_ips: list[str],\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Make the request and follow redirects manually, re-validating every hop.\n\n This closes an SSRF bypass: with ``follow_redirects`` enabled, httpx would\n otherwise auto-follow a redirect from a validated public URL to an internal\n address that was never checked. Here each redirect ``Location`` is resolved\n (relative locations included) and re-validated with ``validate_and_resolve_url``\n — the same private/loopback/link-local denylist and DNS pinning applied to the\n initial request — before any connection to it is made. A blocked hop raises\n ``ValueError``; the number of redirects is capped at ``MAX_REDIRECTS``.\n \"\"\"\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n current_url = url\n current_ips = validated_ips\n redirection_history: list[dict] = []\n\n for _ in range(MAX_REDIRECTS + 1):\n request_params: dict[str, Any] = {\n \"method\": method,\n \"url\": current_url,\n \"headers\": headers,\n \"timeout\": timeout,\n # Never let httpx follow redirects itself; each hop is validated below.\n \"follow_redirects\": False,\n }\n # Only include body for methods that support it (GET must not have a body).\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n\n try:\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.request(**request_params)\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {current_url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n location = response.headers.get(\"Location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n redirection_history.append({\"url\": location, \"status_code\": response.status_code})\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n # Non-http(s) schemes, private/loopback/link-local hosts, and hostnames that\n # resolve to blocked IPs all raise SSRFProtectionError here.\n try:\n _validated_url, current_ips = validate_and_resolve_url(next_url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n method = self._method_for_redirect(method, response.status_code)\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict:\n \"\"\"Update the build config based on the selected mode.\"\"\"\n if field_name != \"mode\":\n if field_name == \"curl_input\" and self.mode == \"cURL\" and self.curl_input:\n return self.parse_curl(self.curl_input, build_config)\n return build_config\n\n if field_value == \"cURL\":\n set_field_display(build_config, \"curl_input\", value=True)\n if build_config[\"curl_input\"][\"value\"]:\n try:\n build_config = self.parse_curl(build_config[\"curl_input\"][\"value\"], build_config)\n except ValueError as e:\n self.log(f\"Failed to parse cURL input: {e}\")\n else:\n set_field_display(build_config, \"curl_input\", value=False)\n\n return set_current_fields(\n build_config=build_config,\n action_fields=MODE_FIELDS,\n selected_action=field_value,\n default_fields=DEFAULT_FIELDS,\n func=set_field_advanced,\n default_value=True,\n )\n\n async def _response_info(\n self, response: httpx.Response, *, with_file_path: bool = False\n ) -> tuple[bool, Path | None]:\n \"\"\"Determine the file path and whether the response content is binary.\n\n Args:\n response (Response): The HTTP response object.\n with_file_path (bool): Whether to save the response content to a file.\n\n Returns:\n Tuple[bool, Path | None]:\n A tuple containing a boolean indicating if the content is binary and the full file path (if applicable).\n \"\"\"\n content_type = response.headers.get(\"Content-Type\", \"\")\n is_binary = \"application/octet-stream\" in content_type or \"application/binary\" in content_type\n\n if not with_file_path:\n return is_binary, None\n\n component_temp_dir = Path(tempfile.gettempdir()) / self.__class__.__name__\n\n # Create directory asynchronously\n await aiofiles_os.makedirs(component_temp_dir, exist_ok=True)\n\n filename = None\n if \"Content-Disposition\" in response.headers:\n content_disposition = response.headers[\"Content-Disposition\"]\n filename_match = re.search(r'filename=\"(.+?)\"', content_disposition)\n if filename_match:\n # The Content-Disposition header is controlled by the (tenant-chosen) remote\n # server. Reduce it to a bare basename so a value like \"../../etc/cron.d/x\"\n # cannot traverse out of component_temp_dir into an arbitrary write location.\n extracted_filename = Path(filename_match.group(1)).name\n filename = extracted_filename or None\n\n # Step 3: Infer file extension or use part of the request URL if no filename\n if not filename:\n # Extract the last segment of the URL path\n url_path = urlparse(str(response.request.url) if response.request else \"\").path\n base_name = Path(url_path).name # Get the last segment of the path\n if not base_name: # If the path ends with a slash or is empty\n base_name = \"response\"\n\n # Infer file extension\n content_type_to_extension = {\n \"text/plain\": \".txt\",\n \"application/json\": \".json\",\n \"image/jpeg\": \".jpg\",\n \"image/png\": \".png\",\n \"application/octet-stream\": \".bin\",\n }\n extension = content_type_to_extension.get(content_type, \".bin\" if is_binary else \".txt\")\n filename = f\"{base_name}{extension}\"\n\n # Step 4: Define the full file path\n file_path = component_temp_dir / filename\n\n # Step 5: Check if file exists asynchronously and handle accordingly\n try:\n # Try to create the file exclusively (x mode) to check existence\n async with aiofiles.open(file_path, \"x\") as _:\n pass # File created successfully, we can use this path\n except FileExistsError:\n # If file exists, append a timestamp to the filename\n timestamp = datetime.now(timezone.utc).strftime(\"%Y%m%d%H%M%S%f\")\n file_path = component_temp_dir / f\"{timestamp}-{filename}\"\n\n return is_binary, file_path\n" }, "curl_input": { "_input_type": "MultilineInput", @@ -57860,7 +57860,7 @@ "icon": "file-spreadsheet", "legacy": true, "metadata": { - "code_hash": "049e2eeb6901", + "code_hash": "f453481bb764", "dependencies": { "dependencies": [ { @@ -57912,7 +57912,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import csv\nimport io\nfrom pathlib import Path\n\nfrom lfx.base.data.storage_utils import read_file_text\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.io import FileInput, MessageTextInput, MultilineInput, Output\nfrom lfx.schema.data import Data\nfrom lfx.utils.async_helpers import run_until_complete\n\n\nclass CSVToDataComponent(Component):\n display_name = \"Load CSV\"\n description = \"Load a CSV file, CSV from a file path, or a valid CSV string and convert it to a list of Data\"\n icon = \"file-spreadsheet\"\n name = \"CSVtoData\"\n legacy = True\n replacement = [\"data.File\"]\n\n inputs = [\n FileInput(\n name=\"csv_file\",\n display_name=\"CSV File\",\n file_types=[\"csv\"],\n info=\"Upload a CSV file to convert to a list of Data objects\",\n ),\n MessageTextInput(\n name=\"csv_path\",\n display_name=\"CSV File Path\",\n info=\"Provide the path to the CSV file as pure text\",\n ),\n MultilineInput(\n name=\"csv_string\",\n display_name=\"CSV String\",\n info=\"Paste a CSV string directly to convert to a list of Data objects\",\n ),\n MessageTextInput(\n name=\"text_key\",\n display_name=\"Text Key\",\n info=\"The key to use for the text column. Defaults to 'text'.\",\n value=\"text\",\n ),\n ]\n\n outputs = [\n Output(name=\"data_list\", display_name=\"JSON List\", method=\"load_csv_to_data\"),\n ]\n\n def load_csv_to_data(self) -> list[Data]:\n if sum(bool(field) for field in [self.csv_file, self.csv_path, self.csv_string]) != 1:\n msg = \"Please provide exactly one of: CSV file, file path, or CSV string.\"\n raise ValueError(msg)\n\n csv_data = None\n try:\n if self.csv_file:\n # FileInput always provides a local file path\n file_path = self.csv_file\n if not file_path.lower().endswith(\".csv\"):\n self.status = \"The provided file must be a CSV file.\"\n else:\n # Resolve to absolute path and read from local filesystem\n resolved_path = self.resolve_path(file_path)\n csv_bytes = Path(resolved_path).read_bytes()\n csv_data = csv_bytes.decode(\"utf-8\")\n\n elif self.csv_path:\n file_path = self.csv_path\n if not file_path.lower().endswith(\".csv\"):\n self.status = \"The provided path must be to a CSV file.\"\n else:\n csv_data = run_until_complete(\n read_file_text(file_path, encoding=\"utf-8\", resolve_path=self.resolve_path, newline=\"\")\n )\n\n else:\n csv_data = self.csv_string\n\n if csv_data:\n csv_reader = csv.DictReader(io.StringIO(csv_data))\n result = [Data(data=row, text_key=self.text_key) for row in csv_reader]\n\n if not result:\n self.status = \"The CSV data is empty.\"\n return []\n\n self.status = result\n return result\n\n except csv.Error as e:\n error_message = f\"CSV parsing error: {e}\"\n self.status = error_message\n raise ValueError(error_message) from e\n\n except Exception as e:\n error_message = f\"An error occurred: {e}\"\n self.status = error_message\n raise ValueError(error_message) from e\n\n # An error occurred\n raise ValueError(self.status)\n" + "value": "import csv\nimport io\nfrom pathlib import Path\n\nfrom lfx.base.data.storage_utils import read_file_text\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.io import FileInput, MessageTextInput, MultilineInput, Output\nfrom lfx.schema.data import Data\nfrom lfx.utils.async_helpers import run_until_complete\nfrom lfx.utils.file_path_security import enforce_local_file_access\n\n\nclass CSVToDataComponent(Component):\n display_name = \"Load CSV\"\n description = \"Load a CSV file, CSV from a file path, or a valid CSV string and convert it to a list of Data\"\n icon = \"file-spreadsheet\"\n name = \"CSVtoData\"\n legacy = True\n replacement = [\"data.File\"]\n\n inputs = [\n FileInput(\n name=\"csv_file\",\n display_name=\"CSV File\",\n file_types=[\"csv\"],\n info=\"Upload a CSV file to convert to a list of Data objects\",\n ),\n MessageTextInput(\n name=\"csv_path\",\n display_name=\"CSV File Path\",\n info=\"Provide the path to the CSV file as pure text\",\n ),\n MultilineInput(\n name=\"csv_string\",\n display_name=\"CSV String\",\n info=\"Paste a CSV string directly to convert to a list of Data objects\",\n ),\n MessageTextInput(\n name=\"text_key\",\n display_name=\"Text Key\",\n info=\"The key to use for the text column. Defaults to 'text'.\",\n value=\"text\",\n ),\n ]\n\n outputs = [\n Output(name=\"data_list\", display_name=\"JSON List\", method=\"load_csv_to_data\"),\n ]\n\n def load_csv_to_data(self) -> list[Data]:\n if sum(bool(field) for field in [self.csv_file, self.csv_path, self.csv_string]) != 1:\n msg = \"Please provide exactly one of: CSV file, file path, or CSV string.\"\n raise ValueError(msg)\n\n csv_data = None\n try:\n if self.csv_file:\n # FileInput always provides a local file path\n file_path = self.csv_file\n if not file_path.lower().endswith(\".csv\"):\n self.status = \"The provided file must be a CSV file.\"\n else:\n # Resolve to absolute path and read from local filesystem\n # (confined to the storage dir in restricted multi-tenant mode).\n resolved_path = enforce_local_file_access(self.resolve_path(file_path))\n csv_bytes = Path(resolved_path).read_bytes()\n csv_data = csv_bytes.decode(\"utf-8\")\n\n elif self.csv_path:\n file_path = self.csv_path\n if not file_path.lower().endswith(\".csv\"):\n self.status = \"The provided path must be to a CSV file.\"\n else:\n\n def _resolve_local_path(p: str) -> str:\n # Confine the resolved path to the storage dir in restricted mode.\n return str(enforce_local_file_access(self.resolve_path(p)))\n\n csv_data = run_until_complete(\n read_file_text(file_path, encoding=\"utf-8\", resolve_path=_resolve_local_path, newline=\"\")\n )\n\n else:\n csv_data = self.csv_string\n\n if csv_data:\n csv_reader = csv.DictReader(io.StringIO(csv_data))\n result = [Data(data=row, text_key=self.text_key) for row in csv_reader]\n\n if not result:\n self.status = \"The CSV data is empty.\"\n return []\n\n self.status = result\n return result\n\n except csv.Error as e:\n error_message = f\"CSV parsing error: {e}\"\n self.status = error_message\n raise ValueError(error_message) from e\n\n except Exception as e:\n error_message = f\"An error occurred: {e}\"\n self.status = error_message\n raise ValueError(error_message) from e\n\n # An error occurred\n raise ValueError(self.status)\n" }, "csv_file": { "_input_type": "FileInput", @@ -58041,7 +58041,7 @@ "icon": "braces", "legacy": true, "metadata": { - "code_hash": "e8d050bde0d0", + "code_hash": "1b2896fabd85", "dependencies": { "dependencies": [ { @@ -58097,7 +58097,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import json\nfrom pathlib import Path\n\nfrom json_repair import repair_json\n\nfrom lfx.base.data.storage_utils import read_file_text\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.io import FileInput, MessageTextInput, MultilineInput, Output\nfrom lfx.schema.data import Data\nfrom lfx.utils.async_helpers import run_until_complete\n\n\nclass JSONToDataComponent(Component):\n display_name = \"Load JSON\"\n description = (\n \"Convert a JSON file, JSON from a file path, or a JSON string to a Data object or a list of Data objects\"\n )\n icon = \"braces\"\n name = \"JSONtoData\"\n legacy = True\n replacement = [\"data.File\"]\n\n inputs = [\n FileInput(\n name=\"json_file\",\n display_name=\"JSON File\",\n file_types=[\"json\"],\n info=\"Upload a JSON file to convert to a Data object or list of Data objects\",\n ),\n MessageTextInput(\n name=\"json_path\",\n display_name=\"JSON File Path\",\n info=\"Provide the path to the JSON file as pure text\",\n ),\n MultilineInput(\n name=\"json_string\",\n display_name=\"JSON String\",\n info=\"Enter a valid JSON string (object or array) to convert to a Data object or list of Data objects\",\n ),\n ]\n\n outputs = [\n Output(name=\"data\", display_name=\"JSON\", method=\"convert_json_to_data\"),\n ]\n\n def convert_json_to_data(self) -> Data | list[Data]:\n if sum(bool(field) for field in [self.json_file, self.json_path, self.json_string]) != 1:\n msg = \"Please provide exactly one of: JSON file, file path, or JSON string.\"\n self.status = msg\n raise ValueError(msg)\n\n json_data = None\n\n try:\n if self.json_file:\n # FileInput always provides a local file path\n file_path = self.json_file\n if not file_path.lower().endswith(\".json\"):\n self.status = \"The provided file must be a JSON file.\"\n else:\n # Resolve to absolute path and read from local filesystem\n resolved_path = self.resolve_path(file_path)\n json_data = Path(resolved_path).read_text(encoding=\"utf-8\")\n\n elif self.json_path:\n # User-provided text path - could be local or S3 key\n file_path = self.json_path\n if not file_path.lower().endswith(\".json\"):\n self.status = \"The provided path must be to a JSON file.\"\n else:\n json_data = run_until_complete(\n read_file_text(file_path, encoding=\"utf-8\", resolve_path=self.resolve_path)\n )\n\n else:\n json_data = self.json_string\n\n if json_data:\n # Try to parse the JSON string\n try:\n parsed_data = json.loads(json_data)\n except json.JSONDecodeError:\n # If JSON parsing fails, try to repair the JSON string\n repaired_json_string = repair_json(json_data)\n parsed_data = json.loads(repaired_json_string)\n\n # Check if the parsed data is a list\n if isinstance(parsed_data, list):\n result = [Data(data=item) for item in parsed_data]\n else:\n result = Data(data=parsed_data)\n self.status = result\n return result\n\n except (json.JSONDecodeError, SyntaxError, ValueError) as e:\n error_message = f\"Invalid JSON or Python literal: {e}\"\n self.status = error_message\n raise ValueError(error_message) from e\n\n except Exception as e:\n error_message = f\"An error occurred: {e}\"\n self.status = error_message\n raise ValueError(error_message) from e\n\n # An error occurred\n raise ValueError(self.status)\n" + "value": "import json\nfrom pathlib import Path\n\nfrom json_repair import repair_json\n\nfrom lfx.base.data.storage_utils import read_file_text\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.io import FileInput, MessageTextInput, MultilineInput, Output\nfrom lfx.schema.data import Data\nfrom lfx.utils.async_helpers import run_until_complete\nfrom lfx.utils.file_path_security import enforce_local_file_access\n\n\nclass JSONToDataComponent(Component):\n display_name = \"Load JSON\"\n description = (\n \"Convert a JSON file, JSON from a file path, or a JSON string to a Data object or a list of Data objects\"\n )\n icon = \"braces\"\n name = \"JSONtoData\"\n legacy = True\n replacement = [\"data.File\"]\n\n inputs = [\n FileInput(\n name=\"json_file\",\n display_name=\"JSON File\",\n file_types=[\"json\"],\n info=\"Upload a JSON file to convert to a Data object or list of Data objects\",\n ),\n MessageTextInput(\n name=\"json_path\",\n display_name=\"JSON File Path\",\n info=\"Provide the path to the JSON file as pure text\",\n ),\n MultilineInput(\n name=\"json_string\",\n display_name=\"JSON String\",\n info=\"Enter a valid JSON string (object or array) to convert to a Data object or list of Data objects\",\n ),\n ]\n\n outputs = [\n Output(name=\"data\", display_name=\"JSON\", method=\"convert_json_to_data\"),\n ]\n\n def convert_json_to_data(self) -> Data | list[Data]:\n if sum(bool(field) for field in [self.json_file, self.json_path, self.json_string]) != 1:\n msg = \"Please provide exactly one of: JSON file, file path, or JSON string.\"\n self.status = msg\n raise ValueError(msg)\n\n json_data = None\n\n try:\n if self.json_file:\n # FileInput always provides a local file path\n file_path = self.json_file\n if not file_path.lower().endswith(\".json\"):\n self.status = \"The provided file must be a JSON file.\"\n else:\n # Resolve to absolute path and read from local filesystem\n # (confined to the storage dir in restricted multi-tenant mode).\n resolved_path = enforce_local_file_access(self.resolve_path(file_path))\n json_data = Path(resolved_path).read_text(encoding=\"utf-8\")\n\n elif self.json_path:\n # User-provided text path - could be local or S3 key\n file_path = self.json_path\n if not file_path.lower().endswith(\".json\"):\n self.status = \"The provided path must be to a JSON file.\"\n else:\n\n def _resolve_local_path(p: str) -> str:\n # Confine the resolved path to the storage dir in restricted mode.\n return str(enforce_local_file_access(self.resolve_path(p)))\n\n json_data = run_until_complete(\n read_file_text(file_path, encoding=\"utf-8\", resolve_path=_resolve_local_path)\n )\n\n else:\n json_data = self.json_string\n\n if json_data:\n # Try to parse the JSON string\n try:\n parsed_data = json.loads(json_data)\n except json.JSONDecodeError:\n # If JSON parsing fails, try to repair the JSON string\n repaired_json_string = repair_json(json_data)\n parsed_data = json.loads(repaired_json_string)\n\n # Check if the parsed data is a list\n if isinstance(parsed_data, list):\n result = [Data(data=item) for item in parsed_data]\n else:\n result = Data(data=parsed_data)\n self.status = result\n return result\n\n except (json.JSONDecodeError, SyntaxError, ValueError) as e:\n error_message = f\"Invalid JSON or Python literal: {e}\"\n self.status = error_message\n raise ValueError(error_message) from e\n\n except Exception as e:\n error_message = f\"An error occurred: {e}\"\n self.status = error_message\n raise ValueError(error_message) from e\n\n # An error occurred\n raise ValueError(self.status)\n" }, "json_file": { "_input_type": "FileInput", @@ -58688,7 +58688,7 @@ "icon": "database", "legacy": false, "metadata": { - "code_hash": "a8dd79af50b8", + "code_hash": "142e40c5968e", "dependencies": { "dependencies": [ { @@ -58772,7 +58772,7 @@ "show": true, "title_case": false, "type": "code", - "value": "from typing import TYPE_CHECKING, Any\n\nfrom langchain_community.utilities import SQLDatabase\nfrom sqlalchemy.exc import SQLAlchemyError\n\nfrom lfx.custom.custom_component.component_with_cache import ComponentWithCache\nfrom lfx.io import BoolInput, MessageTextInput, MultilineInput, Output\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.services.cache.utils import CacheMiss\n\nif TYPE_CHECKING:\n from sqlalchemy.engine import Result\n\n\nclass SQLComponent(ComponentWithCache):\n \"\"\"A sql component.\"\"\"\n\n display_name = \"SQL Database\"\n description = \"Executes SQL queries on SQLAlchemy-compatible databases.\"\n documentation: str = \"https://docs.langflow.org/sql-database\"\n icon = \"database\"\n name = \"SQLComponent\"\n metadata = {\"keywords\": [\"sql\", \"database\", \"query\", \"db\", \"fetch\"]}\n\n def __init__(self, **kwargs) -> None:\n super().__init__(**kwargs)\n self.db: SQLDatabase = None\n\n def maybe_create_db(self):\n if self.database_url != \"\":\n if self._shared_component_cache:\n cached_db = self._shared_component_cache.get(self.database_url)\n if not isinstance(cached_db, CacheMiss):\n self.db = cached_db\n return\n self.log(\"Connecting to database\")\n try:\n self.db = SQLDatabase.from_uri(self.database_url)\n except Exception as e:\n msg = f\"An error occurred while connecting to the database: {e}\"\n raise ValueError(msg) from e\n if self._shared_component_cache:\n self._shared_component_cache.set(self.database_url, self.db)\n\n inputs = [\n MessageTextInput(name=\"database_url\", display_name=\"Database URL\", required=True),\n MultilineInput(name=\"query\", display_name=\"SQL Query\", tool_mode=True, required=True),\n BoolInput(name=\"include_columns\", display_name=\"Include Columns\", value=True, tool_mode=True, advanced=True),\n BoolInput(\n name=\"add_error\",\n display_name=\"Add Error\",\n value=False,\n tool_mode=True,\n info=\"If True, the error will be added to the result\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Result Table\", name=\"run_sql_query\", method=\"run_sql_query\"),\n ]\n\n def build_component(\n self,\n ) -> Message:\n error = None\n self.maybe_create_db()\n try:\n result = self.db.run(self.query, include_columns=self.include_columns)\n self.status = result\n except SQLAlchemyError as e:\n msg = f\"An error occurred while running the SQL Query: {e}\"\n self.log(msg)\n result = str(e)\n self.status = result\n error = repr(e)\n\n if self.add_error and error is not None:\n result = f\"{result}\\n\\nError: {error}\\n\\nQuery: {self.query}\"\n elif error is not None:\n # Then we won't add the error to the result\n result = self.query\n\n return Message(text=result)\n\n def __execute_query(self) -> list[dict[str, Any]]:\n self.maybe_create_db()\n try:\n cursor: Result[Any] = self.db.run(self.query, fetch=\"cursor\")\n return [x._asdict() for x in cursor.fetchall()]\n except SQLAlchemyError as e:\n msg = f\"An error occurred while running the SQL Query: {e}\"\n self.log(msg)\n raise ValueError(msg) from e\n\n def run_sql_query(self) -> DataFrame:\n result = self.__execute_query()\n df_result = DataFrame(result)\n self.status = df_result\n return df_result\n" + "value": "from typing import TYPE_CHECKING, Any\n\nfrom langchain_community.utilities import SQLDatabase\nfrom sqlalchemy.exc import SQLAlchemyError\n\nfrom lfx.custom.custom_component.component_with_cache import ComponentWithCache\nfrom lfx.io import BoolInput, MessageTextInput, MultilineInput, Output\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.services.cache.utils import CacheMiss\nfrom lfx.utils.ssrf_protection import validate_connector_database_url_for_ssrf\n\nif TYPE_CHECKING:\n from sqlalchemy.engine import Result\n\n\nclass SQLComponent(ComponentWithCache):\n \"\"\"A sql component.\"\"\"\n\n display_name = \"SQL Database\"\n description = \"Executes SQL queries on SQLAlchemy-compatible databases.\"\n documentation: str = \"https://docs.langflow.org/sql-database\"\n icon = \"database\"\n name = \"SQLComponent\"\n metadata = {\"keywords\": [\"sql\", \"database\", \"query\", \"db\", \"fetch\"]}\n\n def __init__(self, **kwargs) -> None:\n super().__init__(**kwargs)\n self.db: SQLDatabase = None\n\n def maybe_create_db(self):\n if self.database_url != \"\":\n # Security: a tenant fully controls database_url. Block SSRF to internal\n # databases/services and local-file dialects (sqlite/duckdb -> arbitrary\n # server file read/write) before opening the connection.\n validate_connector_database_url_for_ssrf(self.database_url)\n if self._shared_component_cache:\n cached_db = self._shared_component_cache.get(self.database_url)\n if not isinstance(cached_db, CacheMiss):\n self.db = cached_db\n return\n self.log(\"Connecting to database\")\n try:\n self.db = SQLDatabase.from_uri(self.database_url)\n except Exception as e:\n msg = f\"An error occurred while connecting to the database: {e}\"\n raise ValueError(msg) from e\n if self._shared_component_cache:\n self._shared_component_cache.set(self.database_url, self.db)\n\n inputs = [\n MessageTextInput(name=\"database_url\", display_name=\"Database URL\", required=True),\n MultilineInput(name=\"query\", display_name=\"SQL Query\", tool_mode=True, required=True),\n BoolInput(name=\"include_columns\", display_name=\"Include Columns\", value=True, tool_mode=True, advanced=True),\n BoolInput(\n name=\"add_error\",\n display_name=\"Add Error\",\n value=False,\n tool_mode=True,\n info=\"If True, the error will be added to the result\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Result Table\", name=\"run_sql_query\", method=\"run_sql_query\"),\n ]\n\n def build_component(\n self,\n ) -> Message:\n error = None\n self.maybe_create_db()\n try:\n result = self.db.run(self.query, include_columns=self.include_columns)\n self.status = result\n except SQLAlchemyError as e:\n msg = f\"An error occurred while running the SQL Query: {e}\"\n self.log(msg)\n result = str(e)\n self.status = result\n error = repr(e)\n\n if self.add_error and error is not None:\n result = f\"{result}\\n\\nError: {error}\\n\\nQuery: {self.query}\"\n elif error is not None:\n # Then we won't add the error to the result\n result = self.query\n\n return Message(text=result)\n\n def __execute_query(self) -> list[dict[str, Any]]:\n self.maybe_create_db()\n try:\n cursor: Result[Any] = self.db.run(self.query, fetch=\"cursor\")\n return [x._asdict() for x in cursor.fetchall()]\n except SQLAlchemyError as e:\n msg = f\"An error occurred while running the SQL Query: {e}\"\n self.log(msg)\n raise ValueError(msg) from e\n\n def run_sql_query(self) -> DataFrame:\n result = self.__execute_query()\n df_result = DataFrame(result)\n self.status = df_result\n return df_result\n" }, "database_url": { "_input_type": "MessageTextInput", @@ -59254,7 +59254,7 @@ "icon": "search", "legacy": false, "metadata": { - "code_hash": "cbeeaef8889a", + "code_hash": "3e6dc350b208", "dependencies": { "dependencies": [ { @@ -59338,7 +59338,7 @@ "show": true, "title_case": false, "type": "code", - "value": "\"\"\"Unified Web Search Component.\n\nThis component consolidates Web Search, News Search, and RSS Reader into a single\ncomponent with tabs for different search modes.\n\"\"\"\n\nimport re\nfrom typing import Any\nfrom urllib.parse import parse_qs, quote_plus, unquote, urlparse\n\nimport pandas as pd\nimport requests\nfrom bs4 import BeautifulSoup\n\nfrom lfx.custom import Component\nfrom lfx.io import IntInput, MessageTextInput, Output, TabInput\nfrom lfx.schema import DataFrame\nfrom lfx.utils.request_utils import get_user_agent\n\n\nclass WebSearchComponent(Component):\n display_name = \"Web Search\"\n description = \"Search the web, news, or RSS feeds.\"\n documentation: str = \"https://docs.langflow.org/web-search\"\n icon = \"search\"\n name = \"UnifiedWebSearch\"\n\n inputs = [\n TabInput(\n name=\"search_mode\",\n display_name=\"Search Mode\",\n options=[\"Web\", \"News\", \"RSS\"],\n info=\"Choose search mode: Web (DuckDuckGo), News (Google News), or RSS (Feed Reader)\",\n value=\"Web\",\n real_time_refresh=True,\n tool_mode=True,\n ),\n MessageTextInput(\n name=\"query\",\n display_name=\"Search Query\",\n info=\"Search keywords for news articles.\",\n tool_mode=True,\n required=True,\n ),\n MessageTextInput(\n name=\"hl\",\n display_name=\"Language (hl)\",\n info=\"Language code, e.g. en-US, fr, de. Default: en-US.\",\n tool_mode=False,\n input_types=[],\n required=False,\n advanced=True,\n ),\n MessageTextInput(\n name=\"gl\",\n display_name=\"Country (gl)\",\n info=\"Country code, e.g. US, FR, DE. Default: US.\",\n tool_mode=False,\n input_types=[],\n required=False,\n advanced=True,\n ),\n MessageTextInput(\n name=\"ceid\",\n display_name=\"Country:Language (ceid)\",\n info=\"e.g. US:en, FR:fr. Default: US:en.\",\n tool_mode=False,\n value=\"US:en\",\n input_types=[],\n required=False,\n advanced=True,\n ),\n MessageTextInput(\n name=\"topic\",\n display_name=\"Topic\",\n info=\"One of: WORLD, NATION, BUSINESS, TECHNOLOGY, ENTERTAINMENT, SCIENCE, SPORTS, HEALTH.\",\n tool_mode=False,\n input_types=[],\n required=False,\n advanced=True,\n ),\n MessageTextInput(\n name=\"location\",\n display_name=\"Location (Geo)\",\n info=\"City, state, or country for location-based news. Leave blank for keyword search.\",\n tool_mode=False,\n input_types=[],\n required=False,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=5,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [Output(name=\"results\", display_name=\"Results\", method=\"perform_search\")]\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n \"\"\"Update input visibility based on search mode.\"\"\"\n if field_name == \"search_mode\":\n # Show/hide inputs based on search mode\n is_news = field_value == \"News\"\n is_rss = field_value == \"RSS\"\n\n # Update query field info based on mode\n if is_rss:\n build_config[\"query\"][\"info\"] = \"RSS feed URL to parse\"\n build_config[\"query\"][\"display_name\"] = \"RSS Feed URL\"\n elif is_news:\n build_config[\"query\"][\"info\"] = \"Search keywords for news articles.\"\n build_config[\"query\"][\"display_name\"] = \"Search Query\"\n else: # Web\n build_config[\"query\"][\"info\"] = \"Keywords to search for\"\n build_config[\"query\"][\"display_name\"] = \"Search Query\"\n\n # Keep news-specific fields as advanced (matching original News Search component)\n # They remain advanced=True in all modes, just like in the original component\n\n return build_config\n\n def validate_url(self, string: str) -> bool:\n \"\"\"Validate URL format.\"\"\"\n url_regex = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n )\n return bool(url_regex.match(string))\n\n def ensure_url(self, url: str) -> str:\n \"\"\"Ensure URL has proper protocol.\"\"\"\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n return url\n\n def _sanitize_query(self, query: str) -> str:\n \"\"\"Sanitize search query.\"\"\"\n return re.sub(r'[<>\"\\']', \"\", query.strip())\n\n def clean_html(self, html_string: str) -> str:\n \"\"\"Remove HTML tags from text.\"\"\"\n return BeautifulSoup(html_string, \"html.parser\").get_text(separator=\" \", strip=True)\n\n def perform_web_search(self) -> DataFrame:\n \"\"\"Perform DuckDuckGo web search.\"\"\"\n query = self._sanitize_query(self.query)\n if not query:\n msg = \"Empty search query\"\n raise ValueError(msg)\n\n headers = {\"User-Agent\": get_user_agent()}\n params = {\"q\": query, \"kl\": \"us-en\"}\n url = \"https://html.duckduckgo.com/html/\"\n\n try:\n response = requests.get(url, params=params, headers=headers, timeout=self.timeout)\n response.raise_for_status()\n except requests.RequestException as e:\n self.status = f\"Failed request: {e!s}\"\n return DataFrame(pd.DataFrame([{\"title\": \"Error\", \"link\": \"\", \"snippet\": str(e), \"content\": \"\"}]))\n\n if not response.text or \"text/html\" not in response.headers.get(\"content-type\", \"\").lower():\n self.status = \"No results found\"\n return DataFrame(\n pd.DataFrame([{\"title\": \"Error\", \"link\": \"\", \"snippet\": \"No results found\", \"content\": \"\"}])\n )\n\n soup = BeautifulSoup(response.text, \"html.parser\")\n results = []\n\n for result in soup.select(\"div.result\"):\n title_tag = result.select_one(\"a.result__a\")\n snippet_tag = result.select_one(\"a.result__snippet\")\n if title_tag:\n raw_link = title_tag.get(\"href\", \"\")\n parsed = urlparse(raw_link)\n uddg = parse_qs(parsed.query).get(\"uddg\", [\"\"])[0]\n decoded_link = unquote(uddg) if uddg else raw_link\n\n try:\n final_url = self.ensure_url(decoded_link)\n page = requests.get(final_url, headers=headers, timeout=self.timeout)\n page.raise_for_status()\n content = BeautifulSoup(page.text, \"lxml\").get_text(separator=\" \", strip=True)\n except requests.RequestException as e:\n final_url = decoded_link\n content = f\"(Failed to fetch: {e!s}\"\n\n results.append(\n {\n \"title\": title_tag.get_text(strip=True),\n \"link\": final_url,\n \"snippet\": snippet_tag.get_text(strip=True) if snippet_tag else \"\",\n \"content\": content,\n }\n )\n\n return DataFrame(pd.DataFrame(results))\n\n def perform_news_search(self) -> DataFrame:\n \"\"\"Perform Google News search.\"\"\"\n query = getattr(self, \"query\", \"\")\n hl = getattr(self, \"hl\", \"en-US\") or \"en-US\"\n gl = getattr(self, \"gl\", \"US\") or \"US\"\n topic = getattr(self, \"topic\", None)\n location = getattr(self, \"location\", None)\n\n ceid = f\"{gl}:{hl.split('-')[0]}\"\n\n # Build RSS URL based on parameters\n if topic:\n # Topic-based feed\n base_url = f\"https://news.google.com/rss/headlines/section/topic/{quote_plus(topic.upper())}\"\n params = f\"?hl={hl}&gl={gl}&ceid={ceid}\"\n rss_url = base_url + params\n elif location:\n # Location-based feed\n base_url = f\"https://news.google.com/rss/headlines/section/geo/{quote_plus(location)}\"\n params = f\"?hl={hl}&gl={gl}&ceid={ceid}\"\n rss_url = base_url + params\n elif query:\n # Keyword search feed\n base_url = \"https://news.google.com/rss/search?q=\"\n query_encoded = quote_plus(query)\n params = f\"&hl={hl}&gl={gl}&ceid={ceid}\"\n rss_url = f\"{base_url}{query_encoded}{params}\"\n else:\n self.status = \"No search query, topic, or location provided.\"\n return DataFrame(\n pd.DataFrame(\n [{\"title\": \"Error\", \"link\": \"\", \"published\": \"\", \"summary\": \"No search parameters provided\"}]\n )\n )\n\n try:\n response = requests.get(rss_url, timeout=self.timeout)\n response.raise_for_status()\n soup = BeautifulSoup(response.content, \"xml\")\n items = soup.find_all(\"item\")\n except requests.RequestException as e:\n self.status = f\"Failed to fetch news: {e}\"\n return DataFrame(pd.DataFrame([{\"title\": \"Error\", \"link\": \"\", \"published\": \"\", \"summary\": str(e)}]))\n\n if not items:\n self.status = \"No news articles found.\"\n return DataFrame(pd.DataFrame([{\"title\": \"No articles found\", \"link\": \"\", \"published\": \"\", \"summary\": \"\"}]))\n\n articles = []\n for item in items:\n try:\n title = self.clean_html(item.title.text if item.title else \"\")\n link = item.link.text if item.link else \"\"\n published = item.pubDate.text if item.pubDate else \"\"\n summary = self.clean_html(item.description.text if item.description else \"\")\n articles.append({\"title\": title, \"link\": link, \"published\": published, \"summary\": summary})\n except (AttributeError, ValueError, TypeError) as e:\n self.log(f\"Error parsing article: {e!s}\")\n continue\n\n return DataFrame(pd.DataFrame(articles))\n\n def perform_rss_read(self) -> DataFrame:\n \"\"\"Read RSS feed.\"\"\"\n rss_url = getattr(self, \"query\", \"\")\n if not rss_url:\n return DataFrame(\n pd.DataFrame([{\"title\": \"Error\", \"link\": \"\", \"published\": \"\", \"summary\": \"No RSS URL provided\"}])\n )\n\n try:\n response = requests.get(rss_url, timeout=self.timeout)\n response.raise_for_status()\n if not response.content.strip():\n msg = \"Empty response received\"\n raise ValueError(msg)\n\n # Validate XML\n try:\n BeautifulSoup(response.content, \"xml\")\n except Exception as e:\n msg = f\"Invalid XML response: {e}\"\n raise ValueError(msg) from e\n\n soup = BeautifulSoup(response.content, \"xml\")\n items = soup.find_all(\"item\")\n except (requests.RequestException, ValueError) as e:\n self.status = f\"Failed to fetch RSS: {e}\"\n return DataFrame(pd.DataFrame([{\"title\": \"Error\", \"link\": \"\", \"published\": \"\", \"summary\": str(e)}]))\n\n articles = [\n {\n \"title\": item.title.text if item.title else \"\",\n \"link\": item.link.text if item.link else \"\",\n \"published\": item.pubDate.text if item.pubDate else \"\",\n \"summary\": item.description.text if item.description else \"\",\n }\n for item in items\n ]\n\n # Ensure DataFrame has correct columns even if empty\n df_articles = pd.DataFrame(articles, columns=[\"title\", \"link\", \"published\", \"summary\"])\n self.log(f\"Fetched {len(df_articles)} articles.\")\n return DataFrame(df_articles)\n\n def perform_search(self) -> DataFrame:\n \"\"\"Main search method that routes to appropriate search function based on mode.\"\"\"\n search_mode = getattr(self, \"search_mode\", \"Web\")\n\n if search_mode == \"Web\":\n return self.perform_web_search()\n if search_mode == \"News\":\n return self.perform_news_search()\n if search_mode == \"RSS\":\n return self.perform_rss_read()\n # Fallback to web search\n return self.perform_web_search()\n" + "value": "\"\"\"Unified Web Search Component.\n\nThis component consolidates Web Search, News Search, and RSS Reader into a single\ncomponent with tabs for different search modes.\n\"\"\"\n\nimport re\nfrom typing import Any\nfrom urllib.parse import parse_qs, quote_plus, unquote, urlparse\n\nimport pandas as pd\nimport requests\nfrom bs4 import BeautifulSoup\n\nfrom lfx.custom import Component\nfrom lfx.io import IntInput, MessageTextInput, Output, TabInput\nfrom lfx.schema import DataFrame\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, validate_url_for_ssrf\n\n\nclass WebSearchComponent(Component):\n display_name = \"Web Search\"\n description = \"Search the web, news, or RSS feeds.\"\n documentation: str = \"https://docs.langflow.org/web-search\"\n icon = \"search\"\n name = \"UnifiedWebSearch\"\n\n inputs = [\n TabInput(\n name=\"search_mode\",\n display_name=\"Search Mode\",\n options=[\"Web\", \"News\", \"RSS\"],\n info=\"Choose search mode: Web (DuckDuckGo), News (Google News), or RSS (Feed Reader)\",\n value=\"Web\",\n real_time_refresh=True,\n tool_mode=True,\n ),\n MessageTextInput(\n name=\"query\",\n display_name=\"Search Query\",\n info=\"Search keywords for news articles.\",\n tool_mode=True,\n required=True,\n ),\n MessageTextInput(\n name=\"hl\",\n display_name=\"Language (hl)\",\n info=\"Language code, e.g. en-US, fr, de. Default: en-US.\",\n tool_mode=False,\n input_types=[],\n required=False,\n advanced=True,\n ),\n MessageTextInput(\n name=\"gl\",\n display_name=\"Country (gl)\",\n info=\"Country code, e.g. US, FR, DE. Default: US.\",\n tool_mode=False,\n input_types=[],\n required=False,\n advanced=True,\n ),\n MessageTextInput(\n name=\"ceid\",\n display_name=\"Country:Language (ceid)\",\n info=\"e.g. US:en, FR:fr. Default: US:en.\",\n tool_mode=False,\n value=\"US:en\",\n input_types=[],\n required=False,\n advanced=True,\n ),\n MessageTextInput(\n name=\"topic\",\n display_name=\"Topic\",\n info=\"One of: WORLD, NATION, BUSINESS, TECHNOLOGY, ENTERTAINMENT, SCIENCE, SPORTS, HEALTH.\",\n tool_mode=False,\n input_types=[],\n required=False,\n advanced=True,\n ),\n MessageTextInput(\n name=\"location\",\n display_name=\"Location (Geo)\",\n info=\"City, state, or country for location-based news. Leave blank for keyword search.\",\n tool_mode=False,\n input_types=[],\n required=False,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=5,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [Output(name=\"results\", display_name=\"Results\", method=\"perform_search\")]\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n \"\"\"Update input visibility based on search mode.\"\"\"\n if field_name == \"search_mode\":\n # Show/hide inputs based on search mode\n is_news = field_value == \"News\"\n is_rss = field_value == \"RSS\"\n\n # Update query field info based on mode\n if is_rss:\n build_config[\"query\"][\"info\"] = \"RSS feed URL to parse\"\n build_config[\"query\"][\"display_name\"] = \"RSS Feed URL\"\n elif is_news:\n build_config[\"query\"][\"info\"] = \"Search keywords for news articles.\"\n build_config[\"query\"][\"display_name\"] = \"Search Query\"\n else: # Web\n build_config[\"query\"][\"info\"] = \"Keywords to search for\"\n build_config[\"query\"][\"display_name\"] = \"Search Query\"\n\n # Keep news-specific fields as advanced (matching original News Search component)\n # They remain advanced=True in all modes, just like in the original component\n\n return build_config\n\n def validate_url(self, string: str) -> bool:\n \"\"\"Validate URL format.\"\"\"\n url_regex = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n )\n return bool(url_regex.match(string))\n\n def ensure_url(self, url: str) -> str:\n \"\"\"Ensure URL has proper protocol.\"\"\"\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n return url\n\n def _sanitize_query(self, query: str) -> str:\n \"\"\"Sanitize search query.\"\"\"\n return re.sub(r'[<>\"\\']', \"\", query.strip())\n\n def clean_html(self, html_string: str) -> str:\n \"\"\"Remove HTML tags from text.\"\"\"\n return BeautifulSoup(html_string, \"html.parser\").get_text(separator=\" \", strip=True)\n\n def perform_web_search(self) -> DataFrame:\n \"\"\"Perform DuckDuckGo web search.\"\"\"\n query = self._sanitize_query(self.query)\n if not query:\n msg = \"Empty search query\"\n raise ValueError(msg)\n\n headers = {\"User-Agent\": get_user_agent()}\n params = {\"q\": query, \"kl\": \"us-en\"}\n url = \"https://html.duckduckgo.com/html/\"\n\n try:\n response = requests.get(url, params=params, headers=headers, timeout=self.timeout)\n response.raise_for_status()\n except requests.RequestException as e:\n self.status = f\"Failed request: {e!s}\"\n return DataFrame(pd.DataFrame([{\"title\": \"Error\", \"link\": \"\", \"snippet\": str(e), \"content\": \"\"}]))\n\n if not response.text or \"text/html\" not in response.headers.get(\"content-type\", \"\").lower():\n self.status = \"No results found\"\n return DataFrame(\n pd.DataFrame([{\"title\": \"Error\", \"link\": \"\", \"snippet\": \"No results found\", \"content\": \"\"}])\n )\n\n soup = BeautifulSoup(response.text, \"html.parser\")\n results = []\n\n for result in soup.select(\"div.result\"):\n title_tag = result.select_one(\"a.result__a\")\n snippet_tag = result.select_one(\"a.result__snippet\")\n if title_tag:\n raw_link = title_tag.get(\"href\", \"\")\n parsed = urlparse(raw_link)\n uddg = parse_qs(parsed.query).get(\"uddg\", [\"\"])[0]\n decoded_link = unquote(uddg) if uddg else raw_link\n\n try:\n final_url = self.ensure_url(decoded_link)\n # Security: result links are followed server-side; block SSRF to\n # internal/metadata endpoints before fetching page content.\n validate_url_for_ssrf(final_url)\n # allow_redirects=False: validation only covers the initial host, so a 3xx\n # to an internal/metadata host would otherwise bypass the SSRF guard.\n page = requests.get(final_url, headers=headers, timeout=self.timeout, allow_redirects=False)\n page.raise_for_status()\n content = BeautifulSoup(page.text, \"lxml\").get_text(separator=\" \", strip=True)\n except SSRFProtectionError as e:\n final_url = decoded_link\n content = f\"(Blocked by SSRF protection: {e!s})\"\n except requests.RequestException as e:\n final_url = decoded_link\n content = f\"(Failed to fetch: {e!s}\"\n\n results.append(\n {\n \"title\": title_tag.get_text(strip=True),\n \"link\": final_url,\n \"snippet\": snippet_tag.get_text(strip=True) if snippet_tag else \"\",\n \"content\": content,\n }\n )\n\n return DataFrame(pd.DataFrame(results))\n\n def perform_news_search(self) -> DataFrame:\n \"\"\"Perform Google News search.\"\"\"\n query = getattr(self, \"query\", \"\")\n hl = getattr(self, \"hl\", \"en-US\") or \"en-US\"\n gl = getattr(self, \"gl\", \"US\") or \"US\"\n topic = getattr(self, \"topic\", None)\n location = getattr(self, \"location\", None)\n\n ceid = f\"{gl}:{hl.split('-')[0]}\"\n\n # Build RSS URL based on parameters\n if topic:\n # Topic-based feed\n base_url = f\"https://news.google.com/rss/headlines/section/topic/{quote_plus(topic.upper())}\"\n params = f\"?hl={hl}&gl={gl}&ceid={ceid}\"\n rss_url = base_url + params\n elif location:\n # Location-based feed\n base_url = f\"https://news.google.com/rss/headlines/section/geo/{quote_plus(location)}\"\n params = f\"?hl={hl}&gl={gl}&ceid={ceid}\"\n rss_url = base_url + params\n elif query:\n # Keyword search feed\n base_url = \"https://news.google.com/rss/search?q=\"\n query_encoded = quote_plus(query)\n params = f\"&hl={hl}&gl={gl}&ceid={ceid}\"\n rss_url = f\"{base_url}{query_encoded}{params}\"\n else:\n self.status = \"No search query, topic, or location provided.\"\n return DataFrame(\n pd.DataFrame(\n [{\"title\": \"Error\", \"link\": \"\", \"published\": \"\", \"summary\": \"No search parameters provided\"}]\n )\n )\n\n try:\n response = requests.get(rss_url, timeout=self.timeout)\n response.raise_for_status()\n soup = BeautifulSoup(response.content, \"xml\")\n items = soup.find_all(\"item\")\n except requests.RequestException as e:\n self.status = f\"Failed to fetch news: {e}\"\n return DataFrame(pd.DataFrame([{\"title\": \"Error\", \"link\": \"\", \"published\": \"\", \"summary\": str(e)}]))\n\n if not items:\n self.status = \"No news articles found.\"\n return DataFrame(pd.DataFrame([{\"title\": \"No articles found\", \"link\": \"\", \"published\": \"\", \"summary\": \"\"}]))\n\n articles = []\n for item in items:\n try:\n title = self.clean_html(item.title.text if item.title else \"\")\n link = item.link.text if item.link else \"\"\n published = item.pubDate.text if item.pubDate else \"\"\n summary = self.clean_html(item.description.text if item.description else \"\")\n articles.append({\"title\": title, \"link\": link, \"published\": published, \"summary\": summary})\n except (AttributeError, ValueError, TypeError) as e:\n self.log(f\"Error parsing article: {e!s}\")\n continue\n\n return DataFrame(pd.DataFrame(articles))\n\n def perform_rss_read(self) -> DataFrame:\n \"\"\"Read RSS feed.\"\"\"\n rss_url = getattr(self, \"query\", \"\")\n if not rss_url:\n return DataFrame(\n pd.DataFrame([{\"title\": \"Error\", \"link\": \"\", \"published\": \"\", \"summary\": \"No RSS URL provided\"}])\n )\n\n try:\n # Security: rss_url is fully tenant-controlled. Block SSRF to internal/metadata\n # endpoints before fetching (SSRFProtectionError is a ValueError, caught below).\n validate_url_for_ssrf(rss_url)\n # allow_redirects=False: validation only covers the initial host, so a 3xx to an\n # internal/metadata host would otherwise bypass the SSRF guard.\n response = requests.get(rss_url, timeout=self.timeout, allow_redirects=False)\n response.raise_for_status()\n if not response.content.strip():\n msg = \"Empty response received\"\n raise ValueError(msg)\n\n # Validate XML\n try:\n BeautifulSoup(response.content, \"xml\")\n except Exception as e:\n msg = f\"Invalid XML response: {e}\"\n raise ValueError(msg) from e\n\n soup = BeautifulSoup(response.content, \"xml\")\n items = soup.find_all(\"item\")\n except (requests.RequestException, ValueError) as e:\n self.status = f\"Failed to fetch RSS: {e}\"\n return DataFrame(pd.DataFrame([{\"title\": \"Error\", \"link\": \"\", \"published\": \"\", \"summary\": str(e)}]))\n\n articles = [\n {\n \"title\": item.title.text if item.title else \"\",\n \"link\": item.link.text if item.link else \"\",\n \"published\": item.pubDate.text if item.pubDate else \"\",\n \"summary\": item.description.text if item.description else \"\",\n }\n for item in items\n ]\n\n # Ensure DataFrame has correct columns even if empty\n df_articles = pd.DataFrame(articles, columns=[\"title\", \"link\", \"published\", \"summary\"])\n self.log(f\"Fetched {len(df_articles)} articles.\")\n return DataFrame(df_articles)\n\n def perform_search(self) -> DataFrame:\n \"\"\"Main search method that routes to appropriate search function based on mode.\"\"\"\n search_mode = getattr(self, \"search_mode\", \"Web\")\n\n if search_mode == \"Web\":\n return self.perform_web_search()\n if search_mode == \"News\":\n return self.perform_news_search()\n if search_mode == \"RSS\":\n return self.perform_rss_read()\n # Fallback to web search\n return self.perform_web_search()\n" }, "gl": { "_input_type": "MessageTextInput", @@ -60431,7 +60431,7 @@ "icon": "AstraDB", "legacy": false, "metadata": { - "code_hash": "70c4523f841d", + "code_hash": "22a79e7129ad", "dependencies": { "dependencies": [ { @@ -60570,7 +60570,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import json\nimport urllib\nfrom datetime import datetime, timezone\nfrom http import HTTPStatus\nfrom typing import Any\n\nimport requests\nfrom langchain_core.tools import StructuredTool, Tool\nfrom pydantic import BaseModel, Field, create_model\n\nfrom lfx.base.datastax.astradb_base import AstraDBBaseComponent\nfrom lfx.base.langchain_utilities.model import LCToolComponent\nfrom lfx.io import DictInput, IntInput, StrInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.table import EditMode\n\n\nclass AstraDBCQLToolComponent(AstraDBBaseComponent, LCToolComponent):\n display_name: str = \"Astra DB CQL\"\n description: str = \"Create a tool to get transactional data from DataStax Astra DB CQL Table\"\n documentation: str = \"https://docs.langflow.org/bundles-datastax\"\n icon: str = \"AstraDB\"\n\n inputs = [\n *AstraDBBaseComponent.inputs,\n StrInput(name=\"tool_name\", display_name=\"Tool Name\", info=\"The name of the tool.\", required=True),\n StrInput(\n name=\"tool_description\",\n display_name=\"Tool Description\",\n info=\"The tool description to be passed to the model.\",\n required=True,\n ),\n StrInput(\n name=\"projection_fields\",\n display_name=\"Projection fields\",\n info=\"Attributes to return separated by comma.\",\n required=True,\n value=\"*\",\n advanced=True,\n ),\n TableInput(\n name=\"tools_params\",\n display_name=\"Tools Parameters\",\n info=\"Define the structure for the tool parameters. Describe the parameters \"\n \"in a way the LLM can understand how to use them. Add the parameters \"\n \"respecting the table schema (Partition Keys, Clustering Keys and Indexed Fields).\",\n required=False,\n table_schema=[\n {\n \"name\": \"name\",\n \"display_name\": \"Name\",\n \"type\": \"str\",\n \"description\": \"Name of the field/parameter to be used by the model.\",\n \"default\": \"field\",\n \"edit_mode\": EditMode.INLINE,\n },\n {\n \"name\": \"field_name\",\n \"display_name\": \"Field Name\",\n \"type\": \"str\",\n \"description\": \"Specify the column name to be filtered on the table. \"\n \"Leave empty if the attribute name is the same as the name of the field.\",\n \"default\": \"\",\n \"edit_mode\": EditMode.INLINE,\n },\n {\n \"name\": \"description\",\n \"display_name\": \"Description\",\n \"type\": \"str\",\n \"description\": \"Describe the purpose of the parameter.\",\n \"default\": \"description of tool parameter\",\n \"edit_mode\": EditMode.POPOVER,\n },\n {\n \"name\": \"mandatory\",\n \"display_name\": \"Is Mandatory\",\n \"type\": \"boolean\",\n \"edit_mode\": EditMode.INLINE,\n \"description\": (\"Indicate if the field is mandatory.\"),\n \"options\": [\"True\", \"False\"],\n \"default\": \"False\",\n },\n {\n \"name\": \"is_timestamp\",\n \"display_name\": \"Is Timestamp\",\n \"type\": \"boolean\",\n \"edit_mode\": EditMode.INLINE,\n \"description\": (\"Indicate if the field is a timestamp.\"),\n \"options\": [\"True\", \"False\"],\n \"default\": \"False\",\n },\n {\n \"name\": \"operator\",\n \"display_name\": \"Operator\",\n \"type\": \"str\",\n \"description\": \"Set the operator for the field. \"\n \"https://docs.datastax.com/en/astra-db-serverless/api-reference/documents.html#operators\",\n \"default\": \"$eq\",\n \"options\": [\"$gt\", \"$gte\", \"$lt\", \"$lte\", \"$eq\", \"$ne\", \"$in\", \"$nin\", \"$exists\", \"$all\", \"$size\"],\n \"edit_mode\": EditMode.INLINE,\n },\n ],\n value=[],\n ),\n DictInput(\n name=\"partition_keys\",\n display_name=\"DEPRECATED: Partition Keys\",\n is_list=True,\n info=\"Field name and description to the model\",\n required=False,\n advanced=True,\n ),\n DictInput(\n name=\"clustering_keys\",\n display_name=\"DEPRECATED: Clustering Keys\",\n is_list=True,\n info=\"Field name and description to the model\",\n required=False,\n advanced=True,\n ),\n DictInput(\n name=\"static_filters\",\n display_name=\"Static Filters\",\n is_list=True,\n advanced=True,\n info=\"Field name and value. When filled, it will not be generated by the LLM.\",\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Number of Results\",\n info=\"Number of results to return.\",\n advanced=True,\n value=5,\n ),\n ]\n\n def parse_timestamp(self, timestamp_str: str) -> str:\n \"\"\"Parse a timestamp string into Astra DB REST API format.\n\n Args:\n timestamp_str (str): Input timestamp string\n\n Returns:\n str: Formatted timestamp string in YYYY-MM-DDTHH:MI:SS.000Z format\n\n Raises:\n ValueError: If the timestamp cannot be parsed\n \"\"\"\n # Common datetime formats to try\n formats = [\n \"%Y-%m-%d\", # 2024-03-21\n \"%Y-%m-%dT%H:%M:%S\", # 2024-03-21T15:30:00\n \"%Y-%m-%dT%H:%M:%S%z\", # 2024-03-21T15:30:00+0000\n \"%Y-%m-%d %H:%M:%S\", # 2024-03-21 15:30:00\n \"%d/%m/%Y\", # 21/03/2024\n \"%Y/%m/%d\", # 2024/03/21\n ]\n\n for fmt in formats:\n try:\n # Parse the date string\n date_obj = datetime.strptime(timestamp_str, fmt).astimezone()\n\n # If the parsed date has no timezone info, assume UTC\n if date_obj.tzinfo is None:\n date_obj = date_obj.replace(tzinfo=timezone.utc)\n\n # Convert to UTC and format\n utc_date = date_obj.astimezone(timezone.utc)\n return utc_date.strftime(\"%Y-%m-%dT%H:%M:%S.000Z\")\n except ValueError:\n continue\n\n msg = f\"Could not parse date: {timestamp_str}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def astra_rest(self, args):\n headers = {\"Accept\": \"application/json\", \"X-Cassandra-Token\": f\"{self.token}\"}\n astra_url = f\"{self.get_api_endpoint()}/api/rest/v2/keyspaces/{self.get_keyspace()}/{self.collection_name}/\"\n where = {}\n\n for param in self.tools_params:\n field_name = param[\"field_name\"] if param[\"field_name\"] else param[\"name\"]\n field_value = None\n\n if field_name in self.static_filters:\n field_value = self.static_filters[field_name]\n elif param[\"name\"] in args:\n field_value = args[param[\"name\"]]\n\n if field_value is None:\n continue\n\n if param[\"is_timestamp\"] == True: # noqa: E712\n try:\n field_value = self.parse_timestamp(field_value)\n except ValueError as e:\n msg = f\"Error parsing timestamp: {e} - Use the prompt to specify the date in the correct format\"\n logger.error(msg)\n raise ValueError(msg) from e\n\n if param[\"operator\"] == \"$exists\":\n where[field_name] = {**where.get(field_name, {}), param[\"operator\"]: True}\n elif param[\"operator\"] in [\"$in\", \"$nin\", \"$all\"]:\n where[field_name] = {\n **where.get(field_name, {}),\n param[\"operator\"]: field_value.split(\",\") if isinstance(field_value, str) else field_value,\n }\n else:\n where[field_name] = {**where.get(field_name, {}), param[\"operator\"]: field_value}\n\n url = f\"{astra_url}?page-size={self.number_of_results}\"\n url += f\"&where={json.dumps(where)}\"\n\n if self.projection_fields != \"*\":\n url += f\"&fields={urllib.parse.quote(self.projection_fields.replace(' ', ''))}\"\n\n res = requests.request(\"GET\", url=url, headers=headers, timeout=10)\n\n if int(res.status_code) >= HTTPStatus.BAD_REQUEST:\n msg = f\"Error on Astra DB CQL Tool {self.tool_name} request: {res.text}\"\n logger.error(msg)\n raise ValueError(msg)\n\n try:\n res_data = res.json()\n return res_data[\"data\"]\n except ValueError:\n return res.status_code\n\n def create_args_schema(self) -> dict[str, BaseModel]:\n args: dict[str, tuple[Any, Field]] = {}\n\n for param in self.tools_params:\n field_name = param[\"field_name\"] if param[\"field_name\"] else param[\"name\"]\n if field_name not in self.static_filters:\n if param[\"mandatory\"]:\n args[param[\"name\"]] = (str, Field(description=param[\"description\"]))\n else:\n args[param[\"name\"]] = (str | None, Field(description=param[\"description\"], default=None))\n\n model = create_model(\"ToolInput\", **args, __base__=BaseModel)\n return {\"ToolInput\": model}\n\n def build_tool(self) -> Tool:\n \"\"\"Builds a Astra DB CQL Table tool.\n\n Args:\n name (str, optional): The name of the tool.\n\n Returns:\n Tool: The built Astra DB tool.\n \"\"\"\n schema_dict = self.create_args_schema()\n return StructuredTool.from_function(\n name=self.tool_name,\n args_schema=schema_dict[\"ToolInput\"],\n description=self.tool_description,\n func=self.run_model,\n return_direct=False,\n )\n\n def projection_args(self, input_str: str) -> dict:\n elements = input_str.split(\",\")\n result = {}\n\n for element in elements:\n if element.startswith(\"!\"):\n result[element[1:]] = False\n else:\n result[element] = True\n\n return result\n\n def run_model(self, **args) -> Data | list[Data]:\n results = self.astra_rest(args)\n data: list[Data] = []\n\n if isinstance(results, list):\n data = [Data(data=doc) for doc in results]\n else:\n self.status = results\n return []\n\n self.status = data\n return data\n" + "value": "import json\nimport urllib\nfrom datetime import datetime, timezone\nfrom http import HTTPStatus\nfrom typing import Any\n\nimport requests\nfrom langchain_core.tools import StructuredTool, Tool\nfrom pydantic import BaseModel, Field, create_model\n\nfrom lfx.base.datastax.astradb_base import AstraDBBaseComponent\nfrom lfx.base.langchain_utilities.model import LCToolComponent\nfrom lfx.io import DictInput, IntInput, StrInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.table import EditMode\nfrom lfx.utils.ssrf_protection import validate_connector_url_for_ssrf\n\n\nclass AstraDBCQLToolComponent(AstraDBBaseComponent, LCToolComponent):\n display_name: str = \"Astra DB CQL\"\n description: str = \"Create a tool to get transactional data from DataStax Astra DB CQL Table\"\n documentation: str = \"https://docs.langflow.org/bundles-datastax\"\n icon: str = \"AstraDB\"\n\n inputs = [\n *AstraDBBaseComponent.inputs,\n StrInput(name=\"tool_name\", display_name=\"Tool Name\", info=\"The name of the tool.\", required=True),\n StrInput(\n name=\"tool_description\",\n display_name=\"Tool Description\",\n info=\"The tool description to be passed to the model.\",\n required=True,\n ),\n StrInput(\n name=\"projection_fields\",\n display_name=\"Projection fields\",\n info=\"Attributes to return separated by comma.\",\n required=True,\n value=\"*\",\n advanced=True,\n ),\n TableInput(\n name=\"tools_params\",\n display_name=\"Tools Parameters\",\n info=\"Define the structure for the tool parameters. Describe the parameters \"\n \"in a way the LLM can understand how to use them. Add the parameters \"\n \"respecting the table schema (Partition Keys, Clustering Keys and Indexed Fields).\",\n required=False,\n table_schema=[\n {\n \"name\": \"name\",\n \"display_name\": \"Name\",\n \"type\": \"str\",\n \"description\": \"Name of the field/parameter to be used by the model.\",\n \"default\": \"field\",\n \"edit_mode\": EditMode.INLINE,\n },\n {\n \"name\": \"field_name\",\n \"display_name\": \"Field Name\",\n \"type\": \"str\",\n \"description\": \"Specify the column name to be filtered on the table. \"\n \"Leave empty if the attribute name is the same as the name of the field.\",\n \"default\": \"\",\n \"edit_mode\": EditMode.INLINE,\n },\n {\n \"name\": \"description\",\n \"display_name\": \"Description\",\n \"type\": \"str\",\n \"description\": \"Describe the purpose of the parameter.\",\n \"default\": \"description of tool parameter\",\n \"edit_mode\": EditMode.POPOVER,\n },\n {\n \"name\": \"mandatory\",\n \"display_name\": \"Is Mandatory\",\n \"type\": \"boolean\",\n \"edit_mode\": EditMode.INLINE,\n \"description\": (\"Indicate if the field is mandatory.\"),\n \"options\": [\"True\", \"False\"],\n \"default\": \"False\",\n },\n {\n \"name\": \"is_timestamp\",\n \"display_name\": \"Is Timestamp\",\n \"type\": \"boolean\",\n \"edit_mode\": EditMode.INLINE,\n \"description\": (\"Indicate if the field is a timestamp.\"),\n \"options\": [\"True\", \"False\"],\n \"default\": \"False\",\n },\n {\n \"name\": \"operator\",\n \"display_name\": \"Operator\",\n \"type\": \"str\",\n \"description\": \"Set the operator for the field. \"\n \"https://docs.datastax.com/en/astra-db-serverless/api-reference/documents.html#operators\",\n \"default\": \"$eq\",\n \"options\": [\"$gt\", \"$gte\", \"$lt\", \"$lte\", \"$eq\", \"$ne\", \"$in\", \"$nin\", \"$exists\", \"$all\", \"$size\"],\n \"edit_mode\": EditMode.INLINE,\n },\n ],\n value=[],\n ),\n DictInput(\n name=\"partition_keys\",\n display_name=\"DEPRECATED: Partition Keys\",\n is_list=True,\n info=\"Field name and description to the model\",\n required=False,\n advanced=True,\n ),\n DictInput(\n name=\"clustering_keys\",\n display_name=\"DEPRECATED: Clustering Keys\",\n is_list=True,\n info=\"Field name and description to the model\",\n required=False,\n advanced=True,\n ),\n DictInput(\n name=\"static_filters\",\n display_name=\"Static Filters\",\n is_list=True,\n advanced=True,\n info=\"Field name and value. When filled, it will not be generated by the LLM.\",\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Number of Results\",\n info=\"Number of results to return.\",\n advanced=True,\n value=5,\n ),\n ]\n\n def parse_timestamp(self, timestamp_str: str) -> str:\n \"\"\"Parse a timestamp string into Astra DB REST API format.\n\n Args:\n timestamp_str (str): Input timestamp string\n\n Returns:\n str: Formatted timestamp string in YYYY-MM-DDTHH:MI:SS.000Z format\n\n Raises:\n ValueError: If the timestamp cannot be parsed\n \"\"\"\n # Common datetime formats to try\n formats = [\n \"%Y-%m-%d\", # 2024-03-21\n \"%Y-%m-%dT%H:%M:%S\", # 2024-03-21T15:30:00\n \"%Y-%m-%dT%H:%M:%S%z\", # 2024-03-21T15:30:00+0000\n \"%Y-%m-%d %H:%M:%S\", # 2024-03-21 15:30:00\n \"%d/%m/%Y\", # 21/03/2024\n \"%Y/%m/%d\", # 2024/03/21\n ]\n\n for fmt in formats:\n try:\n # Parse the date string\n date_obj = datetime.strptime(timestamp_str, fmt).astimezone()\n\n # If the parsed date has no timezone info, assume UTC\n if date_obj.tzinfo is None:\n date_obj = date_obj.replace(tzinfo=timezone.utc)\n\n # Convert to UTC and format\n utc_date = date_obj.astimezone(timezone.utc)\n return utc_date.strftime(\"%Y-%m-%dT%H:%M:%S.000Z\")\n except ValueError:\n continue\n\n msg = f\"Could not parse date: {timestamp_str}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def astra_rest(self, args):\n headers = {\"Accept\": \"application/json\", \"X-Cassandra-Token\": f\"{self.token}\"}\n astra_url = f\"{self.get_api_endpoint()}/api/rest/v2/keyspaces/{self.get_keyspace()}/{self.collection_name}/\"\n # api_endpoint is tenant-controlled: block SSRF to internal/cloud-metadata hosts before\n # the Cassandra token is sent to it.\n validate_connector_url_for_ssrf(astra_url)\n where = {}\n\n for param in self.tools_params:\n field_name = param[\"field_name\"] if param[\"field_name\"] else param[\"name\"]\n field_value = None\n\n if field_name in self.static_filters:\n field_value = self.static_filters[field_name]\n elif param[\"name\"] in args:\n field_value = args[param[\"name\"]]\n\n if field_value is None:\n continue\n\n if param[\"is_timestamp\"] == True: # noqa: E712\n try:\n field_value = self.parse_timestamp(field_value)\n except ValueError as e:\n msg = f\"Error parsing timestamp: {e} - Use the prompt to specify the date in the correct format\"\n logger.error(msg)\n raise ValueError(msg) from e\n\n if param[\"operator\"] == \"$exists\":\n where[field_name] = {**where.get(field_name, {}), param[\"operator\"]: True}\n elif param[\"operator\"] in [\"$in\", \"$nin\", \"$all\"]:\n where[field_name] = {\n **where.get(field_name, {}),\n param[\"operator\"]: field_value.split(\",\") if isinstance(field_value, str) else field_value,\n }\n else:\n where[field_name] = {**where.get(field_name, {}), param[\"operator\"]: field_value}\n\n url = f\"{astra_url}?page-size={self.number_of_results}\"\n url += f\"&where={json.dumps(where)}\"\n\n if self.projection_fields != \"*\":\n url += f\"&fields={urllib.parse.quote(self.projection_fields.replace(' ', ''))}\"\n\n res = requests.request(\"GET\", url=url, headers=headers, timeout=10, allow_redirects=False)\n\n if int(res.status_code) >= HTTPStatus.BAD_REQUEST:\n msg = f\"Error on Astra DB CQL Tool {self.tool_name} request: {res.text}\"\n logger.error(msg)\n raise ValueError(msg)\n\n try:\n res_data = res.json()\n return res_data[\"data\"]\n except ValueError:\n return res.status_code\n\n def create_args_schema(self) -> dict[str, BaseModel]:\n args: dict[str, tuple[Any, Field]] = {}\n\n for param in self.tools_params:\n field_name = param[\"field_name\"] if param[\"field_name\"] else param[\"name\"]\n if field_name not in self.static_filters:\n if param[\"mandatory\"]:\n args[param[\"name\"]] = (str, Field(description=param[\"description\"]))\n else:\n args[param[\"name\"]] = (str | None, Field(description=param[\"description\"], default=None))\n\n model = create_model(\"ToolInput\", **args, __base__=BaseModel)\n return {\"ToolInput\": model}\n\n def build_tool(self) -> Tool:\n \"\"\"Builds a Astra DB CQL Table tool.\n\n Args:\n name (str, optional): The name of the tool.\n\n Returns:\n Tool: The built Astra DB tool.\n \"\"\"\n schema_dict = self.create_args_schema()\n return StructuredTool.from_function(\n name=self.tool_name,\n args_schema=schema_dict[\"ToolInput\"],\n description=self.tool_description,\n func=self.run_model,\n return_direct=False,\n )\n\n def projection_args(self, input_str: str) -> dict:\n elements = input_str.split(\",\")\n result = {}\n\n for element in elements:\n if element.startswith(\"!\"):\n result[element[1:]] = False\n else:\n result[element] = True\n\n return result\n\n def run_model(self, **args) -> Data | list[Data]:\n results = self.astra_rest(args)\n data: list[Data] = []\n\n if isinstance(results, list):\n data = [Data(data=doc) for doc in results]\n else:\n self.status = results\n return []\n\n self.status = data\n return data\n" }, "collection_name": { "_input_type": "DropdownInput", @@ -65098,7 +65098,7 @@ "icon": "DeepSeek", "legacy": false, "metadata": { - "code_hash": "c8dac7a258d7", + "code_hash": "23e1a496741a", "dependencies": { "dependencies": [ { @@ -65227,7 +65227,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import requests\nfrom pydantic.v1 import SecretStr\nfrom typing_extensions import override\n\nfrom lfx.base.models.model import LCModelComponent\nfrom lfx.field_typing import LanguageModel\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.inputs.inputs import BoolInput, DictInput, DropdownInput, IntInput, SecretStrInput, SliderInput, StrInput\n\nDEEPSEEK_MODELS = [\"deepseek-chat\"]\n\n\nclass DeepSeekModelComponent(LCModelComponent):\n display_name = \"DeepSeek\"\n description = \"Generate text using DeepSeek LLMs.\"\n icon = \"DeepSeek\"\n\n inputs = [\n *LCModelComponent.get_base_inputs(),\n IntInput(\n name=\"max_tokens\",\n display_name=\"Max Tokens\",\n advanced=True,\n info=\"Maximum number of tokens to generate. Set to 0 for unlimited.\",\n range_spec=RangeSpec(min=0, max=128000),\n ),\n DictInput(\n name=\"model_kwargs\",\n display_name=\"Model Kwargs\",\n advanced=True,\n info=\"Additional keyword arguments to pass to the model.\",\n ),\n BoolInput(\n name=\"json_mode\",\n display_name=\"JSON Mode\",\n advanced=True,\n info=\"If True, it will output JSON regardless of passing a schema.\",\n ),\n DropdownInput(\n name=\"model_name\",\n display_name=\"Model Name\",\n info=\"DeepSeek model to use\",\n options=DEEPSEEK_MODELS,\n value=\"deepseek-chat\",\n refresh_button=True,\n ),\n StrInput(\n name=\"api_base\",\n display_name=\"DeepSeek API Base\",\n advanced=True,\n info=\"Base URL for API requests. Defaults to https://api.deepseek.com\",\n value=\"https://api.deepseek.com\",\n ),\n SecretStrInput(\n name=\"api_key\",\n display_name=\"DeepSeek API Key\",\n info=\"The DeepSeek API Key\",\n advanced=False,\n required=True,\n ),\n SliderInput(\n name=\"temperature\",\n display_name=\"Temperature\",\n info=\"Controls randomness in responses\",\n value=1.0,\n range_spec=RangeSpec(min=0, max=2, step=0.01),\n advanced=True,\n ),\n IntInput(\n name=\"seed\",\n display_name=\"Seed\",\n info=\"The seed controls the reproducibility of the job.\",\n advanced=True,\n value=1,\n ),\n ]\n\n def get_models(self) -> list[str]:\n if not self.api_key:\n return DEEPSEEK_MODELS\n\n url = f\"{self.api_base}/models\"\n headers = {\"Authorization\": f\"Bearer {self.api_key}\", \"Accept\": \"application/json\"}\n\n try:\n response = requests.get(url, headers=headers, timeout=10)\n response.raise_for_status()\n model_list = response.json()\n return [model[\"id\"] for model in model_list.get(\"data\", [])]\n except requests.RequestException as e:\n self.status = f\"Error fetching models: {e}\"\n return DEEPSEEK_MODELS\n\n @override\n def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None):\n if field_name in {\"api_key\", \"api_base\", \"model_name\"}:\n models = self.get_models()\n build_config[\"model_name\"][\"options\"] = models\n return build_config\n\n def build_model(self) -> LanguageModel:\n try:\n from langchain_openai import ChatOpenAI\n except ImportError as e:\n msg = \"langchain-openai not installed. Please install with `pip install langchain-openai`\"\n raise ImportError(msg) from e\n\n api_key = SecretStr(self.api_key).get_secret_value() if self.api_key else None\n output = ChatOpenAI(\n model=self.model_name,\n temperature=self.temperature if self.temperature is not None else 0.1,\n max_tokens=self.max_tokens or None,\n model_kwargs=self.model_kwargs or {},\n base_url=self.api_base,\n api_key=api_key,\n streaming=self.stream if hasattr(self, \"stream\") else False,\n seed=self.seed,\n )\n\n if self.json_mode:\n output = output.bind(response_format={\"type\": \"json_object\"})\n\n return output\n\n def _get_exception_message(self, e: Exception):\n \"\"\"Get message from DeepSeek API exception.\"\"\"\n try:\n from openai import BadRequestError\n\n if isinstance(e, BadRequestError):\n message = e.body.get(\"message\")\n if message:\n return message\n except ImportError:\n pass\n return None\n" + "value": "import requests\nfrom pydantic.v1 import SecretStr\nfrom typing_extensions import override\n\nfrom lfx.base.models.model import LCModelComponent\nfrom lfx.field_typing import LanguageModel\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.inputs.inputs import BoolInput, DictInput, DropdownInput, IntInput, SecretStrInput, SliderInput, StrInput\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, validate_connector_url_for_ssrf\n\nDEEPSEEK_MODELS = [\"deepseek-chat\"]\n\n\nclass DeepSeekModelComponent(LCModelComponent):\n display_name = \"DeepSeek\"\n description = \"Generate text using DeepSeek LLMs.\"\n icon = \"DeepSeek\"\n\n inputs = [\n *LCModelComponent.get_base_inputs(),\n IntInput(\n name=\"max_tokens\",\n display_name=\"Max Tokens\",\n advanced=True,\n info=\"Maximum number of tokens to generate. Set to 0 for unlimited.\",\n range_spec=RangeSpec(min=0, max=128000),\n ),\n DictInput(\n name=\"model_kwargs\",\n display_name=\"Model Kwargs\",\n advanced=True,\n info=\"Additional keyword arguments to pass to the model.\",\n ),\n BoolInput(\n name=\"json_mode\",\n display_name=\"JSON Mode\",\n advanced=True,\n info=\"If True, it will output JSON regardless of passing a schema.\",\n ),\n DropdownInput(\n name=\"model_name\",\n display_name=\"Model Name\",\n info=\"DeepSeek model to use\",\n options=DEEPSEEK_MODELS,\n value=\"deepseek-chat\",\n refresh_button=True,\n ),\n StrInput(\n name=\"api_base\",\n display_name=\"DeepSeek API Base\",\n advanced=True,\n info=\"Base URL for API requests. Defaults to https://api.deepseek.com\",\n value=\"https://api.deepseek.com\",\n ),\n SecretStrInput(\n name=\"api_key\",\n display_name=\"DeepSeek API Key\",\n info=\"The DeepSeek API Key\",\n advanced=False,\n required=True,\n ),\n SliderInput(\n name=\"temperature\",\n display_name=\"Temperature\",\n info=\"Controls randomness in responses\",\n value=1.0,\n range_spec=RangeSpec(min=0, max=2, step=0.01),\n advanced=True,\n ),\n IntInput(\n name=\"seed\",\n display_name=\"Seed\",\n info=\"The seed controls the reproducibility of the job.\",\n advanced=True,\n value=1,\n ),\n ]\n\n def get_models(self) -> list[str]:\n if not self.api_key:\n return DEEPSEEK_MODELS\n\n url = f\"{self.api_base}/models\"\n headers = {\"Authorization\": f\"Bearer {self.api_key}\", \"Accept\": \"application/json\"}\n\n try:\n # api_base is tenant-controlled and fetched during build-config edits: block SSRF\n # to internal/cloud-metadata hosts. allow_redirects=False so a 3xx cannot bypass it.\n validate_connector_url_for_ssrf(url)\n response = requests.get(url, headers=headers, timeout=10, allow_redirects=False)\n response.raise_for_status()\n model_list = response.json()\n return [model[\"id\"] for model in model_list.get(\"data\", [])]\n except SSRFProtectionError as e:\n self.status = f\"api_base blocked by SSRF protection: {e}\"\n return DEEPSEEK_MODELS\n except requests.RequestException as e:\n self.status = f\"Error fetching models: {e}\"\n return DEEPSEEK_MODELS\n\n @override\n def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None):\n if field_name in {\"api_key\", \"api_base\", \"model_name\"}:\n models = self.get_models()\n build_config[\"model_name\"][\"options\"] = models\n return build_config\n\n def build_model(self) -> LanguageModel:\n try:\n from langchain_openai import ChatOpenAI\n except ImportError as e:\n msg = \"langchain-openai not installed. Please install with `pip install langchain-openai`\"\n raise ImportError(msg) from e\n\n api_key = SecretStr(self.api_key).get_secret_value() if self.api_key else None\n output = ChatOpenAI(\n model=self.model_name,\n temperature=self.temperature if self.temperature is not None else 0.1,\n max_tokens=self.max_tokens or None,\n model_kwargs=self.model_kwargs or {},\n base_url=self.api_base,\n api_key=api_key,\n streaming=self.stream if hasattr(self, \"stream\") else False,\n seed=self.seed,\n )\n\n if self.json_mode:\n output = output.bind(response_format={\"type\": \"json_object\"})\n\n return output\n\n def _get_exception_message(self, e: Exception):\n \"\"\"Get message from DeepSeek API exception.\"\"\"\n try:\n from openai import BadRequestError\n\n if isinstance(e, BadRequestError):\n message = e.body.get(\"message\")\n if message:\n return message\n except ImportError:\n pass\n return None\n" }, "input_value": { "_input_type": "MessageInput", @@ -65486,7 +65486,7 @@ "icon": "ElasticsearchStore", "legacy": false, "metadata": { - "code_hash": "23ea4383039e", + "code_hash": "b7bb04ab85e3", "dependencies": { "dependencies": [ { @@ -65599,7 +65599,7 @@ "show": true, "title_case": false, "type": "code", - "value": "from typing import Any\n\nfrom elasticsearch import Elasticsearch\nfrom langchain_core.documents import Document\nfrom langchain_elasticsearch import ElasticsearchStore\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n FloatInput,\n HandleInput,\n IntInput,\n SecretStrInput,\n StrInput,\n)\nfrom lfx.schema.data import Data\n\n\nclass ElasticsearchVectorStoreComponent(LCVectorStoreComponent):\n \"\"\"Elasticsearch Vector Store with with advanced, customizable search capabilities.\"\"\"\n\n display_name: str = \"Elasticsearch\"\n description: str = \"Elasticsearch Vector Store with with advanced, customizable search capabilities.\"\n name = \"Elasticsearch\"\n icon = \"ElasticsearchStore\"\n\n inputs = [\n StrInput(\n name=\"elasticsearch_url\",\n display_name=\"Elasticsearch URL\",\n value=\"http://localhost:9200\",\n info=\"URL for self-managed Elasticsearch deployments (e.g., http://localhost:9200). \"\n \"Do not use with Elastic Cloud deployments, use Elastic Cloud ID instead.\",\n ),\n SecretStrInput(\n name=\"cloud_id\",\n display_name=\"Elastic Cloud ID\",\n value=\"\",\n info=\"Use this for Elastic Cloud deployments. Do not use together with 'Elasticsearch URL'.\",\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=\"The index name where the vectors will be stored in Elasticsearch cluster.\",\n ),\n *LCVectorStoreComponent.inputs,\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"\",\n advanced=False,\n info=(\n \"Elasticsearch username (e.g., 'elastic'). \"\n \"Required for both local and Elastic Cloud setups unless API keys are used.\"\n ),\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"Elasticsearch Password\",\n value=\"\",\n advanced=False,\n info=(\n \"Elasticsearch password for the specified user. \"\n \"Required for both local and Elastic Cloud setups unless API keys are used.\"\n ),\n ),\n HandleInput(\n name=\"embedding\",\n display_name=\"Embedding\",\n input_types=[\"Embeddings\"],\n ),\n DropdownInput(\n name=\"search_type\",\n display_name=\"Search Type\",\n options=[\"similarity\", \"mmr\"],\n value=\"similarity\",\n advanced=True,\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Number of Results\",\n info=\"Number of results to return.\",\n advanced=True,\n value=4,\n ),\n FloatInput(\n name=\"search_score_threshold\",\n display_name=\"Search Score Threshold\",\n info=\"Minimum similarity score threshold for search results.\",\n value=0.0,\n advanced=True,\n ),\n SecretStrInput(\n name=\"api_key\",\n display_name=\"Elastic API Key\",\n value=\"\",\n advanced=True,\n info=\"API Key for Elastic Cloud authentication. If used, 'username' and 'password' are not required.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=True,\n advanced=True,\n info=\"Whether to verify SSL certificates when connecting to Elasticsearch.\",\n ),\n ]\n\n @check_cached_vector_store\n def build_vector_store(self) -> ElasticsearchStore:\n \"\"\"Builds the Elasticsearch Vector Store object.\"\"\"\n if self.cloud_id and self.elasticsearch_url:\n msg = (\n \"Both 'cloud_id' and 'elasticsearch_url' provided. \"\n \"Please use only one based on your deployment (Cloud or Local).\"\n )\n raise ValueError(msg)\n\n es_params = {\n \"index_name\": self.index_name,\n \"embedding\": self.embedding,\n \"es_user\": self.username or None,\n \"es_password\": self.password or None,\n }\n\n if self.cloud_id:\n es_params[\"es_cloud_id\"] = self.cloud_id\n else:\n es_params[\"es_url\"] = self.elasticsearch_url\n\n if self.api_key:\n es_params[\"api_key\"] = self.api_key\n\n # Check if we need to verify SSL certificates\n if self.verify_certs is False:\n # Build client parameters for Elasticsearch constructor\n client_params: dict[str, Any] = {}\n client_params[\"verify_certs\"] = False\n\n if self.cloud_id:\n client_params[\"cloud_id\"] = self.cloud_id\n else:\n client_params[\"hosts\"] = [self.elasticsearch_url]\n\n if self.api_key:\n client_params[\"api_key\"] = self.api_key\n elif self.username and self.password:\n client_params[\"basic_auth\"] = (self.username, self.password)\n\n es_client = Elasticsearch(**client_params)\n es_params[\"es_connection\"] = es_client\n\n elasticsearch = ElasticsearchStore(**es_params)\n\n # If documents are provided, add them to the store\n if self.ingest_data:\n documents = self._prepare_documents()\n if documents:\n elasticsearch.add_documents(documents)\n\n return elasticsearch\n\n def _prepare_documents(self) -> list[Document]:\n \"\"\"Prepares documents from the input data to add to the vector store.\"\"\"\n self.ingest_data = self._prepare_ingest_data()\n\n documents = []\n for data in self.ingest_data:\n if isinstance(data, Data):\n documents.append(data.to_lc_document())\n else:\n error_message = \"Vector Store Inputs must be Data objects.\"\n self.log(error_message)\n raise TypeError(error_message)\n return documents\n\n def _add_documents_to_vector_store(self, vector_store: \"ElasticsearchStore\") -> None:\n \"\"\"Adds documents to the Vector Store.\"\"\"\n documents = self._prepare_documents()\n if documents and self.embedding:\n self.log(f\"Adding {len(documents)} documents to the Vector Store.\")\n vector_store.add_documents(documents)\n else:\n self.log(\"No documents to add to the Vector Store.\")\n\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Search for similar documents in the vector store or retrieve all documents if no query is provided.\"\"\"\n vector_store = self.build_vector_store()\n search_kwargs = {\n \"k\": self.number_of_results,\n \"score_threshold\": self.search_score_threshold,\n }\n\n if query:\n search_type = self.search_type.lower()\n if search_type not in {\"similarity\", \"mmr\"}:\n msg = f\"Invalid search type: {self.search_type}\"\n self.log(msg)\n raise ValueError(msg)\n try:\n if search_type == \"similarity\":\n results = vector_store.similarity_search_with_score(query, **search_kwargs)\n elif search_type == \"mmr\":\n results = vector_store.max_marginal_relevance_search(query, **search_kwargs)\n except Exception as e:\n msg = (\n \"Error occurred while querying the Elasticsearch VectorStore,\"\n \" there is no Data into the VectorStore.\"\n )\n self.log(msg)\n raise ValueError(msg) from e\n return [\n {\"page_content\": doc.page_content, \"metadata\": doc.metadata, \"score\": score} for doc, score in results\n ]\n results = self.get_all_documents(vector_store, **search_kwargs)\n return [{\"page_content\": doc.page_content, \"metadata\": doc.metadata, \"score\": score} for doc, score in results]\n\n def get_all_documents(self, vector_store: ElasticsearchStore, **kwargs) -> list[tuple[Document, float]]:\n \"\"\"Retrieve all documents from the vector store.\"\"\"\n client = vector_store.client\n index_name = self.index_name\n\n query = {\n \"query\": {\"match_all\": {}},\n \"size\": kwargs.get(\"k\", self.number_of_results),\n }\n\n response = client.search(index=index_name, body=query)\n\n results = []\n for hit in response[\"hits\"][\"hits\"]:\n doc = Document(\n page_content=hit[\"_source\"].get(\"text\", \"\"),\n metadata=hit[\"_source\"].get(\"metadata\", {}),\n )\n score = hit[\"_score\"]\n results.append((doc, score))\n\n return results\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search for documents in the vector store based on the search input.\n\n If no search input is provided, retrieve all documents.\n \"\"\"\n results = self.search(self.search_query)\n retrieved_data = [\n Data(\n text=result[\"page_content\"],\n file_path=result[\"metadata\"].get(\"file_path\", \"\"),\n )\n for result in results\n ]\n self.status = retrieved_data\n return retrieved_data\n\n def get_retriever_kwargs(self):\n \"\"\"Get the keyword arguments for the retriever.\"\"\"\n return {\n \"search_type\": self.search_type.lower(),\n \"search_kwargs\": {\n \"k\": self.number_of_results,\n \"score_threshold\": self.search_score_threshold,\n },\n }\n" + "value": "from typing import Any\n\nfrom elasticsearch import Elasticsearch\nfrom langchain_core.documents import Document\nfrom langchain_elasticsearch import ElasticsearchStore\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n FloatInput,\n HandleInput,\n IntInput,\n SecretStrInput,\n StrInput,\n)\nfrom lfx.schema.data import Data\nfrom lfx.utils.ssrf_protection import validate_connector_url_for_ssrf\n\n\nclass ElasticsearchVectorStoreComponent(LCVectorStoreComponent):\n \"\"\"Elasticsearch Vector Store with with advanced, customizable search capabilities.\"\"\"\n\n display_name: str = \"Elasticsearch\"\n description: str = \"Elasticsearch Vector Store with with advanced, customizable search capabilities.\"\n name = \"Elasticsearch\"\n icon = \"ElasticsearchStore\"\n\n inputs = [\n StrInput(\n name=\"elasticsearch_url\",\n display_name=\"Elasticsearch URL\",\n value=\"http://localhost:9200\",\n info=\"URL for self-managed Elasticsearch deployments (e.g., http://localhost:9200). \"\n \"Do not use with Elastic Cloud deployments, use Elastic Cloud ID instead.\",\n ),\n SecretStrInput(\n name=\"cloud_id\",\n display_name=\"Elastic Cloud ID\",\n value=\"\",\n info=\"Use this for Elastic Cloud deployments. Do not use together with 'Elasticsearch URL'.\",\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=\"The index name where the vectors will be stored in Elasticsearch cluster.\",\n ),\n *LCVectorStoreComponent.inputs,\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"\",\n advanced=False,\n info=(\n \"Elasticsearch username (e.g., 'elastic'). \"\n \"Required for both local and Elastic Cloud setups unless API keys are used.\"\n ),\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"Elasticsearch Password\",\n value=\"\",\n advanced=False,\n info=(\n \"Elasticsearch password for the specified user. \"\n \"Required for both local and Elastic Cloud setups unless API keys are used.\"\n ),\n ),\n HandleInput(\n name=\"embedding\",\n display_name=\"Embedding\",\n input_types=[\"Embeddings\"],\n ),\n DropdownInput(\n name=\"search_type\",\n display_name=\"Search Type\",\n options=[\"similarity\", \"mmr\"],\n value=\"similarity\",\n advanced=True,\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Number of Results\",\n info=\"Number of results to return.\",\n advanced=True,\n value=4,\n ),\n FloatInput(\n name=\"search_score_threshold\",\n display_name=\"Search Score Threshold\",\n info=\"Minimum similarity score threshold for search results.\",\n value=0.0,\n advanced=True,\n ),\n SecretStrInput(\n name=\"api_key\",\n display_name=\"Elastic API Key\",\n value=\"\",\n advanced=True,\n info=\"API Key for Elastic Cloud authentication. If used, 'username' and 'password' are not required.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=True,\n advanced=True,\n info=\"Whether to verify SSL certificates when connecting to Elasticsearch.\",\n ),\n ]\n\n @check_cached_vector_store\n def build_vector_store(self) -> ElasticsearchStore:\n \"\"\"Builds the Elasticsearch Vector Store object.\"\"\"\n # elasticsearch_url is tenant-controlled: block SSRF to internal/cloud-metadata hosts.\n if self.elasticsearch_url:\n validate_connector_url_for_ssrf(self.elasticsearch_url)\n if self.cloud_id and self.elasticsearch_url:\n msg = (\n \"Both 'cloud_id' and 'elasticsearch_url' provided. \"\n \"Please use only one based on your deployment (Cloud or Local).\"\n )\n raise ValueError(msg)\n\n es_params = {\n \"index_name\": self.index_name,\n \"embedding\": self.embedding,\n \"es_user\": self.username or None,\n \"es_password\": self.password or None,\n }\n\n if self.cloud_id:\n es_params[\"es_cloud_id\"] = self.cloud_id\n else:\n es_params[\"es_url\"] = self.elasticsearch_url\n\n if self.api_key:\n es_params[\"api_key\"] = self.api_key\n\n # Check if we need to verify SSL certificates\n if self.verify_certs is False:\n # Build client parameters for Elasticsearch constructor\n client_params: dict[str, Any] = {}\n client_params[\"verify_certs\"] = False\n\n if self.cloud_id:\n client_params[\"cloud_id\"] = self.cloud_id\n else:\n client_params[\"hosts\"] = [self.elasticsearch_url]\n\n if self.api_key:\n client_params[\"api_key\"] = self.api_key\n elif self.username and self.password:\n client_params[\"basic_auth\"] = (self.username, self.password)\n\n es_client = Elasticsearch(**client_params)\n es_params[\"es_connection\"] = es_client\n\n elasticsearch = ElasticsearchStore(**es_params)\n\n # If documents are provided, add them to the store\n if self.ingest_data:\n documents = self._prepare_documents()\n if documents:\n elasticsearch.add_documents(documents)\n\n return elasticsearch\n\n def _prepare_documents(self) -> list[Document]:\n \"\"\"Prepares documents from the input data to add to the vector store.\"\"\"\n self.ingest_data = self._prepare_ingest_data()\n\n documents = []\n for data in self.ingest_data:\n if isinstance(data, Data):\n documents.append(data.to_lc_document())\n else:\n error_message = \"Vector Store Inputs must be Data objects.\"\n self.log(error_message)\n raise TypeError(error_message)\n return documents\n\n def _add_documents_to_vector_store(self, vector_store: \"ElasticsearchStore\") -> None:\n \"\"\"Adds documents to the Vector Store.\"\"\"\n documents = self._prepare_documents()\n if documents and self.embedding:\n self.log(f\"Adding {len(documents)} documents to the Vector Store.\")\n vector_store.add_documents(documents)\n else:\n self.log(\"No documents to add to the Vector Store.\")\n\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Search for similar documents in the vector store or retrieve all documents if no query is provided.\"\"\"\n vector_store = self.build_vector_store()\n search_kwargs = {\n \"k\": self.number_of_results,\n \"score_threshold\": self.search_score_threshold,\n }\n\n if query:\n search_type = self.search_type.lower()\n if search_type not in {\"similarity\", \"mmr\"}:\n msg = f\"Invalid search type: {self.search_type}\"\n self.log(msg)\n raise ValueError(msg)\n try:\n if search_type == \"similarity\":\n results = vector_store.similarity_search_with_score(query, **search_kwargs)\n elif search_type == \"mmr\":\n results = vector_store.max_marginal_relevance_search(query, **search_kwargs)\n except Exception as e:\n msg = (\n \"Error occurred while querying the Elasticsearch VectorStore,\"\n \" there is no Data into the VectorStore.\"\n )\n self.log(msg)\n raise ValueError(msg) from e\n return [\n {\"page_content\": doc.page_content, \"metadata\": doc.metadata, \"score\": score} for doc, score in results\n ]\n results = self.get_all_documents(vector_store, **search_kwargs)\n return [{\"page_content\": doc.page_content, \"metadata\": doc.metadata, \"score\": score} for doc, score in results]\n\n def get_all_documents(self, vector_store: ElasticsearchStore, **kwargs) -> list[tuple[Document, float]]:\n \"\"\"Retrieve all documents from the vector store.\"\"\"\n client = vector_store.client\n index_name = self.index_name\n\n query = {\n \"query\": {\"match_all\": {}},\n \"size\": kwargs.get(\"k\", self.number_of_results),\n }\n\n response = client.search(index=index_name, body=query)\n\n results = []\n for hit in response[\"hits\"][\"hits\"]:\n doc = Document(\n page_content=hit[\"_source\"].get(\"text\", \"\"),\n metadata=hit[\"_source\"].get(\"metadata\", {}),\n )\n score = hit[\"_score\"]\n results.append((doc, score))\n\n return results\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search for documents in the vector store based on the search input.\n\n If no search input is provided, retrieve all documents.\n \"\"\"\n results = self.search(self.search_query)\n retrieved_data = [\n Data(\n text=result[\"page_content\"],\n file_path=result[\"metadata\"].get(\"file_path\", \"\"),\n )\n for result in results\n ]\n self.status = retrieved_data\n return retrieved_data\n\n def get_retriever_kwargs(self):\n \"\"\"Get the keyword arguments for the retriever.\"\"\"\n return {\n \"search_type\": self.search_type.lower(),\n \"search_kwargs\": {\n \"k\": self.number_of_results,\n \"score_threshold\": self.search_score_threshold,\n },\n }\n" }, "elasticsearch_url": { "_input_type": "StrInput", @@ -65906,7 +65906,7 @@ "icon": "OpenSearch", "legacy": false, "metadata": { - "code_hash": "f4dfc3668475", + "code_hash": "5d9066e480be", "dependencies": { "dependencies": [ { @@ -66036,7 +66036,7 @@ "show": true, "title_case": false, "type": "code", - "value": "from __future__ import annotations\n\nimport json\nimport uuid\nfrom typing import Any\n\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import RequestError\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import BoolInput, DropdownInput, HandleInput, IntInput, MultilineInput, SecretStrInput, StrInput, TableInput\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponent(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports document\n ingestion, vector embeddings, and advanced filtering with authentication options.\n\n Features:\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Hybrid search combining KNN vector similarity and keyword matching\n - Flexible authentication (Basic auth, JWT tokens)\n - Advanced filtering and aggregations\n - Metadata injection during document ingestion\n \"\"\"\n\n display_name: str = \"OpenSearch\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with hybrid semantic and keyword search capabilities.\"\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"request_timeout\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"docs_metadata\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"]),\n StrInput(\n name=\"vector_field\",\n display_name=\"Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=\"Name of the field in OpenSearch documents that stores the vector embeddings for similarity search.\",\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"basic\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=True,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n IntInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=60,\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch (transport and per-request). \"\n \"Used for the default transport timeout and for bulk ingest HTTP calls. \"\n \"Increase for large bulk ingestion or slow clusters.\"\n ),\n ),\n ]\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n }\n }\n },\n }\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _get_request_timeout(self) -> int:\n \"\"\"Return the configured request timeout in seconds (default 60).\"\"\"\n if not hasattr(self, \"request_timeout\") or self.request_timeout is None:\n return 60\n try:\n t = int(self.request_timeout)\n except (TypeError, ValueError):\n return 60\n else:\n return t if t >= 1 else 60\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(\n client,\n requests,\n max_chunk_bytes=max_chunk_bytes,\n request_timeout=self._get_request_timeout(),\n )\n return return_ids\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._get_request_timeout(),\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our “vector store.”\n self.log(self.ingest_data)\n client = self.build_client()\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings\n - Creates appropriate index mappings\n - Bulk inserts documents with vectors\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n docs = self.ingest_data or []\n if not docs:\n self.log(\"No documents to ingest.\")\n return\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.debug(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.debug(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.debug(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Generate embeddings\n vectors = self.embedding.embed_documents(texts)\n\n if not vectors:\n self.log(\"No vectors generated from documents.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=self.vector_field,\n )\n\n # Ensure index exists with proper KNN mapping (index.knn: true is required for vector search)\n try:\n if not client.indices.exists(index=self.index_name):\n self.log(f\"Creating index '{self.index_name}' with KNN mapping (index.knn: true)\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n raise\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with proper KNN mapping...\")\n\n # Use the LangChain-style bulk ingestion\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=self.vector_field,\n text_field=\"text\",\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n self.log(f\"Successfully indexed {len(return_ids)} documents.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n # ---------- search (single hybrid path matching your tool) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform hybrid search combining vector similarity and keyword matching.\n\n This method executes a sophisticated search that combines:\n - K-nearest neighbor (KNN) vector similarity search (70% weight)\n - Multi-field keyword search with fuzzy matching (30% weight)\n - Optional filtering and score thresholds\n - Aggregations for faceted search results\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression (can be either A or B shape; see _coerce_filter_clauses)\n filter_obj = None\n if getattr(self, \"filter_expression\", \"\") and self.filter_expression.strip():\n try:\n filter_obj = json.loads(self.filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Embed the query\n vec = self.embedding.embed_query(q)\n\n # Build filter clauses (accept both shapes)\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Respect the tool's limit/threshold defaults\n limit = (filter_obj or {}).get(\"limit\", self.number_of_results)\n score_threshold = (filter_obj or {}).get(\"score_threshold\", 0)\n\n # Build the same hybrid body as your SearchService\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"knn\": {\n self.vector_field: {\n \"vector\": vec,\n \"k\": 10, # fixed to match the tool\n \"boost\": 0.7,\n }\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3,\n }\n },\n ],\n \"minimum_should_match\": 1,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": \"filename\", \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n if filter_clauses:\n body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n\n if isinstance(score_threshold, (int, float)) and score_threshold > 0:\n # top-level min_score (matches your tool)\n body[\"min_score\"] = score_threshold\n\n resp = client.search(index=self.index_name, body=body)\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as Data objects.\n\n This is the main interface method that performs the search using the\n configured search_query and returns results in Langflow's Data format.\n\n Returns:\n List of Data objects containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n raw = self.search(self.search_query or \"\")\n return [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n self.log(self.ingest_data)\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n if is_basic:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n" + "value": "from __future__ import annotations\n\nimport json\nimport uuid\nfrom typing import Any\n\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import RequestError\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import BoolInput, DropdownInput, HandleInput, IntInput, MultilineInput, SecretStrInput, StrInput, TableInput\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom lfx.utils.ssrf_protection import validate_connector_url_for_ssrf\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponent(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports document\n ingestion, vector embeddings, and advanced filtering with authentication options.\n\n Features:\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Hybrid search combining KNN vector similarity and keyword matching\n - Flexible authentication (Basic auth, JWT tokens)\n - Advanced filtering and aggregations\n - Metadata injection during document ingestion\n \"\"\"\n\n display_name: str = \"OpenSearch\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with hybrid semantic and keyword search capabilities.\"\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"request_timeout\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"docs_metadata\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"]),\n StrInput(\n name=\"vector_field\",\n display_name=\"Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=\"Name of the field in OpenSearch documents that stores the vector embeddings for similarity search.\",\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"basic\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=True,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n IntInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=60,\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch (transport and per-request). \"\n \"Used for the default transport timeout and for bulk ingest HTTP calls. \"\n \"Increase for large bulk ingestion or slow clusters.\"\n ),\n ),\n ]\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n }\n }\n },\n }\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _get_request_timeout(self) -> int:\n \"\"\"Return the configured request timeout in seconds (default 60).\"\"\"\n if not hasattr(self, \"request_timeout\") or self.request_timeout is None:\n return 60\n try:\n t = int(self.request_timeout)\n except (TypeError, ValueError):\n return 60\n else:\n return t if t >= 1 else 60\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(\n client,\n requests,\n max_chunk_bytes=max_chunk_bytes,\n request_timeout=self._get_request_timeout(),\n )\n return return_ids\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n auth_kwargs = self._build_auth_kwargs()\n # opensearch_url is tenant-controlled: block SSRF to internal/cloud-metadata hosts.\n validate_connector_url_for_ssrf(self.opensearch_url)\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._get_request_timeout(),\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our “vector store.”\n self.log(self.ingest_data)\n client = self.build_client()\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings\n - Creates appropriate index mappings\n - Bulk inserts documents with vectors\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n docs = self.ingest_data or []\n if not docs:\n self.log(\"No documents to ingest.\")\n return\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.debug(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.debug(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.debug(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Generate embeddings\n vectors = self.embedding.embed_documents(texts)\n\n if not vectors:\n self.log(\"No vectors generated from documents.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=self.vector_field,\n )\n\n # Ensure index exists with proper KNN mapping (index.knn: true is required for vector search)\n try:\n if not client.indices.exists(index=self.index_name):\n self.log(f\"Creating index '{self.index_name}' with KNN mapping (index.knn: true)\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n raise\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with proper KNN mapping...\")\n\n # Use the LangChain-style bulk ingestion\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=self.vector_field,\n text_field=\"text\",\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n self.log(f\"Successfully indexed {len(return_ids)} documents.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n # ---------- search (single hybrid path matching your tool) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform hybrid search combining vector similarity and keyword matching.\n\n This method executes a sophisticated search that combines:\n - K-nearest neighbor (KNN) vector similarity search (70% weight)\n - Multi-field keyword search with fuzzy matching (30% weight)\n - Optional filtering and score thresholds\n - Aggregations for faceted search results\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression (can be either A or B shape; see _coerce_filter_clauses)\n filter_obj = None\n if getattr(self, \"filter_expression\", \"\") and self.filter_expression.strip():\n try:\n filter_obj = json.loads(self.filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Embed the query\n vec = self.embedding.embed_query(q)\n\n # Build filter clauses (accept both shapes)\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Respect the tool's limit/threshold defaults\n limit = (filter_obj or {}).get(\"limit\", self.number_of_results)\n score_threshold = (filter_obj or {}).get(\"score_threshold\", 0)\n\n # Build the same hybrid body as your SearchService\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"knn\": {\n self.vector_field: {\n \"vector\": vec,\n \"k\": 10, # fixed to match the tool\n \"boost\": 0.7,\n }\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3,\n }\n },\n ],\n \"minimum_should_match\": 1,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": \"filename\", \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n if filter_clauses:\n body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n\n if isinstance(score_threshold, (int, float)) and score_threshold > 0:\n # top-level min_score (matches your tool)\n body[\"min_score\"] = score_threshold\n\n resp = client.search(index=self.index_name, body=body)\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as Data objects.\n\n This is the main interface method that performs the search using the\n configured search_query and returns results in Langflow's Data format.\n\n Returns:\n List of Data objects containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n raw = self.search(self.search_query or \"\")\n return [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n self.log(self.ingest_data)\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n if is_basic:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n" }, "docs_metadata": { "_input_type": "TableInput", @@ -66569,7 +66569,7 @@ "icon": "OpenSearch", "legacy": false, "metadata": { - "code_hash": "4a05432fb489", + "code_hash": "18458a117b5f", "dependencies": { "dependencies": [ { @@ -66703,7 +66703,7 @@ "show": true, "title_case": false, "type": "code", - "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport time\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import Table\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"jwt\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list) and len(val) > min_vector_length and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\")\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n return return_ids\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\")\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\")\n self.log(\"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\")\n return\n\n logger.debug(f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\")\n self.log(f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\")\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name and self.embedding_model_name.strip():\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\")\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\")\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\")\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Generate embeddings with rate-limit-aware retry logic using tenacity\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is retryable but not a rate limit error.\"\"\"\n # Retry on most exceptions except for specific non-retryable ones\n # Add other non-retryable exceptions here if needed\n return not is_rate_limit_error(exception)\n\n # Create retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Create retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n # Restrict concurrency for IBM/Watsonx models to avoid rate limits\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM: {is_ibm}\")\n\n # For IBM models, use sequential processing with rate limiting\n # For other models, use parallel processing\n vectors: list[list[float]] = [None] * len(texts)\n\n if is_ibm:\n # Sequential processing with inter-request delay for IBM models\n inter_request_delay = 0.6 # ~1.67 req/s, safely under 2 req/s limit\n logger.info(f\"Using sequential processing for IBM model with {inter_request_delay}s delay between requests\")\n\n for idx, chunk in enumerate(texts):\n if idx > 0:\n # Add delay between requests (but not before the first one)\n time.sleep(inter_request_delay)\n vectors[idx] = embed_chunk_with_retry(chunk, idx)\n else:\n # Parallel processing for non-IBM models\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {executor.submit(embed_chunk_with_retry, chunk, idx): idx for idx, chunk in enumerate(texts)}\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n try:\n if not client.indices.exists(index=self.index_name):\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\")\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\"size\": 0, \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}}}\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\")\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(self, properties: dict[str, Any] | None, field_name: str) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (isinstance(self.embedding, list) and all(e is None for e in self.embedding)):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(f\"Mapped available model '{model_str}' to dedicated embedding instance\")\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(f\"Available model '{model_str}' has conflict - used by multiple embeddings\")\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(f\"Identifier '{identifier}' has conflict - used by multiple embeddings\")\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(f\"Created combined identifier '{combined_id}' for embedding object {idx}\")\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\")\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\")\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\")\n except (RuntimeError, ValueError, ConnectionError, TimeoutError, AttributeError, KeyError) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n self.log(f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\")\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\")\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)], \"minimum_should_match\": 1}\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\")\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = knn_queries_without_candidates\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> Table:\n \"\"\"Search documents and return results as Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Table format.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n Table: A table containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n\n return Table(data=raw_list)\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n if is_basic:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n" + "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport time\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import Table\nfrom lfx.utils.ssrf_protection import validate_connector_url_for_ssrf\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"jwt\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list) and len(val) > min_vector_length and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\")\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n return return_ids\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n # opensearch_url is tenant-controlled: block SSRF to internal/cloud-metadata hosts.\n validate_connector_url_for_ssrf(self.opensearch_url)\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\")\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\")\n self.log(\"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\")\n return\n\n logger.debug(f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\")\n self.log(f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\")\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name and self.embedding_model_name.strip():\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\")\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\")\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\")\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Generate embeddings with rate-limit-aware retry logic using tenacity\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is retryable but not a rate limit error.\"\"\"\n # Retry on most exceptions except for specific non-retryable ones\n # Add other non-retryable exceptions here if needed\n return not is_rate_limit_error(exception)\n\n # Create retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Create retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n # Restrict concurrency for IBM/Watsonx models to avoid rate limits\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM: {is_ibm}\")\n\n # For IBM models, use sequential processing with rate limiting\n # For other models, use parallel processing\n vectors: list[list[float]] = [None] * len(texts)\n\n if is_ibm:\n # Sequential processing with inter-request delay for IBM models\n inter_request_delay = 0.6 # ~1.67 req/s, safely under 2 req/s limit\n logger.info(f\"Using sequential processing for IBM model with {inter_request_delay}s delay between requests\")\n\n for idx, chunk in enumerate(texts):\n if idx > 0:\n # Add delay between requests (but not before the first one)\n time.sleep(inter_request_delay)\n vectors[idx] = embed_chunk_with_retry(chunk, idx)\n else:\n # Parallel processing for non-IBM models\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {executor.submit(embed_chunk_with_retry, chunk, idx): idx for idx, chunk in enumerate(texts)}\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n try:\n if not client.indices.exists(index=self.index_name):\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\")\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\"size\": 0, \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}}}\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\")\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(self, properties: dict[str, Any] | None, field_name: str) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (isinstance(self.embedding, list) and all(e is None for e in self.embedding)):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(f\"Mapped available model '{model_str}' to dedicated embedding instance\")\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(f\"Available model '{model_str}' has conflict - used by multiple embeddings\")\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(f\"Identifier '{identifier}' has conflict - used by multiple embeddings\")\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(f\"Created combined identifier '{combined_id}' for embedding object {idx}\")\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\")\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\")\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\")\n except (RuntimeError, ValueError, ConnectionError, TimeoutError, AttributeError, KeyError) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n self.log(f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\")\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\")\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)], \"minimum_should_match\": 1}\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\")\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = knn_queries_without_candidates\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> Table:\n \"\"\"Search documents and return results as Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Table format.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n Table: A table containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n\n return Table(data=raw_list)\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n if is_basic:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n" }, "docs_metadata": { "_input_type": "TableInput", @@ -67712,7 +67712,7 @@ "icon": "folder", "legacy": true, "metadata": { - "code_hash": "328e6f996926", + "code_hash": "1ba25094a25d", "dependencies": { "dependencies": [ { @@ -67764,7 +67764,7 @@ "show": true, "title_case": false, "type": "code", - "value": "from lfx.base.data.utils import TEXT_FILE_TYPES, parallel_load_data, parse_text_file_to_data, retrieve_file_paths\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.io import BoolInput, IntInput, MessageTextInput, MultiselectInput\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.template.field.base import Output\n\n\nclass DirectoryComponent(Component):\n display_name = \"Directory\"\n description = \"Recursively load files from a directory.\"\n documentation: str = \"https://docs.langflow.org/directory\"\n icon = \"folder\"\n name = \"Directory\"\n legacy = True\n replacement = [\"data.File\"]\n\n inputs = [\n MessageTextInput(\n name=\"path\",\n display_name=\"Path\",\n info=\"Path to the directory to load files from. Defaults to current directory ('.')\",\n value=\".\",\n tool_mode=True,\n ),\n MultiselectInput(\n name=\"types\",\n display_name=\"File Types\",\n info=\"File types to load. Select one or more types or leave empty to load all supported types.\",\n options=TEXT_FILE_TYPES,\n value=[],\n ),\n IntInput(\n name=\"depth\",\n display_name=\"Depth\",\n info=\"Depth to search for files.\",\n value=0,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Max Concurrency\",\n advanced=True,\n info=\"Maximum concurrency for loading files.\",\n value=2,\n ),\n BoolInput(\n name=\"load_hidden\",\n display_name=\"Load Hidden\",\n advanced=True,\n info=\"If true, hidden files will be loaded.\",\n ),\n BoolInput(\n name=\"recursive\",\n display_name=\"Recursive\",\n advanced=True,\n info=\"If true, the search will be recursive.\",\n ),\n BoolInput(\n name=\"silent_errors\",\n display_name=\"Silent Errors\",\n advanced=True,\n info=\"If true, errors will not raise an exception.\",\n ),\n BoolInput(\n name=\"use_multithreading\",\n display_name=\"Use Multithreading\",\n advanced=True,\n info=\"If true, multithreading will be used.\",\n ),\n ]\n\n outputs = [\n Output(display_name=\"Loaded Files\", name=\"dataframe\", method=\"as_dataframe\"),\n ]\n\n def load_directory(self) -> list[Data]:\n path = self.path\n types = self.types\n depth = self.depth\n max_concurrency = self.max_concurrency\n load_hidden = self.load_hidden\n recursive = self.recursive\n silent_errors = self.silent_errors\n use_multithreading = self.use_multithreading\n\n resolved_path = self.resolve_path(path)\n\n # If no types are specified, use all supported types\n if not types:\n types = TEXT_FILE_TYPES\n\n # Check if all specified types are valid\n invalid_types = [t for t in types if t not in TEXT_FILE_TYPES]\n if invalid_types:\n msg = f\"Invalid file types specified: {invalid_types}. Valid types are: {TEXT_FILE_TYPES}\"\n raise ValueError(msg)\n\n valid_types = types\n\n file_paths = retrieve_file_paths(\n resolved_path, load_hidden=load_hidden, recursive=recursive, depth=depth, types=valid_types\n )\n\n loaded_data = []\n if use_multithreading:\n loaded_data = parallel_load_data(file_paths, silent_errors=silent_errors, max_concurrency=max_concurrency)\n else:\n loaded_data = [parse_text_file_to_data(file_path, silent_errors=silent_errors) for file_path in file_paths]\n\n valid_data = [x for x in loaded_data if x is not None and isinstance(x, Data)]\n self.status = valid_data\n return valid_data\n\n def as_dataframe(self) -> DataFrame:\n return DataFrame(self.load_directory())\n" + "value": "from lfx.base.data.utils import TEXT_FILE_TYPES, parallel_load_data, parse_text_file_to_data, retrieve_file_paths\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.io import BoolInput, IntInput, MessageTextInput, MultiselectInput\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.template.field.base import Output\nfrom lfx.utils.file_path_security import enforce_local_file_access\n\n\nclass DirectoryComponent(Component):\n display_name = \"Directory\"\n description = \"Recursively load files from a directory.\"\n documentation: str = \"https://docs.langflow.org/directory\"\n icon = \"folder\"\n name = \"Directory\"\n legacy = True\n replacement = [\"data.File\"]\n\n inputs = [\n MessageTextInput(\n name=\"path\",\n display_name=\"Path\",\n info=\"Path to the directory to load files from. Defaults to current directory ('.')\",\n value=\".\",\n tool_mode=True,\n ),\n MultiselectInput(\n name=\"types\",\n display_name=\"File Types\",\n info=\"File types to load. Select one or more types or leave empty to load all supported types.\",\n options=TEXT_FILE_TYPES,\n value=[],\n ),\n IntInput(\n name=\"depth\",\n display_name=\"Depth\",\n info=\"Depth to search for files.\",\n value=0,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Max Concurrency\",\n advanced=True,\n info=\"Maximum concurrency for loading files.\",\n value=2,\n ),\n BoolInput(\n name=\"load_hidden\",\n display_name=\"Load Hidden\",\n advanced=True,\n info=\"If true, hidden files will be loaded.\",\n ),\n BoolInput(\n name=\"recursive\",\n display_name=\"Recursive\",\n advanced=True,\n info=\"If true, the search will be recursive.\",\n ),\n BoolInput(\n name=\"silent_errors\",\n display_name=\"Silent Errors\",\n advanced=True,\n info=\"If true, errors will not raise an exception.\",\n ),\n BoolInput(\n name=\"use_multithreading\",\n display_name=\"Use Multithreading\",\n advanced=True,\n info=\"If true, multithreading will be used.\",\n ),\n ]\n\n outputs = [\n Output(display_name=\"Loaded Files\", name=\"dataframe\", method=\"as_dataframe\"),\n ]\n\n def load_directory(self) -> list[Data]:\n path = self.path\n types = self.types\n depth = self.depth\n max_concurrency = self.max_concurrency\n load_hidden = self.load_hidden\n recursive = self.recursive\n silent_errors = self.silent_errors\n use_multithreading = self.use_multithreading\n\n resolved_path = self.resolve_path(path)\n\n # Security: confine directory reads to the storage dir in restricted (multi-tenant)\n # mode so a tenant cannot recursively read arbitrary server directories.\n resolved_path = str(enforce_local_file_access(resolved_path))\n\n # If no types are specified, use all supported types\n if not types:\n types = TEXT_FILE_TYPES\n\n # Check if all specified types are valid\n invalid_types = [t for t in types if t not in TEXT_FILE_TYPES]\n if invalid_types:\n msg = f\"Invalid file types specified: {invalid_types}. Valid types are: {TEXT_FILE_TYPES}\"\n raise ValueError(msg)\n\n valid_types = types\n\n file_paths = retrieve_file_paths(\n resolved_path, load_hidden=load_hidden, recursive=recursive, depth=depth, types=valid_types\n )\n\n loaded_data = []\n if use_multithreading:\n loaded_data = parallel_load_data(file_paths, silent_errors=silent_errors, max_concurrency=max_concurrency)\n else:\n loaded_data = [parse_text_file_to_data(file_path, silent_errors=silent_errors) for file_path in file_paths]\n\n valid_data = [x for x in loaded_data if x is not None and isinstance(x, Data)]\n self.status = valid_data\n return valid_data\n\n def as_dataframe(self) -> DataFrame:\n return DataFrame(self.load_directory())\n" }, "depth": { "_input_type": "IntInput", @@ -67998,7 +67998,7 @@ "icon": "file-text", "legacy": false, "metadata": { - "code_hash": "f497bdbc749b", + "code_hash": "266ad885c421", "dependencies": { "dependencies": [ { @@ -68160,7 +68160,7 @@ "show": true, "title_case": false, "type": "code", - "value": "\"\"\"Enhanced file component with Docling support and process isolation.\n\nNotes:\n-----\n- ALL Docling parsing/export runs in a separate OS process to prevent memory\n growth and native library state from impacting the main Langflow process.\n- Standard text/structured parsing continues to use existing BaseFileComponent\n utilities (and optional threading via `parallel_load_data`).\n\"\"\"\n\nfrom __future__ import annotations\n\nimport contextlib\nimport json\nimport subprocess\nimport sys\nimport textwrap\nimport time\nfrom copy import deepcopy\nfrom pathlib import Path\nfrom tempfile import NamedTemporaryFile\nfrom typing import Any\n\nfrom lfx.base.data.base_file import BaseFileComponent\nfrom lfx.base.data.storage_utils import parse_storage_path, read_file_bytes, validate_image_content_type\nfrom lfx.base.data.utils import TEXT_FILE_TYPES, parallel_load_data, parse_text_file_to_data\nfrom lfx.inputs import SortableListInput\nfrom lfx.inputs.inputs import DropdownInput, MessageTextInput, StrInput\nfrom lfx.io import BoolInput, FileInput, IntInput, Output, SecretStrInput\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import DataFrame # noqa: TC001\nfrom lfx.schema.message import Message\nfrom lfx.services.deps import get_settings_service, get_storage_service\nfrom lfx.utils.async_helpers import run_until_complete\nfrom lfx.utils.validate_cloud import is_astra_cloud_environment\n\n\ndef _get_storage_location_options():\n \"\"\"Get storage location options, filtering out Local if in Astra cloud environment.\"\"\"\n all_options = [{\"name\": \"AWS\", \"icon\": \"Amazon\"}, {\"name\": \"Google Drive\", \"icon\": \"google\"}]\n if is_astra_cloud_environment():\n return all_options\n return [{\"name\": \"Local\", \"icon\": \"hard-drive\"}, *all_options]\n\n\nclass FileComponent(BaseFileComponent):\n \"\"\"File component with optional Docling processing (isolated in a subprocess).\"\"\"\n\n display_name = \"Read File\"\n # description is now a dynamic property - see get_tool_description()\n _base_description = \"Loads and returns the content from uploaded files.\"\n documentation: str = \"https://docs.langflow.org/read-file\"\n icon = \"file-text\"\n name = \"File\"\n add_tool_output = True # Enable tool mode toggle without requiring tool_mode inputs\n\n # Extensions that can be processed without Docling (using standard text parsing)\n TEXT_EXTENSIONS = TEXT_FILE_TYPES\n\n # Extensions that require Docling for processing (images, advanced office formats, etc.)\n DOCLING_ONLY_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"jpg\",\n \"jpeg\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"webp\",\n ]\n\n # Docling-supported/compatible extensions; TEXT_FILE_TYPES are supported by the base loader.\n VALID_EXTENSIONS = [\n *TEXT_EXTENSIONS,\n *DOCLING_ONLY_EXTENSIONS,\n ]\n\n # Fixed export settings used when markdown export is requested.\n EXPORT_FORMAT = \"Markdown\"\n IMAGE_MODE = \"placeholder\"\n\n _base_inputs = deepcopy(BaseFileComponent.get_base_inputs())\n\n for input_item in _base_inputs:\n if isinstance(input_item, FileInput) and input_item.name == \"path\":\n input_item.real_time_refresh = True\n input_item.tool_mode = False # Disable tool mode for file upload input\n input_item.required = False # Make it optional so it doesn't error in tool mode\n break\n\n inputs = [\n SortableListInput(\n name=\"storage_location\",\n display_name=\"Storage Location\",\n placeholder=\"Select Location\",\n info=\"Choose where to read the file from.\",\n options=_get_storage_location_options(),\n real_time_refresh=True,\n limit=1,\n value=[{\"name\": \"Local\", \"icon\": \"hard-drive\"}],\n advanced=True,\n ),\n *_base_inputs,\n StrInput(\n name=\"file_path_str\",\n display_name=\"File Path\",\n info=(\n \"Path to the file to read. Used when component is called as a tool. \"\n \"If not provided, will use the uploaded file from 'path' input.\"\n ),\n show=False,\n advanced=True,\n tool_mode=True, # Required for Toolset toggle, but _get_tools() ignores this parameter\n required=False,\n ),\n # AWS S3 specific inputs\n SecretStrInput(\n name=\"aws_access_key_id\",\n display_name=\"AWS Access Key ID\",\n info=\"AWS Access key ID.\",\n show=False,\n advanced=False,\n required=True,\n ),\n SecretStrInput(\n name=\"aws_secret_access_key\",\n display_name=\"AWS Secret Key\",\n info=\"AWS Secret Key.\",\n show=False,\n advanced=False,\n required=True,\n ),\n StrInput(\n name=\"bucket_name\",\n display_name=\"S3 Bucket Name\",\n info=\"Enter the name of the S3 bucket.\",\n show=False,\n advanced=False,\n required=True,\n ),\n StrInput(\n name=\"aws_region\",\n display_name=\"AWS Region\",\n info=\"AWS region (e.g., us-east-1, eu-west-1).\",\n show=False,\n advanced=False,\n ),\n StrInput(\n name=\"s3_file_key\",\n display_name=\"S3 File Key\",\n info=\"The key (path) of the file in S3 bucket.\",\n show=False,\n advanced=False,\n required=True,\n ),\n # Google Drive specific inputs\n SecretStrInput(\n name=\"service_account_key\",\n display_name=\"GCP Credentials Secret Key\",\n info=\"Your Google Cloud Platform service account JSON key as a secret string (complete JSON content).\",\n show=False,\n advanced=False,\n required=True,\n ),\n StrInput(\n name=\"file_id\",\n display_name=\"Google Drive File ID\",\n info=(\"The Google Drive file ID to read. The file must be shared with the service account email.\"),\n show=False,\n advanced=False,\n required=True,\n ),\n BoolInput(\n name=\"advanced_mode\",\n display_name=\"Advanced Parser\",\n value=False,\n real_time_refresh=True,\n info=(\n \"Enable advanced document processing and export with Docling for PDFs, images, and office documents. \"\n \"Note that advanced document processing can consume significant resources.\"\n ),\n # Disabled in cloud\n show=not is_astra_cloud_environment(),\n ),\n DropdownInput(\n name=\"pipeline\",\n display_name=\"Pipeline\",\n info=\"Docling pipeline to use\",\n options=[\"standard\", \"vlm\"],\n value=\"standard\",\n advanced=True,\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"ocr_engine\",\n display_name=\"OCR Engine\",\n info=\"OCR engine to use. Only available when pipeline is set to 'standard'.\",\n options=[\"None\", \"easyocr\"],\n value=\"easyocr\",\n show=False,\n advanced=True,\n ),\n StrInput(\n name=\"md_image_placeholder\",\n display_name=\"Image placeholder\",\n info=\"Specify the image placeholder for markdown exports.\",\n value=\"\",\n advanced=True,\n show=False,\n ),\n StrInput(\n name=\"md_page_break_placeholder\",\n display_name=\"Page break placeholder\",\n info=\"Add this placeholder between pages in the markdown output.\",\n value=\"\",\n advanced=True,\n show=False,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n show=False,\n ),\n # Deprecated input retained for backward-compatibility.\n BoolInput(\n name=\"use_multithreading\",\n display_name=\"[Deprecated] Use Multithreading\",\n advanced=True,\n value=True,\n info=\"Set 'Processing Concurrency' greater than 1 to enable multithreading.\",\n ),\n IntInput(\n name=\"concurrency_multithreading\",\n display_name=\"Processing Concurrency\",\n advanced=True,\n info=\"When multiple files are being processed, the number of files to process concurrently.\",\n value=1,\n ),\n BoolInput(\n name=\"markdown\",\n display_name=\"Markdown Export\",\n info=\"Export processed documents to Markdown format. Only available when advanced mode is enabled.\",\n value=False,\n show=False,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\", tool_mode=True),\n ]\n\n # ------------------------------ Tool description with file names --------------\n\n def get_tool_description(self) -> str:\n \"\"\"Return a dynamic description that includes the names of uploaded files.\n\n This helps the Agent understand which files are available to read.\n \"\"\"\n base_description = type(self)._base_description # noqa: SLF001\n\n # Get the list of uploaded file paths\n file_paths = getattr(self, \"path\", None)\n if not file_paths:\n return base_description\n\n # Ensure it's a list\n if not isinstance(file_paths, list):\n file_paths = [file_paths]\n\n # Extract just the file names from the paths\n file_names = []\n for fp in file_paths:\n if fp:\n name = Path(fp).name\n file_names.append(name)\n\n if file_names:\n files_str = \", \".join(file_names)\n return f\"{base_description} Available files: {files_str}. Call this tool to read these files.\"\n\n return base_description\n\n @property\n def description(self) -> str:\n \"\"\"Dynamic description property that includes uploaded file names.\"\"\"\n return self.get_tool_description()\n\n async def _get_tools(self) -> list:\n \"\"\"Override to create a tool without parameters.\n\n The Read File component should use the files already uploaded via UI,\n not accept file paths from the Agent (which wouldn't know the internal paths).\n \"\"\"\n from langchain_core.tools import StructuredTool\n from pydantic import BaseModel\n\n # Empty schema - no parameters needed\n class EmptySchema(BaseModel):\n \"\"\"No parameters required - uses pre-uploaded files.\"\"\"\n\n async def read_files_tool() -> str:\n \"\"\"Read the content of uploaded files.\"\"\"\n try:\n if getattr(self, \"advanced_mode\", False):\n # In advanced mode, use the markdown output path so that the\n # tool shares the same Docling processing as the advanced\n # outputs rather than triggering a second subprocess via\n # load_files_message.\n self.markdown = True\n result = self.load_files_markdown()\n else:\n result = self.load_files_message()\n if hasattr(result, \"get_text\"):\n return result.get_text()\n if hasattr(result, \"text\"):\n return result.text\n return str(result)\n except (FileNotFoundError, ValueError, OSError, RuntimeError) as e:\n return f\"Error reading files: {e}\"\n\n description = self.get_tool_description()\n\n tool = StructuredTool(\n name=\"load_files_message\",\n description=description,\n coroutine=read_files_tool,\n args_schema=EmptySchema,\n handle_tool_error=True,\n tags=[\"load_files_message\"],\n metadata={\n \"display_name\": \"Read File\",\n \"display_description\": description,\n },\n )\n\n return [tool]\n\n # ------------------------------ UI helpers --------------------------------------\n\n def _path_value(self, template: dict) -> list[str]:\n \"\"\"Return the list of currently selected file paths from the template.\"\"\"\n return template.get(\"path\", {}).get(\"file_path\", [])\n\n def _disable_docling_fields_in_cloud(self, build_config: dict[str, Any]) -> None:\n \"\"\"Disable all Docling-related fields in cloud environments.\"\"\"\n if \"advanced_mode\" in build_config:\n build_config[\"advanced_mode\"][\"show\"] = False\n build_config[\"advanced_mode\"][\"value\"] = False\n # Hide all Docling-related fields\n docling_fields = (\"pipeline\", \"ocr_engine\", \"doc_key\", \"md_image_placeholder\", \"md_page_break_placeholder\")\n for field in docling_fields:\n if field in build_config:\n build_config[field][\"show\"] = False\n # Also disable OCR engine specifically\n if \"ocr_engine\" in build_config:\n build_config[\"ocr_engine\"][\"value\"] = \"None\"\n\n def update_build_config(\n self,\n build_config: dict[str, Any],\n field_value: Any,\n field_name: str | None = None,\n ) -> dict[str, Any]:\n \"\"\"Show/hide Advanced Parser and related fields based on selection context.\"\"\"\n # Update storage location options dynamically based on cloud environment\n if \"storage_location\" in build_config:\n updated_options = _get_storage_location_options()\n build_config[\"storage_location\"][\"options\"] = updated_options\n\n # Handle storage location selection\n if field_name == \"storage_location\":\n # Extract selected storage location\n selected = [location[\"name\"] for location in field_value] if isinstance(field_value, list) else []\n\n # Hide all storage-specific fields first\n storage_fields = [\n \"aws_access_key_id\",\n \"aws_secret_access_key\",\n \"bucket_name\",\n \"aws_region\",\n \"s3_file_key\",\n \"service_account_key\",\n \"file_id\",\n ]\n\n for f_name in storage_fields:\n if f_name in build_config:\n build_config[f_name][\"show\"] = False\n\n # Show fields based on selected storage location\n if len(selected) == 1:\n location = selected[0]\n\n if location == \"Local\":\n # Show file upload input for local storage\n if \"path\" in build_config:\n build_config[\"path\"][\"show\"] = True\n\n elif location == \"AWS\":\n # Hide file upload input, show AWS fields\n if \"path\" in build_config:\n build_config[\"path\"][\"show\"] = False\n\n aws_fields = [\n \"aws_access_key_id\",\n \"aws_secret_access_key\",\n \"bucket_name\",\n \"aws_region\",\n \"s3_file_key\",\n ]\n for f_name in aws_fields:\n if f_name in build_config:\n build_config[f_name][\"show\"] = True\n build_config[f_name][\"advanced\"] = False\n\n elif location == \"Google Drive\":\n # Hide file upload input, show Google Drive fields\n if \"path\" in build_config:\n build_config[\"path\"][\"show\"] = False\n\n gdrive_fields = [\"service_account_key\", \"file_id\"]\n for f_name in gdrive_fields:\n if f_name in build_config:\n build_config[f_name][\"show\"] = True\n build_config[f_name][\"advanced\"] = False\n # No storage location selected - show file upload by default\n elif \"path\" in build_config:\n build_config[\"path\"][\"show\"] = True\n\n return build_config\n\n if field_name == \"path\":\n paths = self._path_value(build_config)\n\n # Disable in cloud environments\n if is_astra_cloud_environment():\n self._disable_docling_fields_in_cloud(build_config)\n else:\n # If all files can be processed by docling, do so\n allow_advanced = all(not file_path.endswith((\".csv\", \".xlsx\", \".parquet\")) for file_path in paths)\n build_config[\"advanced_mode\"][\"show\"] = allow_advanced\n if not allow_advanced:\n build_config[\"advanced_mode\"][\"value\"] = False\n docling_fields = (\n \"pipeline\",\n \"ocr_engine\",\n \"doc_key\",\n \"md_image_placeholder\",\n \"md_page_break_placeholder\",\n )\n for field in docling_fields:\n if field in build_config:\n build_config[field][\"show\"] = False\n\n # Docling Processing\n elif field_name == \"advanced_mode\":\n # Disable in cloud environments - don't show Docling fields even if advanced_mode is toggled\n if is_astra_cloud_environment():\n self._disable_docling_fields_in_cloud(build_config)\n else:\n docling_fields = (\n \"pipeline\",\n \"ocr_engine\",\n \"doc_key\",\n \"md_image_placeholder\",\n \"md_page_break_placeholder\",\n )\n for field in docling_fields:\n if field in build_config:\n build_config[field][\"show\"] = bool(field_value)\n if field == \"pipeline\":\n build_config[field][\"advanced\"] = not bool(field_value)\n\n elif field_name == \"pipeline\":\n # Disable in cloud environments - don't show OCR engine even if pipeline is changed\n if is_astra_cloud_environment():\n self._disable_docling_fields_in_cloud(build_config)\n elif field_value == \"standard\":\n build_config[\"ocr_engine\"][\"show\"] = True\n build_config[\"ocr_engine\"][\"value\"] = \"easyocr\"\n else:\n build_config[\"ocr_engine\"][\"show\"] = False\n build_config[\"ocr_engine\"][\"value\"] = \"None\"\n\n return build_config\n\n def update_outputs(self, frontend_node: dict[str, Any], field_name: str, field_value: Any) -> dict[str, Any]: # noqa: ARG002\n \"\"\"Dynamically show outputs based on file count/type and advanced mode.\"\"\"\n if field_name not in [\"path\", \"advanced_mode\", \"pipeline\"]:\n return frontend_node\n\n template = frontend_node.get(\"template\", {})\n paths = self._path_value(template)\n if not paths:\n return frontend_node\n\n frontend_node[\"outputs\"] = []\n if len(paths) == 1:\n file_path = paths[0] if field_name == \"path\" else frontend_node[\"template\"][\"path\"][\"file_path\"][0]\n if file_path.endswith((\".csv\", \".xlsx\", \".parquet\")):\n frontend_node[\"outputs\"].append(\n Output(\n display_name=\"Structured Content\",\n name=\"dataframe\",\n method=\"load_files_structured\",\n tool_mode=True,\n ),\n )\n elif file_path.endswith(\".json\"):\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Content\", name=\"json\", method=\"load_files_json\", tool_mode=True),\n )\n\n advanced_mode = frontend_node.get(\"template\", {}).get(\"advanced_mode\", {}).get(\"value\", False)\n if advanced_mode:\n frontend_node[\"outputs\"].append(\n Output(\n display_name=\"Structured Output\",\n name=\"advanced_dataframe\",\n method=\"load_files_dataframe\",\n tool_mode=True,\n ),\n )\n frontend_node[\"outputs\"].append(\n Output(\n display_name=\"Markdown\", name=\"advanced_markdown\", method=\"load_files_markdown\", tool_mode=True\n ),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\", tool_mode=True),\n )\n else:\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\", tool_mode=True),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\", tool_mode=True),\n )\n else:\n # Multiple files => DataFrame output; advanced parser disabled\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Files\", name=\"dataframe\", method=\"load_files\", tool_mode=True)\n )\n\n return frontend_node\n\n # ------------------------------ Core processing ----------------------------------\n\n def _get_selected_storage_location(self) -> str:\n \"\"\"Get the selected storage location from the SortableListInput.\"\"\"\n if hasattr(self, \"storage_location\") and self.storage_location:\n if isinstance(self.storage_location, list) and len(self.storage_location) > 0:\n return self.storage_location[0].get(\"name\", \"\")\n if isinstance(self.storage_location, dict):\n return self.storage_location.get(\"name\", \"\")\n return \"Local\" # Default to Local if not specified\n\n def _validate_and_resolve_paths(self) -> list[BaseFileComponent.BaseFile]:\n \"\"\"Override to handle file_path_str input from tool mode and cloud storage.\n\n Priority:\n 1. Cloud storage (AWS/Google Drive) if selected\n 2. file_path_str (if provided by the tool call)\n 3. path (uploaded file from UI)\n \"\"\"\n storage_location = self._get_selected_storage_location()\n\n # Handle AWS S3\n if storage_location == \"AWS\":\n return self._read_from_aws_s3()\n\n # Handle Google Drive\n if storage_location == \"Google Drive\":\n return self._read_from_google_drive()\n\n # Handle Local storage\n # Check if file_path_str is provided (from tool mode)\n file_path_str = getattr(self, \"file_path_str\", None)\n if file_path_str:\n # Use the string path from tool mode\n from pathlib import Path\n\n from lfx.schema.data import Data\n\n # Use same resolution logic as BaseFileComponent (support storage paths)\n path_str = str(file_path_str)\n if parse_storage_path(path_str):\n try:\n resolved_path = Path(self.get_full_path(path_str))\n except (ValueError, AttributeError):\n resolved_path = Path(self.resolve_path(path_str))\n else:\n resolved_path = Path(self.resolve_path(path_str))\n\n if not resolved_path.exists():\n msg = f\"File or directory not found: {file_path_str}\"\n self.log(msg)\n if not self.silent_errors:\n raise ValueError(msg)\n return []\n\n data_obj = Data(data={self.SERVER_FILE_PATH_FIELDNAME: str(resolved_path)})\n return [BaseFileComponent.BaseFile(data_obj, resolved_path, delete_after_processing=False)]\n\n # Otherwise use the default implementation (uses path FileInput)\n return super()._validate_and_resolve_paths()\n\n def _read_from_aws_s3(self) -> list[BaseFileComponent.BaseFile]:\n \"\"\"Read file from AWS S3.\"\"\"\n from lfx.base.data.cloud_storage_utils import create_s3_client, validate_aws_credentials\n\n # Validate AWS credentials\n validate_aws_credentials(self)\n if not getattr(self, \"s3_file_key\", None):\n msg = \"S3 File Key is required\"\n raise ValueError(msg)\n\n # Create S3 client\n s3_client = create_s3_client(self)\n\n # Download file to temp location\n import tempfile\n\n # Get file extension from S3 key\n file_extension = Path(self.s3_file_key).suffix or \"\"\n\n with tempfile.NamedTemporaryFile(mode=\"wb\", suffix=file_extension, delete=False) as temp_file:\n temp_file_path = temp_file.name\n try:\n s3_client.download_fileobj(self.bucket_name, self.s3_file_key, temp_file)\n except Exception as e:\n # Clean up temp file on failure\n with contextlib.suppress(OSError):\n Path(temp_file_path).unlink()\n msg = f\"Failed to download file from S3: {e}\"\n raise RuntimeError(msg) from e\n\n # Create BaseFile object\n from lfx.schema.data import Data\n\n temp_path = Path(temp_file_path)\n data_obj = Data(data={self.SERVER_FILE_PATH_FIELDNAME: str(temp_path)})\n return [BaseFileComponent.BaseFile(data_obj, temp_path, delete_after_processing=True)]\n\n def _read_from_google_drive(self) -> list[BaseFileComponent.BaseFile]:\n \"\"\"Read file from Google Drive.\"\"\"\n import tempfile\n\n from googleapiclient.http import MediaIoBaseDownload\n\n from lfx.base.data.cloud_storage_utils import create_google_drive_service\n\n # Validate Google Drive credentials\n if not getattr(self, \"service_account_key\", None):\n msg = \"GCP Credentials Secret Key is required for Google Drive storage\"\n raise ValueError(msg)\n if not getattr(self, \"file_id\", None):\n msg = \"Google Drive File ID is required\"\n raise ValueError(msg)\n\n # Create Google Drive service with read-only scope\n drive_service = create_google_drive_service(\n self.service_account_key, scopes=[\"https://www.googleapis.com/auth/drive.readonly\"]\n )\n\n # Get file metadata to determine file name and extension\n try:\n file_metadata = drive_service.files().get(fileId=self.file_id, fields=\"name,mimeType\").execute()\n file_name = file_metadata.get(\"name\", \"download\")\n except Exception as e:\n msg = (\n f\"Unable to access file with ID '{self.file_id}'. \"\n f\"Error: {e!s}. \"\n \"Please ensure: 1) The file ID is correct, 2) The file exists, \"\n \"3) The service account has been granted access to this file.\"\n )\n raise ValueError(msg) from e\n\n # Download file to temp location\n file_extension = Path(file_name).suffix or \"\"\n with tempfile.NamedTemporaryFile(mode=\"wb\", suffix=file_extension, delete=False) as temp_file:\n temp_file_path = temp_file.name\n try:\n request = drive_service.files().get_media(fileId=self.file_id)\n downloader = MediaIoBaseDownload(temp_file, request)\n done = False\n while not done:\n _status, done = downloader.next_chunk()\n except Exception as e:\n # Clean up temp file on failure\n with contextlib.suppress(OSError):\n Path(temp_file_path).unlink()\n msg = f\"Failed to download file from Google Drive: {e}\"\n raise RuntimeError(msg) from e\n\n # Create BaseFile object\n from lfx.schema.data import Data\n\n temp_path = Path(temp_file_path)\n data_obj = Data(data={self.SERVER_FILE_PATH_FIELDNAME: str(temp_path)})\n return [BaseFileComponent.BaseFile(data_obj, temp_path, delete_after_processing=True)]\n\n def _is_docling_compatible(self, file_path: str) -> bool:\n \"\"\"Lightweight extension gate for Docling-compatible types.\"\"\"\n docling_exts = (\n \".adoc\",\n \".asciidoc\",\n \".asc\",\n \".bmp\",\n \".csv\",\n \".dotx\",\n \".dotm\",\n \".docm\",\n \".docx\",\n \".htm\",\n \".html\",\n \".jpg\",\n \".jpeg\",\n \".json\",\n \".md\",\n \".pdf\",\n \".png\",\n \".potx\",\n \".ppsx\",\n \".pptm\",\n \".potm\",\n \".ppsm\",\n \".pptx\",\n \".tiff\",\n \".txt\",\n \".xls\",\n \".xlsx\",\n \".xhtml\",\n \".xml\",\n \".webp\",\n )\n return file_path.lower().endswith(docling_exts)\n\n async def _get_local_file_for_docling(self, file_path: str) -> tuple[str, bool]:\n \"\"\"Get a local file path for Docling processing, downloading from S3 if needed.\n\n Args:\n file_path: Either a local path or S3 key (format \"flow_id/filename\")\n\n Returns:\n tuple[str, bool]: (local_path, should_delete) where should_delete indicates\n if this is a temporary file that should be cleaned up\n \"\"\"\n settings = get_settings_service().settings\n if settings.storage_type == \"local\":\n return file_path, False\n\n # S3 storage - download to temp file\n parsed = parse_storage_path(file_path)\n if not parsed:\n msg = f\"Invalid S3 path format: {file_path}. Expected 'flow_id/filename'\"\n raise ValueError(msg)\n\n storage_service = get_storage_service()\n flow_id, filename = parsed\n\n # Get file content from S3\n content = await storage_service.get_file(flow_id, filename)\n\n suffix = Path(filename).suffix\n with NamedTemporaryFile(mode=\"wb\", suffix=suffix, delete=False) as tmp_file:\n tmp_file.write(content)\n temp_path = tmp_file.name\n\n return temp_path, True\n\n def _process_docling_in_subprocess(self, file_path: str) -> Data | None:\n \"\"\"Run Docling in a separate OS process and map the result to a Data object.\n\n We avoid multiprocessing pickling by launching `python -c \"