diff --git a/.github/workflows/lint-js.yml b/.github/workflows/lint-js.yml index cdb3fbeeab..70013fe395 100644 --- a/.github/workflows/lint-js.yml +++ b/.github/workflows/lint-js.yml @@ -90,12 +90,16 @@ jobs: RELATIVE_FILES=$(echo "$CHANGED_FILES" | sed 's|^src/frontend/||') cd src/frontend + # NUL-delimit so paths containing spaces (e.g. starter-project + # specs like "Basic Prompting.spec.ts") aren't split by xargs. if ${{ inputs.allow-failure || 'false' }}; then - echo "$RELATIVE_FILES" | xargs npx @biomejs/biome check \ - --files-ignore-unknown=true \ - --diagnostic-level=error || echo "Biome found errors (non-blocking for this run)." + printf '%s\n' "$RELATIVE_FILES" | tr '\n' '\0' \ + | xargs -0 npx @biomejs/biome check \ + --files-ignore-unknown=true \ + --diagnostic-level=error || echo "Biome found errors (non-blocking for this run)." else - echo "$RELATIVE_FILES" | xargs npx @biomejs/biome check \ - --files-ignore-unknown=true \ - --diagnostic-level=error + printf '%s\n' "$RELATIVE_FILES" | tr '\n' '\0' \ + | xargs -0 npx @biomejs/biome check \ + --files-ignore-unknown=true \ + --diagnostic-level=error fi diff --git a/.secrets.baseline b/.secrets.baseline index c65c0b72a0..1a0a677b07 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -2817,7 +2817,7 @@ "filename": "src/backend/base/langflow/services/auth/utils.py", "hashed_secret": "b894b81be94cf8fa8d7536475aaec876addf05c8", "is_verified": false, - "line_number": 59, + "line_number": 75, "is_secret": false } ], @@ -2827,7 +2827,7 @@ "filename": "src/backend/base/langflow/services/database/models/api_key/crud.py", "hashed_secret": "920f8f5815b381ea692e9e7c2f7119f2b1aa620a", "is_verified": false, - "line_number": 112, + "line_number": 134, "is_secret": false } ], @@ -4007,7 +4007,7 @@ "filename": "src/backend/tests/unit/test_login.py", "hashed_secret": "8bb6118f8fd6935ad0876a3be34a717d32708ffd", "is_verified": false, - "line_number": 26, + "line_number": 97, "is_secret": false }, { @@ -4015,7 +4015,7 @@ "filename": "src/backend/tests/unit/test_login.py", "hashed_secret": "d8ecf7db8fc9ec9c31bc5c9ae2929cc599c75f8d", "is_verified": false, - "line_number": 42, + "line_number": 113, "is_secret": false } ], @@ -9287,5 +9287,5 @@ } ] }, - "generated_at": "2026-06-10T18:13:35Z" + "generated_at": "2026-06-17T19:13:55Z" } diff --git a/src/backend/base/langflow/api/v1/api_key.py b/src/backend/base/langflow/api/v1/api_key.py index 83570c6f87..8f65cf3f63 100644 --- a/src/backend/base/langflow/api/v1/api_key.py +++ b/src/backend/base/langflow/api/v1/api_key.py @@ -36,6 +36,8 @@ async def create_api_key_route( try: user_id = current_user.id return await create_api_key(db, req, user_id=user_id) + except PermissionError as e: + raise HTTPException(status_code=403, detail=str(e)) from e except Exception as e: raise HTTPException(status_code=400, detail=str(e)) from e diff --git a/src/backend/base/langflow/api/v1/authz_me.py b/src/backend/base/langflow/api/v1/authz_me.py index cb0ff032f5..5dd0874842 100644 --- a/src/backend/base/langflow/api/v1/authz_me.py +++ b/src/backend/base/langflow/api/v1/authz_me.py @@ -17,6 +17,8 @@ from fastapi import APIRouter, HTTPException, status from pydantic import BaseModel, Field, field_validator from langflow.api.utils import CurrentActiveUser +from langflow.services.auth.context import current_auth_context_for_authz +from langflow.services.authorization.access_ceiling import filter_actions_by_external_access_ceiling from langflow.services.deps import get_authorization_service router = APIRouter(prefix="/authz/me", tags=["Authorization"]) @@ -127,8 +129,15 @@ async def get_effective_permissions( resource_ids=body.resource_ids, actions=actions, domain=body.domain, - context={"is_superuser": getattr(current_user, "is_superuser", False)}, + context={ + **current_auth_context_for_authz(), + "is_superuser": current_user.is_superuser, + }, ) + permissions = { + resource_id: filter_actions_by_external_access_ceiling(allowed_actions) + for resource_id, allowed_actions in permissions.items() + } return EffectivePermissionsResponse( resource_type=body.resource_type, permissions=permissions, diff --git a/src/backend/base/langflow/api/v1/chat.py b/src/backend/base/langflow/api/v1/chat.py index bdcfcd847b..773eab975b 100644 --- a/src/backend/base/langflow/api/v1/chat.py +++ b/src/backend/base/langflow/api/v1/chat.py @@ -887,9 +887,26 @@ async def build_public_tmp( queue_service=queue_service, flow_name=flow_name or f"{authenticated_user_id or client_id}_{flow_id}", ) + # Gate the public events/cancel endpoints to jobs that were actually + # started through this public build path, preventing unauthenticated + # callers from reading or cancelling private-flow builds by job_id. + await queue_service.register_public_job(job_id) except CustomComponentValidationError as exc: await logger.awarning(f"Public flow validation failed: {exc}") raise HTTPException(status_code=400, detail="This flow cannot be executed.") from exc + except JobQueueBackendUnavailableError as exc: + # The public marker could not be persisted to the shared (Redis) backend. + # Returning the job_id anyway would hand back an un-shareable id: on a + # multi-worker deployment every other worker's public events/cancel + # endpoints would 404 it. Cancel the just-started build (best-effort) and + # surface a clean 503 instead of a 500 / an unusable job_id. + try: + await queue_service.cancel_job(job_id) + except Exception as cancel_exc: # noqa: BLE001 + await logger.awarning( + f"Failed to cancel public job {job_id} after marker persistence failed: {cancel_exc!r}" + ) + raise HTTPException(status_code=503, detail=str(exc)) from exc except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc except Exception as exc: @@ -906,6 +923,20 @@ async def build_public_tmp( ) +async def _assert_public_job(job_id: str, queue_service: JobQueueService) -> None: + """Raise HTTP 404 if job_id was not registered through the public build endpoint. + + Prevents unauthenticated callers from reading or cancelling private-flow + builds by guessing or leaking a job_id. + + Why 404 not 403: returning 403 would confirm the job exists under a different + access tier, leaking information about private builds. 404 is neutral. + """ + if not await queue_service.is_public_job_async(job_id): + # Static detail — do not reflect job_id back; avoid confirming which IDs exist. + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found") + + @router.get("/build_public_tmp/{job_id}/events") async def get_build_events_public( job_id: str, @@ -918,6 +949,7 @@ async def get_build_events_public( This endpoint does not require authentication, matching the public build endpoint. It is used by the shareable playground to consume build events. """ + await _assert_public_job(job_id, queue_service) return await get_flow_events_response( job_id=job_id, queue_service=queue_service, @@ -938,6 +970,7 @@ async def cancel_build_public( This endpoint does not require authentication, matching the public build endpoint. It is used by the shareable playground to cancel builds. """ + await _assert_public_job(job_id, queue_service) try: cancellation_success = await cancel_flow_build(job_id=job_id, queue_service=queue_service) diff --git a/src/backend/base/langflow/api/v1/endpoints.py b/src/backend/base/langflow/api/v1/endpoints.py index 054cc0d3e9..7d0fe4082c 100644 --- a/src/backend/base/langflow/api/v1/endpoints.py +++ b/src/backend/base/langflow/api/v1/endpoints.py @@ -61,6 +61,10 @@ from langflow.services.auth.utils import ( get_optional_user, ) from langflow.services.authorization import FlowAction, ensure_flow_permission +from langflow.services.authorization.access_ceiling import ( + external_access_allows, + get_current_external_access_context, +) from langflow.services.cache.utils import save_uploaded_file from langflow.services.database.models.flow.model import Flow, FlowRead from langflow.services.database.models.flow.utils import get_all_webhook_components_in_flow @@ -1226,6 +1230,7 @@ async def get_task_status(_task_id: str) -> TaskStatusResponse: async def create_upload_file( file: UploadFile, flow: Annotated[Flow, Depends(get_flow)], + current_user: CurrentActiveUser, settings_service: Annotated[SettingsService, Depends(get_settings_service)], ) -> UploadFileResponse: """Upload a file for a specific flow (Deprecated). @@ -1237,6 +1242,17 @@ async def create_upload_file( ``/api/v1/files/upload/{flow_id}`` so authenticated callers can't fill disk through this route either. """ + # Writing a file to a flow's storage is a flow mutation: enforce WRITE so + # the external access ceiling (e.g. a "viewer") cannot upload via this + # deprecated route. Mirrors the non-deprecated twin in files.py. + await ensure_flow_permission( + current_user, + FlowAction.WRITE, + flow_id=flow.id, + flow_user_id=flow.user_id, + workspace_id=flow.workspace_id, + folder_id=flow.folder_id, + ) try: max_file_size_upload = settings_service.settings.max_file_size_upload except Exception as exc: @@ -1273,6 +1289,19 @@ async def custom_component( user: CurrentActiveUser, request: Request, ) -> CustomComponentResponse: + # Building a custom component instantiates (and partially executes) posted + # code. That is a create/write-class action, so enforce the external access + # ceiling directly: a "viewer" external identity is denied while + # editor/admin (and all non-external users) pass unchanged. This route is + # not tied to a single owned resource, so the deny-only primitive is used + # instead of an ``ensure_*_permission`` guard. + external_context = get_current_external_access_context() + if external_context is not None and not external_access_allows("create", external_context): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="External credentials do not allow this action", + ) + settings_service = get_settings_service() settings = settings_service.settings all_known = None @@ -1348,6 +1377,17 @@ async def custom_component_update( HTTPException: If an error occurs during component building or updating. SerializationError: If serialization of the updated component node fails. """ + # Updating a custom component instantiates (and partially executes) posted + # code, a create/write-class action. Enforce the external access ceiling + # directly so a "viewer" external identity is denied; editor/admin (and all + # non-external users) pass unchanged. Same action string as ``custom_component``. + external_context = get_current_external_access_context() + if external_context is not None and not external_access_allows("create", external_context): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="External credentials do not allow this action", + ) + settings_service = get_settings_service() all_known = None if _requires_component_hash_lookups(settings_service.settings, user): diff --git a/src/backend/base/langflow/api/v1/files.py b/src/backend/base/langflow/api/v1/files.py index a2d1cdb98f..5e2b521f21 100644 --- a/src/backend/base/langflow/api/v1/files.py +++ b/src/backend/base/langflow/api/v1/files.py @@ -21,6 +21,7 @@ from langflow.api.utils import ( build_content_disposition, ) from langflow.api.v1.schemas import UploadFileResponse +from langflow.services.authorization import FlowAction, ensure_flow_permission from langflow.services.database.models.flow.model import Flow from langflow.services.deps import get_settings_service, get_storage_service from langflow.services.storage.service import StorageService @@ -84,9 +85,20 @@ async def upload_file( *, file: UploadFile, flow: Annotated[Flow, Depends(get_flow)], + current_user: CurrentActiveUser, storage_service: Annotated[StorageService, Depends(get_storage_service)], settings_service: Annotated[SettingsService, Depends(get_settings_service)], ) -> UploadFileResponse: + # Writing a file to a flow's storage is a flow mutation: enforce WRITE so + # the external access ceiling (e.g. a "viewer") cannot upload via this route. + await ensure_flow_permission( + current_user, + FlowAction.WRITE, + flow_id=flow.id, + flow_user_id=flow.user_id, + workspace_id=flow.workspace_id, + folder_id=flow.folder_id, + ) try: max_file_size_upload = settings_service.settings.max_file_size_upload except Exception as e: @@ -290,8 +302,19 @@ async def list_files( async def delete_file( file_name: ValidatedFileName, flow: Annotated[Flow, Depends(get_flow)], + current_user: CurrentActiveUser, storage_service: Annotated[StorageService, Depends(get_storage_service)], ): + # Deleting a file from a flow's storage mutates the flow's attachments; + # enforce WRITE so the external access ceiling (e.g. a "viewer") is honored. + await ensure_flow_permission( + current_user, + FlowAction.WRITE, + flow_id=flow.id, + flow_user_id=flow.user_id, + workspace_id=flow.workspace_id, + folder_id=flow.folder_id, + ) try: await storage_service.delete_file(flow_id=str(flow.id), file_name=file_name) except Exception as e: diff --git a/src/backend/base/langflow/api/v1/flow_version.py b/src/backend/base/langflow/api/v1/flow_version.py index 218789675b..c0d249761e 100644 --- a/src/backend/base/langflow/api/v1/flow_version.py +++ b/src/backend/base/langflow/api/v1/flow_version.py @@ -14,6 +14,7 @@ from langflow.api.utils import CurrentActiveUser, DbSession from langflow.api.utils.core import remove_api_keys from langflow.api.v1.mappers.deployments.helpers import get_owned_provider_account_or_404 from langflow.api.v1.mappers.deployments.sync import sync_flow_version_attachments +from langflow.services.authorization import FlowAction, ensure_flow_permission from langflow.services.database.models.flow.model import Flow, FlowRead from langflow.services.database.models.flow_version.crud import ( create_flow_version_entry, @@ -201,6 +202,14 @@ async def create_snapshot( body: FlowVersionCreate | None = None, ) -> FlowVersionRead: flow = await _get_user_flow(session, flow_id, current_user.id) + await ensure_flow_permission( + current_user, + FlowAction.WRITE, + flow_id=flow.id, + flow_user_id=flow.user_id, + workspace_id=flow.workspace_id, + folder_id=flow.folder_id, + ) description = body.description if body else None try: @@ -234,6 +243,14 @@ async def activate_version( save_draft: Annotated[bool, Query()] = True, ) -> FlowRead: flow = await _get_user_flow(session, flow_id, current_user.id) + await ensure_flow_permission( + current_user, + FlowAction.WRITE, + flow_id=flow.id, + flow_user_id=flow.user_id, + workspace_id=flow.workspace_id, + folder_id=flow.folder_id, + ) # Verify version entry belongs to this flow try: @@ -299,7 +316,15 @@ async def delete_version_entry( current_user: CurrentActiveUser, session: DbSession, ) -> None: - await _get_user_flow(session, flow_id, current_user.id) + flow = await _get_user_flow(session, flow_id, current_user.id) + await ensure_flow_permission( + current_user, + FlowAction.DELETE, + flow_id=flow.id, + flow_user_id=flow.user_id, + workspace_id=flow.workspace_id, + folder_id=flow.folder_id, + ) # Verify entry belongs to this flow, then delete try: diff --git a/src/backend/base/langflow/api/v1/login.py b/src/backend/base/langflow/api/v1/login.py index d74ec9c2e8..adafcf38ef 100644 --- a/src/backend/base/langflow/api/v1/login.py +++ b/src/backend/base/langflow/api/v1/login.py @@ -12,6 +12,7 @@ from slowapi.wrappers import Limit from langflow.api.utils import DbSession from langflow.api.v1.schemas import Token from langflow.initial_setup.setup import get_or_create_default_folder +from langflow.services.auth.exceptions import AuthenticationError from langflow.services.database.models.user.crud import get_user_by_id from langflow.services.database.models.user.model import UserRead from langflow.services.deps import get_auth_service, get_settings_service, get_variable_service @@ -235,16 +236,22 @@ async def get_session( It does not raise an error if unauthenticated, allowing the frontend to gracefully handle the session state. """ - from langflow.services.auth.utils import oauth2_login + from langflow.services.auth.utils import _get_external_token, oauth2_login # Try to get the token from the request (cookie or Authorization header) try: token = await oauth2_login(request) - if not token: + # Extract the external credential separately so a present-but-invalid + # native cookie can't shadow a valid external one (mirrors get_current_user + # and the WS/SSE paths). oauth2_login may already have collapsed to the + # external credential, in which case the service's dedup guard makes the + # fallback a no-op. + external_token = _get_external_token(request.headers, request.cookies) + if not token and not external_token: return SessionResponse(authenticated=False) # Validate the token and get user - user = await get_auth_service().get_current_user_from_access_token(token, db) + user = await get_auth_service().get_current_user_from_access_token(token, db, external_token=external_token) if not user or not user.is_active: return SessionResponse(authenticated=False) @@ -252,7 +259,7 @@ async def get_session( authenticated=True, user=UserRead.model_validate(user, from_attributes=True), ) - except (HTTPException, ValueError) as _: + except (AuthenticationError, HTTPException, ValueError) as _: # Any authentication error means not authenticated return SessionResponse(authenticated=False) diff --git a/src/backend/base/langflow/api/v1/mcp_projects.py b/src/backend/base/langflow/api/v1/mcp_projects.py index 3d1b44049f..527f98e74d 100644 --- a/src/backend/base/langflow/api/v1/mcp_projects.py +++ b/src/backend/base/langflow/api/v1/mcp_projects.py @@ -67,9 +67,17 @@ from langflow.api.v1.schemas import ( MCPSettings, ) from langflow.services.auth.constants import AUTO_LOGIN_WARNING +from langflow.services.auth.context import ( + AUTH_METHOD_AUTO_LOGIN, + AuthCredentialContext, + clear_current_auth_context, + set_current_auth_context, +) from langflow.services.auth.mcp_encryption import decrypt_auth_settings, encrypt_auth_settings +from langflow.services.authorization import ProjectAction, ensure_project_permission +from langflow.services.authorization.access_ceiling import clear_current_external_access_context from langflow.services.database.models import Flow, Folder -from langflow.services.database.models.api_key.crud import check_key, create_api_key +from langflow.services.database.models.api_key.crud import authenticate_api_key, create_api_key from langflow.services.database.models.api_key.model import ApiKeyCreate from langflow.services.database.models.user.crud import get_user_by_username from langflow.services.database.models.user.model import User @@ -93,6 +101,12 @@ async def verify_project_auth( This function provides authentication for MCP endpoints when using MCP Composer and no API key is provided, or checks if the API key is valid. """ + # Mirror the service.py auth entrypoints: reset request-local credential metadata at entry so a + # later branch (e.g. the composer-token fast path) never inherits stale context from a prior call. + clear_current_auth_context() + # Defensive invariant: drop any stale external access ceiling so it can't carry into MCP project auth. + clear_current_external_access_context() + settings_service = get_settings_service() project = (await db.exec(select(Folder).where(Folder.id == project_id))).first() @@ -144,9 +158,11 @@ async def verify_project_auth( ) # Validate the API key - user = await check_key(db, api_key) - if not user: + api_key_result = await authenticate_api_key(db, api_key) + if not api_key_result: raise HTTPException(status_code=401, detail="Invalid API key") + set_current_auth_context(AuthCredentialContext.from_api_key_result(api_key_result)) + user = api_key_result.user # Verify user has access to the project project_access = ( @@ -171,6 +187,7 @@ async def _superuser_fallback(db: AsyncSession, settings_service) -> User: result = await get_user_by_username(db, settings_service.auth_settings.SUPERUSER) if result: logger.warning(AUTO_LOGIN_WARNING) + set_current_auth_context(AuthCredentialContext(method=AUTH_METHOD_AUTO_LOGIN)) return result raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, @@ -546,6 +563,18 @@ async def update_project_mcp_settings( if not project: raise HTTPException(status_code=404, detail="Project not found") + # Mutating flow MCP exposure + project MCP auth settings is a project + # WRITE: enforce so the external access ceiling (e.g. a "viewer") + # cannot change MCP settings. The owner with no ceiling fast-paths via + # owner-override; behavior is unchanged when the feature is off. + await ensure_project_permission( + current_user, + ProjectAction.WRITE, + project_id=project_id, + project_user_id=project.user_id, + workspace_id=project.workspace_id, + ) + # Track if MCP Composer needs to be started or stopped should_handle_mcp_composer = False should_start_composer = False diff --git a/src/backend/base/langflow/api/v1/mcp_utils.py b/src/backend/base/langflow/api/v1/mcp_utils.py index cdbb8a6255..9c444b89fd 100644 --- a/src/backend/base/langflow/api/v1/mcp_utils.py +++ b/src/backend/base/langflow/api/v1/mcp_utils.py @@ -25,6 +25,7 @@ from langflow.api.v1.endpoints import simple_run_flow from langflow.api.v1.schemas import SimplifiedAPIRequest from langflow.helpers.flow import json_schema_from_flow from langflow.schema.message import Message +from langflow.services.authorization import FlowAction, ensure_flow_permission from langflow.services.database.models import Flow from langflow.services.database.models.file.model import File as UserFile from langflow.services.database.models.user.model import User @@ -287,6 +288,18 @@ async def handle_call_tool( msg = f"Flow '{name}' not found in project {project_id}" raise ValueError(msg) + # Enforce execute permission (owner override + external access ceiling) + # before running the flow. Without this an external "viewer" could run a + # flow as a tool, escaping the deny-only access ceiling. + await ensure_flow_permission( + current_user, + FlowAction.EXECUTE, + flow_id=flow.id, + flow_user_id=flow.user_id, + workspace_id=flow.workspace_id, + folder_id=flow.folder_id, + ) + # Process inputs processed_inputs = dict(arguments) diff --git a/src/backend/base/langflow/api/v1/memories.py b/src/backend/base/langflow/api/v1/memories.py index c4b020b12d..1498e566b6 100644 --- a/src/backend/base/langflow/api/v1/memories.py +++ b/src/backend/base/langflow/api/v1/memories.py @@ -32,6 +32,7 @@ from pydantic import BaseModel from sqlmodel import select from langflow.api.utils import CurrentActiveUser +from langflow.services.authorization import KnowledgeBaseAction, ensure_knowledge_base_permission from langflow.services.database.models.memory_base.model import ( MemoryBase, MemoryBaseCreate, @@ -101,6 +102,15 @@ async def create_memory_base( - Returns 409 if a Memory Base with the same name already exists for this user. - Returns 422 if preprocessing=true but preproc_model is missing. """ + # Memory bases are knowledge-base-backed resources; guard via the KB family. + # Passing the actor as owner makes the owner-override path return early when + # no access ceiling is set, preserving existing behavior, while a deny-only + # external ceiling (e.g. a "viewer") is still enforced. + await ensure_knowledge_base_permission( + current_user, + KnowledgeBaseAction.CREATE, + kb_user_id=current_user.id, + ) try: mb = await get_memory_base_service().create(payload, user_id=current_user.id) except PermissionError as exc: @@ -267,6 +277,19 @@ async def update_memory_base( Preprocessing fields (preprocessing, preproc_model, preproc_instructions, preproc_kill_phrase) are immutable after creation and cannot be patched. """ + # Resolve the memory base so the guard receives the REAL owner / kb identity: + # owner-override then fires only for genuine owners, a non-owner reaches the + # registered plugin's enforce path, and audit rows carry the kb id. + mb = await get_memory_base_service().get(memory_base_id, user_id=current_user.id) + if mb is None: + raise HTTPException(status_code=404, detail="Memory base not found") + await ensure_knowledge_base_permission( + current_user, + KnowledgeBaseAction.WRITE, + kb_id=memory_base_id, + kb_user_id=mb.user_id, + kb_name=mb.kb_name, + ) try: mb = await get_memory_base_service().update(memory_base_id, user_id=current_user.id, patch=patch) except PreprocessingValidationError as exc: @@ -287,6 +310,18 @@ async def delete_memory_base( removed. The associated KB directory is deleted from disk afterwards (best-effort — a disk failure will not affect the 204 response). """ + # Resolve the memory base so the guard receives the REAL owner / kb identity + # (owner-override only for genuine owners; non-owners hit plugin enforce). + mb = await get_memory_base_service().get(memory_base_id, user_id=current_user.id) + if mb is None: + raise HTTPException(status_code=404, detail="Memory base not found") + await ensure_knowledge_base_permission( + current_user, + KnowledgeBaseAction.DELETE, + kb_id=memory_base_id, + kb_user_id=mb.user_id, + kb_name=mb.kb_name, + ) deleted = await get_memory_base_service().delete(memory_base_id, user_id=current_user.id) if not deleted: raise HTTPException(status_code=404, detail="Memory base not found") @@ -308,6 +343,18 @@ async def flush_memory_base( Returns 409 Conflict if an ingestion task is already in progress for the given (memory_base_id, session_id) pair to prevent concurrent indexing. """ + # Resolve the memory base so the guard receives the REAL owner / kb identity + # (owner-override only for genuine owners; non-owners hit plugin enforce). + mb = await get_memory_base_service().get(memory_base_id, user_id=current_user.id) + if mb is None: + raise HTTPException(status_code=404, detail="Memory base not found") + await ensure_knowledge_base_permission( + current_user, + KnowledgeBaseAction.INGEST, + kb_id=memory_base_id, + kb_user_id=mb.user_id, + kb_name=mb.kb_name, + ) try: job_id = await get_memory_base_service().trigger_ingestion( memory_base_id=memory_base_id, @@ -355,6 +402,18 @@ async def regenerate_memory_base( Use this to recover from external Chroma directory deletions or vector DB corruption. All MemoryBaseSession.cursor_id values are set to None before re-running ingestion. """ + # Resolve the memory base so the guard receives the REAL owner / kb identity + # (owner-override only for genuine owners; non-owners hit plugin enforce). + mb = await get_memory_base_service().get(memory_base_id, user_id=current_user.id) + if mb is None: + raise HTTPException(status_code=404, detail="Memory base not found") + await ensure_knowledge_base_permission( + current_user, + KnowledgeBaseAction.INGEST, + kb_id=memory_base_id, + kb_user_id=mb.user_id, + kb_name=mb.kb_name, + ) try: job_ids = await get_memory_base_service().regenerate(memory_base_id, user_id=current_user.id) except ValueError as exc: diff --git a/src/backend/base/langflow/api/v1/models.py b/src/backend/base/langflow/api/v1/models.py index a704bc0ff1..968a4545bd 100644 --- a/src/backend/base/langflow/api/v1/models.py +++ b/src/backend/base/langflow/api/v1/models.py @@ -17,6 +17,7 @@ from pydantic import BaseModel, field_validator from langflow.api.utils import CurrentActiveUser, DbSession from langflow.services.auth.utils import get_current_active_user +from langflow.services.authorization import VariableAction, ensure_variable_permission from langflow.services.deps import get_variable_service from langflow.services.variable.constants import GENERIC_TYPE from langflow.services.variable.service import DatabaseVariableService @@ -614,6 +615,14 @@ async def update_enabled_models( Accepts a list of model IDs with their desired enabled status. This only affects model-level enablement - provider credentials must still be configured. """ + # Persists the enabled/disabled model lists as the user's own Variables: a + # variable WRITE. Enforce so the external access ceiling caps a "viewer"; + # the owner with no ceiling fast-paths via owner-override. + await ensure_variable_permission( + current_user, + VariableAction.WRITE, + variable_user_id=current_user.id, + ) variable_service = get_variable_service() if not isinstance(variable_service, DatabaseVariableService): raise HTTPException( @@ -755,6 +764,14 @@ async def set_default_model( request: DefaultModelRequest, ): """Set the default model for the current user.""" + # Creating/updating the default-model Variable is a variable WRITE. Enforce + # so the external access ceiling caps a "viewer"; the owner with no ceiling + # fast-paths via owner-override. + await ensure_variable_permission( + current_user, + VariableAction.WRITE, + variable_user_id=current_user.id, + ) variable_service = get_variable_service() if not isinstance(variable_service, DatabaseVariableService): raise HTTPException( @@ -830,6 +847,14 @@ async def clear_default_model( model_type: Annotated[str, Query(description="Type of model: 'language' or 'embedding'")] = "language", ): """Clear the default model for the current user.""" + # Deleting the default-model Variable is a variable DELETE. Enforce so the + # external access ceiling caps a "viewer"; the owner with no ceiling + # fast-paths via owner-override. + await ensure_variable_permission( + current_user, + VariableAction.DELETE, + variable_user_id=current_user.id, + ) variable_service = get_variable_service() if not isinstance(variable_service, DatabaseVariableService): raise HTTPException( diff --git a/src/backend/base/langflow/initial_setup/starter_projects/Blog Writer.json b/src/backend/base/langflow/initial_setup/starter_projects/Blog Writer.json index f71c5bcbbd..8e4cceb967 100644 --- a/src/backend/base/langflow/initial_setup/starter_projects/Blog Writer.json +++ b/src/backend/base/langflow/initial_setup/starter_projects/Blog Writer.json @@ -850,6 +850,7 @@ "max_depth", "prevent_outside", "use_async", + "follow_redirects", "format", "timeout", "headers", @@ -864,7 +865,7 @@ "legacy": false, "lf_version": "1.4.2", "metadata": { - "code_hash": "2e225a90043a", + "code_hash": "52b5bd602242", "dependencies": { "dependencies": [ { @@ -976,7 +977,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n async with self._build_http_client(url, validated_ips) as client:\n try:\n response = await client.get(url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n else:\n return html_content, metadata\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(start_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(start_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" + "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n# HTTP status codes that carry a redirect Location header (RFC 9110).\nREDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})\n\n# Maximum number of redirects to follow. Each hop is re-validated for SSRF safety;\n# matches httpx's default and the API Request component.\nMAX_REDIRECTS = 20\n\n# Default ports per scheme, used to compare redirect origins.\nDEFAULT_SCHEME_PORTS = {\"http\": 80, \"https\": 443}\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n info=(\n \"If enabled, follows HTTP redirects such as http→https or www/non-www \"\n \"normalization, which most sites rely on to serve their content. When SSRF \"\n \"protection is enabled (the default), every redirect hop is re-validated against \"\n \"the same blocked-IP denylist and DNS-pinned before it is fetched, so following \"\n \"redirects cannot be used to reach internal resources. Disable to capture the \"\n \"first response as-is without following redirects.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different origin.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to an origin other than the one the\n caller intended them for. Headers are kept only when the redirect stays on the\n same origin (scheme, host, port) or is a direct https upgrade of the same host\n on default ports - the exact cases where httpx keeps the Authorization header.\n \"\"\"\n if not headers:\n return headers\n current, nxt = urlparse(current_url), urlparse(next_url)\n current_port = current.port or DEFAULT_SCHEME_PORTS.get(current.scheme)\n next_port = nxt.port or DEFAULT_SCHEME_PORTS.get(nxt.scheme)\n same_origin = (current.scheme, current.hostname, current_port) == (nxt.scheme, nxt.hostname, next_port)\n https_upgrade = (\n current.hostname == nxt.hostname\n and current.scheme == \"http\"\n and nxt.scheme == \"https\"\n and current_port == DEFAULT_SCHEME_PORTS[\"http\"]\n and next_port == DEFAULT_SCHEME_PORTS[\"https\"]\n )\n if same_origin or https_upgrade:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n def _process_response(self, response: httpx.Response) -> tuple[str, dict]:\n \"\"\"Turn a final (non-redirect) response into its HTML content and metadata.\n\n Args:\n response: The HTTP response to process\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {response.url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {response.url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n return html_content, metadata\n\n async def _fetch_with_revalidated_redirects(\n self, url: str, validated_ips: list[str], headers: dict\n ) -> tuple[str, dict]:\n \"\"\"Fetch ``url``, following redirects manually and re-validating every hop.\n\n This closes an SSRF bypass: with redirects enabled, httpx would otherwise\n auto-follow a redirect from a validated public URL to an internal address that\n the pinned transport never checked (the redirect target is a different host, so\n it is not in the pin map). Each redirect ``Location`` is resolved (relative\n locations included) and re-validated with ``ensure_url`` - the same private/\n loopback/link-local denylist and DNS pinning applied to the initial request -\n before any connection to it is made. A blocked hop raises ``ValueError`` (unless\n ``continue_on_failure``); the chain is capped at ``MAX_REDIRECTS`` hops.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning of the initial URL\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n current_url = url\n current_ips = validated_ips\n\n for _ in range(MAX_REDIRECTS + 1):\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.get(current_url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n location = response.headers.get(\"location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n try:\n validated_next_url, current_ips = self.ensure_url(next_url)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping blocked or invalid redirect to {next_url}: {e}\")\n return \"\", {}\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = validated_next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return self._process_response(response)\n\n # Exhausted the redirect budget.\n if self.continue_on_failure:\n logger.warning(f\"Exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\")\n return \"\", {}\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection, following redirects safely.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n try:\n # When SSRF protection is enabled we must follow redirects manually so each hop\n # is re-validated and DNS-pinned; letting httpx auto-follow would connect to the\n # redirect target without pinning (different host, not in the pin map) and re-open\n # the SSRF hole that DNS pinning closes. With protection disabled there is no pin\n # to bypass, so httpx can follow redirects natively.\n if self.follow_redirects and is_ssrf_protection_enabled():\n return await self._fetch_with_revalidated_redirects(url, validated_ips, headers)\n\n async with self._build_http_client(url, validated_ips) as client:\n response = await client.get(\n url, headers=headers, timeout=self.timeout, follow_redirects=self.follow_redirects\n )\n return self._process_response(response)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n # Redirects may have landed on a different canonical URL (http->https, www\n # normalization). Resolve relative links and the prevent_outside check against\n # the URL the content actually came from, and mark it visited so links back to\n # the canonical form are not re-crawled.\n base_url = metadata.get(\"source\") or start_url\n visited.add(base_url)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(base_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(base_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" }, "continue_on_failure": { "_input_type": "BoolInput", @@ -1014,6 +1015,26 @@ "type": "bool", "value": true }, + "follow_redirects": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Follow Redirects", + "dynamic": false, + "info": "If enabled, follows HTTP redirects such as http→https or www/non-www normalization, which most sites rely on to serve their content. When SSRF protection is enabled (the default), every redirect hop is re-validated against the same blocked-IP denylist and DNS-pinned before it is fetched, so following redirects cannot be used to reach internal resources. Disable to capture the first response as-is without following redirects.", + "list": false, + "list_add_label": "Add More", + "name": "follow_redirects", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + }, "format": { "_input_type": "DropdownInput", "advanced": true, diff --git a/src/backend/base/langflow/initial_setup/starter_projects/Custom Component Generator.json b/src/backend/base/langflow/initial_setup/starter_projects/Custom Component Generator.json index 233fcaee9e..9eb3659e04 100644 --- a/src/backend/base/langflow/initial_setup/starter_projects/Custom Component Generator.json +++ b/src/backend/base/langflow/initial_setup/starter_projects/Custom Component Generator.json @@ -908,6 +908,7 @@ "max_depth", "prevent_outside", "use_async", + "follow_redirects", "format", "timeout", "headers", @@ -921,7 +922,7 @@ "legacy": false, "lf_version": "1.6.0", "metadata": { - "code_hash": "2e225a90043a", + "code_hash": "52b5bd602242", "dependencies": { "dependencies": [ { @@ -1031,7 +1032,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n async with self._build_http_client(url, validated_ips) as client:\n try:\n response = await client.get(url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n else:\n return html_content, metadata\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(start_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(start_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" + "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n# HTTP status codes that carry a redirect Location header (RFC 9110).\nREDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})\n\n# Maximum number of redirects to follow. Each hop is re-validated for SSRF safety;\n# matches httpx's default and the API Request component.\nMAX_REDIRECTS = 20\n\n# Default ports per scheme, used to compare redirect origins.\nDEFAULT_SCHEME_PORTS = {\"http\": 80, \"https\": 443}\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n info=(\n \"If enabled, follows HTTP redirects such as http→https or www/non-www \"\n \"normalization, which most sites rely on to serve their content. When SSRF \"\n \"protection is enabled (the default), every redirect hop is re-validated against \"\n \"the same blocked-IP denylist and DNS-pinned before it is fetched, so following \"\n \"redirects cannot be used to reach internal resources. Disable to capture the \"\n \"first response as-is without following redirects.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different origin.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to an origin other than the one the\n caller intended them for. Headers are kept only when the redirect stays on the\n same origin (scheme, host, port) or is a direct https upgrade of the same host\n on default ports - the exact cases where httpx keeps the Authorization header.\n \"\"\"\n if not headers:\n return headers\n current, nxt = urlparse(current_url), urlparse(next_url)\n current_port = current.port or DEFAULT_SCHEME_PORTS.get(current.scheme)\n next_port = nxt.port or DEFAULT_SCHEME_PORTS.get(nxt.scheme)\n same_origin = (current.scheme, current.hostname, current_port) == (nxt.scheme, nxt.hostname, next_port)\n https_upgrade = (\n current.hostname == nxt.hostname\n and current.scheme == \"http\"\n and nxt.scheme == \"https\"\n and current_port == DEFAULT_SCHEME_PORTS[\"http\"]\n and next_port == DEFAULT_SCHEME_PORTS[\"https\"]\n )\n if same_origin or https_upgrade:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n def _process_response(self, response: httpx.Response) -> tuple[str, dict]:\n \"\"\"Turn a final (non-redirect) response into its HTML content and metadata.\n\n Args:\n response: The HTTP response to process\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {response.url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {response.url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n return html_content, metadata\n\n async def _fetch_with_revalidated_redirects(\n self, url: str, validated_ips: list[str], headers: dict\n ) -> tuple[str, dict]:\n \"\"\"Fetch ``url``, following redirects manually and re-validating every hop.\n\n This closes an SSRF bypass: with redirects enabled, httpx would otherwise\n auto-follow a redirect from a validated public URL to an internal address that\n the pinned transport never checked (the redirect target is a different host, so\n it is not in the pin map). Each redirect ``Location`` is resolved (relative\n locations included) and re-validated with ``ensure_url`` - the same private/\n loopback/link-local denylist and DNS pinning applied to the initial request -\n before any connection to it is made. A blocked hop raises ``ValueError`` (unless\n ``continue_on_failure``); the chain is capped at ``MAX_REDIRECTS`` hops.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning of the initial URL\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n current_url = url\n current_ips = validated_ips\n\n for _ in range(MAX_REDIRECTS + 1):\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.get(current_url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n location = response.headers.get(\"location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n try:\n validated_next_url, current_ips = self.ensure_url(next_url)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping blocked or invalid redirect to {next_url}: {e}\")\n return \"\", {}\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = validated_next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return self._process_response(response)\n\n # Exhausted the redirect budget.\n if self.continue_on_failure:\n logger.warning(f\"Exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\")\n return \"\", {}\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection, following redirects safely.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n try:\n # When SSRF protection is enabled we must follow redirects manually so each hop\n # is re-validated and DNS-pinned; letting httpx auto-follow would connect to the\n # redirect target without pinning (different host, not in the pin map) and re-open\n # the SSRF hole that DNS pinning closes. With protection disabled there is no pin\n # to bypass, so httpx can follow redirects natively.\n if self.follow_redirects and is_ssrf_protection_enabled():\n return await self._fetch_with_revalidated_redirects(url, validated_ips, headers)\n\n async with self._build_http_client(url, validated_ips) as client:\n response = await client.get(\n url, headers=headers, timeout=self.timeout, follow_redirects=self.follow_redirects\n )\n return self._process_response(response)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n # Redirects may have landed on a different canonical URL (http->https, www\n # normalization). Resolve relative links and the prevent_outside check against\n # the URL the content actually came from, and mark it visited so links back to\n # the canonical form are not re-crawled.\n base_url = metadata.get(\"source\") or start_url\n visited.add(base_url)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(base_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(base_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" }, "continue_on_failure": { "_input_type": "BoolInput", @@ -1069,6 +1070,26 @@ "type": "bool", "value": true }, + "follow_redirects": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Follow Redirects", + "dynamic": false, + "info": "If enabled, follows HTTP redirects such as http→https or www/non-www normalization, which most sites rely on to serve their content. When SSRF protection is enabled (the default), every redirect hop is re-validated against the same blocked-IP denylist and DNS-pinned before it is fetched, so following redirects cannot be used to reach internal resources. Disable to capture the first response as-is without following redirects.", + "list": false, + "list_add_label": "Add More", + "name": "follow_redirects", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + }, "format": { "_input_type": "DropdownInput", "advanced": true, @@ -1301,6 +1322,7 @@ "max_depth", "prevent_outside", "use_async", + "follow_redirects", "format", "timeout", "headers", @@ -1314,7 +1336,7 @@ "legacy": false, "lf_version": "1.6.0", "metadata": { - "code_hash": "2e225a90043a", + "code_hash": "52b5bd602242", "dependencies": { "dependencies": [ { @@ -1424,7 +1446,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n async with self._build_http_client(url, validated_ips) as client:\n try:\n response = await client.get(url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n else:\n return html_content, metadata\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(start_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(start_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" + "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n# HTTP status codes that carry a redirect Location header (RFC 9110).\nREDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})\n\n# Maximum number of redirects to follow. Each hop is re-validated for SSRF safety;\n# matches httpx's default and the API Request component.\nMAX_REDIRECTS = 20\n\n# Default ports per scheme, used to compare redirect origins.\nDEFAULT_SCHEME_PORTS = {\"http\": 80, \"https\": 443}\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n info=(\n \"If enabled, follows HTTP redirects such as http→https or www/non-www \"\n \"normalization, which most sites rely on to serve their content. When SSRF \"\n \"protection is enabled (the default), every redirect hop is re-validated against \"\n \"the same blocked-IP denylist and DNS-pinned before it is fetched, so following \"\n \"redirects cannot be used to reach internal resources. Disable to capture the \"\n \"first response as-is without following redirects.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different origin.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to an origin other than the one the\n caller intended them for. Headers are kept only when the redirect stays on the\n same origin (scheme, host, port) or is a direct https upgrade of the same host\n on default ports - the exact cases where httpx keeps the Authorization header.\n \"\"\"\n if not headers:\n return headers\n current, nxt = urlparse(current_url), urlparse(next_url)\n current_port = current.port or DEFAULT_SCHEME_PORTS.get(current.scheme)\n next_port = nxt.port or DEFAULT_SCHEME_PORTS.get(nxt.scheme)\n same_origin = (current.scheme, current.hostname, current_port) == (nxt.scheme, nxt.hostname, next_port)\n https_upgrade = (\n current.hostname == nxt.hostname\n and current.scheme == \"http\"\n and nxt.scheme == \"https\"\n and current_port == DEFAULT_SCHEME_PORTS[\"http\"]\n and next_port == DEFAULT_SCHEME_PORTS[\"https\"]\n )\n if same_origin or https_upgrade:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n def _process_response(self, response: httpx.Response) -> tuple[str, dict]:\n \"\"\"Turn a final (non-redirect) response into its HTML content and metadata.\n\n Args:\n response: The HTTP response to process\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {response.url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {response.url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n return html_content, metadata\n\n async def _fetch_with_revalidated_redirects(\n self, url: str, validated_ips: list[str], headers: dict\n ) -> tuple[str, dict]:\n \"\"\"Fetch ``url``, following redirects manually and re-validating every hop.\n\n This closes an SSRF bypass: with redirects enabled, httpx would otherwise\n auto-follow a redirect from a validated public URL to an internal address that\n the pinned transport never checked (the redirect target is a different host, so\n it is not in the pin map). Each redirect ``Location`` is resolved (relative\n locations included) and re-validated with ``ensure_url`` - the same private/\n loopback/link-local denylist and DNS pinning applied to the initial request -\n before any connection to it is made. A blocked hop raises ``ValueError`` (unless\n ``continue_on_failure``); the chain is capped at ``MAX_REDIRECTS`` hops.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning of the initial URL\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n current_url = url\n current_ips = validated_ips\n\n for _ in range(MAX_REDIRECTS + 1):\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.get(current_url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n location = response.headers.get(\"location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n try:\n validated_next_url, current_ips = self.ensure_url(next_url)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping blocked or invalid redirect to {next_url}: {e}\")\n return \"\", {}\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = validated_next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return self._process_response(response)\n\n # Exhausted the redirect budget.\n if self.continue_on_failure:\n logger.warning(f\"Exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\")\n return \"\", {}\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection, following redirects safely.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n try:\n # When SSRF protection is enabled we must follow redirects manually so each hop\n # is re-validated and DNS-pinned; letting httpx auto-follow would connect to the\n # redirect target without pinning (different host, not in the pin map) and re-open\n # the SSRF hole that DNS pinning closes. With protection disabled there is no pin\n # to bypass, so httpx can follow redirects natively.\n if self.follow_redirects and is_ssrf_protection_enabled():\n return await self._fetch_with_revalidated_redirects(url, validated_ips, headers)\n\n async with self._build_http_client(url, validated_ips) as client:\n response = await client.get(\n url, headers=headers, timeout=self.timeout, follow_redirects=self.follow_redirects\n )\n return self._process_response(response)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n # Redirects may have landed on a different canonical URL (http->https, www\n # normalization). Resolve relative links and the prevent_outside check against\n # the URL the content actually came from, and mark it visited so links back to\n # the canonical form are not re-crawled.\n base_url = metadata.get(\"source\") or start_url\n visited.add(base_url)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(base_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(base_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" }, "continue_on_failure": { "_input_type": "BoolInput", @@ -1462,6 +1484,26 @@ "type": "bool", "value": true }, + "follow_redirects": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Follow Redirects", + "dynamic": false, + "info": "If enabled, follows HTTP redirects such as http→https or www/non-www normalization, which most sites rely on to serve their content. When SSRF protection is enabled (the default), every redirect hop is re-validated against the same blocked-IP denylist and DNS-pinned before it is fetched, so following redirects cannot be used to reach internal resources. Disable to capture the first response as-is without following redirects.", + "list": false, + "list_add_label": "Add More", + "name": "follow_redirects", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + }, "format": { "_input_type": "DropdownInput", "advanced": true, @@ -1700,6 +1742,7 @@ "max_depth", "prevent_outside", "use_async", + "follow_redirects", "format", "timeout", "headers", @@ -1713,7 +1756,7 @@ "legacy": false, "lf_version": "1.6.0", "metadata": { - "code_hash": "2e225a90043a", + "code_hash": "52b5bd602242", "dependencies": { "dependencies": [ { @@ -1823,7 +1866,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n async with self._build_http_client(url, validated_ips) as client:\n try:\n response = await client.get(url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n else:\n return html_content, metadata\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(start_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(start_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" + "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n# HTTP status codes that carry a redirect Location header (RFC 9110).\nREDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})\n\n# Maximum number of redirects to follow. Each hop is re-validated for SSRF safety;\n# matches httpx's default and the API Request component.\nMAX_REDIRECTS = 20\n\n# Default ports per scheme, used to compare redirect origins.\nDEFAULT_SCHEME_PORTS = {\"http\": 80, \"https\": 443}\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n info=(\n \"If enabled, follows HTTP redirects such as http→https or www/non-www \"\n \"normalization, which most sites rely on to serve their content. When SSRF \"\n \"protection is enabled (the default), every redirect hop is re-validated against \"\n \"the same blocked-IP denylist and DNS-pinned before it is fetched, so following \"\n \"redirects cannot be used to reach internal resources. Disable to capture the \"\n \"first response as-is without following redirects.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different origin.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to an origin other than the one the\n caller intended them for. Headers are kept only when the redirect stays on the\n same origin (scheme, host, port) or is a direct https upgrade of the same host\n on default ports - the exact cases where httpx keeps the Authorization header.\n \"\"\"\n if not headers:\n return headers\n current, nxt = urlparse(current_url), urlparse(next_url)\n current_port = current.port or DEFAULT_SCHEME_PORTS.get(current.scheme)\n next_port = nxt.port or DEFAULT_SCHEME_PORTS.get(nxt.scheme)\n same_origin = (current.scheme, current.hostname, current_port) == (nxt.scheme, nxt.hostname, next_port)\n https_upgrade = (\n current.hostname == nxt.hostname\n and current.scheme == \"http\"\n and nxt.scheme == \"https\"\n and current_port == DEFAULT_SCHEME_PORTS[\"http\"]\n and next_port == DEFAULT_SCHEME_PORTS[\"https\"]\n )\n if same_origin or https_upgrade:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n def _process_response(self, response: httpx.Response) -> tuple[str, dict]:\n \"\"\"Turn a final (non-redirect) response into its HTML content and metadata.\n\n Args:\n response: The HTTP response to process\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {response.url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {response.url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n return html_content, metadata\n\n async def _fetch_with_revalidated_redirects(\n self, url: str, validated_ips: list[str], headers: dict\n ) -> tuple[str, dict]:\n \"\"\"Fetch ``url``, following redirects manually and re-validating every hop.\n\n This closes an SSRF bypass: with redirects enabled, httpx would otherwise\n auto-follow a redirect from a validated public URL to an internal address that\n the pinned transport never checked (the redirect target is a different host, so\n it is not in the pin map). Each redirect ``Location`` is resolved (relative\n locations included) and re-validated with ``ensure_url`` - the same private/\n loopback/link-local denylist and DNS pinning applied to the initial request -\n before any connection to it is made. A blocked hop raises ``ValueError`` (unless\n ``continue_on_failure``); the chain is capped at ``MAX_REDIRECTS`` hops.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning of the initial URL\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n current_url = url\n current_ips = validated_ips\n\n for _ in range(MAX_REDIRECTS + 1):\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.get(current_url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n location = response.headers.get(\"location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n try:\n validated_next_url, current_ips = self.ensure_url(next_url)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping blocked or invalid redirect to {next_url}: {e}\")\n return \"\", {}\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = validated_next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return self._process_response(response)\n\n # Exhausted the redirect budget.\n if self.continue_on_failure:\n logger.warning(f\"Exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\")\n return \"\", {}\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection, following redirects safely.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n try:\n # When SSRF protection is enabled we must follow redirects manually so each hop\n # is re-validated and DNS-pinned; letting httpx auto-follow would connect to the\n # redirect target without pinning (different host, not in the pin map) and re-open\n # the SSRF hole that DNS pinning closes. With protection disabled there is no pin\n # to bypass, so httpx can follow redirects natively.\n if self.follow_redirects and is_ssrf_protection_enabled():\n return await self._fetch_with_revalidated_redirects(url, validated_ips, headers)\n\n async with self._build_http_client(url, validated_ips) as client:\n response = await client.get(\n url, headers=headers, timeout=self.timeout, follow_redirects=self.follow_redirects\n )\n return self._process_response(response)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n # Redirects may have landed on a different canonical URL (http->https, www\n # normalization). Resolve relative links and the prevent_outside check against\n # the URL the content actually came from, and mark it visited so links back to\n # the canonical form are not re-crawled.\n base_url = metadata.get(\"source\") or start_url\n visited.add(base_url)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(base_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(base_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" }, "continue_on_failure": { "_input_type": "BoolInput", @@ -1861,6 +1904,26 @@ "type": "bool", "value": true }, + "follow_redirects": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Follow Redirects", + "dynamic": false, + "info": "If enabled, follows HTTP redirects such as http→https or www/non-www normalization, which most sites rely on to serve their content. When SSRF protection is enabled (the default), every redirect hop is re-validated against the same blocked-IP denylist and DNS-pinned before it is fetched, so following redirects cannot be used to reach internal resources. Disable to capture the first response as-is without following redirects.", + "list": false, + "list_add_label": "Add More", + "name": "follow_redirects", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + }, "format": { "_input_type": "DropdownInput", "advanced": true, diff --git a/src/backend/base/langflow/initial_setup/starter_projects/Pokédex Agent.json b/src/backend/base/langflow/initial_setup/starter_projects/Pokédex Agent.json index 05ba59a849..3c6f24be8c 100644 --- a/src/backend/base/langflow/initial_setup/starter_projects/Pokédex Agent.json +++ b/src/backend/base/langflow/initial_setup/starter_projects/Pokédex Agent.json @@ -771,7 +771,7 @@ "key": "APIRequest", "legacy": false, "metadata": { - "code_hash": "1a052ebb9519", + "code_hash": "716a753c256e", "dependencies": { "dependencies": [ { @@ -889,7 +889,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import json\nimport re\nimport tempfile\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom typing import Any\nfrom urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse\n\nimport aiofiles\nimport aiofiles.os as aiofiles_os\nimport httpx\nimport validators\n\nfrom lfx.base.curl.parse import parse_context\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs.inputs import TabInput\nfrom lfx.io import (\n BoolInput,\n DataInput,\n DropdownInput,\n IntInput,\n MessageTextInput,\n MultilineInput,\n Output,\n TableInput,\n)\nfrom lfx.schema.data import Data\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.utils.component_utils import set_current_fields, set_field_advanced, set_field_display\n\n# SSRF Protection imports - for preventing Server-Side Request Forgery attacks\nfrom lfx.utils.ssrf_protection import (\n SSRFProtectionError,\n is_ssrf_protection_enabled,\n validate_and_resolve_url,\n)\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Define fields for each mode\nMODE_FIELDS = {\n \"URL\": [\n \"url_input\",\n \"method\",\n ],\n \"cURL\": [\"curl_input\"],\n}\n\n# Fields that should always be visible\nDEFAULT_FIELDS = [\"mode\"]\n\n# HTTP redirect status codes (RFC 9110).\nHTTP_MOVED_PERMANENTLY = 301\nHTTP_FOUND = 302\nHTTP_SEE_OTHER = 303\nHTTP_TEMPORARY_REDIRECT = 307\nHTTP_PERMANENT_REDIRECT = 308\n\n# Maximum number of redirects to follow when re-validating each hop (matches httpx's default).\nMAX_REDIRECTS = 20\n\n# HTTP status codes that represent a redirect carrying a Location header.\nREDIRECT_STATUS_CODES = frozenset(\n {\n HTTP_MOVED_PERMANENTLY,\n HTTP_FOUND,\n HTTP_SEE_OTHER,\n HTTP_TEMPORARY_REDIRECT,\n HTTP_PERMANENT_REDIRECT,\n }\n)\n\n\nclass APIRequestComponent(Component):\n display_name = \"API Request\"\n description = \"Make HTTP requests using URL or cURL commands.\"\n documentation: str = \"https://docs.langflow.org/api-request\"\n icon = \"Globe\"\n name = \"APIRequest\"\n\n inputs = [\n MessageTextInput(\n name=\"url_input\",\n display_name=\"URL\",\n info=\"Enter the URL for the request.\",\n advanced=False,\n tool_mode=True,\n ),\n MultilineInput(\n name=\"curl_input\",\n display_name=\"cURL\",\n info=(\n \"Paste a curl command to populate the fields. \"\n \"This will fill in the dictionary fields for headers and body.\"\n ),\n real_time_refresh=True,\n tool_mode=True,\n advanced=True,\n show=False,\n ),\n DropdownInput(\n name=\"method\",\n display_name=\"Method\",\n options=[\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"],\n value=\"GET\",\n info=\"The HTTP method to use.\",\n real_time_refresh=True,\n ),\n TabInput(\n name=\"mode\",\n display_name=\"Mode\",\n options=[\"URL\", \"cURL\"],\n value=\"URL\",\n info=\"Enable cURL mode to populate fields from a cURL command.\",\n real_time_refresh=True,\n ),\n DataInput(\n name=\"query_params\",\n display_name=\"Query Parameters\",\n info=\"The query parameters to append to the URL.\",\n advanced=True,\n ),\n TableInput(\n name=\"body\",\n display_name=\"Body\",\n info=\"The body to send with the request as a dictionary (for POST, PATCH, PUT).\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Parameter name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"description\": \"Parameter value\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n advanced=True,\n real_time_refresh=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": \"Langflow/1.0\"}],\n advanced=True,\n input_types=[\"Data\", \"JSON\"],\n real_time_refresh=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n value=30,\n info=\"The timeout to use for the request.\",\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n value=False,\n info=(\n \"Whether to follow HTTP redirects. \"\n \"WARNING: Enabling redirects may allow SSRF bypass attacks where a public URL \"\n \"redirects to internal resources. Only enable if you trust the target server. \"\n \"See OWASP SSRF Prevention Cheat Sheet for details.\"\n ),\n advanced=True,\n ),\n BoolInput(\n name=\"save_to_file\",\n display_name=\"Save to File\",\n value=False,\n info=\"Save the API response to a temporary file\",\n advanced=True,\n ),\n BoolInput(\n name=\"include_httpx_metadata\",\n display_name=\"Include HTTPx Metadata\",\n value=False,\n info=(\n \"Include properties such as headers, status_code, response_headers, \"\n \"and redirection_history in the output.\"\n ),\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"API Response\", name=\"data\", method=\"make_api_request\"),\n ]\n\n def _parse_json_value(self, value: Any) -> Any:\n \"\"\"Parse a value that might be a JSON string.\"\"\"\n if not isinstance(value, str):\n return value\n\n try:\n parsed = json.loads(value)\n except json.JSONDecodeError:\n return value\n else:\n return parsed\n\n def _process_body(self, body: Any) -> dict:\n \"\"\"Process the body input into a valid dictionary.\"\"\"\n if body is None:\n return {}\n if hasattr(body, \"data\"):\n body = body.data\n if isinstance(body, dict):\n return self._process_dict_body(body)\n if isinstance(body, str):\n return self._process_string_body(body)\n if isinstance(body, list):\n return self._process_list_body(body)\n return {}\n\n def _process_dict_body(self, body: dict) -> dict:\n \"\"\"Process dictionary body by parsing JSON values.\"\"\"\n return {k: self._parse_json_value(v) for k, v in body.items()}\n\n def _process_string_body(self, body: str) -> dict:\n \"\"\"Process string body by attempting JSON parse.\"\"\"\n try:\n return self._process_body(json.loads(body))\n except json.JSONDecodeError:\n return {\"data\": body}\n\n def _process_list_body(self, body: list) -> dict:\n \"\"\"Process list body by converting to key-value dictionary.\"\"\"\n processed_dict = {}\n try:\n for item in body:\n # Unwrap Data objects\n current_item = item\n if hasattr(item, \"data\"):\n unwrapped_data = item.data\n # If the unwrapped data is a dict but not key-value format, use it directly\n if isinstance(unwrapped_data, dict) and not self._is_valid_key_value_item(unwrapped_data):\n return unwrapped_data\n current_item = unwrapped_data\n if not self._is_valid_key_value_item(current_item):\n continue\n key = current_item[\"key\"]\n value = self._parse_json_value(current_item[\"value\"])\n processed_dict[key] = value\n except (KeyError, TypeError, ValueError) as e:\n self.log(f\"Failed to process body list: {e}\")\n return {}\n return processed_dict\n\n def _is_valid_key_value_item(self, item: Any) -> bool:\n \"\"\"Check if an item is a valid key-value dictionary.\"\"\"\n return isinstance(item, dict) and \"key\" in item and \"value\" in item\n\n def parse_curl(self, curl: str, build_config: dotdict) -> dotdict:\n \"\"\"Parse a cURL command and update build configuration.\"\"\"\n try:\n parsed = parse_context(curl)\n\n # Update basic configuration\n url = parsed.url\n # Normalize URL before setting it\n url = self._normalize_url(url)\n\n build_config[\"url_input\"][\"value\"] = url\n build_config[\"method\"][\"value\"] = parsed.method.upper()\n\n # Process headers\n headers_list = [{\"key\": k, \"value\": v} for k, v in parsed.headers.items()]\n build_config[\"headers\"][\"value\"] = headers_list\n\n # Process body data\n if not parsed.data:\n build_config[\"body\"][\"value\"] = []\n elif parsed.data:\n try:\n json_data = json.loads(parsed.data)\n if isinstance(json_data, dict):\n body_list = [\n {\"key\": k, \"value\": json.dumps(v) if isinstance(v, dict | list) else str(v)}\n for k, v in json_data.items()\n ]\n build_config[\"body\"][\"value\"] = body_list\n else:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": json.dumps(json_data)}]\n except json.JSONDecodeError:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": parsed.data}]\n\n except Exception as exc:\n msg = f\"Error parsing curl: {exc}\"\n self.log(msg)\n raise ValueError(msg) from exc\n\n return build_config\n\n def _normalize_url(self, url: str) -> str:\n \"\"\"Normalize URL by adding https:// if no protocol is specified.\"\"\"\n if not url or not isinstance(url, str):\n msg = \"URL cannot be empty\"\n raise ValueError(msg)\n\n url = url.strip()\n if url.startswith((\"http://\", \"https://\")):\n return url\n return f\"https://{url}\"\n\n async def make_request(\n self,\n client: httpx.AsyncClient,\n method: str,\n url: str,\n headers: dict | None = None,\n body: Any = None,\n timeout: int = 5,\n *,\n follow_redirects: bool = False,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n redirection_history = []\n\n try:\n # Prepare request parameters\n request_params = {\n \"method\": method,\n \"url\": url,\n \"headers\": headers,\n \"timeout\": timeout,\n \"follow_redirects\": follow_redirects,\n }\n # Only include body for methods that support it (GET must not have a body per HTTP spec)\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n response = await client.request(**request_params)\n\n redirection_history = [\n {\n \"url\": redirect.headers.get(\"Location\", str(redirect.url)),\n \"status_code\": redirect.status_code,\n }\n for redirect in response.history\n ]\n\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n async def _build_response_data(\n self,\n response: httpx.Response,\n source_url: str,\n headers: dict | None,\n redirection_history: list,\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Turn an httpx response into the component's ``Data`` output.\n\n Shared by the standard request path (``make_request``) and the redirect\n re-validation path (``_follow_redirects_with_validation``) so both produce\n identical metadata, optional file saving, and body decoding.\n \"\"\"\n is_binary, file_path = await self._response_info(response, with_file_path=save_to_file)\n response_headers = self._headers_to_dict(response.headers)\n\n # Base metadata\n metadata = {\n \"source\": source_url,\n \"status_code\": response.status_code,\n \"response_headers\": response_headers,\n }\n\n if redirection_history:\n metadata[\"redirection_history\"] = redirection_history\n\n if save_to_file:\n mode = \"wb\" if is_binary else \"w\"\n encoding = response.encoding if mode == \"w\" else None\n if file_path:\n await aiofiles_os.makedirs(file_path.parent, exist_ok=True)\n if is_binary:\n async with aiofiles.open(file_path, \"wb\") as f:\n await f.write(response.content)\n await f.flush()\n else:\n async with aiofiles.open(file_path, \"w\", encoding=encoding) as f:\n await f.write(response.text)\n await f.flush()\n metadata[\"file_path\"] = str(file_path)\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n return Data(data=metadata)\n\n # Handle response content\n if is_binary:\n result = response.content\n else:\n try:\n result = response.json()\n except json.JSONDecodeError:\n self.log(\"Failed to decode JSON response\")\n result = response.text.encode(\"utf-8\")\n\n metadata[\"result\"] = result\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n\n return Data(data=metadata)\n\n def add_query_params(self, url: str, params: dict) -> str:\n \"\"\"Add query parameters to URL efficiently.\"\"\"\n if not params:\n return url\n url_parts = list(urlparse(url))\n query = dict(parse_qsl(url_parts[4]))\n query.update(params)\n url_parts[4] = urlencode(query)\n return urlunparse(url_parts)\n\n def _headers_to_dict(self, headers: httpx.Headers) -> dict[str, str]:\n \"\"\"Convert HTTP headers to a dictionary with lowercased keys.\"\"\"\n return {k.lower(): v for k, v in headers.items()}\n\n def _process_headers(self, headers: Any) -> dict:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if headers is None:\n return {}\n if isinstance(headers, dict):\n return headers\n if isinstance(headers, list):\n return {item[\"key\"]: item[\"value\"] for item in headers if self._is_valid_key_value_item(item)}\n return {}\n\n async def make_api_request(self) -> Data:\n \"\"\"Make HTTP request with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. The protection works by:\n 1. Validating the URL and resolving DNS during security check\n 2. Pinning the validated IP address\n 3. Forcing the HTTP client to use the pinned IP for the actual request\n 4. Ignoring any subsequent DNS changes (prevents rebinding attacks)\n\n Returns:\n Data: Response data from the HTTP request\n\n Raises:\n ValueError: If URL is invalid or blocked by SSRF protection\n \"\"\"\n # Extract request parameters\n method = self.method\n url = self.url_input.strip() if isinstance(self.url_input, str) else \"\"\n headers = self.headers or {}\n body = self.body or {}\n timeout = self.timeout\n follow_redirects = self.follow_redirects\n save_to_file = self.save_to_file\n include_httpx_metadata = self.include_httpx_metadata\n\n # Security warning: HTTP redirects can bypass SSRF protection\n # A public URL could redirect to an internal resource\n if follow_redirects:\n self.log(\n \"Security Warning: HTTP redirects are enabled. This may allow SSRF bypass attacks \"\n \"where a public URL redirects to internal resources (e.g., cloud metadata endpoints). \"\n \"Only enable this if you trust the target server.\"\n )\n\n # Normalize URL (add https:// if no protocol specified)\n url = self._normalize_url(url)\n\n # Basic URL format validation\n if not validators.url(url):\n msg = f\"Invalid URL provided: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Pinning the validated IP address\n # 3. Using a custom HTTP transport that forces use of the pinned IP\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IP is pinned: \"example.com = 93.184.216.34\"\n # - HTTP request: uses pinned IP directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IP\n # ============================================================================\n\n try:\n # Validate URL and get validated IPs for DNS pinning\n _validated_url, validated_ips = validate_and_resolve_url(url)\n\n # Log DNS pinning information for security auditing\n if validated_ips:\n self.log(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s)\")\n\n except SSRFProtectionError as e:\n # SSRF protection blocked the request (private IP, internal network, etc.)\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n\n # Process query parameters (from string or Data object)\n if isinstance(self.query_params, str):\n query_params = dict(parse_qsl(self.query_params))\n else:\n query_params = self.query_params.data if self.query_params else {}\n\n # Process headers and body into proper format\n headers = self._process_headers(headers)\n body = self._process_body(body)\n url = self.add_query_params(url, query_params)\n\n # ============================================================================\n # Execute the request (re-validating any redirects when SSRF protection is on)\n # ============================================================================\n # When SSRF protection is enabled we must NOT let httpx auto-follow redirects:\n # a validated public URL can redirect to an internal address (loopback, RFC1918,\n # link-local / cloud metadata) that was never checked, bypassing both the initial\n # validation and DNS pinning. Instead we follow redirects manually so every hop\n # is re-validated with the same denylist + DNS pinning. When protection is\n # disabled, we preserve the previous behavior and let httpx handle redirects.\n if is_ssrf_protection_enabled() and follow_redirects:\n result = await self._follow_redirects_with_validation(\n method,\n url,\n headers,\n body,\n timeout,\n validated_ips,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n else:\n # No redirect re-validation needed:\n # - SSRF protection is disabled (user opted out), or\n # - redirects are disabled, so httpx makes a single request.\n # DNS pinning still applies to the single request when protection is enabled\n # and the host resolved to validated IPs.\n async with self._build_http_client(url, validated_ips) as client:\n result = await self.make_request(\n client,\n method,\n url,\n headers,\n body,\n timeout,\n follow_redirects=follow_redirects,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n self.status = result\n return result\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client, pinning DNS to validated IPs when SSRF protection applies.\n\n Args:\n url: The request URL whose hostname will be pinned.\n validated_ips: IPs validated by ``validate_and_resolve_url`` for this hop.\n\n Returns:\n httpx.AsyncClient: A client that pins DNS to ``validated_ips`` (preventing\n rebinding) when SSRF protection is enabled and the hop has validated IPs;\n otherwise a standard client (protection disabled, allowlisted host, or\n hostname extraction failure).\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n # Extract hostname from the URL so the custom transport can pin it while\n # preserving the Host header for virtual hosting / TLS SNI.\n hostname = urlparse(url).hostname\n if hostname:\n # The custom transport tries validated IPs in order (dual-stack / LB).\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _method_for_redirect(method: str, status_code: int) -> str:\n \"\"\"Return the HTTP method to use after a redirect, mirroring httpx semantics.\n\n A 303 (See Other) always becomes GET; 301/302 downgrade POST to GET for\n browser compatibility; 307/308 preserve the original method (and body).\n \"\"\"\n method = method.upper()\n if status_code == HTTP_SEE_OTHER and method != \"HEAD\":\n return \"GET\"\n if status_code in (HTTP_MOVED_PERMANENTLY, HTTP_FOUND) and method == \"POST\":\n return \"GET\"\n return method\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different host.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to a different host than the one the\n caller intended them for. Same-host redirects keep all headers.\n \"\"\"\n if not headers:\n return headers\n if urlparse(current_url).hostname == urlparse(next_url).hostname:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n async def _follow_redirects_with_validation(\n self,\n method: str,\n url: str,\n headers: dict | None,\n body: Any,\n timeout: int,\n validated_ips: list[str],\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Make the request and follow redirects manually, re-validating every hop.\n\n This closes an SSRF bypass: with ``follow_redirects`` enabled, httpx would\n otherwise auto-follow a redirect from a validated public URL to an internal\n address that was never checked. Here each redirect ``Location`` is resolved\n (relative locations included) and re-validated with ``validate_and_resolve_url``\n — the same private/loopback/link-local denylist and DNS pinning applied to the\n initial request — before any connection to it is made. A blocked hop raises\n ``ValueError``; the number of redirects is capped at ``MAX_REDIRECTS``.\n \"\"\"\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n current_url = url\n current_ips = validated_ips\n redirection_history: list[dict] = []\n\n for _ in range(MAX_REDIRECTS + 1):\n request_params: dict[str, Any] = {\n \"method\": method,\n \"url\": current_url,\n \"headers\": headers,\n \"timeout\": timeout,\n # Never let httpx follow redirects itself; each hop is validated below.\n \"follow_redirects\": False,\n }\n # Only include body for methods that support it (GET must not have a body).\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n\n try:\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.request(**request_params)\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {current_url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n location = response.headers.get(\"Location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n redirection_history.append({\"url\": location, \"status_code\": response.status_code})\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n # Non-http(s) schemes, private/loopback/link-local hosts, and hostnames that\n # resolve to blocked IPs all raise SSRFProtectionError here.\n try:\n _validated_url, current_ips = validate_and_resolve_url(next_url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n method = self._method_for_redirect(method, response.status_code)\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict:\n \"\"\"Update the build config based on the selected mode.\"\"\"\n if field_name != \"mode\":\n if field_name == \"curl_input\" and self.mode == \"cURL\" and self.curl_input:\n return self.parse_curl(self.curl_input, build_config)\n return build_config\n\n if field_value == \"cURL\":\n set_field_display(build_config, \"curl_input\", value=True)\n if build_config[\"curl_input\"][\"value\"]:\n try:\n build_config = self.parse_curl(build_config[\"curl_input\"][\"value\"], build_config)\n except ValueError as e:\n self.log(f\"Failed to parse cURL input: {e}\")\n else:\n set_field_display(build_config, \"curl_input\", value=False)\n\n return set_current_fields(\n build_config=build_config,\n action_fields=MODE_FIELDS,\n selected_action=field_value,\n default_fields=DEFAULT_FIELDS,\n func=set_field_advanced,\n default_value=True,\n )\n\n async def _response_info(\n self, response: httpx.Response, *, with_file_path: bool = False\n ) -> tuple[bool, Path | None]:\n \"\"\"Determine the file path and whether the response content is binary.\n\n Args:\n response (Response): The HTTP response object.\n with_file_path (bool): Whether to save the response content to a file.\n\n Returns:\n Tuple[bool, Path | None]:\n A tuple containing a boolean indicating if the content is binary and the full file path (if applicable).\n \"\"\"\n content_type = response.headers.get(\"Content-Type\", \"\")\n is_binary = \"application/octet-stream\" in content_type or \"application/binary\" in content_type\n\n if not with_file_path:\n return is_binary, None\n\n component_temp_dir = Path(tempfile.gettempdir()) / self.__class__.__name__\n\n # Create directory asynchronously\n await aiofiles_os.makedirs(component_temp_dir, exist_ok=True)\n\n filename = None\n if \"Content-Disposition\" in response.headers:\n content_disposition = response.headers[\"Content-Disposition\"]\n filename_match = re.search(r'filename=\"(.+?)\"', content_disposition)\n if filename_match:\n extracted_filename = filename_match.group(1)\n filename = extracted_filename\n\n # Step 3: Infer file extension or use part of the request URL if no filename\n if not filename:\n # Extract the last segment of the URL path\n url_path = urlparse(str(response.request.url) if response.request else \"\").path\n base_name = Path(url_path).name # Get the last segment of the path\n if not base_name: # If the path ends with a slash or is empty\n base_name = \"response\"\n\n # Infer file extension\n content_type_to_extension = {\n \"text/plain\": \".txt\",\n \"application/json\": \".json\",\n \"image/jpeg\": \".jpg\",\n \"image/png\": \".png\",\n \"application/octet-stream\": \".bin\",\n }\n extension = content_type_to_extension.get(content_type, \".bin\" if is_binary else \".txt\")\n filename = f\"{base_name}{extension}\"\n\n # Step 4: Define the full file path\n file_path = component_temp_dir / filename\n\n # Step 5: Check if file exists asynchronously and handle accordingly\n try:\n # Try to create the file exclusively (x mode) to check existence\n async with aiofiles.open(file_path, \"x\") as _:\n pass # File created successfully, we can use this path\n except FileExistsError:\n # If file exists, append a timestamp to the filename\n timestamp = datetime.now(timezone.utc).strftime(\"%Y%m%d%H%M%S%f\")\n file_path = component_temp_dir / f\"{timestamp}-{filename}\"\n\n return is_binary, file_path\n" + "value": "import json\nimport re\nimport tempfile\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom typing import Any\nfrom urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse\n\nimport aiofiles\nimport aiofiles.os as aiofiles_os\nimport httpx\nimport validators\n\nfrom lfx.base.curl.parse import parse_context\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs.inputs import TabInput\nfrom lfx.io import (\n BoolInput,\n DataInput,\n DropdownInput,\n IntInput,\n MessageTextInput,\n MultilineInput,\n Output,\n TableInput,\n)\nfrom lfx.schema.data import Data\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.utils.component_utils import set_current_fields, set_field_advanced, set_field_display\n\n# SSRF Protection imports - for preventing Server-Side Request Forgery attacks\nfrom lfx.utils.ssrf_protection import (\n SSRFProtectionError,\n is_ssrf_protection_enabled,\n validate_and_resolve_url,\n)\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Define fields for each mode\nMODE_FIELDS = {\n \"URL\": [\n \"url_input\",\n \"method\",\n ],\n \"cURL\": [\"curl_input\"],\n}\n\n# Fields that should always be visible\nDEFAULT_FIELDS = [\"mode\"]\n\n# HTTP redirect status codes (RFC 9110).\nHTTP_MOVED_PERMANENTLY = 301\nHTTP_FOUND = 302\nHTTP_SEE_OTHER = 303\nHTTP_TEMPORARY_REDIRECT = 307\nHTTP_PERMANENT_REDIRECT = 308\n\n# Maximum number of redirects to follow when re-validating each hop (matches httpx's default).\nMAX_REDIRECTS = 20\n\n# HTTP status codes that represent a redirect carrying a Location header.\nREDIRECT_STATUS_CODES = frozenset(\n {\n HTTP_MOVED_PERMANENTLY,\n HTTP_FOUND,\n HTTP_SEE_OTHER,\n HTTP_TEMPORARY_REDIRECT,\n HTTP_PERMANENT_REDIRECT,\n }\n)\n\n# Default ports per scheme, used to compare redirect origins.\nDEFAULT_SCHEME_PORTS = {\"http\": 80, \"https\": 443}\n\n\nclass APIRequestComponent(Component):\n display_name = \"API Request\"\n description = \"Make HTTP requests using URL or cURL commands.\"\n documentation: str = \"https://docs.langflow.org/api-request\"\n icon = \"Globe\"\n name = \"APIRequest\"\n\n inputs = [\n MessageTextInput(\n name=\"url_input\",\n display_name=\"URL\",\n info=\"Enter the URL for the request.\",\n advanced=False,\n tool_mode=True,\n ),\n MultilineInput(\n name=\"curl_input\",\n display_name=\"cURL\",\n info=(\n \"Paste a curl command to populate the fields. \"\n \"This will fill in the dictionary fields for headers and body.\"\n ),\n real_time_refresh=True,\n tool_mode=True,\n advanced=True,\n show=False,\n ),\n DropdownInput(\n name=\"method\",\n display_name=\"Method\",\n options=[\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"],\n value=\"GET\",\n info=\"The HTTP method to use.\",\n real_time_refresh=True,\n ),\n TabInput(\n name=\"mode\",\n display_name=\"Mode\",\n options=[\"URL\", \"cURL\"],\n value=\"URL\",\n info=\"Enable cURL mode to populate fields from a cURL command.\",\n real_time_refresh=True,\n ),\n DataInput(\n name=\"query_params\",\n display_name=\"Query Parameters\",\n info=\"The query parameters to append to the URL.\",\n advanced=True,\n ),\n TableInput(\n name=\"body\",\n display_name=\"Body\",\n info=\"The body to send with the request as a dictionary (for POST, PATCH, PUT).\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Parameter name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"description\": \"Parameter value\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n advanced=True,\n real_time_refresh=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": \"Langflow/1.0\"}],\n advanced=True,\n input_types=[\"Data\", \"JSON\"],\n real_time_refresh=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n value=30,\n info=\"The timeout to use for the request.\",\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n value=False,\n info=(\n \"Whether to follow HTTP redirects. \"\n \"WARNING: Enabling redirects may allow SSRF bypass attacks where a public URL \"\n \"redirects to internal resources. Only enable if you trust the target server. \"\n \"See OWASP SSRF Prevention Cheat Sheet for details.\"\n ),\n advanced=True,\n ),\n BoolInput(\n name=\"save_to_file\",\n display_name=\"Save to File\",\n value=False,\n info=\"Save the API response to a temporary file\",\n advanced=True,\n ),\n BoolInput(\n name=\"include_httpx_metadata\",\n display_name=\"Include HTTPx Metadata\",\n value=False,\n info=(\n \"Include properties such as headers, status_code, response_headers, \"\n \"and redirection_history in the output.\"\n ),\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"API Response\", name=\"data\", method=\"make_api_request\"),\n ]\n\n def _parse_json_value(self, value: Any) -> Any:\n \"\"\"Parse a value that might be a JSON string.\"\"\"\n if not isinstance(value, str):\n return value\n\n try:\n parsed = json.loads(value)\n except json.JSONDecodeError:\n return value\n else:\n return parsed\n\n def _process_body(self, body: Any) -> dict:\n \"\"\"Process the body input into a valid dictionary.\"\"\"\n if body is None:\n return {}\n if hasattr(body, \"data\"):\n body = body.data\n if isinstance(body, dict):\n return self._process_dict_body(body)\n if isinstance(body, str):\n return self._process_string_body(body)\n if isinstance(body, list):\n return self._process_list_body(body)\n return {}\n\n def _process_dict_body(self, body: dict) -> dict:\n \"\"\"Process dictionary body by parsing JSON values.\"\"\"\n return {k: self._parse_json_value(v) for k, v in body.items()}\n\n def _process_string_body(self, body: str) -> dict:\n \"\"\"Process string body by attempting JSON parse.\"\"\"\n try:\n return self._process_body(json.loads(body))\n except json.JSONDecodeError:\n return {\"data\": body}\n\n def _process_list_body(self, body: list) -> dict:\n \"\"\"Process list body by converting to key-value dictionary.\"\"\"\n processed_dict = {}\n try:\n for item in body:\n # Unwrap Data objects\n current_item = item\n if hasattr(item, \"data\"):\n unwrapped_data = item.data\n # If the unwrapped data is a dict but not key-value format, use it directly\n if isinstance(unwrapped_data, dict) and not self._is_valid_key_value_item(unwrapped_data):\n return unwrapped_data\n current_item = unwrapped_data\n if not self._is_valid_key_value_item(current_item):\n continue\n key = current_item[\"key\"]\n value = self._parse_json_value(current_item[\"value\"])\n processed_dict[key] = value\n except (KeyError, TypeError, ValueError) as e:\n self.log(f\"Failed to process body list: {e}\")\n return {}\n return processed_dict\n\n def _is_valid_key_value_item(self, item: Any) -> bool:\n \"\"\"Check if an item is a valid key-value dictionary.\"\"\"\n return isinstance(item, dict) and \"key\" in item and \"value\" in item\n\n def parse_curl(self, curl: str, build_config: dotdict) -> dotdict:\n \"\"\"Parse a cURL command and update build configuration.\"\"\"\n try:\n parsed = parse_context(curl)\n\n # Update basic configuration\n url = parsed.url\n # Normalize URL before setting it\n url = self._normalize_url(url)\n\n build_config[\"url_input\"][\"value\"] = url\n build_config[\"method\"][\"value\"] = parsed.method.upper()\n\n # Process headers\n headers_list = [{\"key\": k, \"value\": v} for k, v in parsed.headers.items()]\n build_config[\"headers\"][\"value\"] = headers_list\n\n # Process body data\n if not parsed.data:\n build_config[\"body\"][\"value\"] = []\n elif parsed.data:\n try:\n json_data = json.loads(parsed.data)\n if isinstance(json_data, dict):\n body_list = [\n {\"key\": k, \"value\": json.dumps(v) if isinstance(v, dict | list) else str(v)}\n for k, v in json_data.items()\n ]\n build_config[\"body\"][\"value\"] = body_list\n else:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": json.dumps(json_data)}]\n except json.JSONDecodeError:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": parsed.data}]\n\n except Exception as exc:\n msg = f\"Error parsing curl: {exc}\"\n self.log(msg)\n raise ValueError(msg) from exc\n\n return build_config\n\n def _normalize_url(self, url: str) -> str:\n \"\"\"Normalize URL by adding https:// if no protocol is specified.\"\"\"\n if not url or not isinstance(url, str):\n msg = \"URL cannot be empty\"\n raise ValueError(msg)\n\n url = url.strip()\n if url.startswith((\"http://\", \"https://\")):\n return url\n return f\"https://{url}\"\n\n async def make_request(\n self,\n client: httpx.AsyncClient,\n method: str,\n url: str,\n headers: dict | None = None,\n body: Any = None,\n timeout: int = 5,\n *,\n follow_redirects: bool = False,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n redirection_history = []\n\n try:\n # Prepare request parameters\n request_params = {\n \"method\": method,\n \"url\": url,\n \"headers\": headers,\n \"timeout\": timeout,\n \"follow_redirects\": follow_redirects,\n }\n # Only include body for methods that support it (GET must not have a body per HTTP spec)\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n response = await client.request(**request_params)\n\n redirection_history = [\n {\n \"url\": redirect.headers.get(\"Location\", str(redirect.url)),\n \"status_code\": redirect.status_code,\n }\n for redirect in response.history\n ]\n\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n async def _build_response_data(\n self,\n response: httpx.Response,\n source_url: str,\n headers: dict | None,\n redirection_history: list,\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Turn an httpx response into the component's ``Data`` output.\n\n Shared by the standard request path (``make_request``) and the redirect\n re-validation path (``_follow_redirects_with_validation``) so both produce\n identical metadata, optional file saving, and body decoding.\n \"\"\"\n is_binary, file_path = await self._response_info(response, with_file_path=save_to_file)\n response_headers = self._headers_to_dict(response.headers)\n\n # Base metadata\n metadata = {\n \"source\": source_url,\n \"status_code\": response.status_code,\n \"response_headers\": response_headers,\n }\n\n if redirection_history:\n metadata[\"redirection_history\"] = redirection_history\n\n if save_to_file:\n mode = \"wb\" if is_binary else \"w\"\n encoding = response.encoding if mode == \"w\" else None\n if file_path:\n await aiofiles_os.makedirs(file_path.parent, exist_ok=True)\n if is_binary:\n async with aiofiles.open(file_path, \"wb\") as f:\n await f.write(response.content)\n await f.flush()\n else:\n async with aiofiles.open(file_path, \"w\", encoding=encoding) as f:\n await f.write(response.text)\n await f.flush()\n metadata[\"file_path\"] = str(file_path)\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n return Data(data=metadata)\n\n # Handle response content\n if is_binary:\n result = response.content\n else:\n try:\n result = response.json()\n except json.JSONDecodeError:\n self.log(\"Failed to decode JSON response\")\n result = response.text.encode(\"utf-8\")\n\n metadata[\"result\"] = result\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n\n return Data(data=metadata)\n\n def add_query_params(self, url: str, params: dict) -> str:\n \"\"\"Add query parameters to URL efficiently.\"\"\"\n if not params:\n return url\n url_parts = list(urlparse(url))\n query = dict(parse_qsl(url_parts[4]))\n query.update(params)\n url_parts[4] = urlencode(query)\n return urlunparse(url_parts)\n\n def _headers_to_dict(self, headers: httpx.Headers) -> dict[str, str]:\n \"\"\"Convert HTTP headers to a dictionary with lowercased keys.\"\"\"\n return {k.lower(): v for k, v in headers.items()}\n\n def _process_headers(self, headers: Any) -> dict:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if headers is None:\n return {}\n if isinstance(headers, dict):\n return headers\n if isinstance(headers, list):\n return {item[\"key\"]: item[\"value\"] for item in headers if self._is_valid_key_value_item(item)}\n return {}\n\n async def make_api_request(self) -> Data:\n \"\"\"Make HTTP request with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. The protection works by:\n 1. Validating the URL and resolving DNS during security check\n 2. Pinning the validated IP address\n 3. Forcing the HTTP client to use the pinned IP for the actual request\n 4. Ignoring any subsequent DNS changes (prevents rebinding attacks)\n\n Returns:\n Data: Response data from the HTTP request\n\n Raises:\n ValueError: If URL is invalid or blocked by SSRF protection\n \"\"\"\n # Extract request parameters\n method = self.method\n url = self.url_input.strip() if isinstance(self.url_input, str) else \"\"\n headers = self.headers or {}\n body = self.body or {}\n timeout = self.timeout\n follow_redirects = self.follow_redirects\n save_to_file = self.save_to_file\n include_httpx_metadata = self.include_httpx_metadata\n\n # Security warning: HTTP redirects can bypass SSRF protection\n # A public URL could redirect to an internal resource\n if follow_redirects:\n self.log(\n \"Security Warning: HTTP redirects are enabled. This may allow SSRF bypass attacks \"\n \"where a public URL redirects to internal resources (e.g., cloud metadata endpoints). \"\n \"Only enable this if you trust the target server.\"\n )\n\n # Normalize URL (add https:// if no protocol specified)\n url = self._normalize_url(url)\n\n # Basic URL format validation\n if not validators.url(url):\n msg = f\"Invalid URL provided: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Pinning the validated IP address\n # 3. Using a custom HTTP transport that forces use of the pinned IP\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IP is pinned: \"example.com = 93.184.216.34\"\n # - HTTP request: uses pinned IP directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IP\n # ============================================================================\n\n try:\n # Validate URL and get validated IPs for DNS pinning\n _validated_url, validated_ips = validate_and_resolve_url(url)\n\n # Log DNS pinning information for security auditing\n if validated_ips:\n self.log(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s)\")\n\n except SSRFProtectionError as e:\n # SSRF protection blocked the request (private IP, internal network, etc.)\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n\n # Process query parameters (from string or Data object)\n if isinstance(self.query_params, str):\n query_params = dict(parse_qsl(self.query_params))\n else:\n query_params = self.query_params.data if self.query_params else {}\n\n # Process headers and body into proper format\n headers = self._process_headers(headers)\n body = self._process_body(body)\n url = self.add_query_params(url, query_params)\n\n # ============================================================================\n # Execute the request (re-validating any redirects when SSRF protection is on)\n # ============================================================================\n # When SSRF protection is enabled we must NOT let httpx auto-follow redirects:\n # a validated public URL can redirect to an internal address (loopback, RFC1918,\n # link-local / cloud metadata) that was never checked, bypassing both the initial\n # validation and DNS pinning. Instead we follow redirects manually so every hop\n # is re-validated with the same denylist + DNS pinning. When protection is\n # disabled, we preserve the previous behavior and let httpx handle redirects.\n if is_ssrf_protection_enabled() and follow_redirects:\n result = await self._follow_redirects_with_validation(\n method,\n url,\n headers,\n body,\n timeout,\n validated_ips,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n else:\n # No redirect re-validation needed:\n # - SSRF protection is disabled (user opted out), or\n # - redirects are disabled, so httpx makes a single request.\n # DNS pinning still applies to the single request when protection is enabled\n # and the host resolved to validated IPs.\n async with self._build_http_client(url, validated_ips) as client:\n result = await self.make_request(\n client,\n method,\n url,\n headers,\n body,\n timeout,\n follow_redirects=follow_redirects,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n self.status = result\n return result\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client, pinning DNS to validated IPs when SSRF protection applies.\n\n Args:\n url: The request URL whose hostname will be pinned.\n validated_ips: IPs validated by ``validate_and_resolve_url`` for this hop.\n\n Returns:\n httpx.AsyncClient: A client that pins DNS to ``validated_ips`` (preventing\n rebinding) when SSRF protection is enabled and the hop has validated IPs;\n otherwise a standard client (protection disabled, allowlisted host, or\n hostname extraction failure).\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n # Extract hostname from the URL so the custom transport can pin it while\n # preserving the Host header for virtual hosting / TLS SNI.\n hostname = urlparse(url).hostname\n if hostname:\n # The custom transport tries validated IPs in order (dual-stack / LB).\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _method_for_redirect(method: str, status_code: int) -> str:\n \"\"\"Return the HTTP method to use after a redirect, mirroring httpx semantics.\n\n A 303 (See Other) always becomes GET; 301/302 downgrade POST to GET for\n browser compatibility; 307/308 preserve the original method (and body).\n \"\"\"\n method = method.upper()\n if status_code == HTTP_SEE_OTHER and method != \"HEAD\":\n return \"GET\"\n if status_code in (HTTP_MOVED_PERMANENTLY, HTTP_FOUND) and method == \"POST\":\n return \"GET\"\n return method\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different origin.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to an origin other than the one the\n caller intended them for. Headers are kept only when the redirect stays on the\n same origin (scheme, host, port) or is a direct https upgrade of the same host\n on default ports - the exact cases where httpx keeps the Authorization header.\n \"\"\"\n if not headers:\n return headers\n current, nxt = urlparse(current_url), urlparse(next_url)\n current_port = current.port or DEFAULT_SCHEME_PORTS.get(current.scheme)\n next_port = nxt.port or DEFAULT_SCHEME_PORTS.get(nxt.scheme)\n same_origin = (current.scheme, current.hostname, current_port) == (nxt.scheme, nxt.hostname, next_port)\n https_upgrade = (\n current.hostname == nxt.hostname\n and current.scheme == \"http\"\n and nxt.scheme == \"https\"\n and current_port == DEFAULT_SCHEME_PORTS[\"http\"]\n and next_port == DEFAULT_SCHEME_PORTS[\"https\"]\n )\n if same_origin or https_upgrade:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n async def _follow_redirects_with_validation(\n self,\n method: str,\n url: str,\n headers: dict | None,\n body: Any,\n timeout: int,\n validated_ips: list[str],\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Make the request and follow redirects manually, re-validating every hop.\n\n This closes an SSRF bypass: with ``follow_redirects`` enabled, httpx would\n otherwise auto-follow a redirect from a validated public URL to an internal\n address that was never checked. Here each redirect ``Location`` is resolved\n (relative locations included) and re-validated with ``validate_and_resolve_url``\n — the same private/loopback/link-local denylist and DNS pinning applied to the\n initial request — before any connection to it is made. A blocked hop raises\n ``ValueError``; the number of redirects is capped at ``MAX_REDIRECTS``.\n \"\"\"\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n current_url = url\n current_ips = validated_ips\n redirection_history: list[dict] = []\n\n for _ in range(MAX_REDIRECTS + 1):\n request_params: dict[str, Any] = {\n \"method\": method,\n \"url\": current_url,\n \"headers\": headers,\n \"timeout\": timeout,\n # Never let httpx follow redirects itself; each hop is validated below.\n \"follow_redirects\": False,\n }\n # Only include body for methods that support it (GET must not have a body).\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n\n try:\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.request(**request_params)\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {current_url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n location = response.headers.get(\"Location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n redirection_history.append({\"url\": location, \"status_code\": response.status_code})\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n # Non-http(s) schemes, private/loopback/link-local hosts, and hostnames that\n # resolve to blocked IPs all raise SSRFProtectionError here.\n try:\n _validated_url, current_ips = validate_and_resolve_url(next_url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n method = self._method_for_redirect(method, response.status_code)\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict:\n \"\"\"Update the build config based on the selected mode.\"\"\"\n if field_name != \"mode\":\n if field_name == \"curl_input\" and self.mode == \"cURL\" and self.curl_input:\n return self.parse_curl(self.curl_input, build_config)\n return build_config\n\n if field_value == \"cURL\":\n set_field_display(build_config, \"curl_input\", value=True)\n if build_config[\"curl_input\"][\"value\"]:\n try:\n build_config = self.parse_curl(build_config[\"curl_input\"][\"value\"], build_config)\n except ValueError as e:\n self.log(f\"Failed to parse cURL input: {e}\")\n else:\n set_field_display(build_config, \"curl_input\", value=False)\n\n return set_current_fields(\n build_config=build_config,\n action_fields=MODE_FIELDS,\n selected_action=field_value,\n default_fields=DEFAULT_FIELDS,\n func=set_field_advanced,\n default_value=True,\n )\n\n async def _response_info(\n self, response: httpx.Response, *, with_file_path: bool = False\n ) -> tuple[bool, Path | None]:\n \"\"\"Determine the file path and whether the response content is binary.\n\n Args:\n response (Response): The HTTP response object.\n with_file_path (bool): Whether to save the response content to a file.\n\n Returns:\n Tuple[bool, Path | None]:\n A tuple containing a boolean indicating if the content is binary and the full file path (if applicable).\n \"\"\"\n content_type = response.headers.get(\"Content-Type\", \"\")\n is_binary = \"application/octet-stream\" in content_type or \"application/binary\" in content_type\n\n if not with_file_path:\n return is_binary, None\n\n component_temp_dir = Path(tempfile.gettempdir()) / self.__class__.__name__\n\n # Create directory asynchronously\n await aiofiles_os.makedirs(component_temp_dir, exist_ok=True)\n\n filename = None\n if \"Content-Disposition\" in response.headers:\n content_disposition = response.headers[\"Content-Disposition\"]\n filename_match = re.search(r'filename=\"(.+?)\"', content_disposition)\n if filename_match:\n extracted_filename = filename_match.group(1)\n filename = extracted_filename\n\n # Step 3: Infer file extension or use part of the request URL if no filename\n if not filename:\n # Extract the last segment of the URL path\n url_path = urlparse(str(response.request.url) if response.request else \"\").path\n base_name = Path(url_path).name # Get the last segment of the path\n if not base_name: # If the path ends with a slash or is empty\n base_name = \"response\"\n\n # Infer file extension\n content_type_to_extension = {\n \"text/plain\": \".txt\",\n \"application/json\": \".json\",\n \"image/jpeg\": \".jpg\",\n \"image/png\": \".png\",\n \"application/octet-stream\": \".bin\",\n }\n extension = content_type_to_extension.get(content_type, \".bin\" if is_binary else \".txt\")\n filename = f\"{base_name}{extension}\"\n\n # Step 4: Define the full file path\n file_path = component_temp_dir / filename\n\n # Step 5: Check if file exists asynchronously and handle accordingly\n try:\n # Try to create the file exclusively (x mode) to check existence\n async with aiofiles.open(file_path, \"x\") as _:\n pass # File created successfully, we can use this path\n except FileExistsError:\n # If file exists, append a timestamp to the filename\n timestamp = datetime.now(timezone.utc).strftime(\"%Y%m%d%H%M%S%f\")\n file_path = component_temp_dir / f\"{timestamp}-{filename}\"\n\n return is_binary, file_path\n" }, "curl_input": { "_input_type": "MultilineInput", diff --git a/src/backend/base/langflow/initial_setup/starter_projects/Simple Agent.json b/src/backend/base/langflow/initial_setup/starter_projects/Simple Agent.json index 40a6f20ca2..d660a1b44b 100644 --- a/src/backend/base/langflow/initial_setup/starter_projects/Simple Agent.json +++ b/src/backend/base/langflow/initial_setup/starter_projects/Simple Agent.json @@ -1516,6 +1516,7 @@ "max_depth", "prevent_outside", "use_async", + "follow_redirects", "format", "timeout", "headers", @@ -1530,7 +1531,7 @@ "last_updated": "2026-02-12T20:48:13.882Z", "legacy": false, "metadata": { - "code_hash": "2e225a90043a", + "code_hash": "52b5bd602242", "dependencies": { "dependencies": [ { @@ -1632,7 +1633,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n async with self._build_http_client(url, validated_ips) as client:\n try:\n response = await client.get(url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n else:\n return html_content, metadata\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(start_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(start_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" + "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n# HTTP status codes that carry a redirect Location header (RFC 9110).\nREDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})\n\n# Maximum number of redirects to follow. Each hop is re-validated for SSRF safety;\n# matches httpx's default and the API Request component.\nMAX_REDIRECTS = 20\n\n# Default ports per scheme, used to compare redirect origins.\nDEFAULT_SCHEME_PORTS = {\"http\": 80, \"https\": 443}\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n info=(\n \"If enabled, follows HTTP redirects such as http→https or www/non-www \"\n \"normalization, which most sites rely on to serve their content. When SSRF \"\n \"protection is enabled (the default), every redirect hop is re-validated against \"\n \"the same blocked-IP denylist and DNS-pinned before it is fetched, so following \"\n \"redirects cannot be used to reach internal resources. Disable to capture the \"\n \"first response as-is without following redirects.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different origin.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to an origin other than the one the\n caller intended them for. Headers are kept only when the redirect stays on the\n same origin (scheme, host, port) or is a direct https upgrade of the same host\n on default ports - the exact cases where httpx keeps the Authorization header.\n \"\"\"\n if not headers:\n return headers\n current, nxt = urlparse(current_url), urlparse(next_url)\n current_port = current.port or DEFAULT_SCHEME_PORTS.get(current.scheme)\n next_port = nxt.port or DEFAULT_SCHEME_PORTS.get(nxt.scheme)\n same_origin = (current.scheme, current.hostname, current_port) == (nxt.scheme, nxt.hostname, next_port)\n https_upgrade = (\n current.hostname == nxt.hostname\n and current.scheme == \"http\"\n and nxt.scheme == \"https\"\n and current_port == DEFAULT_SCHEME_PORTS[\"http\"]\n and next_port == DEFAULT_SCHEME_PORTS[\"https\"]\n )\n if same_origin or https_upgrade:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n def _process_response(self, response: httpx.Response) -> tuple[str, dict]:\n \"\"\"Turn a final (non-redirect) response into its HTML content and metadata.\n\n Args:\n response: The HTTP response to process\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {response.url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {response.url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n return html_content, metadata\n\n async def _fetch_with_revalidated_redirects(\n self, url: str, validated_ips: list[str], headers: dict\n ) -> tuple[str, dict]:\n \"\"\"Fetch ``url``, following redirects manually and re-validating every hop.\n\n This closes an SSRF bypass: with redirects enabled, httpx would otherwise\n auto-follow a redirect from a validated public URL to an internal address that\n the pinned transport never checked (the redirect target is a different host, so\n it is not in the pin map). Each redirect ``Location`` is resolved (relative\n locations included) and re-validated with ``ensure_url`` - the same private/\n loopback/link-local denylist and DNS pinning applied to the initial request -\n before any connection to it is made. A blocked hop raises ``ValueError`` (unless\n ``continue_on_failure``); the chain is capped at ``MAX_REDIRECTS`` hops.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning of the initial URL\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n current_url = url\n current_ips = validated_ips\n\n for _ in range(MAX_REDIRECTS + 1):\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.get(current_url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n location = response.headers.get(\"location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n try:\n validated_next_url, current_ips = self.ensure_url(next_url)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping blocked or invalid redirect to {next_url}: {e}\")\n return \"\", {}\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = validated_next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return self._process_response(response)\n\n # Exhausted the redirect budget.\n if self.continue_on_failure:\n logger.warning(f\"Exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\")\n return \"\", {}\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection, following redirects safely.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n try:\n # When SSRF protection is enabled we must follow redirects manually so each hop\n # is re-validated and DNS-pinned; letting httpx auto-follow would connect to the\n # redirect target without pinning (different host, not in the pin map) and re-open\n # the SSRF hole that DNS pinning closes. With protection disabled there is no pin\n # to bypass, so httpx can follow redirects natively.\n if self.follow_redirects and is_ssrf_protection_enabled():\n return await self._fetch_with_revalidated_redirects(url, validated_ips, headers)\n\n async with self._build_http_client(url, validated_ips) as client:\n response = await client.get(\n url, headers=headers, timeout=self.timeout, follow_redirects=self.follow_redirects\n )\n return self._process_response(response)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n # Redirects may have landed on a different canonical URL (http->https, www\n # normalization). Resolve relative links and the prevent_outside check against\n # the URL the content actually came from, and mark it visited so links back to\n # the canonical form are not re-crawled.\n base_url = metadata.get(\"source\") or start_url\n visited.add(base_url)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(base_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(base_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" }, "continue_on_failure": { "_input_type": "BoolInput", @@ -1670,6 +1671,26 @@ "type": "bool", "value": true }, + "follow_redirects": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Follow Redirects", + "dynamic": false, + "info": "If enabled, follows HTTP redirects such as http→https or www/non-www normalization, which most sites rely on to serve their content. When SSRF protection is enabled (the default), every redirect hop is re-validated against the same blocked-IP denylist and DNS-pinned before it is fetched, so following redirects cannot be used to reach internal resources. Disable to capture the first response as-is without following redirects.", + "list": false, + "list_add_label": "Add More", + "name": "follow_redirects", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + }, "format": { "_input_type": "DropdownInput", "advanced": true, diff --git a/src/backend/base/langflow/initial_setup/starter_projects/Travel Planning Agents.json b/src/backend/base/langflow/initial_setup/starter_projects/Travel Planning Agents.json index e7969703a9..069d5949a0 100644 --- a/src/backend/base/langflow/initial_setup/starter_projects/Travel Planning Agents.json +++ b/src/backend/base/langflow/initial_setup/starter_projects/Travel Planning Agents.json @@ -807,6 +807,7 @@ "max_depth", "prevent_outside", "use_async", + "follow_redirects", "format", "timeout", "headers", @@ -820,7 +821,7 @@ "legacy": false, "lf_version": "1.2.0", "metadata": { - "code_hash": "2e225a90043a", + "code_hash": "52b5bd602242", "dependencies": { "dependencies": [ { @@ -920,7 +921,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n async with self._build_http_client(url, validated_ips) as client:\n try:\n response = await client.get(url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n else:\n return html_content, metadata\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(start_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(start_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" + "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n# HTTP status codes that carry a redirect Location header (RFC 9110).\nREDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})\n\n# Maximum number of redirects to follow. Each hop is re-validated for SSRF safety;\n# matches httpx's default and the API Request component.\nMAX_REDIRECTS = 20\n\n# Default ports per scheme, used to compare redirect origins.\nDEFAULT_SCHEME_PORTS = {\"http\": 80, \"https\": 443}\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n info=(\n \"If enabled, follows HTTP redirects such as http→https or www/non-www \"\n \"normalization, which most sites rely on to serve their content. When SSRF \"\n \"protection is enabled (the default), every redirect hop is re-validated against \"\n \"the same blocked-IP denylist and DNS-pinned before it is fetched, so following \"\n \"redirects cannot be used to reach internal resources. Disable to capture the \"\n \"first response as-is without following redirects.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different origin.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to an origin other than the one the\n caller intended them for. Headers are kept only when the redirect stays on the\n same origin (scheme, host, port) or is a direct https upgrade of the same host\n on default ports - the exact cases where httpx keeps the Authorization header.\n \"\"\"\n if not headers:\n return headers\n current, nxt = urlparse(current_url), urlparse(next_url)\n current_port = current.port or DEFAULT_SCHEME_PORTS.get(current.scheme)\n next_port = nxt.port or DEFAULT_SCHEME_PORTS.get(nxt.scheme)\n same_origin = (current.scheme, current.hostname, current_port) == (nxt.scheme, nxt.hostname, next_port)\n https_upgrade = (\n current.hostname == nxt.hostname\n and current.scheme == \"http\"\n and nxt.scheme == \"https\"\n and current_port == DEFAULT_SCHEME_PORTS[\"http\"]\n and next_port == DEFAULT_SCHEME_PORTS[\"https\"]\n )\n if same_origin or https_upgrade:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n def _process_response(self, response: httpx.Response) -> tuple[str, dict]:\n \"\"\"Turn a final (non-redirect) response into its HTML content and metadata.\n\n Args:\n response: The HTTP response to process\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {response.url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {response.url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n return html_content, metadata\n\n async def _fetch_with_revalidated_redirects(\n self, url: str, validated_ips: list[str], headers: dict\n ) -> tuple[str, dict]:\n \"\"\"Fetch ``url``, following redirects manually and re-validating every hop.\n\n This closes an SSRF bypass: with redirects enabled, httpx would otherwise\n auto-follow a redirect from a validated public URL to an internal address that\n the pinned transport never checked (the redirect target is a different host, so\n it is not in the pin map). Each redirect ``Location`` is resolved (relative\n locations included) and re-validated with ``ensure_url`` - the same private/\n loopback/link-local denylist and DNS pinning applied to the initial request -\n before any connection to it is made. A blocked hop raises ``ValueError`` (unless\n ``continue_on_failure``); the chain is capped at ``MAX_REDIRECTS`` hops.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning of the initial URL\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n current_url = url\n current_ips = validated_ips\n\n for _ in range(MAX_REDIRECTS + 1):\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.get(current_url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n location = response.headers.get(\"location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n try:\n validated_next_url, current_ips = self.ensure_url(next_url)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping blocked or invalid redirect to {next_url}: {e}\")\n return \"\", {}\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = validated_next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return self._process_response(response)\n\n # Exhausted the redirect budget.\n if self.continue_on_failure:\n logger.warning(f\"Exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\")\n return \"\", {}\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection, following redirects safely.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n try:\n # When SSRF protection is enabled we must follow redirects manually so each hop\n # is re-validated and DNS-pinned; letting httpx auto-follow would connect to the\n # redirect target without pinning (different host, not in the pin map) and re-open\n # the SSRF hole that DNS pinning closes. With protection disabled there is no pin\n # to bypass, so httpx can follow redirects natively.\n if self.follow_redirects and is_ssrf_protection_enabled():\n return await self._fetch_with_revalidated_redirects(url, validated_ips, headers)\n\n async with self._build_http_client(url, validated_ips) as client:\n response = await client.get(\n url, headers=headers, timeout=self.timeout, follow_redirects=self.follow_redirects\n )\n return self._process_response(response)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n # Redirects may have landed on a different canonical URL (http->https, www\n # normalization). Resolve relative links and the prevent_outside check against\n # the URL the content actually came from, and mark it visited so links back to\n # the canonical form are not re-crawled.\n base_url = metadata.get(\"source\") or start_url\n visited.add(base_url)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(base_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(base_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" }, "continue_on_failure": { "_input_type": "BoolInput", @@ -958,6 +959,26 @@ "type": "bool", "value": true }, + "follow_redirects": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Follow Redirects", + "dynamic": false, + "info": "If enabled, follows HTTP redirects such as http→https or www/non-www normalization, which most sites rely on to serve their content. When SSRF protection is enabled (the default), every redirect hop is re-validated against the same blocked-IP denylist and DNS-pinned before it is fetched, so following redirects cannot be used to reach internal resources. Disable to capture the first response as-is without following redirects.", + "list": false, + "list_add_label": "Add More", + "name": "follow_redirects", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + }, "format": { "_input_type": "DropdownInput", "advanced": true, diff --git a/src/backend/base/langflow/locales/en.json b/src/backend/base/langflow/locales/en.json index ba5df4bef0..69f01063b4 100644 --- a/src/backend/base/langflow/locales/en.json +++ b/src/backend/base/langflow/locales/en.json @@ -6147,6 +6147,8 @@ "components.urlcomponent.inputs.continue_on_failure.info.635413d8": "If enabled, continues crawling even if some requests fail.", "components.urlcomponent.inputs.filter_text_html.display_name.ee58d47a": "Filter Text/HTML", "components.urlcomponent.inputs.filter_text_html.info.ea4adc6c": "If enabled, filters out text/css content type from the results.", + "components.urlcomponent.inputs.follow_redirects.display_name.0bbacd03": "Follow Redirects", + "components.urlcomponent.inputs.follow_redirects.info.580bb301": "If enabled, follows HTTP redirects such as http→https or www/non-www normalization, which most sites rely on to serve their content. When SSRF protection is enabled (the default), every redirect hop is re-validated against the same blocked-IP denylist and DNS-pinned before it is fetched, so following redirects cannot be used to reach internal resources. Disable to capture the first response as-is without following redirects.", "components.urlcomponent.inputs.format.display_name.ef22a51c": "Output Format", "components.urlcomponent.inputs.format.info.e5559996": "Output Format. Use 'Text' to extract the text from the HTML, 'Markdown' to parse the HTML into Markdown format, or 'HTML' for the raw HTML content.", "components.urlcomponent.inputs.headers.display_name.194e9fe6": "Headers", diff --git a/src/backend/base/langflow/services/auth/context.py b/src/backend/base/langflow/services/auth/context.py new file mode 100644 index 0000000000..c052f09dbe --- /dev/null +++ b/src/backend/base/langflow/services/auth/context.py @@ -0,0 +1,108 @@ +"""Request-local authentication credential context. + +Authentication resolves every credential to a Langflow user, but authorization +plugins sometimes need to know how that user authenticated. Keep that metadata +request-local so API-key caveats can be enforced without changing route +signatures or teaching OSS how to interpret policy. +""" + +from __future__ import annotations + +from contextvars import ContextVar +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from uuid import UUID + + from langflow.services.database.models.api_key.crud import ApiKeyAuthResult + + +AUTH_METHOD_API_KEY = "api_key" # pragma: allowlist secret +AUTH_METHOD_AUTO_LOGIN = "auto_login" +AUTH_METHOD_EXTERNAL = "external" +AUTH_METHOD_JWT = "jwt" + + +@dataclass(frozen=True) +class AuthCredentialContext: + """Metadata about the credential that authenticated the current request.""" + + method: str + api_key_id: UUID | None = None + api_key_source: str | None = None + external_provider: str | None = None + + @classmethod + def from_api_key_result(cls, result: ApiKeyAuthResult) -> AuthCredentialContext: + """Build API-key credential context from an authenticated API-key result. + + Centralizes the API-key projection so every call site stays in sync if the + set of API-key caveat fields ever changes. + """ + return cls( + method=AUTH_METHOD_API_KEY, + api_key_id=result.api_key_id, + api_key_source=result.api_key_source, + ) + + def to_authz_context(self) -> dict[str, Any]: + """Return values safe to pass into authorization plugin context.""" + context: dict[str, Any] = {"auth_method": self.method} + if self.api_key_id is not None: + context["api_key_id"] = self.api_key_id + if self.api_key_source is not None: + context["api_key_source"] = self.api_key_source + if self.external_provider is not None: + context["external_provider"] = self.external_provider + return context + + def to_audit_details(self) -> dict[str, str]: + """Return JSON-friendly values safe for authz audit details.""" + details = {"auth_method": self.method} + if self.api_key_id is not None: + details["api_key_id"] = str(self.api_key_id) + if self.api_key_source is not None: + details["api_key_source"] = self.api_key_source + if self.external_provider is not None: + details["external_provider"] = self.external_provider + return details + + +_current_auth_context = ContextVar["AuthCredentialContext | None"]( + "langflow_auth_credential_context", + default=None, +) + + +def set_current_auth_context(context: AuthCredentialContext | None) -> None: + """Store credential metadata for the current request/task.""" + _current_auth_context.set(context) + + +def clear_current_auth_context() -> None: + """Clear credential metadata for the current request/task.""" + _current_auth_context.set(None) + + +def get_current_auth_context() -> AuthCredentialContext | None: + """Return credential metadata for the current request/task, if any.""" + return _current_auth_context.get() + + +def current_auth_context_for_authz() -> dict[str, Any]: + """Return current credential metadata as an authz context fragment.""" + context = get_current_auth_context() + return context.to_authz_context() if context is not None else {} + + +def current_auth_context_for_audit() -> dict[str, str]: + """Return current credential metadata as an audit details fragment.""" + context = get_current_auth_context() + return context.to_audit_details() if context is not None else {} + + +def current_auth_is_api_key() -> bool: + """Return True when the active request authenticated with a Langflow API key.""" + context = get_current_auth_context() + return context is not None and context.method == AUTH_METHOD_API_KEY diff --git a/src/backend/base/langflow/services/auth/external.py b/src/backend/base/langflow/services/auth/external.py new file mode 100644 index 0000000000..eac4c7ae58 --- /dev/null +++ b/src/backend/base/langflow/services/auth/external.py @@ -0,0 +1,537 @@ +"""External trusted-identity helpers. + +When an upstream identity layer (proxy, gateway, IdP) issues or validates a +credential, Langflow accepts it via this module: extract the token from the +configured header/cookie, validate it (built-in JWT/JWKS or a pluggable +resolver), and return a normalized :class:`ExternalIdentity`. JIT user +provisioning is handled separately by +``BaseAuthService.get_or_create_user_from_claims`` so the auth service stays +the single source of truth for user lifecycle. +""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import time +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Protocol, TypeAlias, TypeVar, cast +from urllib.parse import urlparse + +import httpx +import jwt +from jwt import InvalidTokenError as PyJWTInvalidTokenError +from lfx.log.logger import logger + +from langflow.services.auth.exceptions import InvalidTokenError as AuthInvalidTokenError + +# The request-scoped access ceiling is an authorization primitive. It lives in +# the authorization package so guards can enforce it without importing the auth +# layer; the auth layer (here) only *derives* the ceiling from an identity and +# installs it. These re-exports keep ``langflow.services.auth.external`` a stable +# import site for callers that derive/inspect the ceiling. +from langflow.services.authorization.access_ceiling import ( + EXTERNAL_ACCESS_ADMIN, + EXTERNAL_ACCESS_EDITOR, + EXTERNAL_ACCESS_LEVELS, + EXTERNAL_ACCESS_VIEWER, + ExternalAccessContext, + clear_current_external_access_context, + external_access_allows, + filter_actions_by_external_access_ceiling, + get_current_external_access_context, + set_current_external_access_context, +) + +if TYPE_CHECKING: + from lfx.services.settings.auth import AuthSettings + +__all__ = [ + "EXTERNAL_ACCESS_ADMIN", + "EXTERNAL_ACCESS_EDITOR", + "EXTERNAL_ACCESS_LEVELS", + "EXTERNAL_ACCESS_VIEWER", + "ExternalAccessContext", + "ExternalIdentity", + "ExternalIdentityResolver", + "JwtExternalIdentityResolver", + "access_context_from_identity", + "clear_current_external_access_context", + "decode_external_jwt", + "external_access_allows", + "extract_bearer_or_raw_token", + "extract_external_token", + "filter_actions_by_external_access_ceiling", + "get_current_external_access_context", + "identity_from_claims", + "resolve_external_identity", + "set_current_external_access_context", +] + + +JWKS_CACHE_TTL_SECONDS = 300 +JWKS_MIN_REFRESH_INTERVAL_SECONDS = 30 +_jwks_cache: dict[str, tuple[float, dict[str, Any]]] = {} +# Loopback hosts allowed to use http:// for the JWKS URL in local development. +_JWKS_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) +T = TypeVar("T") + +# Maps raw external claim values to a normalized access level. The level +# vocabulary and the deny-only enforcement live in the authorization package +# (see ``access_ceiling``); this alias table is the auth-side interpretation of +# provider-specific claim strings. +_EXTERNAL_ACCESS_ALIASES = { + "view": EXTERNAL_ACCESS_VIEWER, + "viewer": EXTERNAL_ACCESS_VIEWER, + "read": EXTERNAL_ACCESS_VIEWER, + "readonly": EXTERNAL_ACCESS_VIEWER, + "read_only": EXTERNAL_ACCESS_VIEWER, + "read-only": EXTERNAL_ACCESS_VIEWER, + "edit": EXTERNAL_ACCESS_EDITOR, + "editor": EXTERNAL_ACCESS_EDITOR, + "write": EXTERNAL_ACCESS_EDITOR, + "developer": EXTERNAL_ACCESS_EDITOR, + "admin": EXTERNAL_ACCESS_ADMIN, + "administrator": EXTERNAL_ACCESS_ADMIN, +} + + +@dataclass(frozen=True) +class ExternalIdentity: + """Normalized identity returned by an :class:`ExternalIdentityResolver`.""" + + provider: str + subject: str + username: str + email: str | None = None + name: str | None = None + claims: Mapping[str, Any] = field(default_factory=dict) + + +class ExternalIdentityResolver(Protocol): + """Resolver that turns an external credential into an identity.""" + + async def resolve( + self, + token: str, + auth_settings: AuthSettings, + ) -> ExternalIdentity | Mapping[str, Any]: ... + + +ExternalResolverResult: TypeAlias = ExternalIdentity | Mapping[str, Any] +ExternalResolverCallable: TypeAlias = Callable[ + [str, "AuthSettings"], + ExternalResolverResult | Awaitable[ExternalResolverResult], +] + + +def _split_csv(value: str | None) -> list[str]: + if not value: + return [] + return [item.strip() for item in value.split(",") if item.strip()] + + +def _claim_as_str(claims: Mapping[str, Any], claim_name: str | None) -> str | None: + if not claim_name: + return None + value = claims.get(claim_name) + if isinstance(value, str): + stripped = value.strip() + return stripped or None + if value is None: + return None + return str(value) + + +def _normalize_username(value: str) -> str: + username = value.strip() + if not username: + return "external-user" + return username[:255] + + +def _external_username_fallback(provider: str, subject: str) -> str: + digest = hashlib.sha256(f"{provider}:{subject}".encode()).hexdigest()[:12] + normalized_provider = provider[:200] or "external" + return f"{normalized_provider}-{digest}" + + +def identity_from_claims(claims: Mapping[str, Any], auth_settings: AuthSettings) -> ExternalIdentity: + """Build an :class:`ExternalIdentity` from raw JWT claims using the configured mapping.""" + provider = (auth_settings.EXTERNAL_AUTH_PROVIDER or "external").strip() or "external" + subject = _claim_as_str(claims, auth_settings.EXTERNAL_AUTH_SUBJECT_CLAIM) + if not subject: + msg = f"External credential is missing required claim: {auth_settings.EXTERNAL_AUTH_SUBJECT_CLAIM}" + raise AuthInvalidTokenError(msg) + + email = _claim_as_str(claims, auth_settings.EXTERNAL_AUTH_EMAIL_CLAIM) + preferred_username = _claim_as_str(claims, auth_settings.EXTERNAL_AUTH_USERNAME_CLAIM) + name = _claim_as_str(claims, auth_settings.EXTERNAL_AUTH_NAME_CLAIM) + username_claim = preferred_username or email or name or _external_username_fallback(provider, subject) + + return ExternalIdentity( + provider=provider, + subject=subject, + username=_normalize_username(username_claim), + email=email, + name=name, + claims=dict(claims), + ) + + +def _normalize_access_level(value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip().lower() + if not normalized: + return None + return _EXTERNAL_ACCESS_ALIASES.get(normalized, normalized if normalized in EXTERNAL_ACCESS_LEVELS else None) + + +def _access_claim_mapping(auth_settings: AuthSettings) -> dict[str, str]: + raw_mapping = auth_settings.EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING + if not raw_mapping: + return {} + + mapping: dict[str, str] = {} + try: + loaded = json.loads(raw_mapping) + except json.JSONDecodeError: + loaded = None + + if isinstance(loaded, Mapping): + pairs = loaded.items() + else: + pairs = [] + for item in raw_mapping.split(","): + key, separator, value = item.partition(":") + if not separator: + continue + pairs.append((key, value)) + + for key, value in pairs: + if not isinstance(key, str): + continue + normalized_level = _normalize_access_level(str(value)) + if normalized_level is not None: + mapping[key.strip().lower()] = normalized_level + return mapping + + +def access_context_from_identity( + identity: ExternalIdentity, + auth_settings: AuthSettings, +) -> ExternalAccessContext | None: + """Return the request-local access ceiling for an external identity.""" + if not auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED: + return None + + claim_name = auth_settings.EXTERNAL_AUTH_ACCESS_CLAIM + claim_value = _claim_as_str(identity.claims, claim_name) + mapping = _access_claim_mapping(auth_settings) + # Gate the alias fallthrough on whether the operator CONFIGURED a mapping + # (the raw setting), not on whether it parsed to a non-empty dict. A + # configured-but-all-invalid mapping still parses empty; treating that as + # "no mapping" would let a raw "admin"/"editor" claim self-elevate via the + # alias table, re-opening the hole the authoritative-mapping rule closes. + raw_mapping_configured = bool((auth_settings.EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING or "").strip()) + mapped_level = None + if claim_value is not None: + mapped_level = mapping.get(claim_value.strip().lower()) + # When an explicit mapping is configured it is authoritative: a claim + # value absent from it must NOT be reinterpreted through the built-in + # alias table (otherwise a raw "admin"/"editor" claim would silently + # elevate without an explicit grant). Fall through to the default level + # instead. The alias interpretation is only used when NO mapping is + # configured at all. + if mapped_level is None and not raw_mapping_configured: + mapped_level = _normalize_access_level(claim_value) + level = mapped_level or _normalize_access_level(auth_settings.EXTERNAL_AUTH_DEFAULT_ACCESS_LEVEL) + if level is None: + level = EXTERNAL_ACCESS_VIEWER + + return ExternalAccessContext( + provider=identity.provider, + subject=identity.subject, + level=level, + claim_name=claim_name, + claim_value=claim_value, + ) + + +def _validate_trusted_time_claims(claims: Mapping[str, Any]) -> None: + now = datetime.now(timezone.utc).timestamp() + exp = claims.get("exp") + # A token that omits exp never expires. Require it on the trusted-decode + # path too so a credential without an expiry is rejected rather than + # accepted forever (mirrors the JWKS path's require=["exp"]). + if exp is None: + msg = "External credential is missing exp" + raise AuthInvalidTokenError(msg) + if now > float(exp): + msg = "External credential has expired" + raise AuthInvalidTokenError(msg) + + nbf = claims.get("nbf") + if nbf is not None and now < float(nbf): + msg = "External credential is not valid yet" + raise AuthInvalidTokenError(msg) + + +def _require_https_jwks_url(jwks_url: str) -> None: + """Reject a non-https JWKS URL (http allowed only for loopback hosts). + + Belt-and-suspenders alongside the settings validator: an http:// JWKS lets a + network MITM swap the signing keys and forge tokens, so the fetch itself + refuses anything that is not https (or http to a loopback host for dev). + """ + parsed = urlparse(jwks_url) + scheme = parsed.scheme.lower() + if scheme == "https": + return + if scheme == "http" and parsed.hostname in _JWKS_LOOPBACK_HOSTS: + return + msg = "External JWKS URL must use https (http is allowed only for localhost)" + raise AuthInvalidTokenError(msg) + + +async def _fetch_jwks(jwks_url: str, *, force_refresh: bool = False) -> dict[str, Any]: + _require_https_jwks_url(jwks_url) + cached = _jwks_cache.get(jwks_url) + now = time.monotonic() + if cached and cached[0] > now: + # force_refresh is rate-limited so attacker-supplied kids cannot turn + # every rejected token into a fetch against the IdP's JWKS endpoint. + fetched_at = cached[0] - JWKS_CACHE_TTL_SECONDS + if not force_refresh or now - fetched_at < JWKS_MIN_REFRESH_INTERVAL_SECONDS: + return cached[1] + + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.get(jwks_url) + response.raise_for_status() + jwks = response.json() + + _jwks_cache[jwks_url] = (now + JWKS_CACHE_TTL_SECONDS, jwks) + return jwks + + +def _select_jwk(jwks: dict[str, Any], token: str) -> dict[str, Any]: + keys = jwks.get("keys") + if not isinstance(keys, list) or not keys: + msg = "External JWKS does not contain signing keys" + raise AuthInvalidTokenError(msg) + + header = jwt.get_unverified_header(token) + kid = header.get("kid") + if kid: + for key in keys: + if key.get("kid") == kid: + return key + msg = "External JWT signing key was not found in JWKS" + raise AuthInvalidTokenError(msg) + + if len(keys) == 1: + return keys[0] + + msg = "External JWT is missing kid and JWKS contains multiple keys" + raise AuthInvalidTokenError(msg) + + +async def decode_external_jwt(token: str, auth_settings: AuthSettings) -> dict[str, Any]: + """Validate an external JWT and return its claims. + + If ``EXTERNAL_AUTH_TRUSTED_JWT_DECODE`` is enabled, signature verification + is skipped (the caller has stated an upstream proxy already validated it). + Otherwise ``EXTERNAL_AUTH_JWKS_URL`` and ``EXTERNAL_AUTH_AUDIENCE`` are both + required: the signature is verified against the fetched JWKS using + ``EXTERNAL_AUTH_ALGORITHMS`` and the ``aud`` claim is bound to this + deployment so tokens the IdP minted for other services are rejected. + ``EXTERNAL_AUTH_ISSUER`` is verified when set. ``exp`` is required and + verified on both paths (a token that omits it is rejected); ``nbf`` is + verified when present. + """ + if not auth_settings.EXTERNAL_AUTH_ENABLED: + msg = "External authentication is not enabled" + raise AuthInvalidTokenError(msg) + + try: + if auth_settings.EXTERNAL_AUTH_TRUSTED_JWT_DECODE: + claims = jwt.decode( + token, + options={ + "verify_signature": False, + "verify_aud": False, + "verify_iss": False, + "verify_exp": False, + "verify_nbf": False, + }, + ) + _validate_trusted_time_claims(claims) + return claims + + if not auth_settings.EXTERNAL_AUTH_JWKS_URL: + msg = "External authentication requires EXTERNAL_AUTH_JWKS_URL unless trusted decode is enabled" + raise AuthInvalidTokenError(msg) + + audience = _split_csv(auth_settings.EXTERNAL_AUTH_AUDIENCE) + if not audience: + # Without audience binding, any token the IdP minted for a *different* + # relying party would verify here (same signing keys, valid exp). + # Audience is the control that stops cross-service token reuse, so it + # is required whenever signatures are checked against a JWKS. + msg = ( + "External JWKS verification requires EXTERNAL_AUTH_AUDIENCE so tokens the IdP issued " + "for other services are rejected. Set EXTERNAL_AUTH_AUDIENCE to this deployment's " + "expected audience, or only enable EXTERNAL_AUTH_TRUSTED_JWT_DECODE behind a proxy " + "that already validates audience." + ) + raise AuthInvalidTokenError(msg) + + jwks = await _fetch_jwks(auth_settings.EXTERNAL_AUTH_JWKS_URL) + try: + jwk = _select_jwk(jwks, token) + except AuthInvalidTokenError: + # The token's kid may belong to a key published after the cached + # JWKS was fetched (IdP key rotation). Refetch once; when the + # rate limit suppresses the refetch we get the same cached object + # back and re-raise the original error. + refreshed = await _fetch_jwks(auth_settings.EXTERNAL_AUTH_JWKS_URL, force_refresh=True) + if refreshed is jwks: + raise + jwk = _select_jwk(refreshed, token) + signing_key = jwt.PyJWK.from_dict(jwk).key + issuer = auth_settings.EXTERNAL_AUTH_ISSUER or None + algorithms = _split_csv(auth_settings.EXTERNAL_AUTH_ALGORITHMS) or ["RS256"] + + return jwt.decode( + token, + signing_key, + algorithms=algorithms, + audience=audience, + issuer=issuer, + options={ + "verify_aud": True, + "verify_iss": bool(issuer), + # PyJWT only rejects an *expired* exp; a token that omits exp + # otherwise passes and never expires. require=["exp"] forces the + # claim to be present so unbounded-lifetime tokens are rejected. + "verify_exp": True, + "require": ["exp"], + }, + ) + except AuthInvalidTokenError: + raise + except PyJWTInvalidTokenError as exc: + msg = "External credential validation failed" + raise AuthInvalidTokenError(msg) from exc + except Exception as exc: + logger.debug(f"External credential validation failed: {exc}") + msg = "External credential validation failed" + raise AuthInvalidTokenError(msg) from exc + + +class JwtExternalIdentityResolver: + """Default resolver: validate the credential as a JWT and map claims.""" + + async def resolve(self, token: str, auth_settings: AuthSettings) -> ExternalIdentity: + claims = await decode_external_jwt(token, auth_settings) + return identity_from_claims(claims, auth_settings) + + +async def resolve_external_identity(token: str, auth_settings: AuthSettings) -> ExternalIdentity: + """Resolve a credential to an :class:`ExternalIdentity` using the configured resolver.""" + resolver = _load_external_identity_resolver(auth_settings) + if hasattr(resolver, "resolve"): + result = await _maybe_await(resolver.resolve(token, auth_settings)) + elif callable(resolver): + result = await _maybe_await(resolver(token, auth_settings)) + else: + msg = "External authentication resolver must be callable or expose resolve()" + raise AuthInvalidTokenError(msg) + + if isinstance(result, ExternalIdentity): + return result + if isinstance(result, Mapping): + return identity_from_claims(result, auth_settings) + + msg = "External authentication resolver returned an invalid identity" + raise AuthInvalidTokenError(msg) + + +def _load_external_identity_resolver( + auth_settings: AuthSettings, +) -> ExternalIdentityResolver | ExternalResolverCallable: + resolver_path = auth_settings.EXTERNAL_AUTH_IDENTITY_RESOLVER + if not resolver_path: + return JwtExternalIdentityResolver() + + from lfx.services.config_discovery import load_object_from_import_path + + resolver = load_object_from_import_path( + resolver_path, + object_kind="external auth resolver", + object_key="EXTERNAL_AUTH_IDENTITY_RESOLVER", + ) + if resolver is None: + msg = "External authentication resolver could not be loaded" + raise AuthInvalidTokenError(msg) + + if inspect.isclass(resolver): + signature = inspect.signature(resolver) + required_params = [ + parameter + for parameter in signature.parameters.values() + if parameter.default is inspect.Parameter.empty + and parameter.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + ] + if required_params: + return resolver(auth_settings) + return resolver() + + return resolver + + +async def _maybe_await(value: T | Awaitable[T]) -> T: + if inspect.isawaitable(value): + return await cast("Awaitable[T]", value) + return cast("T", value) + + +def extract_bearer_or_raw_token(value: str | None) -> str | None: + """Strip a 'Bearer ' prefix if present and return the credential.""" + if not value: + return None + stripped = value.strip() + if not stripped: + return None + scheme, _, token = stripped.partition(" ") + if scheme.lower() == "bearer": + return token.strip() or None + return stripped + + +def extract_external_token( + headers: Mapping[str, str], + cookies: Mapping[str, str], + auth_settings: AuthSettings, +) -> str | None: + """Return the external credential from configured header/cookie, header first.""" + if not auth_settings.EXTERNAL_AUTH_ENABLED: + return None + + header_name = auth_settings.EXTERNAL_AUTH_TOKEN_HEADER + if header_name: + header_value = headers.get(header_name) + if token := extract_bearer_or_raw_token(header_value): + return token + + cookie_name = auth_settings.EXTERNAL_AUTH_TOKEN_COOKIE + if cookie_name: + cookie_value = cookies.get(cookie_name) + if token := extract_bearer_or_raw_token(cookie_value): + return token + + return None diff --git a/src/backend/base/langflow/services/auth/service.py b/src/backend/base/langflow/services/auth/service.py index df32a81a40..9cf891a53b 100644 --- a/src/backend/base/langflow/services/auth/service.py +++ b/src/backend/base/langflow/services/auth/service.py @@ -17,6 +17,14 @@ from sqlalchemy.exc import IntegrityError from langflow.helpers.user import get_user_by_flow_id_or_endpoint_name from langflow.services.auth.constants import AUTO_LOGIN_ERROR, AUTO_LOGIN_WARNING +from langflow.services.auth.context import ( + AUTH_METHOD_AUTO_LOGIN, + AUTH_METHOD_EXTERNAL, + AUTH_METHOD_JWT, + AuthCredentialContext, + clear_current_auth_context, + set_current_auth_context, +) from langflow.services.auth.exceptions import ( InactiveUserError, InvalidCredentialsError, @@ -26,7 +34,16 @@ from langflow.services.auth.exceptions import ( from langflow.services.auth.exceptions import ( InvalidTokenError as AuthInvalidTokenError, ) -from langflow.services.database.models.api_key.crud import check_key +from langflow.services.auth.external import ( + ExternalIdentity, + _external_username_fallback, + access_context_from_identity, + clear_current_external_access_context, + identity_from_claims, + resolve_external_identity, + set_current_external_access_context, +) +from langflow.services.database.models.api_key.crud import authenticate_api_key from langflow.services.database.models.user.crud import ( get_user_by_id, get_user_by_username, @@ -40,8 +57,6 @@ if TYPE_CHECKING: from lfx.services.settings.service import SettingsService from sqlmodel.ext.asyncio.session import AsyncSession - from langflow.services.database.models.api_key.model import ApiKey - class AuthService(BaseAuthService): """Default Langflow authentication service (implements LFX BaseAuthService).""" @@ -61,6 +76,7 @@ class AuthService(BaseAuthService): token: str | None, api_key: str | None, db: AsyncSession, + external_token: str | None = None, ) -> User | UserRead: """Framework-agnostic authentication method. @@ -71,6 +87,11 @@ class AuthService(BaseAuthService): token: Access token (JWT, OIDC token, etc.) api_key: API key for authentication db: Database session + external_token: Separately-extracted external credential to try as a + fallback when native token authentication fails for any reason + (expired, invalid, inactive user). When ``None`` behavior is + unchanged. This lets a valid external credential authenticate even + when a present-but-invalid native token would otherwise shadow it. Returns: @@ -84,15 +105,31 @@ class AuthService(BaseAuthService): TokenExpiredError: If token has expired InactiveUserError: If user account is inactive """ + clear_current_auth_context() + clear_current_external_access_context() + # Try token authentication first (if token provided) if token: try: return await self._authenticate_with_token(token, db) - except (AuthInvalidTokenError, TokenExpiredError, InactiveUserError): - # Re-raise our generic exceptions - raise + except (AuthInvalidTokenError, TokenExpiredError, InactiveUserError) as e: + # Native auth failed. If a *distinct* external credential was + # extracted, try it before surfacing the native error so a present + # but invalid/expired native token can't shadow a valid external + # one. When external_token is None or identical to the token we + # already tried, behavior is unchanged. + if external_token and external_token != token: + external_user = await self._authenticate_with_external_token(external_token, db) + if external_user is not None: + return external_user + raise e # noqa: TRY201 except Exception as e: - # Token auth failed; fall back to API key if provided + # Token auth failed for an unexpected reason; try the distinct + # external credential first, then fall back to API key if provided. + if external_token and external_token != token: + external_user = await self._authenticate_with_external_token(external_token, db) + if external_user is not None: + return external_user if api_key: try: user = await self._authenticate_with_api_key(api_key, db) @@ -110,6 +147,13 @@ class AuthService(BaseAuthService): msg = "Token authentication failed" raise AuthInvalidTokenError(msg) from e + # No native token, but a separately-extracted external credential may be + # present (extractors no longer collapse native/external into one string). + if external_token: + external_user = await self._authenticate_with_external_token(external_token, db) + if external_user is not None: + return external_user + # Try API key authentication if api_key: try: @@ -150,6 +194,7 @@ class AuthService(BaseAuthService): msg = "User account is inactive" raise InactiveUserError(msg) logger.warning(AUTO_LOGIN_WARNING) + set_current_auth_context(AuthCredentialContext(method=AUTH_METHOD_AUTO_LOGIN)) return superuser # No credentials provided @@ -198,10 +243,16 @@ class AuthService(BaseAuthService): msg = "Token has expired" raise TokenExpiredError(msg) from e except InvalidTokenError as e: + external_user = await self._authenticate_with_external_token(token, db) + if external_user is not None: + return external_user logger.debug("JWT validation failed: Invalid token format or signature") msg = "Invalid token" raise AuthInvalidTokenError(msg) from e except Exception as e: + external_user = await self._authenticate_with_external_token(token, db) + if external_user is not None: + return external_user logger.error(f"Unexpected error decoding token: {e}") msg = "Token validation failed" raise AuthInvalidTokenError(msg) from e @@ -218,23 +269,179 @@ class AuthService(BaseAuthService): msg = "User account is inactive" raise InactiveUserError(msg) + set_current_auth_context(AuthCredentialContext(method=AUTH_METHOD_JWT)) return user + async def _authenticate_with_external_token(self, token: str, db: AsyncSession) -> User | None: + """Fallback path: try the configured external identity resolver. + + Returns the JIT-provisioned local user when the token resolves to a + valid external identity, ``None`` otherwise. Callers raise the native + JWT error if this returns ``None``. + """ + if not self.settings.auth_settings.EXTERNAL_AUTH_ENABLED: + return None + try: + identity = await resolve_external_identity(token, self.settings.auth_settings) + except AuthInvalidTokenError as exc: + logger.debug(f"External credential rejected: {exc}") + return None + set_current_auth_context( + AuthCredentialContext(method=AUTH_METHOD_EXTERNAL, external_provider=identity.provider) + ) + set_current_external_access_context(access_context_from_identity(identity, self.settings.auth_settings)) + return await self._materialize_external_user(identity, db) + async def _authenticate_with_api_key(self, api_key: str, db: AsyncSession) -> UserRead | None: - """Internal method to authenticate with API key (raises generic exceptions).""" - result = await check_key(db, api_key) + """Internal method to authenticate with API key (raises generic exceptions). + + The EXTERNAL_AUTH access ceiling block for externally-managed users is + enforced inside ``authenticate_api_key`` (the shared chokepoint), which + returns ``None`` for a blocked user so every caller treats it as an auth + failure. No additional ceiling check is needed here. + """ + result = await authenticate_api_key(db, api_key) if not result: return None - if isinstance(result, User): - user_read = UserRead.model_validate(result, from_attributes=True) + if isinstance(result.user, User): + user_read = UserRead.model_validate(result.user, from_attributes=True) if not user_read.is_active: msg = "User account is inactive" raise InactiveUserError(msg) + set_current_auth_context(AuthCredentialContext.from_api_key_result(result)) return user_read return None + # ------------------------------------------------------------------ + # JIT user provisioning via BaseAuthService hook + # ------------------------------------------------------------------ + + def extract_user_info_from_claims(self, claims: dict) -> dict: + """Normalize provider claims using the configured EXTERNAL_AUTH_* mapping. + + Returns a dict with ``provider``, ``subject``, ``username``, ``email``, + and ``name`` keys; raises :class:`AuthInvalidTokenError` when the + subject claim is missing. + """ + identity = identity_from_claims(claims, self.settings.auth_settings) + return { + "provider": identity.provider, + "subject": identity.subject, + "username": identity.username, + "email": identity.email, + "name": identity.name, + } + + async def get_or_create_user_from_claims(self, claims: dict, db: AsyncSession) -> User: + """Return the local Langflow user mapped to these external claims. + + Looks up SSOUserProfile by (provider, sso_user_id). On hit, refreshes + the email + last-login timestamps and returns the existing user. On + miss, JIT-provisions a fresh user, writes a profile row, and seeds + the default folder + variables. + """ + identity = identity_from_claims(claims, self.settings.auth_settings) + return await self._materialize_external_user(identity, db) + + async def _materialize_external_user(self, identity: ExternalIdentity, db: AsyncSession) -> User: + """Find-or-create the local user backing an external identity.""" + import secrets + from datetime import datetime, timezone + + from sqlalchemy.exc import IntegrityError + from sqlmodel import select + + from langflow.services.database.models.auth import SSOUserProfile + + profile_stmt = select(SSOUserProfile).where( + SSOUserProfile.sso_provider == identity.provider, + SSOUserProfile.sso_user_id == identity.subject, + ) + profile = (await db.exec(profile_stmt)).first() + + if profile is not None: + user = await get_user_by_id(db, profile.user_id) + if user is None: + msg = "Mapped external user was not found" + raise AuthInvalidTokenError(msg) + if not user.is_active: + msg = "User account is inactive" + raise InactiveUserError(msg) + now = datetime.now(timezone.utc) + # Only overwrite the stored email when the token carries one; a later + # token that omits the email claim must not erase a previously stored + # address. + if identity.email is not None: + profile.email = identity.email + profile.sso_last_login_at = now + profile.updated_at = now + await update_user_last_login_at(user.id, db) + return user + + username = await self._unique_external_username(db, identity) + random_password = secrets.token_urlsafe(48) + now = datetime.now(timezone.utc) + user = User( + username=username, + password=self.get_password_hash(random_password), + is_active=True, + is_superuser=False, + last_login_at=now, + ) + new_profile = SSOUserProfile( + user_id=user.id, + sso_provider=identity.provider, + sso_user_id=identity.subject, + email=identity.email, + sso_last_login_at=now, + ) + db.add(user) + db.add(new_profile) + try: + await db.flush() + await db.refresh(user) + await self._initialize_jit_user_defaults(user, db) + except IntegrityError: + await db.rollback() + profile = (await db.exec(profile_stmt)).first() + if profile is None: + raise + user = await get_user_by_id(db, profile.user_id) + if user is None: + msg = "Mapped external user was not found" + raise AuthInvalidTokenError(msg) from None + if not user.is_active: + msg = "User account is inactive" + raise InactiveUserError(msg) from None + + return user + + @staticmethod + async def _unique_external_username(db: AsyncSession, identity: ExternalIdentity) -> str: + desired = identity.username + if await get_user_by_username(db, desired) is None: + return desired + fallback = _external_username_fallback(identity.provider, identity.subject) + if await get_user_by_username(db, fallback) is None: + return fallback + # Final tier: fold the desired name into the digest so two providers' + # subjects that collide on the helper's digest still resolve uniquely. + import hashlib + + long_digest = hashlib.sha256(f"{identity.provider}:{identity.subject}:{desired}".encode()).hexdigest()[:16] + normalized_provider = identity.provider[:200] or "external" + return f"{normalized_provider}-{long_digest}" + + @staticmethod + async def _initialize_jit_user_defaults(user: User, db: AsyncSession) -> None: + from langflow.initial_setup.setup import get_or_create_default_folder + from langflow.services.deps import get_variable_service + + await get_or_create_default_folder(db, user.id) + await get_variable_service().initialize_user_variables(user.id, db) + async def api_key_security( self, query_param: str | None, header_param: str | None, db: AsyncSession | None = None ) -> UserRead | None: @@ -254,7 +461,8 @@ class AuthService(BaseAuthService): db: AsyncSession, settings_service, ) -> UserRead | None: - result: ApiKey | User | None + clear_current_auth_context() + clear_current_external_access_context() if settings_service.auth_settings.AUTO_LOGIN: if not settings_service.auth_settings.SUPERUSER: @@ -276,6 +484,7 @@ class AuthService(BaseAuthService): detail="User account is inactive", ) logger.warning(AUTO_LOGIN_WARNING) + set_current_auth_context(AuthCredentialContext(method=AUTH_METHOD_AUTO_LOGIN)) return UserRead.model_validate(result, from_attributes=True) raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, @@ -285,7 +494,7 @@ class AuthService(BaseAuthService): api_key = query_param or header_param if api_key is None: # pragma: no cover - guaranteed by the if-condition above raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid or missing API key") - result = await check_key(db, api_key) + api_key_result = await authenticate_api_key(db, api_key) elif not query_param and not header_param: raise HTTPException( @@ -298,23 +507,27 @@ class AuthService(BaseAuthService): api_key = query_param or header_param if api_key is None: # pragma: no cover - guaranteed by the elif-condition above raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid or missing API key") - result = await check_key(db, api_key) + api_key_result = await authenticate_api_key(db, api_key) - if not result: + if not api_key_result: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Invalid or missing API key", ) - if isinstance(result, User): - return UserRead.model_validate(result, from_attributes=True) + if isinstance(api_key_result.user, User): + set_current_auth_context(AuthCredentialContext.from_api_key_result(api_key_result)) + return UserRead.model_validate(api_key_result.user, from_attributes=True) msg = "Invalid result type" raise ValueError(msg) async def ws_api_key_security(self, api_key: str | None) -> UserRead: settings = self.settings + clear_current_auth_context() + clear_current_external_access_context() async with session_scope() as db: + api_key_result = None if settings.auth_settings.AUTO_LOGIN: if not settings.auth_settings.SUPERUSER: raise WebSocketException( @@ -335,13 +548,15 @@ class AuthService(BaseAuthService): reason="User account is inactive", ) logger.warning(AUTO_LOGIN_WARNING) + set_current_auth_context(AuthCredentialContext(method=AUTH_METHOD_AUTO_LOGIN)) else: raise WebSocketException( code=status.WS_1008_POLICY_VIOLATION, reason=AUTO_LOGIN_ERROR, ) else: - result = await check_key(db, api_key) + api_key_result = await authenticate_api_key(db, api_key) + result = api_key_result.user if api_key_result is not None else None else: if not api_key: @@ -349,7 +564,8 @@ class AuthService(BaseAuthService): code=status.WS_1008_POLICY_VIOLATION, reason="An API key must be passed as query or header", ) - result = await check_key(db, api_key) + api_key_result = await authenticate_api_key(db, api_key) + result = api_key_result.user if api_key_result is not None else None if not result: raise WebSocketException( @@ -358,6 +574,8 @@ class AuthService(BaseAuthService): ) if isinstance(result, User): + if api_key_result is not None: + set_current_auth_context(AuthCredentialContext.from_api_key_result(api_key_result)) return UserRead.model_validate(result, from_attributes=True) raise WebSocketException( @@ -371,6 +589,7 @@ class AuthService(BaseAuthService): query_param: str | None, header_param: str | None, db: AsyncSession, + external_token: str | None = None, ) -> User | UserRead: # Handle coroutine token (FastAPI dependency injection) resolved_token: str | None = None @@ -383,24 +602,31 @@ class AuthService(BaseAuthService): api_key = query_param or header_param # Delegate to framework-agnostic method - return await self.authenticate_with_credentials(resolved_token, api_key, db) + return await self.authenticate_with_credentials(resolved_token, api_key, db, external_token=external_token) async def get_current_user_from_access_token( self, token: str | Coroutine | None, db: AsyncSession, + external_token: str | None = None, ) -> User: """Get user from access token (raises generic exceptions). This method now uses the framework-agnostic _authenticate_with_token() internally. + + ``external_token`` is an optional, separately-extracted external credential + tried as a fallback when native token authentication fails so a + present-but-invalid native token cannot shadow a valid external one. When + ``None`` (or identical to ``token``) behavior is unchanged. """ - if token is None: - msg = "Missing authentication token" - raise MissingCredentialsError(msg) + clear_current_auth_context() + clear_current_external_access_context() # Handle coroutine token (FastAPI dependency injection) - resolved_token: str - if isinstance(token, Coroutine): + resolved_token: str | None + if token is None: + resolved_token = None + elif isinstance(token, Coroutine): resolved_token = await token elif isinstance(token, str): resolved_token = token @@ -408,26 +634,51 @@ class AuthService(BaseAuthService): msg = "Invalid token format" raise AuthInvalidTokenError(msg) - # Use internal authentication method - return await self._authenticate_with_token(resolved_token, db) + # No native token: try a separately-extracted external credential before + # rejecting so a valid external credential authenticates on its own. When + # external_token is None (the default), behavior is unchanged: a missing + # native token raises MissingCredentialsError. + if not resolved_token: + if external_token: + external_user = await self._authenticate_with_external_token(external_token, db) + if external_user is not None: + return external_user + msg = "Missing authentication token" + raise MissingCredentialsError(msg) + + # Use internal authentication method. Try the native token first; on + # failure fall back to a *distinct* external credential before surfacing + # the native error so a stale/invalid native token can't shadow a valid + # external one. When external_token is None or identical, behavior is + # unchanged. + try: + return await self._authenticate_with_token(resolved_token, db) + except (AuthInvalidTokenError, TokenExpiredError, InactiveUserError, InvalidCredentialsError) as e: + if external_token and external_token != resolved_token: + external_user = await self._authenticate_with_external_token(external_token, db) + if external_user is not None: + return external_user + raise e # noqa: TRY201 async def get_current_user_for_websocket( self, token: str | None, api_key: str | None, db: AsyncSession, + external_token: str | None = None, ) -> User | UserRead: """Delegates to authenticate_with_credentials().""" - return await self.authenticate_with_credentials(token, api_key, db) + return await self.authenticate_with_credentials(token, api_key, db, external_token=external_token) async def get_current_user_for_sse( self, token: str | None, api_key: str | None, db: AsyncSession, + external_token: str | None = None, ) -> User | UserRead: """Delegates to authenticate_with_credentials().""" - return await self.authenticate_with_credentials(token, api_key, db) + return await self.authenticate_with_credentials(token, api_key, db, external_token=external_token) async def get_current_active_user(self, current_user: User | UserRead) -> User | UserRead | None: if not current_user.is_active: @@ -441,6 +692,8 @@ class AuthService(BaseAuthService): async def get_webhook_user(self, flow_id: str, request: Request) -> UserRead: settings_service = self.settings + clear_current_auth_context() + clear_current_external_access_context() if not settings_service.auth_settings.WEBHOOK_AUTH_ENABLE: try: @@ -463,12 +716,13 @@ class AuthService(BaseAuthService): try: async with session_scope() as db: - result = await check_key(db, api_key) + result = await authenticate_api_key(db, api_key) if not result: logger.warning("Invalid API key provided for webhook") raise HTTPException(status_code=403, detail="Invalid API key") - authenticated_user = UserRead.model_validate(result, from_attributes=True) + set_current_auth_context(AuthCredentialContext.from_api_key_result(result)) + authenticated_user = UserRead.model_validate(result.user, from_attributes=True) logger.info("Webhook API key validated successfully") except HTTPException: raise @@ -776,11 +1030,14 @@ class AuthService(BaseAuthService): header_param: str | None, db: AsyncSession, ) -> User | UserRead: + clear_current_auth_context() + clear_current_external_access_context() if token: return await self.get_current_user_from_access_token(token, db) settings_service = self.settings - result: ApiKey | User | None + result: User | None + api_key_result = None if settings_service.auth_settings.AUTO_LOGIN: if not settings_service.auth_settings.SUPERUSER: @@ -792,13 +1049,15 @@ class AuthService(BaseAuthService): result = await get_user_by_username(db, settings_service.auth_settings.SUPERUSER) if result: logger.warning(AUTO_LOGIN_WARNING) + set_current_auth_context(AuthCredentialContext(method=AUTH_METHOD_AUTO_LOGIN)) return result else: # At least one of query_param or header_param is truthy api_key = query_param or header_param if api_key is None: # pragma: no cover - guaranteed by the if-condition above raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid or missing API key") - result = await check_key(db, api_key) + api_key_result = await authenticate_api_key(db, api_key) + result = api_key_result.user if api_key_result is not None else None elif not query_param and not header_param: raise HTTPException( @@ -807,13 +1066,15 @@ class AuthService(BaseAuthService): ) elif query_param: - result = await check_key(db, query_param) + api_key_result = await authenticate_api_key(db, query_param) + result = api_key_result.user if api_key_result is not None else None else: # header_param must be truthy here (query_param is falsy, and we passed the not-both-None check) if header_param is None: # pragma: no cover - guaranteed by the elif chain above raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid or missing API key") - result = await check_key(db, header_param) + api_key_result = await authenticate_api_key(db, header_param) + result = api_key_result.user if api_key_result is not None else None if not result: raise HTTPException( @@ -822,6 +1083,8 @@ class AuthService(BaseAuthService): ) if isinstance(result, User): + if api_key_result is not None: + set_current_auth_context(AuthCredentialContext.from_api_key_result(api_key_result)) return result raise HTTPException( diff --git a/src/backend/base/langflow/services/auth/utils.py b/src/backend/base/langflow/services/auth/utils.py index 44ec17f275..b033817dfb 100644 --- a/src/backend/base/langflow/services/auth/utils.py +++ b/src/backend/base/langflow/services/auth/utils.py @@ -17,7 +17,8 @@ from langflow.services.auth.exceptions import ( InvalidCredentialsError, MissingCredentialsError, ) -from langflow.services.deps import get_auth_service +from langflow.services.auth.external import extract_external_token +from langflow.services.deps import get_auth_service, get_settings_service if TYPE_CHECKING: from collections.abc import Coroutine @@ -35,6 +36,8 @@ class OAuth2PasswordBearerCookie(OAuth2PasswordBearer): This allows the application to work with HttpOnly cookies while supporting explicit Authorization headers for backward compatibility and testing scenarios. If an explicit Authorization header is provided, it takes precedence over cookies. + When external trusted auth is enabled, the configured external header/cookie + is consulted last so the native JWT path is always tried first. """ async def __call__(self, request: Request) -> str | None: @@ -49,11 +52,24 @@ class OAuth2PasswordBearerCookie(OAuth2PasswordBearer): if token: return token + # Final fallback: external trusted credential (validated downstream). + if external := _get_external_token(request.headers, request.cookies): + return external + # If auto_error is True, this would raise an exception # Since we set auto_error=False, return None return None +def _get_external_token(headers, cookies) -> str | None: + """Return the configured external credential, swallowing transient failures.""" + try: + auth_settings = get_settings_service().auth_settings + except Exception: # noqa: BLE001 + return None + return extract_external_token(headers, cookies, auth_settings) + + oauth2_login = OAuth2PasswordBearerCookie(tokenUrl="api/v1/login", auto_error=False) API_KEY_NAME = "x-api-key" @@ -155,13 +171,22 @@ def _auth_error_to_http(e: AuthenticationError) -> HTTPException: async def get_current_user( + request: Request, token: Annotated[str | None, Security(oauth2_login)], query_param: Annotated[str | None, Security(api_key_query)], header_param: Annotated[str | None, Security(api_key_header)], db: AsyncSession = Depends(injectable_session_scope), ) -> User: + # Keep the native token (resolved by oauth2_login, which may already have + # collapsed to the external credential) separate from a freshly-extracted + # external credential so a present-but-invalid native cookie cannot shadow a + # valid external one. The auth service tries the native token first and only + # falls back to the external credential when it differs from the token. + external_token = _get_external_token(request.headers, request.cookies) try: - return await _auth_service().get_current_user(token, query_param, header_param, db) + return await _auth_service().get_current_user( + token, query_param, header_param, db, external_token=external_token + ) except AuthenticationError as e: raise _auth_error_to_http(e) from e @@ -169,18 +194,21 @@ async def get_current_user( async def get_current_user_from_access_token( token: str | Coroutine | None, db: AsyncSession, + external_token: str | None = None, ) -> User: """Compatibility helper to resolve a user from an access token. This simply delegates to the active auth service's - `get_current_user_from_access_token` implementation. + `get_current_user_from_access_token` implementation. ``external_token`` is an + optional, separately-extracted external credential tried as a fallback when + native token authentication fails; when ``None`` behavior is unchanged. **For new code, prefer calling `get_auth_service().get_current_user_from_access_token(...)` directly** instead of importing this function. """ try: - return await _auth_service().get_current_user_from_access_token(token, db) + return await _auth_service().get_current_user_from_access_token(token, db, external_token=external_token) except AuthenticationError as e: raise _auth_error_to_http(e) from e @@ -193,7 +221,11 @@ async def get_current_user_for_websocket( db: AsyncSession, ) -> User | UserRead: """Extracts credentials from WebSocket and delegates to auth service.""" + # Keep the native token and the external credential separate so a present but + # invalid/expired native token cannot shadow a valid external credential; the + # auth service tries the external token as a fallback when native auth fails. token = websocket.cookies.get("access_token_lf") or websocket.query_params.get("token") + external_token = _get_external_token(websocket.headers, websocket.cookies) api_key = ( websocket.query_params.get("x-api-key") or websocket.query_params.get("api_key") @@ -202,7 +234,7 @@ async def get_current_user_for_websocket( ) try: - return await _auth_service().get_current_user_for_websocket(token, api_key, db) + return await _auth_service().get_current_user_for_websocket(token, api_key, db, external_token=external_token) except AuthenticationError as e: raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION, reason=WS_AUTH_REASON) from e @@ -215,11 +247,15 @@ async def get_current_user_for_sse( Accepts cookie (access_token_lf) or API key (x-api-key query param). """ + # Keep the native token and the external credential separate (see + # get_current_user_for_websocket) so the external credential remains a usable + # fallback even when a stale native cookie is present. token = request.cookies.get("access_token_lf") + external_token = _get_external_token(request.headers, request.cookies) api_key = request.query_params.get("x-api-key") or request.headers.get("x-api-key") try: - return await _auth_service().get_current_user_for_sse(token, api_key, db) + return await _auth_service().get_current_user_for_sse(token, api_key, db, external_token=external_token) except AuthenticationError as e: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, @@ -285,11 +321,15 @@ async def get_current_user_optional( if auth_header and auth_header.startswith("Bearer "): token = token or auth_header[len("Bearer ") :] - if not token and not api_key: + # Keep the external credential separate so it remains a usable fallback when a + # stale/invalid native token is present (see get_current_user_for_websocket). + external_token = _get_external_token(request.headers, request.cookies) + + if not token and not external_token and not api_key: return None try: - return await _auth_service().get_current_user_for_sse(token, api_key, db) + return await _auth_service().get_current_user_for_sse(token, api_key, db, external_token=external_token) except (AuthenticationError, HTTPException): return None diff --git a/src/backend/base/langflow/services/authorization/access_ceiling.py b/src/backend/base/langflow/services/authorization/access_ceiling.py new file mode 100644 index 0000000000..5622028c8a --- /dev/null +++ b/src/backend/base/langflow/services/authorization/access_ceiling.py @@ -0,0 +1,95 @@ +"""Request-scoped action ceiling (deny-only) enforced by authorization guards. + +This is an authorization primitive: a coarse, deny-only cap on which actions a +request may perform, stored in a context variable for the lifetime of the +request/task. The authentication layer derives the ceiling from a trusted +external identity (see +``langflow.services.auth.external.access_context_from_identity``) and installs +it via :func:`set_current_external_access_context`; the guards in this package +consult it. Keeping the primitive here lets authorization enforce the ceiling +without importing the authentication layer, preserving the auth/authz split +documented in AGENTS.md. +""" + +from __future__ import annotations + +from contextvars import ContextVar +from dataclasses import dataclass + +EXTERNAL_ACCESS_VIEWER = "viewer" +EXTERNAL_ACCESS_EDITOR = "editor" +EXTERNAL_ACCESS_ADMIN = "admin" +EXTERNAL_ACCESS_LEVELS = frozenset( + { + EXTERNAL_ACCESS_VIEWER, + EXTERNAL_ACCESS_EDITOR, + EXTERNAL_ACCESS_ADMIN, + } +) +# Action ceilings per external access level (deny-only caps): +# viewer -> {read} +# editor -> viewer + {write, create, delete, execute, ingest} +# admin -> all actions (no ceiling) +# ``deploy`` is intentionally admin-only and is therefore excluded from the +# editor set: an editor cannot promote a flow to a deployment. +_VIEWER_ALLOWED_ACTIONS = frozenset({"read"}) +_EDITOR_ALLOWED_ACTIONS = frozenset({"read", "write", "create", "delete", "execute", "ingest"}) + + +@dataclass(frozen=True) +class ExternalAccessContext: + """Request-local access ceiling derived from an external identity claim.""" + + provider: str + subject: str + level: str + claim_name: str | None = None + claim_value: str | None = None + + +_current_external_access: ContextVar[ExternalAccessContext | None] = ContextVar( + "langflow_external_access", + default=None, +) + + +def set_current_external_access_context(context: ExternalAccessContext | None) -> None: + """Store the external access ceiling for the current request/task.""" + _current_external_access.set(context) + + +def clear_current_external_access_context() -> None: + """Clear any external access ceiling from the current request/task.""" + _current_external_access.set(None) + + +def get_current_external_access_context() -> ExternalAccessContext | None: + """Return the external access ceiling for the current request/task, if any.""" + return _current_external_access.get() + + +def external_access_allows(action: str, context: ExternalAccessContext | None = None) -> bool: + """Return whether the external access ceiling allows this action. + + This is deliberately action-level and deny-only. It does not grant access to + resources; normal ownership, route guards, and enterprise authz plugins still + decide whether an otherwise allowed action may proceed. + """ + context = context if context is not None else get_current_external_access_context() + if context is None: + return True + + normalized_action = action.strip().lower() + if context.level == EXTERNAL_ACCESS_ADMIN: + return True + if context.level == EXTERNAL_ACCESS_EDITOR: + return normalized_action in _EDITOR_ALLOWED_ACTIONS + return normalized_action in _VIEWER_ALLOWED_ACTIONS + + +def filter_actions_by_external_access_ceiling(actions: list[str] | tuple[str, ...]) -> list[str]: + """Filter an action list through the current external access ceiling.""" + context = get_current_external_access_context() + if context is None: + return list(actions) + return [action for action in actions if external_access_allows(action, context)] diff --git a/src/backend/base/langflow/services/authorization/guards.py b/src/backend/base/langflow/services/authorization/guards.py index eec1e07630..52afcabe19 100644 --- a/src/backend/base/langflow/services/authorization/guards.py +++ b/src/backend/base/langflow/services/authorization/guards.py @@ -8,7 +8,16 @@ from typing import TYPE_CHECKING, Any from fastapi import HTTPException, status from lfx.log.logger import logger +from langflow.services.auth.context import ( + current_auth_context_for_audit, + current_auth_context_for_authz, + current_auth_is_api_key, +) from langflow.services.authorization import audit as _audit +from langflow.services.authorization.access_ceiling import ( + external_access_allows, + get_current_external_access_context, +) from langflow.services.authorization.actions import ( DeploymentAction, FileAction, @@ -56,7 +65,35 @@ _DEFAULT_DENY_DETAIL = "Permission denied" def _auth_context(user: User | UserRead) -> dict[str, Any]: """Build the base context dict passed to authorization enforce calls.""" - return {"is_superuser": getattr(user, "is_superuser", False)} + return { + **current_auth_context_for_authz(), + "is_superuser": getattr(user, "is_superuser", False), + } + + +def _auth_audit_details() -> dict[str, str]: + """Build JSON-friendly auth context for audit details.""" + return current_auth_context_for_audit() + + +async def _api_key_scopes_require_plugin_enforcement() -> bool: + """Return True when owner override must not hide API-key caveats.""" + if not current_auth_is_api_key(): + return False + + settings = get_settings_service() + if not settings.auth_settings.AUTHZ_ENABLED: + return False + + authz = get_authorization_service() + supports_api_key_scopes = getattr(authz, "supports_api_key_scopes", None) + if supports_api_key_scopes is None: + return False + try: + return bool(await supports_api_key_scopes()) + except Exception: # noqa: BLE001 + logger.exception("Authorization plugin failed API-key scope capability check; preserving owner override") + return False def _coerce_action( @@ -114,7 +151,7 @@ async def ensure_permission( merged_context = {**(context or {}), **_auth_context(user)} # Fail closed when enforce() raises (deny + audit, not HTTP 500). audit_action = f"{obj.split(':', 1)[0]}:{act}" if ":" in obj else act - audit_details: dict[str, Any] = {"domain": domain} + audit_details: dict[str, Any] = {"domain": domain, **_auth_audit_details()} for owner_key in _OWNER_CONTEXT_KEYS: if owner_key in merged_context and merged_context[owner_key] is not None: audit_details[owner_key] = str(merged_context[owner_key]) @@ -169,13 +206,35 @@ async def _ensure_resource_permission( """Build object key, apply owner override, else delegate to ensure_permission.""" obj = f"{resource_type}:{resource_id}" if resource_id else f"{resource_type}:*" - if owner_id is not None and getattr(user, "id", None) == owner_id: + external_context = get_current_external_access_context() + if external_context is not None and not external_access_allows(act_str, external_context): + await _audit.audit_decision( + user_id=user.id, + action=f"{resource_type}:{act_str}", + obj=obj, + result=_audit.AUDIT_DENY, + details={ + "domain": resolved_domain, + "external_auth_provider": external_context.provider, + "external_access_level": external_context.level, + }, + ) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="External credentials do not allow this action", + ) + + if ( + owner_id is not None + and getattr(user, "id", None) == owner_id + and not await _api_key_scopes_require_plugin_enforcement() + ): await _audit.audit_decision( user_id=user.id, action=f"{resource_type}:{act_str}", obj=obj, result=_audit.AUDIT_OWNER_OVERRIDE, - details={"domain": resolved_domain}, + details={"domain": resolved_domain, **_auth_audit_details()}, ) return diff --git a/src/backend/base/langflow/services/authorization/listing.py b/src/backend/base/langflow/services/authorization/listing.py index f9945a7c33..5ef737e607 100644 --- a/src/backend/base/langflow/services/authorization/listing.py +++ b/src/backend/base/langflow/services/authorization/listing.py @@ -5,7 +5,11 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, TypeVar from langflow.services.authorization.actions import FlowAction -from langflow.services.authorization.guards import _auth_context, _coerce_action +from langflow.services.authorization.guards import ( + _api_key_scopes_require_plugin_enforcement, + _auth_context, + _coerce_action, +) from langflow.services.deps import get_authorization_service, get_settings_service if TYPE_CHECKING: @@ -43,13 +47,19 @@ async def filter_visible_resources( authz = get_authorization_service() act_str = _coerce_action(act) user_id = getattr(user, "id", None) + owner_override_enabled = not await _api_key_scopes_require_plugin_enforcement() # Owned rows skip batch_enforce (matches direct-read owner override). owned_indices: set[int] = set() enforce_indices: list[int] = [] enforce_items: list[T] = [] for index, item in enumerate(candidates): - if owner_extractor is not None and user_id is not None and owner_extractor(item) == user_id: + if ( + owner_override_enabled + and owner_extractor is not None + and user_id is not None + and owner_extractor(item) == user_id + ): owned_indices.add(index) else: enforce_indices.append(index) diff --git a/src/backend/base/langflow/services/database/models/api_key/crud.py b/src/backend/base/langflow/services/database/models/api_key/crud.py index 078b36a9c2..f3d25abbab 100644 --- a/src/backend/base/langflow/services/database/models/api_key/crud.py +++ b/src/backend/base/langflow/services/database/models/api_key/crud.py @@ -3,6 +3,7 @@ import datetime import hashlib import os import secrets +from dataclasses import dataclass from typing import TYPE_CHECKING from uuid import UUID @@ -20,6 +21,15 @@ if TYPE_CHECKING: from sqlmodel.sql.expression import SelectOfScalar +@dataclass(frozen=True) +class ApiKeyAuthResult: + """Authenticated API-key metadata.""" + + user: User + api_key_source: str + api_key_id: UUID | None = None + + def hash_api_key(api_key: str) -> str: """One-way SHA-256 hash of a plaintext API key for indexed lookup.""" return hashlib.sha256(api_key.encode()).hexdigest() @@ -64,6 +74,18 @@ async def get_api_keys(session: AsyncSession, user_id: UUID) -> list[ApiKeyRead] async def create_api_key(session: AsyncSession, api_key_create: ApiKeyCreate, user_id: UUID) -> UnmaskedApiKeyRead: """Create a new API key, storing an encrypted value and a SHA-256 hash for fast lookup.""" + settings_service = get_settings_service() + auth_settings = settings_service.auth_settings + if ( + auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED + and auth_settings.EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS + ): + from langflow.services.authorization.access_ceiling import get_current_external_access_context + + if get_current_external_access_context() is not None: + msg = "API key creation is disabled for externally authenticated users" + raise PermissionError(msg) + # Generate a random API key with 32 bytes of randomness generated_api_key = f"sk-{secrets.token_urlsafe(32)}" @@ -117,7 +139,56 @@ async def check_key(session: AsyncSession, api_key: str) -> User | None: return await _check_key_from_db(session, api_key, settings_service) +async def authenticate_api_key(session: AsyncSession, api_key: str) -> ApiKeyAuthResult | None: + """Validate an API key and return the user plus non-secret key metadata.""" + settings_service = get_settings_service() + api_key_source = settings_service.auth_settings.API_KEY_SOURCE + + if api_key_source == "env": + user = await _check_key_from_env(session, api_key, settings_service) + if user is not None: + return ApiKeyAuthResult(user=user, api_key_source="env") + # Fallback to database if env validation fails + return await _check_key_from_db_with_context(session, api_key, settings_service) + + +async def _external_access_ceiling_blocks_user(session: AsyncSession, user: User, settings_service) -> bool: + """Return True when API-key auth must be denied for an externally-managed user. + + Single chokepoint for the EXTERNAL_AUTH access ceiling: every DB-path API-key + authentication flows through ``_check_key_from_db_with_context``, so enforcing + here covers ``_api_key_security_impl`` (/run, v2 workflow, openai_responses), + ``ws_api_key_security``, ``get_webhook_user``, ``get_current_user_mcp`` and the + auth service. When the feature is OFF (ceiling disabled OR disable-keys + disabled) this is a no-op. + """ + auth_settings = settings_service.auth_settings + if ( + not auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED + or not auth_settings.EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS + ): + return False + + from langflow.services.database.models.auth import SSOUserProfile + + profile_stmt = select(SSOUserProfile).where( + SSOUserProfile.user_id == user.id, + SSOUserProfile.sso_provider == auth_settings.EXTERNAL_AUTH_PROVIDER, + ) + return (await session.exec(profile_stmt)).first() is not None + + async def _check_key_from_db(session: AsyncSession, api_key: str, settings_service) -> User | None: + """Validate API key against the database and return only the user.""" + result = await _check_key_from_db_with_context(session, api_key, settings_service) + return result.user if result is not None else None + + +async def _check_key_from_db_with_context( + session: AsyncSession, + api_key: str, + settings_service, +) -> ApiKeyAuthResult | None: """Validate API key against the database. Uses hash-based O(1) lookup first. Falls back to decrypt-and-compare @@ -136,12 +207,21 @@ async def _check_key_from_db(session: AsyncSession, api_key: str, settings_servi api_key_obj = matches[0] if _is_expired(api_key_obj.expires_at): return None + # Resolve + authorize the user BEFORE mutating usage counters so a denied + # authentication (missing user or blocked external user) does not bump + # total_uses / last_used_at. + user = await session.get(User, api_key_obj.user_id) + if user is None: + return None + if await _external_access_ceiling_blocks_user(session, user, settings_service): + logger.info("API key rejected for externally managed user while external access ceiling is enabled") + return None if settings_service.settings.disable_track_apikey_usage is not True: api_key_obj.total_uses += 1 api_key_obj.last_used_at = datetime.datetime.now(datetime.timezone.utc) session.add(api_key_obj) await session.flush() - return await session.get(User, api_key_obj.user_id) + return ApiKeyAuthResult(user=user, api_key_source="db", api_key_id=api_key_obj.id) # pragma: allowlist secret if len(matches) > 1: key_ids = [str(m.id) for m in matches] @@ -172,6 +252,15 @@ async def _check_key_from_db(session: AsyncSession, api_key: str, settings_servi if matched: if _is_expired(api_key_obj.expires_at): return None + # Resolve + authorize the user BEFORE mutating usage counters / hash + # backfill so a denied authentication (missing user or blocked + # external user) does not bump total_uses / last_used_at. + user = await session.get(User, api_key_obj.user_id) + if user is None: + return None + if await _external_access_ceiling_blocks_user(session, user, settings_service): + logger.info("API key rejected for externally managed user while external access ceiling is enabled") + return None # Backfill hash for future O(1) lookups api_key_obj.api_key_hash = incoming_hash if settings_service.settings.disable_track_apikey_usage is not True: @@ -179,7 +268,11 @@ async def _check_key_from_db(session: AsyncSession, api_key: str, settings_servi api_key_obj.last_used_at = datetime.datetime.now(datetime.timezone.utc) session.add(api_key_obj) await session.flush() - return await session.get(User, api_key_obj.user_id) + return ApiKeyAuthResult( + user=user, + api_key_source="db", # pragma: allowlist secret + api_key_id=api_key_obj.id, + ) return None diff --git a/src/backend/base/langflow/services/job_queue/service.py b/src/backend/base/langflow/services/job_queue/service.py index a474f953f9..90b0b4d9d4 100644 --- a/src/backend/base/langflow/services/job_queue/service.py +++ b/src/backend/base/langflow/services/job_queue/service.py @@ -29,6 +29,10 @@ _CANCEL_CHANNEL_PREFIX = "langflow:cancel:" # Activity heartbeat key written by polling and streaming responses. The # polling watchdog scans these to detect abandoned builds (client gave up). _ACTIVITY_PREFIX = "langflow:activity:" +# Presence key for jobs started through the public (unauthenticated) build +# endpoint. Allows the public events/cancel endpoints to reject job_ids that +# belong to private-flow builds. Uses the same TTL as the stream/owner keys. +_PUBLIC_JOB_PREFIX = "langflow:public_job:" class JobQueueNotFoundError(Exception): @@ -133,6 +137,7 @@ class JobQueueService(Service): """ self._queues: dict[str, tuple[asyncio.Queue, EventManager, asyncio.Task | None, float | None]] = {} self._job_owners: dict[str, UUID] = {} + self._public_jobs: set[str] = set() self._cleanup_task: asyncio.Task | None = None self._closed = False self.ready = False @@ -352,6 +357,32 @@ class JobQueueService(Service): """Return the user ID that owns a job, or None if not tracked.""" return self._job_owners.get(job_id) + async def register_public_job(self, job_id: str) -> None: + """Mark a job as started through the public (unauthenticated) build endpoint. + + Only jobs registered here may be accessed via the public events/cancel endpoints. + This prevents unauthenticated callers from reading or cancelling private-flow jobs. + + Async (even though the base implementation is a synchronous set add): + RedisJobQueueService overrides this to also persist the marker to Redis + *before returning*, so a request landing on a different worker immediately + after registration sees the marker via is_public_job_async. + """ + self._public_jobs.add(job_id) + + def is_public_job(self, job_id: str) -> bool: + """Return True if the job was started through the public build endpoint.""" + return job_id in self._public_jobs + + async def is_public_job_async(self, job_id: str) -> bool: + """Return True if the job was started through the public build endpoint. + + Base implementation is synchronous (in-memory set lookup). + RedisJobQueueService overrides this to also check Redis for cross-worker + correctness in multi-worker deployments. + """ + return self.is_public_job(job_id) + async def cleanup_job(self, job_id: str) -> None: """Clean up and release resources for a specific job. @@ -407,6 +438,7 @@ class JobQueueService(Service): # Remove the job entry from the registry self._queues.pop(job_id, None) self._job_owners.pop(job_id, None) + self._public_jobs.discard(job_id) await logger.adebug(f"Cleanup successful for job_id {job_id}: resources have been released.") async def cancel_job(self, job_id: str) -> None: @@ -776,6 +808,7 @@ class RedisJobQueueService(JobQueueService): OWNER_PREFIX = _OWNER_PREFIX CANCEL_CHANNEL_PREFIX = _CANCEL_CHANNEL_PREFIX ACTIVITY_PREFIX = _ACTIVITY_PREFIX + PUBLIC_JOB_PREFIX = _PUBLIC_JOB_PREFIX def __init__( self, @@ -853,6 +886,9 @@ class RedisJobQueueService(JobQueueService): def _owner_key(self, job_id: str) -> str: return f"{self.OWNER_PREFIX}{job_id}" + def _public_job_key(self, job_id: str) -> str: + return f"{self.PUBLIC_JOB_PREFIX}{job_id}" + def _cancel_channel(self, job_id: str) -> str: return f"{self.CANCEL_CHANNEL_PREFIX}{job_id}" @@ -1077,6 +1113,16 @@ class RedisJobQueueService(JobQueueService): published = True if needs_ttl_refresh: await self._client.expire(stream_key, self._ttl) + # Why: register_public_job sets the public_job marker with + # ex=self._ttl once at job start. Long-running builds that + # outlive that TTL would have the marker expire while the + # stream itself is kept alive, causing is_public_job_async + # to 404 a still-active public job on other workers. Refresh + # it on the same cadence as the stream TTL. is_public_job is + # the in-memory (sync) check — true on the worker that owns + # this bridge, which is the same worker that registered it. + if self.is_public_job(job_id): + await self._client.expire(self._public_job_key(job_id), self._ttl) last_ttl_refresh = time.monotonic() in_flight_item = None _retry_delay = 0.1 @@ -1662,6 +1708,7 @@ class RedisJobQueueService(JobQueueService): self._stream_key(job_id), self._owner_key(job_id), self._activity_key(job_id), + self._public_job_key(job_id), ) except asyncio.CancelledError: raise @@ -1719,3 +1766,51 @@ class RedisJobQueueService(JobQueueService): raise JobQueueBackendUnavailableError(self._backend_unavailable_message()) from exc raise return None + + async def register_public_job(self, job_id: str) -> None: + """Mark a job as public in both local memory and Redis for cross-worker access. + + Why synchronous (not fire-and-forget): a request for the public events/cancel + endpoint can land on a different worker than the one that registered the job. + If the Redis write were backgrounded, that worker could run is_public_job_async + before the marker exists and incorrectly 404 a legitimate public job. Awaiting + the write here guarantees the marker is visible to every worker by the time + build_public_tmp's response (containing job_id) reaches the client. + + Raises: + JobQueueBackendUnavailableError: if a Redis client is configured but the + marker write fails. Surfacing (instead of swallowing) prevents + build_public_tmp from returning a job_id that only this worker + recognizes — on a multi-worker deployment that would let the public + events/cancel endpoints 404 on every other worker. The caller cleans + up the started job and returns a 503. With no Redis client configured + (single-worker, in-memory) there is no shared marker to persist, so + the base no-op success path is unchanged. + """ + await super().register_public_job(job_id) + if self._client: + await self._set_public_job_key(job_id) + + async def _set_public_job_key(self, job_id: str) -> None: + try: + await self._client.set(self._public_job_key(job_id), b"1", ex=self._ttl) + except Exception as exc: + if _is_backend_connection_error(exc): + raise JobQueueBackendUnavailableError(self._backend_unavailable_message()) from exc + raise + + async def is_public_job_async(self, job_id: str) -> bool: + """Return True if the job was started through the public build endpoint. + + Checks local memory first (fast path), then falls back to Redis so that + a request hitting a different worker than the one that started the build + still works correctly. + """ + if super().is_public_job(job_id): + return True + if self._client: + try: + return bool(await self._client.exists(self._public_job_key(job_id))) + except Exception as exc: # noqa: BLE001 + await logger.awarning(f"Redis public_job check failed for {job_id}: {exc!r}") + return False diff --git a/src/backend/tests/unit/api/v1/test_mcp_utils.py b/src/backend/tests/unit/api/v1/test_mcp_utils.py index 602c46e45b..9dce5a801f 100644 --- a/src/backend/tests/unit/api/v1/test_mcp_utils.py +++ b/src/backend/tests/unit/api/v1/test_mcp_utils.py @@ -391,7 +391,10 @@ def _build_fake_server() -> SimpleNamespace: async def _invoke_handle_call_tool(monkeypatch, arguments: dict) -> AsyncMock: """Run handle_call_tool with all external deps stubbed; return the simple_run_flow mock.""" - flow = SimpleNamespace(id="flow-id-1", name="my_flow", folder_id=None) + # ``user_id`` matches the current user (see ``current_user_ctx`` below) so the + # owner-override path in ``ensure_flow_permission`` is exercised; ``workspace_id`` + # is read by the same guard. + flow = SimpleNamespace(id="flow-id-1", name="my_flow", folder_id=None, user_id="user-1", workspace_id=None) async def fake_get_flow_snake_case(*_args, **_kwargs): return flow diff --git a/src/backend/tests/unit/components/data_source/test_dns_rebinding.py b/src/backend/tests/unit/components/data_source/test_dns_rebinding.py index b20eee0dde..a9b47f92ac 100644 --- a/src/backend/tests/unit/components/data_source/test_dns_rebinding.py +++ b/src/backend/tests/unit/components/data_source/test_dns_rebinding.py @@ -750,6 +750,220 @@ class TestURLComponentDNSRebindingProtection: with pytest.raises(ValueError, match="SSRF Protection"): await component.fetch_url_contents() + @pytest.mark.asyncio + async def test_url_component_follows_redirects_with_per_hop_pinning(self): + """Test that the URL component follows redirects and re-validates/pins every hop. + + Regression test: the component previously sent every request with + follow_redirects=False, so a site that 301s to its canonical URL (http->https or + www normalization - extremely common) returned the redirect stub instead of the + page. Redirects are now followed, and each hop is independently DNS-pinned so a + rebinding attacker who flips the redirect target after validation can never cause + a connection to an unvalidated address. + """ + resolved = [] + + def mock_getaddrinfo(host, *_args, **_kwargs): + """Every host resolves to the same public IP at validation time.""" + resolved.append(host) + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 0))] + + connect_hosts = [] + redirect_response = [ + b"HTTP/1.1 301 Moved Permanently\r\n", + b"Location: https://final.test/\r\n", + b"Content-Length: 0\r\n", + b"\r\n", + ] + final_response = [ + b"HTTP/1.1 200 OK\r\n", + b"Content-Type: text/html\r\n", + b"Content-Length: 39\r\n", + b"\r\n", + b"Final content", + ] + + async def mock_connect_tcp(self, host, port, **kwargs): + """Capture every IP connected to; first hop redirects, second succeeds.""" + connect_hosts.append(host) + stream = redirect_response if len(connect_hosts) == 1 else final_response + return httpcore.AsyncMockStream(list(stream)) + + with ( + patch("socket.getaddrinfo", side_effect=mock_getaddrinfo), + patch.dict(os.environ, {"LANGFLOW_SSRF_PROTECTION_ENABLED": "true"}), + patch.object(httpcore.AnyIOBackend, "connect_tcp", mock_connect_tcp), + ): + component = URLComponent() + component.urls = ["http://redirect.test/"] + component.max_depth = 1 + component.format = "Text" + component.headers = [{"key": "User-Agent", "value": "Test"}] + component.timeout = 30 + component.prevent_outside = True + component.use_async = True + component.follow_redirects = True + component.filter_text_html = False + component.continue_on_failure = False + component.check_response_status = False + component.autoset_encoding = True + + result = await component.fetch_url_contents() + + # The redirect was followed through to the final page's content. + assert len(result) == 1 + assert "Final content" in result[0]["text"] + # Both the initial request and the redirect hop connected to the pinned public IP. + assert connect_hosts == ["8.8.8.8", "8.8.8.8"], connect_hosts + # The redirect target was resolved exactly once (validation only); httpx never + # re-resolved it, so a rebinding flip after validation has no effect. + assert resolved.count("final.test") == 1, resolved + + @pytest.mark.asyncio + async def test_url_component_blocks_redirect_to_internal_ip(self): + """Test that a redirect pointing at an internal address is blocked before connecting. + + A validated public host redirects to a host that resolves to localhost. The hop + is re-validated with the SSRF denylist, so the component raises rather than + fetching the internal resource, and never opens a connection to it. + """ + resolved = [] + + def mock_getaddrinfo(host, *_args, **_kwargs): + """The public redirector resolves to a public IP; the target to localhost.""" + resolved.append(host) + if host == "internal.test": + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))] + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 0))] + + connect_hosts = [] + redirect_response = [ + b"HTTP/1.1 302 Found\r\n", + b"Location: http://internal.test/secret\r\n", + b"Content-Length: 0\r\n", + b"\r\n", + ] + + async def mock_connect_tcp(self, host, port, **kwargs): + """First (and only) connection returns a redirect to the internal host.""" + connect_hosts.append(host) + return httpcore.AsyncMockStream(list(redirect_response)) + + with ( + patch("socket.getaddrinfo", side_effect=mock_getaddrinfo), + patch.dict(os.environ, {"LANGFLOW_SSRF_PROTECTION_ENABLED": "true"}), + patch.object(httpcore.AnyIOBackend, "connect_tcp", mock_connect_tcp), + ): + component = URLComponent() + component.urls = ["http://safe-redirector.test/"] + component.max_depth = 1 + component.format = "Text" + component.headers = [{"key": "User-Agent", "value": "Test"}] + component.timeout = 30 + component.prevent_outside = True + component.use_async = True + component.follow_redirects = True + component.filter_text_html = False + component.continue_on_failure = False + component.check_response_status = False + component.autoset_encoding = True + + # The redirect to a localhost-resolving host must be blocked by SSRF protection. + with pytest.raises(ValueError, match="SSRF Protection"): + await component.fetch_url_contents() + + # Only the first (public) hop ever connected; the internal target was never reached. + assert connect_hosts == ["8.8.8.8"], connect_hosts + assert "internal.test" in resolved, resolved + + @pytest.mark.asyncio + async def test_url_component_crawls_links_from_redirected_base(self): + """Test that depth>1 crawls resolve links against the final post-redirect URL. + + Regression test: after following http://redirect.test/ -> https://final.test/, + relative links were still resolved against the original URL (sending /about to + http://redirect.test/about) and prevent_outside compared link domains against + the pre-redirect host, skipping same-site absolute links. Links must be resolved + against the URL the content actually came from, and the canonical URL itself is + marked visited so links back to it are not re-crawled. + """ + resolved = [] + + def mock_getaddrinfo(host, *_args, **_kwargs): + """Every host resolves to the same public IP at validation time.""" + resolved.append(host) + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 0))] + + def http_ok(body: bytes) -> list[bytes]: + return [ + b"HTTP/1.1 200 OK\r\n", + b"Content-Type: text/html\r\n", + b"Content-Length: " + str(len(body)).encode() + b"\r\n", + b"\r\n", + body, + ] + + responses = [ + [ + b"HTTP/1.1 301 Moved Permanently\r\n", + b"Location: https://final.test/\r\n", + b"Content-Length: 0\r\n", + b"\r\n", + ], + http_ok( + b"Final page" + b'About' + b'Contact' + b'Home' + b"" + ), + http_ok(b"About page"), + http_ok(b"Contact page"), + ] + + connect_hosts = [] + + async def mock_connect_tcp(self, host, port, **kwargs): + """Serve the redirect, the final page, then one page per crawled link.""" + connect_hosts.append(host) + return httpcore.AsyncMockStream(list(responses[len(connect_hosts) - 1])) + + with ( + patch("socket.getaddrinfo", side_effect=mock_getaddrinfo), + patch.dict(os.environ, {"LANGFLOW_SSRF_PROTECTION_ENABLED": "true"}), + patch.object(httpcore.AnyIOBackend, "connect_tcp", mock_connect_tcp), + ): + component = URLComponent() + component.urls = ["http://redirect.test/"] + component.max_depth = 2 + component.format = "Text" + component.headers = [{"key": "User-Agent", "value": "Test"}] + component.timeout = 30 + component.prevent_outside = True + component.use_async = True + component.follow_redirects = True + component.filter_text_html = False + component.continue_on_failure = False + component.check_response_status = False + component.autoset_encoding = True + + result = await component.fetch_url_contents() + + # The final page and both same-site links (relative and absolute) were crawled. + assert [doc["url"] for doc in result] == [ + "https://final.test/", + "https://final.test/about", + "https://final.test/contact", + ] + assert "About page" in result[1]["text"] + # The back-link to the canonical URL was already visited - exactly 4 connections + # (initial + redirect hop + 2 links), all to the pinned public IP. + assert connect_hosts == ["8.8.8.8"] * 4, connect_hosts + # Links validated against the post-redirect host; the pre-redirect host was only + # resolved once, for the initial URL validation. + assert resolved.count("redirect.test") == 1, resolved + assert resolved.count("final.test") == 3, resolved + class TestAPIRequestDNSRebindingEdgeCases: """Additional edge case tests for API Request component DNS rebinding protection.""" diff --git a/src/backend/tests/unit/services/auth/test_auth_service.py b/src/backend/tests/unit/services/auth/test_auth_service.py index 939add8c04..ce98924602 100644 --- a/src/backend/tests/unit/services/auth/test_auth_service.py +++ b/src/backend/tests/unit/services/auth/test_auth_service.py @@ -10,6 +10,11 @@ import jwt import pytest from fastapi import HTTPException, WebSocketException, status from langflow.services.auth.constants import AUTO_LOGIN_WARNING +from langflow.services.auth.context import ( + AUTH_METHOD_API_KEY, + clear_current_auth_context, + get_current_auth_context, +) from langflow.services.auth.exceptions import ( InactiveUserError, InvalidTokenError, @@ -17,6 +22,7 @@ from langflow.services.auth.exceptions import ( TokenExpiredError, ) from langflow.services.auth.service import AuthService +from langflow.services.database.models.api_key.crud import ApiKeyAuthResult from langflow.services.database.models.user.model import User from lfx.services.settings.auth import AuthSettings from pydantic import SecretStr @@ -118,6 +124,38 @@ async def test_authenticate_with_credentials_missing_creds_raises( await auth_service.authenticate_with_credentials(token=None, api_key=None, db=AsyncMock()) +@pytest.mark.anyio +async def test_authenticate_with_api_key_sets_auth_context(auth_service: AuthService): + user = _dummy_user(uuid4()) + api_key_id = uuid4() + + with patch( + "langflow.services.auth.service.authenticate_api_key", + new=AsyncMock( + return_value=ApiKeyAuthResult( + user=user, + api_key_source="db", # pragma: allowlist secret + api_key_id=api_key_id, + ) + ), + ): + try: + result = await auth_service.authenticate_with_credentials( + token=None, + api_key="sk-test-key", # pragma: allowlist secret + db=AsyncMock(), + ) + context = get_current_auth_context() + finally: + clear_current_auth_context() + + assert result.id == user.id + assert context is not None + assert context.method == AUTH_METHOD_API_KEY + assert context.api_key_id == api_key_id + assert context.api_key_source == "db" # pragma: allowlist secret + + @pytest.mark.anyio async def test_authenticate_with_credentials_auto_login_alone_still_rejects( auth_service: AuthService, @@ -781,3 +819,324 @@ async def test_api_key_security_impl_auto_login_skip_rejects_inactive_superuser( ) assert exc.value.status_code == status.HTTP_403_FORBIDDEN + + +# ============================================================================= +# External-user materialization (F1): email is never erased by an email-less token +# ============================================================================= + + +def _external_jwt(auth_service: AuthService, claims: dict) -> str: + """Encode a trusted external JWT signed with the service secret. + + A future ``exp`` is always supplied because the trusted-decode path requires + it (see external._validate_trusted_time_claims). + """ + secret = auth_service.settings.auth_settings.SECRET_KEY.get_secret_value() + payload = {"exp": datetime.now(timezone.utc) + timedelta(minutes=5), **claims} + return jwt.encode(payload, secret, algorithm="HS256") + + +@pytest.mark.anyio +async def test_materialize_external_user_preserves_email_when_token_omits_it( + auth_service: AuthService, + auth_settings: AuthSettings, + async_session, +): + """A later token without an email claim must not erase the stored email (F1).""" + from langflow.services.auth.external import identity_from_claims + from langflow.services.database.models.auth import SSOUserProfile + from sqlmodel import select + + auth_settings.EXTERNAL_AUTH_ENABLED = True + auth_settings.EXTERNAL_AUTH_PROVIDER = "external" + + # First login carries an email and provisions the user + profile. + identity_with_email = identity_from_claims( + {"sub": "ext-subject-1", "email": "alice@example.com", "preferred_username": "alice"}, + auth_settings, + ) + user = await auth_service._materialize_external_user(identity_with_email, async_session) + await async_session.flush() + + profile = (await async_session.exec(select(SSOUserProfile).where(SSOUserProfile.user_id == user.id))).first() + assert profile is not None + assert profile.email == "alice@example.com" + + # Second login with the SAME subject but NO email claim must keep the stored email. + identity_without_email = identity_from_claims( + {"sub": "ext-subject-1", "preferred_username": "alice"}, + auth_settings, + ) + assert identity_without_email.email is None + + same_user = await auth_service._materialize_external_user(identity_without_email, async_session) + await async_session.flush() + await async_session.refresh(profile) + + assert same_user.id == user.id + assert profile.email == "alice@example.com" + + # A later token that DOES carry an email still updates it. + identity_new_email = identity_from_claims( + {"sub": "ext-subject-1", "email": "alice2@example.com"}, + auth_settings, + ) + await auth_service._materialize_external_user(identity_new_email, async_session) + await async_session.flush() + await async_session.refresh(profile) + assert profile.email == "alice2@example.com" + + +# ============================================================================= +# External fallback (F2/F14): a valid external credential is tried when native fails +# ============================================================================= + + +@pytest.mark.anyio +async def test_invalid_native_token_falls_back_to_external_credential( + auth_service: AuthService, + auth_settings: AuthSettings, + async_session, +): + """An invalid native token plus a valid external credential authenticates via external (F2/F14).""" + from langflow.services.auth.external import identity_from_claims + + auth_settings.EXTERNAL_AUTH_ENABLED = True + auth_settings.EXTERNAL_AUTH_TRUSTED_JWT_DECODE = True + auth_settings.EXTERNAL_AUTH_PROVIDER = "external" + + # Pre-provision the external user + profile so the resolver takes the + # existing-profile branch (no folder/variable service needed). + identity = identity_from_claims( + {"sub": "ext-subject-2", "email": "bob@example.com", "preferred_username": "bob"}, + auth_settings, + ) + user = await auth_service._materialize_external_user(identity, async_session) + await async_session.flush() + + # A present-but-invalid native token must NOT shadow the valid external one. + external_token = _external_jwt( + auth_service, {"sub": "ext-subject-2", "email": "bob@example.com", "preferred_username": "bob"} + ) + + try: + result = await auth_service.authenticate_with_credentials( + token="not-a-valid-jwt", # noqa: S106 # native decode fails + api_key=None, + db=async_session, + external_token=external_token, + ) + finally: + clear_current_auth_context() + + assert result.id == user.id + + +@pytest.mark.anyio +async def test_no_external_token_keeps_native_error( + auth_service: AuthService, + auth_settings: AuthSettings, +): + """With external_token=None, an invalid native token still raises (no behavior change).""" + auth_settings.EXTERNAL_AUTH_ENABLED = True + auth_settings.EXTERNAL_AUTH_TRUSTED_JWT_DECODE = True + + with pytest.raises(InvalidTokenError): + await auth_service.authenticate_with_credentials( + token="not-a-valid-jwt", # noqa: S106 + api_key=None, + db=AsyncMock(), + external_token=None, + ) + + +@pytest.mark.anyio +async def test_external_token_only_authenticates_without_native_token( + auth_service: AuthService, + auth_settings: AuthSettings, + async_session, +): + """When no native token is present, the separately-extracted external token still works.""" + from langflow.services.auth.external import identity_from_claims + + auth_settings.EXTERNAL_AUTH_ENABLED = True + auth_settings.EXTERNAL_AUTH_TRUSTED_JWT_DECODE = True + auth_settings.EXTERNAL_AUTH_PROVIDER = "external" + + identity = identity_from_claims( + {"sub": "ext-subject-3", "preferred_username": "carol"}, + auth_settings, + ) + user = await auth_service._materialize_external_user(identity, async_session) + await async_session.flush() + + external_token = _external_jwt(auth_service, {"sub": "ext-subject-3", "preferred_username": "carol"}) + + try: + result = await auth_service.authenticate_with_credentials( + token=None, + api_key=None, + db=async_session, + external_token=external_token, + ) + finally: + clear_current_auth_context() + + assert result.id == user.id + + +# ============================================================================= +# P1: regular HTTP + /session external-credential shadowing +# get_current_user_from_access_token must fall back to a distinct external token +# when the native token is stale/invalid, and accept an external-only credential. +# ============================================================================= + + +@pytest.mark.anyio +async def test_access_token_path_falls_back_to_external_on_invalid_native( + auth_service: AuthService, + auth_settings: AuthSettings, + async_session, +): + """A stale/invalid native token plus a valid external credential recovers (P1).""" + from langflow.services.auth.external import identity_from_claims + + auth_settings.EXTERNAL_AUTH_ENABLED = True + auth_settings.EXTERNAL_AUTH_TRUSTED_JWT_DECODE = True + auth_settings.EXTERNAL_AUTH_PROVIDER = "external" + + identity = identity_from_claims( + {"sub": "ext-p1-1", "preferred_username": "dave"}, + auth_settings, + ) + user = await auth_service._materialize_external_user(identity, async_session) + await async_session.flush() + + external_token = _external_jwt(auth_service, {"sub": "ext-p1-1", "preferred_username": "dave"}) + + try: + result = await auth_service.get_current_user_from_access_token( + "not-a-valid-jwt", # native decode fails + async_session, + external_token=external_token, + ) + finally: + clear_current_auth_context() + + assert result.id == user.id + + +@pytest.mark.anyio +async def test_access_token_path_external_only_authenticates( + auth_service: AuthService, + auth_settings: AuthSettings, + async_session, +): + """No native token but a valid external credential authenticates via /session path (P1).""" + from langflow.services.auth.external import identity_from_claims + + auth_settings.EXTERNAL_AUTH_ENABLED = True + auth_settings.EXTERNAL_AUTH_TRUSTED_JWT_DECODE = True + auth_settings.EXTERNAL_AUTH_PROVIDER = "external" + + identity = identity_from_claims( + {"sub": "ext-p1-2", "preferred_username": "erin"}, + auth_settings, + ) + user = await auth_service._materialize_external_user(identity, async_session) + await async_session.flush() + + external_token = _external_jwt(auth_service, {"sub": "ext-p1-2", "preferred_username": "erin"}) + + try: + result = await auth_service.get_current_user_from_access_token( + None, + async_session, + external_token=external_token, + ) + finally: + clear_current_auth_context() + + assert result.id == user.id + + +@pytest.mark.anyio +async def test_access_token_path_no_external_keeps_native_error( + auth_service: AuthService, + auth_settings: AuthSettings, +): + """With external_token=None, an invalid native token still raises (no behavior change).""" + auth_settings.EXTERNAL_AUTH_ENABLED = True + auth_settings.EXTERNAL_AUTH_TRUSTED_JWT_DECODE = True + + with pytest.raises(InvalidTokenError): + await auth_service.get_current_user_from_access_token( + "not-a-valid-jwt", + AsyncMock(), + external_token=None, + ) + + +@pytest.mark.anyio +async def test_access_token_path_missing_native_token_still_raises( + auth_service: AuthService, +): + """A None native token with no external credential still raises MissingCredentialsError.""" + with pytest.raises(MissingCredentialsError): + await auth_service.get_current_user_from_access_token(None, AsyncMock()) + + +# ============================================================================= +# P2: the external-access ceiling ContextVar is cleared at every auth entrypoint. +# A stale ceiling left over from a prior same-task external auth must not leak +# into a subsequent non-external API-key auth path. +# ============================================================================= + + +@pytest.mark.anyio +async def test_api_key_entrypoint_clears_stale_external_access_ceiling( + auth_service: AuthService, + auth_settings: AuthSettings, +): + """_api_key_security_impl must clear a stale external-access ceiling ContextVar.""" + from langflow.services.auth.external import ( + ExternalAccessContext, + get_current_external_access_context, + set_current_external_access_context, + ) + + auth_settings.AUTO_LOGIN = False + user = _dummy_user(uuid4()) + + # Simulate a stale ceiling left in this task by a prior external auth. + set_current_external_access_context( + ExternalAccessContext(provider="external", subject="stale-subject", level="viewer") + ) + assert get_current_external_access_context() is not None + + try: + with patch( + "langflow.services.auth.service.authenticate_api_key", + new=AsyncMock( + return_value=ApiKeyAuthResult( + user=user, + api_key_source="db", # pragma: allowlist secret + api_key_id=uuid4(), + ) + ), + ): + result = await auth_service._api_key_security_impl( + query_param="sk-test-key", # pragma: allowlist secret + header_param=None, + db=AsyncMock(), + settings_service=auth_service.settings, + ) + + # The stale ceiling must have been cleared by the entrypoint. + assert get_current_external_access_context() is None + finally: + clear_current_auth_context() + set_current_external_access_context(None) + + assert result.id == user.id diff --git a/src/backend/tests/unit/services/auth/test_external_auth.py b/src/backend/tests/unit/services/auth/test_external_auth.py new file mode 100644 index 0000000000..5afcf7a354 --- /dev/null +++ b/src/backend/tests/unit/services/auth/test_external_auth.py @@ -0,0 +1,652 @@ +"""Unit tests for langflow.services.auth.external.""" + +from __future__ import annotations + +import base64 +import time +from datetime import datetime, timedelta, timezone + +import jwt +import pytest +from langflow.services.auth import external +from langflow.services.auth.exceptions import InvalidTokenError +from langflow.services.auth.external import ( + access_context_from_identity, + decode_external_jwt, + external_access_allows, + extract_bearer_or_raw_token, + extract_external_token, + filter_actions_by_external_access_ceiling, + get_current_external_access_context, + identity_from_claims, + resolve_external_identity, + set_current_external_access_context, +) +from lfx.services.settings.auth import AuthSettings +from pydantic import ValidationError + +_TEST_JWT_SECRET = "external-test-secret-with-enough-length" # noqa: S105 # pragma: allowlist secret +_EXTERNAL_AUTH_HEADER = "X-External-Auth" +_EXTERNAL_AUTH_COOKIE = "external-session" +_OPAQUE_CREDENTIAL = "opaque-token" # pragma: allowlist secret +_HEADER_CREDENTIAL = "header-token" # pragma: allowlist secret + + +def _auth_settings(tmp_path, **overrides) -> AuthSettings: + settings = AuthSettings(CONFIG_DIR=str(tmp_path)) + settings.EXTERNAL_AUTH_ENABLED = True + settings.EXTERNAL_AUTH_PROVIDER = "test-provider" + settings.EXTERNAL_AUTH_TRUSTED_JWT_DECODE = True + settings.EXTERNAL_AUTH_TOKEN_HEADER = _EXTERNAL_AUTH_HEADER + settings.EXTERNAL_AUTH_TOKEN_COOKIE = _EXTERNAL_AUTH_COOKIE + for key, value in overrides.items(): + setattr(settings, key, value) + return settings + + +# --------------------------------------------------------------------------- +# extract_external_token / extract_bearer_or_raw_token +# --------------------------------------------------------------------------- + + +def test_extract_external_token_prefers_header_over_cookie(tmp_path): + settings = _auth_settings(tmp_path) + token = extract_external_token( + {_EXTERNAL_AUTH_HEADER: f"Bearer {_HEADER_CREDENTIAL}"}, + {_EXTERNAL_AUTH_COOKIE: "cookie-token"}, + settings, + ) + assert token == _HEADER_CREDENTIAL + + +def test_extract_external_token_returns_none_when_disabled(tmp_path): + settings = _auth_settings(tmp_path, EXTERNAL_AUTH_ENABLED=False) + assert extract_external_token({_EXTERNAL_AUTH_HEADER: "Bearer x"}, {}, settings) is None + + +def test_extract_external_token_falls_back_to_cookie(tmp_path): + settings = _auth_settings(tmp_path) + cookie_value = "cookie-token" # pragma: allowlist secret + token = extract_external_token({}, {_EXTERNAL_AUTH_COOKIE: cookie_value}, settings) + assert token == cookie_value + + +def test_extract_bearer_or_raw_token_handles_variants(): + assert extract_bearer_or_raw_token(None) is None + assert extract_bearer_or_raw_token("") is None + assert extract_bearer_or_raw_token(" ") is None + assert extract_bearer_or_raw_token("Bearer abc") == "abc" + assert extract_bearer_or_raw_token("bearer abc") == "abc" + assert extract_bearer_or_raw_token("opaque") == "opaque" + + +# --------------------------------------------------------------------------- +# decode_external_jwt (trusted-decode path) +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_trusted_jwt_decode_validates_expiry(tmp_path): + settings = _auth_settings(tmp_path) + token = jwt.encode( + {"sub": "expired-subject", "exp": datetime.now(timezone.utc) - timedelta(minutes=1)}, + _TEST_JWT_SECRET, + algorithm="HS256", + ) + with pytest.raises(InvalidTokenError, match="expired"): + await decode_external_jwt(token, settings) + + +@pytest.mark.anyio +async def test_trusted_jwt_decode_validates_not_before(tmp_path): + settings = _auth_settings(tmp_path) + token = jwt.encode( + { + "sub": "future-subject", + "nbf": datetime.now(timezone.utc) + timedelta(minutes=5), + "exp": datetime.now(timezone.utc) + timedelta(minutes=10), + }, + _TEST_JWT_SECRET, + algorithm="HS256", + ) + with pytest.raises(InvalidTokenError, match="not valid yet"): + await decode_external_jwt(token, settings) + + +@pytest.mark.anyio +async def test_trusted_jwt_decode_returns_claims_when_valid(tmp_path): + settings = _auth_settings(tmp_path) + expected_sub = "subject-1" + token = jwt.encode( + { + "sub": expected_sub, + "preferred_username": "alice", + "email": "alice@example.com", + "exp": datetime.now(timezone.utc) + timedelta(minutes=5), + }, + _TEST_JWT_SECRET, + algorithm="HS256", + ) + claims = await decode_external_jwt(token, settings) + assert claims["sub"] == expected_sub + assert claims["preferred_username"] == "alice" + + +@pytest.mark.anyio +async def test_decode_external_jwt_rejects_when_external_auth_disabled(tmp_path): + settings = _auth_settings(tmp_path, EXTERNAL_AUTH_ENABLED=False) + with pytest.raises(InvalidTokenError, match="not enabled"): + await decode_external_jwt("anything", settings) + + +@pytest.mark.anyio +async def test_decode_external_jwt_requires_jwks_when_not_trusted(tmp_path): + settings = _auth_settings( + tmp_path, + EXTERNAL_AUTH_TRUSTED_JWT_DECODE=False, + EXTERNAL_AUTH_JWKS_URL=None, + ) + with pytest.raises(InvalidTokenError, match="EXTERNAL_AUTH_JWKS_URL"): + await decode_external_jwt("anything", settings) + + +# --------------------------------------------------------------------------- +# decode_external_jwt (JWKS path) +# --------------------------------------------------------------------------- + +_JWKS_URL = "https://idp.example.com/.well-known/jwks.json" +_JWKS_AUDIENCE = "langflow" + + +def _jwks_settings(tmp_path) -> AuthSettings: + return _auth_settings( + tmp_path, + EXTERNAL_AUTH_TRUSTED_JWT_DECODE=False, + EXTERNAL_AUTH_JWKS_URL=_JWKS_URL, + EXTERNAL_AUTH_ALGORITHMS="HS256", + EXTERNAL_AUTH_AUDIENCE=_JWKS_AUDIENCE, + ) + + +def _symmetric_jwk(secret: str, kid: str) -> dict: + return { + "kty": "oct", + "alg": "HS256", + "kid": kid, + "k": base64.urlsafe_b64encode(secret.encode()).rstrip(b"=").decode(), + } + + +def _install_fake_jwks_endpoint(monkeypatch, payloads: list[dict]) -> list[str]: + """Serve payloads[i] for the i-th JWKS fetch (last payload repeats); returns the call log.""" + calls: list[str] = [] + + class _FakeResponse: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + class _FakeAsyncClient: + def __init__(self, **_kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def get(self, url): + calls.append(url) + return _FakeResponse(payloads[min(len(calls) - 1, len(payloads) - 1)]) + + monkeypatch.setattr(external.httpx, "AsyncClient", _FakeAsyncClient) + return calls + + +@pytest.mark.anyio +async def test_jwks_decode_verifies_signature(tmp_path, monkeypatch): + settings = _jwks_settings(tmp_path) + monkeypatch.setattr(external, "_jwks_cache", {}) + _install_fake_jwks_endpoint(monkeypatch, [{"keys": [_symmetric_jwk(_TEST_JWT_SECRET, kid="key-1")]}]) + + valid_exp = datetime.now(timezone.utc) + timedelta(minutes=5) + token = jwt.encode( + {"sub": "subject-1", "aud": _JWKS_AUDIENCE, "exp": valid_exp}, + _TEST_JWT_SECRET, + algorithm="HS256", + headers={"kid": "key-1"}, + ) + claims = await decode_external_jwt(token, settings) + assert claims["sub"] == "subject-1" + + wrong_secret = "another-secret-that-did-not-sign-the-jwks-key" # noqa: S105 # pragma: allowlist secret + tampered = jwt.encode( + {"sub": "subject-1", "aud": _JWKS_AUDIENCE, "exp": valid_exp}, + wrong_secret, + algorithm="HS256", + headers={"kid": "key-1"}, + ) + with pytest.raises(InvalidTokenError, match="validation failed"): + await decode_external_jwt(tampered, settings) + + +@pytest.mark.anyio +async def test_jwks_decode_requires_audience(tmp_path, monkeypatch): + """JWKS verification must reject the config when no expected audience is bound.""" + settings = _jwks_settings(tmp_path) + settings.EXTERNAL_AUTH_AUDIENCE = None + monkeypatch.setattr(external, "_jwks_cache", {}) + calls = _install_fake_jwks_endpoint(monkeypatch, [{"keys": [_symmetric_jwk(_TEST_JWT_SECRET, kid="key-1")]}]) + + token = jwt.encode({"sub": "subject-1"}, _TEST_JWT_SECRET, algorithm="HS256", headers={"kid": "key-1"}) + with pytest.raises(InvalidTokenError, match="EXTERNAL_AUTH_AUDIENCE"): + await decode_external_jwt(token, settings) + # Misconfiguration is rejected before any network call to the JWKS endpoint. + assert calls == [] + + +@pytest.mark.anyio +async def test_jwks_decode_rejects_token_for_another_audience(tmp_path, monkeypatch): + """A token the same IdP minted for a different relying party is rejected.""" + settings = _jwks_settings(tmp_path) + monkeypatch.setattr(external, "_jwks_cache", {}) + _install_fake_jwks_endpoint(monkeypatch, [{"keys": [_symmetric_jwk(_TEST_JWT_SECRET, kid="key-1")]}]) + + foreign = jwt.encode( + {"sub": "subject-1", "aud": "some-other-service", "exp": datetime.now(timezone.utc) + timedelta(minutes=5)}, + _TEST_JWT_SECRET, + algorithm="HS256", + headers={"kid": "key-1"}, + ) + with pytest.raises(InvalidTokenError, match="validation failed"): + await decode_external_jwt(foreign, settings) + + +@pytest.mark.anyio +async def test_jwks_refetches_when_kid_is_newer_than_cache(tmp_path, monkeypatch): + """IdP key rotation: an unknown kid forces one cache-busting JWKS refetch.""" + settings = _jwks_settings(tmp_path) + old_jwks = {"keys": [_symmetric_jwk(_TEST_JWT_SECRET, kid="old-key")]} + new_jwks = {"keys": [_symmetric_jwk(_TEST_JWT_SECRET, kid="new-key")]} + # Cached entry is still valid but old enough to be past the refresh rate limit. + fetched_at = time.monotonic() - external.JWKS_MIN_REFRESH_INTERVAL_SECONDS - 1 + monkeypatch.setattr(external, "_jwks_cache", {_JWKS_URL: (fetched_at + external.JWKS_CACHE_TTL_SECONDS, old_jwks)}) + calls = _install_fake_jwks_endpoint(monkeypatch, [new_jwks]) + + token = jwt.encode( + {"sub": "rotated", "aud": _JWKS_AUDIENCE, "exp": datetime.now(timezone.utc) + timedelta(minutes=5)}, + _TEST_JWT_SECRET, + algorithm="HS256", + headers={"kid": "new-key"}, + ) + claims = await decode_external_jwt(token, settings) + + assert claims["sub"] == "rotated" + assert calls == [_JWKS_URL] + + +@pytest.mark.anyio +async def test_jwks_refresh_is_rate_limited_for_unknown_kid(tmp_path, monkeypatch): + """A freshly fetched JWKS is not refetched for an unknown kid (anti-hammering).""" + settings = _jwks_settings(tmp_path) + jwks = {"keys": [_symmetric_jwk(_TEST_JWT_SECRET, kid="key-1")]} + monkeypatch.setattr( + external, "_jwks_cache", {_JWKS_URL: (time.monotonic() + external.JWKS_CACHE_TTL_SECONDS, jwks)} + ) + calls = _install_fake_jwks_endpoint(monkeypatch, [jwks]) + + token = jwt.encode({"sub": "nope"}, _TEST_JWT_SECRET, algorithm="HS256", headers={"kid": "unknown-key"}) + with pytest.raises(InvalidTokenError, match="not found in JWKS"): + await decode_external_jwt(token, settings) + + assert calls == [] + + +# --------------------------------------------------------------------------- +# identity_from_claims / resolve_external_identity +# --------------------------------------------------------------------------- + + +def test_identity_from_claims_uses_configured_mapping(tmp_path): + settings = _auth_settings(tmp_path) + identity = identity_from_claims( + { + "sub": "subject-1", + "preferred_username": "alice", + "email": "alice@example.com", + "name": "Alice", + }, + settings, + ) + assert identity.provider == "test-provider" + assert identity.subject == "subject-1" + assert identity.username == "alice" + assert identity.email == "alice@example.com" + assert identity.name == "Alice" + + +def test_identity_from_claims_requires_subject(tmp_path): + settings = _auth_settings(tmp_path) + with pytest.raises(InvalidTokenError, match="missing required claim"): + identity_from_claims({"preferred_username": "no-sub"}, settings) + + +def test_identity_from_claims_falls_back_to_synthesized_username(tmp_path): + settings = _auth_settings(tmp_path) + identity = identity_from_claims({"sub": "subject-9"}, settings) + assert identity.username.startswith("test-provider-") + assert identity.email is None + assert identity.name is None + + +def test_external_access_context_uses_configured_claim_mapping(tmp_path): + settings = _auth_settings( + tmp_path, + EXTERNAL_AUTH_ACCESS_CEILING_ENABLED=True, + EXTERNAL_AUTH_ACCESS_CLAIM="openrag_mode", + EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING='{"can_view":"viewer","can_edit":"editor"}', + ) + identity = identity_from_claims({"sub": "subject-1", "openrag_mode": "can_edit"}, settings) + + context = access_context_from_identity(identity, settings) + + assert context is not None + assert context.level == "editor" + assert external_access_allows("write", context) + assert external_access_allows("execute", context) + # An editor may delete their own resources; only ``deploy`` stays admin-only. + assert external_access_allows("delete", context) + assert not external_access_allows("deploy", context) + + +def test_external_access_context_defaults_to_viewer_for_missing_claim(tmp_path): + settings = _auth_settings( + tmp_path, + EXTERNAL_AUTH_ACCESS_CEILING_ENABLED=True, + EXTERNAL_AUTH_ACCESS_CLAIM="openrag_mode", + ) + identity = identity_from_claims({"sub": "subject-1"}, settings) + + context = access_context_from_identity(identity, settings) + + assert context is not None + assert context.level == "viewer" + assert external_access_allows("read", context) + assert not external_access_allows("write", context) + + +def test_filter_actions_by_external_access_ceiling_uses_request_context(): + set_current_external_access_context( + external.ExternalAccessContext(provider="test-provider", subject="subject-1", level="viewer") + ) + try: + assert filter_actions_by_external_access_ceiling(["read", "write", "delete"]) == ["read"] + assert get_current_external_access_context() is not None + finally: + set_current_external_access_context(None) + + +@pytest.mark.anyio +async def test_resolve_external_identity_uses_configured_resolver(tmp_path, monkeypatch): + settings = _auth_settings(tmp_path) + settings.EXTERNAL_AUTH_IDENTITY_RESOLVER = "tests.fake:resolver" + + async def resolver(token, auth_settings): + assert token == _OPAQUE_CREDENTIAL + assert auth_settings is settings + return { + "sub": "opaque-subject", + "preferred_username": "opaque-user", + "email": "opaque@example.com", + "name": "Opaque User", + } + + from lfx.services import config_discovery + + def load_resolver(import_path, *, object_kind, object_key): + assert import_path == settings.EXTERNAL_AUTH_IDENTITY_RESOLVER + assert object_kind == "external auth resolver" + assert object_key == "EXTERNAL_AUTH_IDENTITY_RESOLVER" + return resolver + + monkeypatch.setattr(config_discovery, "load_object_from_import_path", load_resolver) + + identity = await resolve_external_identity(_OPAQUE_CREDENTIAL, settings) + + assert identity.provider == "test-provider" + assert identity.subject == "opaque-subject" + assert identity.username == "opaque-user" + assert identity.email == "opaque@example.com" + assert identity.name == "Opaque User" + + +@pytest.mark.anyio +async def test_resolve_external_identity_default_uses_jwt(tmp_path): + settings = _auth_settings(tmp_path) + token = jwt.encode( + { + "sub": "subject-1", + "preferred_username": "bob", + "exp": datetime.now(timezone.utc) + timedelta(minutes=5), + }, + _TEST_JWT_SECRET, + algorithm="HS256", + ) + identity = await resolve_external_identity(token, settings) + assert identity.subject == "subject-1" + assert identity.username == "bob" + + +# --------------------------------------------------------------------------- +# F7: exp is required on both decode paths (a token without exp never expires) +# --------------------------------------------------------------------------- + + +@pytest.mark.anyio +async def test_trusted_jwt_decode_rejects_missing_exp(tmp_path): + """Trusted-decode path must reject a token that omits exp (never expires).""" + settings = _auth_settings(tmp_path) + token = jwt.encode({"sub": "no-exp-subject"}, _TEST_JWT_SECRET, algorithm="HS256") + with pytest.raises(InvalidTokenError, match="missing exp"): + await decode_external_jwt(token, settings) + + +@pytest.mark.anyio +async def test_jwks_decode_rejects_missing_exp(tmp_path, monkeypatch): + """JWKS path must reject a validly-signed token that omits exp.""" + settings = _jwks_settings(tmp_path) + monkeypatch.setattr(external, "_jwks_cache", {}) + _install_fake_jwks_endpoint(monkeypatch, [{"keys": [_symmetric_jwk(_TEST_JWT_SECRET, kid="key-1")]}]) + + # Validly signed, correct audience, but no exp claim. + token = jwt.encode( + {"sub": "subject-1", "aud": _JWKS_AUDIENCE}, + _TEST_JWT_SECRET, + algorithm="HS256", + headers={"kid": "key-1"}, + ) + with pytest.raises(InvalidTokenError, match="validation failed"): + await decode_external_jwt(token, settings) + + +# --------------------------------------------------------------------------- +# F8: EXTERNAL_AUTH_JWKS_URL must be https (http allowed only for loopback) +# --------------------------------------------------------------------------- + + +def test_jwks_url_rejects_http_scheme(tmp_path): + """An http:// JWKS URL is rejected (MITM could swap signing keys).""" + with pytest.raises(ValidationError, match="https"): + AuthSettings( + CONFIG_DIR=str(tmp_path), + EXTERNAL_AUTH_JWKS_URL="http://idp.example.com/.well-known/jwks.json", + ) + + +def test_jwks_url_accepts_https_scheme(tmp_path): + """An https:// JWKS URL is accepted.""" + settings = AuthSettings( + CONFIG_DIR=str(tmp_path), + EXTERNAL_AUTH_JWKS_URL="https://idp.example.com/.well-known/jwks.json", + ) + assert settings.EXTERNAL_AUTH_JWKS_URL == "https://idp.example.com/.well-known/jwks.json" + + +def test_jwks_url_allows_http_for_localhost(tmp_path): + """http:// is permitted for loopback hosts to keep local development usable.""" + for url in ( + "http://localhost:8080/jwks.json", + "http://127.0.0.1:8080/jwks.json", + "http://[::1]:8080/jwks.json", + ): + settings = AuthSettings(CONFIG_DIR=str(tmp_path), EXTERNAL_AUTH_JWKS_URL=url) + assert url == settings.EXTERNAL_AUTH_JWKS_URL + + +def test_jwks_url_none_is_allowed(tmp_path): + """JWKS URL is optional (None / empty) so the validator must not reject it.""" + settings = AuthSettings(CONFIG_DIR=str(tmp_path), EXTERNAL_AUTH_JWKS_URL=None) + assert settings.EXTERNAL_AUTH_JWKS_URL is None + + +@pytest.mark.anyio +async def test_fetch_jwks_guard_rejects_non_https(monkeypatch): + """Belt-and-suspenders: _fetch_jwks itself refuses a non-https URL.""" + monkeypatch.setattr(external, "_jwks_cache", {}) + calls = _install_fake_jwks_endpoint(monkeypatch, [{"keys": []}]) + with pytest.raises(InvalidTokenError, match="https"): + await external._fetch_jwks("http://idp.example.com/jwks.json") + # No network call is made for a rejected scheme. + assert calls == [] + + +# --------------------------------------------------------------------------- +# F11: EXTERNAL_AUTH_PROVIDER normalizes once at the config boundary +# --------------------------------------------------------------------------- + + +def test_external_auth_provider_empty_normalizes_to_external(tmp_path): + settings = AuthSettings(CONFIG_DIR=str(tmp_path), EXTERNAL_AUTH_PROVIDER="") + assert settings.EXTERNAL_AUTH_PROVIDER == "external" + + +def test_external_auth_provider_whitespace_normalizes_to_external(tmp_path): + settings = AuthSettings(CONFIG_DIR=str(tmp_path), EXTERNAL_AUTH_PROVIDER=" ") + assert settings.EXTERNAL_AUTH_PROVIDER == "external" + + +def test_external_auth_provider_strips_whitespace(tmp_path): + settings = AuthSettings(CONFIG_DIR=str(tmp_path), EXTERNAL_AUTH_PROVIDER=" okta ") + assert settings.EXTERNAL_AUTH_PROVIDER == "okta" + + +def test_external_auth_provider_normalizes_on_assignment(tmp_path): + """validate_assignment=True means setattr must normalize too (the test helper path).""" + settings = AuthSettings(CONFIG_DIR=str(tmp_path)) + settings.EXTERNAL_AUTH_PROVIDER = " " + assert settings.EXTERNAL_AUTH_PROVIDER == "external" + + +# --------------------------------------------------------------------------- +# F12: a configured access-claim mapping is authoritative (no alias fallthrough) +# --------------------------------------------------------------------------- + + +def test_access_mapping_unmapped_value_does_not_elevate_via_aliases(tmp_path): + """Unmapped value falls to the default, not the built-in alias table. + + With a non-empty mapping, a claim value absent from it falls to the default + (viewer) instead of being reinterpreted through the built-in alias table. + """ + settings = _auth_settings( + tmp_path, + EXTERNAL_AUTH_ACCESS_CEILING_ENABLED=True, + EXTERNAL_AUTH_ACCESS_CLAIM="role", + EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING='{"can_view":"viewer","can_edit":"editor"}', + ) + # Raw "admin" is an alias for the admin level, but it is NOT a key in the + # configured mapping, so it must NOT elevate. + identity = identity_from_claims({"sub": "subject-1", "role": "admin"}, settings) + + context = access_context_from_identity(identity, settings) + + assert context is not None + assert context.level == "viewer" # default, not admin + assert external_access_allows("read", context) + assert not external_access_allows("write", context) + assert not external_access_allows("delete", context) + + +def test_access_mapping_unmapped_value_uses_configured_default(tmp_path): + """The configured default level is applied for an unmapped claim value. + + The configured default level (not the viewer floor) is applied for an + unmapped claim value when a mapping is present. + """ + settings = _auth_settings( + tmp_path, + EXTERNAL_AUTH_ACCESS_CEILING_ENABLED=True, + EXTERNAL_AUTH_ACCESS_CLAIM="role", + EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING='{"can_edit":"editor"}', + EXTERNAL_AUTH_DEFAULT_ACCESS_LEVEL="editor", + ) + identity = identity_from_claims({"sub": "subject-1", "role": "admin"}, settings) + + context = access_context_from_identity(identity, settings) + + assert context is not None + assert context.level == "editor" + + +def test_no_mapping_still_interprets_raw_claim_via_aliases(tmp_path): + """With no mapping configured, built-in aliases still apply to the raw claim. + + When NO mapping is configured the built-in aliases still apply to the raw + claim value, so a raw 'admin' claim maps to the admin level. + """ + settings = _auth_settings( + tmp_path, + EXTERNAL_AUTH_ACCESS_CEILING_ENABLED=True, + EXTERNAL_AUTH_ACCESS_CLAIM="role", + ) + identity = identity_from_claims({"sub": "subject-1", "role": "admin"}, settings) + + context = access_context_from_identity(identity, settings) + + assert context is not None + assert context.level == "admin" + assert external_access_allows("delete", context) + + +def test_configured_but_all_invalid_mapping_does_not_elevate_via_aliases(tmp_path): + """A configured-but-all-invalid mapping must still suppress alias fallthrough. + + If an operator configures EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING but every entry + is invalid/typo'd, the parsed dict is empty. Gating the alias fallthrough on + the parsed dict (``not mapping``) would treat this as "no mapping" and let a + raw "admin" claim self-elevate through the built-in alias table, re-opening + the hole F12 closes. The gate is on the RAW setting being configured, so an + unmapped claim must fall to the default (viewer floor) instead. + """ + settings = _auth_settings( + tmp_path, + EXTERNAL_AUTH_ACCESS_CEILING_ENABLED=True, + EXTERNAL_AUTH_ACCESS_CLAIM="role", + # Every value is an unrecognized level -> _access_claim_mapping returns {}. + EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING='{"can_view":"bogus","can_edit":"typo"}', + ) + identity = identity_from_claims({"sub": "subject-1", "role": "admin"}, settings) + + context = access_context_from_identity(identity, settings) + + assert context is not None + assert context.level == "viewer" # default/floor, NOT admin + assert external_access_allows("read", context) + assert not external_access_allows("write", context) + assert not external_access_allows("delete", context) diff --git a/src/backend/tests/unit/services/auth/test_pluggable_auth.py b/src/backend/tests/unit/services/auth/test_pluggable_auth.py index 29d9bd7023..68f3ca666d 100644 --- a/src/backend/tests/unit/services/auth/test_pluggable_auth.py +++ b/src/backend/tests/unit/services/auth/test_pluggable_auth.py @@ -24,8 +24,8 @@ class DummyAuthService(Service): self.calls.append(call) return {"call": call} - async def get_current_user(self, token, query_param, header_param, db): - call = ("get_current_user", token, query_param, header_param) + async def get_current_user(self, token, query_param, header_param, db, external_token=None): + call = ("get_current_user", token, query_param, header_param, external_token) self.calls.append(call) return {"user": "dummy", "db": db} @@ -77,8 +77,12 @@ async def test_api_key_security_uses_registered_service(dummy_auth_registration) async def test_get_current_user_delegates_to_service(dummy_auth_registration): dummy = dummy_auth_registration db = MagicMock(spec=AsyncSession) - response = await auth_utils.get_current_user(token=None, query_param="q", header_param=None, db=db) + # get_current_user now extracts the external credential from the request and + # threads it to the service as external_token. With external auth disabled the + # extractor returns None, so delegation records external_token=None. + request = SimpleNamespace(headers={}, cookies={}) + response = await auth_utils.get_current_user(request, token=None, query_param="q", header_param=None, db=db) - assert ("get_current_user", None, "q", None) in dummy.calls + assert ("get_current_user", None, "q", None, None) in dummy.calls assert response["user"] == "dummy" assert response["db"] is db diff --git a/src/backend/tests/unit/services/authorization/_common.py b/src/backend/tests/unit/services/authorization/_common.py index 6953706119..139002f972 100644 --- a/src/backend/tests/unit/services/authorization/_common.py +++ b/src/backend/tests/unit/services/authorization/_common.py @@ -28,9 +28,16 @@ from langflow.services.authorization import listing as authz_listing class _StubAuthorizationService: """Minimal stand-in for BaseAuthorizationService that records calls.""" - def __init__(self, *, allow: bool = True, batch_results: list[bool] | None = None) -> None: + def __init__( + self, + *, + allow: bool = True, + batch_results: list[bool] | None = None, + supports_api_key_scopes: bool = False, + ) -> None: self.allow = allow self.batch_results = batch_results + self._supports_api_key_scopes = supports_api_key_scopes self.calls: list[dict] = [] self.batch_calls: list[dict] = [] @@ -44,6 +51,9 @@ class _StubAuthorizationService: return self.batch_results return [self.allow] * len(kwargs.get("requests", [])) + async def supports_api_key_scopes(self) -> bool: + return self._supports_api_key_scopes + def install_settings(monkeypatch, *, authz_enabled: bool, audit_enabled: bool = False) -> None: """Patch settings on every module that calls ``get_settings_service``.""" diff --git a/src/backend/tests/unit/services/authorization/test_access_ceiling_levels.py b/src/backend/tests/unit/services/authorization/test_access_ceiling_levels.py new file mode 100644 index 0000000000..52d01f892a --- /dev/null +++ b/src/backend/tests/unit/services/authorization/test_access_ceiling_levels.py @@ -0,0 +1,65 @@ +"""Regression tests for the external access-ceiling action vocabulary. + +These guard the deny-only ceiling levels enforced by +``langflow.services.authorization.access_ceiling`` (see PR #13293). The +editor level is expected to permit ``delete`` (a user editing their own +resources should be able to remove them) while ``deploy`` stays admin-only. +""" + +from __future__ import annotations + +import pytest +from langflow.services.authorization.access_ceiling import ( + ExternalAccessContext, + external_access_allows, + filter_actions_by_external_access_ceiling, + set_current_external_access_context, +) + +_ALL_ACTIONS = ("read", "write", "create", "delete", "execute", "ingest", "deploy") + + +def _ctx(level: str) -> ExternalAccessContext: + return ExternalAccessContext(provider="openrag", subject="subject-1", level=level) + + +@pytest.mark.parametrize("action", ["read"]) +def test_viewer_ceiling_allows_read(action: str) -> None: + assert external_access_allows(action, _ctx("viewer")) is True + + +@pytest.mark.parametrize("action", ["write", "create", "delete", "execute", "ingest", "deploy"]) +def test_viewer_ceiling_denies_non_read(action: str) -> None: + assert external_access_allows(action, _ctx("viewer")) is False + + +@pytest.mark.parametrize("action", ["read", "write", "create", "delete", "execute", "ingest"]) +def test_editor_ceiling_allows_crud_and_execute(action: str) -> None: + # ``delete`` is included deliberately (regression for PR #13293): an editor + # may remove their own resources. + assert external_access_allows(action, _ctx("editor")) is True + + +def test_editor_ceiling_denies_deploy() -> None: + # Deploy is intentionally reserved for admin. + assert external_access_allows("deploy", _ctx("editor")) is False + + +@pytest.mark.parametrize("action", _ALL_ACTIONS) +def test_admin_ceiling_allows_everything(action: str) -> None: + assert external_access_allows(action, _ctx("admin")) is True + + +@pytest.mark.parametrize("action", _ALL_ACTIONS) +def test_no_ceiling_allows_everything(action: str) -> None: + """When no external ceiling is installed, every action is permitted.""" + assert external_access_allows(action, None) is True + + +def test_filter_actions_through_editor_ceiling_keeps_delete_drops_deploy() -> None: + set_current_external_access_context(_ctx("editor")) + try: + kept = filter_actions_by_external_access_ceiling(["read", "delete", "deploy"]) + finally: + set_current_external_access_context(None) + assert kept == ["read", "delete"] diff --git a/src/backend/tests/unit/services/authorization/test_capability_flag.py b/src/backend/tests/unit/services/authorization/test_capability_flag.py index 66f5fb6586..28f72cfd5a 100644 --- a/src/backend/tests/unit/services/authorization/test_capability_flag.py +++ b/src/backend/tests/unit/services/authorization/test_capability_flag.py @@ -1,4 +1,4 @@ -"""Tests for the cross-user-fetch capability flag.""" +"""Tests for authorization service capability flags.""" from __future__ import annotations @@ -23,6 +23,7 @@ def _settings(*, authz_enabled: bool = False) -> SimpleNamespace: async def test_base_class_default_is_false(): """The class-level constant defaults False so subclasses must opt in.""" assert BaseAuthorizationService.SUPPORTS_CROSS_USER_FETCH is False + assert BaseAuthorizationService.SUPPORTS_API_KEY_SCOPES is False @pytest.mark.anyio @@ -30,6 +31,7 @@ async def test_lfx_default_service_does_not_support_cross_user_fetch(): """The lfx no-op service inherits the safe default.""" service = LfxDefaultService() assert await service.supports_cross_user_fetch() is False + assert await service.supports_api_key_scopes() is False @pytest.mark.anyio @@ -37,14 +39,17 @@ async def test_langflow_pass_through_does_not_support_cross_user_fetch(): """OSS pass-through must NOT opt in — that is the strict-pass-through contract.""" service = LangflowAuthorizationService(_settings()) assert await service.supports_cross_user_fetch() is False + assert await service.supports_api_key_scopes() is False @pytest.mark.anyio async def test_subclass_can_opt_in(): - """Authorization plugins flip ``SUPPORTS_CROSS_USER_FETCH=True``; the base accepts it.""" + """Authorization plugins flip capability constants; the base accepts them.""" class _Plugin(LangflowAuthorizationService): SUPPORTS_CROSS_USER_FETCH = True + SUPPORTS_API_KEY_SCOPES = True service = _Plugin(_settings()) assert await service.supports_cross_user_fetch() is True + assert await service.supports_api_key_scopes() is True diff --git a/src/backend/tests/unit/services/authorization/test_filter_visible.py b/src/backend/tests/unit/services/authorization/test_filter_visible.py index 14b4adc0c2..9d4dc55c69 100644 --- a/src/backend/tests/unit/services/authorization/test_filter_visible.py +++ b/src/backend/tests/unit/services/authorization/test_filter_visible.py @@ -6,6 +6,12 @@ from types import SimpleNamespace from uuid import uuid4 import pytest +from langflow.services.auth.context import ( + AUTH_METHOD_API_KEY, + AuthCredentialContext, + clear_current_auth_context, + set_current_auth_context, +) from langflow.services.authorization import guards as authz_guards from langflow.services.authorization import listing as authz_listing from langflow.services.authorization.actions import FlowAction @@ -170,3 +176,41 @@ async def test_filter_visible_resources_owner_override_skips_enforcer(monkeypatc # Enforcer was consulted only for the non-owned item. assert len(service.batch_calls) == 1 assert len(service.batch_calls[0]["requests"]) == 1 + + +@pytest.mark.anyio +async def test_filter_visible_resources_api_key_scope_plugin_evaluates_owned_rows(monkeypatch, fake_user): + """API-key scope plugins can filter owned rows instead of owner-skipping them.""" + install_settings(monkeypatch, authz_enabled=True) + items = [ + SimpleNamespace(id=uuid4(), user_id=fake_user.id), + SimpleNamespace(id=uuid4(), user_id=fake_user.id), + ] + service = _StubAuthorizationService(batch_results=[True, False], supports_api_key_scopes=True) + install_authz(monkeypatch, service) + set_current_auth_context( + AuthCredentialContext( + method=AUTH_METHOD_API_KEY, + api_key_id=uuid4(), + api_key_source="db", # pragma: allowlist secret + ) + ) + + try: + result = await authz_listing.filter_visible_resources( + fake_user, + resource_type="flow", + candidates=items, + owner_extractor=lambda item: item.user_id, + act=FlowAction.READ, + ) + finally: + clear_current_auth_context() + + assert result == [items[0]] + assert len(service.batch_calls) == 1 + assert service.batch_calls[0]["requests"] == [ + (f"flow:{items[0].id}", "read"), + (f"flow:{items[1].id}", "read"), + ] + assert service.batch_calls[0]["context"]["auth_method"] == "api_key" diff --git a/src/backend/tests/unit/services/authorization/test_guards.py b/src/backend/tests/unit/services/authorization/test_guards.py index 5f85f5a399..5f3843e7f0 100644 --- a/src/backend/tests/unit/services/authorization/test_guards.py +++ b/src/backend/tests/unit/services/authorization/test_guards.py @@ -6,7 +6,17 @@ from uuid import uuid4 import pytest from fastapi import HTTPException +from langflow.services.auth.context import ( + AUTH_METHOD_API_KEY, + AuthCredentialContext, + clear_current_auth_context, + set_current_auth_context, +) from langflow.services.authorization import guards as authz_guards +from langflow.services.authorization.access_ceiling import ( + ExternalAccessContext, + set_current_external_access_context, +) from langflow.services.authorization.actions import ( DeploymentAction, FileAction, @@ -161,6 +171,35 @@ async def test_ensure_permission_writes_audit_on_deny(monkeypatch, fake_user): assert audit_calls[0]["result"] == "deny" +@pytest.mark.anyio +async def test_ensure_permission_forwards_auth_context(monkeypatch, fake_user): + """Credential metadata is available to plugins and audit without route signature churn.""" + install_settings(monkeypatch, authz_enabled=True) + service = _StubAuthorizationService(allow=True) + install_authz(monkeypatch, service) + audit_calls = install_audit_recorder(monkeypatch) + api_key_id = uuid4() + set_current_auth_context( + AuthCredentialContext( + method=AUTH_METHOD_API_KEY, + api_key_id=api_key_id, + api_key_source="db", # pragma: allowlist secret + ) + ) + + try: + await authz_guards.ensure_permission(fake_user, domain="*", obj="flow:abc", act="read") + finally: + clear_current_auth_context() + + forwarded_context = service.calls[0]["context"] + assert forwarded_context["auth_method"] == "api_key" + assert forwarded_context["api_key_id"] == api_key_id + assert forwarded_context["api_key_source"] == "db" # pragma: allowlist secret + assert audit_calls[0]["details"]["auth_method"] == "api_key" + assert audit_calls[0]["details"]["api_key_id"] == str(api_key_id) + + # ----------------------------------------------------------------------------- # # ensure_flow_permission — enum coercion, domain, owner override # ----------------------------------------------------------------------------- # @@ -294,6 +333,88 @@ async def test_owner_override_skips_enforce(monkeypatch, fake_user): assert audit_calls[0]["result"] == "owner_override" +@pytest.mark.anyio +async def test_api_key_scope_plugin_sees_owner_resource_instead_of_owner_override(monkeypatch, fake_user): + """API-key scopes can be narrower than the owning user when a plugin opts in.""" + install_settings(monkeypatch, authz_enabled=True) + service = _StubAuthorizationService(allow=False, supports_api_key_scopes=True) + install_authz(monkeypatch, service) + install_audit_recorder(monkeypatch) + set_current_auth_context( + AuthCredentialContext( + method=AUTH_METHOD_API_KEY, + api_key_id=uuid4(), + api_key_source="db", # pragma: allowlist secret + ) + ) + + try: + with pytest.raises(HTTPException) as exc_info: + await authz_guards.ensure_flow_permission( + fake_user, + FlowAction.DELETE, + flow_id=uuid4(), + flow_user_id=fake_user.id, + ) + finally: + clear_current_auth_context() + + assert exc_info.value.status_code == 403 + assert len(service.calls) == 1 + assert service.calls[0]["context"]["flow_user_id"] == fake_user.id + assert service.calls[0]["context"]["auth_method"] == "api_key" + + +@pytest.mark.anyio +async def test_external_viewer_ceiling_denies_before_owner_override(monkeypatch, fake_user): + """A viewer claim is a deny-only ceiling even when the user owns the flow.""" + install_settings(monkeypatch, authz_enabled=False, audit_enabled=True) + service = _StubAuthorizationService(allow=True) + install_authz(monkeypatch, service) + audit_calls = install_audit_recorder(monkeypatch) + set_current_external_access_context(ExternalAccessContext(provider="openrag", subject="subject-1", level="viewer")) + + try: + with pytest.raises(HTTPException) as exc_info: + await authz_guards.ensure_flow_permission( + fake_user, + FlowAction.WRITE, + flow_id=uuid4(), + flow_user_id=fake_user.id, + ) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + assert "External credentials" in exc_info.value.detail + assert service.calls == [] + assert audit_calls[0]["result"] == "deny" + assert audit_calls[0]["details"]["external_access_level"] == "viewer" + + +@pytest.mark.anyio +async def test_external_editor_ceiling_allows_write_then_owner_override(monkeypatch, fake_user): + """Editor is still only a ceiling; normal owner/plugin logic continues after it passes.""" + install_settings(monkeypatch, authz_enabled=True, audit_enabled=True) + service = _StubAuthorizationService(allow=False) + install_authz(monkeypatch, service) + audit_calls = install_audit_recorder(monkeypatch) + set_current_external_access_context(ExternalAccessContext(provider="openrag", subject="subject-1", level="editor")) + + try: + await authz_guards.ensure_flow_permission( + fake_user, + FlowAction.WRITE, + flow_id=uuid4(), + flow_user_id=fake_user.id, + ) + finally: + set_current_external_access_context(None) + + assert service.calls == [] + assert audit_calls[0]["result"] == "owner_override" + + @pytest.mark.anyio async def test_owner_override_audits_even_when_authz_disabled(monkeypatch, fake_user): """Owner override on a disabled-authz install still writes an audit row. diff --git a/src/backend/tests/unit/services/authorization/test_route_ceiling_guards.py b/src/backend/tests/unit/services/authorization/test_route_ceiling_guards.py new file mode 100644 index 0000000000..39f3fec715 --- /dev/null +++ b/src/backend/tests/unit/services/authorization/test_route_ceiling_guards.py @@ -0,0 +1,610 @@ +"""Regression tests for access-ceiling guards added to execution / mutation routes. + +PR #13293 added ``ensure_*_permission`` guards to four route areas that +previously escaped the external access ceiling: + +* ``mcp_utils.handle_call_tool`` (flow EXECUTE) +* ``flow_version`` create/activate/delete (flow WRITE/DELETE) +* ``files`` upload/delete (flow WRITE) +* ``memories`` create/update/delete/flush/regenerate (knowledge-base actions) + +These tests drive the real route handlers with the data services mocked, +verifying two invariants for each newly-guarded action: + +1. A viewer-ceiling caller is denied with HTTP 403 *before* the data service is + touched (the deny-only ceiling fires ahead of owner-override and the + ``AUTHZ_ENABLED`` gate). +2. With no ceiling installed, the owner-override path returns early and the + route proceeds to the underlying service — preserving feature-off behavior. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock +from uuid import uuid4 + +import pytest +from fastapi import HTTPException +from langflow.services.authorization.access_ceiling import ( + ExternalAccessContext, + set_current_external_access_context, +) + +from ._common import ( + _StubAuthorizationService, + install_audit_recorder, + install_authz, + install_settings, +) + + +@pytest.fixture +def owner(): + """A non-superuser acting as the resource owner.""" + return SimpleNamespace(id=uuid4(), username="owner", is_superuser=False) + + +@pytest.fixture(autouse=True) +def _wire_guard_services(monkeypatch): + """Install pass-through authz services so guards run the real ceiling path. + + ``AUTHZ_ENABLED=False`` mirrors the OSS default: the only thing that can deny + is the external access ceiling, which is checked before the gate. + """ + install_settings(monkeypatch, authz_enabled=False, audit_enabled=True) + install_authz(monkeypatch, _StubAuthorizationService(allow=True)) + install_audit_recorder(monkeypatch) + + +def _viewer_ceiling() -> ExternalAccessContext: + return ExternalAccessContext(provider="openrag", subject="s-1", level="viewer") + + +def _make_flow(owner_id): + return SimpleNamespace( + id=uuid4(), + user_id=owner_id, + workspace_id=None, + folder_id=None, + data={"nodes": [], "edges": []}, + name="flow", + ) + + +# --------------------------------------------------------------------------- # +# C1 — mcp_utils.handle_call_tool (flow EXECUTE) +# --------------------------------------------------------------------------- # + + +@pytest.mark.anyio +async def test_mcp_call_tool_viewer_denied_before_run(monkeypatch, owner): + from langflow.api.v1 import mcp_utils + + flow = _make_flow(owner.id) + monkeypatch.setattr(mcp_utils, "get_flow_snake_case", AsyncMock(return_value=flow)) + monkeypatch.setattr(mcp_utils, "get_mcp_config", lambda: SimpleNamespace(enable_progress_notifications=False)) + run_spy = AsyncMock() + monkeypatch.setattr(mcp_utils, "simple_run_flow", run_spy) + monkeypatch.setattr(mcp_utils, "with_db_session", lambda fn: fn(MagicMock())) + mcp_utils.current_user_ctx.set(owner) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await mcp_utils.handle_call_tool(name="flow", arguments={}, server=MagicMock()) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + run_spy.assert_not_awaited() + + +@pytest.mark.anyio +async def test_mcp_call_tool_owner_runs_without_ceiling(monkeypatch, owner): + from langflow.api.v1 import mcp_utils + + flow = _make_flow(owner.id) + monkeypatch.setattr(mcp_utils, "get_flow_snake_case", AsyncMock(return_value=flow)) + monkeypatch.setattr(mcp_utils, "get_mcp_config", lambda: SimpleNamespace(enable_progress_notifications=False)) + run_spy = AsyncMock(return_value=SimpleNamespace(outputs=[])) + monkeypatch.setattr(mcp_utils, "simple_run_flow", run_spy) + monkeypatch.setattr(mcp_utils, "with_db_session", lambda fn: fn(MagicMock())) + mcp_utils.current_user_ctx.set(owner) + + await mcp_utils.handle_call_tool(name="flow", arguments={}, server=MagicMock()) + + run_spy.assert_awaited_once() + + +# --------------------------------------------------------------------------- # +# H2 — flow_version create / activate / delete +# --------------------------------------------------------------------------- # + + +@pytest.mark.anyio +async def test_flow_version_create_snapshot_viewer_denied(monkeypatch, owner): + from langflow.api.v1 import flow_version + + flow = _make_flow(owner.id) + monkeypatch.setattr(flow_version, "_get_user_flow", AsyncMock(return_value=flow)) + create_spy = AsyncMock() + monkeypatch.setattr(flow_version, "create_flow_version_entry", create_spy) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await flow_version.create_snapshot(flow_id=flow.id, current_user=owner, session=MagicMock(), body=None) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + create_spy.assert_not_awaited() + + +@pytest.mark.anyio +async def test_flow_version_delete_viewer_denied(monkeypatch, owner): + from langflow.api.v1 import flow_version + + flow = _make_flow(owner.id) + monkeypatch.setattr(flow_version, "_get_user_flow", AsyncMock(return_value=flow)) + delete_spy = AsyncMock() + monkeypatch.setattr(flow_version, "delete_flow_version_entry", delete_spy) + monkeypatch.setattr(flow_version, "get_flow_version_entry_or_raise", AsyncMock()) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await flow_version.delete_version_entry( + flow_id=flow.id, version_id=uuid4(), current_user=owner, session=MagicMock() + ) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + delete_spy.assert_not_awaited() + + +@pytest.mark.anyio +async def test_flow_version_create_snapshot_owner_proceeds(monkeypatch, owner): + from langflow.api.v1 import flow_version + + flow = _make_flow(owner.id) + monkeypatch.setattr(flow_version, "_get_user_flow", AsyncMock(return_value=flow)) + entry = SimpleNamespace(id=uuid4()) + create_spy = AsyncMock(return_value=entry) + monkeypatch.setattr(flow_version, "create_flow_version_entry", create_spy) + monkeypatch.setattr(flow_version, "_version_to_read", lambda e: e) + + result = await flow_version.create_snapshot(flow_id=flow.id, current_user=owner, session=MagicMock(), body=None) + + create_spy.assert_awaited_once() + assert result is entry + + +# --------------------------------------------------------------------------- # +# F19 — files upload / delete (flow WRITE) +# --------------------------------------------------------------------------- # + + +@pytest.mark.anyio +async def test_files_upload_viewer_denied(owner): + from langflow.api.v1 import files + + flow = _make_flow(owner.id) + storage = MagicMock(save_file=AsyncMock()) + settings_service = SimpleNamespace(settings=SimpleNamespace(max_file_size_upload=100)) + upload = SimpleNamespace(size=1, filename="x.txt", read=AsyncMock(return_value=b"data")) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await files.upload_file( + file=upload, + flow=flow, + current_user=owner, + storage_service=storage, + settings_service=settings_service, + ) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + storage.save_file.assert_not_awaited() + + +@pytest.mark.anyio +async def test_files_delete_viewer_denied(owner): + from langflow.api.v1 import files + + flow = _make_flow(owner.id) + storage = MagicMock(delete_file=AsyncMock()) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await files.delete_file(file_name="x.txt", flow=flow, current_user=owner, storage_service=storage) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + storage.delete_file.assert_not_awaited() + + +@pytest.mark.anyio +async def test_files_delete_owner_proceeds(owner): + from langflow.api.v1 import files + + flow = _make_flow(owner.id) + storage = MagicMock(delete_file=AsyncMock()) + + result = await files.delete_file(file_name="x.txt", flow=flow, current_user=owner, storage_service=storage) + + storage.delete_file.assert_awaited_once() + assert "deleted successfully" in result["message"] + + +# --------------------------------------------------------------------------- # +# F20 — memories create / update / delete / flush / regenerate +# --------------------------------------------------------------------------- # + + +def _install_memory_service(monkeypatch, **methods): + from langflow.api.v1 import memories + + service = SimpleNamespace(**methods) + monkeypatch.setattr(memories, "get_memory_base_service", lambda: service) + return service + + +@pytest.mark.anyio +async def test_memory_create_viewer_denied(monkeypatch, owner): + from langflow.api.v1 import memories + + create_spy = AsyncMock() + _install_memory_service(monkeypatch, create=create_spy) + payload = SimpleNamespace() + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await memories.create_memory_base(current_user=owner, payload=payload) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + create_spy.assert_not_awaited() + + +@pytest.mark.anyio +async def test_memory_delete_viewer_denied(monkeypatch, owner): + from langflow.api.v1 import memories + + delete_spy = AsyncMock(return_value=True) + mb = SimpleNamespace(user_id=owner.id, kb_name="kb") + _install_memory_service(monkeypatch, delete=delete_spy, get=AsyncMock(return_value=mb)) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await memories.delete_memory_base(memory_base_id=uuid4(), current_user=owner) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + delete_spy.assert_not_awaited() + + +@pytest.mark.anyio +async def test_memory_flush_viewer_denied(monkeypatch, owner): + from langflow.api.v1 import memories + + trigger_spy = AsyncMock(return_value=uuid4()) + mb = SimpleNamespace(user_id=owner.id, kb_name="kb") + _install_memory_service(monkeypatch, trigger_ingestion=trigger_spy, get=AsyncMock(return_value=mb)) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await memories.flush_memory_base( + memory_base_id=uuid4(), current_user=owner, body=SimpleNamespace(session_id="s") + ) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + trigger_spy.assert_not_awaited() + + +@pytest.mark.anyio +async def test_memory_regenerate_viewer_denied(monkeypatch, owner): + from langflow.api.v1 import memories + + regen_spy = AsyncMock(return_value=[]) + mb = SimpleNamespace(user_id=owner.id, kb_name="kb") + _install_memory_service(monkeypatch, regenerate=regen_spy, get=AsyncMock(return_value=mb)) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await memories.regenerate_memory_base(memory_base_id=uuid4(), current_user=owner) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + regen_spy.assert_not_awaited() + + +@pytest.mark.anyio +async def test_memory_delete_owner_proceeds(monkeypatch, owner): + """An editor (delete is now in the editor ceiling) deletes their own memory base.""" + from langflow.api.v1 import memories + + delete_spy = AsyncMock(return_value=True) + mb = SimpleNamespace(user_id=owner.id, kb_name="kb") + _install_memory_service(monkeypatch, delete=delete_spy, get=AsyncMock(return_value=mb)) + + set_current_external_access_context(ExternalAccessContext(provider="openrag", subject="s-1", level="editor")) + try: + await memories.delete_memory_base(memory_base_id=uuid4(), current_user=owner) + finally: + set_current_external_access_context(None) + + delete_spy.assert_awaited_once() + + +# --------------------------------------------------------------------------- # +# endpoints.custom_component / custom_component_update (direct ceiling, "create") +# +# These routes instantiate posted component code and are not tied to a single +# owned resource, so they enforce the deny-only ceiling primitive directly +# instead of an ``ensure_*_permission`` guard. +# --------------------------------------------------------------------------- # + + +@pytest.mark.anyio +async def test_custom_component_viewer_denied_before_build(monkeypatch, owner): + from langflow.api.v1 import endpoints + + build_spy = MagicMock() + monkeypatch.setattr(endpoints, "build_custom_component_template", build_spy) + raw_code = SimpleNamespace(code="print('x')", frontend_node=None) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await endpoints.custom_component(raw_code=raw_code, user=owner, request=MagicMock()) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + build_spy.assert_not_called() + + +@pytest.mark.anyio +async def test_custom_component_update_viewer_denied_before_build(monkeypatch, owner): + from langflow.api.v1 import endpoints + + build_spy = MagicMock() + monkeypatch.setattr(endpoints, "build_custom_component_template", build_spy) + code_request = SimpleNamespace(code="print('x')", tool_mode=False, get_template=dict) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await endpoints.custom_component_update(code_request=code_request, user=owner, request=MagicMock()) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + build_spy.assert_not_called() + + +@pytest.mark.anyio +async def test_custom_component_editor_passes_ceiling(monkeypatch, owner): + """An editor ceiling allows component instantiation (create is in the editor set).""" + from langflow.api.v1 import endpoints + + # Stub the build pipeline so the route returns without real execution; the + # point is that the ceiling check does NOT short-circuit for an editor. + instance = MagicMock() + instance.update_frontend_node = AsyncMock(return_value={"tool_mode": False}) + monkeypatch.setattr( + endpoints, + "build_custom_component_template", + MagicMock(return_value=({"tool_mode": False}, instance)), + ) + monkeypatch.setattr(endpoints, "get_instance_name", lambda _i: "MyComponent") + monkeypatch.setattr(endpoints, "_requires_component_hash_lookups", lambda *_a, **_k: False) + # Settings: custom components allowed, no admin-only gate. + settings = SimpleNamespace(allow_custom_components=True, custom_component_admin_only=False) + monkeypatch.setattr(endpoints, "get_settings_service", lambda: SimpleNamespace(settings=settings)) + # ``isinstance(instance, Component)`` must be False so the optional + # run_and_validate_update_outputs branch is skipped for this MagicMock. + raw_code = SimpleNamespace(code="print('x')", frontend_node=None) + request = SimpleNamespace(state=SimpleNamespace(locale="en")) + + set_current_external_access_context(ExternalAccessContext(provider="openrag", subject="s-1", level="editor")) + try: + result = await endpoints.custom_component(raw_code=raw_code, user=owner, request=request) + finally: + set_current_external_access_context(None) + + assert result.type == "MyComponent" + + +# --------------------------------------------------------------------------- # +# endpoints.create_upload_file — deprecated upload (flow WRITE) +# --------------------------------------------------------------------------- # + + +@pytest.mark.anyio +async def test_deprecated_upload_viewer_denied(monkeypatch, owner): + from langflow.api.v1 import endpoints + + flow = _make_flow(owner.id) + save_spy = MagicMock() + monkeypatch.setattr(endpoints, "save_uploaded_file", save_spy) + settings_service = SimpleNamespace(settings=SimpleNamespace(max_file_size_upload=100)) + upload = SimpleNamespace(size=1, filename="x.txt") + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await endpoints.create_upload_file( + file=upload, flow=flow, current_user=owner, settings_service=settings_service + ) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + save_spy.assert_not_called() + + +@pytest.mark.anyio +async def test_deprecated_upload_owner_proceeds(monkeypatch, owner): + from pathlib import Path + + from langflow.api.v1 import endpoints + + flow = _make_flow(owner.id) + save_spy = MagicMock(return_value=Path("flow/x.txt")) + monkeypatch.setattr(endpoints, "save_uploaded_file", save_spy) + settings_service = SimpleNamespace(settings=SimpleNamespace(max_file_size_upload=100)) + upload = SimpleNamespace(size=1, filename="x.txt") + + result = await endpoints.create_upload_file( + file=upload, flow=flow, current_user=owner, settings_service=settings_service + ) + + save_spy.assert_called_once() + assert result.flow_id == str(flow.id) + + +# --------------------------------------------------------------------------- # +# mcp_projects.update_project_mcp_settings — project WRITE +# --------------------------------------------------------------------------- # + + +def _install_mcp_session(monkeypatch, project): + """Patch ``session_scope`` so the route's project fetch returns ``project``.""" + from contextlib import asynccontextmanager + + from langflow.api.v1 import mcp_projects + + exec_result = SimpleNamespace(first=lambda: project, all=list) + session = SimpleNamespace( + exec=AsyncMock(return_value=exec_result), + flush=AsyncMock(), + add=MagicMock(), + ) + + @asynccontextmanager + async def _scope(): + yield session + + monkeypatch.setattr(mcp_projects, "session_scope", _scope) + return session + + +@pytest.mark.anyio +async def test_mcp_update_settings_viewer_denied(monkeypatch, owner): + from langflow.api.v1 import mcp_projects + + project = SimpleNamespace( + id=uuid4(), + user_id=owner.id, + workspace_id=None, + auth_settings=None, + flows=[], + name="proj", + ) + session = _install_mcp_session(monkeypatch, project) + request = SimpleNamespace(settings=[], auth_settings=None, model_fields_set=set()) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await mcp_projects.update_project_mcp_settings(project_id=project.id, request=request, current_user=owner) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + session.flush.assert_not_awaited() + + +# --------------------------------------------------------------------------- # +# models.py variable-mutating routes (variable WRITE / DELETE) +# --------------------------------------------------------------------------- # + + +@pytest.mark.anyio +async def test_update_enabled_models_viewer_denied(monkeypatch, owner): + from langflow.api.v1 import models + + svc_spy = MagicMock() + monkeypatch.setattr(models, "get_variable_service", svc_spy) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await models.update_enabled_models(session=MagicMock(), current_user=owner, updates=[]) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + svc_spy.assert_not_called() + + +@pytest.mark.anyio +async def test_set_default_model_viewer_denied(monkeypatch, owner): + from langflow.api.v1 import models + + svc_spy = MagicMock() + monkeypatch.setattr(models, "get_variable_service", svc_spy) + request = SimpleNamespace(model_type="language", model_name="m", provider="p") + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await models.set_default_model(session=MagicMock(), current_user=owner, request=request) + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + svc_spy.assert_not_called() + + +@pytest.mark.anyio +async def test_clear_default_model_viewer_denied(monkeypatch, owner): + from langflow.api.v1 import models + + svc_spy = MagicMock() + monkeypatch.setattr(models, "get_variable_service", svc_spy) + + set_current_external_access_context(_viewer_ceiling()) + try: + with pytest.raises(HTTPException) as exc_info: + await models.clear_default_model(session=MagicMock(), current_user=owner, model_type="language") + finally: + set_current_external_access_context(None) + + assert exc_info.value.status_code == 403 + svc_spy.assert_not_called() + + +@pytest.mark.anyio +async def test_set_default_model_owner_proceeds(monkeypatch, owner): + """Owner with no ceiling reaches the variable service (owner-override path).""" + from langflow.api.v1 import models + + var_service = MagicMock(spec=models.DatabaseVariableService) + var_service.get_variable_object = AsyncMock(return_value=SimpleNamespace(id=uuid4())) + var_service.update_variable_fields = AsyncMock() + monkeypatch.setattr(models, "get_variable_service", lambda: var_service) + request = SimpleNamespace(model_type="language", model_name="m", provider="p") + + result = await models.set_default_model(session=MagicMock(), current_user=owner, request=request) + + var_service.update_variable_fields.assert_awaited_once() + assert result["default_model"]["model_name"] == "m" diff --git a/src/backend/tests/unit/services/database/models/api_key/test_crud.py b/src/backend/tests/unit/services/database/models/api_key/test_crud.py index c9f5fac837..c700bf1500 100644 --- a/src/backend/tests/unit/services/database/models/api_key/test_crud.py +++ b/src/backend/tests/unit/services/database/models/api_key/test_crud.py @@ -10,6 +10,7 @@ from uuid import uuid4 import pytest from langflow.services.database.models.api_key.crud import ( _check_key_from_db, + authenticate_api_key, create_api_key, hash_api_key, ) @@ -55,6 +56,12 @@ def mock_settings(monkeypatch): settings = SimpleNamespace( auth_settings=SimpleNamespace( SECRET_KEY=SimpleNamespace(get_secret_value=lambda: "a" * 43), + API_KEY_SOURCE="db", # pragma: allowlist secret + # Non-optional AuthSettings fields read directly by create_api_key + # and the external-access-ceiling chokepoint. + EXTERNAL_AUTH_ACCESS_CEILING_ENABLED=False, + EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS=True, + EXTERNAL_AUTH_PROVIDER="external", ), settings=SimpleNamespace(disable_track_apikey_usage=False), ) @@ -111,6 +118,32 @@ async def test_check_key_finds_by_hash(async_session, mock_settings): assert result.id == user.id +@pytest.mark.anyio +async def test_authenticate_api_key_returns_db_key_metadata(async_session, mock_settings): # noqa: ARG001 + """The richer resolver preserves the DB API-key id for authorization context.""" + user = _make_user() + async_session.add(user) + await async_session.flush() + + plaintext = "sk-test-12345" # pragma: allowlist secret + api_key = ApiKey( + api_key="encrypted-value", # pragma: allowlist secret + api_key_hash=hash_api_key(plaintext), + name="test", + user_id=user.id, + created_at=datetime.now(timezone.utc), + ) + async_session.add(api_key) + await async_session.flush() + + result = await authenticate_api_key(async_session, plaintext) + + assert result is not None + assert result.user.id == user.id + assert result.api_key_source == "db" # pragma: allowlist secret + assert result.api_key_id == api_key.id + + @pytest.mark.anyio async def test_check_key_fallback_for_legacy_keys(async_session, mock_settings, monkeypatch): """Legacy keys without hash must still match via decrypt-and-compare.""" @@ -390,3 +423,215 @@ async def test_create_api_key_no_expires_at_is_none(async_session, mock_settings assert row is not None assert row.expires_at is None assert result.expires_at is None + + +# ============================================================================= +# External access ceiling chokepoint (H1): API keys disabled for external users +# ============================================================================= + + +async def _seed_external_user_with_key(async_session, *, provider: str = "external"): + """Create a user with an external SSO profile and an active API key.""" + from langflow.services.database.models.auth import SSOUserProfile + + user = _make_user(username=f"ext-{uuid4().hex[:8]}") + async_session.add(user) + await async_session.flush() + + async_session.add( + SSOUserProfile( + user_id=user.id, + sso_provider=provider, + sso_user_id=f"subject-{uuid4().hex[:8]}", + ) + ) + plaintext = f"sk-ext-{uuid4().hex}" # pragma: allowlist secret + async_session.add( + ApiKey( + api_key="encrypted-ext", # pragma: allowlist secret + api_key_hash=hash_api_key(plaintext), + name="ext-key", + user_id=user.id, + created_at=datetime.now(timezone.utc), + ) + ) + await async_session.flush() + return user, plaintext + + +@pytest.mark.anyio +async def test_authenticate_api_key_rejects_external_user_when_ceiling_enabled(async_session, mock_settings): + """When ceiling + disable-keys are ON, an external user's API key fails at auth time.""" + mock_settings.auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED = True + mock_settings.auth_settings.EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS = True + + _user, plaintext = await _seed_external_user_with_key(async_session) + + # authenticate_api_key is the shared chokepoint used by every API-key caller. + assert await authenticate_api_key(async_session, plaintext) is None + + +@pytest.mark.anyio +async def test_authenticate_api_key_allows_external_user_when_ceiling_disabled(async_session, mock_settings): + """When the ceiling is OFF, the external user's API key still authenticates (no behavior change).""" + mock_settings.auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED = False + mock_settings.auth_settings.EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS = True + + user, plaintext = await _seed_external_user_with_key(async_session) + + result = await authenticate_api_key(async_session, plaintext) + assert result is not None + assert result.user.id == user.id + + +@pytest.mark.anyio +async def test_authenticate_api_key_allows_external_user_when_disable_keys_disabled(async_session, mock_settings): + """Ceiling ON but disable-keys OFF must not block the external user (feature is opt-in).""" + mock_settings.auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED = True + mock_settings.auth_settings.EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS = False + + user, plaintext = await _seed_external_user_with_key(async_session) + + result = await authenticate_api_key(async_session, plaintext) + assert result is not None + assert result.user.id == user.id + + +@pytest.mark.anyio +async def test_authenticate_api_key_allows_non_external_user_when_ceiling_enabled(async_session, mock_settings): + """A user with no external SSO profile is unaffected even with the ceiling fully enabled.""" + mock_settings.auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED = True + mock_settings.auth_settings.EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS = True + + user = _make_user() + async_session.add(user) + await async_session.flush() + plaintext = "sk-native-key" # pragma: allowlist secret + async_session.add( + ApiKey( + api_key="encrypted-native", # pragma: allowlist secret + api_key_hash=hash_api_key(plaintext), + name="native-key", + user_id=user.id, + created_at=datetime.now(timezone.utc), + ) + ) + await async_session.flush() + + result = await authenticate_api_key(async_session, plaintext) + assert result is not None + assert result.user.id == user.id + + +@pytest.mark.anyio +async def test_authenticate_api_key_ignores_profile_for_other_provider(async_session, mock_settings): + """A profile under a different provider key must not trip the configured-provider block.""" + mock_settings.auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED = True + mock_settings.auth_settings.EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS = True + mock_settings.auth_settings.EXTERNAL_AUTH_PROVIDER = "external" + + # Profile is stored under "some-other-provider", not the configured "external". + user, plaintext = await _seed_external_user_with_key(async_session, provider="some-other-provider") + + result = await authenticate_api_key(async_session, plaintext) + assert result is not None + assert result.user.id == user.id + + +# ============================================================================= +# LOW: a DENIED API-key auth must not mutate usage counters (total_uses/last_used_at) +# ============================================================================= + + +@pytest.mark.anyio +async def test_blocked_external_user_key_does_not_increment_usage(async_session, mock_settings): + """A ceiling-blocked external user's key must NOT bump total_uses / last_used_at (fast hash path).""" + from sqlmodel import select + + mock_settings.auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED = True + mock_settings.auth_settings.EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS = True + + _user, plaintext = await _seed_external_user_with_key(async_session) + + # Denied auth. + assert await authenticate_api_key(async_session, plaintext) is None + + row = (await async_session.exec(select(ApiKey).where(ApiKey.api_key_hash == hash_api_key(plaintext)))).first() + assert row is not None + assert row.total_uses == 0 + assert row.last_used_at is None + + +@pytest.mark.anyio +async def test_blocked_external_user_legacy_key_does_not_increment_usage(async_session, mock_settings, monkeypatch): + """A ceiling-blocked external user's legacy (hashless) key must NOT bump usage / backfill hash.""" + from langflow.services.database.models.auth import SSOUserProfile + from sqlmodel import select + + mock_settings.auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED = True + mock_settings.auth_settings.EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS = True + + user = _make_user(username=f"ext-{uuid4().hex[:8]}") + async_session.add(user) + await async_session.flush() + async_session.add( + SSOUserProfile( + user_id=user.id, + sso_provider="external", + sso_user_id=f"subject-{uuid4().hex[:8]}", + ) + ) + + plaintext = "sk-legacy-blocked" # pragma: allowlist secret + legacy = ApiKey( + api_key=plaintext, + api_key_hash=None, + name="legacy-blocked", + user_id=user.id, + created_at=datetime.now(timezone.utc), + ) + async_session.add(legacy) + await async_session.flush() + + monkeypatch.setattr( + "langflow.services.database.models.api_key.crud.auth_utils.decrypt_api_key", + lambda val, **_kwargs: val, + ) + + assert await authenticate_api_key(async_session, plaintext) is None + + row = (await async_session.exec(select(ApiKey).where(ApiKey.id == legacy.id))).first() + assert row is not None + assert row.total_uses == 0 + assert row.last_used_at is None + # The hash backfill is a success-path side effect and must not run on a denial. + assert row.api_key_hash is None + + +@pytest.mark.anyio +async def test_allowed_user_key_still_increments_usage(async_session, mock_settings): # noqa: ARG001 + """Sanity: a non-blocked user's key still records usage (success-path unchanged).""" + from sqlmodel import select + + user = _make_user() + async_session.add(user) + await async_session.flush() + plaintext = "sk-usage-tracked" # pragma: allowlist secret + async_session.add( + ApiKey( + api_key="encrypted-tracked", # pragma: allowlist secret + api_key_hash=hash_api_key(plaintext), + name="tracked", + user_id=user.id, + created_at=datetime.now(timezone.utc), + ) + ) + await async_session.flush() + + result = await authenticate_api_key(async_session, plaintext) + assert result is not None + + row = (await async_session.exec(select(ApiKey).where(ApiKey.api_key_hash == hash_api_key(plaintext)))).first() + assert row is not None + assert row.total_uses == 1 + assert row.last_used_at is not None diff --git a/src/backend/tests/unit/test_chat_endpoint.py b/src/backend/tests/unit/test_chat_endpoint.py index 3df3aafacd..50476a5532 100644 --- a/src/backend/tests/unit/test_chat_endpoint.py +++ b/src/backend/tests/unit/test_chat_endpoint.py @@ -1319,3 +1319,138 @@ def test_scope_session_to_namespace_helper(): assert scope_session_to_namespace("victim-session", "namespace-B") == "namespace-B:victim-session" # A foreign-namespace prefix is treated as out-of-namespace and gets re-wrapped. assert scope_session_to_namespace("namespace-B:victim", "namespace-A") == "namespace-A:namespace-B:victim" + + +# ── Public job registry unit tests ─────────────────────────────────────────── +# CVE fix: unauthenticated callers must not access private-flow job streams +# by guessing or leaking a job_id from the authenticated build endpoint. + + +async def test_job_queue_service_register_and_check_public_job(): + """register_public_job marks a job as public; is_public_job reflects that.""" + svc = JobQueueService() + job_id = str(uuid.uuid4()) + + # Why: job not registered yet — must return False before registration + assert svc.is_public_job(job_id) is False + + await svc.register_public_job(job_id) + + # Why: job registered — must return True after registration + assert svc.is_public_job(job_id) is True + + +def test_job_queue_service_unregistered_job_not_public(): + """A job_id that was never registered is not considered public.""" + svc = JobQueueService() + assert svc.is_public_job(str(uuid.uuid4())) is False + + +async def test_job_queue_service_is_public_job_async_base(): + """is_public_job_async on base class delegates to in-memory is_public_job.""" + svc = JobQueueService() + job_id = str(uuid.uuid4()) + + # Why: async variant must mirror sync variant — False before, True after + assert await svc.is_public_job_async(job_id) is False + await svc.register_public_job(job_id) + assert await svc.is_public_job_async(job_id) is True + + +async def test_job_queue_service_cleanup_removes_public_registration(): + """cleanup_job discards the public registration so the job_id cannot be reused. + + Why: tests the actual cleanup_job contract — not the internal set. + If cleanup_job stops calling discard, this test must catch it. + """ + svc = JobQueueService() + job_id = str(uuid.uuid4()) + await svc.register_public_job(job_id) + assert svc.is_public_job(job_id) is True + + # Call the real cleanup path — not svc._public_jobs.discard directly. + # cleanup_job early-returns when job_id is not in _queues, but the + # _public_jobs.discard call is unconditional (after the early-return guard), + # so we need to reach it. Seed a minimal queue entry first. + svc._queues[job_id] = (asyncio.Queue(), None, None, None) # type: ignore[arg-type] + await svc.cleanup_job(job_id) + + # Why: if cleanup_job ever drops the discard call, is_public_job still returns True here + assert svc.is_public_job(job_id) is False + + +@pytest.mark.benchmark +@pytest.mark.security +async def test_private_job_id_blocked_on_public_events_endpoint(client, json_memory_chatbot_no_llm, logged_in_headers): + """A job_id started via the authenticated build endpoint must be rejected by the public events endpoint. + + Security proof: before the fix, any caller who knew or guessed a private job_id + could read the live event stream (LLM output, API keys, tracebacks) without auth. + After the fix, _assert_public_job returns HTTP 404 because the job was never + registered via register_public_job. + + Why 404 not 403: returning 403 would confirm the job exists under a different + access tier, leaking information about private builds. + """ + flow_id = await create_flow(client, json_memory_chatbot_no_llm, logged_in_headers) + + # Start a PRIVATE (authenticated) build — job_id never passed through build_public_tmp + private_start = await client.post( + f"api/v1/build/{flow_id}/flow", + json={}, + headers={**logged_in_headers, "Content-Type": "application/json"}, + ) + assert private_start.status_code == codes.OK + private_job_id = private_start.json()["job_id"] + + # Why: the shared AsyncClient persists access-token cookies from logged_in_headers. + # Without clearing them, get_current_user_optional could resolve a user on this + # "public" request, which would not exercise the unauthenticated attack path. + client.cookies.clear() + + # Attempt to read the private job's events via the unauthenticated public endpoint + # Why: this is the exact attack vector — attacker has job_id, tries public endpoint + events_response = await client.get( + f"api/v1/build_public_tmp/{private_job_id}/events?event_delivery=polling", + headers={"Accept": "application/x-ndjson"}, + ) + + # Must be 404 — gate blocks private job from public endpoint + assert events_response.status_code == codes.NOT_FOUND + assert events_response.json()["detail"] == "Job not found" + + +@pytest.mark.benchmark +@pytest.mark.security +async def test_private_job_id_blocked_on_public_cancel_endpoint(client, json_memory_chatbot_no_llm, logged_in_headers): + """A job_id started via the authenticated build endpoint must be rejected by the public cancel endpoint. + + Security proof: before the fix, an unauthenticated attacker could cancel any + in-flight private build as a denial-of-service by supplying a known job_id. + After the fix, _assert_public_job returns HTTP 404. + """ + flow_id = await create_flow(client, json_memory_chatbot_no_llm, logged_in_headers) + + # Start a PRIVATE (authenticated) build + private_start = await client.post( + f"api/v1/build/{flow_id}/flow", + json={}, + headers={**logged_in_headers, "Content-Type": "application/json"}, + ) + assert private_start.status_code == codes.OK + private_job_id = private_start.json()["job_id"] + + # Why: the shared AsyncClient persists access-token cookies from logged_in_headers. + # Without clearing them, get_current_user_optional could resolve a user on this + # "public" request, which would not exercise the unauthenticated attack path. + client.cookies.clear() + + # Attempt to cancel the private job via the unauthenticated public endpoint + cancel_response = await client.post( + f"api/v1/build_public_tmp/{private_job_id}/cancel", + headers={"Content-Type": "application/json"}, + ) + + # Must be 404 — gate blocks private job from public cancel endpoint + assert cancel_response.status_code == codes.NOT_FOUND + assert cancel_response.json()["detail"] == "Job not found" diff --git a/src/backend/tests/unit/test_login.py b/src/backend/tests/unit/test_login.py index 03af2b93ec..1316a78496 100644 --- a/src/backend/tests/unit/test_login.py +++ b/src/backend/tests/unit/test_login.py @@ -1,7 +1,78 @@ +from datetime import datetime, timedelta, timezone + +import jwt import pytest +from langflow.services.database.models.auth import SSOUserProfile from langflow.services.database.models.user import User -from langflow.services.deps import get_auth_service, session_scope +from langflow.services.deps import get_auth_service, get_settings_service, session_scope from sqlalchemy.exc import IntegrityError +from sqlmodel import select + +_EXTERNAL_TEST_SECRET = "external-test-secret-with-enough-length" # noqa: S105 # pragma: allowlist secret +_EXTERNAL_AUTH_HEADER = "X-Langflow-External-Auth" +_EXTERNAL_AUTH_COOKIE = "external-session" + + +def _external_token(**claims) -> str: + payload = { + "exp": datetime.now(timezone.utc) + timedelta(minutes=5), + **claims, + } + return jwt.encode(payload, _EXTERNAL_TEST_SECRET, algorithm="HS256") + + +_EXTERNAL_FIELDS = ( + "EXTERNAL_AUTH_ENABLED", + "EXTERNAL_AUTH_PROVIDER", + "EXTERNAL_AUTH_TOKEN_HEADER", + "EXTERNAL_AUTH_TOKEN_COOKIE", + "EXTERNAL_AUTH_IDENTITY_RESOLVER", + "EXTERNAL_AUTH_TRUSTED_JWT_DECODE", + "EXTERNAL_AUTH_JWKS_URL", + "EXTERNAL_AUTH_ISSUER", + "EXTERNAL_AUTH_AUDIENCE", + "EXTERNAL_AUTH_ALGORITHMS", + "EXTERNAL_AUTH_SUBJECT_CLAIM", + "EXTERNAL_AUTH_USERNAME_CLAIM", + "EXTERNAL_AUTH_EMAIL_CLAIM", + "EXTERNAL_AUTH_NAME_CLAIM", + "EXTERNAL_AUTH_ACCESS_CEILING_ENABLED", + "EXTERNAL_AUTH_ACCESS_CLAIM", + "EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING", + "EXTERNAL_AUTH_DEFAULT_ACCESS_LEVEL", + "EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS", +) + + +@pytest.fixture +async def external_auth_settings(client): # noqa: ARG001 + auth_settings = get_settings_service().auth_settings + original = {field: getattr(auth_settings, field) for field in _EXTERNAL_FIELDS} + + auth_settings.EXTERNAL_AUTH_ENABLED = True + auth_settings.EXTERNAL_AUTH_PROVIDER = "test-provider" + auth_settings.EXTERNAL_AUTH_TOKEN_HEADER = _EXTERNAL_AUTH_HEADER + auth_settings.EXTERNAL_AUTH_TOKEN_COOKIE = None + auth_settings.EXTERNAL_AUTH_IDENTITY_RESOLVER = None + auth_settings.EXTERNAL_AUTH_TRUSTED_JWT_DECODE = True + auth_settings.EXTERNAL_AUTH_JWKS_URL = None + auth_settings.EXTERNAL_AUTH_ISSUER = None + auth_settings.EXTERNAL_AUTH_AUDIENCE = None + auth_settings.EXTERNAL_AUTH_ALGORITHMS = "RS256" + auth_settings.EXTERNAL_AUTH_SUBJECT_CLAIM = "sub" + auth_settings.EXTERNAL_AUTH_USERNAME_CLAIM = "preferred_username" + auth_settings.EXTERNAL_AUTH_EMAIL_CLAIM = "email" + auth_settings.EXTERNAL_AUTH_NAME_CLAIM = "name" + auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED = False + auth_settings.EXTERNAL_AUTH_ACCESS_CLAIM = None + auth_settings.EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING = None + auth_settings.EXTERNAL_AUTH_DEFAULT_ACCESS_LEVEL = "viewer" + auth_settings.EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS = True + + yield auth_settings + + for field, value in original.items(): + setattr(auth_settings, field, value) @pytest.fixture @@ -77,3 +148,171 @@ async def test_session_endpoint_no_api_key_in_response(client, logged_in_headers assert data["authenticated"] is True # Verify store_api_key field is None or not present in response assert data.get("store_api_key") is None + + +# --------------------------------------------------------------------------- +# External trusted auth + JIT provisioning +# --------------------------------------------------------------------------- + + +async def test_session_endpoint_jit_creates_user_for_external_header(client, external_auth_settings): # noqa: ARG001 + token = _external_token( + sub="subject-1", + preferred_username="external-user", + email="external@example.com", + name="External User", + ) + + response = await client.get("api/v1/session", headers={_EXTERNAL_AUTH_HEADER: f"Bearer {token}"}) + + assert response.status_code == 200 + data = response.json() + assert data["authenticated"] is True + assert data["user"]["username"] == "external-user" + assert data["user"]["is_active"] is True + + async with session_scope() as session: + statement = select(SSOUserProfile).where( + SSOUserProfile.sso_provider == "test-provider", + SSOUserProfile.sso_user_id == "subject-1", + ) + profiles = (await session.exec(statement)).all() + assert len(profiles) == 1 + assert str(profiles[0].user_id) == data["user"]["id"] + assert profiles[0].email == "external@example.com" + + # Second request reuses the same user. + second_response = await client.get("api/v1/session", headers={_EXTERNAL_AUTH_HEADER: f"Bearer {token}"}) + assert second_response.status_code == 200 + assert second_response.json()["user"]["id"] == data["user"]["id"] + + +async def test_session_endpoint_accepts_external_cookie_with_custom_claim_mapping(client, external_auth_settings): + external_auth_settings.EXTERNAL_AUTH_TOKEN_HEADER = "X-Unused-External-Auth" # noqa: S105 + external_auth_settings.EXTERNAL_AUTH_TOKEN_COOKIE = _EXTERNAL_AUTH_COOKIE + external_auth_settings.EXTERNAL_AUTH_USERNAME_CLAIM = "username" + external_auth_settings.EXTERNAL_AUTH_EMAIL_CLAIM = "username" + external_auth_settings.EXTERNAL_AUTH_NAME_CLAIM = "display_name" + + token = _external_token(sub="cookie-subject", username="person@example.com", display_name="Person Name") + + response = await client.get("api/v1/session", headers={"Cookie": f"{_EXTERNAL_AUTH_COOKIE}={token}"}) + + assert response.status_code == 200 + data = response.json() + assert data["authenticated"] is True + assert data["user"]["username"] == "person@example.com" + + async with session_scope() as session: + statement = select(SSOUserProfile).where( + SSOUserProfile.sso_provider == "test-provider", + SSOUserProfile.sso_user_id == "cookie-subject", + ) + profile = (await session.exec(statement)).first() + assert profile is not None + assert profile.email == "person@example.com" + + +async def test_session_endpoint_rejects_expired_external_token(client, external_auth_settings): # noqa: ARG001 + token = jwt.encode( + { + "sub": "expired-subject", + "preferred_username": "expired-user", + "exp": datetime.now(timezone.utc) - timedelta(minutes=1), + }, + _EXTERNAL_TEST_SECRET, + algorithm="HS256", + ) + + response = await client.get("api/v1/session", headers={_EXTERNAL_AUTH_HEADER: f"Bearer {token}"}) + + assert response.status_code == 200 + assert response.json()["authenticated"] is False + + +async def test_external_access_ceiling_filters_effective_permissions(client, external_auth_settings): + external_auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED = True + external_auth_settings.EXTERNAL_AUTH_ACCESS_CLAIM = "openrag_mode" + external_auth_settings.EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING = '{"view":"viewer","edit":"editor"}' + token = _external_token( + sub="viewer-subject", + preferred_username="viewer-user", + openrag_mode="view", + ) + + response = await client.post( + "api/v1/authz/me/permissions", + headers={_EXTERNAL_AUTH_HEADER: f"Bearer {token}"}, + json={ + "resource_type": "flow", + "resource_ids": ["00000000-0000-0000-0000-000000000001"], + "actions": ["read", "write", "delete"], + }, + ) + + assert response.status_code == 200 + permissions = response.json()["permissions"] + assert permissions["00000000-0000-0000-0000-000000000001"] == ["read"] + + +async def test_session_external_recovers_despite_stale_native_cookie(client, external_auth_settings): # noqa: ARG001 + """A stale/invalid native cookie must not shadow a valid external credential on /session (P1).""" + token = _external_token( + sub="recover-subject", + preferred_username="recover-user", + email="recover@example.com", + ) + + response = await client.get( + "api/v1/session", + headers={ + # A stale/invalid native cookie is present alongside the valid external header. + "Cookie": "access_token_lf=stale-invalid-native-token", + _EXTERNAL_AUTH_HEADER: f"Bearer {token}", + }, + ) + + assert response.status_code == 200 + data = response.json() + assert data["authenticated"] is True + assert data["user"]["username"] == "recover-user" + + +async def test_get_current_user_external_recovers_despite_stale_native_cookie(client, external_auth_settings): # noqa: ARG001 + """A get_current_user-protected endpoint accepts a valid external credential despite a stale cookie (P1).""" + token = _external_token( + sub="recover-subject-2", + preferred_username="recover-user-2", + email="recover2@example.com", + ) + + # /api/v1/users/whoami is guarded by CurrentActiveUser -> get_current_user. + response = await client.get( + "api/v1/users/whoami", + headers={ + "Cookie": "access_token_lf=stale-invalid-native-token", + _EXTERNAL_AUTH_HEADER: f"Bearer {token}", + }, + ) + + assert response.status_code == 200 + assert response.json()["username"] == "recover-user-2" + + +async def test_external_access_ceiling_blocks_api_key_creation(client, external_auth_settings): + external_auth_settings.EXTERNAL_AUTH_ACCESS_CEILING_ENABLED = True + external_auth_settings.EXTERNAL_AUTH_ACCESS_CLAIM = "openrag_mode" + token = _external_token( + sub="viewer-api-key-subject", + preferred_username="viewer-api-key-user", + openrag_mode="viewer", + ) + + response = await client.post( + "api/v1/api_key/", + headers={_EXTERNAL_AUTH_HEADER: f"Bearer {token}"}, + json={"name": "should-not-work"}, + ) + + assert response.status_code == 403 + assert "API key creation is disabled" in response.json()["detail"] diff --git a/src/backend/tests/unit/test_memory_bases.py b/src/backend/tests/unit/test_memory_bases.py index 2001e7788e..82bfa0bb07 100644 --- a/src/backend/tests/unit/test_memory_bases.py +++ b/src/backend/tests/unit/test_memory_bases.py @@ -451,6 +451,164 @@ class TestMemoryBaseCreateFlowOwnership: assert exc_info.value.status_code == 404 +class TestMemoryBaseGuardPassesRealKbIdentity: + """The ID-bearing guards must pass the REAL kb identity, not actor-as-owner. + + Previously the five F20 guards called ensure_knowledge_base_permission with + kb_user_id=current_user.id and no kb_id, so the owner-override path always + fired (a registered plugin's enforce never ran) and audit rows lacked the kb + id. The fix resolves the memory base first (via the owner-scoped service.get) + and passes kb_id / kb_user_id (the real owner) / kb_name so owner-override is + taken only for genuine owners. + """ + + @pytest.mark.asyncio + async def test_update_passes_real_kb_identity_to_guard(self): + from langflow.api.v1.memories import update_memory_base + from langflow.services.database.models.user.model import User + + owner_id = uuid.uuid4() + mb = _make_mb(user_id=owner_id) + actor = User(id=uuid.uuid4(), username="actor") + + mock_service = MagicMock() + mock_service.get = AsyncMock(return_value=mb) + mock_service.update = AsyncMock(return_value=mb) + captured = {} + + async def _capture_guard(_user, _act, **kwargs): + captured.update(kwargs) + + with ( + patch("langflow.api.v1.memories.get_memory_base_service", return_value=mock_service), + patch("langflow.api.v1.memories.ensure_knowledge_base_permission", _capture_guard), + ): + await update_memory_base( + memory_base_id=mb.id, + current_user=actor, + patch=MemoryBaseUpdate(threshold=99), + ) + + assert captured["kb_id"] == mb.id + assert captured["kb_user_id"] == owner_id, "guard must receive the real owner, not the actor" + assert captured["kb_name"] == mb.kb_name + + @pytest.mark.asyncio + async def test_delete_passes_real_kb_identity_to_guard(self): + from langflow.api.v1.memories import delete_memory_base + from langflow.services.database.models.user.model import User + + owner_id = uuid.uuid4() + mb = _make_mb(user_id=owner_id) + actor = User(id=uuid.uuid4(), username="actor") + + mock_service = MagicMock() + mock_service.get = AsyncMock(return_value=mb) + mock_service.delete = AsyncMock(return_value=True) + captured = {} + + async def _capture_guard(_user, _act, **kwargs): + captured.update(kwargs) + + with ( + patch("langflow.api.v1.memories.get_memory_base_service", return_value=mock_service), + patch("langflow.api.v1.memories.ensure_knowledge_base_permission", _capture_guard), + ): + await delete_memory_base(memory_base_id=mb.id, current_user=actor) + + assert captured["kb_id"] == mb.id + assert captured["kb_user_id"] == owner_id + assert captured["kb_name"] == mb.kb_name + + @pytest.mark.asyncio + async def test_flush_passes_real_kb_identity_to_guard(self): + from langflow.api.v1.memories import FlushRequest, flush_memory_base + from langflow.services.database.models.user.model import User + + owner_id = uuid.uuid4() + mb = _make_mb(user_id=owner_id) + actor = User(id=uuid.uuid4(), username="actor") + + mock_service = MagicMock() + mock_service.get = AsyncMock(return_value=mb) + mock_service.trigger_ingestion = AsyncMock(return_value="job-1") + captured = {} + + async def _capture_guard(_user, _act, **kwargs): + captured.update(kwargs) + + with ( + patch("langflow.api.v1.memories.get_memory_base_service", return_value=mock_service), + patch("langflow.api.v1.memories.ensure_knowledge_base_permission", _capture_guard), + ): + await flush_memory_base( + memory_base_id=mb.id, + current_user=actor, + body=FlushRequest(session_id="sess-1"), + ) + + assert captured["kb_id"] == mb.id + assert captured["kb_user_id"] == owner_id + assert captured["kb_name"] == mb.kb_name + + @pytest.mark.asyncio + async def test_regenerate_passes_real_kb_identity_to_guard(self): + from langflow.api.v1.memories import regenerate_memory_base + from langflow.services.database.models.user.model import User + + owner_id = uuid.uuid4() + mb = _make_mb(user_id=owner_id) + actor = User(id=uuid.uuid4(), username="actor") + + mock_service = MagicMock() + mock_service.get = AsyncMock(return_value=mb) + mock_service.regenerate = AsyncMock(return_value=["job-1"]) + captured = {} + + async def _capture_guard(_user, _act, **kwargs): + captured.update(kwargs) + + with ( + patch("langflow.api.v1.memories.get_memory_base_service", return_value=mock_service), + patch("langflow.api.v1.memories.ensure_knowledge_base_permission", _capture_guard), + ): + await regenerate_memory_base(memory_base_id=mb.id, current_user=actor) + + assert captured["kb_id"] == mb.id + assert captured["kb_user_id"] == owner_id + assert captured["kb_name"] == mb.kb_name + + @pytest.mark.asyncio + async def test_update_returns_404_when_memory_base_not_found(self): + """If the resolve lookup returns None, the handler 404s before the guard runs.""" + from fastapi import HTTPException + from langflow.api.v1.memories import update_memory_base + from langflow.services.database.models.user.model import User + + actor = User(id=uuid.uuid4(), username="actor") + mock_service = MagicMock() + mock_service.get = AsyncMock(return_value=None) + guard_called = False + + async def _guard(*_a, **_k): + nonlocal guard_called + guard_called = True + + with ( + patch("langflow.api.v1.memories.get_memory_base_service", return_value=mock_service), + patch("langflow.api.v1.memories.ensure_knowledge_base_permission", _guard), + pytest.raises(HTTPException) as exc_info, + ): + await update_memory_base( + memory_base_id=uuid.uuid4(), + current_user=actor, + patch=MemoryBaseUpdate(threshold=1), + ) + + assert exc_info.value.status_code == 404 + assert not guard_called, "guard must not run for a non-existent memory base" + + class TestMemoryBaseServiceConcurrency: """409 guard: only one active ingestion per (memory_base_id, session_id).""" @@ -1510,12 +1668,15 @@ class TestMemoriesAPIRouting: """trigger_ingestion raising RuntimeError should map to HTTP 409.""" from langflow.api.v1.memories import flush_memory_base - patched_service.trigger_ingestion = AsyncMock(side_effect=RuntimeError("already in progress")) - # We call the handler directly to test the error mapping mock_user = MagicMock() mock_user.id = uuid.uuid4() + # The guard now resolves the memory base first; let it pass through so the + # error mapping under test (trigger_ingestion -> 409) is exercised. + patched_service.get = AsyncMock(return_value=_make_mb(user_id=mock_user.id)) + patched_service.trigger_ingestion = AsyncMock(side_effect=RuntimeError("already in progress")) + from fastapi import HTTPException from langflow.api.v1.memories import FlushRequest @@ -1607,6 +1768,7 @@ class TestMemoriesAPIHandlers: from langflow.services.memory_base.service import PreprocessingValidationError svc = MagicMock() + svc.get = AsyncMock(return_value=_make_mb(user_id=mock_user.id)) svc.update = AsyncMock(side_effect=PreprocessingValidationError("No API key found for provider 'OpenAI'")) with ( patch("langflow.api.v1.memories.get_memory_base_service", return_value=svc), @@ -1664,6 +1826,7 @@ class TestMemoriesAPIHandlers: mb = _make_mb(user_id=mock_user.id, threshold=99) svc = MagicMock() + svc.get = AsyncMock(return_value=mb) svc.update = AsyncMock(return_value=mb) with patch("langflow.api.v1.memories.get_memory_base_service", return_value=svc): result = await update_memory_base( @@ -1680,6 +1843,7 @@ class TestMemoriesAPIHandlers: from langflow.api.v1.memories import update_memory_base svc = MagicMock() + svc.get = AsyncMock(return_value=_make_mb(user_id=mock_user.id)) svc.update = AsyncMock(return_value=None) with ( patch("langflow.api.v1.memories.get_memory_base_service", return_value=svc), @@ -1702,6 +1866,7 @@ class TestMemoriesAPIHandlers: from langflow.api.v1.memories import delete_memory_base svc = MagicMock() + svc.get = AsyncMock(return_value=_make_mb(user_id=mock_user.id)) svc.delete = AsyncMock(return_value=True) with patch("langflow.api.v1.memories.get_memory_base_service", return_value=svc): result = await delete_memory_base(memory_base_id=uuid.uuid4(), current_user=mock_user) @@ -1714,6 +1879,7 @@ class TestMemoriesAPIHandlers: from langflow.api.v1.memories import delete_memory_base svc = MagicMock() + svc.get = AsyncMock(return_value=_make_mb(user_id=mock_user.id)) svc.delete = AsyncMock(return_value=False) with ( patch("langflow.api.v1.memories.get_memory_base_service", return_value=svc), @@ -1734,6 +1900,7 @@ class TestMemoriesAPIHandlers: job_id = str(uuid.uuid4()) svc = MagicMock() + svc.get = AsyncMock(return_value=_make_mb(user_id=mock_user.id)) svc.trigger_ingestion = AsyncMock(return_value=job_id) with patch("langflow.api.v1.memories.get_memory_base_service", return_value=svc): result = await flush_memory_base( @@ -1750,6 +1917,7 @@ class TestMemoriesAPIHandlers: from langflow.api.v1.memories import FlushRequest, flush_memory_base svc = MagicMock() + svc.get = AsyncMock(return_value=_make_mb(user_id=mock_user.id)) svc.trigger_ingestion = AsyncMock(side_effect=ValueError("memory base not found")) with ( patch("langflow.api.v1.memories.get_memory_base_service", return_value=svc), @@ -1770,6 +1938,7 @@ class TestMemoriesAPIHandlers: from langflow.services.jobs import DuplicateJobError svc = MagicMock() + svc.get = AsyncMock(return_value=_make_mb(user_id=mock_user.id)) svc.trigger_ingestion = AsyncMock(side_effect=DuplicateJobError("already running")) with ( patch("langflow.api.v1.memories.get_memory_base_service", return_value=svc), @@ -1835,6 +2004,7 @@ class TestMemoriesAPIHandlers: job_ids = [str(uuid.uuid4()), str(uuid.uuid4())] svc = MagicMock() + svc.get = AsyncMock(return_value=_make_mb(user_id=mock_user.id)) svc.regenerate = AsyncMock(return_value=job_ids) with patch("langflow.api.v1.memories.get_memory_base_service", return_value=svc): result = await regenerate_memory_base(memory_base_id=uuid.uuid4(), current_user=mock_user) @@ -1847,6 +2017,7 @@ class TestMemoriesAPIHandlers: from langflow.api.v1.memories import regenerate_memory_base svc = MagicMock() + svc.get = AsyncMock(return_value=_make_mb(user_id=mock_user.id)) svc.regenerate = AsyncMock(side_effect=ValueError("not found")) with ( patch("langflow.api.v1.memories.get_memory_base_service", return_value=svc), diff --git a/src/backend/tests/unit/test_redis_job_queue_service.py b/src/backend/tests/unit/test_redis_job_queue_service.py index 403c27202c..83180a7ef7 100644 --- a/src/backend/tests/unit/test_redis_job_queue_service.py +++ b/src/backend/tests/unit/test_redis_job_queue_service.py @@ -2408,6 +2408,200 @@ async def test_polling_watchdog_runs_when_cancel_channel_disabled(): await fake_client.aclose() +# ── Public job registry — Redis-specific tests ─────────────────────────────── +# Complement the base-class unit tests in test_chat_endpoint.py. +# These tests verify the Redis-specific paths: cross-worker fallback (#6) +# and cleanup deleting the Redis key (#7). + + +async def test_redis_public_job_cross_worker_fallback(): + """is_public_job_async returns True for Worker B even when its in-memory set is empty. + + Why: Worker A registers the job (writes local memory + Redis key via background task). + Worker B has no local memory entry — it must fall back to Redis. + This is the multi-worker correctness guarantee of RedisJobQueueService. + """ + fake_client = fakeredis_aio.FakeRedis() + # Worker A — registers the public job + svc_a, _ = await _make_service(shared_client=fake_client, cancel_channel_enabled=False) + # Worker B — shares same Redis, but has empty local memory + svc_b, _ = await _make_service(shared_client=fake_client, cancel_channel_enabled=False) + + try: + job_id = str(uuid.uuid4()) + # register_public_job awaits the Redis write directly (no background task + # to drain) — see Why comment on RedisJobQueueService.register_public_job. + await svc_a.register_public_job(job_id) + + # Worker B must have no in-memory entry (no shared memory between workers) + assert not svc_b.is_public_job(job_id), "Worker B must have no in-memory entry" + + # Why: is_public_job_async on Worker B must hit Redis fallback and return True + assert await svc_b.is_public_job_async(job_id) is True + finally: + await _stop_service(svc_a) + await _stop_service(svc_b) + await fake_client.aclose() + + +async def test_redis_cleanup_removes_public_job_key(): + """cleanup_job deletes the public_job Redis key so the job_id cannot be reused cross-worker. + + Why: after cleanup the in-memory discard is proven by the base-class test in + test_chat_endpoint.py. This test proves the Redis key (cross-worker marker) is + also removed. A missing delete would let a cross-worker is_public_job_async + return True for a finished/evicted job. + """ + svc, fake_client = await _make_service(cancel_channel_enabled=False) + + try: + job_id = str(uuid.uuid4()) + # register_public_job awaits the Redis write directly (no background task + # to drain) — see Why comment on RedisJobQueueService.register_public_job. + await svc.register_public_job(job_id) + + # Confirm the key exists in Redis before cleanup + pub_key = svc._public_job_key(job_id) + assert await fake_client.exists(pub_key), "public_job Redis key must exist after registration" + + # Seed a minimal queue entry so cleanup_job doesn't early-return + svc._queues[job_id] = (asyncio.Queue(), None, None, None) # type: ignore[arg-type] + await svc.cleanup_job(job_id) + + # Drain any background tasks spawned by cleanup + for task in list(svc._background_tasks): + with contextlib.suppress(Exception): + await task + + # Why: if cleanup_job's Redis DEL call ever drops public_job_key, this catches it + assert not await fake_client.exists(pub_key), "public_job Redis key must be deleted after cleanup" + # In-memory also cleared + assert svc.is_public_job(job_id) is False + finally: + await _stop_service(svc) + await fake_client.aclose() + + +@pytest.mark.asyncio +async def test_register_public_job_raises_backend_unavailable_when_marker_write_fails() -> None: + """register_public_job must surface (not swallow) a failed Redis marker write. + + Swallowing the failure would let build_public_tmp return a job_id that only + this worker recognizes — on a multi-worker deployment the public events/cancel + endpoints would 404 it on every other worker. With a Redis client configured, + the failure must raise JobQueueBackendUnavailableError. + """ + from langflow.services.job_queue.service import JobQueueBackendUnavailableError + + service = RedisJobQueueService() + service._client = _PingFailRedis() + with pytest.raises(JobQueueBackendUnavailableError): + await service.register_public_job(str(uuid.uuid4())) + + +@pytest.mark.asyncio +async def test_register_public_job_is_noop_success_for_in_memory_backend() -> None: + """The in-memory base class stays a pure no-op success (no shared marker to persist). + + Single-worker deployments have no Redis client; there is no shared marker, so + register_public_job must not raise and must record the job locally. + """ + service = JobQueueService() + job_id = str(uuid.uuid4()) + await service.register_public_job(job_id) # must not raise + assert service.is_public_job(job_id) is True + + +@pytest.mark.asyncio +async def test_build_public_tmp_returns_503_when_public_marker_persist_fails(monkeypatch) -> None: + """build_public_tmp returns 503 (not an un-shareable job_id) when the marker write fails. + + The build task is started before the public marker is persisted. If the shared + backend write fails, the handler must cancel the just-started build and surface + a clean 503 instead of returning a job_id that other workers cannot resolve. + """ + from fastapi import HTTPException + from langflow.api.v1 import chat as chat_module + + service, fake_client = await _make_service(cancel_channel_enabled=False) + service._POST_CANCEL_CLEANUP_TIMEOUT_S = 0.5 + try: + flow_id = uuid.uuid4() + new_flow_id = uuid.uuid4() + job_id = str(uuid.uuid4()) + service.create_queue(job_id) + + started = asyncio.Event() + cancelled = asyncio.Event() + + async def _build() -> None: + started.set() + try: + await asyncio.sleep(30) + except asyncio.CancelledError: + cancelled.set() + raise + + class _Owner: + id = uuid.uuid4() + + async def _fake_verify_public_flow_and_get_user(**_kwargs): + return _Owner(), new_flow_id + + async def _fake_start_flow_build(**_kwargs): + service.start_job(job_id, _build()) + await asyncio.wait_for(started.wait(), timeout=5) + return job_id + + class _FakeFlow: + data = None + + class _FakeSession: + async def get(self, *_args, **_kwargs): + return _FakeFlow() + + @contextlib.asynccontextmanager + async def _fake_session_scope(): + yield _FakeSession() + + class _FakeSettingsService: + class auth_settings: # noqa: N801 + AUTO_LOGIN = True + + monkeypatch.setattr(chat_module, "verify_public_flow_and_get_user", _fake_verify_public_flow_and_get_user) + monkeypatch.setattr(chat_module, "start_flow_build", _fake_start_flow_build) + monkeypatch.setattr(chat_module, "session_scope", _fake_session_scope) + monkeypatch.setattr(chat_module, "get_settings_service", lambda: _FakeSettingsService()) + + # Redis marker write fails: register_public_job raises JobQueueBackendUnavailableError. + service._client = _PingFailRedis() + + class _FakeRequest: + cookies: dict[str, str] = {"client_id": "test-client"} + + with pytest.raises(HTTPException) as exc_info: + await chat_module.build_public_tmp( + background_tasks=None, + flow_id=flow_id, + inputs=None, + files=None, + stop_component_id=None, + start_component_id=None, + log_builds=False, + flow_name=None, + request=_FakeRequest(), + queue_service=service, + authenticated_user=None, + event_delivery=EventDeliveryType.POLLING, + ) + assert exc_info.value.status_code == 503 + # The just-started build must have been cancelled, not left running unreachable. + await asyncio.wait_for(cancelled.wait(), timeout=5) + finally: + await _stop_service(service) + await fake_client.aclose() + + # --------------------------------------------------------------------------- # Startup connectivity probe + runtime backstop (LE-1396) # --------------------------------------------------------------------------- diff --git a/src/frontend/src/CustomNodes/NoteNode/components/select-items.tsx b/src/frontend/src/CustomNodes/NoteNode/components/select-items.tsx index 183a6c54d6..f6065ac3a8 100644 --- a/src/frontend/src/CustomNodes/NoteNode/components/select-items.tsx +++ b/src/frontend/src/CustomNodes/NoteNode/components/select-items.tsx @@ -9,7 +9,13 @@ import ToolbarSelectItem from "@/pages/FlowPage/components/nodeToolbarComponent/ import type { NoteDataType } from "@/types/flow"; export const SelectItems = memo( - ({ shortcuts, data }: { shortcuts: any[]; data: NoteDataType }) => { + ({ + shortcuts, + data, + }: { + shortcuts: Array<{ name: string; display_name: string; shortcut: string }>; + data: NoteDataType; + }) => { const { t } = useTranslation(); return ( diff --git a/src/frontend/src/components/core/assistantPanel/components/__tests__/assistant-file-card.test.tsx b/src/frontend/src/components/core/assistantPanel/components/__tests__/assistant-file-card.test.tsx index 79e4f87c05..c0b86a9cfe 100644 --- a/src/frontend/src/components/core/assistantPanel/components/__tests__/assistant-file-card.test.tsx +++ b/src/frontend/src/components/core/assistantPanel/components/__tests__/assistant-file-card.test.tsx @@ -9,9 +9,8 @@ import "@testing-library/jest-dom"; import { fireEvent, render, screen } from "@testing-library/react"; - -import { AssistantFileCard } from "../assistant-file-card"; import type { WrittenFile } from "../../assistant-panel.types"; +import { AssistantFileCard } from "../assistant-file-card"; function makeFile(overrides: Partial = {}): WrittenFile { return { @@ -25,7 +24,9 @@ function makeFile(overrides: Partial = {}): WrittenFile { } beforeEach(() => { + // biome-ignore lint/suspicious/noExplicitAny: stubbing global URL method in test (global.URL as any).createObjectURL = jest.fn().mockReturnValue("blob:mock"); + // biome-ignore lint/suspicious/noExplicitAny: stubbing global URL method in test (global.URL as any).revokeObjectURL = jest.fn(); }); diff --git a/src/frontend/src/components/core/assistantPanel/components/__tests__/assistant-proposal-integration.test.tsx b/src/frontend/src/components/core/assistantPanel/components/__tests__/assistant-proposal-integration.test.tsx index e920dc1816..348512fb85 100644 --- a/src/frontend/src/components/core/assistantPanel/components/__tests__/assistant-proposal-integration.test.tsx +++ b/src/frontend/src/components/core/assistantPanel/components/__tests__/assistant-proposal-integration.test.tsx @@ -131,8 +131,8 @@ jest.mock("short-unique-id", () => { }; }); -import { AssistantMessageBody } from "../assistant-message-body"; import { useAssistantChat } from "../../hooks/use-assistant-chat"; +import { AssistantMessageBody } from "../assistant-message-body"; const TEST_MODEL = { id: "openai/gpt-4o-mini", diff --git a/src/frontend/src/components/core/assistantPanel/components/assistant-file-card.tsx b/src/frontend/src/components/core/assistantPanel/components/assistant-file-card.tsx index 4a4c74874a..03049ec5d9 100644 --- a/src/frontend/src/components/core/assistantPanel/components/assistant-file-card.tsx +++ b/src/frontend/src/components/core/assistantPanel/components/assistant-file-card.tsx @@ -15,10 +15,10 @@ */ import { - FileText, - FilePen, Download as DownloadIcon, ExternalLink, + FilePen, + FileText, } from "lucide-react"; import { useCallback } from "react"; diff --git a/src/frontend/src/components/core/assistantPanel/hooks/__tests__/use-assistant-chat-file-events.test.ts b/src/frontend/src/components/core/assistantPanel/hooks/__tests__/use-assistant-chat-file-events.test.ts index a4f004945c..e7e783c0a0 100644 --- a/src/frontend/src/components/core/assistantPanel/hooks/__tests__/use-assistant-chat-file-events.test.ts +++ b/src/frontend/src/components/core/assistantPanel/hooks/__tests__/use-assistant-chat-file-events.test.ts @@ -21,7 +21,9 @@ jest.mock("@xyflow/react", () => ({ })); // Capture the StreamCallbacks the hook passes in so we can drive them in tests. +// biome-ignore lint/suspicious/noExplicitAny: captures the hook's StreamCallbacks (interface not exported) let capturedCallbacks: any = null; +// biome-ignore lint/suspicious/noExplicitAny: mock signature mirrors postAssistStream's untyped callbacks arg const mockPostAssistStream = jest.fn(async (_req: unknown, callbacks: any) => { capturedCallbacks = callbacks; }); diff --git a/src/frontend/src/components/core/assistantPanel/hooks/use-session-history.ts b/src/frontend/src/components/core/assistantPanel/hooks/use-session-history.ts index a8186e842f..42b1372736 100644 --- a/src/frontend/src/components/core/assistantPanel/hooks/use-session-history.ts +++ b/src/frontend/src/components/core/assistantPanel/hooks/use-session-history.ts @@ -6,8 +6,8 @@ import { useCallback, useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; import { ASSISTANT_MAX_SESSIONS, - ASSISTANT_SESSIONS_STORAGE_KEY, ASSISTANT_SESSION_PREVIEW_LENGTH, + ASSISTANT_SESSIONS_STORAGE_KEY, } from "../assistant-panel.constants"; import type { AssistantMessage, diff --git a/src/frontend/src/controllers/API/queries/agentic/index.ts b/src/frontend/src/controllers/API/queries/agentic/index.ts index 59ee823664..58598ac9c6 100644 --- a/src/frontend/src/controllers/API/queries/agentic/index.ts +++ b/src/frontend/src/controllers/API/queries/agentic/index.ts @@ -1,4 +1,3 @@ -export { postAssistStream } from "./use-post-assist-stream"; export type { AgenticAssistRequest, AgenticCancelledEvent, @@ -15,3 +14,4 @@ export type { AgenticTokenEvent, FlowAction, } from "./types"; +export { postAssistStream } from "./use-post-assist-stream"; diff --git a/src/frontend/src/modals/IOModal/components/chatView/chatInput/components/voice-assistant/hooks/use-handle-websocket-message.ts b/src/frontend/src/modals/IOModal/components/chatView/chatInput/components/voice-assistant/hooks/use-handle-websocket-message.ts index 4ba5b9065f..a7af1f90cd 100644 --- a/src/frontend/src/modals/IOModal/components/chatView/chatInput/components/voice-assistant/hooks/use-handle-websocket-message.ts +++ b/src/frontend/src/modals/IOModal/components/chatView/chatInput/components/voice-assistant/hooks/use-handle-websocket-message.ts @@ -1,5 +1,6 @@ -import i18n from "@/i18n"; import { BuildStatus } from "@/constants/enums"; +import i18n from "@/i18n"; +import type { VertexBuildTypeAPI } from "@/types/api"; import { base64ToFloat32Array } from "../helpers/utils"; export const useHandleWebsocketMessage = ( @@ -17,7 +18,7 @@ export const useHandleWebsocketMessage = ( setStatus: React.Dispatch>, messagesStore, setEdges, - addDataToFlowPool: (data: any, nodeId: string) => void, + addDataToFlowPool: (data: VertexBuildTypeAPI, nodeId: string) => void, updateEdgesRunningByNodes: (nodeIds: string[], isRunning: boolean) => void, updateBuildStatus: (nodeIds: string[], status: BuildStatus) => void, hasOpenAIAPIKey: boolean, diff --git a/src/frontend/src/modals/textModal/components/textEditorArea/index.tsx b/src/frontend/src/modals/textModal/components/textEditorArea/index.tsx index 243d6a3db5..65d0607cab 100644 --- a/src/frontend/src/modals/textModal/components/textEditorArea/index.tsx +++ b/src/frontend/src/modals/textModal/components/textEditorArea/index.tsx @@ -10,6 +10,7 @@ const TextEditorArea = ({ }: { left: boolean | undefined; resizable?: boolean; + // biome-ignore lint/suspicious/noExplicitAny: polymorphic value (string | object | ...) rendered as text value: any; onChange?: (string) => void; readonly: boolean; diff --git a/src/frontend/src/pages/FlowPage/components/UpdateAllComponents/index.tsx b/src/frontend/src/pages/FlowPage/components/UpdateAllComponents/index.tsx index 41675227a2..141aea80a2 100644 --- a/src/frontend/src/pages/FlowPage/components/UpdateAllComponents/index.tsx +++ b/src/frontend/src/pages/FlowPage/components/UpdateAllComponents/index.tsx @@ -1,6 +1,6 @@ import { useUpdateNodeInternals } from "@xyflow/react"; -import { cloneDeep } from "lodash"; import { AnimatePresence, motion } from "framer-motion"; +import { cloneDeep } from "lodash"; import { useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { processNodeAdvancedFields } from "@/CustomNodes/helpers/process-node-advanced-fields"; @@ -12,8 +12,8 @@ import { usePostValidateComponentCode } from "@/controllers/API/queries/nodes/us import UpdateComponentModal from "@/modals/updateComponentModal"; import useAlertStore from "@/stores/alertStore"; import useFlowStore, { - registerNodeUpdate, completeNodeUpdate, + registerNodeUpdate, } from "@/stores/flowStore"; import useFlowsManagerStore from "@/stores/flowsManagerStore"; import { useTypesStore } from "@/stores/typesStore"; diff --git a/src/frontend/src/pages/FlowPage/components/flowSidebarComponent/components/McpSidebarGroup.tsx b/src/frontend/src/pages/FlowPage/components/flowSidebarComponent/components/McpSidebarGroup.tsx index e47a4fe50c..1d6133c796 100644 --- a/src/frontend/src/pages/FlowPage/components/flowSidebarComponent/components/McpSidebarGroup.tsx +++ b/src/frontend/src/pages/FlowPage/components/flowSidebarComponent/components/McpSidebarGroup.tsx @@ -18,10 +18,11 @@ import { SearchConfigTrigger } from "./searchConfigTrigger"; import SidebarDraggableComponent from "./sidebarDraggableComponent"; type McpSidebarGroupProps = { + // biome-ignore lint/suspicious/noExplicitAny: MCP component shape extends APIClassType with server-specific fields mcpComponents?: any[]; - nodeColors: any; + nodeColors: Record; onDragStart: ( - event: React.DragEvent, + event: React.DragEvent, data: { type: string; node?: APIClassType }, ) => void; openCategories: string[]; diff --git a/src/frontend/src/pages/FlowPage/index.tsx b/src/frontend/src/pages/FlowPage/index.tsx index c4477f4616..5a0c6291c6 100644 --- a/src/frontend/src/pages/FlowPage/index.tsx +++ b/src/frontend/src/pages/FlowPage/index.tsx @@ -20,9 +20,9 @@ import { useWebhookEvents } from "@/hooks/use-webhook-events"; import { SaveChangesModal } from "@/modals/saveChangesModal"; import useAlertStore from "@/stores/alertStore"; import useAssistantManagerStore from "@/stores/assistantManagerStore"; +import useFlowBuilderWelcomeStore from "@/stores/flowBuilderWelcomeStore"; import { usePlaygroundStore } from "@/stores/playgroundStore"; import { useShortcutsStore } from "@/stores/shortcuts"; -import useFlowBuilderWelcomeStore from "@/stores/flowBuilderWelcomeStore"; import { useTypesStore } from "@/stores/typesStore"; import { customStringify } from "@/utils/reactflowUtils"; import { cn } from "@/utils/utils"; diff --git a/src/frontend/src/shared/components/textOutputView/index.tsx b/src/frontend/src/shared/components/textOutputView/index.tsx index 0cd5abb60d..07fdf63549 100644 --- a/src/frontend/src/shared/components/textOutputView/index.tsx +++ b/src/frontend/src/shared/components/textOutputView/index.tsx @@ -6,6 +6,7 @@ const TextOutputView = ({ value, }: { left: boolean | undefined; + // biome-ignore lint/suspicious/noExplicitAny: polymorphic output value (string | object | ...) rendered as text value: any; }) => { const { t } = useTranslation(); diff --git a/src/frontend/src/utils/utils.ts b/src/frontend/src/utils/utils.ts index 53bc472ec4..be0692de1d 100644 --- a/src/frontend/src/utils/utils.ts +++ b/src/frontend/src/utils/utils.ts @@ -380,7 +380,7 @@ export function getSetFromObject(obj: object, key?: string): Set { return set; } -export function freezeObject(obj: any) { +export function freezeObject(obj: T): T { if (!obj) return obj; return JSON.parse(JSON.stringify(obj)); } @@ -395,8 +395,8 @@ export function extractColumnsFromRows( rows: object[], mode: "intersection" | "union", excludeColumns?: Array, -): ColDef[] { - const columnsKeys: { [key: string]: ColDef | ColGroupDef } = {}; +): ColDef[] { + const columnsKeys: { [key: string]: ColDef | ColGroupDef } = {}; if (rows.length === 0) { return []; } @@ -451,9 +451,9 @@ export function isThereModal(): boolean { return modal.length > 0; } -export function messagesSorter(a: any, b: any) { - const indexA = MESSAGES_TABLE_ORDER.indexOf(a.field); - const indexB = MESSAGES_TABLE_ORDER.indexOf(b.field); +export function messagesSorter(a: { field?: string }, b: { field?: string }) { + const indexA = a.field ? MESSAGES_TABLE_ORDER.indexOf(a.field) : -1; + const indexB = b.field ? MESSAGES_TABLE_ORDER.indexOf(b.field) : -1; // If the field is not in the MESSAGES_TABLE_ORDER, we can place it at the end. const orderA = indexA === -1 ? MESSAGES_TABLE_ORDER.length : indexA; @@ -542,7 +542,7 @@ export function brokenEdgeMessage({ source.outputDisplayName ? " | " + source.outputDisplayName : "" } -> ${target.displayName}${target.field ? " | " + target.field : ""}`; } -export function FormatColumns(columns: ColumnField[]): ColDef[] { +export function FormatColumns(columns: ColumnField[]): ColDef[] { if (!columns) return []; const basic_types = new Set(["date", "number"]); const colDefs = columns.map((col) => { @@ -568,10 +568,10 @@ export function FormatColumns(columns: ColumnField[]): ColDef[] { newValue, context.field_parsers[colDef.field ?? ""], ); - } catch (error: any) { + } catch (error) { useAlertStore.getState().setErrorData({ title: i18n.t("errors.errorParsingString"), - list: [String(error.message ?? error)], + list: [String(error instanceof Error ? error.message : error)], }); return oldValue; } @@ -659,7 +659,7 @@ export function generateBackendColumnsFromValue( // Determine the formatter based on the sample value if (rows[0] && rows[0][column.field ?? ""]) { - const value = rows[0][column.field ?? ""] as any; + const value = rows[0][column.field ?? ""] as unknown; if (typeof value === "string") { if (isTimeStampString(value)) { newColumn.formatter = FormatterType.date; @@ -805,7 +805,7 @@ export const isStringArray = (value: unknown): value is string[] => { export const stringToBool = (str) => (str === "false" ? false : true); // Filter out null/undefined options -export const filterNullOptions = (opts: any[]): any[] => { +export const filterNullOptions = (opts: T[]): T[] => { return opts.filter((opt) => opt !== null && opt !== undefined); }; diff --git a/src/lfx/src/lfx/_assets/component_index.json b/src/lfx/src/lfx/_assets/component_index.json index 65cc46b042..e9e22a2228 100644 --- a/src/lfx/src/lfx/_assets/component_index.json +++ b/src/lfx/src/lfx/_assets/component_index.json @@ -57471,7 +57471,7 @@ "icon": "Globe", "legacy": false, "metadata": { - "code_hash": "1a052ebb9519", + "code_hash": "716a753c256e", "dependencies": { "dependencies": [ { @@ -57573,7 +57573,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import json\nimport re\nimport tempfile\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom typing import Any\nfrom urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse\n\nimport aiofiles\nimport aiofiles.os as aiofiles_os\nimport httpx\nimport validators\n\nfrom lfx.base.curl.parse import parse_context\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs.inputs import TabInput\nfrom lfx.io import (\n BoolInput,\n DataInput,\n DropdownInput,\n IntInput,\n MessageTextInput,\n MultilineInput,\n Output,\n TableInput,\n)\nfrom lfx.schema.data import Data\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.utils.component_utils import set_current_fields, set_field_advanced, set_field_display\n\n# SSRF Protection imports - for preventing Server-Side Request Forgery attacks\nfrom lfx.utils.ssrf_protection import (\n SSRFProtectionError,\n is_ssrf_protection_enabled,\n validate_and_resolve_url,\n)\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Define fields for each mode\nMODE_FIELDS = {\n \"URL\": [\n \"url_input\",\n \"method\",\n ],\n \"cURL\": [\"curl_input\"],\n}\n\n# Fields that should always be visible\nDEFAULT_FIELDS = [\"mode\"]\n\n# HTTP redirect status codes (RFC 9110).\nHTTP_MOVED_PERMANENTLY = 301\nHTTP_FOUND = 302\nHTTP_SEE_OTHER = 303\nHTTP_TEMPORARY_REDIRECT = 307\nHTTP_PERMANENT_REDIRECT = 308\n\n# Maximum number of redirects to follow when re-validating each hop (matches httpx's default).\nMAX_REDIRECTS = 20\n\n# HTTP status codes that represent a redirect carrying a Location header.\nREDIRECT_STATUS_CODES = frozenset(\n {\n HTTP_MOVED_PERMANENTLY,\n HTTP_FOUND,\n HTTP_SEE_OTHER,\n HTTP_TEMPORARY_REDIRECT,\n HTTP_PERMANENT_REDIRECT,\n }\n)\n\n\nclass APIRequestComponent(Component):\n display_name = \"API Request\"\n description = \"Make HTTP requests using URL or cURL commands.\"\n documentation: str = \"https://docs.langflow.org/api-request\"\n icon = \"Globe\"\n name = \"APIRequest\"\n\n inputs = [\n MessageTextInput(\n name=\"url_input\",\n display_name=\"URL\",\n info=\"Enter the URL for the request.\",\n advanced=False,\n tool_mode=True,\n ),\n MultilineInput(\n name=\"curl_input\",\n display_name=\"cURL\",\n info=(\n \"Paste a curl command to populate the fields. \"\n \"This will fill in the dictionary fields for headers and body.\"\n ),\n real_time_refresh=True,\n tool_mode=True,\n advanced=True,\n show=False,\n ),\n DropdownInput(\n name=\"method\",\n display_name=\"Method\",\n options=[\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"],\n value=\"GET\",\n info=\"The HTTP method to use.\",\n real_time_refresh=True,\n ),\n TabInput(\n name=\"mode\",\n display_name=\"Mode\",\n options=[\"URL\", \"cURL\"],\n value=\"URL\",\n info=\"Enable cURL mode to populate fields from a cURL command.\",\n real_time_refresh=True,\n ),\n DataInput(\n name=\"query_params\",\n display_name=\"Query Parameters\",\n info=\"The query parameters to append to the URL.\",\n advanced=True,\n ),\n TableInput(\n name=\"body\",\n display_name=\"Body\",\n info=\"The body to send with the request as a dictionary (for POST, PATCH, PUT).\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Parameter name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"description\": \"Parameter value\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n advanced=True,\n real_time_refresh=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": \"Langflow/1.0\"}],\n advanced=True,\n input_types=[\"Data\", \"JSON\"],\n real_time_refresh=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n value=30,\n info=\"The timeout to use for the request.\",\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n value=False,\n info=(\n \"Whether to follow HTTP redirects. \"\n \"WARNING: Enabling redirects may allow SSRF bypass attacks where a public URL \"\n \"redirects to internal resources. Only enable if you trust the target server. \"\n \"See OWASP SSRF Prevention Cheat Sheet for details.\"\n ),\n advanced=True,\n ),\n BoolInput(\n name=\"save_to_file\",\n display_name=\"Save to File\",\n value=False,\n info=\"Save the API response to a temporary file\",\n advanced=True,\n ),\n BoolInput(\n name=\"include_httpx_metadata\",\n display_name=\"Include HTTPx Metadata\",\n value=False,\n info=(\n \"Include properties such as headers, status_code, response_headers, \"\n \"and redirection_history in the output.\"\n ),\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"API Response\", name=\"data\", method=\"make_api_request\"),\n ]\n\n def _parse_json_value(self, value: Any) -> Any:\n \"\"\"Parse a value that might be a JSON string.\"\"\"\n if not isinstance(value, str):\n return value\n\n try:\n parsed = json.loads(value)\n except json.JSONDecodeError:\n return value\n else:\n return parsed\n\n def _process_body(self, body: Any) -> dict:\n \"\"\"Process the body input into a valid dictionary.\"\"\"\n if body is None:\n return {}\n if hasattr(body, \"data\"):\n body = body.data\n if isinstance(body, dict):\n return self._process_dict_body(body)\n if isinstance(body, str):\n return self._process_string_body(body)\n if isinstance(body, list):\n return self._process_list_body(body)\n return {}\n\n def _process_dict_body(self, body: dict) -> dict:\n \"\"\"Process dictionary body by parsing JSON values.\"\"\"\n return {k: self._parse_json_value(v) for k, v in body.items()}\n\n def _process_string_body(self, body: str) -> dict:\n \"\"\"Process string body by attempting JSON parse.\"\"\"\n try:\n return self._process_body(json.loads(body))\n except json.JSONDecodeError:\n return {\"data\": body}\n\n def _process_list_body(self, body: list) -> dict:\n \"\"\"Process list body by converting to key-value dictionary.\"\"\"\n processed_dict = {}\n try:\n for item in body:\n # Unwrap Data objects\n current_item = item\n if hasattr(item, \"data\"):\n unwrapped_data = item.data\n # If the unwrapped data is a dict but not key-value format, use it directly\n if isinstance(unwrapped_data, dict) and not self._is_valid_key_value_item(unwrapped_data):\n return unwrapped_data\n current_item = unwrapped_data\n if not self._is_valid_key_value_item(current_item):\n continue\n key = current_item[\"key\"]\n value = self._parse_json_value(current_item[\"value\"])\n processed_dict[key] = value\n except (KeyError, TypeError, ValueError) as e:\n self.log(f\"Failed to process body list: {e}\")\n return {}\n return processed_dict\n\n def _is_valid_key_value_item(self, item: Any) -> bool:\n \"\"\"Check if an item is a valid key-value dictionary.\"\"\"\n return isinstance(item, dict) and \"key\" in item and \"value\" in item\n\n def parse_curl(self, curl: str, build_config: dotdict) -> dotdict:\n \"\"\"Parse a cURL command and update build configuration.\"\"\"\n try:\n parsed = parse_context(curl)\n\n # Update basic configuration\n url = parsed.url\n # Normalize URL before setting it\n url = self._normalize_url(url)\n\n build_config[\"url_input\"][\"value\"] = url\n build_config[\"method\"][\"value\"] = parsed.method.upper()\n\n # Process headers\n headers_list = [{\"key\": k, \"value\": v} for k, v in parsed.headers.items()]\n build_config[\"headers\"][\"value\"] = headers_list\n\n # Process body data\n if not parsed.data:\n build_config[\"body\"][\"value\"] = []\n elif parsed.data:\n try:\n json_data = json.loads(parsed.data)\n if isinstance(json_data, dict):\n body_list = [\n {\"key\": k, \"value\": json.dumps(v) if isinstance(v, dict | list) else str(v)}\n for k, v in json_data.items()\n ]\n build_config[\"body\"][\"value\"] = body_list\n else:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": json.dumps(json_data)}]\n except json.JSONDecodeError:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": parsed.data}]\n\n except Exception as exc:\n msg = f\"Error parsing curl: {exc}\"\n self.log(msg)\n raise ValueError(msg) from exc\n\n return build_config\n\n def _normalize_url(self, url: str) -> str:\n \"\"\"Normalize URL by adding https:// if no protocol is specified.\"\"\"\n if not url or not isinstance(url, str):\n msg = \"URL cannot be empty\"\n raise ValueError(msg)\n\n url = url.strip()\n if url.startswith((\"http://\", \"https://\")):\n return url\n return f\"https://{url}\"\n\n async def make_request(\n self,\n client: httpx.AsyncClient,\n method: str,\n url: str,\n headers: dict | None = None,\n body: Any = None,\n timeout: int = 5,\n *,\n follow_redirects: bool = False,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n redirection_history = []\n\n try:\n # Prepare request parameters\n request_params = {\n \"method\": method,\n \"url\": url,\n \"headers\": headers,\n \"timeout\": timeout,\n \"follow_redirects\": follow_redirects,\n }\n # Only include body for methods that support it (GET must not have a body per HTTP spec)\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n response = await client.request(**request_params)\n\n redirection_history = [\n {\n \"url\": redirect.headers.get(\"Location\", str(redirect.url)),\n \"status_code\": redirect.status_code,\n }\n for redirect in response.history\n ]\n\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n async def _build_response_data(\n self,\n response: httpx.Response,\n source_url: str,\n headers: dict | None,\n redirection_history: list,\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Turn an httpx response into the component's ``Data`` output.\n\n Shared by the standard request path (``make_request``) and the redirect\n re-validation path (``_follow_redirects_with_validation``) so both produce\n identical metadata, optional file saving, and body decoding.\n \"\"\"\n is_binary, file_path = await self._response_info(response, with_file_path=save_to_file)\n response_headers = self._headers_to_dict(response.headers)\n\n # Base metadata\n metadata = {\n \"source\": source_url,\n \"status_code\": response.status_code,\n \"response_headers\": response_headers,\n }\n\n if redirection_history:\n metadata[\"redirection_history\"] = redirection_history\n\n if save_to_file:\n mode = \"wb\" if is_binary else \"w\"\n encoding = response.encoding if mode == \"w\" else None\n if file_path:\n await aiofiles_os.makedirs(file_path.parent, exist_ok=True)\n if is_binary:\n async with aiofiles.open(file_path, \"wb\") as f:\n await f.write(response.content)\n await f.flush()\n else:\n async with aiofiles.open(file_path, \"w\", encoding=encoding) as f:\n await f.write(response.text)\n await f.flush()\n metadata[\"file_path\"] = str(file_path)\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n return Data(data=metadata)\n\n # Handle response content\n if is_binary:\n result = response.content\n else:\n try:\n result = response.json()\n except json.JSONDecodeError:\n self.log(\"Failed to decode JSON response\")\n result = response.text.encode(\"utf-8\")\n\n metadata[\"result\"] = result\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n\n return Data(data=metadata)\n\n def add_query_params(self, url: str, params: dict) -> str:\n \"\"\"Add query parameters to URL efficiently.\"\"\"\n if not params:\n return url\n url_parts = list(urlparse(url))\n query = dict(parse_qsl(url_parts[4]))\n query.update(params)\n url_parts[4] = urlencode(query)\n return urlunparse(url_parts)\n\n def _headers_to_dict(self, headers: httpx.Headers) -> dict[str, str]:\n \"\"\"Convert HTTP headers to a dictionary with lowercased keys.\"\"\"\n return {k.lower(): v for k, v in headers.items()}\n\n def _process_headers(self, headers: Any) -> dict:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if headers is None:\n return {}\n if isinstance(headers, dict):\n return headers\n if isinstance(headers, list):\n return {item[\"key\"]: item[\"value\"] for item in headers if self._is_valid_key_value_item(item)}\n return {}\n\n async def make_api_request(self) -> Data:\n \"\"\"Make HTTP request with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. The protection works by:\n 1. Validating the URL and resolving DNS during security check\n 2. Pinning the validated IP address\n 3. Forcing the HTTP client to use the pinned IP for the actual request\n 4. Ignoring any subsequent DNS changes (prevents rebinding attacks)\n\n Returns:\n Data: Response data from the HTTP request\n\n Raises:\n ValueError: If URL is invalid or blocked by SSRF protection\n \"\"\"\n # Extract request parameters\n method = self.method\n url = self.url_input.strip() if isinstance(self.url_input, str) else \"\"\n headers = self.headers or {}\n body = self.body or {}\n timeout = self.timeout\n follow_redirects = self.follow_redirects\n save_to_file = self.save_to_file\n include_httpx_metadata = self.include_httpx_metadata\n\n # Security warning: HTTP redirects can bypass SSRF protection\n # A public URL could redirect to an internal resource\n if follow_redirects:\n self.log(\n \"Security Warning: HTTP redirects are enabled. This may allow SSRF bypass attacks \"\n \"where a public URL redirects to internal resources (e.g., cloud metadata endpoints). \"\n \"Only enable this if you trust the target server.\"\n )\n\n # Normalize URL (add https:// if no protocol specified)\n url = self._normalize_url(url)\n\n # Basic URL format validation\n if not validators.url(url):\n msg = f\"Invalid URL provided: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Pinning the validated IP address\n # 3. Using a custom HTTP transport that forces use of the pinned IP\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IP is pinned: \"example.com = 93.184.216.34\"\n # - HTTP request: uses pinned IP directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IP\n # ============================================================================\n\n try:\n # Validate URL and get validated IPs for DNS pinning\n _validated_url, validated_ips = validate_and_resolve_url(url)\n\n # Log DNS pinning information for security auditing\n if validated_ips:\n self.log(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s)\")\n\n except SSRFProtectionError as e:\n # SSRF protection blocked the request (private IP, internal network, etc.)\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n\n # Process query parameters (from string or Data object)\n if isinstance(self.query_params, str):\n query_params = dict(parse_qsl(self.query_params))\n else:\n query_params = self.query_params.data if self.query_params else {}\n\n # Process headers and body into proper format\n headers = self._process_headers(headers)\n body = self._process_body(body)\n url = self.add_query_params(url, query_params)\n\n # ============================================================================\n # Execute the request (re-validating any redirects when SSRF protection is on)\n # ============================================================================\n # When SSRF protection is enabled we must NOT let httpx auto-follow redirects:\n # a validated public URL can redirect to an internal address (loopback, RFC1918,\n # link-local / cloud metadata) that was never checked, bypassing both the initial\n # validation and DNS pinning. Instead we follow redirects manually so every hop\n # is re-validated with the same denylist + DNS pinning. When protection is\n # disabled, we preserve the previous behavior and let httpx handle redirects.\n if is_ssrf_protection_enabled() and follow_redirects:\n result = await self._follow_redirects_with_validation(\n method,\n url,\n headers,\n body,\n timeout,\n validated_ips,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n else:\n # No redirect re-validation needed:\n # - SSRF protection is disabled (user opted out), or\n # - redirects are disabled, so httpx makes a single request.\n # DNS pinning still applies to the single request when protection is enabled\n # and the host resolved to validated IPs.\n async with self._build_http_client(url, validated_ips) as client:\n result = await self.make_request(\n client,\n method,\n url,\n headers,\n body,\n timeout,\n follow_redirects=follow_redirects,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n self.status = result\n return result\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client, pinning DNS to validated IPs when SSRF protection applies.\n\n Args:\n url: The request URL whose hostname will be pinned.\n validated_ips: IPs validated by ``validate_and_resolve_url`` for this hop.\n\n Returns:\n httpx.AsyncClient: A client that pins DNS to ``validated_ips`` (preventing\n rebinding) when SSRF protection is enabled and the hop has validated IPs;\n otherwise a standard client (protection disabled, allowlisted host, or\n hostname extraction failure).\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n # Extract hostname from the URL so the custom transport can pin it while\n # preserving the Host header for virtual hosting / TLS SNI.\n hostname = urlparse(url).hostname\n if hostname:\n # The custom transport tries validated IPs in order (dual-stack / LB).\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _method_for_redirect(method: str, status_code: int) -> str:\n \"\"\"Return the HTTP method to use after a redirect, mirroring httpx semantics.\n\n A 303 (See Other) always becomes GET; 301/302 downgrade POST to GET for\n browser compatibility; 307/308 preserve the original method (and body).\n \"\"\"\n method = method.upper()\n if status_code == HTTP_SEE_OTHER and method != \"HEAD\":\n return \"GET\"\n if status_code in (HTTP_MOVED_PERMANENTLY, HTTP_FOUND) and method == \"POST\":\n return \"GET\"\n return method\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different host.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to a different host than the one the\n caller intended them for. Same-host redirects keep all headers.\n \"\"\"\n if not headers:\n return headers\n if urlparse(current_url).hostname == urlparse(next_url).hostname:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n async def _follow_redirects_with_validation(\n self,\n method: str,\n url: str,\n headers: dict | None,\n body: Any,\n timeout: int,\n validated_ips: list[str],\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Make the request and follow redirects manually, re-validating every hop.\n\n This closes an SSRF bypass: with ``follow_redirects`` enabled, httpx would\n otherwise auto-follow a redirect from a validated public URL to an internal\n address that was never checked. Here each redirect ``Location`` is resolved\n (relative locations included) and re-validated with ``validate_and_resolve_url``\n — the same private/loopback/link-local denylist and DNS pinning applied to the\n initial request — before any connection to it is made. A blocked hop raises\n ``ValueError``; the number of redirects is capped at ``MAX_REDIRECTS``.\n \"\"\"\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n current_url = url\n current_ips = validated_ips\n redirection_history: list[dict] = []\n\n for _ in range(MAX_REDIRECTS + 1):\n request_params: dict[str, Any] = {\n \"method\": method,\n \"url\": current_url,\n \"headers\": headers,\n \"timeout\": timeout,\n # Never let httpx follow redirects itself; each hop is validated below.\n \"follow_redirects\": False,\n }\n # Only include body for methods that support it (GET must not have a body).\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n\n try:\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.request(**request_params)\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {current_url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n location = response.headers.get(\"Location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n redirection_history.append({\"url\": location, \"status_code\": response.status_code})\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n # Non-http(s) schemes, private/loopback/link-local hosts, and hostnames that\n # resolve to blocked IPs all raise SSRFProtectionError here.\n try:\n _validated_url, current_ips = validate_and_resolve_url(next_url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n method = self._method_for_redirect(method, response.status_code)\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict:\n \"\"\"Update the build config based on the selected mode.\"\"\"\n if field_name != \"mode\":\n if field_name == \"curl_input\" and self.mode == \"cURL\" and self.curl_input:\n return self.parse_curl(self.curl_input, build_config)\n return build_config\n\n if field_value == \"cURL\":\n set_field_display(build_config, \"curl_input\", value=True)\n if build_config[\"curl_input\"][\"value\"]:\n try:\n build_config = self.parse_curl(build_config[\"curl_input\"][\"value\"], build_config)\n except ValueError as e:\n self.log(f\"Failed to parse cURL input: {e}\")\n else:\n set_field_display(build_config, \"curl_input\", value=False)\n\n return set_current_fields(\n build_config=build_config,\n action_fields=MODE_FIELDS,\n selected_action=field_value,\n default_fields=DEFAULT_FIELDS,\n func=set_field_advanced,\n default_value=True,\n )\n\n async def _response_info(\n self, response: httpx.Response, *, with_file_path: bool = False\n ) -> tuple[bool, Path | None]:\n \"\"\"Determine the file path and whether the response content is binary.\n\n Args:\n response (Response): The HTTP response object.\n with_file_path (bool): Whether to save the response content to a file.\n\n Returns:\n Tuple[bool, Path | None]:\n A tuple containing a boolean indicating if the content is binary and the full file path (if applicable).\n \"\"\"\n content_type = response.headers.get(\"Content-Type\", \"\")\n is_binary = \"application/octet-stream\" in content_type or \"application/binary\" in content_type\n\n if not with_file_path:\n return is_binary, None\n\n component_temp_dir = Path(tempfile.gettempdir()) / self.__class__.__name__\n\n # Create directory asynchronously\n await aiofiles_os.makedirs(component_temp_dir, exist_ok=True)\n\n filename = None\n if \"Content-Disposition\" in response.headers:\n content_disposition = response.headers[\"Content-Disposition\"]\n filename_match = re.search(r'filename=\"(.+?)\"', content_disposition)\n if filename_match:\n extracted_filename = filename_match.group(1)\n filename = extracted_filename\n\n # Step 3: Infer file extension or use part of the request URL if no filename\n if not filename:\n # Extract the last segment of the URL path\n url_path = urlparse(str(response.request.url) if response.request else \"\").path\n base_name = Path(url_path).name # Get the last segment of the path\n if not base_name: # If the path ends with a slash or is empty\n base_name = \"response\"\n\n # Infer file extension\n content_type_to_extension = {\n \"text/plain\": \".txt\",\n \"application/json\": \".json\",\n \"image/jpeg\": \".jpg\",\n \"image/png\": \".png\",\n \"application/octet-stream\": \".bin\",\n }\n extension = content_type_to_extension.get(content_type, \".bin\" if is_binary else \".txt\")\n filename = f\"{base_name}{extension}\"\n\n # Step 4: Define the full file path\n file_path = component_temp_dir / filename\n\n # Step 5: Check if file exists asynchronously and handle accordingly\n try:\n # Try to create the file exclusively (x mode) to check existence\n async with aiofiles.open(file_path, \"x\") as _:\n pass # File created successfully, we can use this path\n except FileExistsError:\n # If file exists, append a timestamp to the filename\n timestamp = datetime.now(timezone.utc).strftime(\"%Y%m%d%H%M%S%f\")\n file_path = component_temp_dir / f\"{timestamp}-{filename}\"\n\n return is_binary, file_path\n" + "value": "import json\nimport re\nimport tempfile\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom typing import Any\nfrom urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse\n\nimport aiofiles\nimport aiofiles.os as aiofiles_os\nimport httpx\nimport validators\n\nfrom lfx.base.curl.parse import parse_context\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs.inputs import TabInput\nfrom lfx.io import (\n BoolInput,\n DataInput,\n DropdownInput,\n IntInput,\n MessageTextInput,\n MultilineInput,\n Output,\n TableInput,\n)\nfrom lfx.schema.data import Data\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.utils.component_utils import set_current_fields, set_field_advanced, set_field_display\n\n# SSRF Protection imports - for preventing Server-Side Request Forgery attacks\nfrom lfx.utils.ssrf_protection import (\n SSRFProtectionError,\n is_ssrf_protection_enabled,\n validate_and_resolve_url,\n)\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Define fields for each mode\nMODE_FIELDS = {\n \"URL\": [\n \"url_input\",\n \"method\",\n ],\n \"cURL\": [\"curl_input\"],\n}\n\n# Fields that should always be visible\nDEFAULT_FIELDS = [\"mode\"]\n\n# HTTP redirect status codes (RFC 9110).\nHTTP_MOVED_PERMANENTLY = 301\nHTTP_FOUND = 302\nHTTP_SEE_OTHER = 303\nHTTP_TEMPORARY_REDIRECT = 307\nHTTP_PERMANENT_REDIRECT = 308\n\n# Maximum number of redirects to follow when re-validating each hop (matches httpx's default).\nMAX_REDIRECTS = 20\n\n# HTTP status codes that represent a redirect carrying a Location header.\nREDIRECT_STATUS_CODES = frozenset(\n {\n HTTP_MOVED_PERMANENTLY,\n HTTP_FOUND,\n HTTP_SEE_OTHER,\n HTTP_TEMPORARY_REDIRECT,\n HTTP_PERMANENT_REDIRECT,\n }\n)\n\n# Default ports per scheme, used to compare redirect origins.\nDEFAULT_SCHEME_PORTS = {\"http\": 80, \"https\": 443}\n\n\nclass APIRequestComponent(Component):\n display_name = \"API Request\"\n description = \"Make HTTP requests using URL or cURL commands.\"\n documentation: str = \"https://docs.langflow.org/api-request\"\n icon = \"Globe\"\n name = \"APIRequest\"\n\n inputs = [\n MessageTextInput(\n name=\"url_input\",\n display_name=\"URL\",\n info=\"Enter the URL for the request.\",\n advanced=False,\n tool_mode=True,\n ),\n MultilineInput(\n name=\"curl_input\",\n display_name=\"cURL\",\n info=(\n \"Paste a curl command to populate the fields. \"\n \"This will fill in the dictionary fields for headers and body.\"\n ),\n real_time_refresh=True,\n tool_mode=True,\n advanced=True,\n show=False,\n ),\n DropdownInput(\n name=\"method\",\n display_name=\"Method\",\n options=[\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"],\n value=\"GET\",\n info=\"The HTTP method to use.\",\n real_time_refresh=True,\n ),\n TabInput(\n name=\"mode\",\n display_name=\"Mode\",\n options=[\"URL\", \"cURL\"],\n value=\"URL\",\n info=\"Enable cURL mode to populate fields from a cURL command.\",\n real_time_refresh=True,\n ),\n DataInput(\n name=\"query_params\",\n display_name=\"Query Parameters\",\n info=\"The query parameters to append to the URL.\",\n advanced=True,\n ),\n TableInput(\n name=\"body\",\n display_name=\"Body\",\n info=\"The body to send with the request as a dictionary (for POST, PATCH, PUT).\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Parameter name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"description\": \"Parameter value\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n advanced=True,\n real_time_refresh=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": \"Langflow/1.0\"}],\n advanced=True,\n input_types=[\"Data\", \"JSON\"],\n real_time_refresh=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n value=30,\n info=\"The timeout to use for the request.\",\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n value=False,\n info=(\n \"Whether to follow HTTP redirects. \"\n \"WARNING: Enabling redirects may allow SSRF bypass attacks where a public URL \"\n \"redirects to internal resources. Only enable if you trust the target server. \"\n \"See OWASP SSRF Prevention Cheat Sheet for details.\"\n ),\n advanced=True,\n ),\n BoolInput(\n name=\"save_to_file\",\n display_name=\"Save to File\",\n value=False,\n info=\"Save the API response to a temporary file\",\n advanced=True,\n ),\n BoolInput(\n name=\"include_httpx_metadata\",\n display_name=\"Include HTTPx Metadata\",\n value=False,\n info=(\n \"Include properties such as headers, status_code, response_headers, \"\n \"and redirection_history in the output.\"\n ),\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"API Response\", name=\"data\", method=\"make_api_request\"),\n ]\n\n def _parse_json_value(self, value: Any) -> Any:\n \"\"\"Parse a value that might be a JSON string.\"\"\"\n if not isinstance(value, str):\n return value\n\n try:\n parsed = json.loads(value)\n except json.JSONDecodeError:\n return value\n else:\n return parsed\n\n def _process_body(self, body: Any) -> dict:\n \"\"\"Process the body input into a valid dictionary.\"\"\"\n if body is None:\n return {}\n if hasattr(body, \"data\"):\n body = body.data\n if isinstance(body, dict):\n return self._process_dict_body(body)\n if isinstance(body, str):\n return self._process_string_body(body)\n if isinstance(body, list):\n return self._process_list_body(body)\n return {}\n\n def _process_dict_body(self, body: dict) -> dict:\n \"\"\"Process dictionary body by parsing JSON values.\"\"\"\n return {k: self._parse_json_value(v) for k, v in body.items()}\n\n def _process_string_body(self, body: str) -> dict:\n \"\"\"Process string body by attempting JSON parse.\"\"\"\n try:\n return self._process_body(json.loads(body))\n except json.JSONDecodeError:\n return {\"data\": body}\n\n def _process_list_body(self, body: list) -> dict:\n \"\"\"Process list body by converting to key-value dictionary.\"\"\"\n processed_dict = {}\n try:\n for item in body:\n # Unwrap Data objects\n current_item = item\n if hasattr(item, \"data\"):\n unwrapped_data = item.data\n # If the unwrapped data is a dict but not key-value format, use it directly\n if isinstance(unwrapped_data, dict) and not self._is_valid_key_value_item(unwrapped_data):\n return unwrapped_data\n current_item = unwrapped_data\n if not self._is_valid_key_value_item(current_item):\n continue\n key = current_item[\"key\"]\n value = self._parse_json_value(current_item[\"value\"])\n processed_dict[key] = value\n except (KeyError, TypeError, ValueError) as e:\n self.log(f\"Failed to process body list: {e}\")\n return {}\n return processed_dict\n\n def _is_valid_key_value_item(self, item: Any) -> bool:\n \"\"\"Check if an item is a valid key-value dictionary.\"\"\"\n return isinstance(item, dict) and \"key\" in item and \"value\" in item\n\n def parse_curl(self, curl: str, build_config: dotdict) -> dotdict:\n \"\"\"Parse a cURL command and update build configuration.\"\"\"\n try:\n parsed = parse_context(curl)\n\n # Update basic configuration\n url = parsed.url\n # Normalize URL before setting it\n url = self._normalize_url(url)\n\n build_config[\"url_input\"][\"value\"] = url\n build_config[\"method\"][\"value\"] = parsed.method.upper()\n\n # Process headers\n headers_list = [{\"key\": k, \"value\": v} for k, v in parsed.headers.items()]\n build_config[\"headers\"][\"value\"] = headers_list\n\n # Process body data\n if not parsed.data:\n build_config[\"body\"][\"value\"] = []\n elif parsed.data:\n try:\n json_data = json.loads(parsed.data)\n if isinstance(json_data, dict):\n body_list = [\n {\"key\": k, \"value\": json.dumps(v) if isinstance(v, dict | list) else str(v)}\n for k, v in json_data.items()\n ]\n build_config[\"body\"][\"value\"] = body_list\n else:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": json.dumps(json_data)}]\n except json.JSONDecodeError:\n build_config[\"body\"][\"value\"] = [{\"key\": \"data\", \"value\": parsed.data}]\n\n except Exception as exc:\n msg = f\"Error parsing curl: {exc}\"\n self.log(msg)\n raise ValueError(msg) from exc\n\n return build_config\n\n def _normalize_url(self, url: str) -> str:\n \"\"\"Normalize URL by adding https:// if no protocol is specified.\"\"\"\n if not url or not isinstance(url, str):\n msg = \"URL cannot be empty\"\n raise ValueError(msg)\n\n url = url.strip()\n if url.startswith((\"http://\", \"https://\")):\n return url\n return f\"https://{url}\"\n\n async def make_request(\n self,\n client: httpx.AsyncClient,\n method: str,\n url: str,\n headers: dict | None = None,\n body: Any = None,\n timeout: int = 5,\n *,\n follow_redirects: bool = False,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n redirection_history = []\n\n try:\n # Prepare request parameters\n request_params = {\n \"method\": method,\n \"url\": url,\n \"headers\": headers,\n \"timeout\": timeout,\n \"follow_redirects\": follow_redirects,\n }\n # Only include body for methods that support it (GET must not have a body per HTTP spec)\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n response = await client.request(**request_params)\n\n redirection_history = [\n {\n \"url\": redirect.headers.get(\"Location\", str(redirect.url)),\n \"status_code\": redirect.status_code,\n }\n for redirect in response.history\n ]\n\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n async def _build_response_data(\n self,\n response: httpx.Response,\n source_url: str,\n headers: dict | None,\n redirection_history: list,\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Turn an httpx response into the component's ``Data`` output.\n\n Shared by the standard request path (``make_request``) and the redirect\n re-validation path (``_follow_redirects_with_validation``) so both produce\n identical metadata, optional file saving, and body decoding.\n \"\"\"\n is_binary, file_path = await self._response_info(response, with_file_path=save_to_file)\n response_headers = self._headers_to_dict(response.headers)\n\n # Base metadata\n metadata = {\n \"source\": source_url,\n \"status_code\": response.status_code,\n \"response_headers\": response_headers,\n }\n\n if redirection_history:\n metadata[\"redirection_history\"] = redirection_history\n\n if save_to_file:\n mode = \"wb\" if is_binary else \"w\"\n encoding = response.encoding if mode == \"w\" else None\n if file_path:\n await aiofiles_os.makedirs(file_path.parent, exist_ok=True)\n if is_binary:\n async with aiofiles.open(file_path, \"wb\") as f:\n await f.write(response.content)\n await f.flush()\n else:\n async with aiofiles.open(file_path, \"w\", encoding=encoding) as f:\n await f.write(response.text)\n await f.flush()\n metadata[\"file_path\"] = str(file_path)\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n return Data(data=metadata)\n\n # Handle response content\n if is_binary:\n result = response.content\n else:\n try:\n result = response.json()\n except json.JSONDecodeError:\n self.log(\"Failed to decode JSON response\")\n result = response.text.encode(\"utf-8\")\n\n metadata[\"result\"] = result\n\n if include_httpx_metadata:\n metadata.update({\"headers\": headers})\n\n return Data(data=metadata)\n\n def add_query_params(self, url: str, params: dict) -> str:\n \"\"\"Add query parameters to URL efficiently.\"\"\"\n if not params:\n return url\n url_parts = list(urlparse(url))\n query = dict(parse_qsl(url_parts[4]))\n query.update(params)\n url_parts[4] = urlencode(query)\n return urlunparse(url_parts)\n\n def _headers_to_dict(self, headers: httpx.Headers) -> dict[str, str]:\n \"\"\"Convert HTTP headers to a dictionary with lowercased keys.\"\"\"\n return {k.lower(): v for k, v in headers.items()}\n\n def _process_headers(self, headers: Any) -> dict:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if headers is None:\n return {}\n if isinstance(headers, dict):\n return headers\n if isinstance(headers, list):\n return {item[\"key\"]: item[\"value\"] for item in headers if self._is_valid_key_value_item(item)}\n return {}\n\n async def make_api_request(self) -> Data:\n \"\"\"Make HTTP request with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. The protection works by:\n 1. Validating the URL and resolving DNS during security check\n 2. Pinning the validated IP address\n 3. Forcing the HTTP client to use the pinned IP for the actual request\n 4. Ignoring any subsequent DNS changes (prevents rebinding attacks)\n\n Returns:\n Data: Response data from the HTTP request\n\n Raises:\n ValueError: If URL is invalid or blocked by SSRF protection\n \"\"\"\n # Extract request parameters\n method = self.method\n url = self.url_input.strip() if isinstance(self.url_input, str) else \"\"\n headers = self.headers or {}\n body = self.body or {}\n timeout = self.timeout\n follow_redirects = self.follow_redirects\n save_to_file = self.save_to_file\n include_httpx_metadata = self.include_httpx_metadata\n\n # Security warning: HTTP redirects can bypass SSRF protection\n # A public URL could redirect to an internal resource\n if follow_redirects:\n self.log(\n \"Security Warning: HTTP redirects are enabled. This may allow SSRF bypass attacks \"\n \"where a public URL redirects to internal resources (e.g., cloud metadata endpoints). \"\n \"Only enable this if you trust the target server.\"\n )\n\n # Normalize URL (add https:// if no protocol specified)\n url = self._normalize_url(url)\n\n # Basic URL format validation\n if not validators.url(url):\n msg = f\"Invalid URL provided: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Pinning the validated IP address\n # 3. Using a custom HTTP transport that forces use of the pinned IP\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IP is pinned: \"example.com = 93.184.216.34\"\n # - HTTP request: uses pinned IP directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IP\n # ============================================================================\n\n try:\n # Validate URL and get validated IPs for DNS pinning\n _validated_url, validated_ips = validate_and_resolve_url(url)\n\n # Log DNS pinning information for security auditing\n if validated_ips:\n self.log(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s)\")\n\n except SSRFProtectionError as e:\n # SSRF protection blocked the request (private IP, internal network, etc.)\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n\n # Process query parameters (from string or Data object)\n if isinstance(self.query_params, str):\n query_params = dict(parse_qsl(self.query_params))\n else:\n query_params = self.query_params.data if self.query_params else {}\n\n # Process headers and body into proper format\n headers = self._process_headers(headers)\n body = self._process_body(body)\n url = self.add_query_params(url, query_params)\n\n # ============================================================================\n # Execute the request (re-validating any redirects when SSRF protection is on)\n # ============================================================================\n # When SSRF protection is enabled we must NOT let httpx auto-follow redirects:\n # a validated public URL can redirect to an internal address (loopback, RFC1918,\n # link-local / cloud metadata) that was never checked, bypassing both the initial\n # validation and DNS pinning. Instead we follow redirects manually so every hop\n # is re-validated with the same denylist + DNS pinning. When protection is\n # disabled, we preserve the previous behavior and let httpx handle redirects.\n if is_ssrf_protection_enabled() and follow_redirects:\n result = await self._follow_redirects_with_validation(\n method,\n url,\n headers,\n body,\n timeout,\n validated_ips,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n else:\n # No redirect re-validation needed:\n # - SSRF protection is disabled (user opted out), or\n # - redirects are disabled, so httpx makes a single request.\n # DNS pinning still applies to the single request when protection is enabled\n # and the host resolved to validated IPs.\n async with self._build_http_client(url, validated_ips) as client:\n result = await self.make_request(\n client,\n method,\n url,\n headers,\n body,\n timeout,\n follow_redirects=follow_redirects,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n self.status = result\n return result\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client, pinning DNS to validated IPs when SSRF protection applies.\n\n Args:\n url: The request URL whose hostname will be pinned.\n validated_ips: IPs validated by ``validate_and_resolve_url`` for this hop.\n\n Returns:\n httpx.AsyncClient: A client that pins DNS to ``validated_ips`` (preventing\n rebinding) when SSRF protection is enabled and the hop has validated IPs;\n otherwise a standard client (protection disabled, allowlisted host, or\n hostname extraction failure).\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n # Extract hostname from the URL so the custom transport can pin it while\n # preserving the Host header for virtual hosting / TLS SNI.\n hostname = urlparse(url).hostname\n if hostname:\n # The custom transport tries validated IPs in order (dual-stack / LB).\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _method_for_redirect(method: str, status_code: int) -> str:\n \"\"\"Return the HTTP method to use after a redirect, mirroring httpx semantics.\n\n A 303 (See Other) always becomes GET; 301/302 downgrade POST to GET for\n browser compatibility; 307/308 preserve the original method (and body).\n \"\"\"\n method = method.upper()\n if status_code == HTTP_SEE_OTHER and method != \"HEAD\":\n return \"GET\"\n if status_code in (HTTP_MOVED_PERMANENTLY, HTTP_FOUND) and method == \"POST\":\n return \"GET\"\n return method\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different origin.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to an origin other than the one the\n caller intended them for. Headers are kept only when the redirect stays on the\n same origin (scheme, host, port) or is a direct https upgrade of the same host\n on default ports - the exact cases where httpx keeps the Authorization header.\n \"\"\"\n if not headers:\n return headers\n current, nxt = urlparse(current_url), urlparse(next_url)\n current_port = current.port or DEFAULT_SCHEME_PORTS.get(current.scheme)\n next_port = nxt.port or DEFAULT_SCHEME_PORTS.get(nxt.scheme)\n same_origin = (current.scheme, current.hostname, current_port) == (nxt.scheme, nxt.hostname, next_port)\n https_upgrade = (\n current.hostname == nxt.hostname\n and current.scheme == \"http\"\n and nxt.scheme == \"https\"\n and current_port == DEFAULT_SCHEME_PORTS[\"http\"]\n and next_port == DEFAULT_SCHEME_PORTS[\"https\"]\n )\n if same_origin or https_upgrade:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n async def _follow_redirects_with_validation(\n self,\n method: str,\n url: str,\n headers: dict | None,\n body: Any,\n timeout: int,\n validated_ips: list[str],\n *,\n save_to_file: bool = False,\n include_httpx_metadata: bool = False,\n ) -> Data:\n \"\"\"Make the request and follow redirects manually, re-validating every hop.\n\n This closes an SSRF bypass: with ``follow_redirects`` enabled, httpx would\n otherwise auto-follow a redirect from a validated public URL to an internal\n address that was never checked. Here each redirect ``Location`` is resolved\n (relative locations included) and re-validated with ``validate_and_resolve_url``\n — the same private/loopback/link-local denylist and DNS pinning applied to the\n initial request — before any connection to it is made. A blocked hop raises\n ``ValueError``; the number of redirects is capped at ``MAX_REDIRECTS``.\n \"\"\"\n method = method.upper()\n if method not in {\"GET\", \"POST\", \"PATCH\", \"PUT\", \"DELETE\"}:\n msg = f\"Unsupported method: {method}\"\n raise ValueError(msg)\n\n processed_body = self._process_body(body)\n current_url = url\n current_ips = validated_ips\n redirection_history: list[dict] = []\n\n for _ in range(MAX_REDIRECTS + 1):\n request_params: dict[str, Any] = {\n \"method\": method,\n \"url\": current_url,\n \"headers\": headers,\n \"timeout\": timeout,\n # Never let httpx follow redirects itself; each hop is validated below.\n \"follow_redirects\": False,\n }\n # Only include body for methods that support it (GET must not have a body).\n if method in {\"POST\", \"PATCH\", \"PUT\", \"DELETE\"} and processed_body is not None:\n request_params[\"json\"] = processed_body\n\n try:\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.request(**request_params)\n except (httpx.HTTPError, httpx.RequestError, httpx.TimeoutException) as exc:\n self.log(f\"Error making request to {current_url}\")\n return Data(\n data={\n \"source\": url,\n \"headers\": headers,\n \"status_code\": 500,\n \"error\": str(exc),\n **({\"redirection_history\": redirection_history} if redirection_history else {}),\n },\n )\n\n location = response.headers.get(\"Location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n redirection_history.append({\"url\": location, \"status_code\": response.status_code})\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n # Non-http(s) schemes, private/loopback/link-local hosts, and hostnames that\n # resolve to blocked IPs all raise SSRFProtectionError here.\n try:\n _validated_url, current_ips = validate_and_resolve_url(next_url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n method = self._method_for_redirect(method, response.status_code)\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return await self._build_response_data(\n response,\n url,\n headers,\n redirection_history,\n save_to_file=save_to_file,\n include_httpx_metadata=include_httpx_metadata,\n )\n\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict:\n \"\"\"Update the build config based on the selected mode.\"\"\"\n if field_name != \"mode\":\n if field_name == \"curl_input\" and self.mode == \"cURL\" and self.curl_input:\n return self.parse_curl(self.curl_input, build_config)\n return build_config\n\n if field_value == \"cURL\":\n set_field_display(build_config, \"curl_input\", value=True)\n if build_config[\"curl_input\"][\"value\"]:\n try:\n build_config = self.parse_curl(build_config[\"curl_input\"][\"value\"], build_config)\n except ValueError as e:\n self.log(f\"Failed to parse cURL input: {e}\")\n else:\n set_field_display(build_config, \"curl_input\", value=False)\n\n return set_current_fields(\n build_config=build_config,\n action_fields=MODE_FIELDS,\n selected_action=field_value,\n default_fields=DEFAULT_FIELDS,\n func=set_field_advanced,\n default_value=True,\n )\n\n async def _response_info(\n self, response: httpx.Response, *, with_file_path: bool = False\n ) -> tuple[bool, Path | None]:\n \"\"\"Determine the file path and whether the response content is binary.\n\n Args:\n response (Response): The HTTP response object.\n with_file_path (bool): Whether to save the response content to a file.\n\n Returns:\n Tuple[bool, Path | None]:\n A tuple containing a boolean indicating if the content is binary and the full file path (if applicable).\n \"\"\"\n content_type = response.headers.get(\"Content-Type\", \"\")\n is_binary = \"application/octet-stream\" in content_type or \"application/binary\" in content_type\n\n if not with_file_path:\n return is_binary, None\n\n component_temp_dir = Path(tempfile.gettempdir()) / self.__class__.__name__\n\n # Create directory asynchronously\n await aiofiles_os.makedirs(component_temp_dir, exist_ok=True)\n\n filename = None\n if \"Content-Disposition\" in response.headers:\n content_disposition = response.headers[\"Content-Disposition\"]\n filename_match = re.search(r'filename=\"(.+?)\"', content_disposition)\n if filename_match:\n extracted_filename = filename_match.group(1)\n filename = extracted_filename\n\n # Step 3: Infer file extension or use part of the request URL if no filename\n if not filename:\n # Extract the last segment of the URL path\n url_path = urlparse(str(response.request.url) if response.request else \"\").path\n base_name = Path(url_path).name # Get the last segment of the path\n if not base_name: # If the path ends with a slash or is empty\n base_name = \"response\"\n\n # Infer file extension\n content_type_to_extension = {\n \"text/plain\": \".txt\",\n \"application/json\": \".json\",\n \"image/jpeg\": \".jpg\",\n \"image/png\": \".png\",\n \"application/octet-stream\": \".bin\",\n }\n extension = content_type_to_extension.get(content_type, \".bin\" if is_binary else \".txt\")\n filename = f\"{base_name}{extension}\"\n\n # Step 4: Define the full file path\n file_path = component_temp_dir / filename\n\n # Step 5: Check if file exists asynchronously and handle accordingly\n try:\n # Try to create the file exclusively (x mode) to check existence\n async with aiofiles.open(file_path, \"x\") as _:\n pass # File created successfully, we can use this path\n except FileExistsError:\n # If file exists, append a timestamp to the filename\n timestamp = datetime.now(timezone.utc).strftime(\"%Y%m%d%H%M%S%f\")\n file_path = component_temp_dir / f\"{timestamp}-{filename}\"\n\n return is_binary, file_path\n" }, "curl_input": { "_input_type": "MultilineInput", @@ -58868,6 +58868,7 @@ "max_depth", "prevent_outside", "use_async", + "follow_redirects", "format", "timeout", "headers", @@ -58880,7 +58881,7 @@ "icon": "layout-template", "legacy": false, "metadata": { - "code_hash": "2e225a90043a", + "code_hash": "52b5bd602242", "dependencies": { "dependencies": [ { @@ -58995,7 +58996,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n async with self._build_http_client(url, validated_ips) as client:\n try:\n response = await client.get(url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n else:\n return html_content, metadata\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(start_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(start_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" + "value": "import importlib.util\nimport io\nimport re\nfrom urllib.parse import urljoin, urlparse\n\nimport httpx\nfrom bs4 import BeautifulSoup\nfrom markitdown import MarkItDown\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\nfrom lfx.utils.ssrf_protection import SSRFProtectionError, is_ssrf_protection_enabled, validate_and_resolve_url\nfrom lfx.utils.ssrf_transport import create_ssrf_protected_client\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n# HTTP status codes that carry a redirect Location header (RFC 9110).\nREDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})\n\n# Maximum number of redirects to follow. Each hop is re-validated for SSRF safety;\n# matches httpx's default and the API Request component.\nMAX_REDIRECTS = 20\n\n# Default ports per scheme, used to compare redirect origins.\nDEFAULT_SCHEME_PORTS = {\"http\": 80, \"https\": 443}\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"follow_redirects\",\n display_name=\"Follow Redirects\",\n info=(\n \"If enabled, follows HTTP redirects such as http→https or www/non-www \"\n \"normalization, which most sites rely on to serve their content. When SSRF \"\n \"protection is enabled (the default), every redirect hop is re-validated against \"\n \"the same blocked-IP denylist and DNS-pinned before it is fetched, so following \"\n \"redirects cannot be used to reach internal resources. Disable to capture the \"\n \"first response as-is without following redirects.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=(\n \"Output Format. Use 'Text' to extract the text from the HTML, \"\n \"'Markdown' to parse the HTML into Markdown format, or 'HTML' \"\n \"for the raw HTML content.\"\n ),\n options=[\"Text\", \"HTML\", \"Markdown\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\", \"Table\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def _html_extractor(x: str) -> str:\n \"\"\"Extract raw HTML content.\"\"\"\n return x\n\n @staticmethod\n def _text_extractor(x: str) -> str:\n \"\"\"Extract clean text from HTML.\"\"\"\n return BeautifulSoup(x, \"lxml\").get_text()\n\n @staticmethod\n def _markdown_extractor(x: str) -> str:\n \"\"\"Convert HTML to Markdown format.\"\"\"\n stream = io.BytesIO(x.encode(\"utf-8\"))\n result = MarkItDown(enable_plugins=False).convert_stream(stream)\n return result.markdown\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> tuple[str, list[str]]:\n \"\"\"Ensures the given string is a valid URL and returns validated IPs for DNS pinning.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n tuple[str, list[str]]: The normalized URL and list of validated IPs for DNS pinning\n\n Raises:\n ValueError: If the URL is invalid or blocked by SSRF protection\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n # ============================================================================\n # SSRF Protection with DNS Pinning\n # ============================================================================\n # This prevents DNS rebinding attacks by:\n # 1. Resolving DNS and validating IPs during security check\n # 2. Returning the validated IP addresses for DNS pinning\n # 3. Using a custom HTTP transport that forces use of the pinned IPs\n # 4. Ignoring any new DNS resolutions (prevents rebinding)\n #\n # Without DNS pinning, an attacker could:\n # - First DNS lookup: returns public IP (passes validation)\n # - Second DNS lookup: returns internal IP (bypasses protection)\n # - Attack succeeds: accesses internal services\n #\n # With DNS pinning:\n # - First DNS lookup: returns public IP (passes validation)\n # - IPs are pinned and returned\n # - HTTP request: uses pinned IPs directly (no new DNS lookup)\n # - Attack fails: even if DNS changes, we use the validated IPs\n # ============================================================================\n try:\n _validated_url, validated_ips = validate_and_resolve_url(url)\n except SSRFProtectionError as e:\n msg = f\"SSRF Protection: {e}\"\n raise ValueError(msg) from e\n else:\n # Log DNS pinning information for security auditing\n if validated_ips:\n logger.debug(f\"SSRF Protection: Using DNS pinning with {len(validated_ips)} validated IP(s) for {url}\")\n\n return url, validated_ips\n\n def _build_http_client(self, url: str, validated_ips: list[str]) -> httpx.AsyncClient:\n \"\"\"Create an HTTP client with DNS pinning for SSRF protection.\n\n Args:\n url: The request URL whose hostname will be pinned\n validated_ips: IPs validated by validate_and_resolve_url for this URL\n\n Returns:\n httpx.AsyncClient: A client with DNS pinning when SSRF protection is enabled\n \"\"\"\n if is_ssrf_protection_enabled() and validated_ips:\n hostname = urlparse(url).hostname\n if hostname:\n return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips)\n return httpx.AsyncClient()\n\n @staticmethod\n def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None:\n \"\"\"Drop sensitive headers when a redirect crosses to a different origin.\n\n Mirrors httpx's auto-follow behavior so manually following redirects does not\n leak credentials (Authorization / Cookie) to an origin other than the one the\n caller intended them for. Headers are kept only when the redirect stays on the\n same origin (scheme, host, port) or is a direct https upgrade of the same host\n on default ports - the exact cases where httpx keeps the Authorization header.\n \"\"\"\n if not headers:\n return headers\n current, nxt = urlparse(current_url), urlparse(next_url)\n current_port = current.port or DEFAULT_SCHEME_PORTS.get(current.scheme)\n next_port = nxt.port or DEFAULT_SCHEME_PORTS.get(nxt.scheme)\n same_origin = (current.scheme, current.hostname, current_port) == (nxt.scheme, nxt.hostname, next_port)\n https_upgrade = (\n current.hostname == nxt.hostname\n and current.scheme == \"http\"\n and nxt.scheme == \"https\"\n and current_port == DEFAULT_SCHEME_PORTS[\"http\"]\n and next_port == DEFAULT_SCHEME_PORTS[\"https\"]\n )\n if same_origin or https_upgrade:\n return headers\n sensitive = {\"authorization\", \"proxy-authorization\", \"cookie\"}\n return {k: v for k, v in headers.items() if k.lower() not in sensitive}\n\n def _process_response(self, response: httpx.Response) -> tuple[str, dict]:\n \"\"\"Turn a final (non-redirect) response into its HTML content and metadata.\n\n Args:\n response: The HTTP response to process\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n if self.check_response_status:\n response.raise_for_status()\n\n content_type = response.headers.get(\"content-type\", \"\").lower()\n\n # Filter out CSS files if requested\n if self.filter_text_html and \"text/css\" in content_type:\n logger.debug(f\"Skipping CSS file: {response.url}\")\n return \"\", {}\n\n # Get the HTML content\n html_content = response.text\n\n # Extract metadata\n metadata = {\n \"source\": str(response.url),\n \"title\": \"\",\n \"description\": \"\",\n \"content_type\": content_type,\n \"language\": \"\",\n }\n\n # Try to extract title and description from HTML\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - metadata extraction is optional\n # and we don't want to fail the entire request if it fails\n logger.debug(f\"Failed to extract metadata from {response.url}\")\n else:\n if soup.title:\n metadata[\"title\"] = soup.title.string or \"\"\n\n # Try to get description from meta tags\n meta_desc = soup.find(\"meta\", attrs={\"name\": \"description\"})\n if meta_desc and meta_desc.get(\"content\"):\n metadata[\"description\"] = meta_desc[\"content\"]\n\n # Try to get language\n html_tag = soup.find(\"html\")\n if html_tag and html_tag.get(\"lang\"):\n metadata[\"language\"] = html_tag[\"lang\"]\n\n return html_content, metadata\n\n async def _fetch_with_revalidated_redirects(\n self, url: str, validated_ips: list[str], headers: dict\n ) -> tuple[str, dict]:\n \"\"\"Fetch ``url``, following redirects manually and re-validating every hop.\n\n This closes an SSRF bypass: with redirects enabled, httpx would otherwise\n auto-follow a redirect from a validated public URL to an internal address that\n the pinned transport never checked (the redirect target is a different host, so\n it is not in the pin map). Each redirect ``Location`` is resolved (relative\n locations included) and re-validated with ``ensure_url`` - the same private/\n loopback/link-local denylist and DNS pinning applied to the initial request -\n before any connection to it is made. A blocked hop raises ``ValueError`` (unless\n ``continue_on_failure``); the chain is capped at ``MAX_REDIRECTS`` hops.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning of the initial URL\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n current_url = url\n current_ips = validated_ips\n\n for _ in range(MAX_REDIRECTS + 1):\n async with self._build_http_client(current_url, current_ips) as client:\n response = await client.get(current_url, headers=headers, timeout=self.timeout, follow_redirects=False)\n\n location = response.headers.get(\"location\")\n if response.status_code in REDIRECT_STATUS_CODES and location:\n # Resolve relative redirects against the current URL.\n next_url = urljoin(current_url, location)\n\n # Re-validate the redirect target with the same SSRF denylist + DNS pinning.\n try:\n validated_next_url, current_ips = self.ensure_url(next_url)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping blocked or invalid redirect to {next_url}: {e}\")\n return \"\", {}\n msg = f\"SSRF Protection: blocked redirect to {next_url}: {e}\"\n raise ValueError(msg) from e\n\n headers = self._headers_for_redirect(headers, current_url, next_url)\n current_url = validated_next_url\n continue\n\n # Not a redirect (or no Location header) - this is the final response.\n return self._process_response(response)\n\n # Exhausted the redirect budget.\n if self.continue_on_failure:\n logger.warning(f\"Exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\")\n return \"\", {}\n msg = f\"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}\"\n raise ValueError(msg)\n\n async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]:\n \"\"\"Fetch a single URL with DNS pinning protection, following redirects safely.\n\n Args:\n url: The URL to fetch\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n\n Returns:\n tuple[str, dict]: The HTML content and metadata\n \"\"\"\n try:\n # When SSRF protection is enabled we must follow redirects manually so each hop\n # is re-validated and DNS-pinned; letting httpx auto-follow would connect to the\n # redirect target without pinning (different host, not in the pin map) and re-open\n # the SSRF hole that DNS pinning closes. With protection disabled there is no pin\n # to bypass, so httpx can follow redirects natively.\n if self.follow_redirects and is_ssrf_protection_enabled():\n return await self._fetch_with_revalidated_redirects(url, validated_ips, headers)\n\n async with self._build_http_client(url, validated_ips) as client:\n response = await client.get(\n url, headers=headers, timeout=self.timeout, follow_redirects=self.follow_redirects\n )\n return self._process_response(response)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Failed to fetch {url}: {e}\")\n return \"\", {}\n raise\n\n async def _crawl_recursive(\n self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0\n ) -> list[dict]:\n \"\"\"Recursively crawl URLs with DNS pinning protection.\n\n Args:\n start_url: The URL to start crawling from\n validated_ips: Validated IPs for DNS pinning\n headers: HTTP headers to send\n visited: Set of already visited URLs\n depth: Current crawl depth\n\n Returns:\n list[dict]: List of documents with content and metadata\n \"\"\"\n if depth >= self.max_depth or start_url in visited:\n return []\n\n visited.add(start_url)\n documents = []\n\n # Fetch the current URL\n html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers)\n\n # Redirects may have landed on a different canonical URL (http->https, www\n # normalization). Resolve relative links and the prevent_outside check against\n # the URL the content actually came from, and mark it visited so links back to\n # the canonical form are not re-crawled.\n base_url = metadata.get(\"source\") or start_url\n visited.add(base_url)\n\n if not html_content:\n return documents\n\n # Extract content based on format\n extractors = {\n \"HTML\": self._html_extractor,\n \"Markdown\": self._markdown_extractor,\n \"Text\": self._text_extractor,\n }\n extractor = extractors.get(self.format, self._text_extractor)\n extracted_content = extractor(html_content)\n\n # Add the document\n documents.append(\n {\n \"page_content\": extracted_content,\n \"metadata\": metadata,\n }\n )\n\n # If we haven't reached max depth, extract and follow links\n if depth < self.max_depth - 1:\n try:\n soup = BeautifulSoup(html_content, \"lxml\")\n links = soup.find_all(\"a\", href=True)\n\n for link in links:\n href = link[\"href\"]\n # Resolve relative URLs\n absolute_url = urljoin(base_url, href)\n\n # Skip if already visited\n if absolute_url in visited:\n continue\n\n # Check if we should prevent going outside the domain\n if self.prevent_outside:\n start_domain = urlparse(base_url).netloc\n link_domain = urlparse(absolute_url).netloc\n if start_domain != link_domain:\n continue\n\n # Validate and crawl the linked URL\n try:\n _, link_validated_ips = self.ensure_url(absolute_url)\n sub_docs = await self._crawl_recursive(\n absolute_url, link_validated_ips, headers, visited, depth + 1\n )\n documents.extend(sub_docs)\n except (ValueError, SSRFProtectionError) as e:\n if self.continue_on_failure:\n logger.warning(f\"Skipping {absolute_url}: {e}\")\n continue\n raise\n\n except Exception: # noqa: BLE001\n # Broad exception is acceptable here - link extraction is optional\n # and we don't want to fail the entire crawl if one page has issues\n logger.debug(f\"Failed to extract links from {start_url}\")\n\n return documents\n\n async def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs with SSRF protection and DNS pinning.\n\n This method implements comprehensive SSRF (Server-Side Request Forgery) protection\n using DNS pinning to prevent DNS rebinding attacks. Each URL is validated and its\n DNS resolution is pinned before any HTTP requests are made.\n\n Returns:\n list[dict]: List of documents with content and metadata\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n # Validate all URLs and get their validated IPs for DNS pinning\n validated_urls = []\n first_validation_error: Exception | None = None\n for url in self.urls:\n if not url.strip():\n continue\n try:\n normalized_url, validated_ips = self.ensure_url(url)\n validated_urls.append((normalized_url, validated_ips))\n except (ValueError, SSRFProtectionError) as e:\n # Remember the first rejection so its real cause (for example an\n # SSRF block) can be surfaced if no URL survives validation,\n # instead of being hidden behind \"No valid URLs provided\".\n if first_validation_error is None:\n first_validation_error = e\n if self.continue_on_failure:\n logger.warning(f\"Skipping invalid URL {url}: {e}\")\n continue\n raise\n\n # Remove duplicates while preserving order\n seen = set()\n unique_validated_urls = []\n for url, ips in validated_urls:\n if url not in seen:\n seen.add(url)\n unique_validated_urls.append((url, ips))\n\n logger.debug(f\"Validated {len(unique_validated_urls)} unique URL(s)\")\n\n if not unique_validated_urls:\n # Surface the actual reason every candidate URL was rejected (e.g. an\n # SSRF block) rather than a generic message, so a security failure is\n # not silently swallowed when continue_on_failure is enabled.\n if first_validation_error is not None:\n raise first_validation_error\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n # Prepare headers\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n\n # Crawl all URLs\n all_docs = []\n for url, validated_ips in unique_validated_urls:\n logger.debug(f\"Crawling {url} with max_depth={self.max_depth}\")\n\n try:\n visited = set()\n docs = await self._crawl_recursive(url, validated_ips, headers_dict, visited, depth=0)\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} document(s) from {url}\")\n all_docs.extend(docs)\n\n except httpx.HTTPError as e:\n if self.continue_on_failure:\n logger.warning(f\"Error loading documents from {url}: {e}\")\n continue\n msg = f\"Error loading documents from {url}: {e}\"\n raise ValueError(msg) from e\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # Convert to output format\n return [\n {\n \"text\": safe_convert(doc[\"page_content\"], clean_data=True),\n \"url\": doc[\"metadata\"].get(\"source\", \"\"),\n \"title\": doc[\"metadata\"].get(\"title\", \"\"),\n \"description\": doc[\"metadata\"].get(\"description\", \"\"),\n \"content_type\": doc[\"metadata\"].get(\"content_type\", \"\"),\n \"language\": doc[\"metadata\"].get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else str(e)\n msg = f\"Error loading documents: {error_msg}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n\n async def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n url_contents = await self.fetch_url_contents()\n return DataFrame(data=url_contents)\n\n async def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = await self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" }, "continue_on_failure": { "_input_type": "BoolInput", @@ -59037,6 +59038,26 @@ "type": "bool", "value": true }, + "follow_redirects": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Follow Redirects", + "dynamic": false, + "info": "If enabled, follows HTTP redirects such as http→https or www/non-www normalization, which most sites rely on to serve their content. When SSRF protection is enabled (the default), every redirect hop is re-validated against the same blocked-IP denylist and DNS-pinned before it is fetched, so following redirects cannot be used to reach internal resources. Disable to capture the first response as-is without following redirects.", + "list": false, + "list_add_label": "Add More", + "name": "follow_redirects", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + }, "format": { "_input_type": "DropdownInput", "advanced": true, @@ -68684,7 +68705,7 @@ "icon": "folder", "legacy": false, "metadata": { - "code_hash": "8689a573f75f", + "code_hash": "705242f98bad", "dependencies": { "dependencies": [ { @@ -68742,7 +68763,7 @@ "show": true, "title_case": false, "type": "code", - "value": "\"\"\"Sandboxed filesystem tool component exposing 5 file I/O tools to agents.\"\"\"\n\nfrom __future__ import annotations\n\nimport contextvars\nimport json\nimport os\nimport re\nfrom pathlib import Path, PureWindowsPath\nfrom typing import TYPE_CHECKING\n\nfrom langchain_core.tools import StructuredTool\nfrom pydantic import BaseModel, Field\n\nfrom lfx.components.files_and_knowledge._filesystem_isolation import (\n IsolationConfig,\n load_isolation_config,\n)\nfrom lfx.components.files_and_knowledge._filesystem_namespace import (\n compute_user_namespace,\n load_or_create_pepper,\n)\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs.inputs import BoolInput, StrInput\nfrom lfx.io import Output\nfrom lfx.schema.data import Data\nfrom lfx.services.deps import get_settings_service\n\nif TYPE_CHECKING:\n from lfx.field_typing import Tool\n\n# Sub-directory under used when AUTO_LOGIN=True. Stays parallel to\n# users// so flipping AUTO_LOGIN at deploy-time does not mix file trees.\nSHARED_NAMESPACE = \"shared\"\n\n\n# L2 binding atomicity: the binding check captures `user_id` once at the\n# start of every tool invocation; subsequent reads of `_resolve_user_id`\n# during the same call (in _validate_root, _validate_path, etc.) read this\n# pinned value instead of re-resolving. Without this pin, a concurrent\n# mutation of `self._user_id` between the binding check and the path\n# resolution would let an attacker write into a foreign user's namespace\n# while the check still passed for the original user.\n_pinned_user_id_var: contextvars.ContextVar[str | None] = contextvars.ContextVar(\n \"_filesystem_pinned_user_id\",\n default=None,\n)\n\n\n# Reserved on-disk segments inside every user namespace.\n#\n# `.lfsig` is the forward hook for an HMAC sidecar tree (L3 in the FS plan).\n# `.components` is the storage location for user-generated component code\n# (written by a privileged backend helper after Layer-2 code validation;\n# read by the registry overlay so build_flow / search_components see the\n# user's custom types). Allowing the agent's FS tools to touch either\n# directory would either poison a security-critical namespace or let the\n# agent plant arbitrary code into its own executable namespace.\n#\n# Kept as a singular constant for backwards-compat with prior message text\n# (\"Path component '.lfsig' is reserved\") and as a tuple for the actual\n# check — both names point at the same casefold set.\nRESERVED_SEGMENT = \".lfsig\"\nRESERVED_SEGMENTS: tuple[str, ...] = (\".lfsig\", \".components\")\n\n\ndef _default_config_dir() -> Path:\n \"\"\"Pick a sensible config dir when no env var is set.\n\n Operators set ``LANGFLOW_FS_TOOL_BASE_DIR`` / ``LANGFLOW_FS_TOOL_PEPPER_PATH``\n explicitly in any real deployment; this fallback exists so the OSS desktop\n install just works without any setup.\n \"\"\"\n return Path.home() / \".langflow\" / \"fs_tool\"\n\n\nMAX_FILE_SIZE_BYTES = 10 * 1024 * 1024\nBINARY_SNIFF_BYTES = 8 * 1024\nGLOB_RESULT_LIMIT = 100\n# Hard upper bound on the number of glob matches collected before truncation.\n# Without scanning past `GLOB_RESULT_LIMIT`, the first big branch hit by\n# `os.scandir` order fills the cap and entire nested branches are silently\n# dropped (BUG-02 / T2-001). The ceiling bounds memory/time on pathological\n# trees while leaving headroom to surface diverse branches to the agent.\nGLOB_SCAN_CEILING = GLOB_RESULT_LIMIT * 10\nGREP_LINE_LIMIT = 250\nGREP_OUTPUT_MODES = (\"files_with_matches\", \"content\", \"count\")\n# Hard ceiling on user-supplied regex pattern length. Long patterns are a\n# common ReDoS vector and have no legitimate need against single text lines.\nGREP_PATTERN_MAX_LEN = 1024\n# Skip regex matching on lines longer than this — exponential backtracking\n# requires non-trivial input length, and oversized lines tend to be data\n# blobs (minified JS/JSON) rather than meaningful matches.\nGREP_REGEX_LINE_MAX_LEN = 4096\n# Cap on total lines scanned per file in regex mode, regardless of matches.\n# Bounds the work even for benign-looking patterns on large files.\nGREP_REGEX_LINES_PER_FILE = 50_000\n\n# Heuristic ReDoS detector.\n#\n# Stdlib `re` has no native timeout; we cannot kill a runaway match because\n# the engine holds the GIL. Instead we reject patterns whose structure makes\n# catastrophic backtracking possible BEFORE compiling them.\n#\n# The classic ReDoS shape is a nested unbounded quantifier — a parenthesized\n# group whose body is itself quantifiable, followed by an outer `+`/`*`/`{n,}`.\n# Examples: `(a+)+`, `(a*)*`, `(.*)+`, `(\\w+)+`, `(a|aa)+`. We reject any\n# group that ends with `)+`/`)*`/`){n,}` AND whose body contains an\n# unescaped quantifier (`+`, `*`, `{n,}`) or alternation (`|`).\n#\n# This is intentionally conservative — it rules out some safe patterns\n# (`(ab)+` is fine; `(ab+)+` is not, but our rule rejects both). Users who\n# need those patterns can rewrite without the outer quantifier. Trading a\n# bit of expressiveness for a hard guarantee that no user pattern can\n# DoS the worker.\n_REDOS_GROUP = re.compile(\n r\"\"\"\n \\( # outer group open\n (?: # body of the group:\n (?:\\\\.) # - any escaped char (so \\+ etc. don't trip us)\n | [^()] # - or any non-paren char\n )*?\n (?: # ... that contains at least one of:\n (? bool:\n \"\"\"Return True if `pattern` matches a known catastrophic-backtracking shape.\"\"\"\n return bool(_REDOS_GROUP.search(pattern))\n\n\n# Windows portability rules (applied on every host OS so flows authored on\n# macOS/Linux do not silently break when run on Windows). Pure-string checks.\n_WINDOWS_RESERVED_NAMES = frozenset(\n {\"CON\", \"PRN\", \"AUX\", \"NUL\", *(f\"COM{i}\" for i in range(1, 10)), *(f\"LPT{i}\" for i in range(1, 10))}\n)\n# Note: ':' is NOT included — drive letters (C:) are valid; bare ':' inside a\n# basename will be caught by the OS at write time anyway.\n_WINDOWS_FORBIDDEN_CHARS = frozenset('<>\"|?*')\n\n# Deny-list: even when a path lies inside `root_path`, refuse access to\n# basenames or path components that match well-known credential / secret\n# patterns. This is the \"default-deny inside an allowed root\" pattern from\n# Claude Code Sandboxing — it limits the blast radius of a flow author who\n# misconfigures `root_path` to cover $HOME or the project root. Pure-string\n# checks; runs before any I/O so the agent never observes the file's\n# existence, contents, or absence-vs-denied distinction beyond the error\n# string itself.\n#\n# Match semantics (all case-insensitive):\n# * literals — exact basename match (e.g. `.env`, `.netrc`)\n# * prefixes — basename startswith (e.g. `id_rsa`, `id_rsa.pub`, `id_ed25519`)\n# * suffixes — basename endswith (e.g. `cert.pem`, `private.key`)\n# * fragments — any DIRECTORY component equals the fragment (e.g. `.ssh/`,\n# `.aws/config`); the basename itself is not matched against fragments.\n_DENY_BASENAME_LITERALS = frozenset({\".env\", \".netrc\", \".pgpass\", \".htpasswd\", \"authorized_keys\"})\n_DENY_BASENAME_PREFIXES = (\"id_rsa\", \"id_dsa\", \"id_ecdsa\", \"id_ed25519\", \"credentials\")\n_DENY_BASENAME_SUFFIXES = (\".pem\", \".key\", \".pfx\", \".p12\")\n_DENY_PATH_FRAGMENTS = frozenset({\".ssh\", \".aws\", \".gnupg\", \".docker\", \".kube\", \".git\"})\n\n\ndef _looks_binary(head: bytes) -> bool:\n return b\"\\x00\" in head\n\n\ndef _check_hardlink(candidate: Path) -> str | None:\n \"\"\"Return an error string when ``candidate`` is a multi-hardlink file.\n\n Why we refuse multi-hardlink files: an attacker with write access to a\n location outside the sandbox can pre-create an extra hardlink pointing\n at a sandbox path. Subsequent writes through the sandbox name then\n also clobber the external name, defeating the boundary. There is no\n legitimate flow that depends on a multi-link inode inside the\n sandbox, so refusing fails closed.\n\n Restricted to **regular files**: directories on POSIX always have\n ``st_nlink >= 2`` (`.`, plus one per subdirectory entry), so checking\n them would refuse every nested path. Symlinks fall through to the\n boundary check and the no-follow helpers. Non-existent paths return\n ``None`` — creation is handled by the O_NOFOLLOW write helper.\n \"\"\"\n try:\n st = os.lstat(candidate)\n except FileNotFoundError:\n return None\n except OSError as exc:\n return f\"Cannot stat path: {exc.strerror or exc}\"\n import stat as _stat_module\n\n if _stat_module.S_ISREG(st.st_mode) and st.st_nlink > 1:\n return f\"Refusing to operate on multi-hardlink file (nlink={st.st_nlink})\"\n return None\n\n\ndef _write_bytes_no_follow(target: Path, data: bytes) -> None:\n \"\"\"Write ``data`` to ``target`` without following symlinks.\n\n Uses ``O_NOFOLLOW`` on POSIX so that any symlink at ``target`` —\n including one created by a concurrent process between path\n validation and this open — raises ``ELOOP`` and fails the write\n closed. On Windows the flag is unavailable; we lstat and refuse if\n the target is already a symlink (best-effort against the non-racy\n attacker).\n\n The file is created with mode 0600 so that, even on shared hosts,\n other users cannot read it without explicit operator action.\n \"\"\"\n flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC\n if hasattr(os, \"O_NOFOLLOW\"):\n flags |= os.O_NOFOLLOW\n elif target.is_symlink(): # pragma: no cover — Windows-only branch\n msg = f\"Refusing to write through symlink: {target}\"\n raise PermissionError(msg)\n fd = os.open(str(target), flags, 0o600)\n try:\n os.write(fd, data)\n finally:\n os.close(fd)\n\n\ndef _read_bytes_no_follow(target: Path) -> bytes:\n \"\"\"Read the entire file at ``target`` without following symlinks.\n\n Same TOCTOU rationale as ``_write_bytes_no_follow``: with\n ``O_NOFOLLOW`` the open fails if a symlink was substituted between\n path validation and this read. Reads up to ``MAX_FILE_SIZE_BYTES``\n in a single syscall — the caller has already enforced the cap via\n ``stat().st_size`` checks.\n \"\"\"\n flags = os.O_RDONLY\n if hasattr(os, \"O_NOFOLLOW\"):\n flags |= os.O_NOFOLLOW\n elif target.is_symlink(): # pragma: no cover — Windows-only branch\n msg = f\"Refusing to read through symlink: {target}\"\n raise PermissionError(msg)\n fd = os.open(str(target), flags)\n try:\n chunks: list[bytes] = []\n while True:\n chunk = os.read(fd, 1 << 20) # 1 MiB per syscall\n if not chunk:\n break\n chunks.append(chunk)\n return b\"\".join(chunks)\n finally:\n os.close(fd)\n\n\ndef _read_head_no_follow(target: Path, n: int) -> bytes:\n \"\"\"Read the first ``n`` bytes of ``target`` without following symlinks.\n\n Used for the binary-content sniff before deciding whether a file is\n safe to surface as text — the same TOCTOU defence applies.\n \"\"\"\n flags = os.O_RDONLY\n if hasattr(os, \"O_NOFOLLOW\"):\n flags |= os.O_NOFOLLOW\n elif target.is_symlink(): # pragma: no cover — Windows-only branch\n msg = f\"Refusing to read through symlink: {target}\"\n raise PermissionError(msg)\n fd = os.open(str(target), flags)\n try:\n return os.read(fd, n)\n finally:\n os.close(fd)\n\n\ndef _check_windows_portability(path: str) -> str | None:\n \"\"\"Return a human-readable error if the path has Windows-portability issues.\n\n Why we run this on every host: paths that work on macOS/Linux can silently\n fail on Windows (reserved names, forbidden characters, trailing dot/space\n silently stripped by the OS). The agent should see the same structured\n error regardless of where the flow runs.\n \"\"\"\n # Use PureWindowsPath to parse separators consistently — it splits on both\n # `/` and `\\` and exposes drive markers as components ending in '\\'.\n for component in PureWindowsPath(path).parts:\n # Skip root markers ('\\\\', '/', 'C:\\\\') and relative markers ('.', '..')\n if component.endswith((\"\\\\\", \"/\")) or component in (\".\", \"..\"):\n continue\n # Reserved name (case-insensitive, with or without extension).\n stem = component.split(\".\", 1)[0].upper()\n if stem in _WINDOWS_RESERVED_NAMES:\n return f\"Path component {component!r} is a Windows reserved name\"\n # Forbidden characters in basename.\n bad = sorted(set(component) & _WINDOWS_FORBIDDEN_CHARS)\n if bad:\n return f\"Path component {component!r} contains forbidden character(s): {bad}\"\n # Trailing dot or space — Windows silently strips them.\n if component != component.rstrip(\". \"):\n return f\"Path component {component!r} has trailing dot or space (silently stripped on Windows)\"\n return None\n\n\nclass _ReadFileArgs(BaseModel):\n path: str = Field(..., description=\"Path to the file, relative to the sandbox root.\")\n offset: int | None = Field(default=None, description=\"1-based line number to start reading from.\")\n limit: int | None = Field(default=None, description=\"Maximum number of lines to return.\")\n\n\nclass _WriteFileArgs(BaseModel):\n path: str = Field(..., description=\"Path to the file, relative to the sandbox root.\")\n content: str = Field(..., description=\"Text content to write. Overwrites existing file.\")\n\n\nclass _EditFileArgs(BaseModel):\n path: str = Field(..., description=\"Path to the file, relative to the sandbox root.\")\n old_string: str = Field(..., description=\"Exact string to replace.\")\n new_string: str = Field(..., description=\"Replacement string.\")\n replace_all: bool = Field(default=False, description=\"Replace every occurrence instead of failing on ambiguity.\")\n\n\nclass _GlobSearchArgs(BaseModel):\n pattern: str = Field(..., description=\"Glob pattern, e.g. '**/*.py'.\")\n path: str | None = Field(default=None, description=\"Optional sub-directory to scope the search.\")\n\n\nclass _GrepSearchArgs(BaseModel):\n pattern: str = Field(..., description=\"Pattern to match against file contents (literal substring by default).\")\n path: str | None = Field(default=None, description=\"Optional file or directory to scope the search.\")\n glob: str | None = Field(default=None, description=\"Optional glob filter, e.g. '*.py'.\")\n case_insensitive: bool = Field(default=False, description=\"If true, the pattern is matched case-insensitively.\")\n output_mode: str = Field(\n default=\"files_with_matches\",\n description=\"One of 'files_with_matches', 'content', 'count'.\",\n )\n is_regex: bool = Field(\n default=False,\n description=(\n \"Treat `pattern` as a Python regex. Disabled by default — when enabled, \"\n \"patterns are validated against catastrophic-backtracking shapes \"\n \"(rejecting e.g. `(a+)+` or `(.*)*`) and oversized lines are skipped.\"\n ),\n )\n\n\nclass FileSystemToolComponent(Component):\n display_name = \"File System\"\n description = \"Sandboxed filesystem access for agents.\"\n icon = \"folder\"\n name = \"FileSystemTool\"\n\n # Enables the \"Tool Mode\" toggle on the node header. When ON the framework\n # calls _get_tools() and emits a Toolset output that connects to an Agent's\n # \"Tools\" handle. When OFF, the JSON metadata output is the only handle.\n add_tool_output = True\n\n inputs = [\n StrInput(\n name=\"root_path\",\n display_name=\"Workspace Sub-path\",\n required=False,\n value=\"\",\n info=(\n \"Sub-folder inside your sandboxed workspace. Leave empty to use the \"\n \"root of the workspace.\\n\\n\"\n \"Resolves under /shared// when AUTO_LOGIN=True \"\n \"(single-user / desktop) or under /users/// \"\n \"when AUTO_LOGIN=False (multi-user, per-user isolation). \"\n \"BASE_DIR is operator-controlled via LANGFLOW_FS_TOOL_BASE_DIR.\"\n ),\n ),\n BoolInput(\n name=\"read_only\",\n display_name=\"Read Only\",\n value=False,\n advanced=True,\n info=\"If true, write and edit operations are disabled.\",\n ),\n # Synthetic hidden input. Exists ONLY to make the \"Tool Mode\" toggle\n # appear on the node header — see frontend rule in\n # `src/frontend/src/CustomNodes/helpers/parameter-filtering.ts :: isHidden`\n # and the backend gate in `src/lfx/src/lfx/template/utils.py` (toggle\n # visibility = `any(input.tool_mode for input in inputs)`).\n # `show=False` keeps it hidden from the config UI; `tool_mode=True`\n # would otherwise hide a real input in tool mode (see same isHidden\n # rule), so we keep root_path / read_only WITHOUT tool_mode=True so\n # they remain user-editable when the toggle is on. _get_tools() ignores\n # this field — each StructuredTool has its own per-operation schema.\n StrInput(\n name=\"tool_mode_trigger\",\n display_name=\"\",\n show=False,\n tool_mode=True,\n required=False,\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"JSON\",\n name=\"metadata\",\n method=\"build_metadata\",\n types=[\"Data\"],\n ),\n ]\n\n def build_metadata(self) -> Data:\n \"\"\"Return introspective metadata about the sandbox. No file I/O.\n\n Surfaces the live AUTO_LOGIN-driven layout so flow authors can see\n whether per-user scoping is active and where their files actually land\n without needing to grep env vars.\n \"\"\"\n registered = [\"read_file\", \"glob_search\", \"grep_search\"]\n if not self.read_only:\n registered.extend([\"write_file\", \"edit_file\"])\n\n auto_login = self._resolve_auto_login()\n user_id = self._resolve_user_id()\n if auto_login:\n mode = \"shared\"\n elif user_id:\n mode = \"isolated\"\n else:\n mode = \"refused\"\n\n # Best-effort effective_root resolution. We never raise from metadata —\n # if the policy refuses the call (AUTO_LOGIN=False without user_id,\n # unwritable BASE_DIR, etc.) we surface the reason instead of crashing\n # the node preview / Agent build pipeline.\n effective_root: str | None\n resolution_error: str | None = None\n try:\n effective_root = str(self._validate_root())\n except Exception as exc: # noqa: BLE001 — metadata must never raise\n effective_root = None\n resolution_error = str(exc) or exc.__class__.__name__\n\n return Data(\n data={\n \"root_path\": self.root_path,\n \"read_only\": bool(self.read_only),\n \"tools_registered\": registered,\n \"auto_login\": auto_login,\n \"mode\": mode,\n \"effective_root\": effective_root,\n \"resolution_error\": resolution_error,\n }\n )\n\n async def _get_tools(self) -> list[Tool]:\n \"\"\"Tool Mode entrypoint. Called by Component.to_toolkit().\"\"\"\n # Capture the bound user_id ONCE per build. Each StructuredTool closure\n # below re-checks the live ``self._user_id`` against this value at call\n # time, so a tool issued for user A cannot be invoked after the\n # component has been reused for user B (defense for L2 in the plan).\n bound_user_id = self._resolve_user_id()\n tools: list[Tool] = [\n self._make_read_tool(bound_user_id=bound_user_id),\n self._make_glob_tool(bound_user_id=bound_user_id),\n self._make_grep_tool(bound_user_id=bound_user_id),\n ]\n if not self.read_only:\n tools.append(self._make_write_tool(bound_user_id=bound_user_id))\n tools.append(self._make_edit_tool(bound_user_id=bound_user_id))\n return tools\n\n def _user_binding_error(self, bound_user_id: str | None) -> dict | None:\n \"\"\"Return a structured error dict if the live user_id has shifted.\n\n In shared mode (AUTO_LOGIN=True) user_id is not part of the security\n boundary, so the binding check is a no-op. In isolated mode we compare\n the captured user_id to the live one and refuse if they diverge —\n defends against a worker pool reusing one component instance across\n sessions for different users.\n \"\"\"\n _, err = self._user_binding_check(bound_user_id)\n return err\n\n def _user_binding_check(self, bound_user_id: str | None) -> tuple[str | None, dict | None]:\n \"\"\"Atomic capture of the user_id for the current invocation.\n\n Returns ``(captured_user_id, error_or_none)``. The captured value is\n the SINGLE read of ``_resolve_user_id`` for this call; downstream\n path-resolution code MUST reuse it (via the ContextVar pin) instead\n of reading again — otherwise a mid-call mutation of ``self._user_id``\n would let a write land in a different user's namespace while this\n check still passed for the original user.\n \"\"\"\n # When force_isolation is set, every invocation MUST carry the user_id\n # that was captured at binding time — AUTO_LOGIN does not relax the\n # check here, otherwise the per-user root in _validate_root would be\n # reached with the wrong identity.\n if getattr(self, \"_force_isolation\", False):\n current = self._resolve_user_id()\n if current and current == bound_user_id:\n return current, None\n return None, {\n \"error\": (\n \"tool/user-id mismatch: this tool was bound to a different user session and cannot be reused\"\n ),\n }\n if self._resolve_auto_login():\n return bound_user_id, None\n current = self._resolve_user_id()\n if current == bound_user_id:\n return current, None\n return None, {\n \"error\": (\"tool/user-id mismatch: this tool was bound to a different user session and cannot be reused\"),\n }\n\n def _make_read_tool(self, *, bound_user_id: str | None) -> StructuredTool:\n def _run(path: str, offset: int | None = None, limit: int | None = None) -> str:\n captured, err = self._user_binding_check(bound_user_id)\n if err is not None:\n return json.dumps(err)\n token = _pinned_user_id_var.set(captured)\n try:\n return json.dumps(self._read_file(path, offset=offset, limit=limit))\n finally:\n _pinned_user_id_var.reset(token)\n\n return StructuredTool.from_function(\n name=\"read_file\",\n description=(\n \"Read a text file from the sandboxed workspace. \"\n \"Returns content prefixed with line numbers plus metadata \"\n \"(total_lines, start_line, num_lines).\"\n ),\n func=_run,\n args_schema=_ReadFileArgs,\n tags=[\"read_file\"],\n )\n\n def _make_write_tool(self, *, bound_user_id: str | None) -> StructuredTool:\n def _run(path: str, content: str) -> str:\n captured, err = self._user_binding_check(bound_user_id)\n if err is not None:\n return json.dumps(err)\n token = _pinned_user_id_var.set(captured)\n try:\n return json.dumps(self._write_file(path, content))\n finally:\n _pinned_user_id_var.reset(token)\n\n return StructuredTool.from_function(\n name=\"write_file\",\n description=\"Create or overwrite a text file inside the sandboxed workspace.\",\n func=_run,\n args_schema=_WriteFileArgs,\n tags=[\"write_file\"],\n )\n\n def _make_edit_tool(self, *, bound_user_id: str | None) -> StructuredTool:\n def _run(path: str, old_string: str, new_string: str, *, replace_all: bool = False) -> str:\n captured, err = self._user_binding_check(bound_user_id)\n if err is not None:\n return json.dumps(err)\n token = _pinned_user_id_var.set(captured)\n try:\n return json.dumps(\n self._edit_file(path, old_string=old_string, new_string=new_string, replace_all=replace_all)\n )\n finally:\n _pinned_user_id_var.reset(token)\n\n return StructuredTool.from_function(\n name=\"edit_file\",\n description=(\n \"Edit a text file by replacing an exact old_string with new_string. \"\n \"Fails on ambiguous matches unless replace_all=True.\"\n ),\n func=_run,\n args_schema=_EditFileArgs,\n tags=[\"edit_file\"],\n )\n\n def _make_glob_tool(self, *, bound_user_id: str | None) -> StructuredTool:\n def _run(pattern: str, path: str | None = None) -> str:\n captured, err = self._user_binding_check(bound_user_id)\n if err is not None:\n return json.dumps(err)\n token = _pinned_user_id_var.set(captured)\n try:\n return json.dumps(self._glob_search(pattern, path=path))\n finally:\n _pinned_user_id_var.reset(token)\n\n return StructuredTool.from_function(\n name=\"glob_search\",\n description=(\n \"List files matching a glob pattern inside the sandboxed workspace. \"\n f\"Results are truncated at {GLOB_RESULT_LIMIT} entries.\"\n ),\n func=_run,\n args_schema=_GlobSearchArgs,\n tags=[\"glob_search\"],\n )\n\n def _make_grep_tool(self, *, bound_user_id: str | None) -> StructuredTool:\n def _run(\n pattern: str,\n path: str | None = None,\n glob: str | None = None,\n *,\n case_insensitive: bool = False,\n output_mode: str = \"files_with_matches\",\n is_regex: bool = False,\n ) -> str:\n captured, err = self._user_binding_check(bound_user_id)\n if err is not None:\n return json.dumps(err)\n token = _pinned_user_id_var.set(captured)\n try:\n return json.dumps(\n self._grep_search(\n pattern,\n path=path,\n glob=glob,\n case_insensitive=case_insensitive,\n output_mode=output_mode,\n is_regex=is_regex,\n )\n )\n finally:\n _pinned_user_id_var.reset(token)\n\n return StructuredTool.from_function(\n name=\"grep_search\",\n description=(\n \"Search file contents. Default is literal substring match (safe for any pattern); \"\n \"set is_regex=True to opt into Python regex. Patterns with nested unbounded \"\n \"quantifiers (e.g. (a+)+) are rejected to prevent catastrophic backtracking. \"\n \"output_mode: 'files_with_matches' (default), 'content', or 'count'. \"\n f\"Content mode is capped at {GREP_LINE_LIMIT} lines.\"\n ),\n func=_run,\n args_schema=_GrepSearchArgs,\n tags=[\"grep_search\"],\n )\n\n def _resolve_user_id(self) -> str | None:\n \"\"\"Best-effort lookup of the calling user's id.\n\n The Component base class populates ``_user_id`` during instantiation,\n and the property cascades to ``self.graph.user_id``. In tests there is\n no graph; in scheduled/anonymous runs there is no user_id at all. We\n treat all of those as \"anonymous\" — the isolation mode decides what to\n do with that information.\n\n Why we filter ``\"none\"`` / ``\"null\"``: Langflow's ``PlaceholderGraph``\n stringifies a missing user as ``\"None\"`` rather than the Python\n ``None`` value, so a naive truthiness check would mistake \"no user\" for\n a real user named \"None\" and create a spurious shared namespace.\n\n L2 binding atomicity: when a tool invocation is in progress, the\n pinned ContextVar value takes precedence so every read inside the\n same call sees the user_id captured at the binding check — even if\n ``self._user_id`` is mutated mid-call by a reused component instance.\n \"\"\"\n pinned = _pinned_user_id_var.get()\n if pinned is not None:\n return pinned\n candidates: list[object | None] = [getattr(self, \"_user_id\", None)]\n graph = getattr(self, \"graph\", None)\n if graph is not None:\n candidates.append(getattr(graph, \"user_id\", None))\n\n for value in candidates:\n if value is None:\n continue\n cleaned = str(value).strip()\n if not cleaned or cleaned.lower() in {\"none\", \"null\"}:\n continue\n return cleaned\n return None\n\n def _isolation_config(self) -> IsolationConfig:\n \"\"\"Read the on-disk layout config from the environment on every call.\n\n Re-read intentionally: tests and live operators tweak env vars between\n runs; caching would create stale-config bugs that are painful to\n diagnose. The cost is a handful of dict lookups per tool call.\n \"\"\"\n return load_isolation_config(env=os.environ, default_config_dir=_default_config_dir())\n\n def _resolve_auto_login(self) -> bool:\n \"\"\"Return AUTO_LOGIN from the live settings service.\n\n Defaults to True when the settings service is unavailable (tests, very\n early bootstrap) — mirrors the platform-wide default and keeps the\n component usable in lightweight contexts. Tests can override this\n method per-instance to pin the desired mode.\n \"\"\"\n try:\n settings_service = get_settings_service()\n except Exception: # noqa: BLE001 — service registry may not be ready\n return True\n if settings_service is None:\n return True\n try:\n return bool(settings_service.auth_settings.AUTO_LOGIN)\n except AttributeError:\n return True\n\n def _validate_root(self) -> Path:\n \"\"\"Resolve and authorize the effective sandbox root.\n\n Dispatch:\n - ``_force_isolation=True`` → /users//\n - AUTO_LOGIN=True → /shared/\n - AUTO_LOGIN=False + user_id → /users//\n - any mode with no user_id → PermissionError (caught by callers)\n\n ``_force_isolation`` exists for callers that carry an authenticated\n user identity and need per-user isolation regardless of the global\n AUTO_LOGIN flag (e.g. the agentic file router + the agent's write\n tools). It defaults to False so other call sites keep their current\n AUTO_LOGIN-driven behavior unchanged.\n \"\"\"\n config = self._isolation_config()\n\n if getattr(self, \"_force_isolation\", False):\n user_id = self._resolve_user_id()\n if not user_id:\n msg = \"FileSystemTool requires an authenticated user when _force_isolation is set\"\n raise PermissionError(msg)\n return self._isolated_user_root(config=config, user_id=user_id)\n\n if self._resolve_auto_login():\n return self._shared_root(config=config)\n\n user_id = self._resolve_user_id()\n if not user_id:\n msg = \"FileSystemTool requires an authenticated user when AUTO_LOGIN=False\"\n raise PermissionError(msg)\n return self._isolated_user_root(config=config, user_id=user_id)\n\n def _shared_root(self, *, config: IsolationConfig) -> Path:\n \"\"\"Materialize ``/shared/`` and verify the boundary.\n\n Why a fixed ``shared/`` prefix (and not just ``base_dir`` directly):\n keeps the on-disk layout symmetric with isolated mode (``users//``)\n so flipping AUTO_LOGIN at deploy time never mixes the two trees.\n \"\"\"\n shared_root = (config.base_dir / SHARED_NAMESPACE).resolve()\n try:\n shared_root.mkdir(parents=True, exist_ok=True)\n except OSError as exc:\n msg = (\n f\"Cannot create shared workspace at {shared_root}: \"\n f\"{exc.strerror or exc}. \"\n f\"Check that LANGFLOW_FS_TOOL_BASE_DIR ({config.base_dir}) is writable \"\n f\"by the Langflow process user.\"\n )\n raise PermissionError(msg) from exc\n\n sub_raw = (self.root_path or \"\").strip()\n sub = sub_raw.lstrip(\"/\\\\\")\n candidate = (shared_root / sub).resolve() if sub else shared_root\n if not candidate.is_relative_to(shared_root):\n msg = f\"sub_path {self.root_path!r} escapes shared workspace\"\n raise PermissionError(msg)\n try:\n candidate.mkdir(parents=True, exist_ok=True)\n except OSError as exc:\n msg = f\"Cannot create sub-path {candidate}: {exc.strerror or exc}\"\n raise PermissionError(msg) from exc\n return candidate\n\n def _isolated_user_root(self, *, config: IsolationConfig, user_id: str) -> Path:\n \"\"\"Materialize ``/users//`` and verify the boundary.\"\"\"\n try:\n pepper = load_or_create_pepper(config.pepper_path)\n except OSError as exc:\n msg = (\n f\"Cannot access pepper file at {config.pepper_path}: \"\n f\"{exc.strerror or exc}. \"\n f\"Check that LANGFLOW_FS_TOOL_PEPPER_PATH points to a writable location.\"\n )\n raise PermissionError(msg) from exc\n namespace = compute_user_namespace(user_id, pepper=pepper)\n user_root = (config.base_dir / namespace).resolve()\n try:\n user_root.mkdir(parents=True, exist_ok=True)\n except OSError as exc:\n msg = (\n f\"Cannot create user namespace at {user_root}: \"\n f\"{exc.strerror or exc}. \"\n f\"Check that LANGFLOW_FS_TOOL_BASE_DIR ({config.base_dir}) is writable \"\n f\"by the Langflow process user.\"\n )\n raise PermissionError(msg) from exc\n\n sub_raw = (self.root_path or \"\").strip()\n # Strip leading separators so absolute-looking sub_paths are pinned\n # under the user root rather than escaping to the host filesystem.\n sub = sub_raw.lstrip(\"/\\\\\")\n candidate = (user_root / sub).resolve() if sub else user_root\n if not candidate.is_relative_to(user_root):\n msg = f\"sub_path {self.root_path!r} escapes user namespace\"\n raise PermissionError(msg)\n try:\n candidate.mkdir(parents=True, exist_ok=True)\n except OSError as exc:\n msg = f\"Cannot create sub-path {candidate}: {exc.strerror or exc}\"\n raise PermissionError(msg) from exc\n return candidate\n\n def _validate_path(self, path: str) -> Path:\n \"\"\"Resolve `path` relative to the sandbox root and reject any escape.\n\n Why raise: internal helper for control flow; each tool operation catches\n PermissionError and translates it into a structured error for the agent.\n \"\"\"\n if \"\\x00\" in path:\n msg = \"Path contains NUL byte\"\n raise PermissionError(msg)\n if portability_error := _check_windows_portability(path):\n raise PermissionError(portability_error)\n # Block the reserved segments (`.lfsig` integrity hook, `.components`\n # registered-user-component store). We forbid traversal even by users\n # with valid credentials — agents and humans both — because the\n # privilege model depends on these directories being unreachable from\n # the public tool surface.\n # Compare case-insensitively: APFS and NTFS resolve `.LFSIG` to the\n # same directory as `.lfsig`, so a case-sensitive equality check\n # lets uppercase variants slip past the reservation on those\n # filesystems. `casefold` is the locale-aware lowercase used for\n # caseless string comparison — broader than `.lower()`.\n reserved_folds = {seg.casefold() for seg in RESERVED_SEGMENTS}\n for part in PureWindowsPath(path).parts:\n folded = part.casefold()\n if folded in reserved_folds:\n # Surface the canonical segment name (with leading dot) so\n # error messages stay stable across modes and OSes.\n canonical = next(seg for seg in RESERVED_SEGMENTS if seg.casefold() == folded)\n msg = f\"Path component {canonical!r} is reserved\"\n raise PermissionError(msg)\n root_resolved = self._validate_root()\n candidate = (root_resolved / path).resolve()\n if not candidate.is_relative_to(root_resolved):\n msg = f\"Path escapes workspace boundary: {path}\"\n raise PermissionError(msg)\n if hardlink_error := _check_hardlink(candidate):\n raise PermissionError(hardlink_error)\n return candidate\n\n def _read_file(self, path: str, offset: int | None = None, limit: int | None = None) -> dict:\n try:\n resolved = self._validate_path(path)\n except PermissionError as exc:\n return {\"error\": str(exc), \"path\": path}\n\n if not resolved.exists():\n return {\"error\": f\"File not found: {path}\", \"path\": path}\n if resolved.is_dir():\n return {\"error\": f\"Path is a directory, not a file: {path}\", \"path\": path}\n\n try:\n size = resolved.stat().st_size\n except OSError as exc:\n return {\"error\": f\"Cannot stat file: {exc.strerror or exc}\", \"path\": path}\n if size > MAX_FILE_SIZE_BYTES:\n return {\n \"error\": f\"File size {size} exceeds limit of {MAX_FILE_SIZE_BYTES} bytes\",\n \"path\": path,\n }\n\n try:\n head = _read_head_no_follow(resolved, BINARY_SNIFF_BYTES)\n except OSError as exc:\n return {\"error\": f\"Cannot read file: {exc.strerror or exc}\", \"path\": path}\n if _looks_binary(head):\n return {\"error\": f\"Refusing to read binary file: {path}\", \"path\": path}\n\n try:\n text = _read_bytes_no_follow(resolved).decode(\"utf-8\", errors=\"replace\")\n except OSError as exc:\n return {\"error\": f\"Cannot read file: {exc.strerror or exc}\", \"path\": path}\n\n lines = text.splitlines()\n total = len(lines)\n start = offset if offset and offset > 0 else 1\n end = total if limit is None else min(total, start - 1 + limit)\n window = lines[start - 1 : end]\n numbered = \"\\n\".join(f\"{i:>6}→{line}\" for i, line in enumerate(window, start=start))\n return {\n \"status\": \"ok\",\n \"path\": path,\n \"content\": numbered,\n \"total_lines\": total,\n \"start_line\": start,\n \"num_lines\": len(window),\n }\n\n def _write_file(self, path: str, content: str) -> dict:\n try:\n resolved = self._validate_path(path)\n except PermissionError as exc:\n return {\"error\": str(exc), \"path\": path}\n\n encoded = content.encode(\"utf-8\")\n if len(encoded) > MAX_FILE_SIZE_BYTES:\n return {\n \"error\": f\"Content size {len(encoded)} exceeds limit of {MAX_FILE_SIZE_BYTES} bytes\",\n \"path\": path,\n }\n\n if resolved.is_dir():\n return {\"error\": f\"Path is a directory, not a file: {path}\", \"path\": path}\n # Auto-create missing parent directories. `_validate_path` has already\n # confirmed `resolved` lies inside the sandbox root, so every ancestor\n # of `resolved.parent` is also inside the root — `mkdir(parents=True)`\n # cannot escape the sandbox.\n try:\n resolved.parent.mkdir(parents=True, exist_ok=True)\n except OSError as exc:\n return {\"error\": f\"Cannot create parent directory: {exc.strerror or exc}\", \"path\": path}\n\n existed = resolved.exists()\n try:\n _write_bytes_no_follow(resolved, encoded)\n except OSError as exc:\n return {\"error\": f\"Cannot write file: {exc.strerror or exc}\", \"path\": path}\n\n return {\n \"status\": \"updated\" if existed else \"created\",\n \"path\": path,\n \"bytes_written\": len(encoded),\n }\n\n def _edit_file(\n self,\n path: str,\n old_string: str,\n new_string: str,\n *,\n replace_all: bool = False,\n ) -> dict:\n try:\n resolved = self._validate_path(path)\n except PermissionError as exc:\n return {\"error\": str(exc), \"path\": path}\n\n if not resolved.exists():\n return {\"error\": f\"File not found: {path}\", \"path\": path}\n if resolved.is_dir():\n return {\"error\": f\"Path is a directory, not a file: {path}\", \"path\": path}\n\n # Reject empty old_string outright. With `old_string=\"\"`, str.replace\n # inserts new_string between every character (and at both ends),\n # producing roughly N*len(new_string) extra bytes — a small file plus\n # a large new_string can blow far past MAX_FILE_SIZE_BYTES before the\n # post-construction guard runs. Empty old_string also has no\n # well-defined semantics for \"edit this exact match\".\n if old_string == \"\":\n return {\"error\": \"old_string must not be empty\", \"path\": path}\n\n try:\n current = _read_bytes_no_follow(resolved).decode(\"utf-8\", errors=\"replace\")\n except OSError as exc:\n return {\"error\": f\"Cannot read file: {exc.strerror or exc}\", \"path\": path}\n\n occurrences = current.count(old_string)\n if occurrences == 0:\n return {\"error\": f\"old_string not found in file: {path}\", \"path\": path}\n if occurrences > 1 and not replace_all:\n return {\n \"error\": (\n f\"old_string is ambiguous: {occurrences} multiple matches in {path}. \"\n \"Pass replace_all=True to replace every occurrence.\"\n ),\n \"path\": path,\n \"matches\": occurrences,\n }\n\n # Project the encoded size BEFORE building the replacement string.\n # str.replace allocates the full result up front, so a small file\n # with replace_all=True and a large new_string can balloon memory\n # before the post-allocation length check runs. UTF-8 byte counts are\n # exact (no estimation) so the projection is precise.\n old_bytes = len(old_string.encode(\"utf-8\"))\n new_bytes = len(new_string.encode(\"utf-8\"))\n current_bytes = len(current.encode(\"utf-8\"))\n replacements_planned = occurrences if replace_all else 1\n projected_bytes = current_bytes + replacements_planned * (new_bytes - old_bytes)\n if projected_bytes > MAX_FILE_SIZE_BYTES:\n return {\n \"error\": (\n f\"Resulting content size {projected_bytes} would exceed limit of {MAX_FILE_SIZE_BYTES} bytes\"\n ),\n \"path\": path,\n }\n\n updated = current.replace(old_string, new_string) if replace_all else current.replace(old_string, new_string, 1)\n encoded = updated.encode(\"utf-8\")\n # Defensive re-check (should be unreachable given the projection above,\n # but keeps the original invariant explicit).\n if len(encoded) > MAX_FILE_SIZE_BYTES:\n return {\n \"error\": f\"Resulting content size {len(encoded)} exceeds limit of {MAX_FILE_SIZE_BYTES} bytes\",\n \"path\": path,\n }\n\n try:\n _write_bytes_no_follow(resolved, encoded)\n except OSError as exc:\n return {\"error\": f\"Cannot write file: {exc.strerror or exc}\", \"path\": path}\n\n return {\n \"status\": \"ok\",\n \"path\": path,\n \"replacements\": occurrences if replace_all else 1,\n \"old_string\": old_string,\n \"new_string\": new_string,\n }\n\n def _glob_search(self, pattern: str, path: str | None = None) -> dict:\n # Empty patterns make ``Path.glob`` raise ``ValueError`` instead of\n # returning an empty match list, which would propagate uncaught and\n # crash the agent. Surface a structured error instead.\n if not pattern or not pattern.strip():\n return {\"error\": \"Pattern must not be empty\", \"pattern\": pattern, \"path\": path}\n # Reject traversal segments in the pattern itself. Without this guard,\n # ``base.glob(\"../*\")`` walks one directory up; one of the resulting\n # paths (``/../``) resolves back to ``base`` and\n # is surfaced to the agent as ``\".\"`` — a silent, misleading hit.\n # ``PureWindowsPath`` splits on both ``/`` and ``\\`` so the check\n # covers Windows-authored patterns as well.\n if any(part == \"..\" for part in PureWindowsPath(pattern).parts):\n return {\n \"error\": \"Pattern must not contain '..' (path traversal not allowed)\",\n \"pattern\": pattern,\n \"path\": path,\n }\n try:\n root_resolved = self._validate_root()\n base = self._validate_path(path) if path else root_resolved\n except PermissionError as exc:\n return {\"error\": str(exc), \"pattern\": pattern, \"path\": path}\n\n if not base.exists() or not base.is_dir():\n return {\"error\": f\"Base path is not a directory: {path or '.'}\", \"pattern\": pattern}\n\n # Collect up to GLOB_SCAN_CEILING (>> GLOB_RESULT_LIMIT) so we can\n # surface diverse branches even when one directory dominates the\n # iteration order. See BUG-02 / T2-001.\n collected: list[str] = []\n for match in base.glob(pattern):\n try:\n resolved_match = match.resolve()\n except OSError:\n continue\n if not resolved_match.is_relative_to(root_resolved):\n continue\n collected.append(str(resolved_match.relative_to(root_resolved)))\n if len(collected) >= GLOB_SCAN_CEILING:\n break\n\n # Sort for cross-filesystem determinism, then truncate to the visible\n # cap. Emit `truncated_branches` so the agent has a non-silent signal:\n # top-level branches under `base` whose files appear in the omitted\n # tail. The agent can narrow with `path=` to recover them.\n collected.sort()\n truncated = len(collected) > GLOB_RESULT_LIMIT\n if truncated:\n omitted = collected[GLOB_RESULT_LIMIT:]\n matches = collected[:GLOB_RESULT_LIMIT]\n truncated_branches = sorted({Path(p).parts[0] for p in omitted if Path(p).parts})\n else:\n matches = collected\n truncated_branches = []\n\n return {\n \"status\": \"ok\",\n \"pattern\": pattern,\n \"matches\": matches,\n \"truncated\": truncated,\n \"truncated_branches\": truncated_branches,\n }\n\n def _grep_search(\n self,\n pattern: str,\n path: str | None = None,\n glob: str | None = None,\n *,\n case_insensitive: bool = False,\n output_mode: str = \"files_with_matches\",\n is_regex: bool = False,\n ) -> dict:\n if output_mode not in GREP_OUTPUT_MODES:\n return {\n \"error\": f\"Invalid output_mode '{output_mode}'. Must be one of {GREP_OUTPUT_MODES}\",\n \"pattern\": pattern,\n }\n\n # Build a per-line matcher. Default is a literal substring check —\n # safe for any user-supplied pattern. Regex is opt-in and is gated by\n # a static heuristic plus per-line/per-file caps. Stdlib `re` cannot\n # be cancelled (it holds the GIL), so prevention is the only viable\n # mitigation: reject pathological patterns at compile time and bound\n # the input we hand to the engine.\n if is_regex:\n if len(pattern) > GREP_PATTERN_MAX_LEN:\n return {\n \"error\": (\n f\"Regex pattern exceeds {GREP_PATTERN_MAX_LEN} chars; \"\n \"shorten it or use literal mode (is_regex=False).\"\n ),\n \"pattern\": pattern,\n }\n if _looks_like_redos(pattern):\n return {\n \"error\": (\n \"Regex pattern rejected: nested unbounded quantifier \"\n \"(catastrophic-backtracking risk). Rewrite without an \"\n \"outer +/* on a group that already contains +, *, {n,}, \"\n \"or |, or use literal mode (is_regex=False).\"\n ),\n \"pattern\": pattern,\n }\n try:\n flags = re.IGNORECASE if case_insensitive else 0\n regex = re.compile(pattern, flags)\n except re.error as exc:\n return {\"error\": f\"Invalid regex pattern: {exc}\", \"pattern\": pattern}\n\n def line_matches(line: str) -> bool:\n # Skip regex matching on oversized lines — they are usually\n # data blobs, and a long input is a prerequisite for most\n # practical exponential-backtracking attacks.\n if len(line) > GREP_REGEX_LINE_MAX_LEN:\n return False\n return regex.search(line) is not None\n else:\n needle = pattern.lower() if case_insensitive else pattern\n\n def line_matches(line: str) -> bool:\n hay = line.lower() if case_insensitive else line\n return needle in hay\n\n try:\n root_resolved = self._validate_root()\n base = self._validate_path(path) if path else root_resolved\n except PermissionError as exc:\n return {\"error\": str(exc), \"pattern\": pattern, \"path\": path}\n\n if not base.exists():\n return {\"error\": f\"Path does not exist: {path or '.'}\", \"pattern\": pattern}\n\n targets = self._collect_grep_targets(base, glob)\n files_with_matches: list[str] = []\n content_matches: list[dict] = []\n count_matches: list[dict] = []\n line_budget = GREP_LINE_LIMIT\n truncated = False\n\n for file in targets:\n try:\n resolved_file = file.resolve()\n except OSError:\n continue\n if not resolved_file.is_relative_to(root_resolved):\n continue\n try:\n size = resolved_file.stat().st_size\n except OSError:\n continue\n if size > MAX_FILE_SIZE_BYTES:\n continue\n try:\n with resolved_file.open(\"rb\") as fh:\n head = fh.read(BINARY_SNIFF_BYTES)\n except OSError:\n continue\n if _looks_binary(head):\n continue\n try:\n text = resolved_file.read_text(encoding=\"utf-8\", errors=\"replace\")\n except OSError:\n continue\n\n rel_path = str(resolved_file.relative_to(root_resolved))\n per_file_count = 0\n matched_any = False\n for idx, line in enumerate(text.splitlines(), start=1):\n # Per-file scan cap (regex only — literal scans are O(n)).\n if is_regex and idx > GREP_REGEX_LINES_PER_FILE:\n break\n if not line_matches(line):\n continue\n matched_any = True\n per_file_count += 1\n if output_mode == \"content\":\n if line_budget <= 0:\n truncated = True\n break\n content_matches.append({\"path\": rel_path, \"line_number\": idx, \"line\": line})\n line_budget -= 1\n\n if matched_any:\n files_with_matches.append(rel_path)\n if output_mode == \"count\":\n count_matches.append({\"path\": rel_path, \"count\": per_file_count})\n\n if output_mode == \"content\" and truncated:\n break\n\n if output_mode == \"files_with_matches\":\n matches: list = files_with_matches\n elif output_mode == \"content\":\n matches = content_matches\n else: # count\n matches = count_matches\n\n return {\n \"status\": \"ok\",\n \"pattern\": pattern,\n \"output_mode\": output_mode,\n \"matches\": matches,\n \"truncated\": truncated,\n }\n\n def _collect_grep_targets(self, base: Path, glob: str | None) -> list[Path]:\n if base.is_file():\n return [base]\n all_files = [p for p in base.rglob(\"*\") if p.is_file()]\n if glob:\n return [p for p in all_files if p.match(glob)]\n return all_files\n" + "value": "\"\"\"Sandboxed filesystem tool component exposing 5 file I/O tools to agents.\"\"\"\n\nfrom __future__ import annotations\n\nimport contextvars\nimport json\nimport os\nimport re\nfrom pathlib import Path, PureWindowsPath\nfrom typing import TYPE_CHECKING\n\nfrom langchain_core.tools import StructuredTool\nfrom pydantic import BaseModel, Field\n\nfrom lfx.components.files_and_knowledge._filesystem_isolation import (\n IsolationConfig,\n load_isolation_config,\n)\nfrom lfx.components.files_and_knowledge._filesystem_namespace import (\n compute_user_namespace,\n load_or_create_pepper,\n)\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs.inputs import BoolInput, StrInput\nfrom lfx.io import Output\nfrom lfx.schema.data import Data\nfrom lfx.services.deps import get_settings_service\n\nif TYPE_CHECKING:\n from lfx.field_typing import Tool\n\n# Sub-directory under used when AUTO_LOGIN=True. Stays parallel to\n# users// so flipping AUTO_LOGIN at deploy-time does not mix file trees.\nSHARED_NAMESPACE = \"shared\"\n\n\n# L2 binding atomicity: the binding check captures `user_id` once at the\n# start of every tool invocation; subsequent reads of `_resolve_user_id`\n# during the same call (in _validate_root, _validate_path, etc.) read this\n# pinned value instead of re-resolving. Without this pin, a concurrent\n# mutation of `self._user_id` between the binding check and the path\n# resolution would let an attacker write into a foreign user's namespace\n# while the check still passed for the original user.\n_pinned_user_id_var: contextvars.ContextVar[str | None] = contextvars.ContextVar(\n \"_filesystem_pinned_user_id\",\n default=None,\n)\n\n\n# Reserved on-disk segments inside every user namespace.\n#\n# `.lfsig` is the forward hook for an HMAC sidecar tree (L3 in the FS plan).\n# `.components` is the storage location for user-generated component code\n# (written by a privileged backend helper after Layer-2 code validation;\n# read by the registry overlay so build_flow / search_components see the\n# user's custom types). Allowing the agent's FS tools to touch either\n# directory would either poison a security-critical namespace or let the\n# agent plant arbitrary code into its own executable namespace.\n#\n# Kept as a singular constant for backwards-compat with prior message text\n# (\"Path component '.lfsig' is reserved\") and as a tuple for the actual\n# check — both names point at the same casefold set.\nRESERVED_SEGMENT = \".lfsig\"\nRESERVED_SEGMENTS: tuple[str, ...] = (\".lfsig\", \".components\")\n\n\ndef _default_config_dir() -> Path:\n \"\"\"Pick a sensible config dir when no env var is set.\n\n Operators set ``LANGFLOW_FS_TOOL_BASE_DIR`` / ``LANGFLOW_FS_TOOL_PEPPER_PATH``\n explicitly in any real deployment; this fallback exists so the OSS desktop\n install just works without any setup.\n \"\"\"\n return Path.home() / \".langflow\" / \"fs_tool\"\n\n\nMAX_FILE_SIZE_BYTES = 10 * 1024 * 1024\nBINARY_SNIFF_BYTES = 8 * 1024\nGLOB_RESULT_LIMIT = 100\n# Hard upper bound on the number of glob matches collected before truncation.\n# Without scanning past `GLOB_RESULT_LIMIT`, the first big branch hit by\n# `os.scandir` order fills the cap and entire nested branches are silently\n# dropped (BUG-02 / T2-001). The ceiling bounds memory/time on pathological\n# trees while leaving headroom to surface diverse branches to the agent.\nGLOB_SCAN_CEILING = GLOB_RESULT_LIMIT * 10\nGREP_LINE_LIMIT = 250\nGREP_OUTPUT_MODES = (\"files_with_matches\", \"content\", \"count\")\n# Hard ceiling on user-supplied regex pattern length. Long patterns are a\n# common ReDoS vector and have no legitimate need against single text lines.\nGREP_PATTERN_MAX_LEN = 1024\n# Skip regex matching on lines longer than this — exponential backtracking\n# requires non-trivial input length, and oversized lines tend to be data\n# blobs (minified JS/JSON) rather than meaningful matches.\nGREP_REGEX_LINE_MAX_LEN = 4096\n# Cap on total lines scanned per file in regex mode, regardless of matches.\n# Bounds the work even for benign-looking patterns on large files.\nGREP_REGEX_LINES_PER_FILE = 50_000\n\n# Heuristic ReDoS detector.\n#\n# Stdlib `re` has no native timeout; we cannot kill a runaway match because\n# the engine holds the GIL. Instead we reject patterns whose structure makes\n# catastrophic backtracking possible BEFORE compiling them.\n#\n# The classic ReDoS shape is a nested unbounded quantifier — a parenthesized\n# group whose body is itself quantifiable, followed by an outer `+`/`*`/`{n,}`.\n# Examples: `(a+)+`, `(a*)*`, `(.*)+`, `(\\w+)+`, `(a|aa)+`. We reject any\n# group that ends with `)+`/`)*`/`){n,}` AND whose body contains an\n# unescaped quantifier (`+`, `*`, `{n,}`) or alternation (`|`).\n#\n# This is intentionally conservative — it rules out some safe patterns\n# (`(ab)+` is fine; `(ab+)+` is not, but our rule rejects both). Users who\n# need those patterns can rewrite without the outer quantifier. Trading a\n# bit of expressiveness for a hard guarantee that no user pattern can\n# DoS the worker.\n_REDOS_GROUP = re.compile(\n r\"\"\"\n \\( # outer group open\n (?: # body of the group:\n (?:\\\\.) # - any escaped char (so \\+ etc. don't trip us)\n | [^()] # - or any non-paren char\n )*?\n (?: # ... that contains at least one of:\n (? bool:\n \"\"\"Return True if `pattern` matches a known catastrophic-backtracking shape.\"\"\"\n return bool(_REDOS_GROUP.search(pattern))\n\n\n# Windows portability rules (applied on every host OS so flows authored on\n# macOS/Linux do not silently break when run on Windows). Pure-string checks.\n_WINDOWS_RESERVED_NAMES = frozenset(\n {\"CON\", \"PRN\", \"AUX\", \"NUL\", *(f\"COM{i}\" for i in range(1, 10)), *(f\"LPT{i}\" for i in range(1, 10))}\n)\n# Note: ':' is NOT included — drive letters (C:) are valid; bare ':' inside a\n# basename will be caught by the OS at write time anyway.\n_WINDOWS_FORBIDDEN_CHARS = frozenset('<>\"|?*')\n\n# Deny-list: even when a path lies inside `root_path`, refuse access to\n# basenames or path components that match well-known credential / secret\n# patterns. This is the \"default-deny inside an allowed root\" pattern from\n# Claude Code Sandboxing — it limits the blast radius of a flow author who\n# misconfigures `root_path` to cover $HOME or the project root. Pure-string\n# checks; runs before any I/O so the agent never observes the file's\n# existence, contents, or absence-vs-denied distinction beyond the error\n# string itself.\n#\n# Match semantics (all case-insensitive):\n# * literals — exact basename match (e.g. `.env`, `.netrc`)\n# * prefixes — basename startswith (e.g. `id_rsa`, `id_rsa.pub`, `id_ed25519`)\n# * suffixes — basename endswith (e.g. `cert.pem`, `private.key`)\n# * fragments — any path component equals the fragment, including the\n# basename (e.g. `.ssh/`, `.aws/config`, or a bare `.ssh` request). Matching\n# the basename too prevents requesting a protected directory directly (which\n# would leak its existence) and stops a `**/*` glob from returning the\n# protected directory entry itself even when its children are filtered.\n_DENY_BASENAME_LITERALS = frozenset({\".env\", \".netrc\", \".pgpass\", \".htpasswd\", \"authorized_keys\"})\n_DENY_BASENAME_PREFIXES = (\"id_rsa\", \"id_dsa\", \"id_ecdsa\", \"id_ed25519\", \"credentials\")\n_DENY_BASENAME_SUFFIXES = (\".pem\", \".key\", \".pfx\", \".p12\")\n_DENY_PATH_FRAGMENTS = frozenset({\".ssh\", \".aws\", \".gnupg\", \".docker\", \".kube\", \".git\"})\n\n\ndef _check_deny_list(path: str) -> str | None:\n parts = PureWindowsPath(path).parts\n if not parts:\n return None\n *directories, basename = parts\n folded = basename.casefold()\n if folded in _DENY_PATH_FRAGMENTS:\n return f\"Access to {basename!r} is denied: name matches a protected directory\"\n if (\n folded in _DENY_BASENAME_LITERALS\n or folded.startswith(_DENY_BASENAME_PREFIXES)\n or folded.endswith(_DENY_BASENAME_SUFFIXES)\n ):\n return f\"Access to {basename!r} is denied: filename matches a protected credential pattern\"\n for segment in directories:\n if segment.casefold() in _DENY_PATH_FRAGMENTS:\n return f\"Access to {path!r} is denied: path contains protected directory {segment!r}\"\n return None\n\n\ndef _looks_binary(head: bytes) -> bool:\n return b\"\\x00\" in head\n\n\ndef _check_hardlink(candidate: Path) -> str | None:\n \"\"\"Return an error string when ``candidate`` is a multi-hardlink file.\n\n Why we refuse multi-hardlink files: an attacker with write access to a\n location outside the sandbox can pre-create an extra hardlink pointing\n at a sandbox path. Subsequent writes through the sandbox name then\n also clobber the external name, defeating the boundary. There is no\n legitimate flow that depends on a multi-link inode inside the\n sandbox, so refusing fails closed.\n\n Restricted to **regular files**: directories on POSIX always have\n ``st_nlink >= 2`` (`.`, plus one per subdirectory entry), so checking\n them would refuse every nested path. Symlinks fall through to the\n boundary check and the no-follow helpers. Non-existent paths return\n ``None`` — creation is handled by the O_NOFOLLOW write helper.\n \"\"\"\n try:\n st = os.lstat(candidate)\n except FileNotFoundError:\n return None\n except OSError as exc:\n return f\"Cannot stat path: {exc.strerror or exc}\"\n import stat as _stat_module\n\n if _stat_module.S_ISREG(st.st_mode) and st.st_nlink > 1:\n return f\"Refusing to operate on multi-hardlink file (nlink={st.st_nlink})\"\n return None\n\n\ndef _write_bytes_no_follow(target: Path, data: bytes) -> None:\n \"\"\"Write ``data`` to ``target`` without following symlinks.\n\n Uses ``O_NOFOLLOW`` on POSIX so that any symlink at ``target`` —\n including one created by a concurrent process between path\n validation and this open — raises ``ELOOP`` and fails the write\n closed. On Windows the flag is unavailable; we lstat and refuse if\n the target is already a symlink (best-effort against the non-racy\n attacker).\n\n The file is created with mode 0600 so that, even on shared hosts,\n other users cannot read it without explicit operator action.\n \"\"\"\n flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC\n if hasattr(os, \"O_NOFOLLOW\"):\n flags |= os.O_NOFOLLOW\n elif target.is_symlink(): # pragma: no cover — Windows-only branch\n msg = f\"Refusing to write through symlink: {target}\"\n raise PermissionError(msg)\n fd = os.open(str(target), flags, 0o600)\n try:\n os.write(fd, data)\n finally:\n os.close(fd)\n\n\ndef _read_bytes_no_follow(target: Path) -> bytes:\n \"\"\"Read the entire file at ``target`` without following symlinks.\n\n Same TOCTOU rationale as ``_write_bytes_no_follow``: with\n ``O_NOFOLLOW`` the open fails if a symlink was substituted between\n path validation and this read. Reads up to ``MAX_FILE_SIZE_BYTES``\n in a single syscall — the caller has already enforced the cap via\n ``stat().st_size`` checks.\n \"\"\"\n flags = os.O_RDONLY\n if hasattr(os, \"O_NOFOLLOW\"):\n flags |= os.O_NOFOLLOW\n elif target.is_symlink(): # pragma: no cover — Windows-only branch\n msg = f\"Refusing to read through symlink: {target}\"\n raise PermissionError(msg)\n fd = os.open(str(target), flags)\n try:\n chunks: list[bytes] = []\n while True:\n chunk = os.read(fd, 1 << 20) # 1 MiB per syscall\n if not chunk:\n break\n chunks.append(chunk)\n return b\"\".join(chunks)\n finally:\n os.close(fd)\n\n\ndef _read_head_no_follow(target: Path, n: int) -> bytes:\n \"\"\"Read the first ``n`` bytes of ``target`` without following symlinks.\n\n Used for the binary-content sniff before deciding whether a file is\n safe to surface as text — the same TOCTOU defence applies.\n \"\"\"\n flags = os.O_RDONLY\n if hasattr(os, \"O_NOFOLLOW\"):\n flags |= os.O_NOFOLLOW\n elif target.is_symlink(): # pragma: no cover — Windows-only branch\n msg = f\"Refusing to read through symlink: {target}\"\n raise PermissionError(msg)\n fd = os.open(str(target), flags)\n try:\n return os.read(fd, n)\n finally:\n os.close(fd)\n\n\ndef _check_windows_portability(path: str) -> str | None:\n \"\"\"Return a human-readable error if the path has Windows-portability issues.\n\n Why we run this on every host: paths that work on macOS/Linux can silently\n fail on Windows (reserved names, forbidden characters, trailing dot/space\n silently stripped by the OS). The agent should see the same structured\n error regardless of where the flow runs.\n \"\"\"\n # Use PureWindowsPath to parse separators consistently — it splits on both\n # `/` and `\\` and exposes drive markers as components ending in '\\'.\n for component in PureWindowsPath(path).parts:\n # Skip root markers ('\\\\', '/', 'C:\\\\') and relative markers ('.', '..')\n if component.endswith((\"\\\\\", \"/\")) or component in (\".\", \"..\"):\n continue\n # Reserved name (case-insensitive, with or without extension).\n stem = component.split(\".\", 1)[0].upper()\n if stem in _WINDOWS_RESERVED_NAMES:\n return f\"Path component {component!r} is a Windows reserved name\"\n # Forbidden characters in basename.\n bad = sorted(set(component) & _WINDOWS_FORBIDDEN_CHARS)\n if bad:\n return f\"Path component {component!r} contains forbidden character(s): {bad}\"\n # Trailing dot or space — Windows silently strips them.\n if component != component.rstrip(\". \"):\n return f\"Path component {component!r} has trailing dot or space (silently stripped on Windows)\"\n return None\n\n\nclass _ReadFileArgs(BaseModel):\n path: str = Field(..., description=\"Path to the file, relative to the sandbox root.\")\n offset: int | None = Field(default=None, description=\"1-based line number to start reading from.\")\n limit: int | None = Field(default=None, description=\"Maximum number of lines to return.\")\n\n\nclass _WriteFileArgs(BaseModel):\n path: str = Field(..., description=\"Path to the file, relative to the sandbox root.\")\n content: str = Field(..., description=\"Text content to write. Overwrites existing file.\")\n\n\nclass _EditFileArgs(BaseModel):\n path: str = Field(..., description=\"Path to the file, relative to the sandbox root.\")\n old_string: str = Field(..., description=\"Exact string to replace.\")\n new_string: str = Field(..., description=\"Replacement string.\")\n replace_all: bool = Field(default=False, description=\"Replace every occurrence instead of failing on ambiguity.\")\n\n\nclass _GlobSearchArgs(BaseModel):\n pattern: str = Field(..., description=\"Glob pattern, e.g. '**/*.py'.\")\n path: str | None = Field(default=None, description=\"Optional sub-directory to scope the search.\")\n\n\nclass _GrepSearchArgs(BaseModel):\n pattern: str = Field(..., description=\"Pattern to match against file contents (literal substring by default).\")\n path: str | None = Field(default=None, description=\"Optional file or directory to scope the search.\")\n glob: str | None = Field(default=None, description=\"Optional glob filter, e.g. '*.py'.\")\n case_insensitive: bool = Field(default=False, description=\"If true, the pattern is matched case-insensitively.\")\n output_mode: str = Field(\n default=\"files_with_matches\",\n description=\"One of 'files_with_matches', 'content', 'count'.\",\n )\n is_regex: bool = Field(\n default=False,\n description=(\n \"Treat `pattern` as a Python regex. Disabled by default — when enabled, \"\n \"patterns are validated against catastrophic-backtracking shapes \"\n \"(rejecting e.g. `(a+)+` or `(.*)*`) and oversized lines are skipped.\"\n ),\n )\n\n\nclass FileSystemToolComponent(Component):\n display_name = \"File System\"\n description = \"Sandboxed filesystem access for agents.\"\n icon = \"folder\"\n name = \"FileSystemTool\"\n\n # Enables the \"Tool Mode\" toggle on the node header. When ON the framework\n # calls _get_tools() and emits a Toolset output that connects to an Agent's\n # \"Tools\" handle. When OFF, the JSON metadata output is the only handle.\n add_tool_output = True\n\n inputs = [\n StrInput(\n name=\"root_path\",\n display_name=\"Workspace Sub-path\",\n required=False,\n value=\"\",\n info=(\n \"Sub-folder inside your sandboxed workspace. Leave empty to use the \"\n \"root of the workspace.\\n\\n\"\n \"Resolves under /shared// when AUTO_LOGIN=True \"\n \"(single-user / desktop) or under /users/// \"\n \"when AUTO_LOGIN=False (multi-user, per-user isolation). \"\n \"BASE_DIR is operator-controlled via LANGFLOW_FS_TOOL_BASE_DIR.\"\n ),\n ),\n BoolInput(\n name=\"read_only\",\n display_name=\"Read Only\",\n value=False,\n advanced=True,\n info=\"If true, write and edit operations are disabled.\",\n ),\n # Synthetic hidden input. Exists ONLY to make the \"Tool Mode\" toggle\n # appear on the node header — see frontend rule in\n # `src/frontend/src/CustomNodes/helpers/parameter-filtering.ts :: isHidden`\n # and the backend gate in `src/lfx/src/lfx/template/utils.py` (toggle\n # visibility = `any(input.tool_mode for input in inputs)`).\n # `show=False` keeps it hidden from the config UI; `tool_mode=True`\n # would otherwise hide a real input in tool mode (see same isHidden\n # rule), so we keep root_path / read_only WITHOUT tool_mode=True so\n # they remain user-editable when the toggle is on. _get_tools() ignores\n # this field — each StructuredTool has its own per-operation schema.\n StrInput(\n name=\"tool_mode_trigger\",\n display_name=\"\",\n show=False,\n tool_mode=True,\n required=False,\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"JSON\",\n name=\"metadata\",\n method=\"build_metadata\",\n types=[\"Data\"],\n ),\n ]\n\n def build_metadata(self) -> Data:\n \"\"\"Return introspective metadata about the sandbox. No file I/O.\n\n Surfaces the live AUTO_LOGIN-driven layout so flow authors can see\n whether per-user scoping is active and where their files actually land\n without needing to grep env vars.\n \"\"\"\n registered = [\"read_file\", \"glob_search\", \"grep_search\"]\n if not self.read_only:\n registered.extend([\"write_file\", \"edit_file\"])\n\n auto_login = self._resolve_auto_login()\n user_id = self._resolve_user_id()\n if auto_login:\n mode = \"shared\"\n elif user_id:\n mode = \"isolated\"\n else:\n mode = \"refused\"\n\n # Best-effort effective_root resolution. We never raise from metadata —\n # if the policy refuses the call (AUTO_LOGIN=False without user_id,\n # unwritable BASE_DIR, etc.) we surface the reason instead of crashing\n # the node preview / Agent build pipeline.\n effective_root: str | None\n resolution_error: str | None = None\n try:\n effective_root = str(self._validate_root())\n except Exception as exc: # noqa: BLE001 — metadata must never raise\n effective_root = None\n resolution_error = str(exc) or exc.__class__.__name__\n\n return Data(\n data={\n \"root_path\": self.root_path,\n \"read_only\": bool(self.read_only),\n \"tools_registered\": registered,\n \"auto_login\": auto_login,\n \"mode\": mode,\n \"effective_root\": effective_root,\n \"resolution_error\": resolution_error,\n }\n )\n\n async def _get_tools(self) -> list[Tool]:\n \"\"\"Tool Mode entrypoint. Called by Component.to_toolkit().\"\"\"\n # Capture the bound user_id ONCE per build. Each StructuredTool closure\n # below re-checks the live ``self._user_id`` against this value at call\n # time, so a tool issued for user A cannot be invoked after the\n # component has been reused for user B (defense for L2 in the plan).\n bound_user_id = self._resolve_user_id()\n tools: list[Tool] = [\n self._make_read_tool(bound_user_id=bound_user_id),\n self._make_glob_tool(bound_user_id=bound_user_id),\n self._make_grep_tool(bound_user_id=bound_user_id),\n ]\n if not self.read_only:\n tools.append(self._make_write_tool(bound_user_id=bound_user_id))\n tools.append(self._make_edit_tool(bound_user_id=bound_user_id))\n return tools\n\n def _user_binding_error(self, bound_user_id: str | None) -> dict | None:\n \"\"\"Return a structured error dict if the live user_id has shifted.\n\n In shared mode (AUTO_LOGIN=True) user_id is not part of the security\n boundary, so the binding check is a no-op. In isolated mode we compare\n the captured user_id to the live one and refuse if they diverge —\n defends against a worker pool reusing one component instance across\n sessions for different users.\n \"\"\"\n _, err = self._user_binding_check(bound_user_id)\n return err\n\n def _user_binding_check(self, bound_user_id: str | None) -> tuple[str | None, dict | None]:\n \"\"\"Atomic capture of the user_id for the current invocation.\n\n Returns ``(captured_user_id, error_or_none)``. The captured value is\n the SINGLE read of ``_resolve_user_id`` for this call; downstream\n path-resolution code MUST reuse it (via the ContextVar pin) instead\n of reading again — otherwise a mid-call mutation of ``self._user_id``\n would let a write land in a different user's namespace while this\n check still passed for the original user.\n \"\"\"\n # When force_isolation is set, every invocation MUST carry the user_id\n # that was captured at binding time — AUTO_LOGIN does not relax the\n # check here, otherwise the per-user root in _validate_root would be\n # reached with the wrong identity.\n if getattr(self, \"_force_isolation\", False):\n current = self._resolve_user_id()\n if current and current == bound_user_id:\n return current, None\n return None, {\n \"error\": (\n \"tool/user-id mismatch: this tool was bound to a different user session and cannot be reused\"\n ),\n }\n if self._resolve_auto_login():\n return bound_user_id, None\n current = self._resolve_user_id()\n if current == bound_user_id:\n return current, None\n return None, {\n \"error\": (\"tool/user-id mismatch: this tool was bound to a different user session and cannot be reused\"),\n }\n\n def _make_read_tool(self, *, bound_user_id: str | None) -> StructuredTool:\n def _run(path: str, offset: int | None = None, limit: int | None = None) -> str:\n captured, err = self._user_binding_check(bound_user_id)\n if err is not None:\n return json.dumps(err)\n token = _pinned_user_id_var.set(captured)\n try:\n return json.dumps(self._read_file(path, offset=offset, limit=limit))\n finally:\n _pinned_user_id_var.reset(token)\n\n return StructuredTool.from_function(\n name=\"read_file\",\n description=(\n \"Read a text file from the sandboxed workspace. \"\n \"Returns content prefixed with line numbers plus metadata \"\n \"(total_lines, start_line, num_lines).\"\n ),\n func=_run,\n args_schema=_ReadFileArgs,\n tags=[\"read_file\"],\n )\n\n def _make_write_tool(self, *, bound_user_id: str | None) -> StructuredTool:\n def _run(path: str, content: str) -> str:\n captured, err = self._user_binding_check(bound_user_id)\n if err is not None:\n return json.dumps(err)\n token = _pinned_user_id_var.set(captured)\n try:\n return json.dumps(self._write_file(path, content))\n finally:\n _pinned_user_id_var.reset(token)\n\n return StructuredTool.from_function(\n name=\"write_file\",\n description=\"Create or overwrite a text file inside the sandboxed workspace.\",\n func=_run,\n args_schema=_WriteFileArgs,\n tags=[\"write_file\"],\n )\n\n def _make_edit_tool(self, *, bound_user_id: str | None) -> StructuredTool:\n def _run(path: str, old_string: str, new_string: str, *, replace_all: bool = False) -> str:\n captured, err = self._user_binding_check(bound_user_id)\n if err is not None:\n return json.dumps(err)\n token = _pinned_user_id_var.set(captured)\n try:\n return json.dumps(\n self._edit_file(path, old_string=old_string, new_string=new_string, replace_all=replace_all)\n )\n finally:\n _pinned_user_id_var.reset(token)\n\n return StructuredTool.from_function(\n name=\"edit_file\",\n description=(\n \"Edit a text file by replacing an exact old_string with new_string. \"\n \"Fails on ambiguous matches unless replace_all=True.\"\n ),\n func=_run,\n args_schema=_EditFileArgs,\n tags=[\"edit_file\"],\n )\n\n def _make_glob_tool(self, *, bound_user_id: str | None) -> StructuredTool:\n def _run(pattern: str, path: str | None = None) -> str:\n captured, err = self._user_binding_check(bound_user_id)\n if err is not None:\n return json.dumps(err)\n token = _pinned_user_id_var.set(captured)\n try:\n return json.dumps(self._glob_search(pattern, path=path))\n finally:\n _pinned_user_id_var.reset(token)\n\n return StructuredTool.from_function(\n name=\"glob_search\",\n description=(\n \"List files matching a glob pattern inside the sandboxed workspace. \"\n f\"Results are truncated at {GLOB_RESULT_LIMIT} entries.\"\n ),\n func=_run,\n args_schema=_GlobSearchArgs,\n tags=[\"glob_search\"],\n )\n\n def _make_grep_tool(self, *, bound_user_id: str | None) -> StructuredTool:\n def _run(\n pattern: str,\n path: str | None = None,\n glob: str | None = None,\n *,\n case_insensitive: bool = False,\n output_mode: str = \"files_with_matches\",\n is_regex: bool = False,\n ) -> str:\n captured, err = self._user_binding_check(bound_user_id)\n if err is not None:\n return json.dumps(err)\n token = _pinned_user_id_var.set(captured)\n try:\n return json.dumps(\n self._grep_search(\n pattern,\n path=path,\n glob=glob,\n case_insensitive=case_insensitive,\n output_mode=output_mode,\n is_regex=is_regex,\n )\n )\n finally:\n _pinned_user_id_var.reset(token)\n\n return StructuredTool.from_function(\n name=\"grep_search\",\n description=(\n \"Search file contents. Default is literal substring match (safe for any pattern); \"\n \"set is_regex=True to opt into Python regex. Patterns with nested unbounded \"\n \"quantifiers (e.g. (a+)+) are rejected to prevent catastrophic backtracking. \"\n \"output_mode: 'files_with_matches' (default), 'content', or 'count'. \"\n f\"Content mode is capped at {GREP_LINE_LIMIT} lines.\"\n ),\n func=_run,\n args_schema=_GrepSearchArgs,\n tags=[\"grep_search\"],\n )\n\n def _resolve_user_id(self) -> str | None:\n \"\"\"Best-effort lookup of the calling user's id.\n\n The Component base class populates ``_user_id`` during instantiation,\n and the property cascades to ``self.graph.user_id``. In tests there is\n no graph; in scheduled/anonymous runs there is no user_id at all. We\n treat all of those as \"anonymous\" — the isolation mode decides what to\n do with that information.\n\n Why we filter ``\"none\"`` / ``\"null\"``: Langflow's ``PlaceholderGraph``\n stringifies a missing user as ``\"None\"`` rather than the Python\n ``None`` value, so a naive truthiness check would mistake \"no user\" for\n a real user named \"None\" and create a spurious shared namespace.\n\n L2 binding atomicity: when a tool invocation is in progress, the\n pinned ContextVar value takes precedence so every read inside the\n same call sees the user_id captured at the binding check — even if\n ``self._user_id`` is mutated mid-call by a reused component instance.\n \"\"\"\n pinned = _pinned_user_id_var.get()\n if pinned is not None:\n return pinned\n candidates: list[object | None] = [getattr(self, \"_user_id\", None)]\n graph = getattr(self, \"graph\", None)\n if graph is not None:\n candidates.append(getattr(graph, \"user_id\", None))\n\n for value in candidates:\n if value is None:\n continue\n cleaned = str(value).strip()\n if not cleaned or cleaned.lower() in {\"none\", \"null\"}:\n continue\n return cleaned\n return None\n\n def _isolation_config(self) -> IsolationConfig:\n \"\"\"Read the on-disk layout config from the environment on every call.\n\n Re-read intentionally: tests and live operators tweak env vars between\n runs; caching would create stale-config bugs that are painful to\n diagnose. The cost is a handful of dict lookups per tool call.\n \"\"\"\n return load_isolation_config(env=os.environ, default_config_dir=_default_config_dir())\n\n def _resolve_auto_login(self) -> bool:\n \"\"\"Return AUTO_LOGIN from the live settings service.\n\n Defaults to True when the settings service is unavailable (tests, very\n early bootstrap) — mirrors the platform-wide default and keeps the\n component usable in lightweight contexts. Tests can override this\n method per-instance to pin the desired mode.\n \"\"\"\n try:\n settings_service = get_settings_service()\n except Exception: # noqa: BLE001 — service registry may not be ready\n return True\n if settings_service is None:\n return True\n try:\n return bool(settings_service.auth_settings.AUTO_LOGIN)\n except AttributeError:\n return True\n\n def _validate_root(self) -> Path:\n \"\"\"Resolve and authorize the effective sandbox root.\n\n Dispatch:\n - ``_force_isolation=True`` → /users//\n - AUTO_LOGIN=True → /shared/\n - AUTO_LOGIN=False + user_id → /users//\n - any mode with no user_id → PermissionError (caught by callers)\n\n ``_force_isolation`` exists for callers that carry an authenticated\n user identity and need per-user isolation regardless of the global\n AUTO_LOGIN flag (e.g. the agentic file router + the agent's write\n tools). It defaults to False so other call sites keep their current\n AUTO_LOGIN-driven behavior unchanged.\n \"\"\"\n config = self._isolation_config()\n\n if getattr(self, \"_force_isolation\", False):\n user_id = self._resolve_user_id()\n if not user_id:\n msg = \"FileSystemTool requires an authenticated user when _force_isolation is set\"\n raise PermissionError(msg)\n return self._isolated_user_root(config=config, user_id=user_id)\n\n if self._resolve_auto_login():\n return self._shared_root(config=config)\n\n user_id = self._resolve_user_id()\n if not user_id:\n msg = \"FileSystemTool requires an authenticated user when AUTO_LOGIN=False\"\n raise PermissionError(msg)\n return self._isolated_user_root(config=config, user_id=user_id)\n\n def _shared_root(self, *, config: IsolationConfig) -> Path:\n \"\"\"Materialize ``/shared/`` and verify the boundary.\n\n Why a fixed ``shared/`` prefix (and not just ``base_dir`` directly):\n keeps the on-disk layout symmetric with isolated mode (``users//``)\n so flipping AUTO_LOGIN at deploy time never mixes the two trees.\n \"\"\"\n shared_root = (config.base_dir / SHARED_NAMESPACE).resolve()\n try:\n shared_root.mkdir(parents=True, exist_ok=True)\n except OSError as exc:\n msg = (\n f\"Cannot create shared workspace at {shared_root}: \"\n f\"{exc.strerror or exc}. \"\n f\"Check that LANGFLOW_FS_TOOL_BASE_DIR ({config.base_dir}) is writable \"\n f\"by the Langflow process user.\"\n )\n raise PermissionError(msg) from exc\n\n sub_raw = (self.root_path or \"\").strip()\n sub = sub_raw.lstrip(\"/\\\\\")\n candidate = (shared_root / sub).resolve() if sub else shared_root\n if not candidate.is_relative_to(shared_root):\n msg = f\"sub_path {self.root_path!r} escapes shared workspace\"\n raise PermissionError(msg)\n try:\n candidate.mkdir(parents=True, exist_ok=True)\n except OSError as exc:\n msg = f\"Cannot create sub-path {candidate}: {exc.strerror or exc}\"\n raise PermissionError(msg) from exc\n return candidate\n\n def _isolated_user_root(self, *, config: IsolationConfig, user_id: str) -> Path:\n \"\"\"Materialize ``/users//`` and verify the boundary.\"\"\"\n try:\n pepper = load_or_create_pepper(config.pepper_path)\n except OSError as exc:\n msg = (\n f\"Cannot access pepper file at {config.pepper_path}: \"\n f\"{exc.strerror or exc}. \"\n f\"Check that LANGFLOW_FS_TOOL_PEPPER_PATH points to a writable location.\"\n )\n raise PermissionError(msg) from exc\n namespace = compute_user_namespace(user_id, pepper=pepper)\n user_root = (config.base_dir / namespace).resolve()\n try:\n user_root.mkdir(parents=True, exist_ok=True)\n except OSError as exc:\n msg = (\n f\"Cannot create user namespace at {user_root}: \"\n f\"{exc.strerror or exc}. \"\n f\"Check that LANGFLOW_FS_TOOL_BASE_DIR ({config.base_dir}) is writable \"\n f\"by the Langflow process user.\"\n )\n raise PermissionError(msg) from exc\n\n sub_raw = (self.root_path or \"\").strip()\n # Strip leading separators so absolute-looking sub_paths are pinned\n # under the user root rather than escaping to the host filesystem.\n sub = sub_raw.lstrip(\"/\\\\\")\n candidate = (user_root / sub).resolve() if sub else user_root\n if not candidate.is_relative_to(user_root):\n msg = f\"sub_path {self.root_path!r} escapes user namespace\"\n raise PermissionError(msg)\n try:\n candidate.mkdir(parents=True, exist_ok=True)\n except OSError as exc:\n msg = f\"Cannot create sub-path {candidate}: {exc.strerror or exc}\"\n raise PermissionError(msg) from exc\n return candidate\n\n def _validate_path(self, path: str) -> Path:\n \"\"\"Resolve `path` relative to the sandbox root and reject any escape.\n\n Why raise: internal helper for control flow; each tool operation catches\n PermissionError and translates it into a structured error for the agent.\n \"\"\"\n if \"\\x00\" in path:\n msg = \"Path contains NUL byte\"\n raise PermissionError(msg)\n if portability_error := _check_windows_portability(path):\n raise PermissionError(portability_error)\n # Block the reserved segments (`.lfsig` integrity hook, `.components`\n # registered-user-component store). We forbid traversal even by users\n # with valid credentials — agents and humans both — because the\n # privilege model depends on these directories being unreachable from\n # the public tool surface.\n # Compare case-insensitively: APFS and NTFS resolve `.LFSIG` to the\n # same directory as `.lfsig`, so a case-sensitive equality check\n # lets uppercase variants slip past the reservation on those\n # filesystems. `casefold` is the locale-aware lowercase used for\n # caseless string comparison — broader than `.lower()`.\n reserved_folds = {seg.casefold() for seg in RESERVED_SEGMENTS}\n for part in PureWindowsPath(path).parts:\n folded = part.casefold()\n if folded in reserved_folds:\n # Surface the canonical segment name (with leading dot) so\n # error messages stay stable across modes and OSes.\n canonical = next(seg for seg in RESERVED_SEGMENTS if seg.casefold() == folded)\n msg = f\"Path component {canonical!r} is reserved\"\n raise PermissionError(msg)\n if deny_error := _check_deny_list(path):\n raise PermissionError(deny_error)\n root_resolved = self._validate_root()\n candidate = (root_resolved / path).resolve()\n if not candidate.is_relative_to(root_resolved):\n msg = f\"Path escapes workspace boundary: {path}\"\n raise PermissionError(msg)\n # Re-check the resolved name: a symlink with an innocuous basename can\n # alias a denied target that `.resolve()` has already followed.\n if deny_error := _check_deny_list(str(candidate.relative_to(root_resolved))):\n raise PermissionError(deny_error)\n if hardlink_error := _check_hardlink(candidate):\n raise PermissionError(hardlink_error)\n return candidate\n\n def _read_file(self, path: str, offset: int | None = None, limit: int | None = None) -> dict:\n try:\n resolved = self._validate_path(path)\n except PermissionError as exc:\n return {\"error\": str(exc), \"path\": path}\n\n if not resolved.exists():\n return {\"error\": f\"File not found: {path}\", \"path\": path}\n if resolved.is_dir():\n return {\"error\": f\"Path is a directory, not a file: {path}\", \"path\": path}\n\n try:\n size = resolved.stat().st_size\n except OSError as exc:\n return {\"error\": f\"Cannot stat file: {exc.strerror or exc}\", \"path\": path}\n if size > MAX_FILE_SIZE_BYTES:\n return {\n \"error\": f\"File size {size} exceeds limit of {MAX_FILE_SIZE_BYTES} bytes\",\n \"path\": path,\n }\n\n try:\n head = _read_head_no_follow(resolved, BINARY_SNIFF_BYTES)\n except OSError as exc:\n return {\"error\": f\"Cannot read file: {exc.strerror or exc}\", \"path\": path}\n if _looks_binary(head):\n return {\"error\": f\"Refusing to read binary file: {path}\", \"path\": path}\n\n try:\n text = _read_bytes_no_follow(resolved).decode(\"utf-8\", errors=\"replace\")\n except OSError as exc:\n return {\"error\": f\"Cannot read file: {exc.strerror or exc}\", \"path\": path}\n\n lines = text.splitlines()\n total = len(lines)\n start = offset if offset and offset > 0 else 1\n end = total if limit is None else min(total, start - 1 + limit)\n window = lines[start - 1 : end]\n numbered = \"\\n\".join(f\"{i:>6}→{line}\" for i, line in enumerate(window, start=start))\n return {\n \"status\": \"ok\",\n \"path\": path,\n \"content\": numbered,\n \"total_lines\": total,\n \"start_line\": start,\n \"num_lines\": len(window),\n }\n\n def _write_file(self, path: str, content: str) -> dict:\n try:\n resolved = self._validate_path(path)\n except PermissionError as exc:\n return {\"error\": str(exc), \"path\": path}\n\n encoded = content.encode(\"utf-8\")\n if len(encoded) > MAX_FILE_SIZE_BYTES:\n return {\n \"error\": f\"Content size {len(encoded)} exceeds limit of {MAX_FILE_SIZE_BYTES} bytes\",\n \"path\": path,\n }\n\n if resolved.is_dir():\n return {\"error\": f\"Path is a directory, not a file: {path}\", \"path\": path}\n # Auto-create missing parent directories. `_validate_path` has already\n # confirmed `resolved` lies inside the sandbox root, so every ancestor\n # of `resolved.parent` is also inside the root — `mkdir(parents=True)`\n # cannot escape the sandbox.\n try:\n resolved.parent.mkdir(parents=True, exist_ok=True)\n except OSError as exc:\n return {\"error\": f\"Cannot create parent directory: {exc.strerror or exc}\", \"path\": path}\n\n existed = resolved.exists()\n try:\n _write_bytes_no_follow(resolved, encoded)\n except OSError as exc:\n return {\"error\": f\"Cannot write file: {exc.strerror or exc}\", \"path\": path}\n\n return {\n \"status\": \"updated\" if existed else \"created\",\n \"path\": path,\n \"bytes_written\": len(encoded),\n }\n\n def _edit_file(\n self,\n path: str,\n old_string: str,\n new_string: str,\n *,\n replace_all: bool = False,\n ) -> dict:\n try:\n resolved = self._validate_path(path)\n except PermissionError as exc:\n return {\"error\": str(exc), \"path\": path}\n\n if not resolved.exists():\n return {\"error\": f\"File not found: {path}\", \"path\": path}\n if resolved.is_dir():\n return {\"error\": f\"Path is a directory, not a file: {path}\", \"path\": path}\n\n # Reject empty old_string outright. With `old_string=\"\"`, str.replace\n # inserts new_string between every character (and at both ends),\n # producing roughly N*len(new_string) extra bytes — a small file plus\n # a large new_string can blow far past MAX_FILE_SIZE_BYTES before the\n # post-construction guard runs. Empty old_string also has no\n # well-defined semantics for \"edit this exact match\".\n if old_string == \"\":\n return {\"error\": \"old_string must not be empty\", \"path\": path}\n\n try:\n current = _read_bytes_no_follow(resolved).decode(\"utf-8\", errors=\"replace\")\n except OSError as exc:\n return {\"error\": f\"Cannot read file: {exc.strerror or exc}\", \"path\": path}\n\n occurrences = current.count(old_string)\n if occurrences == 0:\n return {\"error\": f\"old_string not found in file: {path}\", \"path\": path}\n if occurrences > 1 and not replace_all:\n return {\n \"error\": (\n f\"old_string is ambiguous: {occurrences} multiple matches in {path}. \"\n \"Pass replace_all=True to replace every occurrence.\"\n ),\n \"path\": path,\n \"matches\": occurrences,\n }\n\n # Project the encoded size BEFORE building the replacement string.\n # str.replace allocates the full result up front, so a small file\n # with replace_all=True and a large new_string can balloon memory\n # before the post-allocation length check runs. UTF-8 byte counts are\n # exact (no estimation) so the projection is precise.\n old_bytes = len(old_string.encode(\"utf-8\"))\n new_bytes = len(new_string.encode(\"utf-8\"))\n current_bytes = len(current.encode(\"utf-8\"))\n replacements_planned = occurrences if replace_all else 1\n projected_bytes = current_bytes + replacements_planned * (new_bytes - old_bytes)\n if projected_bytes > MAX_FILE_SIZE_BYTES:\n return {\n \"error\": (\n f\"Resulting content size {projected_bytes} would exceed limit of {MAX_FILE_SIZE_BYTES} bytes\"\n ),\n \"path\": path,\n }\n\n updated = current.replace(old_string, new_string) if replace_all else current.replace(old_string, new_string, 1)\n encoded = updated.encode(\"utf-8\")\n # Defensive re-check (should be unreachable given the projection above,\n # but keeps the original invariant explicit).\n if len(encoded) > MAX_FILE_SIZE_BYTES:\n return {\n \"error\": f\"Resulting content size {len(encoded)} exceeds limit of {MAX_FILE_SIZE_BYTES} bytes\",\n \"path\": path,\n }\n\n try:\n _write_bytes_no_follow(resolved, encoded)\n except OSError as exc:\n return {\"error\": f\"Cannot write file: {exc.strerror or exc}\", \"path\": path}\n\n return {\n \"status\": \"ok\",\n \"path\": path,\n \"replacements\": occurrences if replace_all else 1,\n \"old_string\": old_string,\n \"new_string\": new_string,\n }\n\n def _glob_search(self, pattern: str, path: str | None = None) -> dict:\n # Empty patterns make ``Path.glob`` raise ``ValueError`` instead of\n # returning an empty match list, which would propagate uncaught and\n # crash the agent. Surface a structured error instead.\n if not pattern or not pattern.strip():\n return {\"error\": \"Pattern must not be empty\", \"pattern\": pattern, \"path\": path}\n # Reject traversal segments in the pattern itself. Without this guard,\n # ``base.glob(\"../*\")`` walks one directory up; one of the resulting\n # paths (``/../``) resolves back to ``base`` and\n # is surfaced to the agent as ``\".\"`` — a silent, misleading hit.\n # ``PureWindowsPath`` splits on both ``/`` and ``\\`` so the check\n # covers Windows-authored patterns as well.\n if any(part == \"..\" for part in PureWindowsPath(pattern).parts):\n return {\n \"error\": \"Pattern must not contain '..' (path traversal not allowed)\",\n \"pattern\": pattern,\n \"path\": path,\n }\n try:\n root_resolved = self._validate_root()\n base = self._validate_path(path) if path else root_resolved\n except PermissionError as exc:\n return {\"error\": str(exc), \"pattern\": pattern, \"path\": path}\n\n if not base.exists() or not base.is_dir():\n return {\"error\": f\"Base path is not a directory: {path or '.'}\", \"pattern\": pattern}\n\n # Collect up to GLOB_SCAN_CEILING (>> GLOB_RESULT_LIMIT) so we can\n # surface diverse branches even when one directory dominates the\n # iteration order. See BUG-02 / T2-001.\n collected: list[str] = []\n for match in base.glob(pattern):\n try:\n resolved_match = match.resolve()\n except OSError:\n continue\n if not resolved_match.is_relative_to(root_resolved):\n continue\n relative_match = str(resolved_match.relative_to(root_resolved))\n if _check_deny_list(relative_match):\n continue\n collected.append(relative_match)\n if len(collected) >= GLOB_SCAN_CEILING:\n break\n\n # Sort for cross-filesystem determinism, then truncate to the visible\n # cap. Emit `truncated_branches` so the agent has a non-silent signal:\n # top-level branches under `base` whose files appear in the omitted\n # tail. The agent can narrow with `path=` to recover them.\n collected.sort()\n truncated = len(collected) > GLOB_RESULT_LIMIT\n if truncated:\n omitted = collected[GLOB_RESULT_LIMIT:]\n matches = collected[:GLOB_RESULT_LIMIT]\n truncated_branches = sorted({Path(p).parts[0] for p in omitted if Path(p).parts})\n else:\n matches = collected\n truncated_branches = []\n\n return {\n \"status\": \"ok\",\n \"pattern\": pattern,\n \"matches\": matches,\n \"truncated\": truncated,\n \"truncated_branches\": truncated_branches,\n }\n\n def _grep_search(\n self,\n pattern: str,\n path: str | None = None,\n glob: str | None = None,\n *,\n case_insensitive: bool = False,\n output_mode: str = \"files_with_matches\",\n is_regex: bool = False,\n ) -> dict:\n if output_mode not in GREP_OUTPUT_MODES:\n return {\n \"error\": f\"Invalid output_mode '{output_mode}'. Must be one of {GREP_OUTPUT_MODES}\",\n \"pattern\": pattern,\n }\n\n # Build a per-line matcher. Default is a literal substring check —\n # safe for any user-supplied pattern. Regex is opt-in and is gated by\n # a static heuristic plus per-line/per-file caps. Stdlib `re` cannot\n # be cancelled (it holds the GIL), so prevention is the only viable\n # mitigation: reject pathological patterns at compile time and bound\n # the input we hand to the engine.\n if is_regex:\n if len(pattern) > GREP_PATTERN_MAX_LEN:\n return {\n \"error\": (\n f\"Regex pattern exceeds {GREP_PATTERN_MAX_LEN} chars; \"\n \"shorten it or use literal mode (is_regex=False).\"\n ),\n \"pattern\": pattern,\n }\n if _looks_like_redos(pattern):\n return {\n \"error\": (\n \"Regex pattern rejected: nested unbounded quantifier \"\n \"(catastrophic-backtracking risk). Rewrite without an \"\n \"outer +/* on a group that already contains +, *, {n,}, \"\n \"or |, or use literal mode (is_regex=False).\"\n ),\n \"pattern\": pattern,\n }\n try:\n flags = re.IGNORECASE if case_insensitive else 0\n regex = re.compile(pattern, flags)\n except re.error as exc:\n return {\"error\": f\"Invalid regex pattern: {exc}\", \"pattern\": pattern}\n\n def line_matches(line: str) -> bool:\n # Skip regex matching on oversized lines — they are usually\n # data blobs, and a long input is a prerequisite for most\n # practical exponential-backtracking attacks.\n if len(line) > GREP_REGEX_LINE_MAX_LEN:\n return False\n return regex.search(line) is not None\n else:\n needle = pattern.lower() if case_insensitive else pattern\n\n def line_matches(line: str) -> bool:\n hay = line.lower() if case_insensitive else line\n return needle in hay\n\n try:\n root_resolved = self._validate_root()\n base = self._validate_path(path) if path else root_resolved\n except PermissionError as exc:\n return {\"error\": str(exc), \"pattern\": pattern, \"path\": path}\n\n if not base.exists():\n return {\"error\": f\"Path does not exist: {path or '.'}\", \"pattern\": pattern}\n\n targets = self._collect_grep_targets(base, glob)\n files_with_matches: list[str] = []\n content_matches: list[dict] = []\n count_matches: list[dict] = []\n line_budget = GREP_LINE_LIMIT\n truncated = False\n\n for file in targets:\n try:\n resolved_file = file.resolve()\n except OSError:\n continue\n if not resolved_file.is_relative_to(root_resolved):\n continue\n rel_path = str(resolved_file.relative_to(root_resolved))\n if _check_deny_list(rel_path):\n continue\n try:\n size = resolved_file.stat().st_size\n except OSError:\n continue\n if size > MAX_FILE_SIZE_BYTES:\n continue\n try:\n with resolved_file.open(\"rb\") as fh:\n head = fh.read(BINARY_SNIFF_BYTES)\n except OSError:\n continue\n if _looks_binary(head):\n continue\n try:\n text = resolved_file.read_text(encoding=\"utf-8\", errors=\"replace\")\n except OSError:\n continue\n\n per_file_count = 0\n matched_any = False\n for idx, line in enumerate(text.splitlines(), start=1):\n # Per-file scan cap (regex only — literal scans are O(n)).\n if is_regex and idx > GREP_REGEX_LINES_PER_FILE:\n break\n if not line_matches(line):\n continue\n matched_any = True\n per_file_count += 1\n if output_mode == \"content\":\n if line_budget <= 0:\n truncated = True\n break\n content_matches.append({\"path\": rel_path, \"line_number\": idx, \"line\": line})\n line_budget -= 1\n\n if matched_any:\n files_with_matches.append(rel_path)\n if output_mode == \"count\":\n count_matches.append({\"path\": rel_path, \"count\": per_file_count})\n\n if output_mode == \"content\" and truncated:\n break\n\n if output_mode == \"files_with_matches\":\n matches: list = files_with_matches\n elif output_mode == \"content\":\n matches = content_matches\n else: # count\n matches = count_matches\n\n return {\n \"status\": \"ok\",\n \"pattern\": pattern,\n \"output_mode\": output_mode,\n \"matches\": matches,\n \"truncated\": truncated,\n }\n\n def _collect_grep_targets(self, base: Path, glob: str | None) -> list[Path]:\n if base.is_file():\n return [base]\n all_files = [p for p in base.rglob(\"*\") if p.is_file()]\n if glob:\n return [p for p in all_files if p.match(glob)]\n return all_files\n" }, "read_only": { "_input_type": "BoolInput", @@ -117645,6 +117666,6 @@ "num_components": 350, "num_modules": 94 }, - "sha256": "0d7e995401cee6bc0ad9f46fd036e18bbcec09c37d5db9d8e38bd3b350a3d7f6", + "sha256": "72bec3913e70fb8fb049c511f285f2c9a35dd367bd97296bb8ac44ff8b2df4e0", "version": "1.11.0" } diff --git a/src/lfx/src/lfx/components/data_source/api_request.py b/src/lfx/src/lfx/components/data_source/api_request.py index ba6dab8382..9e779a29b0 100644 --- a/src/lfx/src/lfx/components/data_source/api_request.py +++ b/src/lfx/src/lfx/components/data_source/api_request.py @@ -69,6 +69,9 @@ REDIRECT_STATUS_CODES = frozenset( } ) +# Default ports per scheme, used to compare redirect origins. +DEFAULT_SCHEME_PORTS = {"http": 80, "https": 443} + class APIRequestComponent(Component): display_name = "API Request" @@ -642,15 +645,28 @@ class APIRequestComponent(Component): @staticmethod def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None: - """Drop sensitive headers when a redirect crosses to a different host. + """Drop sensitive headers when a redirect crosses to a different origin. Mirrors httpx's auto-follow behavior so manually following redirects does not - leak credentials (Authorization / Cookie) to a different host than the one the - caller intended them for. Same-host redirects keep all headers. + leak credentials (Authorization / Cookie) to an origin other than the one the + caller intended them for. Headers are kept only when the redirect stays on the + same origin (scheme, host, port) or is a direct https upgrade of the same host + on default ports - the exact cases where httpx keeps the Authorization header. """ if not headers: return headers - if urlparse(current_url).hostname == urlparse(next_url).hostname: + current, nxt = urlparse(current_url), urlparse(next_url) + current_port = current.port or DEFAULT_SCHEME_PORTS.get(current.scheme) + next_port = nxt.port or DEFAULT_SCHEME_PORTS.get(nxt.scheme) + same_origin = (current.scheme, current.hostname, current_port) == (nxt.scheme, nxt.hostname, next_port) + https_upgrade = ( + current.hostname == nxt.hostname + and current.scheme == "http" + and nxt.scheme == "https" + and current_port == DEFAULT_SCHEME_PORTS["http"] + and next_port == DEFAULT_SCHEME_PORTS["https"] + ) + if same_origin or https_upgrade: return headers sensitive = {"authorization", "proxy-authorization", "cookie"} return {k: v for k, v in headers.items() if k.lower() not in sensitive} diff --git a/src/lfx/src/lfx/components/data_source/url.py b/src/lfx/src/lfx/components/data_source/url.py index 9000234125..d2cfdda9a7 100644 --- a/src/lfx/src/lfx/components/data_source/url.py +++ b/src/lfx/src/lfx/components/data_source/url.py @@ -23,6 +23,16 @@ DEFAULT_TIMEOUT = 30 DEFAULT_MAX_DEPTH = 1 DEFAULT_FORMAT = "Text" +# HTTP status codes that carry a redirect Location header (RFC 9110). +REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308}) + +# Maximum number of redirects to follow. Each hop is re-validated for SSRF safety; +# matches httpx's default and the API Request component. +MAX_REDIRECTS = 20 + +# Default ports per scheme, used to compare redirect origins. +DEFAULT_SCHEME_PORTS = {"http": 80, "https": 443} + URL_REGEX = re.compile( r"^(https?:\/\/)?" r"(www\.)?" r"([a-zA-Z0-9.-]+)" r"(\.[a-zA-Z]{2,})?" r"(:\d+)?" r"(\/[^\s]*)?$", @@ -108,6 +118,21 @@ class URLComponent(Component): required=False, advanced=True, ), + BoolInput( + name="follow_redirects", + display_name="Follow Redirects", + info=( + "If enabled, follows HTTP redirects such as http→https or www/non-www " + "normalization, which most sites rely on to serve their content. When SSRF " + "protection is enabled (the default), every redirect hop is re-validated against " + "the same blocked-IP denylist and DNS-pinned before it is fetched, so following " + "redirects cannot be used to reach internal resources. Disable to capture the " + "first response as-is without following redirects." + ), + value=True, + required=False, + advanced=True, + ), DropdownInput( name="format", display_name="Output Format", @@ -286,8 +311,148 @@ class URLComponent(Component): return create_ssrf_protected_client(hostname=hostname, validated_ips=validated_ips) return httpx.AsyncClient() + @staticmethod + def _headers_for_redirect(headers: dict | None, current_url: str, next_url: str) -> dict | None: + """Drop sensitive headers when a redirect crosses to a different origin. + + Mirrors httpx's auto-follow behavior so manually following redirects does not + leak credentials (Authorization / Cookie) to an origin other than the one the + caller intended them for. Headers are kept only when the redirect stays on the + same origin (scheme, host, port) or is a direct https upgrade of the same host + on default ports - the exact cases where httpx keeps the Authorization header. + """ + if not headers: + return headers + current, nxt = urlparse(current_url), urlparse(next_url) + current_port = current.port or DEFAULT_SCHEME_PORTS.get(current.scheme) + next_port = nxt.port or DEFAULT_SCHEME_PORTS.get(nxt.scheme) + same_origin = (current.scheme, current.hostname, current_port) == (nxt.scheme, nxt.hostname, next_port) + https_upgrade = ( + current.hostname == nxt.hostname + and current.scheme == "http" + and nxt.scheme == "https" + and current_port == DEFAULT_SCHEME_PORTS["http"] + and next_port == DEFAULT_SCHEME_PORTS["https"] + ) + if same_origin or https_upgrade: + return headers + sensitive = {"authorization", "proxy-authorization", "cookie"} + return {k: v for k, v in headers.items() if k.lower() not in sensitive} + + def _process_response(self, response: httpx.Response) -> tuple[str, dict]: + """Turn a final (non-redirect) response into its HTML content and metadata. + + Args: + response: The HTTP response to process + + Returns: + tuple[str, dict]: The HTML content and metadata + """ + if self.check_response_status: + response.raise_for_status() + + content_type = response.headers.get("content-type", "").lower() + + # Filter out CSS files if requested + if self.filter_text_html and "text/css" in content_type: + logger.debug(f"Skipping CSS file: {response.url}") + return "", {} + + # Get the HTML content + html_content = response.text + + # Extract metadata + metadata = { + "source": str(response.url), + "title": "", + "description": "", + "content_type": content_type, + "language": "", + } + + # Try to extract title and description from HTML + try: + soup = BeautifulSoup(html_content, "lxml") + except Exception: # noqa: BLE001 + # Broad exception is acceptable here - metadata extraction is optional + # and we don't want to fail the entire request if it fails + logger.debug(f"Failed to extract metadata from {response.url}") + else: + if soup.title: + metadata["title"] = soup.title.string or "" + + # Try to get description from meta tags + meta_desc = soup.find("meta", attrs={"name": "description"}) + if meta_desc and meta_desc.get("content"): + metadata["description"] = meta_desc["content"] + + # Try to get language + html_tag = soup.find("html") + if html_tag and html_tag.get("lang"): + metadata["language"] = html_tag["lang"] + + return html_content, metadata + + async def _fetch_with_revalidated_redirects( + self, url: str, validated_ips: list[str], headers: dict + ) -> tuple[str, dict]: + """Fetch ``url``, following redirects manually and re-validating every hop. + + This closes an SSRF bypass: with redirects enabled, httpx would otherwise + auto-follow a redirect from a validated public URL to an internal address that + the pinned transport never checked (the redirect target is a different host, so + it is not in the pin map). Each redirect ``Location`` is resolved (relative + locations included) and re-validated with ``ensure_url`` - the same private/ + loopback/link-local denylist and DNS pinning applied to the initial request - + before any connection to it is made. A blocked hop raises ``ValueError`` (unless + ``continue_on_failure``); the chain is capped at ``MAX_REDIRECTS`` hops. + + Args: + url: The URL to fetch + validated_ips: Validated IPs for DNS pinning of the initial URL + headers: HTTP headers to send + + Returns: + tuple[str, dict]: The HTML content and metadata + """ + current_url = url + current_ips = validated_ips + + for _ in range(MAX_REDIRECTS + 1): + async with self._build_http_client(current_url, current_ips) as client: + response = await client.get(current_url, headers=headers, timeout=self.timeout, follow_redirects=False) + + location = response.headers.get("location") + if response.status_code in REDIRECT_STATUS_CODES and location: + # Resolve relative redirects against the current URL. + next_url = urljoin(current_url, location) + + # Re-validate the redirect target with the same SSRF denylist + DNS pinning. + try: + validated_next_url, current_ips = self.ensure_url(next_url) + except (ValueError, SSRFProtectionError) as e: + if self.continue_on_failure: + logger.warning(f"Skipping blocked or invalid redirect to {next_url}: {e}") + return "", {} + msg = f"SSRF Protection: blocked redirect to {next_url}: {e}" + raise ValueError(msg) from e + + headers = self._headers_for_redirect(headers, current_url, next_url) + current_url = validated_next_url + continue + + # Not a redirect (or no Location header) - this is the final response. + return self._process_response(response) + + # Exhausted the redirect budget. + if self.continue_on_failure: + logger.warning(f"Exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}") + return "", {} + msg = f"SSRF Protection: exceeded the maximum of {MAX_REDIRECTS} redirects while requesting {url}" + raise ValueError(msg) + async def _fetch_url_with_pinning(self, url: str, validated_ips: list[str], headers: dict) -> tuple[str, dict]: - """Fetch a single URL with DNS pinning protection. + """Fetch a single URL with DNS pinning protection, following redirects safely. Args: url: The URL to fetch @@ -297,60 +462,26 @@ class URLComponent(Component): Returns: tuple[str, dict]: The HTML content and metadata """ - async with self._build_http_client(url, validated_ips) as client: - try: - response = await client.get(url, headers=headers, timeout=self.timeout, follow_redirects=False) + try: + # When SSRF protection is enabled we must follow redirects manually so each hop + # is re-validated and DNS-pinned; letting httpx auto-follow would connect to the + # redirect target without pinning (different host, not in the pin map) and re-open + # the SSRF hole that DNS pinning closes. With protection disabled there is no pin + # to bypass, so httpx can follow redirects natively. + if self.follow_redirects and is_ssrf_protection_enabled(): + return await self._fetch_with_revalidated_redirects(url, validated_ips, headers) - if self.check_response_status: - response.raise_for_status() + async with self._build_http_client(url, validated_ips) as client: + response = await client.get( + url, headers=headers, timeout=self.timeout, follow_redirects=self.follow_redirects + ) + return self._process_response(response) - content_type = response.headers.get("content-type", "").lower() - - # Filter out CSS files if requested - if self.filter_text_html and "text/css" in content_type: - logger.debug(f"Skipping CSS file: {url}") - return "", {} - - # Get the HTML content - html_content = response.text - - # Extract metadata - metadata = { - "source": str(response.url), - "title": "", - "description": "", - "content_type": content_type, - "language": "", - } - - # Try to extract title and description from HTML - try: - soup = BeautifulSoup(html_content, "lxml") - except Exception: # noqa: BLE001 - # Broad exception is acceptable here - metadata extraction is optional - # and we don't want to fail the entire request if it fails - logger.debug(f"Failed to extract metadata from {url}") - else: - if soup.title: - metadata["title"] = soup.title.string or "" - - # Try to get description from meta tags - meta_desc = soup.find("meta", attrs={"name": "description"}) - if meta_desc and meta_desc.get("content"): - metadata["description"] = meta_desc["content"] - - # Try to get language - html_tag = soup.find("html") - if html_tag and html_tag.get("lang"): - metadata["language"] = html_tag["lang"] - - except httpx.HTTPError as e: - if self.continue_on_failure: - logger.warning(f"Failed to fetch {url}: {e}") - return "", {} - raise - else: - return html_content, metadata + except httpx.HTTPError as e: + if self.continue_on_failure: + logger.warning(f"Failed to fetch {url}: {e}") + return "", {} + raise async def _crawl_recursive( self, start_url: str, validated_ips: list[str], headers: dict, visited: set, depth: int = 0 @@ -376,6 +507,13 @@ class URLComponent(Component): # Fetch the current URL html_content, metadata = await self._fetch_url_with_pinning(start_url, validated_ips, headers) + # Redirects may have landed on a different canonical URL (http->https, www + # normalization). Resolve relative links and the prevent_outside check against + # the URL the content actually came from, and mark it visited so links back to + # the canonical form are not re-crawled. + base_url = metadata.get("source") or start_url + visited.add(base_url) + if not html_content: return documents @@ -405,7 +543,7 @@ class URLComponent(Component): for link in links: href = link["href"] # Resolve relative URLs - absolute_url = urljoin(start_url, href) + absolute_url = urljoin(base_url, href) # Skip if already visited if absolute_url in visited: @@ -413,7 +551,7 @@ class URLComponent(Component): # Check if we should prevent going outside the domain if self.prevent_outside: - start_domain = urlparse(start_url).netloc + start_domain = urlparse(base_url).netloc link_domain = urlparse(absolute_url).netloc if start_domain != link_domain: continue diff --git a/src/lfx/src/lfx/components/files_and_knowledge/filesystem.py b/src/lfx/src/lfx/components/files_and_knowledge/filesystem.py index 5870fac75d..e8bb8add14 100644 --- a/src/lfx/src/lfx/components/files_and_knowledge/filesystem.py +++ b/src/lfx/src/lfx/components/files_and_knowledge/filesystem.py @@ -160,14 +160,37 @@ _WINDOWS_FORBIDDEN_CHARS = frozenset('<>"|?*') # * literals — exact basename match (e.g. `.env`, `.netrc`) # * prefixes — basename startswith (e.g. `id_rsa`, `id_rsa.pub`, `id_ed25519`) # * suffixes — basename endswith (e.g. `cert.pem`, `private.key`) -# * fragments — any DIRECTORY component equals the fragment (e.g. `.ssh/`, -# `.aws/config`); the basename itself is not matched against fragments. +# * fragments — any path component equals the fragment, including the +# basename (e.g. `.ssh/`, `.aws/config`, or a bare `.ssh` request). Matching +# the basename too prevents requesting a protected directory directly (which +# would leak its existence) and stops a `**/*` glob from returning the +# protected directory entry itself even when its children are filtered. _DENY_BASENAME_LITERALS = frozenset({".env", ".netrc", ".pgpass", ".htpasswd", "authorized_keys"}) _DENY_BASENAME_PREFIXES = ("id_rsa", "id_dsa", "id_ecdsa", "id_ed25519", "credentials") _DENY_BASENAME_SUFFIXES = (".pem", ".key", ".pfx", ".p12") _DENY_PATH_FRAGMENTS = frozenset({".ssh", ".aws", ".gnupg", ".docker", ".kube", ".git"}) +def _check_deny_list(path: str) -> str | None: + parts = PureWindowsPath(path).parts + if not parts: + return None + *directories, basename = parts + folded = basename.casefold() + if folded in _DENY_PATH_FRAGMENTS: + return f"Access to {basename!r} is denied: name matches a protected directory" + if ( + folded in _DENY_BASENAME_LITERALS + or folded.startswith(_DENY_BASENAME_PREFIXES) + or folded.endswith(_DENY_BASENAME_SUFFIXES) + ): + return f"Access to {basename!r} is denied: filename matches a protected credential pattern" + for segment in directories: + if segment.casefold() in _DENY_PATH_FRAGMENTS: + return f"Access to {path!r} is denied: path contains protected directory {segment!r}" + return None + + def _looks_binary(head: bytes) -> bool: return b"\x00" in head @@ -838,11 +861,17 @@ class FileSystemToolComponent(Component): canonical = next(seg for seg in RESERVED_SEGMENTS if seg.casefold() == folded) msg = f"Path component {canonical!r} is reserved" raise PermissionError(msg) + if deny_error := _check_deny_list(path): + raise PermissionError(deny_error) root_resolved = self._validate_root() candidate = (root_resolved / path).resolve() if not candidate.is_relative_to(root_resolved): msg = f"Path escapes workspace boundary: {path}" raise PermissionError(msg) + # Re-check the resolved name: a symlink with an innocuous basename can + # alias a denied target that `.resolve()` has already followed. + if deny_error := _check_deny_list(str(candidate.relative_to(root_resolved))): + raise PermissionError(deny_error) if hardlink_error := _check_hardlink(candidate): raise PermissionError(hardlink_error) return candidate @@ -1055,7 +1084,10 @@ class FileSystemToolComponent(Component): continue if not resolved_match.is_relative_to(root_resolved): continue - collected.append(str(resolved_match.relative_to(root_resolved))) + relative_match = str(resolved_match.relative_to(root_resolved)) + if _check_deny_list(relative_match): + continue + collected.append(relative_match) if len(collected) >= GLOB_SCAN_CEILING: break @@ -1165,6 +1197,9 @@ class FileSystemToolComponent(Component): continue if not resolved_file.is_relative_to(root_resolved): continue + rel_path = str(resolved_file.relative_to(root_resolved)) + if _check_deny_list(rel_path): + continue try: size = resolved_file.stat().st_size except OSError: @@ -1183,7 +1218,6 @@ class FileSystemToolComponent(Component): except OSError: continue - rel_path = str(resolved_file.relative_to(root_resolved)) per_file_count = 0 matched_any = False for idx, line in enumerate(text.splitlines(), start=1): diff --git a/src/lfx/src/lfx/services/authorization/base.py b/src/lfx/src/lfx/services/authorization/base.py index 97a997a058..7f5357d3d9 100644 --- a/src/lfx/src/lfx/services/authorization/base.py +++ b/src/lfx/src/lfx/services/authorization/base.py @@ -27,6 +27,10 @@ class AuthzContext(TypedDict, total=False): share_user_id: _UUID | None workspace_id: _UUID | None folder_id: _UUID | None + auth_method: str | None + api_key_id: _UUID | None + api_key_source: str | None + external_provider: str | None class BaseAuthorizationService(Service, abc.ABC): @@ -36,11 +40,20 @@ class BaseAuthorizationService(Service, abc.ABC): # True when the service can authorize non-owner access (share-aware fetch). SUPPORTS_CROSS_USER_FETCH: ClassVar[bool] = False + # True when the service honors API-key credential context as a possible + # restriction on top of the resolved user. When enabled, Langflow lets the + # plugin evaluate owner-owned resources for API-key requests instead of + # applying the built-in owner override first. + SUPPORTS_API_KEY_SCOPES: ClassVar[bool] = False async def supports_cross_user_fetch(self) -> bool: """Return True when this service can authorize non-owner resource access.""" return self.SUPPORTS_CROSS_USER_FETCH + async def supports_api_key_scopes(self) -> bool: + """Return True when API-key requests should be enforced even for owners.""" + return self.SUPPORTS_API_KEY_SCOPES + @abc.abstractmethod async def is_enabled(self) -> bool: """Return True when authorization enforcement is active.""" diff --git a/src/lfx/src/lfx/services/settings/auth.py b/src/lfx/src/lfx/services/settings/auth.py index 21b8a73310..8f9f3d1dfc 100644 --- a/src/lfx/src/lfx/services/settings/auth.py +++ b/src/lfx/src/lfx/services/settings/auth.py @@ -2,6 +2,7 @@ import secrets from enum import Enum from pathlib import Path from typing import Literal +from urllib.parse import urlparse from passlib.context import CryptContext from pydantic import Field, SecretStr, field_validator, model_validator @@ -133,6 +134,108 @@ class AuthSettings(BaseSettings): ) """Path to YAML configuration file for SSO settings. Contains provider-specific configuration.""" + # External trusted-identity settings. + # Used when an upstream identity layer (proxy, gateway, IdP) issues or + # validates a credential and Langflow needs to accept it and map it to + # a local user via SSOUserProfile (JIT provisioning). + EXTERNAL_AUTH_ENABLED: bool = Field( + default=False, + description="Enable trusted external request authentication and JIT local user mapping.", + ) + EXTERNAL_AUTH_PROVIDER: str = Field( + default="external", + description="Stable provider key written to SSOUserProfile.sso_provider for external identities.", + ) + EXTERNAL_AUTH_TOKEN_HEADER: str = Field( + default="Authorization", + description=( + "Header containing the external credential. The native Langflow JWT path is tried first; " + "if it fails, the external path is attempted as a fallback. Bearer-prefixed values are supported." + ), + ) + EXTERNAL_AUTH_TOKEN_COOKIE: str | None = Field( + default=None, + description="Optional cookie name containing the external credential.", + ) + EXTERNAL_AUTH_IDENTITY_RESOLVER: str | None = Field( + default=None, + description=( + "Optional 'module:attribute' import path for a custom resolver that converts the external " + "credential into an identity. Defaults to built-in JWT validation when unset." + ), + ) + EXTERNAL_AUTH_TRUSTED_JWT_DECODE: bool = Field( + default=False, + description=( + "When True, decode the external JWT without verifying its signature. ONLY safe behind a trusted " + "upstream proxy that already validates the token. Off by default." + ), + ) + EXTERNAL_AUTH_JWKS_URL: str | None = Field( + default=None, + description="JWKS URL used to verify external JWT signatures when trusted decode is disabled.", + ) + EXTERNAL_AUTH_ISSUER: str | None = Field( + default=None, + description="Expected JWT issuer (iss). Leave empty to skip issuer validation.", + ) + EXTERNAL_AUTH_AUDIENCE: str | None = Field( + default=None, + description="Expected JWT audience (aud). Comma-separated audiences are supported.", + ) + EXTERNAL_AUTH_ALGORITHMS: str = Field( + default="RS256", + description="Comma-separated JWT algorithms accepted for external JWT validation.", + ) + EXTERNAL_AUTH_SUBJECT_CLAIM: str = Field( + default="sub", + description="JWT claim used as the stable external user id.", + ) + EXTERNAL_AUTH_USERNAME_CLAIM: str = Field( + default="preferred_username", + description="JWT claim preferred for the local Langflow username on JIT provisioning.", + ) + EXTERNAL_AUTH_EMAIL_CLAIM: str = Field( + default="email", + description="JWT claim containing the user's email.", + ) + EXTERNAL_AUTH_NAME_CLAIM: str = Field( + default="name", + description="JWT claim containing the user's display name.", + ) + EXTERNAL_AUTH_ACCESS_CEILING_ENABLED: bool = Field( + default=False, + description=( + "Enable a coarse deny-only action ceiling for users authenticated by external trusted identity. " + "This is not an RBAC engine; it only caps actions above a mapped access level." + ), + ) + EXTERNAL_AUTH_ACCESS_CLAIM: str | None = Field( + default=None, + description=( + "JWT claim used to derive the external access ceiling. Claim values are mapped through " + "EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING." + ), + ) + EXTERNAL_AUTH_ACCESS_CLAIM_MAPPING: str | None = Field( + default=None, + description=( + "JSON object or comma-separated value map from external claim values to one of: viewer, editor, admin. " + 'Example: \'{"view":"viewer","edit":"editor"}\'. Built-in aliases cover common values.' + ), + ) + EXTERNAL_AUTH_DEFAULT_ACCESS_LEVEL: str = Field( + default="viewer", + description="Fallback access level used when the access claim is missing or unmapped.", + ) + EXTERNAL_AUTH_DISABLE_API_KEYS_FOR_EXTERNAL_USERS: bool = Field( + default=True, + description=( + "When the external access ceiling is enabled, reject Langflow API-key authentication for users mapped " + "through the configured external provider so API keys cannot bypass the JWT claim ceiling." + ), + ) + # Authorization (RBAC) feature flags — enforcement via authorization_service plugin AUTHZ_ENABLED: bool = Field( default=False, @@ -233,6 +336,51 @@ class AuthSettings(BaseSettings): return value if isinstance(value, SecretStr) else SecretStr(value).get_secret_value() + @field_validator("EXTERNAL_AUTH_PROVIDER", mode="before") + @classmethod + def normalize_external_auth_provider(cls, value): + """Normalize the external provider key once at the config boundary. + + Every consumer (JIT provisioning, the API-key floor in the auth service, + SSOUserProfile.sso_provider lookups) reads this value directly, so an + empty/whitespace value must resolve to the same canonical string here. + Otherwise the value written ("external") and the value compared against + ("") would diverge and silently disable a security floor. + """ + if value is None: + return "external" + normalized = str(value).strip() + return normalized or "external" + + @field_validator("EXTERNAL_AUTH_JWKS_URL", mode="before") + @classmethod + def validate_external_auth_jwks_url(cls, value): + """Reject non-HTTPS JWKS URLs to stop a network MITM swapping signing keys. + + An ``http://`` JWKS endpoint lets an on-path attacker substitute their own + keys and forge tokens. HTTPS is required; ``http`` is permitted only for + loopback hosts (localhost / 127.0.0.1 / ::1) to keep local development + usable. + """ + if value is None: + return value + url = str(value).strip() + if not url: + return None + + parsed = urlparse(url) + scheme = parsed.scheme.lower() + if scheme == "https": + return url + if scheme == "http" and parsed.hostname in {"localhost", "127.0.0.1", "::1"}: + return url + + msg = ( + "EXTERNAL_AUTH_JWKS_URL must use https so a network attacker cannot swap the JWKS " + "signing keys and forge tokens. http is allowed only for localhost/127.0.0.1/::1." + ) + raise ValueError(msg) + @model_validator(mode="after") def setup_rsa_keys(self): """Generate or load RSA keys when using RS256/RS512 algorithm.""" diff --git a/src/lfx/tests/unit/components/data_source/test_url_component.py b/src/lfx/tests/unit/components/data_source/test_url_component.py index c162320937..4f0c761bf5 100644 --- a/src/lfx/tests/unit/components/data_source/test_url_component.py +++ b/src/lfx/tests/unit/components/data_source/test_url_component.py @@ -33,3 +33,41 @@ class TestURLComponentInputTypes: def test_urls_input_is_list(self, url_component): urls_input = next(inp for inp in url_component.inputs if inp.name == "urls") assert urls_input.is_list is True + + def test_follow_redirects_input_defaults_to_true(self, url_component): + """Redirects must be followed by default so canonical http->https / www hops resolve.""" + follow_redirects_input = next(inp for inp in url_component.inputs if inp.name == "follow_redirects") + assert follow_redirects_input.value is True + + +class TestHeadersForRedirect: + """Verify sensitive headers are only kept across same-origin (or https-upgrade) redirects.""" + + HEADERS = {"Authorization": "Bearer token", "Cookie": "session=1", "User-Agent": "Test"} + + def test_same_origin_keeps_headers(self, url_component): + result = url_component._headers_for_redirect(self.HEADERS, "https://a.test/x", "https://a.test/y") + assert result == self.HEADERS + + def test_https_upgrade_keeps_headers(self, url_component): + """Direct http->https upgrade on default ports keeps headers, matching httpx.""" + result = url_component._headers_for_redirect(self.HEADERS, "http://a.test/x", "https://a.test/x") + assert result == self.HEADERS + + def test_https_downgrade_drops_sensitive_headers(self, url_component): + """https->http is a different origin; credentials must not leak to plaintext.""" + result = url_component._headers_for_redirect(self.HEADERS, "https://a.test/x", "http://a.test/x") + assert result == {"User-Agent": "Test"} + + def test_port_change_drops_sensitive_headers(self, url_component): + """Same host on another port is a different origin (possibly a different service).""" + result = url_component._headers_for_redirect(self.HEADERS, "https://a.test/x", "https://a.test:8443/x") + assert result == {"User-Agent": "Test"} + + def test_cross_host_drops_sensitive_headers(self, url_component): + result = url_component._headers_for_redirect(self.HEADERS, "https://a.test/x", "https://b.test/x") + assert result == {"User-Agent": "Test"} + + def test_explicit_default_port_is_same_origin(self, url_component): + result = url_component._headers_for_redirect(self.HEADERS, "https://a.test/x", "https://a.test:443/y") + assert result == self.HEADERS diff --git a/src/lfx/tests/unit/components/tools/test_filesystem_deny_list.py b/src/lfx/tests/unit/components/tools/test_filesystem_deny_list.py new file mode 100644 index 0000000000..fbd56c9b90 --- /dev/null +++ b/src/lfx/tests/unit/components/tools/test_filesystem_deny_list.py @@ -0,0 +1,213 @@ +"""Regression for the unenforced credential deny-list (dead code). + +``filesystem.py`` defines ``_DENY_BASENAME_LITERALS`` / ``_PREFIXES`` / +``_SUFFIXES`` / ``_DENY_PATH_FRAGMENTS`` documented as the +"default-deny inside an allowed root" control, but no code referenced +them: any ``.env`` / private key / ``.ssh`` file inside the sandbox was +fully readable, writable, editable, glob-able and grep-able. + +The deny check must be pure-string and run before any I/O, so the agent +cannot distinguish a denied-existing file from a denied-absent one. +""" + +import json +from pathlib import Path + +import pytest +from lfx.components.files_and_knowledge.filesystem import FileSystemToolComponent + +SENTINEL_VALUE = "sk-fake-secret-12345" + + +@pytest.fixture +def sandbox(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.setenv("LANGFLOW_FS_TOOL_BASE_DIR", str(tmp_path)) + shared = tmp_path / "shared" + shared.mkdir(parents=True, exist_ok=True) + (shared / ".env").write_text(f"API_KEY={SENTINEL_VALUE}\n", encoding="utf-8") + (shared / "id_rsa").write_text(f"FAKE PRIVATE KEY {SENTINEL_VALUE}\n", encoding="utf-8") + (shared / "secret.pem").write_text(f"-----BEGIN FAKE KEY----- {SENTINEL_VALUE}\n", encoding="utf-8") + ssh_dir = shared / ".ssh" + ssh_dir.mkdir() + (ssh_dir / "config").write_text(f"Host fakehost {SENTINEL_VALUE}\n", encoding="utf-8") + (shared / "notes.txt").write_text("plain allowed file\n", encoding="utf-8") + return shared + + +@pytest.fixture +def component(sandbox: Path, monkeypatch: pytest.MonkeyPatch) -> FileSystemToolComponent: # noqa: ARG001 + c = FileSystemToolComponent(root_path="", read_only=False) + monkeypatch.setattr(c, "_resolve_auto_login", lambda: True) + return c + + +DENIED_PATHS = [ + pytest.param(".env", id="literal-dotenv"), + pytest.param("id_rsa", id="prefix-id_rsa"), + pytest.param("id_rsa.pub", id="prefix-id_rsa-pub"), + pytest.param("secret.pem", id="suffix-pem"), + pytest.param(".ssh/config", id="fragment-ssh"), + pytest.param(".ENV", id="literal-case-insensitive"), + pytest.param("ID_RSA", id="prefix-case-insensitive"), + pytest.param("SECRET.PEM", id="suffix-case-insensitive"), + pytest.param(".SSH/config", id="fragment-case-insensitive"), + pytest.param("nested/.aws/credentials", id="fragment-nested-aws"), +] + + +class TestDenyListRead: + @pytest.mark.parametrize("path", DENIED_PATHS) + def test_should_deny_read_when_path_matches_credential_pattern( + self, component: FileSystemToolComponent, path: str + ) -> None: + result = component._read_file(path) + + assert "error" in result, f"deny-list did not fire for {path!r}; result={result!r}" + assert SENTINEL_VALUE not in json.dumps(result) + + def test_should_deny_before_io_when_denied_file_does_not_exist(self, component: FileSystemToolComponent) -> None: + result = component._read_file(".netrc") + + assert "error" in result + assert "not found" not in result["error"].lower(), ( + "deny must fire before existence is observed, error was: " + result["error"] + ) + + def test_should_still_read_allowed_file(self, component: FileSystemToolComponent) -> None: + result = component._read_file("notes.txt") + + assert result.get("status") == "ok", f"allowed file was blocked: {result!r}" + + def test_should_not_deny_when_literal_is_only_a_name_prefix(self, component: FileSystemToolComponent) -> None: + result = component._read_file(".env.local") + + assert "error" in result + assert "denied" not in result["error"].lower(), ".env.local must not match the exact-literal rule" + + +class TestDenyListWrite: + def test_should_deny_write_when_basename_matches_literal( + self, component: FileSystemToolComponent, sandbox: Path + ) -> None: + result = component._write_file(".pgpass", "db:5432:*:user:hunter2") + + assert "error" in result + assert not (sandbox / ".pgpass").exists() + + def test_should_deny_write_when_directory_matches_fragment( + self, component: FileSystemToolComponent, sandbox: Path + ) -> None: + result = component._write_file(".kube/config", "apiVersion: v1") + + assert "error" in result + assert not (sandbox / ".kube").exists() + + +class TestDenyListEdit: + def test_should_deny_edit_when_path_matches_credential_pattern( + self, component: FileSystemToolComponent, sandbox: Path + ) -> None: + result = component._edit_file(".env", old_string=SENTINEL_VALUE, new_string="changed") + + assert "error" in result + assert SENTINEL_VALUE in (sandbox / ".env").read_text(encoding="utf-8") + + +class TestDenyListSymlinkAlias: + """A symlink whose own name is innocuous must not alias a denied target. + + The deny check runs on the input string; ``.resolve()`` follows the link + afterwards, so without a post-resolution check an ``alias -> .env`` symlink + (which a shared-root workspace can acquire out-of-band) reads ``.env``. + """ + + def test_should_deny_read_when_symlink_aliases_a_denied_file( + self, component: FileSystemToolComponent, sandbox: Path + ) -> None: + alias = sandbox / "alias" + alias.symlink_to(sandbox / ".env") + + result = component._read_file("alias") + + assert "error" in result, f"symlink alias bypassed the deny-list; result={result!r}" + assert SENTINEL_VALUE not in json.dumps(result) + + def test_should_still_read_allowed_symlink_target(self, component: FileSystemToolComponent, sandbox: Path) -> None: + alias = sandbox / "notes_link" + alias.symlink_to(sandbox / "notes.txt") + + result = component._read_file("notes_link") + + assert result.get("status") == "ok", f"allowed symlink target was blocked: {result!r}" + + +class TestDenyListProtectedDirectoryBasename: + """A path whose final component IS a protected directory name must be denied. + + Previously the basename was matched only against literal/prefix/suffix rules, + never against ``_DENY_PATH_FRAGMENTS``. Requesting ``.ssh`` / ``.aws`` / + ``.git`` directly therefore fell through to I/O and leaked a + "directory vs not-found" existence signal, and a ``**/*`` glob could surface + the protected directory entry itself even when its children were filtered. + """ + + PROTECTED_DIR_BASENAMES = [ + pytest.param(".ssh", id="ssh"), + pytest.param(".aws", id="aws"), + pytest.param(".git", id="git"), + pytest.param(".SSH", id="ssh-case-insensitive"), + pytest.param("nested/.aws", id="nested-aws"), + ] + + @pytest.mark.parametrize("path", PROTECTED_DIR_BASENAMES) + def test_should_deny_read_when_basename_is_protected_directory( + self, component: FileSystemToolComponent, path: str + ) -> None: + result = component._read_file(path) + + assert "error" in result, f"deny-list did not fire for protected directory {path!r}; result={result!r}" + assert "not found" not in result["error"].lower(), ( + "deny must fire before existence is observed (no directory-vs-not-found leak), " + f"error was: {result['error']}" + ) + + +class TestDenyListGlob: + def test_should_omit_denied_files_when_globbing_workspace(self, component: FileSystemToolComponent) -> None: + result = component._glob_search("**/*") + + assert result.get("status") == "ok" + denied_hits = [m for m in result["matches"] if Path(m).name in {".env", "id_rsa", "secret.pem", "config"}] + assert denied_hits == [], f"glob leaked denied files: {denied_hits}" + assert "notes.txt" in result["matches"] + + def test_should_omit_protected_directory_entry_when_globbing(self, component: FileSystemToolComponent) -> None: + result = component._glob_search("**/*") + + assert result.get("status") == "ok" + fragment_entries = [m for m in result["matches"] if Path(m).name in {".ssh", ".aws", ".git"}] + assert fragment_entries == [], f"glob returned a protected directory entry: {fragment_entries}" + + +class TestDenyListGrep: + def test_should_not_leak_denied_content_when_grepping_parent_directory( + self, component: FileSystemToolComponent + ) -> None: + result = component._grep_search(SENTINEL_VALUE, output_mode="content") + + assert result.get("status") == "ok" + assert result["matches"] == [], f"grep leaked denied file content: {result['matches']!r}" + + def test_should_not_list_denied_files_when_grepping_parent_directory( + self, component: FileSystemToolComponent + ) -> None: + result = component._grep_search(SENTINEL_VALUE, output_mode="files_with_matches") + + assert result.get("status") == "ok" + assert result["matches"] == [] + + def test_should_still_grep_allowed_files(self, component: FileSystemToolComponent) -> None: + result = component._grep_search("plain allowed", output_mode="files_with_matches") + + assert result.get("status") == "ok" + assert result["matches"] == ["notes.txt"]