From e5d42e54fc06dae2d9d6406d5f329dd664173d46 Mon Sep 17 00:00:00 2001 From: Rakshith Ramprakash Date: Wed, 17 Jun 2026 18:25:01 +0530 Subject: [PATCH] feat: upgrade Firecrawl to v2 SDK and extract into the lfx-firecrawl extension bundle (#13495) * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) * docs: add Firecrawl Search API to the bundle page Co-Authored-By: Claude Opus 4.8 (1M context) * [autofix.ci] apply automated fixes * fix(firecrawl): D205 docstring + defensive .get('data') in crawl Co-Authored-By: Claude Opus 4.8 (1M context) * fix(firecrawl): D205 docstring in scrape Co-Authored-By: Claude Opus 4.8 (1M context) * perf(firecrawl): use list.extend in map (PERF401) Co-Authored-By: Claude Opus 4.8 (1M context) * test: add unit tests for Firecrawl v2 components Co-Authored-By: Claude Opus 4.8 (1M context) * [autofix.ci] apply automated fixes * test: align Firecrawl component tests with lfx _attributes pattern Co-Authored-By: Claude Opus 4.8 (1M context) * [autofix.ci] apply automated fixes * refactor(firecrawl): remove deprecated extract endpoint component Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(firecrawl): remove deprecated extract endpoint component Co-Authored-By: Claude Opus 4.8 (1M context) * test: drop extract component test (extract endpoint removed) Co-Authored-By: Claude Opus 4.8 (1M context) * docs: remove Firecrawl Extract API section (deprecated) Co-Authored-By: Claude Opus 4.8 (1M context) * [autofix.ci] apply automated fixes * fix(firecrawl): map Search component to Firecrawl icon, drop Extract The frontend icon map in styleUtils.ts still referenced the removed FirecrawlExtractApi and lacked the new FirecrawlSearchApi, so the Search node would not render the Firecrawl logo. * chore: regenerate component index for Firecrawl v2 (Search added, Extract removed) * refactor(firecrawl): extract components into the lfx-firecrawl extension bundle Port the firecrawl provider out of lfx.components into a standalone Extension Bundle at src/bundles/firecrawl (distribution: lfx-firecrawl), following src/bundles/PORTING.md: - Move the v2-SDK components (Scrape, Crawl, Map, Search) to src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/ and remove the in-tree provider + its lfx.components registration. - Own the firecrawl-py>=4,<5 pin in the bundle; drop it from langflow-base. - Wire the workspace (root pyproject deps/sources/members) and uv.lock. - Append migration-table entries (bare name, both import paths, pre-a slot) for the four classes; FirecrawlExtractApi gets no entries since the v2 SDK removed the extract endpoint and the component was dropped. - Regenerate the component index (firecrawl category removed) and locales/en.json (54 firecrawl keys move out of core). - Bundle-local unit tests + test_pilot_firecrawl_upgrade integration test; update Dockerfile bundle enumeration comments. * fix(lfx): repair migration table entry fused during release-1.11.0 merge The release-1.11.0 back-merge collided the FirecrawlSearchApi legacy_slot entry with the NextPlaidVectorStoreComponent bare_class_name entry, mashing both into a single object with duplicate 'target' keys, populating two of {bare_class_name, import_path, legacy_slot}. That fails the MigrationTable validator (entries.51) and broke the lfx loader tests. Split it into the two intended entries so each populates exactly one key-field. Restores the FirecrawlSearchApi and NextPlaidVectorStoreComponent quads (60 entries total, +16 vs base). Append-only and bare-name guards pass. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Eric Hare --- .secrets.baseline | 4 +- docker/build_and_push_backend.Dockerfile | 2 +- docker/build_and_push_base.Dockerfile | 2 +- docs/docs/Components/bundles-firecrawl.mdx | 13 +- pyproject.toml | 3 + src/backend/base/langflow/locales/en.json | 54 -- src/backend/base/pyproject.toml | 1 - src/bundles/firecrawl/README.md | 29 + src/bundles/firecrawl/pyproject.toml | 55 ++ .../firecrawl/src/lfx_firecrawl/__init__.py | 14 + .../components/firecrawl/__init__.py | 13 + .../firecrawl/firecrawl_crawl_api.py | 57 +- .../components/firecrawl/firecrawl_map_api.py | 27 +- .../firecrawl/firecrawl_scrape_api.py | 40 +- .../firecrawl/firecrawl_search_api.py | 76 ++ .../src/lfx_firecrawl/extension.json | 16 + .../tests/test_firecrawl_components.py | 182 ++++ src/frontend/src/utils/styleUtils.ts | 2 +- src/lfx/src/lfx/_assets/component_index.json | 815 +----------------- src/lfx/src/lfx/components/__init__.py | 3 - .../src/lfx/components/firecrawl/__init__.py | 43 - .../firecrawl/firecrawl_extract_api.py | 136 --- .../extension/migration/migration_table.json | 80 ++ .../extension/test_pilot_firecrawl_upgrade.py | 128 +++ uv.lock | 28 +- 25 files changed, 720 insertions(+), 1103 deletions(-) create mode 100644 src/bundles/firecrawl/README.md create mode 100644 src/bundles/firecrawl/pyproject.toml create mode 100644 src/bundles/firecrawl/src/lfx_firecrawl/__init__.py create mode 100644 src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/__init__.py rename src/{lfx/src/lfx => bundles/firecrawl/src/lfx_firecrawl}/components/firecrawl/firecrawl_crawl_api.py (52%) rename src/{lfx/src/lfx => bundles/firecrawl/src/lfx_firecrawl}/components/firecrawl/firecrawl_map_api.py (71%) rename src/{lfx/src/lfx => bundles/firecrawl/src/lfx_firecrawl}/components/firecrawl/firecrawl_scrape_api.py (61%) create mode 100644 src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_search_api.py create mode 100644 src/bundles/firecrawl/src/lfx_firecrawl/extension.json create mode 100644 src/bundles/firecrawl/tests/test_firecrawl_components.py delete mode 100644 src/lfx/src/lfx/components/firecrawl/__init__.py delete mode 100644 src/lfx/src/lfx/components/firecrawl/firecrawl_extract_api.py create mode 100644 src/lfx/tests/integration/extension/test_pilot_firecrawl_upgrade.py diff --git a/.secrets.baseline b/.secrets.baseline index 8c0c458bf5..c65c0b72a0 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -2763,7 +2763,7 @@ "filename": "src/backend/base/langflow/locales/en.json", "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", "is_verified": false, - "line_number": 4606 + "line_number": 4552 } ], "src/backend/base/langflow/locales/es.json": [ @@ -9287,5 +9287,5 @@ } ] }, - "generated_at": "2026-06-10T08:36:23Z" + "generated_at": "2026-06-10T18:13:35Z" } diff --git a/docker/build_and_push_backend.Dockerfile b/docker/build_and_push_backend.Dockerfile index 2ef2c94bab..508795bb35 100644 --- a/docker/build_and_push_backend.Dockerfile +++ b/docker/build_and_push_backend.Dockerfile @@ -39,7 +39,7 @@ ENV VIRTUAL_ENV="/app/.venv" # Install langflow-base with all extras except dev (which includes Playwright). # This image ships the langflow-base core only. Extension bundles -# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling) are intentionally NOT +# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling, lfx-firecrawl) are intentionally NOT # installed here -- they belong to the full ``langflow`` distribution, not # the lean core. Use the ``langflow`` image, or ``pip install`` the bundle # alongside this image, to add those components. diff --git a/docker/build_and_push_base.Dockerfile b/docker/build_and_push_base.Dockerfile index be08fc154d..a81df8188b 100644 --- a/docker/build_and_push_base.Dockerfile +++ b/docker/build_and_push_base.Dockerfile @@ -75,7 +75,7 @@ RUN npm install \ WORKDIR /app/src/backend/base # langflow-base ships the core framework only. The extension bundles -# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling) are intentionally NOT +# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling, lfx-firecrawl) are intentionally NOT # installed in this image: they are dependencies of the full ``langflow`` # distribution, not of the lean ``langflow-base`` core, and we keep that # boundary at the image layer too. Consumers who want those components diff --git a/docs/docs/Components/bundles-firecrawl.mdx b/docs/docs/Components/bundles-firecrawl.mdx index 6ba3d4f734..a80bf9c753 100644 --- a/docs/docs/Components/bundles-firecrawl.mdx +++ b/docs/docs/Components/bundles-firecrawl.mdx @@ -60,18 +60,17 @@ It outputs the API response as [`Data`](/data-types#data). | Sitemap Only (`sitemap_only`) | Boolean | Input parameter. When true, only links found in the sitemap are returned. | | Include Subdomains (`include_subdomains`) | Boolean | Input parameter. When true, subdomains of the provided URL are also scanned. | -## Firecrawl Extract API +## Firecrawl Search API -This component extracts structured data from one or more URLs using a prompt or schema. +This component searches the web and returns the results. It outputs the API response as [`Data`](/data-types#data). -### Firecrawl Extract API parameters +### Firecrawl Search API parameters | Name | Type | Description | |------|------|-------------| | Firecrawl API Key (`api_key`) | SecretString | Input parameter. The API key to use Firecrawl API. | -| URLs (`urls`) | String | Input parameter. List of URLs to extract data from (separated by commas or new lines). | -| Prompt (`prompt`) | String | Input parameter. Prompt to guide the extraction process. | -| Schema (`schema`) | Data | Input parameter. Schema to define the structure of the extracted data. | -| Enable Web Search (`enable_web_search`) | Boolean | Input parameter. When true, the extraction uses web search to find additional data. | \ No newline at end of file +| Query (`query`) | String | Input parameter. The search query to run. | +| Limit (`limit`) | Integer | Input parameter. The maximum number of results to return. | +| Location (`location`) | String | Input parameter. Location to bias the search results (for example, a country or region). | diff --git a/pyproject.toml b/pyproject.toml index f7421444dd..3f6768c01b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "lfx-arxiv>=0.1.0", "lfx-ibm>=0.1.0", "lfx-docling>=0.1.0", + "lfx-firecrawl>=0.1.0", "lfx-nextplaid>=0.1.0", # langflow-extensions:bundle-deps-end ] @@ -85,6 +86,7 @@ lfx-duckduckgo = { workspace = true } lfx-arxiv = { workspace = true } lfx-ibm = { workspace = true } lfx-docling = { workspace = true } +lfx-firecrawl = { workspace = true } lfx-nextplaid = { workspace = true } # langflow-extensions:bundle-sources-end torch = { index = "pytorch-cpu" } @@ -102,6 +104,7 @@ members = [ "src/bundles/arxiv", "src/bundles/ibm", "src/bundles/docling", + "src/bundles/firecrawl", "src/bundles/nextplaid", # langflow-extensions:bundle-members-end ] diff --git a/src/backend/base/langflow/locales/en.json b/src/backend/base/langflow/locales/en.json index 817c874b15..ba5df4bef0 100644 --- a/src/backend/base/langflow/locales/en.json +++ b/src/backend/base/langflow/locales/en.json @@ -3567,60 +3567,6 @@ "components.filterdatavalues.inputs.operator.display_name.d6b063d9": "Comparison Operator", "components.filterdatavalues.inputs.operator.info.81652313": "The operator to apply for comparing the values.", "components.filterdatavalues.outputs.filtered_data.display_name.8acf8a3c": "Filtered Data", - "components.firecrawlcrawlapi.description.e9b9ca89": "Crawls a URL and returns the results.", - "components.firecrawlcrawlapi.display_name.1c8af0bd": "Firecrawl Crawl API", - "components.firecrawlcrawlapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key", - "components.firecrawlcrawlapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.", - "components.firecrawlcrawlapi.inputs.crawlerOptions.display_name.77b95f41": "Crawler Options", - "components.firecrawlcrawlapi.inputs.crawlerOptions.info.b3898e10": "The crawler options to send with the request.", - "components.firecrawlcrawlapi.inputs.idempotency_key.display_name.0409cf23": "Idempotency Key", - "components.firecrawlcrawlapi.inputs.idempotency_key.info.7fa2bc03": "Optional idempotency key to ensure unique requests.", - "components.firecrawlcrawlapi.inputs.scrapeOptions.display_name.f6227db1": "Scrape Options", - "components.firecrawlcrawlapi.inputs.scrapeOptions.info.034e1d4f": "The page options to send with the request.", - "components.firecrawlcrawlapi.inputs.timeout.display_name.70594d93": "Timeout", - "components.firecrawlcrawlapi.inputs.timeout.info.11f2ddb2": "Timeout in milliseconds for the request.", - "components.firecrawlcrawlapi.inputs.url.display_name.e7a241de": "URL", - "components.firecrawlcrawlapi.inputs.url.info.b260224a": "The URL to scrape.", - "components.firecrawlcrawlapi.outputs.data.display_name.db1a21a0": "JSON", - "components.firecrawlextractapi.description.eed57be9": "Extracts data from a URL.", - "components.firecrawlextractapi.display_name.ae3a11a0": "Firecrawl Extract API", - "components.firecrawlextractapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key", - "components.firecrawlextractapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.", - "components.firecrawlextractapi.inputs.enable_web_search.display_name.a53c041a": "Enable Web Search", - "components.firecrawlextractapi.inputs.enable_web_search.info.07671323": "When true, the extraction will use web search to find additional data.", - "components.firecrawlextractapi.inputs.prompt.display_name.5c391238": "Prompt", - "components.firecrawlextractapi.inputs.prompt.info.30c68d4d": "Prompt to guide the extraction process.", - "components.firecrawlextractapi.inputs.schema.display_name.07b091a3": "Schema", - "components.firecrawlextractapi.inputs.schema.info.fc0ac9cc": "Schema to define the structure of the extracted data.", - "components.firecrawlextractapi.inputs.urls.display_name.1240054e": "URLs", - "components.firecrawlextractapi.inputs.urls.info.a35e321b": "List of URLs to extract data from (separated by commas or new lines).", - "components.firecrawlextractapi.outputs.data.display_name.db1a21a0": "JSON", - "components.firecrawlmapapi.description.acbcf5bc": "Maps a URL and returns the results.", - "components.firecrawlmapapi.display_name.30b2bee7": "Firecrawl Map API", - "components.firecrawlmapapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key", - "components.firecrawlmapapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.", - "components.firecrawlmapapi.inputs.ignore_sitemap.display_name.6d48c522": "Ignore Sitemap", - "components.firecrawlmapapi.inputs.ignore_sitemap.info.02a79d48": "When true, the sitemap.xml file will be ignored during crawling.", - "components.firecrawlmapapi.inputs.include_subdomains.display_name.5cf2dc9e": "Include Subdomains", - "components.firecrawlmapapi.inputs.include_subdomains.info.073e9171": "When true, subdomains of the provided URL will also be scanned.", - "components.firecrawlmapapi.inputs.sitemap_only.display_name.021a3604": "Sitemap Only", - "components.firecrawlmapapi.inputs.sitemap_only.info.7ca2f13c": "When true, only links found in the sitemap will be returned.", - "components.firecrawlmapapi.inputs.urls.display_name.1240054e": "URLs", - "components.firecrawlmapapi.inputs.urls.info.71719292": "List of URLs to create maps from (separated by commas or new lines).", - "components.firecrawlmapapi.outputs.data.display_name.db1a21a0": "JSON", - "components.firecrawlscrapeapi.description.017ffc83": "Scrapes a URL and returns the results.", - "components.firecrawlscrapeapi.display_name.39a1c0de": "Firecrawl Scrape API", - "components.firecrawlscrapeapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key", - "components.firecrawlscrapeapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.", - "components.firecrawlscrapeapi.inputs.extractorOptions.display_name.80a369a3": "Extractor Options", - "components.firecrawlscrapeapi.inputs.extractorOptions.info.9fd7d66f": "The extractor options to send with the request.", - "components.firecrawlscrapeapi.inputs.scrapeOptions.display_name.f6227db1": "Scrape Options", - "components.firecrawlscrapeapi.inputs.scrapeOptions.info.034e1d4f": "The page options to send with the request.", - "components.firecrawlscrapeapi.inputs.timeout.display_name.70594d93": "Timeout", - "components.firecrawlscrapeapi.inputs.timeout.info.11f2ddb2": "Timeout in milliseconds for the request.", - "components.firecrawlscrapeapi.inputs.url.display_name.e7a241de": "URL", - "components.firecrawlscrapeapi.inputs.url.info.b260224a": "The URL to scrape.", - "components.firecrawlscrapeapi.outputs.data.display_name.db1a21a0": "JSON", "components.flowtool.description.d3907775": "Construct a Tool from a function that runs the loaded Flow.", "components.flowtool.display_name.21209f51": "Flow as Tool", "components.flowtool.inputs.flow_name.display_name.e5fcec99": "Flow Name", diff --git a/src/backend/base/pyproject.toml b/src/backend/base/pyproject.toml index 5dea4eaa90..8be6961883 100644 --- a/src/backend/base/pyproject.toml +++ b/src/backend/base/pyproject.toml @@ -64,7 +64,6 @@ dependencies = [ "uncurl>=0.0.11,<1.0.0", "sentry-sdk[fastapi,loguru]>=2.5.1,<3.0.0", "chardet>=7.3.0", - "firecrawl-py>=1.0.16,<2.0.0", "opentelemetry-api>=1.30.0,<2.0.0", "opentelemetry-sdk>=1.30.0,<2.0.0", "opentelemetry-exporter-prometheus>=0.50b0,<1.0.0", diff --git a/src/bundles/firecrawl/README.md b/src/bundles/firecrawl/README.md new file mode 100644 index 0000000000..86b40755b7 --- /dev/null +++ b/src/bundles/firecrawl/README.md @@ -0,0 +1,29 @@ +# lfx-firecrawl + +Firecrawl components (Scrape, Crawl, Map, and Search APIs) as a standalone +Langflow Extension Bundle, built against the firecrawl-py v2 SDK. + +## Install + +```bash +pip install lfx-firecrawl +``` + +The bundle is registered automatically via the `langflow.extensions` +entry-point. After install, restart your Langflow server; the bundle's +components will appear in the palette under the `firecrawl` group. + +## Develop + +```bash +cd src/bundles/firecrawl +pip install -e . +lfx extension validate src/lfx_firecrawl +``` + +## Migration + +Saved flows referencing the legacy class name(s) or the old import paths +under `lfx.components.firecrawl.*` are rewritten to the new namespaced +IDs by the migration table in +`src/lfx/src/lfx/extension/migration/migration_table.json`. diff --git a/src/bundles/firecrawl/pyproject.toml b/src/bundles/firecrawl/pyproject.toml new file mode 100644 index 0000000000..613860194d --- /dev/null +++ b/src/bundles/firecrawl/pyproject.toml @@ -0,0 +1,55 @@ +[project] +name = "lfx-firecrawl" +version = "0.1.0" +description = "Firecrawl components (Scrape, Crawl, Map, and Search APIs) as a standalone Langflow Extension Bundle." +readme = "README.md" +requires-python = ">=3.10,<3.15" +license = { text = "MIT" } +authors = [ + { name = "Langflow", email = "contact@langflow.org" }, +] +keywords = ["langflow", "lfx", "extension", "bundle", "firecrawl"] + +# Runtime deps: lfx (the BUNDLE_API surface) plus any third-party imports +# the bundle's components rely on. REVIEW THIS LIST -- the script ports +# only ``lfx``; add any other deps the moved component(s) import. +# lfx is floored at the current major.minor line and capped below the next +# lfx major (e.g. "lfx>=1.10.0,<2.0.0"); the floor is read from +# src/lfx/pyproject.toml at port time and re-synced on ``make patch`` via +# scripts/ci/sync_bundle_lfx_pin.py. Fine-grained BUNDLE_API compatibility +# is enforced via extension.json's lfx.compat list against BUNDLE_API_VERSION. +dependencies = [ + "lfx>=1.11.0.dev0,<2.0.0", + "firecrawl-py>=4.0.0,<5.0.0", +] + +[project.urls] +Homepage = "https://github.com/langflow-ai/langflow" +Documentation = "https://docs.langflow.org/extensions" +Repository = "https://github.com/langflow-ai/langflow" + +# Manifest-shipping distributions are discovered via the +# ``langflow.extensions`` entry-point. Editable installs whose +# ``dist.files`` only surfaces dist-info entries fall back to this +# entry-point to find the manifest. +[project.entry-points."langflow.extensions"] +lfx-firecrawl = "lfx_firecrawl" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +# extension.json + components live inside the lfx_firecrawl package so +# ``importlib.metadata.files(dist)`` finds them and the loader resolves +# bundles[].path relative to the manifest's directory. +packages = ["src/lfx_firecrawl"] +include = ["src/lfx_firecrawl/extension.json", "src/lfx_firecrawl/components/**/*.py"] + +[tool.hatch.build.targets.sdist] +include = [ + "src/lfx_firecrawl", + "extension.json", + "README.md", + "pyproject.toml", +] diff --git a/src/bundles/firecrawl/src/lfx_firecrawl/__init__.py b/src/bundles/firecrawl/src/lfx_firecrawl/__init__.py new file mode 100644 index 0000000000..1df9f385e0 --- /dev/null +++ b/src/bundles/firecrawl/src/lfx_firecrawl/__init__.py @@ -0,0 +1,14 @@ +"""lfx-firecrawl: Firecrawl bundle. + +Distribution unit ``lfx-firecrawl``. At runtime Langflow's loader +discovers ``extension.json`` shipped alongside this ``__init__.py`` and +registers the bundle's components under the namespaced IDs +``ext:firecrawl:@official``. +""" + +from lfx_firecrawl.components.firecrawl.firecrawl_crawl_api import FirecrawlCrawlApi +from lfx_firecrawl.components.firecrawl.firecrawl_map_api import FirecrawlMapApi +from lfx_firecrawl.components.firecrawl.firecrawl_scrape_api import FirecrawlScrapeApi +from lfx_firecrawl.components.firecrawl.firecrawl_search_api import FirecrawlSearchApi + +__all__ = ["FirecrawlCrawlApi", "FirecrawlMapApi", "FirecrawlScrapeApi", "FirecrawlSearchApi"] diff --git a/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/__init__.py b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/__init__.py new file mode 100644 index 0000000000..c80de80888 --- /dev/null +++ b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/__init__.py @@ -0,0 +1,13 @@ +"""Component re-exports for the ``firecrawl`` bundle. + +Saved-flow migration entries that target ``lfx.components.firecrawl.`` +resolve through this package, so the moved Component class(es) must be +importable from here by name. +""" + +from .firecrawl_crawl_api import FirecrawlCrawlApi +from .firecrawl_map_api import FirecrawlMapApi +from .firecrawl_scrape_api import FirecrawlScrapeApi +from .firecrawl_search_api import FirecrawlSearchApi + +__all__ = ["FirecrawlCrawlApi", "FirecrawlMapApi", "FirecrawlScrapeApi", "FirecrawlSearchApi"] diff --git a/src/lfx/src/lfx/components/firecrawl/firecrawl_crawl_api.py b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_crawl_api.py similarity index 52% rename from src/lfx/src/lfx/components/firecrawl/firecrawl_crawl_api.py rename to src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_crawl_api.py index a1315c964c..82c8c7e62a 100644 --- a/src/lfx/src/lfx/components/firecrawl/firecrawl_crawl_api.py +++ b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_crawl_api.py @@ -1,9 +1,21 @@ -import uuid +import re from lfx.custom.custom_component.component import Component from lfx.io import DataInput, IntInput, MultilineInput, Output, SecretStrInput, StrInput from lfx.schema.data import Data +_CAMEL_TO_SNAKE_RE = re.compile(r"(? dict: + """Convert camelCase option keys to snake_case keyword arguments. + + The firecrawl-py v1 convention uses camelCase, while the v2 SDK expects + snake_case keyword arguments. Keys that are already snake_case are passed + through unchanged. + """ + return {_CAMEL_TO_SNAKE_RE.sub("_", key).lower(): value for key, value in params.items()} + class FirecrawlCrawlApi(Component): display_name: str = "Firecrawl Crawl API" @@ -56,33 +68,42 @@ class FirecrawlCrawlApi(Component): def crawl(self) -> Data: try: - from firecrawl import FirecrawlApp + from firecrawl import Firecrawl + from firecrawl.v2.types import ScrapeOptions except ImportError as e: msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`." raise ImportError(msg) from e - params = self.crawlerOptions.__dict__["data"] if self.crawlerOptions else {} - scrape_options_dict = self.scrapeOptions.__dict__["data"] if self.scrapeOptions else {} - if scrape_options_dict: - params["scrapeOptions"] = scrape_options_dict + params = dict(self.crawlerOptions.__dict__.get("data", {})) if self.crawlerOptions else {} + scrape_options_dict = dict(self.scrapeOptions.__dict__.get("data", {})) if self.scrapeOptions else {} - # Set default values for new parameters in v1 + # Set default values for crawl parameters. + # Note: firecrawl-py v2 renamed several options. "maxDepth" -> "max_discovery_depth" + # and "allowBackwardLinks" -> "crawl_entire_domain". params.setdefault("maxDepth", 2) params.setdefault("limit", 10000) params.setdefault("allowExternalLinks", False) params.setdefault("allowBackwardLinks", False) - params.setdefault("ignoreSitemap", False) params.setdefault("ignoreQueryParameters", False) - # Ensure onlyMainContent is explicitly set if not provided - if "scrapeOptions" in params: - params["scrapeOptions"].setdefault("onlyMainContent", True) - else: - params["scrapeOptions"] = {"onlyMainContent": True} + # Ensure onlyMainContent is explicitly set if not provided. + scrape_options_dict.setdefault("onlyMainContent", True) - if not self.idempotency_key: - self.idempotency_key = str(uuid.uuid4()) + # Translate legacy v1 camelCase option names to the firecrawl-py v2 keyword args. + kwargs = _to_snake_case_kwargs(params) + if "max_depth" in kwargs: + kwargs["max_discovery_depth"] = kwargs.pop("max_depth") + if "allow_backward_links" in kwargs: + kwargs["crawl_entire_domain"] = kwargs.pop("allow_backward_links") + # v2 removed "ignore_sitemap"; it is now the "sitemap" mode enum. + if kwargs.pop("ignore_sitemap", False): + kwargs["sitemap"] = "skip" - app = FirecrawlApp(api_key=self.api_key) - crawl_result = app.crawl_url(self.url, params=params, idempotency_key=self.idempotency_key) - return Data(data={"results": crawl_result}) + # Build the typed ScrapeOptions object expected by v2 from the (snake_cased) dict. + scrape_kwargs = _to_snake_case_kwargs(scrape_options_dict) + kwargs["scrape_options"] = ScrapeOptions(**scrape_kwargs) + + app = Firecrawl(api_key=self.api_key) + # v2 polls to completion and returns a typed CrawlJob object. + crawl_job = app.crawl(self.url, **kwargs) + return Data(data={"results": crawl_job.model_dump()}) diff --git a/src/lfx/src/lfx/components/firecrawl/firecrawl_map_api.py b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_map_api.py similarity index 71% rename from src/lfx/src/lfx/components/firecrawl/firecrawl_map_api.py rename to src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_map_api.py index 9f33514487..59c0361044 100644 --- a/src/lfx/src/lfx/components/firecrawl/firecrawl_map_api.py +++ b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_map_api.py @@ -53,7 +53,7 @@ class FirecrawlMapApi(Component): def map(self) -> Data: try: - from firecrawl import FirecrawlApp + from firecrawl import Firecrawl except ImportError as e: msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`." raise ImportError(msg) from e @@ -69,20 +69,25 @@ class FirecrawlMapApi(Component): msg = "No valid URLs provided" raise ValueError(msg) - params = { - "ignoreSitemap": self.ignore_sitemap, - "sitemapOnly": self.sitemap_only, - "includeSubdomains": self.include_subdomains, - } + # firecrawl-py v2 replaced the separate "ignoreSitemap"/"sitemapOnly" flags with a + # single "sitemap" mode: "only" (sitemap only), "skip" (ignore sitemap), or the + # default mixed behavior. + kwargs: dict = {"include_subdomains": self.include_subdomains} + if self.sitemap_only: + kwargs["sitemap"] = "only" + elif self.ignore_sitemap: + kwargs["sitemap"] = "skip" - app = FirecrawlApp(api_key=self.api_key) + app = Firecrawl(api_key=self.api_key) - # Map all provided URLs and combine results + # Map all provided URLs and combine results. combined_links = [] for url in urls: - result = app.map_url(url, params=params) - if isinstance(result, dict) and "links" in result: - combined_links.extend(result["links"]) + result = app.map(url, **kwargs) + # v2 returns a typed MapData object whose .links is a list of typed + # SearchResult objects; serialize each to a dict for downstream consumers. + links = getattr(result, "links", None) or [] + combined_links.extend(link.model_dump() if hasattr(link, "model_dump") else link for link in links) map_result = {"success": True, "links": combined_links} diff --git a/src/lfx/src/lfx/components/firecrawl/firecrawl_scrape_api.py b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_scrape_api.py similarity index 61% rename from src/lfx/src/lfx/components/firecrawl/firecrawl_scrape_api.py rename to src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_scrape_api.py index 718146ea6d..04c97627bb 100644 --- a/src/lfx/src/lfx/components/firecrawl/firecrawl_scrape_api.py +++ b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_scrape_api.py @@ -1,3 +1,5 @@ +import re + from lfx.custom.custom_component.component import Component from lfx.io import ( DataInput, @@ -8,6 +10,18 @@ from lfx.io import ( ) from lfx.schema.data import Data +_CAMEL_TO_SNAKE_RE = re.compile(r"(? dict: + """Convert camelCase option keys to snake_case keyword arguments. + + The firecrawl-py v1 convention uses camelCase, while the v2 SDK expects + snake_case keyword arguments. Keys that are already snake_case are passed + through unchanged. + """ + return {_CAMEL_TO_SNAKE_RE.sub("_", key).lower(): value for key, value in params.items()} + class FirecrawlScrapeApi(Component): display_name: str = "Firecrawl Scrape API" @@ -54,20 +68,30 @@ class FirecrawlScrapeApi(Component): def scrape(self) -> Data: try: - from firecrawl import FirecrawlApp + from firecrawl import Firecrawl except ImportError as e: msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`." raise ImportError(msg) from e - params = self.scrapeOptions.__dict__.get("data", {}) if self.scrapeOptions else {} - extractor_options_dict = self.extractorOptions.__dict__.get("data", {}) if self.extractorOptions else {} - if extractor_options_dict: - params["extract"] = extractor_options_dict + params = dict(self.scrapeOptions.__dict__.get("data", {})) if self.scrapeOptions else {} # Set default values for parameters params.setdefault("formats", ["markdown"]) # Default output format params.setdefault("onlyMainContent", True) # Default to only main content + if self.timeout: + params.setdefault("timeout", self.timeout) - app = FirecrawlApp(api_key=self.api_key) - results = app.scrape_url(self.url, params=params) - return Data(data=results) + kwargs = _to_snake_case_kwargs(params) + + # In firecrawl-py v2, structured extraction is requested via a "json" format entry + # on the scrape call rather than a separate extractor option. + extractor_options_dict = self.extractorOptions.__dict__.get("data", {}) if self.extractorOptions else {} + if extractor_options_dict: + formats = list(kwargs.get("formats", [])) + formats.append({"type": "json", **_to_snake_case_kwargs(extractor_options_dict)}) + kwargs["formats"] = formats + + app = Firecrawl(api_key=self.api_key) + document = app.scrape(self.url, **kwargs) + # v2 returns a typed Document object; serialize to a dict for downstream consumers. + return Data(data=document.model_dump()) diff --git a/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_search_api.py b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_search_api.py new file mode 100644 index 0000000000..aaaded0b6b --- /dev/null +++ b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_search_api.py @@ -0,0 +1,76 @@ +from lfx.custom.custom_component.component import Component +from lfx.io import ( + IntInput, + MultilineInput, + Output, + SecretStrInput, + StrInput, +) +from lfx.schema.data import Data + + +class FirecrawlSearchApi(Component): + display_name: str = "Firecrawl Search API" + description: str = "Searches the web and returns the results." + name = "FirecrawlSearchApi" + + documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/search" + + inputs = [ + SecretStrInput( + name="api_key", + display_name="Firecrawl API Key", + required=True, + password=True, + info="The API key to use Firecrawl API.", + ), + MultilineInput( + name="query", + display_name="Query", + required=True, + info="The search query to run.", + tool_mode=True, + ), + IntInput( + name="limit", + display_name="Limit", + info="Maximum number of results to return.", + value=5, + ), + StrInput( + name="location", + display_name="Location", + info="Location to bias the search results (e.g. a country or region).", + advanced=True, + ), + ] + + outputs = [ + Output(display_name="JSON", name="data", method="search"), + ] + + def search(self) -> Data: + try: + from firecrawl import Firecrawl + except ImportError as e: + msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`." + raise ImportError(msg) from e + + if not self.query: + msg = "Query is required" + raise ValueError(msg) + + kwargs: dict = {} + if self.limit: + kwargs["limit"] = self.limit + if self.location: + kwargs["location"] = self.location + + app = Firecrawl(api_key=self.api_key) + result = app.search(self.query, **kwargs) + + # v2 returns a typed SearchData object (results grouped by source); + # serialize to a dict for downstream consumers. + search_result = result.model_dump() if hasattr(result, "model_dump") else result + + return Data(data=search_result) diff --git a/src/bundles/firecrawl/src/lfx_firecrawl/extension.json b/src/bundles/firecrawl/src/lfx_firecrawl/extension.json new file mode 100644 index 0000000000..8530f8fa81 --- /dev/null +++ b/src/bundles/firecrawl/src/lfx_firecrawl/extension.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://schemas.langflow.org/extension/v1.json", + "id": "lfx-firecrawl", + "version": "0.1.0", + "name": "Firecrawl", + "description": "Firecrawl components (Scrape, Crawl, Map, and Search APIs) as a standalone Langflow Extension Bundle.", + "lfx": { + "compat": ["1"] + }, + "bundles": [ + { + "name": "firecrawl", + "path": "components/firecrawl" + } + ] +} diff --git a/src/bundles/firecrawl/tests/test_firecrawl_components.py b/src/bundles/firecrawl/tests/test_firecrawl_components.py new file mode 100644 index 0000000000..c8ec7f2e2e --- /dev/null +++ b/src/bundles/firecrawl/tests/test_firecrawl_components.py @@ -0,0 +1,182 @@ +"""Unit tests for the Firecrawl extension bundle (``lfx-firecrawl``). + +The components used to live at ``lfx.components.firecrawl``; they have since +been extracted into a standalone bundle, so these tests travel with the +bundle and import the public bundle entry point. + +They verify v2 SDK usage, the v1->v2 parameter mapping, response +serialization, and error handling using a mocked Firecrawl SDK. No network +access or real ``firecrawl-py`` install is required: the SDK is injected via +``sys.modules`` so the components' in-method ``from firecrawl import ...`` +statements resolve to the mocks. +""" + +import sys +import types +from unittest.mock import MagicMock + +import pytest +from lfx_firecrawl import FirecrawlCrawlApi, FirecrawlMapApi, FirecrawlScrapeApi, FirecrawlSearchApi + + +@pytest.fixture +def mock_firecrawl(monkeypatch): + """Inject a fake ``firecrawl`` SDK and return the mocked ``Firecrawl`` class. + + ``mock_firecrawl.return_value`` is the client instance whose + ``.scrape``/``.crawl``/``.map``/``.extract``/``.search`` methods can be + stubbed per test. + """ + client_cls = MagicMock(name="Firecrawl") + + firecrawl_mod = types.ModuleType("firecrawl") + firecrawl_mod.Firecrawl = client_cls + + v2_mod = types.ModuleType("firecrawl.v2") + types_mod = types.ModuleType("firecrawl.v2.types") + types_mod.ScrapeOptions = MagicMock(name="ScrapeOptions") + v2_mod.types = types_mod + + monkeypatch.setitem(sys.modules, "firecrawl", firecrawl_mod) + monkeypatch.setitem(sys.modules, "firecrawl.v2", v2_mod) + monkeypatch.setitem(sys.modules, "firecrawl.v2.types", types_mod) + return client_cls + + +def _typed(dump): + """Return a fake v2 typed response whose ``model_dump()`` yields ``dump``.""" + obj = MagicMock() + obj.model_dump.return_value = dump + return obj + + +class TestFirecrawlScrapeApi: + def test_scrape_calls_v2_and_serializes(self, mock_firecrawl): + client = mock_firecrawl.return_value + client.scrape.return_value = _typed({"markdown": "# Hi", "metadata": {}}) + + component = FirecrawlScrapeApi() + component._attributes = { + "api_key": "test-key", # pragma: allowlist secret + "url": "https://example.com", + "timeout": 0, + "scrapeOptions": None, + "extractorOptions": None, + } + + result = component.scrape() + + mock_firecrawl.assert_called_once_with(api_key="test-key") # pragma: allowlist secret + client.scrape.assert_called_once() + assert client.scrape.call_args.args[0] == "https://example.com" + assert result.data == {"markdown": "# Hi", "metadata": {}} + + def test_scrape_propagates_sdk_error(self, mock_firecrawl): + client = mock_firecrawl.return_value + client.scrape.side_effect = RuntimeError("boom") + + component = FirecrawlScrapeApi() + component._attributes = { + "api_key": "k", # pragma: allowlist secret + "url": "https://example.com", + "timeout": 0, + "scrapeOptions": None, + "extractorOptions": None, + } + + with pytest.raises(RuntimeError): + component.scrape() + + +class TestFirecrawlCrawlApi: + def test_crawl_maps_legacy_params_and_serializes(self, mock_firecrawl): + client = mock_firecrawl.return_value + client.crawl.return_value = _typed({"status": "completed", "data": []}) + + component = FirecrawlCrawlApi() + component._attributes = { + "api_key": "test-key", # pragma: allowlist secret + "url": "https://example.com", + "timeout": 0, + "idempotency_key": "", + "crawlerOptions": None, + "scrapeOptions": None, + } + + result = component.crawl() + + mock_firecrawl.assert_called_once_with(api_key="test-key") # pragma: allowlist secret + client.crawl.assert_called_once() + assert client.crawl.call_args.args[0] == "https://example.com" + kwargs = client.crawl.call_args.kwargs + # v1 "maxDepth" was renamed to v2 "max_discovery_depth". + assert "max_discovery_depth" in kwargs + assert "max_depth" not in kwargs + # v1 "allowBackwardLinks" was renamed to v2 "crawl_entire_domain". + assert "crawl_entire_domain" in kwargs + assert result.data["results"] == {"status": "completed", "data": []} + + +class TestFirecrawlMapApi: + def test_map_sitemap_only_maps_to_enum(self, mock_firecrawl): + client = mock_firecrawl.return_value + result_obj = MagicMock() + result_obj.links = [_typed({"url": "https://example.com/a"})] + client.map.return_value = result_obj + + component = FirecrawlMapApi() + component._attributes = { + "api_key": "test-key", # pragma: allowlist secret + "urls": "https://example.com", + "ignore_sitemap": False, + "sitemap_only": True, + "include_subdomains": False, + } + + result = component.map() + + client.map.assert_called_once() + assert client.map.call_args.kwargs.get("sitemap") == "only" + assert result.data["links"] == [{"url": "https://example.com/a"}] + + def test_map_ignore_sitemap_maps_to_skip(self, mock_firecrawl): + client = mock_firecrawl.return_value + result_obj = MagicMock() + result_obj.links = [] + client.map.return_value = result_obj + + component = FirecrawlMapApi() + component._attributes = { + "api_key": "k", # pragma: allowlist secret + "urls": "https://example.com", + "ignore_sitemap": True, + "sitemap_only": False, + "include_subdomains": False, + } + + component.map() + + assert client.map.call_args.kwargs.get("sitemap") == "skip" + + +class TestFirecrawlSearchApi: + def test_search_calls_v2_with_query_limit_location(self, mock_firecrawl): + client = mock_firecrawl.return_value + client.search.return_value = _typed({"web": [{"url": "https://example.com"}]}) + + component = FirecrawlSearchApi() + component._attributes = { + "api_key": "test-key", # pragma: allowlist secret + "query": "firecrawl", + "limit": 5, + "location": "US", + } + + result = component.search() + + mock_firecrawl.assert_called_once_with(api_key="test-key") # pragma: allowlist secret + client.search.assert_called_once() + assert client.search.call_args.args[0] == "firecrawl" + assert client.search.call_args.kwargs.get("limit") == 5 + assert client.search.call_args.kwargs.get("location") == "US" + assert result.data == {"web": [{"url": "https://example.com"}]} diff --git a/src/frontend/src/utils/styleUtils.ts b/src/frontend/src/utils/styleUtils.ts index 9b3e5d3054..792bf7e9a0 100644 --- a/src/frontend/src/utils/styleUtils.ts +++ b/src/frontend/src/utils/styleUtils.ts @@ -620,9 +620,9 @@ export const nodeIconToDisplayIconMap: Record = { FAISS: "Meta", FaissSearch: "Meta", FirecrawlCrawlApi: "Firecrawl", - FirecrawlExtractApi: "Firecrawl", FirecrawlMapApi: "Firecrawl", FirecrawlScrapeApi: "Firecrawl", + FirecrawlSearchApi: "Firecrawl", GitbookLoader: "GitBook", GoogleGenerativeAI: "GoogleGenerativeAI", Googlesheets: "Googlesheets", diff --git a/src/lfx/src/lfx/_assets/component_index.json b/src/lfx/src/lfx/_assets/component_index.json index bce86b23c4..65cc46b042 100644 --- a/src/lfx/src/lfx/_assets/component_index.json +++ b/src/lfx/src/lfx/_assets/component_index.json @@ -71522,815 +71522,6 @@ } } ], - [ - "firecrawl", - { - "FirecrawlCrawlApi": { - "base_classes": [ - "JSON" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Crawls a URL and returns the results.", - "display_name": "Firecrawl Crawl API", - "documentation": "https://docs.firecrawl.dev/v1/api-reference/endpoint/crawl-post", - "edited": false, - "field_order": [ - "api_key", - "url", - "timeout", - "idempotency_key", - "crawlerOptions", - "scrapeOptions" - ], - "frozen": false, - "legacy": false, - "metadata": { - "code_hash": "21b4965f8b53", - "dependencies": { - "dependencies": [ - { - "name": "lfx", - "version": null - }, - { - "name": "firecrawl", - "version": "1.17.0" - } - ], - "total_dependencies": 2 - }, - "module": "lfx.components.firecrawl.firecrawl_crawl_api.FirecrawlCrawlApi" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "JSON", - "group_outputs": false, - "method": "crawl", - "name": "data", - "selected": "JSON", - "tool_mode": true, - "types": [ - "JSON" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "api_key": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Firecrawl API Key", - "dynamic": false, - "info": "The API key to use Firecrawl API.", - "input_types": [], - "load_from_db": true, - "name": "api_key", - "override_skip": false, - "password": true, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "import uuid\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.io import DataInput, IntInput, MultilineInput, Output, SecretStrInput, StrInput\nfrom lfx.schema.data import Data\n\n\nclass FirecrawlCrawlApi(Component):\n display_name: str = \"Firecrawl Crawl API\"\n description: str = \"Crawls a URL and returns the results.\"\n name = \"FirecrawlCrawlApi\"\n\n documentation: str = \"https://docs.firecrawl.dev/v1/api-reference/endpoint/crawl-post\"\n\n inputs = [\n SecretStrInput(\n name=\"api_key\",\n display_name=\"Firecrawl API Key\",\n required=True,\n password=True,\n info=\"The API key to use Firecrawl API.\",\n ),\n MultilineInput(\n name=\"url\",\n display_name=\"URL\",\n required=True,\n info=\"The URL to scrape.\",\n tool_mode=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout in milliseconds for the request.\",\n ),\n StrInput(\n name=\"idempotency_key\",\n display_name=\"Idempotency Key\",\n info=\"Optional idempotency key to ensure unique requests.\",\n ),\n DataInput(\n name=\"crawlerOptions\",\n display_name=\"Crawler Options\",\n info=\"The crawler options to send with the request.\",\n ),\n DataInput(\n name=\"scrapeOptions\",\n display_name=\"Scrape Options\",\n info=\"The page options to send with the request.\",\n ),\n ]\n\n outputs = [\n Output(display_name=\"JSON\", name=\"data\", method=\"crawl\"),\n ]\n idempotency_key: str | None = None\n\n def crawl(self) -> Data:\n try:\n from firecrawl import FirecrawlApp\n except ImportError as e:\n msg = \"Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`.\"\n raise ImportError(msg) from e\n\n params = self.crawlerOptions.__dict__[\"data\"] if self.crawlerOptions else {}\n scrape_options_dict = self.scrapeOptions.__dict__[\"data\"] if self.scrapeOptions else {}\n if scrape_options_dict:\n params[\"scrapeOptions\"] = scrape_options_dict\n\n # Set default values for new parameters in v1\n params.setdefault(\"maxDepth\", 2)\n params.setdefault(\"limit\", 10000)\n params.setdefault(\"allowExternalLinks\", False)\n params.setdefault(\"allowBackwardLinks\", False)\n params.setdefault(\"ignoreSitemap\", False)\n params.setdefault(\"ignoreQueryParameters\", False)\n\n # Ensure onlyMainContent is explicitly set if not provided\n if \"scrapeOptions\" in params:\n params[\"scrapeOptions\"].setdefault(\"onlyMainContent\", True)\n else:\n params[\"scrapeOptions\"] = {\"onlyMainContent\": True}\n\n if not self.idempotency_key:\n self.idempotency_key = str(uuid.uuid4())\n\n app = FirecrawlApp(api_key=self.api_key)\n crawl_result = app.crawl_url(self.url, params=params, idempotency_key=self.idempotency_key)\n return Data(data={\"results\": crawl_result})\n" - }, - "crawlerOptions": { - "_input_type": "JSONInput", - "advanced": false, - "display_name": "Crawler Options", - "dynamic": false, - "info": "The crawler options to send with the request.", - "input_types": [ - "Data", - "JSON" - ], - "list": false, - "list_add_label": "Add More", - "name": "crawlerOptions", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "other", - "value": "" - }, - "idempotency_key": { - "_input_type": "StrInput", - "advanced": false, - "display_name": "Idempotency Key", - "dynamic": false, - "info": "Optional idempotency key to ensure unique requests.", - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "idempotency_key", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "scrapeOptions": { - "_input_type": "JSONInput", - "advanced": false, - "display_name": "Scrape Options", - "dynamic": false, - "info": "The page options to send with the request.", - "input_types": [ - "Data", - "JSON" - ], - "list": false, - "list_add_label": "Add More", - "name": "scrapeOptions", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "other", - "value": "" - }, - "timeout": { - "_input_type": "IntInput", - "advanced": false, - "display_name": "Timeout", - "dynamic": false, - "info": "Timeout in milliseconds for the request.", - "list": false, - "list_add_label": "Add More", - "name": "timeout", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "int", - "value": 0 - }, - "url": { - "_input_type": "MultilineInput", - "advanced": false, - "ai_enabled": false, - "copy_field": false, - "display_name": "URL", - "dynamic": false, - "info": "The URL to scrape.", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "multiline": true, - "name": "url", - "override_skip": false, - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "tool_mode": true, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - } - }, - "tool_mode": false - }, - "FirecrawlExtractApi": { - "base_classes": [ - "JSON" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Extracts data from a URL.", - "display_name": "Firecrawl Extract API", - "documentation": "https://docs.firecrawl.dev/api-reference/endpoint/extract", - "edited": false, - "field_order": [ - "api_key", - "urls", - "prompt", - "schema", - "enable_web_search" - ], - "frozen": false, - "legacy": false, - "metadata": { - "code_hash": "1363b7da7bf7", - "dependencies": { - "dependencies": [ - { - "name": "lfx", - "version": null - }, - { - "name": "firecrawl", - "version": "1.17.0" - } - ], - "total_dependencies": 2 - }, - "module": "lfx.components.firecrawl.firecrawl_extract_api.FirecrawlExtractApi" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "JSON", - "group_outputs": false, - "method": "extract", - "name": "data", - "selected": "JSON", - "tool_mode": true, - "types": [ - "JSON" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "api_key": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Firecrawl API Key", - "dynamic": false, - "info": "The API key to use Firecrawl API.", - "input_types": [], - "load_from_db": true, - "name": "api_key", - "override_skip": false, - "password": true, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from lfx.custom.custom_component.component import Component\nfrom lfx.io import BoolInput, DataInput, MultilineInput, Output, SecretStrInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.data import Data\n\n\nclass FirecrawlExtractApi(Component):\n display_name: str = \"Firecrawl Extract API\"\n description: str = \"Extracts data from a URL.\"\n name = \"FirecrawlExtractApi\"\n\n documentation: str = \"https://docs.firecrawl.dev/api-reference/endpoint/extract\"\n\n inputs = [\n SecretStrInput(\n name=\"api_key\",\n display_name=\"Firecrawl API Key\",\n required=True,\n password=True,\n info=\"The API key to use Firecrawl API.\",\n ),\n MultilineInput(\n name=\"urls\",\n display_name=\"URLs\",\n required=True,\n info=\"List of URLs to extract data from (separated by commas or new lines).\",\n tool_mode=True,\n ),\n MultilineInput(\n name=\"prompt\",\n display_name=\"Prompt\",\n required=True,\n info=\"Prompt to guide the extraction process.\",\n tool_mode=True,\n ),\n DataInput(\n name=\"schema\",\n display_name=\"Schema\",\n required=False,\n info=\"Schema to define the structure of the extracted data.\",\n ),\n BoolInput(\n name=\"enable_web_search\",\n display_name=\"Enable Web Search\",\n info=\"When true, the extraction will use web search to find additional data.\",\n ),\n # # Optional: Not essential for basic extraction\n # BoolInput(\n # name=\"ignore_sitemap\",\n # display_name=\"Ignore Sitemap\",\n # info=\"When true, sitemap.xml files will be ignored during website scanning.\",\n # ),\n # # Optional: Not essential for basic extraction\n # BoolInput(\n # name=\"include_subdomains\",\n # display_name=\"Include Subdomains\",\n # info=\"When true, subdomains of the provided URLs will also be scanned.\",\n # ),\n # # Optional: Not essential for basic extraction\n # BoolInput(\n # name=\"show_sources\",\n # display_name=\"Show Sources\",\n # info=\"When true, the sources used to extract the data will be included in the response.\",\n # ),\n ]\n\n outputs = [\n Output(display_name=\"JSON\", name=\"data\", method=\"extract\"),\n ]\n\n def extract(self) -> Data:\n try:\n from firecrawl import FirecrawlApp\n except ImportError as e:\n msg = \"Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`.\"\n raise ImportError(msg) from e\n\n # Validate API key\n if not self.api_key:\n msg = \"API key is required\"\n raise ValueError(msg)\n\n # Validate URLs\n if not self.urls:\n msg = \"URLs are required\"\n raise ValueError(msg)\n\n # Split and validate URLs (handle both commas and newlines)\n urls = [url.strip() for url in self.urls.replace(\"\\n\", \",\").split(\",\") if url.strip()]\n if not urls:\n msg = \"No valid URLs provided\"\n raise ValueError(msg)\n\n # Validate and process prompt\n if not self.prompt:\n msg = \"Prompt is required\"\n raise ValueError(msg)\n\n # Get the prompt text (handling both string and multiline input)\n prompt_text = self.prompt.strip()\n\n # Enhance the prompt to encourage comprehensive extraction\n enhanced_prompt = prompt_text\n if \"schema\" not in prompt_text.lower():\n enhanced_prompt = f\"{prompt_text}. Please extract all instances in a comprehensive, structured format.\"\n\n params = {\n \"prompt\": enhanced_prompt,\n \"enableWebSearch\": self.enable_web_search,\n # Optional parameters - not essential for basic extraction\n \"ignoreSitemap\": self.ignore_sitemap,\n \"includeSubdomains\": self.include_subdomains,\n \"showSources\": self.show_sources,\n \"timeout\": 300,\n }\n\n # Only add schema to params if it's provided and is a valid schema structure\n if self.schema:\n try:\n if isinstance(self.schema, dict) and \"type\" in self.schema:\n params[\"schema\"] = self.schema\n elif hasattr(self.schema, \"dict\") and \"type\" in self.schema.dict():\n params[\"schema\"] = self.schema.dict()\n else:\n # Skip invalid schema without raising an error\n pass\n except Exception as e: # noqa: BLE001\n logger.error(f\"Invalid schema: {e!s}\")\n\n try:\n app = FirecrawlApp(api_key=self.api_key)\n extract_result = app.extract(urls, params=params)\n return Data(data=extract_result)\n except Exception as e:\n msg = f\"Error during extraction: {e!s}\"\n raise ValueError(msg) from e\n" - }, - "enable_web_search": { - "_input_type": "BoolInput", - "advanced": false, - "display_name": "Enable Web Search", - "dynamic": false, - "info": "When true, the extraction will use web search to find additional data.", - "list": false, - "list_add_label": "Add More", - "name": "enable_web_search", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": false - }, - "prompt": { - "_input_type": "MultilineInput", - "advanced": false, - "ai_enabled": false, - "copy_field": false, - "display_name": "Prompt", - "dynamic": false, - "info": "Prompt to guide the extraction process.", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "multiline": true, - "name": "prompt", - "override_skip": false, - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "tool_mode": true, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "schema": { - "_input_type": "JSONInput", - "advanced": false, - "display_name": "Schema", - "dynamic": false, - "info": "Schema to define the structure of the extracted data.", - "input_types": [ - "Data", - "JSON" - ], - "list": false, - "list_add_label": "Add More", - "name": "schema", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "other", - "value": "" - }, - "urls": { - "_input_type": "MultilineInput", - "advanced": false, - "ai_enabled": false, - "copy_field": false, - "display_name": "URLs", - "dynamic": false, - "info": "List of URLs to extract data from (separated by commas or new lines).", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "multiline": true, - "name": "urls", - "override_skip": false, - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "tool_mode": true, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - } - }, - "tool_mode": false - }, - "FirecrawlMapApi": { - "base_classes": [ - "JSON" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Maps a URL and returns the results.", - "display_name": "Firecrawl Map API", - "documentation": "https://docs.firecrawl.dev/api-reference/endpoint/map", - "edited": false, - "field_order": [ - "api_key", - "urls", - "ignore_sitemap", - "sitemap_only", - "include_subdomains" - ], - "frozen": false, - "legacy": false, - "metadata": { - "code_hash": "31e75312e67e", - "dependencies": { - "dependencies": [ - { - "name": "lfx", - "version": null - }, - { - "name": "firecrawl", - "version": "1.17.0" - } - ], - "total_dependencies": 2 - }, - "module": "lfx.components.firecrawl.firecrawl_map_api.FirecrawlMapApi" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "JSON", - "group_outputs": false, - "method": "map", - "name": "data", - "selected": "JSON", - "tool_mode": true, - "types": [ - "JSON" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "api_key": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Firecrawl API Key", - "dynamic": false, - "info": "The API key to use Firecrawl API.", - "input_types": [], - "load_from_db": true, - "name": "api_key", - "override_skip": false, - "password": true, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from lfx.custom.custom_component.component import Component\nfrom lfx.io import (\n BoolInput,\n MultilineInput,\n Output,\n SecretStrInput,\n)\nfrom lfx.schema.data import Data\n\n\nclass FirecrawlMapApi(Component):\n display_name: str = \"Firecrawl Map API\"\n description: str = \"Maps a URL and returns the results.\"\n name = \"FirecrawlMapApi\"\n\n documentation: str = \"https://docs.firecrawl.dev/api-reference/endpoint/map\"\n\n inputs = [\n SecretStrInput(\n name=\"api_key\",\n display_name=\"Firecrawl API Key\",\n required=True,\n password=True,\n info=\"The API key to use Firecrawl API.\",\n ),\n MultilineInput(\n name=\"urls\",\n display_name=\"URLs\",\n required=True,\n info=\"List of URLs to create maps from (separated by commas or new lines).\",\n tool_mode=True,\n ),\n BoolInput(\n name=\"ignore_sitemap\",\n display_name=\"Ignore Sitemap\",\n info=\"When true, the sitemap.xml file will be ignored during crawling.\",\n ),\n BoolInput(\n name=\"sitemap_only\",\n display_name=\"Sitemap Only\",\n info=\"When true, only links found in the sitemap will be returned.\",\n ),\n BoolInput(\n name=\"include_subdomains\",\n display_name=\"Include Subdomains\",\n info=\"When true, subdomains of the provided URL will also be scanned.\",\n ),\n ]\n\n outputs = [\n Output(display_name=\"JSON\", name=\"data\", method=\"map\"),\n ]\n\n def map(self) -> Data:\n try:\n from firecrawl import FirecrawlApp\n except ImportError as e:\n msg = \"Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`.\"\n raise ImportError(msg) from e\n\n # Validate URLs\n if not self.urls:\n msg = \"URLs are required\"\n raise ValueError(msg)\n\n # Split and validate URLs (handle both commas and newlines)\n urls = [url.strip() for url in self.urls.replace(\"\\n\", \",\").split(\",\") if url.strip()]\n if not urls:\n msg = \"No valid URLs provided\"\n raise ValueError(msg)\n\n params = {\n \"ignoreSitemap\": self.ignore_sitemap,\n \"sitemapOnly\": self.sitemap_only,\n \"includeSubdomains\": self.include_subdomains,\n }\n\n app = FirecrawlApp(api_key=self.api_key)\n\n # Map all provided URLs and combine results\n combined_links = []\n for url in urls:\n result = app.map_url(url, params=params)\n if isinstance(result, dict) and \"links\" in result:\n combined_links.extend(result[\"links\"])\n\n map_result = {\"success\": True, \"links\": combined_links}\n\n return Data(data=map_result)\n" - }, - "ignore_sitemap": { - "_input_type": "BoolInput", - "advanced": false, - "display_name": "Ignore Sitemap", - "dynamic": false, - "info": "When true, the sitemap.xml file will be ignored during crawling.", - "list": false, - "list_add_label": "Add More", - "name": "ignore_sitemap", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": false - }, - "include_subdomains": { - "_input_type": "BoolInput", - "advanced": false, - "display_name": "Include Subdomains", - "dynamic": false, - "info": "When true, subdomains of the provided URL will also be scanned.", - "list": false, - "list_add_label": "Add More", - "name": "include_subdomains", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": false - }, - "sitemap_only": { - "_input_type": "BoolInput", - "advanced": false, - "display_name": "Sitemap Only", - "dynamic": false, - "info": "When true, only links found in the sitemap will be returned.", - "list": false, - "list_add_label": "Add More", - "name": "sitemap_only", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": false - }, - "urls": { - "_input_type": "MultilineInput", - "advanced": false, - "ai_enabled": false, - "copy_field": false, - "display_name": "URLs", - "dynamic": false, - "info": "List of URLs to create maps from (separated by commas or new lines).", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "multiline": true, - "name": "urls", - "override_skip": false, - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "tool_mode": true, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - } - }, - "tool_mode": false - }, - "FirecrawlScrapeApi": { - "base_classes": [ - "JSON" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Scrapes a URL and returns the results.", - "display_name": "Firecrawl Scrape API", - "documentation": "https://docs.firecrawl.dev/api-reference/endpoint/scrape", - "edited": false, - "field_order": [ - "api_key", - "url", - "timeout", - "scrapeOptions", - "extractorOptions" - ], - "frozen": false, - "legacy": false, - "metadata": { - "code_hash": "a56c999d7a42", - "dependencies": { - "dependencies": [ - { - "name": "lfx", - "version": null - }, - { - "name": "firecrawl", - "version": "1.17.0" - } - ], - "total_dependencies": 2 - }, - "module": "lfx.components.firecrawl.firecrawl_scrape_api.FirecrawlScrapeApi" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "JSON", - "group_outputs": false, - "method": "scrape", - "name": "data", - "selected": "JSON", - "tool_mode": true, - "types": [ - "JSON" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "api_key": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Firecrawl API Key", - "dynamic": false, - "info": "The API key to use Firecrawl API.", - "input_types": [], - "load_from_db": true, - "name": "api_key", - "override_skip": false, - "password": true, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from lfx.custom.custom_component.component import Component\nfrom lfx.io import (\n DataInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n)\nfrom lfx.schema.data import Data\n\n\nclass FirecrawlScrapeApi(Component):\n display_name: str = \"Firecrawl Scrape API\"\n description: str = \"Scrapes a URL and returns the results.\"\n name = \"FirecrawlScrapeApi\"\n\n documentation: str = \"https://docs.firecrawl.dev/api-reference/endpoint/scrape\"\n\n inputs = [\n SecretStrInput(\n name=\"api_key\",\n display_name=\"Firecrawl API Key\",\n required=True,\n password=True,\n info=\"The API key to use Firecrawl API.\",\n ),\n MultilineInput(\n name=\"url\",\n display_name=\"URL\",\n required=True,\n info=\"The URL to scrape.\",\n tool_mode=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout in milliseconds for the request.\",\n ),\n DataInput(\n name=\"scrapeOptions\",\n display_name=\"Scrape Options\",\n info=\"The page options to send with the request.\",\n ),\n DataInput(\n name=\"extractorOptions\",\n display_name=\"Extractor Options\",\n info=\"The extractor options to send with the request.\",\n ),\n ]\n\n outputs = [\n Output(display_name=\"JSON\", name=\"data\", method=\"scrape\"),\n ]\n\n def scrape(self) -> Data:\n try:\n from firecrawl import FirecrawlApp\n except ImportError as e:\n msg = \"Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`.\"\n raise ImportError(msg) from e\n\n params = self.scrapeOptions.__dict__.get(\"data\", {}) if self.scrapeOptions else {}\n extractor_options_dict = self.extractorOptions.__dict__.get(\"data\", {}) if self.extractorOptions else {}\n if extractor_options_dict:\n params[\"extract\"] = extractor_options_dict\n\n # Set default values for parameters\n params.setdefault(\"formats\", [\"markdown\"]) # Default output format\n params.setdefault(\"onlyMainContent\", True) # Default to only main content\n\n app = FirecrawlApp(api_key=self.api_key)\n results = app.scrape_url(self.url, params=params)\n return Data(data=results)\n" - }, - "extractorOptions": { - "_input_type": "JSONInput", - "advanced": false, - "display_name": "Extractor Options", - "dynamic": false, - "info": "The extractor options to send with the request.", - "input_types": [ - "Data", - "JSON" - ], - "list": false, - "list_add_label": "Add More", - "name": "extractorOptions", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "other", - "value": "" - }, - "scrapeOptions": { - "_input_type": "JSONInput", - "advanced": false, - "display_name": "Scrape Options", - "dynamic": false, - "info": "The page options to send with the request.", - "input_types": [ - "Data", - "JSON" - ], - "list": false, - "list_add_label": "Add More", - "name": "scrapeOptions", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "other", - "value": "" - }, - "timeout": { - "_input_type": "IntInput", - "advanced": false, - "display_name": "Timeout", - "dynamic": false, - "info": "Timeout in milliseconds for the request.", - "list": false, - "list_add_label": "Add More", - "name": "timeout", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "int", - "value": 0 - }, - "url": { - "_input_type": "MultilineInput", - "advanced": false, - "ai_enabled": false, - "copy_field": false, - "display_name": "URL", - "dynamic": false, - "info": "The URL to scrape.", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "multiline": true, - "name": "url", - "override_skip": false, - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "tool_mode": true, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - } - }, - "tool_mode": false - } - } - ], [ "flow_controls", { @@ -118451,9 +117642,9 @@ ] ], "metadata": { - "num_components": 354, - "num_modules": 95 + "num_components": 350, + "num_modules": 94 }, - "sha256": "87d65557fa5c4182f09f07270bbe8624d5f9b65a0322c3251f75822b6f1546ac", + "sha256": "0d7e995401cee6bc0ad9f46fd036e18bbcec09c37d5db9d8e38bd3b350a3d7f6", "version": "1.11.0" } diff --git a/src/lfx/src/lfx/components/__init__.py b/src/lfx/src/lfx/components/__init__.py index 10554a0b47..8befe38859 100644 --- a/src/lfx/src/lfx/components/__init__.py +++ b/src/lfx/src/lfx/components/__init__.py @@ -42,7 +42,6 @@ if TYPE_CHECKING: embeddings, exa, files_ingestion, - firecrawl, git, glean, google, @@ -143,7 +142,6 @@ _dynamic_imports = { "embeddings": "__module__", "exa": "__module__", "FAISS": "__module__", - "firecrawl": "__module__", "files_ingestion": "__module__", "git": "__module__", "glean": "__module__", @@ -274,7 +272,6 @@ __all__ = [ "embeddings", "exa", "files_ingestion", - "firecrawl", "git", "glean", "google", diff --git a/src/lfx/src/lfx/components/firecrawl/__init__.py b/src/lfx/src/lfx/components/firecrawl/__init__.py deleted file mode 100644 index b1e754d7ab..0000000000 --- a/src/lfx/src/lfx/components/firecrawl/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from lfx.components._importing import import_mod - -if TYPE_CHECKING: - from .firecrawl_crawl_api import FirecrawlCrawlApi - from .firecrawl_extract_api import FirecrawlExtractApi - from .firecrawl_map_api import FirecrawlMapApi - from .firecrawl_scrape_api import FirecrawlScrapeApi - -_dynamic_imports = { - "FirecrawlCrawlApi": "firecrawl_crawl_api", - "FirecrawlExtractApi": "firecrawl_extract_api", - "FirecrawlMapApi": "firecrawl_map_api", - "FirecrawlScrapeApi": "firecrawl_scrape_api", -} - -__all__ = [ - "FirecrawlCrawlApi", - "FirecrawlExtractApi", - "FirecrawlMapApi", - "FirecrawlScrapeApi", -] - - -def __getattr__(attr_name: str) -> Any: - """Lazily import firecrawl components on attribute access.""" - if attr_name not in _dynamic_imports: - msg = f"module '{__name__}' has no attribute '{attr_name}'" - raise AttributeError(msg) - try: - result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) - except (ModuleNotFoundError, ImportError, AttributeError) as e: - msg = f"Could not import '{attr_name}' from '{__name__}': {e}" - raise AttributeError(msg) from e - globals()[attr_name] = result - return result - - -def __dir__() -> list[str]: - return list(__all__) diff --git a/src/lfx/src/lfx/components/firecrawl/firecrawl_extract_api.py b/src/lfx/src/lfx/components/firecrawl/firecrawl_extract_api.py deleted file mode 100644 index 107c6cbc65..0000000000 --- a/src/lfx/src/lfx/components/firecrawl/firecrawl_extract_api.py +++ /dev/null @@ -1,136 +0,0 @@ -from lfx.custom.custom_component.component import Component -from lfx.io import BoolInput, DataInput, MultilineInput, Output, SecretStrInput -from lfx.log.logger import logger -from lfx.schema.data import Data - - -class FirecrawlExtractApi(Component): - display_name: str = "Firecrawl Extract API" - description: str = "Extracts data from a URL." - name = "FirecrawlExtractApi" - - documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/extract" - - inputs = [ - SecretStrInput( - name="api_key", - display_name="Firecrawl API Key", - required=True, - password=True, - info="The API key to use Firecrawl API.", - ), - MultilineInput( - name="urls", - display_name="URLs", - required=True, - info="List of URLs to extract data from (separated by commas or new lines).", - tool_mode=True, - ), - MultilineInput( - name="prompt", - display_name="Prompt", - required=True, - info="Prompt to guide the extraction process.", - tool_mode=True, - ), - DataInput( - name="schema", - display_name="Schema", - required=False, - info="Schema to define the structure of the extracted data.", - ), - BoolInput( - name="enable_web_search", - display_name="Enable Web Search", - info="When true, the extraction will use web search to find additional data.", - ), - # # Optional: Not essential for basic extraction - # BoolInput( - # name="ignore_sitemap", - # display_name="Ignore Sitemap", - # info="When true, sitemap.xml files will be ignored during website scanning.", - # ), - # # Optional: Not essential for basic extraction - # BoolInput( - # name="include_subdomains", - # display_name="Include Subdomains", - # info="When true, subdomains of the provided URLs will also be scanned.", - # ), - # # Optional: Not essential for basic extraction - # BoolInput( - # name="show_sources", - # display_name="Show Sources", - # info="When true, the sources used to extract the data will be included in the response.", - # ), - ] - - outputs = [ - Output(display_name="JSON", name="data", method="extract"), - ] - - def extract(self) -> Data: - try: - from firecrawl import FirecrawlApp - except ImportError as e: - msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`." - raise ImportError(msg) from e - - # Validate API key - if not self.api_key: - msg = "API key is required" - raise ValueError(msg) - - # Validate URLs - if not self.urls: - msg = "URLs are required" - raise ValueError(msg) - - # Split and validate URLs (handle both commas and newlines) - urls = [url.strip() for url in self.urls.replace("\n", ",").split(",") if url.strip()] - if not urls: - msg = "No valid URLs provided" - raise ValueError(msg) - - # Validate and process prompt - if not self.prompt: - msg = "Prompt is required" - raise ValueError(msg) - - # Get the prompt text (handling both string and multiline input) - prompt_text = self.prompt.strip() - - # Enhance the prompt to encourage comprehensive extraction - enhanced_prompt = prompt_text - if "schema" not in prompt_text.lower(): - enhanced_prompt = f"{prompt_text}. Please extract all instances in a comprehensive, structured format." - - params = { - "prompt": enhanced_prompt, - "enableWebSearch": self.enable_web_search, - # Optional parameters - not essential for basic extraction - "ignoreSitemap": self.ignore_sitemap, - "includeSubdomains": self.include_subdomains, - "showSources": self.show_sources, - "timeout": 300, - } - - # Only add schema to params if it's provided and is a valid schema structure - if self.schema: - try: - if isinstance(self.schema, dict) and "type" in self.schema: - params["schema"] = self.schema - elif hasattr(self.schema, "dict") and "type" in self.schema.dict(): - params["schema"] = self.schema.dict() - else: - # Skip invalid schema without raising an error - pass - except Exception as e: # noqa: BLE001 - logger.error(f"Invalid schema: {e!s}") - - try: - app = FirecrawlApp(api_key=self.api_key) - extract_result = app.extract(urls, params=params) - return Data(data=extract_result) - except Exception as e: - msg = f"Error during extraction: {e!s}" - raise ValueError(msg) from e diff --git a/src/lfx/src/lfx/extension/migration/migration_table.json b/src/lfx/src/lfx/extension/migration/migration_table.json index 7798cb6c8f..2ca0990632 100644 --- a/src/lfx/src/lfx/extension/migration/migration_table.json +++ b/src/lfx/src/lfx/extension/migration/migration_table.json @@ -181,6 +181,86 @@ "target": "ext:docling:ExportDoclingDocumentComponent@official", "added_in": "1.10.0" }, + { + "bare_class_name": "FirecrawlCrawlApi", + "target": "ext:firecrawl:FirecrawlCrawlApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.firecrawl_crawl_api.FirecrawlCrawlApi", + "target": "ext:firecrawl:FirecrawlCrawlApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.FirecrawlCrawlApi", + "target": "ext:firecrawl:FirecrawlCrawlApi@official", + "added_in": "1.11.0" + }, + { + "legacy_slot": "ext:firecrawl:FirecrawlCrawlApi@official-pre-a", + "target": "ext:firecrawl:FirecrawlCrawlApi@official", + "added_in": "1.11.0" + }, + { + "bare_class_name": "FirecrawlMapApi", + "target": "ext:firecrawl:FirecrawlMapApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.firecrawl_map_api.FirecrawlMapApi", + "target": "ext:firecrawl:FirecrawlMapApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.FirecrawlMapApi", + "target": "ext:firecrawl:FirecrawlMapApi@official", + "added_in": "1.11.0" + }, + { + "legacy_slot": "ext:firecrawl:FirecrawlMapApi@official-pre-a", + "target": "ext:firecrawl:FirecrawlMapApi@official", + "added_in": "1.11.0" + }, + { + "bare_class_name": "FirecrawlScrapeApi", + "target": "ext:firecrawl:FirecrawlScrapeApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.firecrawl_scrape_api.FirecrawlScrapeApi", + "target": "ext:firecrawl:FirecrawlScrapeApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.FirecrawlScrapeApi", + "target": "ext:firecrawl:FirecrawlScrapeApi@official", + "added_in": "1.11.0" + }, + { + "legacy_slot": "ext:firecrawl:FirecrawlScrapeApi@official-pre-a", + "target": "ext:firecrawl:FirecrawlScrapeApi@official", + "added_in": "1.11.0" + }, + { + "bare_class_name": "FirecrawlSearchApi", + "target": "ext:firecrawl:FirecrawlSearchApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.firecrawl_search_api.FirecrawlSearchApi", + "target": "ext:firecrawl:FirecrawlSearchApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.FirecrawlSearchApi", + "target": "ext:firecrawl:FirecrawlSearchApi@official", + "added_in": "1.11.0" + }, + { + "legacy_slot": "ext:firecrawl:FirecrawlSearchApi@official-pre-a", + "target": "ext:firecrawl:FirecrawlSearchApi@official", + "added_in": "1.11.0" + }, { "bare_class_name": "NextPlaidVectorStoreComponent", "target": "ext:nextplaid:NextPlaidVectorStoreComponent@official", diff --git a/src/lfx/tests/integration/extension/test_pilot_firecrawl_upgrade.py b/src/lfx/tests/integration/extension/test_pilot_firecrawl_upgrade.py new file mode 100644 index 0000000000..cfdcd51a3a --- /dev/null +++ b/src/lfx/tests/integration/extension/test_pilot_firecrawl_upgrade.py @@ -0,0 +1,128 @@ +"""Integration test: legacy firecrawl flows upgrade cleanly. + +Mirrors ``test_pilot_duckduckgo_upgrade.py`` for ``FirecrawlCrawlApi``. +""" + +from __future__ import annotations + +import json +from importlib import metadata as importlib_metadata +from pathlib import Path + +import pytest +from lfx.extension.migration.loader import load_migration_table + +REPO_ROOT = Path(__file__).resolve().parents[5] +TABLE_PATH = REPO_ROOT / "src" / "lfx" / "src" / "lfx" / "extension" / "migration" / "migration_table.json" +EXPECTED_TARGET = "ext:firecrawl:FirecrawlCrawlApi@official" + + +@pytest.fixture(scope="module") +def migration_table(): + table, error = load_migration_table(TABLE_PATH) + assert error is None, f"failed to load migration table: {error}" + assert table is not None + return table + + +def _saved_flow_node(node_id: str, type_value: str) -> dict: + """Build a minimal saved-flow node skeleton for testing.""" + return { + "id": node_id, + "type": "genericNode", + "data": {"id": node_id, "type": type_value, "node": {"template": {}}}, + } + + +def _saved_flow(*nodes: dict) -> dict: + return {"data": {"nodes": list(nodes), "edges": []}} + + +@pytest.mark.integration +def test_legacy_bare_name_flow_upgrades(migration_table) -> None: + """Pre-Phase-A flow with the bare class name upgrades to the canonical ID.""" + from lfx.extension.migration.rewrite import migrate_flow_payload + + flow = _saved_flow(_saved_flow_node("firecrawl-1", "FirecrawlCrawlApi")) + report = migrate_flow_payload(flow, table=migration_table) + + assert report.rewritten_count == 1 + assert flow["data"]["nodes"][0]["data"]["type"] == EXPECTED_TARGET + [record] = report.records + assert record.legacy_form_kind == "bare_class_name" + assert record.new_value == EXPECTED_TARGET + + +@pytest.mark.integration +def test_legacy_import_path_flow_upgrades(migration_table) -> None: + """Dotted import-path form upgrades to the canonical ID.""" + from lfx.extension.migration.rewrite import migrate_flow_payload + + flow = _saved_flow( + _saved_flow_node("firecrawl-2", "lfx.components.firecrawl.firecrawl_crawl_api.FirecrawlCrawlApi") + ) + report = migrate_flow_payload(flow, table=migration_table) + + assert report.rewritten_count == 1 + assert flow["data"]["nodes"][0]["data"]["type"] == EXPECTED_TARGET + assert report.records[0].legacy_form_kind == "import_path" + + +@pytest.mark.integration +def test_short_import_path_flow_upgrades(migration_table) -> None: + """Package-level import-path form (via ``__init__.py`` re-export) upgrades.""" + from lfx.extension.migration.rewrite import migrate_flow_payload + + flow = _saved_flow(_saved_flow_node("firecrawl-3", "lfx.components.firecrawl.FirecrawlCrawlApi")) + report = migrate_flow_payload(flow, table=migration_table) + + assert report.rewritten_count == 1 + assert flow["data"]["nodes"][0]["data"]["type"] == EXPECTED_TARGET + + +@pytest.mark.integration +def test_lfx_firecrawl_distribution_is_importable() -> None: + """The bundle's package is importable in the development workspace.""" + try: + from lfx_firecrawl import FirecrawlCrawlApi + except ImportError: + pytest.skip("lfx-firecrawl not installed in this test environment") + + assert FirecrawlCrawlApi.__name__ == "FirecrawlCrawlApi" + + +def _is_editable_install(dist: importlib_metadata.Distribution) -> bool: + direct_url = dist.read_text("direct_url.json") + if not direct_url: + return False + try: + payload = json.loads(direct_url) + except json.JSONDecodeError: + return False + return bool(payload.get("dir_info", {}).get("editable")) + + +@pytest.mark.integration +def test_lfx_firecrawl_ships_manifest() -> None: + """``importlib.metadata`` can find ``extension.json`` for the installed dist.""" + try: + dist = importlib_metadata.distribution("lfx-firecrawl") + except importlib_metadata.PackageNotFoundError: + pytest.skip("lfx-firecrawl not installed in this test environment") + + if _is_editable_install(dist): + import lfx_firecrawl + + package_dir = Path(lfx_firecrawl.__file__).parent + manifest_path = package_dir / "extension.json" + assert manifest_path.is_file() + else: + files = dist.files or [] + manifests = [f for f in files if f.parts and f.parts[-1] == "extension.json"] + assert manifests + manifest_path = Path(dist.locate_file(manifests[0])) + + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert manifest["id"] == "lfx-firecrawl" + assert manifest["lfx"]["compat"] == ["1"] + assert any(b["name"] == "firecrawl" for b in manifest["bundles"]) diff --git a/uv.lock b/uv.lock index e2e9abccdf..243832b032 100644 --- a/uv.lock +++ b/uv.lock @@ -33,6 +33,7 @@ members = [ "lfx-arxiv", "lfx-docling", "lfx-duckduckgo", + "lfx-firecrawl", "lfx-ibm", "lfx-nextplaid", ] @@ -4260,18 +4261,20 @@ wheels = [ [[package]] name = "firecrawl-py" -version = "1.17.0" +version = "4.28.2" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "aiohttp" }, + { name = "httpx" }, { name = "nest-asyncio" }, { name = "pydantic" }, { name = "python-dotenv" }, { name = "requests" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4d/08/61df5c8357742d8bee504db166b633ef7972b40c831e8c2f2a26feb9021c/firecrawl_py-1.17.0.tar.gz", hash = "sha256:5e2f50ec1f0e67514cdf6f0afc7df6be36eb8277fbec9e1f5a283fc01fae7875", size = 22191, upload-time = "2025-04-17T18:13:01.068Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/09/e7c667875c9a9dac09480e2f69fd46bdd9c1139555b835c3036a9858aff2/firecrawl_py-4.28.2.tar.gz", hash = "sha256:7e6181e2129b63c8d6aec5728d9b2fcf16ea82cb854372ad824b278efd258696", size = 191869, upload-time = "2026-05-28T00:03:48.822Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/53/a6/9431b62571b4ff4cd0d09f70c7df1253abf611e829f68d194e1ea69ea57e/firecrawl_py-1.17.0-py3-none-any.whl", hash = "sha256:0392822fbd906731f4c0876f91a9c3cce7624279c81948e4e3f8bc60b4e1c855", size = 21295, upload-time = "2025-04-17T18:12:59.463Z" }, + { url = "https://files.pythonhosted.org/packages/80/08/384ca104c8004ca0541aacd3470e855bd28666d6bdb2101e08446dfcb407/firecrawl_py-4.28.2-py3-none-any.whl", hash = "sha256:0689080cb01672370e5a97963e0df479f6102137aa088857eac0fa287a4269b6", size = 239166, upload-time = "2026-05-28T00:03:47.479Z" }, ] [[package]] @@ -7421,6 +7424,7 @@ dependencies = [ { name = "lfx-arxiv" }, { name = "lfx-docling" }, { name = "lfx-duckduckgo" }, + { name = "lfx-firecrawl" }, { name = "lfx-ibm" }, { name = "lfx-nextplaid" }, ] @@ -7515,6 +7519,7 @@ requires-dist = [ { name = "lfx-docling", extras = ["image-description"], marker = "extra == 'docling-image-description'", editable = "src/bundles/docling" }, { name = "lfx-docling", extras = ["local"], marker = "extra == 'docling'", editable = "src/bundles/docling" }, { name = "lfx-duckduckgo", editable = "src/bundles/duckduckgo" }, + { name = "lfx-firecrawl", editable = "src/bundles/firecrawl" }, { name = "lfx-ibm", editable = "src/bundles/ibm" }, { name = "lfx-nextplaid", editable = "src/bundles/nextplaid" }, { name = "nv-ingest-api", marker = "python_full_version >= '3.12' and extra == 'nv-ingest'", specifier = ">=26.1.0,<27.0.0" }, @@ -7596,7 +7601,6 @@ dependencies = [ { name = "fastapi" }, { name = "fastapi-pagination" }, { name = "filelock" }, - { name = "firecrawl-py" }, { name = "grandalf" }, { name = "greenlet" }, { name = "gunicorn" }, @@ -8347,7 +8351,6 @@ requires-dist = [ { name = "fastmcp", marker = "extra == 'fastmcp'", specifier = ">=3.2.0" }, { name = "fastparquet", marker = "extra == 'fastparquet'", specifier = ">=2024.11.0,<2025.0.0" }, { name = "filelock", specifier = ">=3.20.1,<4.0.0" }, - { name = "firecrawl-py", specifier = ">=1.0.16,<2.0.0" }, { name = "gassist", marker = "sys_platform == 'win32' and extra == 'gassist'", specifier = ">=0.0.1" }, { name = "gitpython", marker = "extra == 'gitpython'", specifier = ">=3.1.50" }, { name = "google-api-python-client", marker = "extra == 'google'", specifier = "~=2.161" }, @@ -9155,6 +9158,21 @@ requires-dist = [ { name = "lfx", editable = "src/lfx" }, ] +[[package]] +name = "lfx-firecrawl" +version = "0.1.0" +source = { editable = "src/bundles/firecrawl" } +dependencies = [ + { name = "firecrawl-py" }, + { name = "lfx" }, +] + +[package.metadata] +requires-dist = [ + { name = "firecrawl-py", specifier = ">=4.0.0,<5.0.0" }, + { name = "lfx", editable = "src/lfx" }, +] + [[package]] name = "lfx-ibm" version = "0.1.1"