mirror of
https://github.com/langflow-ai/langflow.git
synced 2026-07-23 16:10:27 +08:00
feat: upgrade Firecrawl to v2 SDK and extract into the lfx-firecrawl extension bundle (#13495)
* Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * Upgrade Firecrawl components to firecrawl-py v2 + add Search component Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: add Firecrawl Search API to the bundle page Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * [autofix.ci] apply automated fixes * fix(firecrawl): D205 docstring + defensive .get('data') in crawl Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(firecrawl): D205 docstring in scrape Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * perf(firecrawl): use list.extend in map (PERF401) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * test: add unit tests for Firecrawl v2 components Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * [autofix.ci] apply automated fixes * test: align Firecrawl component tests with lfx _attributes pattern Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * [autofix.ci] apply automated fixes * refactor(firecrawl): remove deprecated extract endpoint component Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * refactor(firecrawl): remove deprecated extract endpoint component Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * test: drop extract component test (extract endpoint removed) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: remove Firecrawl Extract API section (deprecated) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * [autofix.ci] apply automated fixes * fix(firecrawl): map Search component to Firecrawl icon, drop Extract The frontend icon map in styleUtils.ts still referenced the removed FirecrawlExtractApi and lacked the new FirecrawlSearchApi, so the Search node would not render the Firecrawl logo. * chore: regenerate component index for Firecrawl v2 (Search added, Extract removed) * refactor(firecrawl): extract components into the lfx-firecrawl extension bundle Port the firecrawl provider out of lfx.components into a standalone Extension Bundle at src/bundles/firecrawl (distribution: lfx-firecrawl), following src/bundles/PORTING.md: - Move the v2-SDK components (Scrape, Crawl, Map, Search) to src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/ and remove the in-tree provider + its lfx.components registration. - Own the firecrawl-py>=4,<5 pin in the bundle; drop it from langflow-base. - Wire the workspace (root pyproject deps/sources/members) and uv.lock. - Append migration-table entries (bare name, both import paths, pre-a slot) for the four classes; FirecrawlExtractApi gets no entries since the v2 SDK removed the extract endpoint and the component was dropped. - Regenerate the component index (firecrawl category removed) and locales/en.json (54 firecrawl keys move out of core). - Bundle-local unit tests + test_pilot_firecrawl_upgrade integration test; update Dockerfile bundle enumeration comments. * fix(lfx): repair migration table entry fused during release-1.11.0 merge The release-1.11.0 back-merge collided the FirecrawlSearchApi legacy_slot entry with the NextPlaidVectorStoreComponent bare_class_name entry, mashing both into a single object with duplicate 'target' keys, populating two of {bare_class_name, import_path, legacy_slot}. That fails the MigrationTable validator (entries.51) and broke the lfx loader tests. Split it into the two intended entries so each populates exactly one key-field. Restores the FirecrawlSearchApi and NextPlaidVectorStoreComponent quads (60 entries total, +16 vs base). Append-only and bare-name guards pass. --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Eric Hare <ericrhare@gmail.com>
This commit is contained in:
Rakshith Ramprakash
committed by
GitHub
GitHub
parent
97c5598ee3
commit
e5d42e54fc
@ -2763,7 +2763,7 @@
|
||||
"filename": "src/backend/base/langflow/locales/en.json",
|
||||
"hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
|
||||
"is_verified": false,
|
||||
"line_number": 4606
|
||||
"line_number": 4552
|
||||
}
|
||||
],
|
||||
"src/backend/base/langflow/locales/es.json": [
|
||||
@ -9287,5 +9287,5 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"generated_at": "2026-06-10T08:36:23Z"
|
||||
"generated_at": "2026-06-10T18:13:35Z"
|
||||
}
|
||||
|
||||
@ -39,7 +39,7 @@ ENV VIRTUAL_ENV="/app/.venv"
|
||||
|
||||
# Install langflow-base with all extras except dev (which includes Playwright).
|
||||
# This image ships the langflow-base core only. Extension bundles
|
||||
# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling) are intentionally NOT
|
||||
# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling, lfx-firecrawl) are intentionally NOT
|
||||
# installed here -- they belong to the full ``langflow`` distribution, not
|
||||
# the lean core. Use the ``langflow`` image, or ``pip install`` the bundle
|
||||
# alongside this image, to add those components.
|
||||
|
||||
@ -75,7 +75,7 @@ RUN npm install \
|
||||
|
||||
WORKDIR /app/src/backend/base
|
||||
# langflow-base ships the core framework only. The extension bundles
|
||||
# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling) are intentionally NOT
|
||||
# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling, lfx-firecrawl) are intentionally NOT
|
||||
# installed in this image: they are dependencies of the full ``langflow``
|
||||
# distribution, not of the lean ``langflow-base`` core, and we keep that
|
||||
# boundary at the image layer too. Consumers who want those components
|
||||
|
||||
@ -60,18 +60,17 @@ It outputs the API response as [`Data`](/data-types#data).
|
||||
| Sitemap Only (`sitemap_only`) | Boolean | Input parameter. When true, only links found in the sitemap are returned. |
|
||||
| Include Subdomains (`include_subdomains`) | Boolean | Input parameter. When true, subdomains of the provided URL are also scanned. |
|
||||
|
||||
## Firecrawl Extract API
|
||||
## Firecrawl Search API
|
||||
|
||||
This component extracts structured data from one or more URLs using a prompt or schema.
|
||||
This component searches the web and returns the results.
|
||||
|
||||
It outputs the API response as [`Data`](/data-types#data).
|
||||
|
||||
### Firecrawl Extract API parameters
|
||||
### Firecrawl Search API parameters
|
||||
|
||||
| Name | Type | Description |
|
||||
|------|------|-------------|
|
||||
| Firecrawl API Key (`api_key`) | SecretString | Input parameter. The API key to use Firecrawl API. |
|
||||
| URLs (`urls`) | String | Input parameter. List of URLs to extract data from (separated by commas or new lines). |
|
||||
| Prompt (`prompt`) | String | Input parameter. Prompt to guide the extraction process. |
|
||||
| Schema (`schema`) | Data | Input parameter. Schema to define the structure of the extracted data. |
|
||||
| Enable Web Search (`enable_web_search`) | Boolean | Input parameter. When true, the extraction uses web search to find additional data. |
|
||||
| Query (`query`) | String | Input parameter. The search query to run. |
|
||||
| Limit (`limit`) | Integer | Input parameter. The maximum number of results to return. |
|
||||
| Location (`location`) | String | Input parameter. Location to bias the search results (for example, a country or region). |
|
||||
|
||||
@ -23,6 +23,7 @@ dependencies = [
|
||||
"lfx-arxiv>=0.1.0",
|
||||
"lfx-ibm>=0.1.0",
|
||||
"lfx-docling>=0.1.0",
|
||||
"lfx-firecrawl>=0.1.0",
|
||||
"lfx-nextplaid>=0.1.0",
|
||||
# langflow-extensions:bundle-deps-end
|
||||
]
|
||||
@ -85,6 +86,7 @@ lfx-duckduckgo = { workspace = true }
|
||||
lfx-arxiv = { workspace = true }
|
||||
lfx-ibm = { workspace = true }
|
||||
lfx-docling = { workspace = true }
|
||||
lfx-firecrawl = { workspace = true }
|
||||
lfx-nextplaid = { workspace = true }
|
||||
# langflow-extensions:bundle-sources-end
|
||||
torch = { index = "pytorch-cpu" }
|
||||
@ -102,6 +104,7 @@ members = [
|
||||
"src/bundles/arxiv",
|
||||
"src/bundles/ibm",
|
||||
"src/bundles/docling",
|
||||
"src/bundles/firecrawl",
|
||||
"src/bundles/nextplaid",
|
||||
# langflow-extensions:bundle-members-end
|
||||
]
|
||||
|
||||
@ -3567,60 +3567,6 @@
|
||||
"components.filterdatavalues.inputs.operator.display_name.d6b063d9": "Comparison Operator",
|
||||
"components.filterdatavalues.inputs.operator.info.81652313": "The operator to apply for comparing the values.",
|
||||
"components.filterdatavalues.outputs.filtered_data.display_name.8acf8a3c": "Filtered Data",
|
||||
"components.firecrawlcrawlapi.description.e9b9ca89": "Crawls a URL and returns the results.",
|
||||
"components.firecrawlcrawlapi.display_name.1c8af0bd": "Firecrawl Crawl API",
|
||||
"components.firecrawlcrawlapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key",
|
||||
"components.firecrawlcrawlapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.",
|
||||
"components.firecrawlcrawlapi.inputs.crawlerOptions.display_name.77b95f41": "Crawler Options",
|
||||
"components.firecrawlcrawlapi.inputs.crawlerOptions.info.b3898e10": "The crawler options to send with the request.",
|
||||
"components.firecrawlcrawlapi.inputs.idempotency_key.display_name.0409cf23": "Idempotency Key",
|
||||
"components.firecrawlcrawlapi.inputs.idempotency_key.info.7fa2bc03": "Optional idempotency key to ensure unique requests.",
|
||||
"components.firecrawlcrawlapi.inputs.scrapeOptions.display_name.f6227db1": "Scrape Options",
|
||||
"components.firecrawlcrawlapi.inputs.scrapeOptions.info.034e1d4f": "The page options to send with the request.",
|
||||
"components.firecrawlcrawlapi.inputs.timeout.display_name.70594d93": "Timeout",
|
||||
"components.firecrawlcrawlapi.inputs.timeout.info.11f2ddb2": "Timeout in milliseconds for the request.",
|
||||
"components.firecrawlcrawlapi.inputs.url.display_name.e7a241de": "URL",
|
||||
"components.firecrawlcrawlapi.inputs.url.info.b260224a": "The URL to scrape.",
|
||||
"components.firecrawlcrawlapi.outputs.data.display_name.db1a21a0": "JSON",
|
||||
"components.firecrawlextractapi.description.eed57be9": "Extracts data from a URL.",
|
||||
"components.firecrawlextractapi.display_name.ae3a11a0": "Firecrawl Extract API",
|
||||
"components.firecrawlextractapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key",
|
||||
"components.firecrawlextractapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.",
|
||||
"components.firecrawlextractapi.inputs.enable_web_search.display_name.a53c041a": "Enable Web Search",
|
||||
"components.firecrawlextractapi.inputs.enable_web_search.info.07671323": "When true, the extraction will use web search to find additional data.",
|
||||
"components.firecrawlextractapi.inputs.prompt.display_name.5c391238": "Prompt",
|
||||
"components.firecrawlextractapi.inputs.prompt.info.30c68d4d": "Prompt to guide the extraction process.",
|
||||
"components.firecrawlextractapi.inputs.schema.display_name.07b091a3": "Schema",
|
||||
"components.firecrawlextractapi.inputs.schema.info.fc0ac9cc": "Schema to define the structure of the extracted data.",
|
||||
"components.firecrawlextractapi.inputs.urls.display_name.1240054e": "URLs",
|
||||
"components.firecrawlextractapi.inputs.urls.info.a35e321b": "List of URLs to extract data from (separated by commas or new lines).",
|
||||
"components.firecrawlextractapi.outputs.data.display_name.db1a21a0": "JSON",
|
||||
"components.firecrawlmapapi.description.acbcf5bc": "Maps a URL and returns the results.",
|
||||
"components.firecrawlmapapi.display_name.30b2bee7": "Firecrawl Map API",
|
||||
"components.firecrawlmapapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key",
|
||||
"components.firecrawlmapapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.",
|
||||
"components.firecrawlmapapi.inputs.ignore_sitemap.display_name.6d48c522": "Ignore Sitemap",
|
||||
"components.firecrawlmapapi.inputs.ignore_sitemap.info.02a79d48": "When true, the sitemap.xml file will be ignored during crawling.",
|
||||
"components.firecrawlmapapi.inputs.include_subdomains.display_name.5cf2dc9e": "Include Subdomains",
|
||||
"components.firecrawlmapapi.inputs.include_subdomains.info.073e9171": "When true, subdomains of the provided URL will also be scanned.",
|
||||
"components.firecrawlmapapi.inputs.sitemap_only.display_name.021a3604": "Sitemap Only",
|
||||
"components.firecrawlmapapi.inputs.sitemap_only.info.7ca2f13c": "When true, only links found in the sitemap will be returned.",
|
||||
"components.firecrawlmapapi.inputs.urls.display_name.1240054e": "URLs",
|
||||
"components.firecrawlmapapi.inputs.urls.info.71719292": "List of URLs to create maps from (separated by commas or new lines).",
|
||||
"components.firecrawlmapapi.outputs.data.display_name.db1a21a0": "JSON",
|
||||
"components.firecrawlscrapeapi.description.017ffc83": "Scrapes a URL and returns the results.",
|
||||
"components.firecrawlscrapeapi.display_name.39a1c0de": "Firecrawl Scrape API",
|
||||
"components.firecrawlscrapeapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key",
|
||||
"components.firecrawlscrapeapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.",
|
||||
"components.firecrawlscrapeapi.inputs.extractorOptions.display_name.80a369a3": "Extractor Options",
|
||||
"components.firecrawlscrapeapi.inputs.extractorOptions.info.9fd7d66f": "The extractor options to send with the request.",
|
||||
"components.firecrawlscrapeapi.inputs.scrapeOptions.display_name.f6227db1": "Scrape Options",
|
||||
"components.firecrawlscrapeapi.inputs.scrapeOptions.info.034e1d4f": "The page options to send with the request.",
|
||||
"components.firecrawlscrapeapi.inputs.timeout.display_name.70594d93": "Timeout",
|
||||
"components.firecrawlscrapeapi.inputs.timeout.info.11f2ddb2": "Timeout in milliseconds for the request.",
|
||||
"components.firecrawlscrapeapi.inputs.url.display_name.e7a241de": "URL",
|
||||
"components.firecrawlscrapeapi.inputs.url.info.b260224a": "The URL to scrape.",
|
||||
"components.firecrawlscrapeapi.outputs.data.display_name.db1a21a0": "JSON",
|
||||
"components.flowtool.description.d3907775": "Construct a Tool from a function that runs the loaded Flow.",
|
||||
"components.flowtool.display_name.21209f51": "Flow as Tool",
|
||||
"components.flowtool.inputs.flow_name.display_name.e5fcec99": "Flow Name",
|
||||
|
||||
@ -64,7 +64,6 @@ dependencies = [
|
||||
"uncurl>=0.0.11,<1.0.0",
|
||||
"sentry-sdk[fastapi,loguru]>=2.5.1,<3.0.0",
|
||||
"chardet>=7.3.0",
|
||||
"firecrawl-py>=1.0.16,<2.0.0",
|
||||
"opentelemetry-api>=1.30.0,<2.0.0",
|
||||
"opentelemetry-sdk>=1.30.0,<2.0.0",
|
||||
"opentelemetry-exporter-prometheus>=0.50b0,<1.0.0",
|
||||
|
||||
29
src/bundles/firecrawl/README.md
Normal file
29
src/bundles/firecrawl/README.md
Normal file
@ -0,0 +1,29 @@
|
||||
# lfx-firecrawl
|
||||
|
||||
Firecrawl components (Scrape, Crawl, Map, and Search APIs) as a standalone
|
||||
Langflow Extension Bundle, built against the firecrawl-py v2 SDK.
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
pip install lfx-firecrawl
|
||||
```
|
||||
|
||||
The bundle is registered automatically via the `langflow.extensions`
|
||||
entry-point. After install, restart your Langflow server; the bundle's
|
||||
components will appear in the palette under the `firecrawl` group.
|
||||
|
||||
## Develop
|
||||
|
||||
```bash
|
||||
cd src/bundles/firecrawl
|
||||
pip install -e .
|
||||
lfx extension validate src/lfx_firecrawl
|
||||
```
|
||||
|
||||
## Migration
|
||||
|
||||
Saved flows referencing the legacy class name(s) or the old import paths
|
||||
under `lfx.components.firecrawl.*` are rewritten to the new namespaced
|
||||
IDs by the migration table in
|
||||
`src/lfx/src/lfx/extension/migration/migration_table.json`.
|
||||
55
src/bundles/firecrawl/pyproject.toml
Normal file
55
src/bundles/firecrawl/pyproject.toml
Normal file
@ -0,0 +1,55 @@
|
||||
[project]
|
||||
name = "lfx-firecrawl"
|
||||
version = "0.1.0"
|
||||
description = "Firecrawl components (Scrape, Crawl, Map, and Search APIs) as a standalone Langflow Extension Bundle."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10,<3.15"
|
||||
license = { text = "MIT" }
|
||||
authors = [
|
||||
{ name = "Langflow", email = "contact@langflow.org" },
|
||||
]
|
||||
keywords = ["langflow", "lfx", "extension", "bundle", "firecrawl"]
|
||||
|
||||
# Runtime deps: lfx (the BUNDLE_API surface) plus any third-party imports
|
||||
# the bundle's components rely on. REVIEW THIS LIST -- the script ports
|
||||
# only ``lfx``; add any other deps the moved component(s) import.
|
||||
# lfx is floored at the current major.minor line and capped below the next
|
||||
# lfx major (e.g. "lfx>=1.10.0,<2.0.0"); the floor is read from
|
||||
# src/lfx/pyproject.toml at port time and re-synced on ``make patch`` via
|
||||
# scripts/ci/sync_bundle_lfx_pin.py. Fine-grained BUNDLE_API compatibility
|
||||
# is enforced via extension.json's lfx.compat list against BUNDLE_API_VERSION.
|
||||
dependencies = [
|
||||
"lfx>=1.11.0.dev0,<2.0.0",
|
||||
"firecrawl-py>=4.0.0,<5.0.0",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/langflow-ai/langflow"
|
||||
Documentation = "https://docs.langflow.org/extensions"
|
||||
Repository = "https://github.com/langflow-ai/langflow"
|
||||
|
||||
# Manifest-shipping distributions are discovered via the
|
||||
# ``langflow.extensions`` entry-point. Editable installs whose
|
||||
# ``dist.files`` only surfaces dist-info entries fall back to this
|
||||
# entry-point to find the manifest.
|
||||
[project.entry-points."langflow.extensions"]
|
||||
lfx-firecrawl = "lfx_firecrawl"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
# extension.json + components live inside the lfx_firecrawl package so
|
||||
# ``importlib.metadata.files(dist)`` finds them and the loader resolves
|
||||
# bundles[].path relative to the manifest's directory.
|
||||
packages = ["src/lfx_firecrawl"]
|
||||
include = ["src/lfx_firecrawl/extension.json", "src/lfx_firecrawl/components/**/*.py"]
|
||||
|
||||
[tool.hatch.build.targets.sdist]
|
||||
include = [
|
||||
"src/lfx_firecrawl",
|
||||
"extension.json",
|
||||
"README.md",
|
||||
"pyproject.toml",
|
||||
]
|
||||
14
src/bundles/firecrawl/src/lfx_firecrawl/__init__.py
Normal file
14
src/bundles/firecrawl/src/lfx_firecrawl/__init__.py
Normal file
@ -0,0 +1,14 @@
|
||||
"""lfx-firecrawl: Firecrawl bundle.
|
||||
|
||||
Distribution unit ``lfx-firecrawl``. At runtime Langflow's loader
|
||||
discovers ``extension.json`` shipped alongside this ``__init__.py`` and
|
||||
registers the bundle's components under the namespaced IDs
|
||||
``ext:firecrawl:<Class>@official``.
|
||||
"""
|
||||
|
||||
from lfx_firecrawl.components.firecrawl.firecrawl_crawl_api import FirecrawlCrawlApi
|
||||
from lfx_firecrawl.components.firecrawl.firecrawl_map_api import FirecrawlMapApi
|
||||
from lfx_firecrawl.components.firecrawl.firecrawl_scrape_api import FirecrawlScrapeApi
|
||||
from lfx_firecrawl.components.firecrawl.firecrawl_search_api import FirecrawlSearchApi
|
||||
|
||||
__all__ = ["FirecrawlCrawlApi", "FirecrawlMapApi", "FirecrawlScrapeApi", "FirecrawlSearchApi"]
|
||||
@ -0,0 +1,13 @@
|
||||
"""Component re-exports for the ``firecrawl`` bundle.
|
||||
|
||||
Saved-flow migration entries that target ``lfx.components.firecrawl.<Class>``
|
||||
resolve through this package, so the moved Component class(es) must be
|
||||
importable from here by name.
|
||||
"""
|
||||
|
||||
from .firecrawl_crawl_api import FirecrawlCrawlApi
|
||||
from .firecrawl_map_api import FirecrawlMapApi
|
||||
from .firecrawl_scrape_api import FirecrawlScrapeApi
|
||||
from .firecrawl_search_api import FirecrawlSearchApi
|
||||
|
||||
__all__ = ["FirecrawlCrawlApi", "FirecrawlMapApi", "FirecrawlScrapeApi", "FirecrawlSearchApi"]
|
||||
@ -1,9 +1,21 @@
|
||||
import uuid
|
||||
import re
|
||||
|
||||
from lfx.custom.custom_component.component import Component
|
||||
from lfx.io import DataInput, IntInput, MultilineInput, Output, SecretStrInput, StrInput
|
||||
from lfx.schema.data import Data
|
||||
|
||||
_CAMEL_TO_SNAKE_RE = re.compile(r"(?<!^)(?=[A-Z])")
|
||||
|
||||
|
||||
def _to_snake_case_kwargs(params: dict) -> dict:
|
||||
"""Convert camelCase option keys to snake_case keyword arguments.
|
||||
|
||||
The firecrawl-py v1 convention uses camelCase, while the v2 SDK expects
|
||||
snake_case keyword arguments. Keys that are already snake_case are passed
|
||||
through unchanged.
|
||||
"""
|
||||
return {_CAMEL_TO_SNAKE_RE.sub("_", key).lower(): value for key, value in params.items()}
|
||||
|
||||
|
||||
class FirecrawlCrawlApi(Component):
|
||||
display_name: str = "Firecrawl Crawl API"
|
||||
@ -56,33 +68,42 @@ class FirecrawlCrawlApi(Component):
|
||||
|
||||
def crawl(self) -> Data:
|
||||
try:
|
||||
from firecrawl import FirecrawlApp
|
||||
from firecrawl import Firecrawl
|
||||
from firecrawl.v2.types import ScrapeOptions
|
||||
except ImportError as e:
|
||||
msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
|
||||
raise ImportError(msg) from e
|
||||
|
||||
params = self.crawlerOptions.__dict__["data"] if self.crawlerOptions else {}
|
||||
scrape_options_dict = self.scrapeOptions.__dict__["data"] if self.scrapeOptions else {}
|
||||
if scrape_options_dict:
|
||||
params["scrapeOptions"] = scrape_options_dict
|
||||
params = dict(self.crawlerOptions.__dict__.get("data", {})) if self.crawlerOptions else {}
|
||||
scrape_options_dict = dict(self.scrapeOptions.__dict__.get("data", {})) if self.scrapeOptions else {}
|
||||
|
||||
# Set default values for new parameters in v1
|
||||
# Set default values for crawl parameters.
|
||||
# Note: firecrawl-py v2 renamed several options. "maxDepth" -> "max_discovery_depth"
|
||||
# and "allowBackwardLinks" -> "crawl_entire_domain".
|
||||
params.setdefault("maxDepth", 2)
|
||||
params.setdefault("limit", 10000)
|
||||
params.setdefault("allowExternalLinks", False)
|
||||
params.setdefault("allowBackwardLinks", False)
|
||||
params.setdefault("ignoreSitemap", False)
|
||||
params.setdefault("ignoreQueryParameters", False)
|
||||
|
||||
# Ensure onlyMainContent is explicitly set if not provided
|
||||
if "scrapeOptions" in params:
|
||||
params["scrapeOptions"].setdefault("onlyMainContent", True)
|
||||
else:
|
||||
params["scrapeOptions"] = {"onlyMainContent": True}
|
||||
# Ensure onlyMainContent is explicitly set if not provided.
|
||||
scrape_options_dict.setdefault("onlyMainContent", True)
|
||||
|
||||
if not self.idempotency_key:
|
||||
self.idempotency_key = str(uuid.uuid4())
|
||||
# Translate legacy v1 camelCase option names to the firecrawl-py v2 keyword args.
|
||||
kwargs = _to_snake_case_kwargs(params)
|
||||
if "max_depth" in kwargs:
|
||||
kwargs["max_discovery_depth"] = kwargs.pop("max_depth")
|
||||
if "allow_backward_links" in kwargs:
|
||||
kwargs["crawl_entire_domain"] = kwargs.pop("allow_backward_links")
|
||||
# v2 removed "ignore_sitemap"; it is now the "sitemap" mode enum.
|
||||
if kwargs.pop("ignore_sitemap", False):
|
||||
kwargs["sitemap"] = "skip"
|
||||
|
||||
app = FirecrawlApp(api_key=self.api_key)
|
||||
crawl_result = app.crawl_url(self.url, params=params, idempotency_key=self.idempotency_key)
|
||||
return Data(data={"results": crawl_result})
|
||||
# Build the typed ScrapeOptions object expected by v2 from the (snake_cased) dict.
|
||||
scrape_kwargs = _to_snake_case_kwargs(scrape_options_dict)
|
||||
kwargs["scrape_options"] = ScrapeOptions(**scrape_kwargs)
|
||||
|
||||
app = Firecrawl(api_key=self.api_key)
|
||||
# v2 polls to completion and returns a typed CrawlJob object.
|
||||
crawl_job = app.crawl(self.url, **kwargs)
|
||||
return Data(data={"results": crawl_job.model_dump()})
|
||||
@ -53,7 +53,7 @@ class FirecrawlMapApi(Component):
|
||||
|
||||
def map(self) -> Data:
|
||||
try:
|
||||
from firecrawl import FirecrawlApp
|
||||
from firecrawl import Firecrawl
|
||||
except ImportError as e:
|
||||
msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
|
||||
raise ImportError(msg) from e
|
||||
@ -69,20 +69,25 @@ class FirecrawlMapApi(Component):
|
||||
msg = "No valid URLs provided"
|
||||
raise ValueError(msg)
|
||||
|
||||
params = {
|
||||
"ignoreSitemap": self.ignore_sitemap,
|
||||
"sitemapOnly": self.sitemap_only,
|
||||
"includeSubdomains": self.include_subdomains,
|
||||
}
|
||||
# firecrawl-py v2 replaced the separate "ignoreSitemap"/"sitemapOnly" flags with a
|
||||
# single "sitemap" mode: "only" (sitemap only), "skip" (ignore sitemap), or the
|
||||
# default mixed behavior.
|
||||
kwargs: dict = {"include_subdomains": self.include_subdomains}
|
||||
if self.sitemap_only:
|
||||
kwargs["sitemap"] = "only"
|
||||
elif self.ignore_sitemap:
|
||||
kwargs["sitemap"] = "skip"
|
||||
|
||||
app = FirecrawlApp(api_key=self.api_key)
|
||||
app = Firecrawl(api_key=self.api_key)
|
||||
|
||||
# Map all provided URLs and combine results
|
||||
# Map all provided URLs and combine results.
|
||||
combined_links = []
|
||||
for url in urls:
|
||||
result = app.map_url(url, params=params)
|
||||
if isinstance(result, dict) and "links" in result:
|
||||
combined_links.extend(result["links"])
|
||||
result = app.map(url, **kwargs)
|
||||
# v2 returns a typed MapData object whose .links is a list of typed
|
||||
# SearchResult objects; serialize each to a dict for downstream consumers.
|
||||
links = getattr(result, "links", None) or []
|
||||
combined_links.extend(link.model_dump() if hasattr(link, "model_dump") else link for link in links)
|
||||
|
||||
map_result = {"success": True, "links": combined_links}
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
import re
|
||||
|
||||
from lfx.custom.custom_component.component import Component
|
||||
from lfx.io import (
|
||||
DataInput,
|
||||
@ -8,6 +10,18 @@ from lfx.io import (
|
||||
)
|
||||
from lfx.schema.data import Data
|
||||
|
||||
_CAMEL_TO_SNAKE_RE = re.compile(r"(?<!^)(?=[A-Z])")
|
||||
|
||||
|
||||
def _to_snake_case_kwargs(params: dict) -> dict:
|
||||
"""Convert camelCase option keys to snake_case keyword arguments.
|
||||
|
||||
The firecrawl-py v1 convention uses camelCase, while the v2 SDK expects
|
||||
snake_case keyword arguments. Keys that are already snake_case are passed
|
||||
through unchanged.
|
||||
"""
|
||||
return {_CAMEL_TO_SNAKE_RE.sub("_", key).lower(): value for key, value in params.items()}
|
||||
|
||||
|
||||
class FirecrawlScrapeApi(Component):
|
||||
display_name: str = "Firecrawl Scrape API"
|
||||
@ -54,20 +68,30 @@ class FirecrawlScrapeApi(Component):
|
||||
|
||||
def scrape(self) -> Data:
|
||||
try:
|
||||
from firecrawl import FirecrawlApp
|
||||
from firecrawl import Firecrawl
|
||||
except ImportError as e:
|
||||
msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
|
||||
raise ImportError(msg) from e
|
||||
|
||||
params = self.scrapeOptions.__dict__.get("data", {}) if self.scrapeOptions else {}
|
||||
extractor_options_dict = self.extractorOptions.__dict__.get("data", {}) if self.extractorOptions else {}
|
||||
if extractor_options_dict:
|
||||
params["extract"] = extractor_options_dict
|
||||
params = dict(self.scrapeOptions.__dict__.get("data", {})) if self.scrapeOptions else {}
|
||||
|
||||
# Set default values for parameters
|
||||
params.setdefault("formats", ["markdown"]) # Default output format
|
||||
params.setdefault("onlyMainContent", True) # Default to only main content
|
||||
if self.timeout:
|
||||
params.setdefault("timeout", self.timeout)
|
||||
|
||||
app = FirecrawlApp(api_key=self.api_key)
|
||||
results = app.scrape_url(self.url, params=params)
|
||||
return Data(data=results)
|
||||
kwargs = _to_snake_case_kwargs(params)
|
||||
|
||||
# In firecrawl-py v2, structured extraction is requested via a "json" format entry
|
||||
# on the scrape call rather than a separate extractor option.
|
||||
extractor_options_dict = self.extractorOptions.__dict__.get("data", {}) if self.extractorOptions else {}
|
||||
if extractor_options_dict:
|
||||
formats = list(kwargs.get("formats", []))
|
||||
formats.append({"type": "json", **_to_snake_case_kwargs(extractor_options_dict)})
|
||||
kwargs["formats"] = formats
|
||||
|
||||
app = Firecrawl(api_key=self.api_key)
|
||||
document = app.scrape(self.url, **kwargs)
|
||||
# v2 returns a typed Document object; serialize to a dict for downstream consumers.
|
||||
return Data(data=document.model_dump())
|
||||
@ -0,0 +1,76 @@
|
||||
from lfx.custom.custom_component.component import Component
|
||||
from lfx.io import (
|
||||
IntInput,
|
||||
MultilineInput,
|
||||
Output,
|
||||
SecretStrInput,
|
||||
StrInput,
|
||||
)
|
||||
from lfx.schema.data import Data
|
||||
|
||||
|
||||
class FirecrawlSearchApi(Component):
|
||||
display_name: str = "Firecrawl Search API"
|
||||
description: str = "Searches the web and returns the results."
|
||||
name = "FirecrawlSearchApi"
|
||||
|
||||
documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/search"
|
||||
|
||||
inputs = [
|
||||
SecretStrInput(
|
||||
name="api_key",
|
||||
display_name="Firecrawl API Key",
|
||||
required=True,
|
||||
password=True,
|
||||
info="The API key to use Firecrawl API.",
|
||||
),
|
||||
MultilineInput(
|
||||
name="query",
|
||||
display_name="Query",
|
||||
required=True,
|
||||
info="The search query to run.",
|
||||
tool_mode=True,
|
||||
),
|
||||
IntInput(
|
||||
name="limit",
|
||||
display_name="Limit",
|
||||
info="Maximum number of results to return.",
|
||||
value=5,
|
||||
),
|
||||
StrInput(
|
||||
name="location",
|
||||
display_name="Location",
|
||||
info="Location to bias the search results (e.g. a country or region).",
|
||||
advanced=True,
|
||||
),
|
||||
]
|
||||
|
||||
outputs = [
|
||||
Output(display_name="JSON", name="data", method="search"),
|
||||
]
|
||||
|
||||
def search(self) -> Data:
|
||||
try:
|
||||
from firecrawl import Firecrawl
|
||||
except ImportError as e:
|
||||
msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
|
||||
raise ImportError(msg) from e
|
||||
|
||||
if not self.query:
|
||||
msg = "Query is required"
|
||||
raise ValueError(msg)
|
||||
|
||||
kwargs: dict = {}
|
||||
if self.limit:
|
||||
kwargs["limit"] = self.limit
|
||||
if self.location:
|
||||
kwargs["location"] = self.location
|
||||
|
||||
app = Firecrawl(api_key=self.api_key)
|
||||
result = app.search(self.query, **kwargs)
|
||||
|
||||
# v2 returns a typed SearchData object (results grouped by source);
|
||||
# serialize to a dict for downstream consumers.
|
||||
search_result = result.model_dump() if hasattr(result, "model_dump") else result
|
||||
|
||||
return Data(data=search_result)
|
||||
16
src/bundles/firecrawl/src/lfx_firecrawl/extension.json
Normal file
16
src/bundles/firecrawl/src/lfx_firecrawl/extension.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"$schema": "https://schemas.langflow.org/extension/v1.json",
|
||||
"id": "lfx-firecrawl",
|
||||
"version": "0.1.0",
|
||||
"name": "Firecrawl",
|
||||
"description": "Firecrawl components (Scrape, Crawl, Map, and Search APIs) as a standalone Langflow Extension Bundle.",
|
||||
"lfx": {
|
||||
"compat": ["1"]
|
||||
},
|
||||
"bundles": [
|
||||
{
|
||||
"name": "firecrawl",
|
||||
"path": "components/firecrawl"
|
||||
}
|
||||
]
|
||||
}
|
||||
182
src/bundles/firecrawl/tests/test_firecrawl_components.py
Normal file
182
src/bundles/firecrawl/tests/test_firecrawl_components.py
Normal file
@ -0,0 +1,182 @@
|
||||
"""Unit tests for the Firecrawl extension bundle (``lfx-firecrawl``).
|
||||
|
||||
The components used to live at ``lfx.components.firecrawl``; they have since
|
||||
been extracted into a standalone bundle, so these tests travel with the
|
||||
bundle and import the public bundle entry point.
|
||||
|
||||
They verify v2 SDK usage, the v1->v2 parameter mapping, response
|
||||
serialization, and error handling using a mocked Firecrawl SDK. No network
|
||||
access or real ``firecrawl-py`` install is required: the SDK is injected via
|
||||
``sys.modules`` so the components' in-method ``from firecrawl import ...``
|
||||
statements resolve to the mocks.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import types
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from lfx_firecrawl import FirecrawlCrawlApi, FirecrawlMapApi, FirecrawlScrapeApi, FirecrawlSearchApi
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_firecrawl(monkeypatch):
|
||||
"""Inject a fake ``firecrawl`` SDK and return the mocked ``Firecrawl`` class.
|
||||
|
||||
``mock_firecrawl.return_value`` is the client instance whose
|
||||
``.scrape``/``.crawl``/``.map``/``.extract``/``.search`` methods can be
|
||||
stubbed per test.
|
||||
"""
|
||||
client_cls = MagicMock(name="Firecrawl")
|
||||
|
||||
firecrawl_mod = types.ModuleType("firecrawl")
|
||||
firecrawl_mod.Firecrawl = client_cls
|
||||
|
||||
v2_mod = types.ModuleType("firecrawl.v2")
|
||||
types_mod = types.ModuleType("firecrawl.v2.types")
|
||||
types_mod.ScrapeOptions = MagicMock(name="ScrapeOptions")
|
||||
v2_mod.types = types_mod
|
||||
|
||||
monkeypatch.setitem(sys.modules, "firecrawl", firecrawl_mod)
|
||||
monkeypatch.setitem(sys.modules, "firecrawl.v2", v2_mod)
|
||||
monkeypatch.setitem(sys.modules, "firecrawl.v2.types", types_mod)
|
||||
return client_cls
|
||||
|
||||
|
||||
def _typed(dump):
|
||||
"""Return a fake v2 typed response whose ``model_dump()`` yields ``dump``."""
|
||||
obj = MagicMock()
|
||||
obj.model_dump.return_value = dump
|
||||
return obj
|
||||
|
||||
|
||||
class TestFirecrawlScrapeApi:
|
||||
def test_scrape_calls_v2_and_serializes(self, mock_firecrawl):
|
||||
client = mock_firecrawl.return_value
|
||||
client.scrape.return_value = _typed({"markdown": "# Hi", "metadata": {}})
|
||||
|
||||
component = FirecrawlScrapeApi()
|
||||
component._attributes = {
|
||||
"api_key": "test-key", # pragma: allowlist secret
|
||||
"url": "https://example.com",
|
||||
"timeout": 0,
|
||||
"scrapeOptions": None,
|
||||
"extractorOptions": None,
|
||||
}
|
||||
|
||||
result = component.scrape()
|
||||
|
||||
mock_firecrawl.assert_called_once_with(api_key="test-key") # pragma: allowlist secret
|
||||
client.scrape.assert_called_once()
|
||||
assert client.scrape.call_args.args[0] == "https://example.com"
|
||||
assert result.data == {"markdown": "# Hi", "metadata": {}}
|
||||
|
||||
def test_scrape_propagates_sdk_error(self, mock_firecrawl):
|
||||
client = mock_firecrawl.return_value
|
||||
client.scrape.side_effect = RuntimeError("boom")
|
||||
|
||||
component = FirecrawlScrapeApi()
|
||||
component._attributes = {
|
||||
"api_key": "k", # pragma: allowlist secret
|
||||
"url": "https://example.com",
|
||||
"timeout": 0,
|
||||
"scrapeOptions": None,
|
||||
"extractorOptions": None,
|
||||
}
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
component.scrape()
|
||||
|
||||
|
||||
class TestFirecrawlCrawlApi:
|
||||
def test_crawl_maps_legacy_params_and_serializes(self, mock_firecrawl):
|
||||
client = mock_firecrawl.return_value
|
||||
client.crawl.return_value = _typed({"status": "completed", "data": []})
|
||||
|
||||
component = FirecrawlCrawlApi()
|
||||
component._attributes = {
|
||||
"api_key": "test-key", # pragma: allowlist secret
|
||||
"url": "https://example.com",
|
||||
"timeout": 0,
|
||||
"idempotency_key": "",
|
||||
"crawlerOptions": None,
|
||||
"scrapeOptions": None,
|
||||
}
|
||||
|
||||
result = component.crawl()
|
||||
|
||||
mock_firecrawl.assert_called_once_with(api_key="test-key") # pragma: allowlist secret
|
||||
client.crawl.assert_called_once()
|
||||
assert client.crawl.call_args.args[0] == "https://example.com"
|
||||
kwargs = client.crawl.call_args.kwargs
|
||||
# v1 "maxDepth" was renamed to v2 "max_discovery_depth".
|
||||
assert "max_discovery_depth" in kwargs
|
||||
assert "max_depth" not in kwargs
|
||||
# v1 "allowBackwardLinks" was renamed to v2 "crawl_entire_domain".
|
||||
assert "crawl_entire_domain" in kwargs
|
||||
assert result.data["results"] == {"status": "completed", "data": []}
|
||||
|
||||
|
||||
class TestFirecrawlMapApi:
|
||||
def test_map_sitemap_only_maps_to_enum(self, mock_firecrawl):
|
||||
client = mock_firecrawl.return_value
|
||||
result_obj = MagicMock()
|
||||
result_obj.links = [_typed({"url": "https://example.com/a"})]
|
||||
client.map.return_value = result_obj
|
||||
|
||||
component = FirecrawlMapApi()
|
||||
component._attributes = {
|
||||
"api_key": "test-key", # pragma: allowlist secret
|
||||
"urls": "https://example.com",
|
||||
"ignore_sitemap": False,
|
||||
"sitemap_only": True,
|
||||
"include_subdomains": False,
|
||||
}
|
||||
|
||||
result = component.map()
|
||||
|
||||
client.map.assert_called_once()
|
||||
assert client.map.call_args.kwargs.get("sitemap") == "only"
|
||||
assert result.data["links"] == [{"url": "https://example.com/a"}]
|
||||
|
||||
def test_map_ignore_sitemap_maps_to_skip(self, mock_firecrawl):
|
||||
client = mock_firecrawl.return_value
|
||||
result_obj = MagicMock()
|
||||
result_obj.links = []
|
||||
client.map.return_value = result_obj
|
||||
|
||||
component = FirecrawlMapApi()
|
||||
component._attributes = {
|
||||
"api_key": "k", # pragma: allowlist secret
|
||||
"urls": "https://example.com",
|
||||
"ignore_sitemap": True,
|
||||
"sitemap_only": False,
|
||||
"include_subdomains": False,
|
||||
}
|
||||
|
||||
component.map()
|
||||
|
||||
assert client.map.call_args.kwargs.get("sitemap") == "skip"
|
||||
|
||||
|
||||
class TestFirecrawlSearchApi:
|
||||
def test_search_calls_v2_with_query_limit_location(self, mock_firecrawl):
|
||||
client = mock_firecrawl.return_value
|
||||
client.search.return_value = _typed({"web": [{"url": "https://example.com"}]})
|
||||
|
||||
component = FirecrawlSearchApi()
|
||||
component._attributes = {
|
||||
"api_key": "test-key", # pragma: allowlist secret
|
||||
"query": "firecrawl",
|
||||
"limit": 5,
|
||||
"location": "US",
|
||||
}
|
||||
|
||||
result = component.search()
|
||||
|
||||
mock_firecrawl.assert_called_once_with(api_key="test-key") # pragma: allowlist secret
|
||||
client.search.assert_called_once()
|
||||
assert client.search.call_args.args[0] == "firecrawl"
|
||||
assert client.search.call_args.kwargs.get("limit") == 5
|
||||
assert client.search.call_args.kwargs.get("location") == "US"
|
||||
assert result.data == {"web": [{"url": "https://example.com"}]}
|
||||
@ -620,9 +620,9 @@ export const nodeIconToDisplayIconMap: Record<string, string> = {
|
||||
FAISS: "Meta",
|
||||
FaissSearch: "Meta",
|
||||
FirecrawlCrawlApi: "Firecrawl",
|
||||
FirecrawlExtractApi: "Firecrawl",
|
||||
FirecrawlMapApi: "Firecrawl",
|
||||
FirecrawlScrapeApi: "Firecrawl",
|
||||
FirecrawlSearchApi: "Firecrawl",
|
||||
GitbookLoader: "GitBook",
|
||||
GoogleGenerativeAI: "GoogleGenerativeAI",
|
||||
Googlesheets: "Googlesheets",
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -42,7 +42,6 @@ if TYPE_CHECKING:
|
||||
embeddings,
|
||||
exa,
|
||||
files_ingestion,
|
||||
firecrawl,
|
||||
git,
|
||||
glean,
|
||||
google,
|
||||
@ -143,7 +142,6 @@ _dynamic_imports = {
|
||||
"embeddings": "__module__",
|
||||
"exa": "__module__",
|
||||
"FAISS": "__module__",
|
||||
"firecrawl": "__module__",
|
||||
"files_ingestion": "__module__",
|
||||
"git": "__module__",
|
||||
"glean": "__module__",
|
||||
@ -274,7 +272,6 @@ __all__ = [
|
||||
"embeddings",
|
||||
"exa",
|
||||
"files_ingestion",
|
||||
"firecrawl",
|
||||
"git",
|
||||
"glean",
|
||||
"google",
|
||||
|
||||
@ -1,43 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from lfx.components._importing import import_mod
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .firecrawl_crawl_api import FirecrawlCrawlApi
|
||||
from .firecrawl_extract_api import FirecrawlExtractApi
|
||||
from .firecrawl_map_api import FirecrawlMapApi
|
||||
from .firecrawl_scrape_api import FirecrawlScrapeApi
|
||||
|
||||
_dynamic_imports = {
|
||||
"FirecrawlCrawlApi": "firecrawl_crawl_api",
|
||||
"FirecrawlExtractApi": "firecrawl_extract_api",
|
||||
"FirecrawlMapApi": "firecrawl_map_api",
|
||||
"FirecrawlScrapeApi": "firecrawl_scrape_api",
|
||||
}
|
||||
|
||||
__all__ = [
|
||||
"FirecrawlCrawlApi",
|
||||
"FirecrawlExtractApi",
|
||||
"FirecrawlMapApi",
|
||||
"FirecrawlScrapeApi",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(attr_name: str) -> Any:
|
||||
"""Lazily import firecrawl components on attribute access."""
|
||||
if attr_name not in _dynamic_imports:
|
||||
msg = f"module '{__name__}' has no attribute '{attr_name}'"
|
||||
raise AttributeError(msg)
|
||||
try:
|
||||
result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
|
||||
except (ModuleNotFoundError, ImportError, AttributeError) as e:
|
||||
msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
|
||||
raise AttributeError(msg) from e
|
||||
globals()[attr_name] = result
|
||||
return result
|
||||
|
||||
|
||||
def __dir__() -> list[str]:
|
||||
return list(__all__)
|
||||
@ -1,136 +0,0 @@
|
||||
from lfx.custom.custom_component.component import Component
|
||||
from lfx.io import BoolInput, DataInput, MultilineInput, Output, SecretStrInput
|
||||
from lfx.log.logger import logger
|
||||
from lfx.schema.data import Data
|
||||
|
||||
|
||||
class FirecrawlExtractApi(Component):
|
||||
display_name: str = "Firecrawl Extract API"
|
||||
description: str = "Extracts data from a URL."
|
||||
name = "FirecrawlExtractApi"
|
||||
|
||||
documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/extract"
|
||||
|
||||
inputs = [
|
||||
SecretStrInput(
|
||||
name="api_key",
|
||||
display_name="Firecrawl API Key",
|
||||
required=True,
|
||||
password=True,
|
||||
info="The API key to use Firecrawl API.",
|
||||
),
|
||||
MultilineInput(
|
||||
name="urls",
|
||||
display_name="URLs",
|
||||
required=True,
|
||||
info="List of URLs to extract data from (separated by commas or new lines).",
|
||||
tool_mode=True,
|
||||
),
|
||||
MultilineInput(
|
||||
name="prompt",
|
||||
display_name="Prompt",
|
||||
required=True,
|
||||
info="Prompt to guide the extraction process.",
|
||||
tool_mode=True,
|
||||
),
|
||||
DataInput(
|
||||
name="schema",
|
||||
display_name="Schema",
|
||||
required=False,
|
||||
info="Schema to define the structure of the extracted data.",
|
||||
),
|
||||
BoolInput(
|
||||
name="enable_web_search",
|
||||
display_name="Enable Web Search",
|
||||
info="When true, the extraction will use web search to find additional data.",
|
||||
),
|
||||
# # Optional: Not essential for basic extraction
|
||||
# BoolInput(
|
||||
# name="ignore_sitemap",
|
||||
# display_name="Ignore Sitemap",
|
||||
# info="When true, sitemap.xml files will be ignored during website scanning.",
|
||||
# ),
|
||||
# # Optional: Not essential for basic extraction
|
||||
# BoolInput(
|
||||
# name="include_subdomains",
|
||||
# display_name="Include Subdomains",
|
||||
# info="When true, subdomains of the provided URLs will also be scanned.",
|
||||
# ),
|
||||
# # Optional: Not essential for basic extraction
|
||||
# BoolInput(
|
||||
# name="show_sources",
|
||||
# display_name="Show Sources",
|
||||
# info="When true, the sources used to extract the data will be included in the response.",
|
||||
# ),
|
||||
]
|
||||
|
||||
outputs = [
|
||||
Output(display_name="JSON", name="data", method="extract"),
|
||||
]
|
||||
|
||||
def extract(self) -> Data:
|
||||
try:
|
||||
from firecrawl import FirecrawlApp
|
||||
except ImportError as e:
|
||||
msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
|
||||
raise ImportError(msg) from e
|
||||
|
||||
# Validate API key
|
||||
if not self.api_key:
|
||||
msg = "API key is required"
|
||||
raise ValueError(msg)
|
||||
|
||||
# Validate URLs
|
||||
if not self.urls:
|
||||
msg = "URLs are required"
|
||||
raise ValueError(msg)
|
||||
|
||||
# Split and validate URLs (handle both commas and newlines)
|
||||
urls = [url.strip() for url in self.urls.replace("\n", ",").split(",") if url.strip()]
|
||||
if not urls:
|
||||
msg = "No valid URLs provided"
|
||||
raise ValueError(msg)
|
||||
|
||||
# Validate and process prompt
|
||||
if not self.prompt:
|
||||
msg = "Prompt is required"
|
||||
raise ValueError(msg)
|
||||
|
||||
# Get the prompt text (handling both string and multiline input)
|
||||
prompt_text = self.prompt.strip()
|
||||
|
||||
# Enhance the prompt to encourage comprehensive extraction
|
||||
enhanced_prompt = prompt_text
|
||||
if "schema" not in prompt_text.lower():
|
||||
enhanced_prompt = f"{prompt_text}. Please extract all instances in a comprehensive, structured format."
|
||||
|
||||
params = {
|
||||
"prompt": enhanced_prompt,
|
||||
"enableWebSearch": self.enable_web_search,
|
||||
# Optional parameters - not essential for basic extraction
|
||||
"ignoreSitemap": self.ignore_sitemap,
|
||||
"includeSubdomains": self.include_subdomains,
|
||||
"showSources": self.show_sources,
|
||||
"timeout": 300,
|
||||
}
|
||||
|
||||
# Only add schema to params if it's provided and is a valid schema structure
|
||||
if self.schema:
|
||||
try:
|
||||
if isinstance(self.schema, dict) and "type" in self.schema:
|
||||
params["schema"] = self.schema
|
||||
elif hasattr(self.schema, "dict") and "type" in self.schema.dict():
|
||||
params["schema"] = self.schema.dict()
|
||||
else:
|
||||
# Skip invalid schema without raising an error
|
||||
pass
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"Invalid schema: {e!s}")
|
||||
|
||||
try:
|
||||
app = FirecrawlApp(api_key=self.api_key)
|
||||
extract_result = app.extract(urls, params=params)
|
||||
return Data(data=extract_result)
|
||||
except Exception as e:
|
||||
msg = f"Error during extraction: {e!s}"
|
||||
raise ValueError(msg) from e
|
||||
@ -181,6 +181,86 @@
|
||||
"target": "ext:docling:ExportDoclingDocumentComponent@official",
|
||||
"added_in": "1.10.0"
|
||||
},
|
||||
{
|
||||
"bare_class_name": "FirecrawlCrawlApi",
|
||||
"target": "ext:firecrawl:FirecrawlCrawlApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"import_path": "lfx.components.firecrawl.firecrawl_crawl_api.FirecrawlCrawlApi",
|
||||
"target": "ext:firecrawl:FirecrawlCrawlApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"import_path": "lfx.components.firecrawl.FirecrawlCrawlApi",
|
||||
"target": "ext:firecrawl:FirecrawlCrawlApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"legacy_slot": "ext:firecrawl:FirecrawlCrawlApi@official-pre-a",
|
||||
"target": "ext:firecrawl:FirecrawlCrawlApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"bare_class_name": "FirecrawlMapApi",
|
||||
"target": "ext:firecrawl:FirecrawlMapApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"import_path": "lfx.components.firecrawl.firecrawl_map_api.FirecrawlMapApi",
|
||||
"target": "ext:firecrawl:FirecrawlMapApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"import_path": "lfx.components.firecrawl.FirecrawlMapApi",
|
||||
"target": "ext:firecrawl:FirecrawlMapApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"legacy_slot": "ext:firecrawl:FirecrawlMapApi@official-pre-a",
|
||||
"target": "ext:firecrawl:FirecrawlMapApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"bare_class_name": "FirecrawlScrapeApi",
|
||||
"target": "ext:firecrawl:FirecrawlScrapeApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"import_path": "lfx.components.firecrawl.firecrawl_scrape_api.FirecrawlScrapeApi",
|
||||
"target": "ext:firecrawl:FirecrawlScrapeApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"import_path": "lfx.components.firecrawl.FirecrawlScrapeApi",
|
||||
"target": "ext:firecrawl:FirecrawlScrapeApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"legacy_slot": "ext:firecrawl:FirecrawlScrapeApi@official-pre-a",
|
||||
"target": "ext:firecrawl:FirecrawlScrapeApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"bare_class_name": "FirecrawlSearchApi",
|
||||
"target": "ext:firecrawl:FirecrawlSearchApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"import_path": "lfx.components.firecrawl.firecrawl_search_api.FirecrawlSearchApi",
|
||||
"target": "ext:firecrawl:FirecrawlSearchApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"import_path": "lfx.components.firecrawl.FirecrawlSearchApi",
|
||||
"target": "ext:firecrawl:FirecrawlSearchApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"legacy_slot": "ext:firecrawl:FirecrawlSearchApi@official-pre-a",
|
||||
"target": "ext:firecrawl:FirecrawlSearchApi@official",
|
||||
"added_in": "1.11.0"
|
||||
},
|
||||
{
|
||||
"bare_class_name": "NextPlaidVectorStoreComponent",
|
||||
"target": "ext:nextplaid:NextPlaidVectorStoreComponent@official",
|
||||
|
||||
@ -0,0 +1,128 @@
|
||||
"""Integration test: legacy firecrawl flows upgrade cleanly.
|
||||
|
||||
Mirrors ``test_pilot_duckduckgo_upgrade.py`` for ``FirecrawlCrawlApi``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from importlib import metadata as importlib_metadata
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from lfx.extension.migration.loader import load_migration_table
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[5]
|
||||
TABLE_PATH = REPO_ROOT / "src" / "lfx" / "src" / "lfx" / "extension" / "migration" / "migration_table.json"
|
||||
EXPECTED_TARGET = "ext:firecrawl:FirecrawlCrawlApi@official"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def migration_table():
|
||||
table, error = load_migration_table(TABLE_PATH)
|
||||
assert error is None, f"failed to load migration table: {error}"
|
||||
assert table is not None
|
||||
return table
|
||||
|
||||
|
||||
def _saved_flow_node(node_id: str, type_value: str) -> dict:
|
||||
"""Build a minimal saved-flow node skeleton for testing."""
|
||||
return {
|
||||
"id": node_id,
|
||||
"type": "genericNode",
|
||||
"data": {"id": node_id, "type": type_value, "node": {"template": {}}},
|
||||
}
|
||||
|
||||
|
||||
def _saved_flow(*nodes: dict) -> dict:
|
||||
return {"data": {"nodes": list(nodes), "edges": []}}
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_legacy_bare_name_flow_upgrades(migration_table) -> None:
|
||||
"""Pre-Phase-A flow with the bare class name upgrades to the canonical ID."""
|
||||
from lfx.extension.migration.rewrite import migrate_flow_payload
|
||||
|
||||
flow = _saved_flow(_saved_flow_node("firecrawl-1", "FirecrawlCrawlApi"))
|
||||
report = migrate_flow_payload(flow, table=migration_table)
|
||||
|
||||
assert report.rewritten_count == 1
|
||||
assert flow["data"]["nodes"][0]["data"]["type"] == EXPECTED_TARGET
|
||||
[record] = report.records
|
||||
assert record.legacy_form_kind == "bare_class_name"
|
||||
assert record.new_value == EXPECTED_TARGET
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_legacy_import_path_flow_upgrades(migration_table) -> None:
|
||||
"""Dotted import-path form upgrades to the canonical ID."""
|
||||
from lfx.extension.migration.rewrite import migrate_flow_payload
|
||||
|
||||
flow = _saved_flow(
|
||||
_saved_flow_node("firecrawl-2", "lfx.components.firecrawl.firecrawl_crawl_api.FirecrawlCrawlApi")
|
||||
)
|
||||
report = migrate_flow_payload(flow, table=migration_table)
|
||||
|
||||
assert report.rewritten_count == 1
|
||||
assert flow["data"]["nodes"][0]["data"]["type"] == EXPECTED_TARGET
|
||||
assert report.records[0].legacy_form_kind == "import_path"
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_short_import_path_flow_upgrades(migration_table) -> None:
|
||||
"""Package-level import-path form (via ``__init__.py`` re-export) upgrades."""
|
||||
from lfx.extension.migration.rewrite import migrate_flow_payload
|
||||
|
||||
flow = _saved_flow(_saved_flow_node("firecrawl-3", "lfx.components.firecrawl.FirecrawlCrawlApi"))
|
||||
report = migrate_flow_payload(flow, table=migration_table)
|
||||
|
||||
assert report.rewritten_count == 1
|
||||
assert flow["data"]["nodes"][0]["data"]["type"] == EXPECTED_TARGET
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_lfx_firecrawl_distribution_is_importable() -> None:
|
||||
"""The bundle's package is importable in the development workspace."""
|
||||
try:
|
||||
from lfx_firecrawl import FirecrawlCrawlApi
|
||||
except ImportError:
|
||||
pytest.skip("lfx-firecrawl not installed in this test environment")
|
||||
|
||||
assert FirecrawlCrawlApi.__name__ == "FirecrawlCrawlApi"
|
||||
|
||||
|
||||
def _is_editable_install(dist: importlib_metadata.Distribution) -> bool:
|
||||
direct_url = dist.read_text("direct_url.json")
|
||||
if not direct_url:
|
||||
return False
|
||||
try:
|
||||
payload = json.loads(direct_url)
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
return bool(payload.get("dir_info", {}).get("editable"))
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_lfx_firecrawl_ships_manifest() -> None:
|
||||
"""``importlib.metadata`` can find ``extension.json`` for the installed dist."""
|
||||
try:
|
||||
dist = importlib_metadata.distribution("lfx-firecrawl")
|
||||
except importlib_metadata.PackageNotFoundError:
|
||||
pytest.skip("lfx-firecrawl not installed in this test environment")
|
||||
|
||||
if _is_editable_install(dist):
|
||||
import lfx_firecrawl
|
||||
|
||||
package_dir = Path(lfx_firecrawl.__file__).parent
|
||||
manifest_path = package_dir / "extension.json"
|
||||
assert manifest_path.is_file()
|
||||
else:
|
||||
files = dist.files or []
|
||||
manifests = [f for f in files if f.parts and f.parts[-1] == "extension.json"]
|
||||
assert manifests
|
||||
manifest_path = Path(dist.locate_file(manifests[0]))
|
||||
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
assert manifest["id"] == "lfx-firecrawl"
|
||||
assert manifest["lfx"]["compat"] == ["1"]
|
||||
assert any(b["name"] == "firecrawl" for b in manifest["bundles"])
|
||||
28
uv.lock
generated
28
uv.lock
generated
@ -33,6 +33,7 @@ members = [
|
||||
"lfx-arxiv",
|
||||
"lfx-docling",
|
||||
"lfx-duckduckgo",
|
||||
"lfx-firecrawl",
|
||||
"lfx-ibm",
|
||||
"lfx-nextplaid",
|
||||
]
|
||||
@ -4260,18 +4261,20 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "firecrawl-py"
|
||||
version = "1.17.0"
|
||||
version = "4.28.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
{ name = "httpx" },
|
||||
{ name = "nest-asyncio" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "python-dotenv" },
|
||||
{ name = "requests" },
|
||||
{ name = "websockets" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/4d/08/61df5c8357742d8bee504db166b633ef7972b40c831e8c2f2a26feb9021c/firecrawl_py-1.17.0.tar.gz", hash = "sha256:5e2f50ec1f0e67514cdf6f0afc7df6be36eb8277fbec9e1f5a283fc01fae7875", size = 22191, upload-time = "2025-04-17T18:13:01.068Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/2d/09/e7c667875c9a9dac09480e2f69fd46bdd9c1139555b835c3036a9858aff2/firecrawl_py-4.28.2.tar.gz", hash = "sha256:7e6181e2129b63c8d6aec5728d9b2fcf16ea82cb854372ad824b278efd258696", size = 191869, upload-time = "2026-05-28T00:03:48.822Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/53/a6/9431b62571b4ff4cd0d09f70c7df1253abf611e829f68d194e1ea69ea57e/firecrawl_py-1.17.0-py3-none-any.whl", hash = "sha256:0392822fbd906731f4c0876f91a9c3cce7624279c81948e4e3f8bc60b4e1c855", size = 21295, upload-time = "2025-04-17T18:12:59.463Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/08/384ca104c8004ca0541aacd3470e855bd28666d6bdb2101e08446dfcb407/firecrawl_py-4.28.2-py3-none-any.whl", hash = "sha256:0689080cb01672370e5a97963e0df479f6102137aa088857eac0fa287a4269b6", size = 239166, upload-time = "2026-05-28T00:03:47.479Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -7421,6 +7424,7 @@ dependencies = [
|
||||
{ name = "lfx-arxiv" },
|
||||
{ name = "lfx-docling" },
|
||||
{ name = "lfx-duckduckgo" },
|
||||
{ name = "lfx-firecrawl" },
|
||||
{ name = "lfx-ibm" },
|
||||
{ name = "lfx-nextplaid" },
|
||||
]
|
||||
@ -7515,6 +7519,7 @@ requires-dist = [
|
||||
{ name = "lfx-docling", extras = ["image-description"], marker = "extra == 'docling-image-description'", editable = "src/bundles/docling" },
|
||||
{ name = "lfx-docling", extras = ["local"], marker = "extra == 'docling'", editable = "src/bundles/docling" },
|
||||
{ name = "lfx-duckduckgo", editable = "src/bundles/duckduckgo" },
|
||||
{ name = "lfx-firecrawl", editable = "src/bundles/firecrawl" },
|
||||
{ name = "lfx-ibm", editable = "src/bundles/ibm" },
|
||||
{ name = "lfx-nextplaid", editable = "src/bundles/nextplaid" },
|
||||
{ name = "nv-ingest-api", marker = "python_full_version >= '3.12' and extra == 'nv-ingest'", specifier = ">=26.1.0,<27.0.0" },
|
||||
@ -7596,7 +7601,6 @@ dependencies = [
|
||||
{ name = "fastapi" },
|
||||
{ name = "fastapi-pagination" },
|
||||
{ name = "filelock" },
|
||||
{ name = "firecrawl-py" },
|
||||
{ name = "grandalf" },
|
||||
{ name = "greenlet" },
|
||||
{ name = "gunicorn" },
|
||||
@ -8347,7 +8351,6 @@ requires-dist = [
|
||||
{ name = "fastmcp", marker = "extra == 'fastmcp'", specifier = ">=3.2.0" },
|
||||
{ name = "fastparquet", marker = "extra == 'fastparquet'", specifier = ">=2024.11.0,<2025.0.0" },
|
||||
{ name = "filelock", specifier = ">=3.20.1,<4.0.0" },
|
||||
{ name = "firecrawl-py", specifier = ">=1.0.16,<2.0.0" },
|
||||
{ name = "gassist", marker = "sys_platform == 'win32' and extra == 'gassist'", specifier = ">=0.0.1" },
|
||||
{ name = "gitpython", marker = "extra == 'gitpython'", specifier = ">=3.1.50" },
|
||||
{ name = "google-api-python-client", marker = "extra == 'google'", specifier = "~=2.161" },
|
||||
@ -9155,6 +9158,21 @@ requires-dist = [
|
||||
{ name = "lfx", editable = "src/lfx" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lfx-firecrawl"
|
||||
version = "0.1.0"
|
||||
source = { editable = "src/bundles/firecrawl" }
|
||||
dependencies = [
|
||||
{ name = "firecrawl-py" },
|
||||
{ name = "lfx" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "firecrawl-py", specifier = ">=4.0.0,<5.0.0" },
|
||||
{ name = "lfx", editable = "src/lfx" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lfx-ibm"
|
||||
version = "0.1.1"
|
||||
|
||||
Reference in New Issue
Block a user