diff --git a/.secrets.baseline b/.secrets.baseline index 8c0c458bf5..c65c0b72a0 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -2763,7 +2763,7 @@ "filename": "src/backend/base/langflow/locales/en.json", "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", "is_verified": false, - "line_number": 4606 + "line_number": 4552 } ], "src/backend/base/langflow/locales/es.json": [ @@ -9287,5 +9287,5 @@ } ] }, - "generated_at": "2026-06-10T08:36:23Z" + "generated_at": "2026-06-10T18:13:35Z" } diff --git a/docker/build_and_push_backend.Dockerfile b/docker/build_and_push_backend.Dockerfile index 2ef2c94bab..508795bb35 100644 --- a/docker/build_and_push_backend.Dockerfile +++ b/docker/build_and_push_backend.Dockerfile @@ -39,7 +39,7 @@ ENV VIRTUAL_ENV="/app/.venv" # Install langflow-base with all extras except dev (which includes Playwright). # This image ships the langflow-base core only. Extension bundles -# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling) are intentionally NOT +# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling, lfx-firecrawl) are intentionally NOT # installed here -- they belong to the full ``langflow`` distribution, not # the lean core. Use the ``langflow`` image, or ``pip install`` the bundle # alongside this image, to add those components. diff --git a/docker/build_and_push_base.Dockerfile b/docker/build_and_push_base.Dockerfile index be08fc154d..a81df8188b 100644 --- a/docker/build_and_push_base.Dockerfile +++ b/docker/build_and_push_base.Dockerfile @@ -75,7 +75,7 @@ RUN npm install \ WORKDIR /app/src/backend/base # langflow-base ships the core framework only. The extension bundles -# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling) are intentionally NOT +# (lfx-duckduckgo, lfx-arxiv, lfx-ibm, lfx-docling, lfx-firecrawl) are intentionally NOT # installed in this image: they are dependencies of the full ``langflow`` # distribution, not of the lean ``langflow-base`` core, and we keep that # boundary at the image layer too. Consumers who want those components diff --git a/docs/docs/Components/bundles-firecrawl.mdx b/docs/docs/Components/bundles-firecrawl.mdx index 6ba3d4f734..a80bf9c753 100644 --- a/docs/docs/Components/bundles-firecrawl.mdx +++ b/docs/docs/Components/bundles-firecrawl.mdx @@ -60,18 +60,17 @@ It outputs the API response as [`Data`](/data-types#data). | Sitemap Only (`sitemap_only`) | Boolean | Input parameter. When true, only links found in the sitemap are returned. | | Include Subdomains (`include_subdomains`) | Boolean | Input parameter. When true, subdomains of the provided URL are also scanned. | -## Firecrawl Extract API +## Firecrawl Search API -This component extracts structured data from one or more URLs using a prompt or schema. +This component searches the web and returns the results. It outputs the API response as [`Data`](/data-types#data). -### Firecrawl Extract API parameters +### Firecrawl Search API parameters | Name | Type | Description | |------|------|-------------| | Firecrawl API Key (`api_key`) | SecretString | Input parameter. The API key to use Firecrawl API. | -| URLs (`urls`) | String | Input parameter. List of URLs to extract data from (separated by commas or new lines). | -| Prompt (`prompt`) | String | Input parameter. Prompt to guide the extraction process. | -| Schema (`schema`) | Data | Input parameter. Schema to define the structure of the extracted data. | -| Enable Web Search (`enable_web_search`) | Boolean | Input parameter. When true, the extraction uses web search to find additional data. | \ No newline at end of file +| Query (`query`) | String | Input parameter. The search query to run. | +| Limit (`limit`) | Integer | Input parameter. The maximum number of results to return. | +| Location (`location`) | String | Input parameter. Location to bias the search results (for example, a country or region). | diff --git a/pyproject.toml b/pyproject.toml index f7421444dd..3f6768c01b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "lfx-arxiv>=0.1.0", "lfx-ibm>=0.1.0", "lfx-docling>=0.1.0", + "lfx-firecrawl>=0.1.0", "lfx-nextplaid>=0.1.0", # langflow-extensions:bundle-deps-end ] @@ -85,6 +86,7 @@ lfx-duckduckgo = { workspace = true } lfx-arxiv = { workspace = true } lfx-ibm = { workspace = true } lfx-docling = { workspace = true } +lfx-firecrawl = { workspace = true } lfx-nextplaid = { workspace = true } # langflow-extensions:bundle-sources-end torch = { index = "pytorch-cpu" } @@ -102,6 +104,7 @@ members = [ "src/bundles/arxiv", "src/bundles/ibm", "src/bundles/docling", + "src/bundles/firecrawl", "src/bundles/nextplaid", # langflow-extensions:bundle-members-end ] diff --git a/src/backend/base/langflow/locales/en.json b/src/backend/base/langflow/locales/en.json index 817c874b15..ba5df4bef0 100644 --- a/src/backend/base/langflow/locales/en.json +++ b/src/backend/base/langflow/locales/en.json @@ -3567,60 +3567,6 @@ "components.filterdatavalues.inputs.operator.display_name.d6b063d9": "Comparison Operator", "components.filterdatavalues.inputs.operator.info.81652313": "The operator to apply for comparing the values.", "components.filterdatavalues.outputs.filtered_data.display_name.8acf8a3c": "Filtered Data", - "components.firecrawlcrawlapi.description.e9b9ca89": "Crawls a URL and returns the results.", - "components.firecrawlcrawlapi.display_name.1c8af0bd": "Firecrawl Crawl API", - "components.firecrawlcrawlapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key", - "components.firecrawlcrawlapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.", - "components.firecrawlcrawlapi.inputs.crawlerOptions.display_name.77b95f41": "Crawler Options", - "components.firecrawlcrawlapi.inputs.crawlerOptions.info.b3898e10": "The crawler options to send with the request.", - "components.firecrawlcrawlapi.inputs.idempotency_key.display_name.0409cf23": "Idempotency Key", - "components.firecrawlcrawlapi.inputs.idempotency_key.info.7fa2bc03": "Optional idempotency key to ensure unique requests.", - "components.firecrawlcrawlapi.inputs.scrapeOptions.display_name.f6227db1": "Scrape Options", - "components.firecrawlcrawlapi.inputs.scrapeOptions.info.034e1d4f": "The page options to send with the request.", - "components.firecrawlcrawlapi.inputs.timeout.display_name.70594d93": "Timeout", - "components.firecrawlcrawlapi.inputs.timeout.info.11f2ddb2": "Timeout in milliseconds for the request.", - "components.firecrawlcrawlapi.inputs.url.display_name.e7a241de": "URL", - "components.firecrawlcrawlapi.inputs.url.info.b260224a": "The URL to scrape.", - "components.firecrawlcrawlapi.outputs.data.display_name.db1a21a0": "JSON", - "components.firecrawlextractapi.description.eed57be9": "Extracts data from a URL.", - "components.firecrawlextractapi.display_name.ae3a11a0": "Firecrawl Extract API", - "components.firecrawlextractapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key", - "components.firecrawlextractapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.", - "components.firecrawlextractapi.inputs.enable_web_search.display_name.a53c041a": "Enable Web Search", - "components.firecrawlextractapi.inputs.enable_web_search.info.07671323": "When true, the extraction will use web search to find additional data.", - "components.firecrawlextractapi.inputs.prompt.display_name.5c391238": "Prompt", - "components.firecrawlextractapi.inputs.prompt.info.30c68d4d": "Prompt to guide the extraction process.", - "components.firecrawlextractapi.inputs.schema.display_name.07b091a3": "Schema", - "components.firecrawlextractapi.inputs.schema.info.fc0ac9cc": "Schema to define the structure of the extracted data.", - "components.firecrawlextractapi.inputs.urls.display_name.1240054e": "URLs", - "components.firecrawlextractapi.inputs.urls.info.a35e321b": "List of URLs to extract data from (separated by commas or new lines).", - "components.firecrawlextractapi.outputs.data.display_name.db1a21a0": "JSON", - "components.firecrawlmapapi.description.acbcf5bc": "Maps a URL and returns the results.", - "components.firecrawlmapapi.display_name.30b2bee7": "Firecrawl Map API", - "components.firecrawlmapapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key", - "components.firecrawlmapapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.", - "components.firecrawlmapapi.inputs.ignore_sitemap.display_name.6d48c522": "Ignore Sitemap", - "components.firecrawlmapapi.inputs.ignore_sitemap.info.02a79d48": "When true, the sitemap.xml file will be ignored during crawling.", - "components.firecrawlmapapi.inputs.include_subdomains.display_name.5cf2dc9e": "Include Subdomains", - "components.firecrawlmapapi.inputs.include_subdomains.info.073e9171": "When true, subdomains of the provided URL will also be scanned.", - "components.firecrawlmapapi.inputs.sitemap_only.display_name.021a3604": "Sitemap Only", - "components.firecrawlmapapi.inputs.sitemap_only.info.7ca2f13c": "When true, only links found in the sitemap will be returned.", - "components.firecrawlmapapi.inputs.urls.display_name.1240054e": "URLs", - "components.firecrawlmapapi.inputs.urls.info.71719292": "List of URLs to create maps from (separated by commas or new lines).", - "components.firecrawlmapapi.outputs.data.display_name.db1a21a0": "JSON", - "components.firecrawlscrapeapi.description.017ffc83": "Scrapes a URL and returns the results.", - "components.firecrawlscrapeapi.display_name.39a1c0de": "Firecrawl Scrape API", - "components.firecrawlscrapeapi.inputs.api_key.display_name.bda16463": "Firecrawl API Key", - "components.firecrawlscrapeapi.inputs.api_key.info.0801284e": "The API key to use Firecrawl API.", - "components.firecrawlscrapeapi.inputs.extractorOptions.display_name.80a369a3": "Extractor Options", - "components.firecrawlscrapeapi.inputs.extractorOptions.info.9fd7d66f": "The extractor options to send with the request.", - "components.firecrawlscrapeapi.inputs.scrapeOptions.display_name.f6227db1": "Scrape Options", - "components.firecrawlscrapeapi.inputs.scrapeOptions.info.034e1d4f": "The page options to send with the request.", - "components.firecrawlscrapeapi.inputs.timeout.display_name.70594d93": "Timeout", - "components.firecrawlscrapeapi.inputs.timeout.info.11f2ddb2": "Timeout in milliseconds for the request.", - "components.firecrawlscrapeapi.inputs.url.display_name.e7a241de": "URL", - "components.firecrawlscrapeapi.inputs.url.info.b260224a": "The URL to scrape.", - "components.firecrawlscrapeapi.outputs.data.display_name.db1a21a0": "JSON", "components.flowtool.description.d3907775": "Construct a Tool from a function that runs the loaded Flow.", "components.flowtool.display_name.21209f51": "Flow as Tool", "components.flowtool.inputs.flow_name.display_name.e5fcec99": "Flow Name", diff --git a/src/backend/base/pyproject.toml b/src/backend/base/pyproject.toml index 5dea4eaa90..8be6961883 100644 --- a/src/backend/base/pyproject.toml +++ b/src/backend/base/pyproject.toml @@ -64,7 +64,6 @@ dependencies = [ "uncurl>=0.0.11,<1.0.0", "sentry-sdk[fastapi,loguru]>=2.5.1,<3.0.0", "chardet>=7.3.0", - "firecrawl-py>=1.0.16,<2.0.0", "opentelemetry-api>=1.30.0,<2.0.0", "opentelemetry-sdk>=1.30.0,<2.0.0", "opentelemetry-exporter-prometheus>=0.50b0,<1.0.0", diff --git a/src/bundles/firecrawl/README.md b/src/bundles/firecrawl/README.md new file mode 100644 index 0000000000..86b40755b7 --- /dev/null +++ b/src/bundles/firecrawl/README.md @@ -0,0 +1,29 @@ +# lfx-firecrawl + +Firecrawl components (Scrape, Crawl, Map, and Search APIs) as a standalone +Langflow Extension Bundle, built against the firecrawl-py v2 SDK. + +## Install + +```bash +pip install lfx-firecrawl +``` + +The bundle is registered automatically via the `langflow.extensions` +entry-point. After install, restart your Langflow server; the bundle's +components will appear in the palette under the `firecrawl` group. + +## Develop + +```bash +cd src/bundles/firecrawl +pip install -e . +lfx extension validate src/lfx_firecrawl +``` + +## Migration + +Saved flows referencing the legacy class name(s) or the old import paths +under `lfx.components.firecrawl.*` are rewritten to the new namespaced +IDs by the migration table in +`src/lfx/src/lfx/extension/migration/migration_table.json`. diff --git a/src/bundles/firecrawl/pyproject.toml b/src/bundles/firecrawl/pyproject.toml new file mode 100644 index 0000000000..613860194d --- /dev/null +++ b/src/bundles/firecrawl/pyproject.toml @@ -0,0 +1,55 @@ +[project] +name = "lfx-firecrawl" +version = "0.1.0" +description = "Firecrawl components (Scrape, Crawl, Map, and Search APIs) as a standalone Langflow Extension Bundle." +readme = "README.md" +requires-python = ">=3.10,<3.15" +license = { text = "MIT" } +authors = [ + { name = "Langflow", email = "contact@langflow.org" }, +] +keywords = ["langflow", "lfx", "extension", "bundle", "firecrawl"] + +# Runtime deps: lfx (the BUNDLE_API surface) plus any third-party imports +# the bundle's components rely on. REVIEW THIS LIST -- the script ports +# only ``lfx``; add any other deps the moved component(s) import. +# lfx is floored at the current major.minor line and capped below the next +# lfx major (e.g. "lfx>=1.10.0,<2.0.0"); the floor is read from +# src/lfx/pyproject.toml at port time and re-synced on ``make patch`` via +# scripts/ci/sync_bundle_lfx_pin.py. Fine-grained BUNDLE_API compatibility +# is enforced via extension.json's lfx.compat list against BUNDLE_API_VERSION. +dependencies = [ + "lfx>=1.11.0.dev0,<2.0.0", + "firecrawl-py>=4.0.0,<5.0.0", +] + +[project.urls] +Homepage = "https://github.com/langflow-ai/langflow" +Documentation = "https://docs.langflow.org/extensions" +Repository = "https://github.com/langflow-ai/langflow" + +# Manifest-shipping distributions are discovered via the +# ``langflow.extensions`` entry-point. Editable installs whose +# ``dist.files`` only surfaces dist-info entries fall back to this +# entry-point to find the manifest. +[project.entry-points."langflow.extensions"] +lfx-firecrawl = "lfx_firecrawl" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +# extension.json + components live inside the lfx_firecrawl package so +# ``importlib.metadata.files(dist)`` finds them and the loader resolves +# bundles[].path relative to the manifest's directory. +packages = ["src/lfx_firecrawl"] +include = ["src/lfx_firecrawl/extension.json", "src/lfx_firecrawl/components/**/*.py"] + +[tool.hatch.build.targets.sdist] +include = [ + "src/lfx_firecrawl", + "extension.json", + "README.md", + "pyproject.toml", +] diff --git a/src/bundles/firecrawl/src/lfx_firecrawl/__init__.py b/src/bundles/firecrawl/src/lfx_firecrawl/__init__.py new file mode 100644 index 0000000000..1df9f385e0 --- /dev/null +++ b/src/bundles/firecrawl/src/lfx_firecrawl/__init__.py @@ -0,0 +1,14 @@ +"""lfx-firecrawl: Firecrawl bundle. + +Distribution unit ``lfx-firecrawl``. At runtime Langflow's loader +discovers ``extension.json`` shipped alongside this ``__init__.py`` and +registers the bundle's components under the namespaced IDs +``ext:firecrawl:@official``. +""" + +from lfx_firecrawl.components.firecrawl.firecrawl_crawl_api import FirecrawlCrawlApi +from lfx_firecrawl.components.firecrawl.firecrawl_map_api import FirecrawlMapApi +from lfx_firecrawl.components.firecrawl.firecrawl_scrape_api import FirecrawlScrapeApi +from lfx_firecrawl.components.firecrawl.firecrawl_search_api import FirecrawlSearchApi + +__all__ = ["FirecrawlCrawlApi", "FirecrawlMapApi", "FirecrawlScrapeApi", "FirecrawlSearchApi"] diff --git a/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/__init__.py b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/__init__.py new file mode 100644 index 0000000000..c80de80888 --- /dev/null +++ b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/__init__.py @@ -0,0 +1,13 @@ +"""Component re-exports for the ``firecrawl`` bundle. + +Saved-flow migration entries that target ``lfx.components.firecrawl.`` +resolve through this package, so the moved Component class(es) must be +importable from here by name. +""" + +from .firecrawl_crawl_api import FirecrawlCrawlApi +from .firecrawl_map_api import FirecrawlMapApi +from .firecrawl_scrape_api import FirecrawlScrapeApi +from .firecrawl_search_api import FirecrawlSearchApi + +__all__ = ["FirecrawlCrawlApi", "FirecrawlMapApi", "FirecrawlScrapeApi", "FirecrawlSearchApi"] diff --git a/src/lfx/src/lfx/components/firecrawl/firecrawl_crawl_api.py b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_crawl_api.py similarity index 52% rename from src/lfx/src/lfx/components/firecrawl/firecrawl_crawl_api.py rename to src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_crawl_api.py index a1315c964c..82c8c7e62a 100644 --- a/src/lfx/src/lfx/components/firecrawl/firecrawl_crawl_api.py +++ b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_crawl_api.py @@ -1,9 +1,21 @@ -import uuid +import re from lfx.custom.custom_component.component import Component from lfx.io import DataInput, IntInput, MultilineInput, Output, SecretStrInput, StrInput from lfx.schema.data import Data +_CAMEL_TO_SNAKE_RE = re.compile(r"(? dict: + """Convert camelCase option keys to snake_case keyword arguments. + + The firecrawl-py v1 convention uses camelCase, while the v2 SDK expects + snake_case keyword arguments. Keys that are already snake_case are passed + through unchanged. + """ + return {_CAMEL_TO_SNAKE_RE.sub("_", key).lower(): value for key, value in params.items()} + class FirecrawlCrawlApi(Component): display_name: str = "Firecrawl Crawl API" @@ -56,33 +68,42 @@ class FirecrawlCrawlApi(Component): def crawl(self) -> Data: try: - from firecrawl import FirecrawlApp + from firecrawl import Firecrawl + from firecrawl.v2.types import ScrapeOptions except ImportError as e: msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`." raise ImportError(msg) from e - params = self.crawlerOptions.__dict__["data"] if self.crawlerOptions else {} - scrape_options_dict = self.scrapeOptions.__dict__["data"] if self.scrapeOptions else {} - if scrape_options_dict: - params["scrapeOptions"] = scrape_options_dict + params = dict(self.crawlerOptions.__dict__.get("data", {})) if self.crawlerOptions else {} + scrape_options_dict = dict(self.scrapeOptions.__dict__.get("data", {})) if self.scrapeOptions else {} - # Set default values for new parameters in v1 + # Set default values for crawl parameters. + # Note: firecrawl-py v2 renamed several options. "maxDepth" -> "max_discovery_depth" + # and "allowBackwardLinks" -> "crawl_entire_domain". params.setdefault("maxDepth", 2) params.setdefault("limit", 10000) params.setdefault("allowExternalLinks", False) params.setdefault("allowBackwardLinks", False) - params.setdefault("ignoreSitemap", False) params.setdefault("ignoreQueryParameters", False) - # Ensure onlyMainContent is explicitly set if not provided - if "scrapeOptions" in params: - params["scrapeOptions"].setdefault("onlyMainContent", True) - else: - params["scrapeOptions"] = {"onlyMainContent": True} + # Ensure onlyMainContent is explicitly set if not provided. + scrape_options_dict.setdefault("onlyMainContent", True) - if not self.idempotency_key: - self.idempotency_key = str(uuid.uuid4()) + # Translate legacy v1 camelCase option names to the firecrawl-py v2 keyword args. + kwargs = _to_snake_case_kwargs(params) + if "max_depth" in kwargs: + kwargs["max_discovery_depth"] = kwargs.pop("max_depth") + if "allow_backward_links" in kwargs: + kwargs["crawl_entire_domain"] = kwargs.pop("allow_backward_links") + # v2 removed "ignore_sitemap"; it is now the "sitemap" mode enum. + if kwargs.pop("ignore_sitemap", False): + kwargs["sitemap"] = "skip" - app = FirecrawlApp(api_key=self.api_key) - crawl_result = app.crawl_url(self.url, params=params, idempotency_key=self.idempotency_key) - return Data(data={"results": crawl_result}) + # Build the typed ScrapeOptions object expected by v2 from the (snake_cased) dict. + scrape_kwargs = _to_snake_case_kwargs(scrape_options_dict) + kwargs["scrape_options"] = ScrapeOptions(**scrape_kwargs) + + app = Firecrawl(api_key=self.api_key) + # v2 polls to completion and returns a typed CrawlJob object. + crawl_job = app.crawl(self.url, **kwargs) + return Data(data={"results": crawl_job.model_dump()}) diff --git a/src/lfx/src/lfx/components/firecrawl/firecrawl_map_api.py b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_map_api.py similarity index 71% rename from src/lfx/src/lfx/components/firecrawl/firecrawl_map_api.py rename to src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_map_api.py index 9f33514487..59c0361044 100644 --- a/src/lfx/src/lfx/components/firecrawl/firecrawl_map_api.py +++ b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_map_api.py @@ -53,7 +53,7 @@ class FirecrawlMapApi(Component): def map(self) -> Data: try: - from firecrawl import FirecrawlApp + from firecrawl import Firecrawl except ImportError as e: msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`." raise ImportError(msg) from e @@ -69,20 +69,25 @@ class FirecrawlMapApi(Component): msg = "No valid URLs provided" raise ValueError(msg) - params = { - "ignoreSitemap": self.ignore_sitemap, - "sitemapOnly": self.sitemap_only, - "includeSubdomains": self.include_subdomains, - } + # firecrawl-py v2 replaced the separate "ignoreSitemap"/"sitemapOnly" flags with a + # single "sitemap" mode: "only" (sitemap only), "skip" (ignore sitemap), or the + # default mixed behavior. + kwargs: dict = {"include_subdomains": self.include_subdomains} + if self.sitemap_only: + kwargs["sitemap"] = "only" + elif self.ignore_sitemap: + kwargs["sitemap"] = "skip" - app = FirecrawlApp(api_key=self.api_key) + app = Firecrawl(api_key=self.api_key) - # Map all provided URLs and combine results + # Map all provided URLs and combine results. combined_links = [] for url in urls: - result = app.map_url(url, params=params) - if isinstance(result, dict) and "links" in result: - combined_links.extend(result["links"]) + result = app.map(url, **kwargs) + # v2 returns a typed MapData object whose .links is a list of typed + # SearchResult objects; serialize each to a dict for downstream consumers. + links = getattr(result, "links", None) or [] + combined_links.extend(link.model_dump() if hasattr(link, "model_dump") else link for link in links) map_result = {"success": True, "links": combined_links} diff --git a/src/lfx/src/lfx/components/firecrawl/firecrawl_scrape_api.py b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_scrape_api.py similarity index 61% rename from src/lfx/src/lfx/components/firecrawl/firecrawl_scrape_api.py rename to src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_scrape_api.py index 718146ea6d..04c97627bb 100644 --- a/src/lfx/src/lfx/components/firecrawl/firecrawl_scrape_api.py +++ b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_scrape_api.py @@ -1,3 +1,5 @@ +import re + from lfx.custom.custom_component.component import Component from lfx.io import ( DataInput, @@ -8,6 +10,18 @@ from lfx.io import ( ) from lfx.schema.data import Data +_CAMEL_TO_SNAKE_RE = re.compile(r"(? dict: + """Convert camelCase option keys to snake_case keyword arguments. + + The firecrawl-py v1 convention uses camelCase, while the v2 SDK expects + snake_case keyword arguments. Keys that are already snake_case are passed + through unchanged. + """ + return {_CAMEL_TO_SNAKE_RE.sub("_", key).lower(): value for key, value in params.items()} + class FirecrawlScrapeApi(Component): display_name: str = "Firecrawl Scrape API" @@ -54,20 +68,30 @@ class FirecrawlScrapeApi(Component): def scrape(self) -> Data: try: - from firecrawl import FirecrawlApp + from firecrawl import Firecrawl except ImportError as e: msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`." raise ImportError(msg) from e - params = self.scrapeOptions.__dict__.get("data", {}) if self.scrapeOptions else {} - extractor_options_dict = self.extractorOptions.__dict__.get("data", {}) if self.extractorOptions else {} - if extractor_options_dict: - params["extract"] = extractor_options_dict + params = dict(self.scrapeOptions.__dict__.get("data", {})) if self.scrapeOptions else {} # Set default values for parameters params.setdefault("formats", ["markdown"]) # Default output format params.setdefault("onlyMainContent", True) # Default to only main content + if self.timeout: + params.setdefault("timeout", self.timeout) - app = FirecrawlApp(api_key=self.api_key) - results = app.scrape_url(self.url, params=params) - return Data(data=results) + kwargs = _to_snake_case_kwargs(params) + + # In firecrawl-py v2, structured extraction is requested via a "json" format entry + # on the scrape call rather than a separate extractor option. + extractor_options_dict = self.extractorOptions.__dict__.get("data", {}) if self.extractorOptions else {} + if extractor_options_dict: + formats = list(kwargs.get("formats", [])) + formats.append({"type": "json", **_to_snake_case_kwargs(extractor_options_dict)}) + kwargs["formats"] = formats + + app = Firecrawl(api_key=self.api_key) + document = app.scrape(self.url, **kwargs) + # v2 returns a typed Document object; serialize to a dict for downstream consumers. + return Data(data=document.model_dump()) diff --git a/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_search_api.py b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_search_api.py new file mode 100644 index 0000000000..aaaded0b6b --- /dev/null +++ b/src/bundles/firecrawl/src/lfx_firecrawl/components/firecrawl/firecrawl_search_api.py @@ -0,0 +1,76 @@ +from lfx.custom.custom_component.component import Component +from lfx.io import ( + IntInput, + MultilineInput, + Output, + SecretStrInput, + StrInput, +) +from lfx.schema.data import Data + + +class FirecrawlSearchApi(Component): + display_name: str = "Firecrawl Search API" + description: str = "Searches the web and returns the results." + name = "FirecrawlSearchApi" + + documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/search" + + inputs = [ + SecretStrInput( + name="api_key", + display_name="Firecrawl API Key", + required=True, + password=True, + info="The API key to use Firecrawl API.", + ), + MultilineInput( + name="query", + display_name="Query", + required=True, + info="The search query to run.", + tool_mode=True, + ), + IntInput( + name="limit", + display_name="Limit", + info="Maximum number of results to return.", + value=5, + ), + StrInput( + name="location", + display_name="Location", + info="Location to bias the search results (e.g. a country or region).", + advanced=True, + ), + ] + + outputs = [ + Output(display_name="JSON", name="data", method="search"), + ] + + def search(self) -> Data: + try: + from firecrawl import Firecrawl + except ImportError as e: + msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`." + raise ImportError(msg) from e + + if not self.query: + msg = "Query is required" + raise ValueError(msg) + + kwargs: dict = {} + if self.limit: + kwargs["limit"] = self.limit + if self.location: + kwargs["location"] = self.location + + app = Firecrawl(api_key=self.api_key) + result = app.search(self.query, **kwargs) + + # v2 returns a typed SearchData object (results grouped by source); + # serialize to a dict for downstream consumers. + search_result = result.model_dump() if hasattr(result, "model_dump") else result + + return Data(data=search_result) diff --git a/src/bundles/firecrawl/src/lfx_firecrawl/extension.json b/src/bundles/firecrawl/src/lfx_firecrawl/extension.json new file mode 100644 index 0000000000..8530f8fa81 --- /dev/null +++ b/src/bundles/firecrawl/src/lfx_firecrawl/extension.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://schemas.langflow.org/extension/v1.json", + "id": "lfx-firecrawl", + "version": "0.1.0", + "name": "Firecrawl", + "description": "Firecrawl components (Scrape, Crawl, Map, and Search APIs) as a standalone Langflow Extension Bundle.", + "lfx": { + "compat": ["1"] + }, + "bundles": [ + { + "name": "firecrawl", + "path": "components/firecrawl" + } + ] +} diff --git a/src/bundles/firecrawl/tests/test_firecrawl_components.py b/src/bundles/firecrawl/tests/test_firecrawl_components.py new file mode 100644 index 0000000000..c8ec7f2e2e --- /dev/null +++ b/src/bundles/firecrawl/tests/test_firecrawl_components.py @@ -0,0 +1,182 @@ +"""Unit tests for the Firecrawl extension bundle (``lfx-firecrawl``). + +The components used to live at ``lfx.components.firecrawl``; they have since +been extracted into a standalone bundle, so these tests travel with the +bundle and import the public bundle entry point. + +They verify v2 SDK usage, the v1->v2 parameter mapping, response +serialization, and error handling using a mocked Firecrawl SDK. No network +access or real ``firecrawl-py`` install is required: the SDK is injected via +``sys.modules`` so the components' in-method ``from firecrawl import ...`` +statements resolve to the mocks. +""" + +import sys +import types +from unittest.mock import MagicMock + +import pytest +from lfx_firecrawl import FirecrawlCrawlApi, FirecrawlMapApi, FirecrawlScrapeApi, FirecrawlSearchApi + + +@pytest.fixture +def mock_firecrawl(monkeypatch): + """Inject a fake ``firecrawl`` SDK and return the mocked ``Firecrawl`` class. + + ``mock_firecrawl.return_value`` is the client instance whose + ``.scrape``/``.crawl``/``.map``/``.extract``/``.search`` methods can be + stubbed per test. + """ + client_cls = MagicMock(name="Firecrawl") + + firecrawl_mod = types.ModuleType("firecrawl") + firecrawl_mod.Firecrawl = client_cls + + v2_mod = types.ModuleType("firecrawl.v2") + types_mod = types.ModuleType("firecrawl.v2.types") + types_mod.ScrapeOptions = MagicMock(name="ScrapeOptions") + v2_mod.types = types_mod + + monkeypatch.setitem(sys.modules, "firecrawl", firecrawl_mod) + monkeypatch.setitem(sys.modules, "firecrawl.v2", v2_mod) + monkeypatch.setitem(sys.modules, "firecrawl.v2.types", types_mod) + return client_cls + + +def _typed(dump): + """Return a fake v2 typed response whose ``model_dump()`` yields ``dump``.""" + obj = MagicMock() + obj.model_dump.return_value = dump + return obj + + +class TestFirecrawlScrapeApi: + def test_scrape_calls_v2_and_serializes(self, mock_firecrawl): + client = mock_firecrawl.return_value + client.scrape.return_value = _typed({"markdown": "# Hi", "metadata": {}}) + + component = FirecrawlScrapeApi() + component._attributes = { + "api_key": "test-key", # pragma: allowlist secret + "url": "https://example.com", + "timeout": 0, + "scrapeOptions": None, + "extractorOptions": None, + } + + result = component.scrape() + + mock_firecrawl.assert_called_once_with(api_key="test-key") # pragma: allowlist secret + client.scrape.assert_called_once() + assert client.scrape.call_args.args[0] == "https://example.com" + assert result.data == {"markdown": "# Hi", "metadata": {}} + + def test_scrape_propagates_sdk_error(self, mock_firecrawl): + client = mock_firecrawl.return_value + client.scrape.side_effect = RuntimeError("boom") + + component = FirecrawlScrapeApi() + component._attributes = { + "api_key": "k", # pragma: allowlist secret + "url": "https://example.com", + "timeout": 0, + "scrapeOptions": None, + "extractorOptions": None, + } + + with pytest.raises(RuntimeError): + component.scrape() + + +class TestFirecrawlCrawlApi: + def test_crawl_maps_legacy_params_and_serializes(self, mock_firecrawl): + client = mock_firecrawl.return_value + client.crawl.return_value = _typed({"status": "completed", "data": []}) + + component = FirecrawlCrawlApi() + component._attributes = { + "api_key": "test-key", # pragma: allowlist secret + "url": "https://example.com", + "timeout": 0, + "idempotency_key": "", + "crawlerOptions": None, + "scrapeOptions": None, + } + + result = component.crawl() + + mock_firecrawl.assert_called_once_with(api_key="test-key") # pragma: allowlist secret + client.crawl.assert_called_once() + assert client.crawl.call_args.args[0] == "https://example.com" + kwargs = client.crawl.call_args.kwargs + # v1 "maxDepth" was renamed to v2 "max_discovery_depth". + assert "max_discovery_depth" in kwargs + assert "max_depth" not in kwargs + # v1 "allowBackwardLinks" was renamed to v2 "crawl_entire_domain". + assert "crawl_entire_domain" in kwargs + assert result.data["results"] == {"status": "completed", "data": []} + + +class TestFirecrawlMapApi: + def test_map_sitemap_only_maps_to_enum(self, mock_firecrawl): + client = mock_firecrawl.return_value + result_obj = MagicMock() + result_obj.links = [_typed({"url": "https://example.com/a"})] + client.map.return_value = result_obj + + component = FirecrawlMapApi() + component._attributes = { + "api_key": "test-key", # pragma: allowlist secret + "urls": "https://example.com", + "ignore_sitemap": False, + "sitemap_only": True, + "include_subdomains": False, + } + + result = component.map() + + client.map.assert_called_once() + assert client.map.call_args.kwargs.get("sitemap") == "only" + assert result.data["links"] == [{"url": "https://example.com/a"}] + + def test_map_ignore_sitemap_maps_to_skip(self, mock_firecrawl): + client = mock_firecrawl.return_value + result_obj = MagicMock() + result_obj.links = [] + client.map.return_value = result_obj + + component = FirecrawlMapApi() + component._attributes = { + "api_key": "k", # pragma: allowlist secret + "urls": "https://example.com", + "ignore_sitemap": True, + "sitemap_only": False, + "include_subdomains": False, + } + + component.map() + + assert client.map.call_args.kwargs.get("sitemap") == "skip" + + +class TestFirecrawlSearchApi: + def test_search_calls_v2_with_query_limit_location(self, mock_firecrawl): + client = mock_firecrawl.return_value + client.search.return_value = _typed({"web": [{"url": "https://example.com"}]}) + + component = FirecrawlSearchApi() + component._attributes = { + "api_key": "test-key", # pragma: allowlist secret + "query": "firecrawl", + "limit": 5, + "location": "US", + } + + result = component.search() + + mock_firecrawl.assert_called_once_with(api_key="test-key") # pragma: allowlist secret + client.search.assert_called_once() + assert client.search.call_args.args[0] == "firecrawl" + assert client.search.call_args.kwargs.get("limit") == 5 + assert client.search.call_args.kwargs.get("location") == "US" + assert result.data == {"web": [{"url": "https://example.com"}]} diff --git a/src/frontend/src/utils/styleUtils.ts b/src/frontend/src/utils/styleUtils.ts index 9b3e5d3054..792bf7e9a0 100644 --- a/src/frontend/src/utils/styleUtils.ts +++ b/src/frontend/src/utils/styleUtils.ts @@ -620,9 +620,9 @@ export const nodeIconToDisplayIconMap: Record = { FAISS: "Meta", FaissSearch: "Meta", FirecrawlCrawlApi: "Firecrawl", - FirecrawlExtractApi: "Firecrawl", FirecrawlMapApi: "Firecrawl", FirecrawlScrapeApi: "Firecrawl", + FirecrawlSearchApi: "Firecrawl", GitbookLoader: "GitBook", GoogleGenerativeAI: "GoogleGenerativeAI", Googlesheets: "Googlesheets", diff --git a/src/lfx/src/lfx/_assets/component_index.json b/src/lfx/src/lfx/_assets/component_index.json index bce86b23c4..65cc46b042 100644 --- a/src/lfx/src/lfx/_assets/component_index.json +++ b/src/lfx/src/lfx/_assets/component_index.json @@ -71522,815 +71522,6 @@ } } ], - [ - "firecrawl", - { - "FirecrawlCrawlApi": { - "base_classes": [ - "JSON" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Crawls a URL and returns the results.", - "display_name": "Firecrawl Crawl API", - "documentation": "https://docs.firecrawl.dev/v1/api-reference/endpoint/crawl-post", - "edited": false, - "field_order": [ - "api_key", - "url", - "timeout", - "idempotency_key", - "crawlerOptions", - "scrapeOptions" - ], - "frozen": false, - "legacy": false, - "metadata": { - "code_hash": "21b4965f8b53", - "dependencies": { - "dependencies": [ - { - "name": "lfx", - "version": null - }, - { - "name": "firecrawl", - "version": "1.17.0" - } - ], - "total_dependencies": 2 - }, - "module": "lfx.components.firecrawl.firecrawl_crawl_api.FirecrawlCrawlApi" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "JSON", - "group_outputs": false, - "method": "crawl", - "name": "data", - "selected": "JSON", - "tool_mode": true, - "types": [ - "JSON" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "api_key": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Firecrawl API Key", - "dynamic": false, - "info": "The API key to use Firecrawl API.", - "input_types": [], - "load_from_db": true, - "name": "api_key", - "override_skip": false, - "password": true, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "import uuid\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.io import DataInput, IntInput, MultilineInput, Output, SecretStrInput, StrInput\nfrom lfx.schema.data import Data\n\n\nclass FirecrawlCrawlApi(Component):\n display_name: str = \"Firecrawl Crawl API\"\n description: str = \"Crawls a URL and returns the results.\"\n name = \"FirecrawlCrawlApi\"\n\n documentation: str = \"https://docs.firecrawl.dev/v1/api-reference/endpoint/crawl-post\"\n\n inputs = [\n SecretStrInput(\n name=\"api_key\",\n display_name=\"Firecrawl API Key\",\n required=True,\n password=True,\n info=\"The API key to use Firecrawl API.\",\n ),\n MultilineInput(\n name=\"url\",\n display_name=\"URL\",\n required=True,\n info=\"The URL to scrape.\",\n tool_mode=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout in milliseconds for the request.\",\n ),\n StrInput(\n name=\"idempotency_key\",\n display_name=\"Idempotency Key\",\n info=\"Optional idempotency key to ensure unique requests.\",\n ),\n DataInput(\n name=\"crawlerOptions\",\n display_name=\"Crawler Options\",\n info=\"The crawler options to send with the request.\",\n ),\n DataInput(\n name=\"scrapeOptions\",\n display_name=\"Scrape Options\",\n info=\"The page options to send with the request.\",\n ),\n ]\n\n outputs = [\n Output(display_name=\"JSON\", name=\"data\", method=\"crawl\"),\n ]\n idempotency_key: str | None = None\n\n def crawl(self) -> Data:\n try:\n from firecrawl import FirecrawlApp\n except ImportError as e:\n msg = \"Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`.\"\n raise ImportError(msg) from e\n\n params = self.crawlerOptions.__dict__[\"data\"] if self.crawlerOptions else {}\n scrape_options_dict = self.scrapeOptions.__dict__[\"data\"] if self.scrapeOptions else {}\n if scrape_options_dict:\n params[\"scrapeOptions\"] = scrape_options_dict\n\n # Set default values for new parameters in v1\n params.setdefault(\"maxDepth\", 2)\n params.setdefault(\"limit\", 10000)\n params.setdefault(\"allowExternalLinks\", False)\n params.setdefault(\"allowBackwardLinks\", False)\n params.setdefault(\"ignoreSitemap\", False)\n params.setdefault(\"ignoreQueryParameters\", False)\n\n # Ensure onlyMainContent is explicitly set if not provided\n if \"scrapeOptions\" in params:\n params[\"scrapeOptions\"].setdefault(\"onlyMainContent\", True)\n else:\n params[\"scrapeOptions\"] = {\"onlyMainContent\": True}\n\n if not self.idempotency_key:\n self.idempotency_key = str(uuid.uuid4())\n\n app = FirecrawlApp(api_key=self.api_key)\n crawl_result = app.crawl_url(self.url, params=params, idempotency_key=self.idempotency_key)\n return Data(data={\"results\": crawl_result})\n" - }, - "crawlerOptions": { - "_input_type": "JSONInput", - "advanced": false, - "display_name": "Crawler Options", - "dynamic": false, - "info": "The crawler options to send with the request.", - "input_types": [ - "Data", - "JSON" - ], - "list": false, - "list_add_label": "Add More", - "name": "crawlerOptions", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "other", - "value": "" - }, - "idempotency_key": { - "_input_type": "StrInput", - "advanced": false, - "display_name": "Idempotency Key", - "dynamic": false, - "info": "Optional idempotency key to ensure unique requests.", - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "idempotency_key", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "scrapeOptions": { - "_input_type": "JSONInput", - "advanced": false, - "display_name": "Scrape Options", - "dynamic": false, - "info": "The page options to send with the request.", - "input_types": [ - "Data", - "JSON" - ], - "list": false, - "list_add_label": "Add More", - "name": "scrapeOptions", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "other", - "value": "" - }, - "timeout": { - "_input_type": "IntInput", - "advanced": false, - "display_name": "Timeout", - "dynamic": false, - "info": "Timeout in milliseconds for the request.", - "list": false, - "list_add_label": "Add More", - "name": "timeout", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "int", - "value": 0 - }, - "url": { - "_input_type": "MultilineInput", - "advanced": false, - "ai_enabled": false, - "copy_field": false, - "display_name": "URL", - "dynamic": false, - "info": "The URL to scrape.", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "multiline": true, - "name": "url", - "override_skip": false, - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "tool_mode": true, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - } - }, - "tool_mode": false - }, - "FirecrawlExtractApi": { - "base_classes": [ - "JSON" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Extracts data from a URL.", - "display_name": "Firecrawl Extract API", - "documentation": "https://docs.firecrawl.dev/api-reference/endpoint/extract", - "edited": false, - "field_order": [ - "api_key", - "urls", - "prompt", - "schema", - "enable_web_search" - ], - "frozen": false, - "legacy": false, - "metadata": { - "code_hash": "1363b7da7bf7", - "dependencies": { - "dependencies": [ - { - "name": "lfx", - "version": null - }, - { - "name": "firecrawl", - "version": "1.17.0" - } - ], - "total_dependencies": 2 - }, - "module": "lfx.components.firecrawl.firecrawl_extract_api.FirecrawlExtractApi" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "JSON", - "group_outputs": false, - "method": "extract", - "name": "data", - "selected": "JSON", - "tool_mode": true, - "types": [ - "JSON" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "api_key": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Firecrawl API Key", - "dynamic": false, - "info": "The API key to use Firecrawl API.", - "input_types": [], - "load_from_db": true, - "name": "api_key", - "override_skip": false, - "password": true, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from lfx.custom.custom_component.component import Component\nfrom lfx.io import BoolInput, DataInput, MultilineInput, Output, SecretStrInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.data import Data\n\n\nclass FirecrawlExtractApi(Component):\n display_name: str = \"Firecrawl Extract API\"\n description: str = \"Extracts data from a URL.\"\n name = \"FirecrawlExtractApi\"\n\n documentation: str = \"https://docs.firecrawl.dev/api-reference/endpoint/extract\"\n\n inputs = [\n SecretStrInput(\n name=\"api_key\",\n display_name=\"Firecrawl API Key\",\n required=True,\n password=True,\n info=\"The API key to use Firecrawl API.\",\n ),\n MultilineInput(\n name=\"urls\",\n display_name=\"URLs\",\n required=True,\n info=\"List of URLs to extract data from (separated by commas or new lines).\",\n tool_mode=True,\n ),\n MultilineInput(\n name=\"prompt\",\n display_name=\"Prompt\",\n required=True,\n info=\"Prompt to guide the extraction process.\",\n tool_mode=True,\n ),\n DataInput(\n name=\"schema\",\n display_name=\"Schema\",\n required=False,\n info=\"Schema to define the structure of the extracted data.\",\n ),\n BoolInput(\n name=\"enable_web_search\",\n display_name=\"Enable Web Search\",\n info=\"When true, the extraction will use web search to find additional data.\",\n ),\n # # Optional: Not essential for basic extraction\n # BoolInput(\n # name=\"ignore_sitemap\",\n # display_name=\"Ignore Sitemap\",\n # info=\"When true, sitemap.xml files will be ignored during website scanning.\",\n # ),\n # # Optional: Not essential for basic extraction\n # BoolInput(\n # name=\"include_subdomains\",\n # display_name=\"Include Subdomains\",\n # info=\"When true, subdomains of the provided URLs will also be scanned.\",\n # ),\n # # Optional: Not essential for basic extraction\n # BoolInput(\n # name=\"show_sources\",\n # display_name=\"Show Sources\",\n # info=\"When true, the sources used to extract the data will be included in the response.\",\n # ),\n ]\n\n outputs = [\n Output(display_name=\"JSON\", name=\"data\", method=\"extract\"),\n ]\n\n def extract(self) -> Data:\n try:\n from firecrawl import FirecrawlApp\n except ImportError as e:\n msg = \"Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`.\"\n raise ImportError(msg) from e\n\n # Validate API key\n if not self.api_key:\n msg = \"API key is required\"\n raise ValueError(msg)\n\n # Validate URLs\n if not self.urls:\n msg = \"URLs are required\"\n raise ValueError(msg)\n\n # Split and validate URLs (handle both commas and newlines)\n urls = [url.strip() for url in self.urls.replace(\"\\n\", \",\").split(\",\") if url.strip()]\n if not urls:\n msg = \"No valid URLs provided\"\n raise ValueError(msg)\n\n # Validate and process prompt\n if not self.prompt:\n msg = \"Prompt is required\"\n raise ValueError(msg)\n\n # Get the prompt text (handling both string and multiline input)\n prompt_text = self.prompt.strip()\n\n # Enhance the prompt to encourage comprehensive extraction\n enhanced_prompt = prompt_text\n if \"schema\" not in prompt_text.lower():\n enhanced_prompt = f\"{prompt_text}. Please extract all instances in a comprehensive, structured format.\"\n\n params = {\n \"prompt\": enhanced_prompt,\n \"enableWebSearch\": self.enable_web_search,\n # Optional parameters - not essential for basic extraction\n \"ignoreSitemap\": self.ignore_sitemap,\n \"includeSubdomains\": self.include_subdomains,\n \"showSources\": self.show_sources,\n \"timeout\": 300,\n }\n\n # Only add schema to params if it's provided and is a valid schema structure\n if self.schema:\n try:\n if isinstance(self.schema, dict) and \"type\" in self.schema:\n params[\"schema\"] = self.schema\n elif hasattr(self.schema, \"dict\") and \"type\" in self.schema.dict():\n params[\"schema\"] = self.schema.dict()\n else:\n # Skip invalid schema without raising an error\n pass\n except Exception as e: # noqa: BLE001\n logger.error(f\"Invalid schema: {e!s}\")\n\n try:\n app = FirecrawlApp(api_key=self.api_key)\n extract_result = app.extract(urls, params=params)\n return Data(data=extract_result)\n except Exception as e:\n msg = f\"Error during extraction: {e!s}\"\n raise ValueError(msg) from e\n" - }, - "enable_web_search": { - "_input_type": "BoolInput", - "advanced": false, - "display_name": "Enable Web Search", - "dynamic": false, - "info": "When true, the extraction will use web search to find additional data.", - "list": false, - "list_add_label": "Add More", - "name": "enable_web_search", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": false - }, - "prompt": { - "_input_type": "MultilineInput", - "advanced": false, - "ai_enabled": false, - "copy_field": false, - "display_name": "Prompt", - "dynamic": false, - "info": "Prompt to guide the extraction process.", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "multiline": true, - "name": "prompt", - "override_skip": false, - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "tool_mode": true, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "schema": { - "_input_type": "JSONInput", - "advanced": false, - "display_name": "Schema", - "dynamic": false, - "info": "Schema to define the structure of the extracted data.", - "input_types": [ - "Data", - "JSON" - ], - "list": false, - "list_add_label": "Add More", - "name": "schema", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "other", - "value": "" - }, - "urls": { - "_input_type": "MultilineInput", - "advanced": false, - "ai_enabled": false, - "copy_field": false, - "display_name": "URLs", - "dynamic": false, - "info": "List of URLs to extract data from (separated by commas or new lines).", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "multiline": true, - "name": "urls", - "override_skip": false, - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "tool_mode": true, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - } - }, - "tool_mode": false - }, - "FirecrawlMapApi": { - "base_classes": [ - "JSON" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Maps a URL and returns the results.", - "display_name": "Firecrawl Map API", - "documentation": "https://docs.firecrawl.dev/api-reference/endpoint/map", - "edited": false, - "field_order": [ - "api_key", - "urls", - "ignore_sitemap", - "sitemap_only", - "include_subdomains" - ], - "frozen": false, - "legacy": false, - "metadata": { - "code_hash": "31e75312e67e", - "dependencies": { - "dependencies": [ - { - "name": "lfx", - "version": null - }, - { - "name": "firecrawl", - "version": "1.17.0" - } - ], - "total_dependencies": 2 - }, - "module": "lfx.components.firecrawl.firecrawl_map_api.FirecrawlMapApi" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "JSON", - "group_outputs": false, - "method": "map", - "name": "data", - "selected": "JSON", - "tool_mode": true, - "types": [ - "JSON" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "api_key": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Firecrawl API Key", - "dynamic": false, - "info": "The API key to use Firecrawl API.", - "input_types": [], - "load_from_db": true, - "name": "api_key", - "override_skip": false, - "password": true, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from lfx.custom.custom_component.component import Component\nfrom lfx.io import (\n BoolInput,\n MultilineInput,\n Output,\n SecretStrInput,\n)\nfrom lfx.schema.data import Data\n\n\nclass FirecrawlMapApi(Component):\n display_name: str = \"Firecrawl Map API\"\n description: str = \"Maps a URL and returns the results.\"\n name = \"FirecrawlMapApi\"\n\n documentation: str = \"https://docs.firecrawl.dev/api-reference/endpoint/map\"\n\n inputs = [\n SecretStrInput(\n name=\"api_key\",\n display_name=\"Firecrawl API Key\",\n required=True,\n password=True,\n info=\"The API key to use Firecrawl API.\",\n ),\n MultilineInput(\n name=\"urls\",\n display_name=\"URLs\",\n required=True,\n info=\"List of URLs to create maps from (separated by commas or new lines).\",\n tool_mode=True,\n ),\n BoolInput(\n name=\"ignore_sitemap\",\n display_name=\"Ignore Sitemap\",\n info=\"When true, the sitemap.xml file will be ignored during crawling.\",\n ),\n BoolInput(\n name=\"sitemap_only\",\n display_name=\"Sitemap Only\",\n info=\"When true, only links found in the sitemap will be returned.\",\n ),\n BoolInput(\n name=\"include_subdomains\",\n display_name=\"Include Subdomains\",\n info=\"When true, subdomains of the provided URL will also be scanned.\",\n ),\n ]\n\n outputs = [\n Output(display_name=\"JSON\", name=\"data\", method=\"map\"),\n ]\n\n def map(self) -> Data:\n try:\n from firecrawl import FirecrawlApp\n except ImportError as e:\n msg = \"Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`.\"\n raise ImportError(msg) from e\n\n # Validate URLs\n if not self.urls:\n msg = \"URLs are required\"\n raise ValueError(msg)\n\n # Split and validate URLs (handle both commas and newlines)\n urls = [url.strip() for url in self.urls.replace(\"\\n\", \",\").split(\",\") if url.strip()]\n if not urls:\n msg = \"No valid URLs provided\"\n raise ValueError(msg)\n\n params = {\n \"ignoreSitemap\": self.ignore_sitemap,\n \"sitemapOnly\": self.sitemap_only,\n \"includeSubdomains\": self.include_subdomains,\n }\n\n app = FirecrawlApp(api_key=self.api_key)\n\n # Map all provided URLs and combine results\n combined_links = []\n for url in urls:\n result = app.map_url(url, params=params)\n if isinstance(result, dict) and \"links\" in result:\n combined_links.extend(result[\"links\"])\n\n map_result = {\"success\": True, \"links\": combined_links}\n\n return Data(data=map_result)\n" - }, - "ignore_sitemap": { - "_input_type": "BoolInput", - "advanced": false, - "display_name": "Ignore Sitemap", - "dynamic": false, - "info": "When true, the sitemap.xml file will be ignored during crawling.", - "list": false, - "list_add_label": "Add More", - "name": "ignore_sitemap", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": false - }, - "include_subdomains": { - "_input_type": "BoolInput", - "advanced": false, - "display_name": "Include Subdomains", - "dynamic": false, - "info": "When true, subdomains of the provided URL will also be scanned.", - "list": false, - "list_add_label": "Add More", - "name": "include_subdomains", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": false - }, - "sitemap_only": { - "_input_type": "BoolInput", - "advanced": false, - "display_name": "Sitemap Only", - "dynamic": false, - "info": "When true, only links found in the sitemap will be returned.", - "list": false, - "list_add_label": "Add More", - "name": "sitemap_only", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": false - }, - "urls": { - "_input_type": "MultilineInput", - "advanced": false, - "ai_enabled": false, - "copy_field": false, - "display_name": "URLs", - "dynamic": false, - "info": "List of URLs to create maps from (separated by commas or new lines).", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "multiline": true, - "name": "urls", - "override_skip": false, - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "tool_mode": true, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - } - }, - "tool_mode": false - }, - "FirecrawlScrapeApi": { - "base_classes": [ - "JSON" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Scrapes a URL and returns the results.", - "display_name": "Firecrawl Scrape API", - "documentation": "https://docs.firecrawl.dev/api-reference/endpoint/scrape", - "edited": false, - "field_order": [ - "api_key", - "url", - "timeout", - "scrapeOptions", - "extractorOptions" - ], - "frozen": false, - "legacy": false, - "metadata": { - "code_hash": "a56c999d7a42", - "dependencies": { - "dependencies": [ - { - "name": "lfx", - "version": null - }, - { - "name": "firecrawl", - "version": "1.17.0" - } - ], - "total_dependencies": 2 - }, - "module": "lfx.components.firecrawl.firecrawl_scrape_api.FirecrawlScrapeApi" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "JSON", - "group_outputs": false, - "method": "scrape", - "name": "data", - "selected": "JSON", - "tool_mode": true, - "types": [ - "JSON" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "api_key": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Firecrawl API Key", - "dynamic": false, - "info": "The API key to use Firecrawl API.", - "input_types": [], - "load_from_db": true, - "name": "api_key", - "override_skip": false, - "password": true, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from lfx.custom.custom_component.component import Component\nfrom lfx.io import (\n DataInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n)\nfrom lfx.schema.data import Data\n\n\nclass FirecrawlScrapeApi(Component):\n display_name: str = \"Firecrawl Scrape API\"\n description: str = \"Scrapes a URL and returns the results.\"\n name = \"FirecrawlScrapeApi\"\n\n documentation: str = \"https://docs.firecrawl.dev/api-reference/endpoint/scrape\"\n\n inputs = [\n SecretStrInput(\n name=\"api_key\",\n display_name=\"Firecrawl API Key\",\n required=True,\n password=True,\n info=\"The API key to use Firecrawl API.\",\n ),\n MultilineInput(\n name=\"url\",\n display_name=\"URL\",\n required=True,\n info=\"The URL to scrape.\",\n tool_mode=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout in milliseconds for the request.\",\n ),\n DataInput(\n name=\"scrapeOptions\",\n display_name=\"Scrape Options\",\n info=\"The page options to send with the request.\",\n ),\n DataInput(\n name=\"extractorOptions\",\n display_name=\"Extractor Options\",\n info=\"The extractor options to send with the request.\",\n ),\n ]\n\n outputs = [\n Output(display_name=\"JSON\", name=\"data\", method=\"scrape\"),\n ]\n\n def scrape(self) -> Data:\n try:\n from firecrawl import FirecrawlApp\n except ImportError as e:\n msg = \"Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`.\"\n raise ImportError(msg) from e\n\n params = self.scrapeOptions.__dict__.get(\"data\", {}) if self.scrapeOptions else {}\n extractor_options_dict = self.extractorOptions.__dict__.get(\"data\", {}) if self.extractorOptions else {}\n if extractor_options_dict:\n params[\"extract\"] = extractor_options_dict\n\n # Set default values for parameters\n params.setdefault(\"formats\", [\"markdown\"]) # Default output format\n params.setdefault(\"onlyMainContent\", True) # Default to only main content\n\n app = FirecrawlApp(api_key=self.api_key)\n results = app.scrape_url(self.url, params=params)\n return Data(data=results)\n" - }, - "extractorOptions": { - "_input_type": "JSONInput", - "advanced": false, - "display_name": "Extractor Options", - "dynamic": false, - "info": "The extractor options to send with the request.", - "input_types": [ - "Data", - "JSON" - ], - "list": false, - "list_add_label": "Add More", - "name": "extractorOptions", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "other", - "value": "" - }, - "scrapeOptions": { - "_input_type": "JSONInput", - "advanced": false, - "display_name": "Scrape Options", - "dynamic": false, - "info": "The page options to send with the request.", - "input_types": [ - "Data", - "JSON" - ], - "list": false, - "list_add_label": "Add More", - "name": "scrapeOptions", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "other", - "value": "" - }, - "timeout": { - "_input_type": "IntInput", - "advanced": false, - "display_name": "Timeout", - "dynamic": false, - "info": "Timeout in milliseconds for the request.", - "list": false, - "list_add_label": "Add More", - "name": "timeout", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "int", - "value": 0 - }, - "url": { - "_input_type": "MultilineInput", - "advanced": false, - "ai_enabled": false, - "copy_field": false, - "display_name": "URL", - "dynamic": false, - "info": "The URL to scrape.", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "multiline": true, - "name": "url", - "override_skip": false, - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "tool_mode": true, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - } - }, - "tool_mode": false - } - } - ], [ "flow_controls", { @@ -118451,9 +117642,9 @@ ] ], "metadata": { - "num_components": 354, - "num_modules": 95 + "num_components": 350, + "num_modules": 94 }, - "sha256": "87d65557fa5c4182f09f07270bbe8624d5f9b65a0322c3251f75822b6f1546ac", + "sha256": "0d7e995401cee6bc0ad9f46fd036e18bbcec09c37d5db9d8e38bd3b350a3d7f6", "version": "1.11.0" } diff --git a/src/lfx/src/lfx/components/__init__.py b/src/lfx/src/lfx/components/__init__.py index 10554a0b47..8befe38859 100644 --- a/src/lfx/src/lfx/components/__init__.py +++ b/src/lfx/src/lfx/components/__init__.py @@ -42,7 +42,6 @@ if TYPE_CHECKING: embeddings, exa, files_ingestion, - firecrawl, git, glean, google, @@ -143,7 +142,6 @@ _dynamic_imports = { "embeddings": "__module__", "exa": "__module__", "FAISS": "__module__", - "firecrawl": "__module__", "files_ingestion": "__module__", "git": "__module__", "glean": "__module__", @@ -274,7 +272,6 @@ __all__ = [ "embeddings", "exa", "files_ingestion", - "firecrawl", "git", "glean", "google", diff --git a/src/lfx/src/lfx/components/firecrawl/__init__.py b/src/lfx/src/lfx/components/firecrawl/__init__.py deleted file mode 100644 index b1e754d7ab..0000000000 --- a/src/lfx/src/lfx/components/firecrawl/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from lfx.components._importing import import_mod - -if TYPE_CHECKING: - from .firecrawl_crawl_api import FirecrawlCrawlApi - from .firecrawl_extract_api import FirecrawlExtractApi - from .firecrawl_map_api import FirecrawlMapApi - from .firecrawl_scrape_api import FirecrawlScrapeApi - -_dynamic_imports = { - "FirecrawlCrawlApi": "firecrawl_crawl_api", - "FirecrawlExtractApi": "firecrawl_extract_api", - "FirecrawlMapApi": "firecrawl_map_api", - "FirecrawlScrapeApi": "firecrawl_scrape_api", -} - -__all__ = [ - "FirecrawlCrawlApi", - "FirecrawlExtractApi", - "FirecrawlMapApi", - "FirecrawlScrapeApi", -] - - -def __getattr__(attr_name: str) -> Any: - """Lazily import firecrawl components on attribute access.""" - if attr_name not in _dynamic_imports: - msg = f"module '{__name__}' has no attribute '{attr_name}'" - raise AttributeError(msg) - try: - result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) - except (ModuleNotFoundError, ImportError, AttributeError) as e: - msg = f"Could not import '{attr_name}' from '{__name__}': {e}" - raise AttributeError(msg) from e - globals()[attr_name] = result - return result - - -def __dir__() -> list[str]: - return list(__all__) diff --git a/src/lfx/src/lfx/components/firecrawl/firecrawl_extract_api.py b/src/lfx/src/lfx/components/firecrawl/firecrawl_extract_api.py deleted file mode 100644 index 107c6cbc65..0000000000 --- a/src/lfx/src/lfx/components/firecrawl/firecrawl_extract_api.py +++ /dev/null @@ -1,136 +0,0 @@ -from lfx.custom.custom_component.component import Component -from lfx.io import BoolInput, DataInput, MultilineInput, Output, SecretStrInput -from lfx.log.logger import logger -from lfx.schema.data import Data - - -class FirecrawlExtractApi(Component): - display_name: str = "Firecrawl Extract API" - description: str = "Extracts data from a URL." - name = "FirecrawlExtractApi" - - documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/extract" - - inputs = [ - SecretStrInput( - name="api_key", - display_name="Firecrawl API Key", - required=True, - password=True, - info="The API key to use Firecrawl API.", - ), - MultilineInput( - name="urls", - display_name="URLs", - required=True, - info="List of URLs to extract data from (separated by commas or new lines).", - tool_mode=True, - ), - MultilineInput( - name="prompt", - display_name="Prompt", - required=True, - info="Prompt to guide the extraction process.", - tool_mode=True, - ), - DataInput( - name="schema", - display_name="Schema", - required=False, - info="Schema to define the structure of the extracted data.", - ), - BoolInput( - name="enable_web_search", - display_name="Enable Web Search", - info="When true, the extraction will use web search to find additional data.", - ), - # # Optional: Not essential for basic extraction - # BoolInput( - # name="ignore_sitemap", - # display_name="Ignore Sitemap", - # info="When true, sitemap.xml files will be ignored during website scanning.", - # ), - # # Optional: Not essential for basic extraction - # BoolInput( - # name="include_subdomains", - # display_name="Include Subdomains", - # info="When true, subdomains of the provided URLs will also be scanned.", - # ), - # # Optional: Not essential for basic extraction - # BoolInput( - # name="show_sources", - # display_name="Show Sources", - # info="When true, the sources used to extract the data will be included in the response.", - # ), - ] - - outputs = [ - Output(display_name="JSON", name="data", method="extract"), - ] - - def extract(self) -> Data: - try: - from firecrawl import FirecrawlApp - except ImportError as e: - msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`." - raise ImportError(msg) from e - - # Validate API key - if not self.api_key: - msg = "API key is required" - raise ValueError(msg) - - # Validate URLs - if not self.urls: - msg = "URLs are required" - raise ValueError(msg) - - # Split and validate URLs (handle both commas and newlines) - urls = [url.strip() for url in self.urls.replace("\n", ",").split(",") if url.strip()] - if not urls: - msg = "No valid URLs provided" - raise ValueError(msg) - - # Validate and process prompt - if not self.prompt: - msg = "Prompt is required" - raise ValueError(msg) - - # Get the prompt text (handling both string and multiline input) - prompt_text = self.prompt.strip() - - # Enhance the prompt to encourage comprehensive extraction - enhanced_prompt = prompt_text - if "schema" not in prompt_text.lower(): - enhanced_prompt = f"{prompt_text}. Please extract all instances in a comprehensive, structured format." - - params = { - "prompt": enhanced_prompt, - "enableWebSearch": self.enable_web_search, - # Optional parameters - not essential for basic extraction - "ignoreSitemap": self.ignore_sitemap, - "includeSubdomains": self.include_subdomains, - "showSources": self.show_sources, - "timeout": 300, - } - - # Only add schema to params if it's provided and is a valid schema structure - if self.schema: - try: - if isinstance(self.schema, dict) and "type" in self.schema: - params["schema"] = self.schema - elif hasattr(self.schema, "dict") and "type" in self.schema.dict(): - params["schema"] = self.schema.dict() - else: - # Skip invalid schema without raising an error - pass - except Exception as e: # noqa: BLE001 - logger.error(f"Invalid schema: {e!s}") - - try: - app = FirecrawlApp(api_key=self.api_key) - extract_result = app.extract(urls, params=params) - return Data(data=extract_result) - except Exception as e: - msg = f"Error during extraction: {e!s}" - raise ValueError(msg) from e diff --git a/src/lfx/src/lfx/extension/migration/migration_table.json b/src/lfx/src/lfx/extension/migration/migration_table.json index 7798cb6c8f..2ca0990632 100644 --- a/src/lfx/src/lfx/extension/migration/migration_table.json +++ b/src/lfx/src/lfx/extension/migration/migration_table.json @@ -181,6 +181,86 @@ "target": "ext:docling:ExportDoclingDocumentComponent@official", "added_in": "1.10.0" }, + { + "bare_class_name": "FirecrawlCrawlApi", + "target": "ext:firecrawl:FirecrawlCrawlApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.firecrawl_crawl_api.FirecrawlCrawlApi", + "target": "ext:firecrawl:FirecrawlCrawlApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.FirecrawlCrawlApi", + "target": "ext:firecrawl:FirecrawlCrawlApi@official", + "added_in": "1.11.0" + }, + { + "legacy_slot": "ext:firecrawl:FirecrawlCrawlApi@official-pre-a", + "target": "ext:firecrawl:FirecrawlCrawlApi@official", + "added_in": "1.11.0" + }, + { + "bare_class_name": "FirecrawlMapApi", + "target": "ext:firecrawl:FirecrawlMapApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.firecrawl_map_api.FirecrawlMapApi", + "target": "ext:firecrawl:FirecrawlMapApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.FirecrawlMapApi", + "target": "ext:firecrawl:FirecrawlMapApi@official", + "added_in": "1.11.0" + }, + { + "legacy_slot": "ext:firecrawl:FirecrawlMapApi@official-pre-a", + "target": "ext:firecrawl:FirecrawlMapApi@official", + "added_in": "1.11.0" + }, + { + "bare_class_name": "FirecrawlScrapeApi", + "target": "ext:firecrawl:FirecrawlScrapeApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.firecrawl_scrape_api.FirecrawlScrapeApi", + "target": "ext:firecrawl:FirecrawlScrapeApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.FirecrawlScrapeApi", + "target": "ext:firecrawl:FirecrawlScrapeApi@official", + "added_in": "1.11.0" + }, + { + "legacy_slot": "ext:firecrawl:FirecrawlScrapeApi@official-pre-a", + "target": "ext:firecrawl:FirecrawlScrapeApi@official", + "added_in": "1.11.0" + }, + { + "bare_class_name": "FirecrawlSearchApi", + "target": "ext:firecrawl:FirecrawlSearchApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.firecrawl_search_api.FirecrawlSearchApi", + "target": "ext:firecrawl:FirecrawlSearchApi@official", + "added_in": "1.11.0" + }, + { + "import_path": "lfx.components.firecrawl.FirecrawlSearchApi", + "target": "ext:firecrawl:FirecrawlSearchApi@official", + "added_in": "1.11.0" + }, + { + "legacy_slot": "ext:firecrawl:FirecrawlSearchApi@official-pre-a", + "target": "ext:firecrawl:FirecrawlSearchApi@official", + "added_in": "1.11.0" + }, { "bare_class_name": "NextPlaidVectorStoreComponent", "target": "ext:nextplaid:NextPlaidVectorStoreComponent@official", diff --git a/src/lfx/tests/integration/extension/test_pilot_firecrawl_upgrade.py b/src/lfx/tests/integration/extension/test_pilot_firecrawl_upgrade.py new file mode 100644 index 0000000000..cfdcd51a3a --- /dev/null +++ b/src/lfx/tests/integration/extension/test_pilot_firecrawl_upgrade.py @@ -0,0 +1,128 @@ +"""Integration test: legacy firecrawl flows upgrade cleanly. + +Mirrors ``test_pilot_duckduckgo_upgrade.py`` for ``FirecrawlCrawlApi``. +""" + +from __future__ import annotations + +import json +from importlib import metadata as importlib_metadata +from pathlib import Path + +import pytest +from lfx.extension.migration.loader import load_migration_table + +REPO_ROOT = Path(__file__).resolve().parents[5] +TABLE_PATH = REPO_ROOT / "src" / "lfx" / "src" / "lfx" / "extension" / "migration" / "migration_table.json" +EXPECTED_TARGET = "ext:firecrawl:FirecrawlCrawlApi@official" + + +@pytest.fixture(scope="module") +def migration_table(): + table, error = load_migration_table(TABLE_PATH) + assert error is None, f"failed to load migration table: {error}" + assert table is not None + return table + + +def _saved_flow_node(node_id: str, type_value: str) -> dict: + """Build a minimal saved-flow node skeleton for testing.""" + return { + "id": node_id, + "type": "genericNode", + "data": {"id": node_id, "type": type_value, "node": {"template": {}}}, + } + + +def _saved_flow(*nodes: dict) -> dict: + return {"data": {"nodes": list(nodes), "edges": []}} + + +@pytest.mark.integration +def test_legacy_bare_name_flow_upgrades(migration_table) -> None: + """Pre-Phase-A flow with the bare class name upgrades to the canonical ID.""" + from lfx.extension.migration.rewrite import migrate_flow_payload + + flow = _saved_flow(_saved_flow_node("firecrawl-1", "FirecrawlCrawlApi")) + report = migrate_flow_payload(flow, table=migration_table) + + assert report.rewritten_count == 1 + assert flow["data"]["nodes"][0]["data"]["type"] == EXPECTED_TARGET + [record] = report.records + assert record.legacy_form_kind == "bare_class_name" + assert record.new_value == EXPECTED_TARGET + + +@pytest.mark.integration +def test_legacy_import_path_flow_upgrades(migration_table) -> None: + """Dotted import-path form upgrades to the canonical ID.""" + from lfx.extension.migration.rewrite import migrate_flow_payload + + flow = _saved_flow( + _saved_flow_node("firecrawl-2", "lfx.components.firecrawl.firecrawl_crawl_api.FirecrawlCrawlApi") + ) + report = migrate_flow_payload(flow, table=migration_table) + + assert report.rewritten_count == 1 + assert flow["data"]["nodes"][0]["data"]["type"] == EXPECTED_TARGET + assert report.records[0].legacy_form_kind == "import_path" + + +@pytest.mark.integration +def test_short_import_path_flow_upgrades(migration_table) -> None: + """Package-level import-path form (via ``__init__.py`` re-export) upgrades.""" + from lfx.extension.migration.rewrite import migrate_flow_payload + + flow = _saved_flow(_saved_flow_node("firecrawl-3", "lfx.components.firecrawl.FirecrawlCrawlApi")) + report = migrate_flow_payload(flow, table=migration_table) + + assert report.rewritten_count == 1 + assert flow["data"]["nodes"][0]["data"]["type"] == EXPECTED_TARGET + + +@pytest.mark.integration +def test_lfx_firecrawl_distribution_is_importable() -> None: + """The bundle's package is importable in the development workspace.""" + try: + from lfx_firecrawl import FirecrawlCrawlApi + except ImportError: + pytest.skip("lfx-firecrawl not installed in this test environment") + + assert FirecrawlCrawlApi.__name__ == "FirecrawlCrawlApi" + + +def _is_editable_install(dist: importlib_metadata.Distribution) -> bool: + direct_url = dist.read_text("direct_url.json") + if not direct_url: + return False + try: + payload = json.loads(direct_url) + except json.JSONDecodeError: + return False + return bool(payload.get("dir_info", {}).get("editable")) + + +@pytest.mark.integration +def test_lfx_firecrawl_ships_manifest() -> None: + """``importlib.metadata`` can find ``extension.json`` for the installed dist.""" + try: + dist = importlib_metadata.distribution("lfx-firecrawl") + except importlib_metadata.PackageNotFoundError: + pytest.skip("lfx-firecrawl not installed in this test environment") + + if _is_editable_install(dist): + import lfx_firecrawl + + package_dir = Path(lfx_firecrawl.__file__).parent + manifest_path = package_dir / "extension.json" + assert manifest_path.is_file() + else: + files = dist.files or [] + manifests = [f for f in files if f.parts and f.parts[-1] == "extension.json"] + assert manifests + manifest_path = Path(dist.locate_file(manifests[0])) + + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert manifest["id"] == "lfx-firecrawl" + assert manifest["lfx"]["compat"] == ["1"] + assert any(b["name"] == "firecrawl" for b in manifest["bundles"]) diff --git a/uv.lock b/uv.lock index e2e9abccdf..243832b032 100644 --- a/uv.lock +++ b/uv.lock @@ -33,6 +33,7 @@ members = [ "lfx-arxiv", "lfx-docling", "lfx-duckduckgo", + "lfx-firecrawl", "lfx-ibm", "lfx-nextplaid", ] @@ -4260,18 +4261,20 @@ wheels = [ [[package]] name = "firecrawl-py" -version = "1.17.0" +version = "4.28.2" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "aiohttp" }, + { name = "httpx" }, { name = "nest-asyncio" }, { name = "pydantic" }, { name = "python-dotenv" }, { name = "requests" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4d/08/61df5c8357742d8bee504db166b633ef7972b40c831e8c2f2a26feb9021c/firecrawl_py-1.17.0.tar.gz", hash = "sha256:5e2f50ec1f0e67514cdf6f0afc7df6be36eb8277fbec9e1f5a283fc01fae7875", size = 22191, upload-time = "2025-04-17T18:13:01.068Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/09/e7c667875c9a9dac09480e2f69fd46bdd9c1139555b835c3036a9858aff2/firecrawl_py-4.28.2.tar.gz", hash = "sha256:7e6181e2129b63c8d6aec5728d9b2fcf16ea82cb854372ad824b278efd258696", size = 191869, upload-time = "2026-05-28T00:03:48.822Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/53/a6/9431b62571b4ff4cd0d09f70c7df1253abf611e829f68d194e1ea69ea57e/firecrawl_py-1.17.0-py3-none-any.whl", hash = "sha256:0392822fbd906731f4c0876f91a9c3cce7624279c81948e4e3f8bc60b4e1c855", size = 21295, upload-time = "2025-04-17T18:12:59.463Z" }, + { url = "https://files.pythonhosted.org/packages/80/08/384ca104c8004ca0541aacd3470e855bd28666d6bdb2101e08446dfcb407/firecrawl_py-4.28.2-py3-none-any.whl", hash = "sha256:0689080cb01672370e5a97963e0df479f6102137aa088857eac0fa287a4269b6", size = 239166, upload-time = "2026-05-28T00:03:47.479Z" }, ] [[package]] @@ -7421,6 +7424,7 @@ dependencies = [ { name = "lfx-arxiv" }, { name = "lfx-docling" }, { name = "lfx-duckduckgo" }, + { name = "lfx-firecrawl" }, { name = "lfx-ibm" }, { name = "lfx-nextplaid" }, ] @@ -7515,6 +7519,7 @@ requires-dist = [ { name = "lfx-docling", extras = ["image-description"], marker = "extra == 'docling-image-description'", editable = "src/bundles/docling" }, { name = "lfx-docling", extras = ["local"], marker = "extra == 'docling'", editable = "src/bundles/docling" }, { name = "lfx-duckduckgo", editable = "src/bundles/duckduckgo" }, + { name = "lfx-firecrawl", editable = "src/bundles/firecrawl" }, { name = "lfx-ibm", editable = "src/bundles/ibm" }, { name = "lfx-nextplaid", editable = "src/bundles/nextplaid" }, { name = "nv-ingest-api", marker = "python_full_version >= '3.12' and extra == 'nv-ingest'", specifier = ">=26.1.0,<27.0.0" }, @@ -7596,7 +7601,6 @@ dependencies = [ { name = "fastapi" }, { name = "fastapi-pagination" }, { name = "filelock" }, - { name = "firecrawl-py" }, { name = "grandalf" }, { name = "greenlet" }, { name = "gunicorn" }, @@ -8347,7 +8351,6 @@ requires-dist = [ { name = "fastmcp", marker = "extra == 'fastmcp'", specifier = ">=3.2.0" }, { name = "fastparquet", marker = "extra == 'fastparquet'", specifier = ">=2024.11.0,<2025.0.0" }, { name = "filelock", specifier = ">=3.20.1,<4.0.0" }, - { name = "firecrawl-py", specifier = ">=1.0.16,<2.0.0" }, { name = "gassist", marker = "sys_platform == 'win32' and extra == 'gassist'", specifier = ">=0.0.1" }, { name = "gitpython", marker = "extra == 'gitpython'", specifier = ">=3.1.50" }, { name = "google-api-python-client", marker = "extra == 'google'", specifier = "~=2.161" }, @@ -9155,6 +9158,21 @@ requires-dist = [ { name = "lfx", editable = "src/lfx" }, ] +[[package]] +name = "lfx-firecrawl" +version = "0.1.0" +source = { editable = "src/bundles/firecrawl" } +dependencies = [ + { name = "firecrawl-py" }, + { name = "lfx" }, +] + +[package.metadata] +requires-dist = [ + { name = "firecrawl-py", specifier = ">=4.0.0,<5.0.0" }, + { name = "lfx", editable = "src/lfx" }, +] + [[package]] name = "lfx-ibm" version = "0.1.1"