diff --git a/src/backend/tests/unit/components/data/test_news_search.py b/src/backend/tests/unit/components/data/test_news_search.py deleted file mode 100644 index 93ca1be088..0000000000 --- a/src/backend/tests/unit/components/data/test_news_search.py +++ /dev/null @@ -1,88 +0,0 @@ -from unittest.mock import Mock, patch - -import pytest -import requests -from lfx.components.data.news_search import NewsSearchComponent -from lfx.schema import DataFrame - -from tests.base import ComponentTestBaseWithoutClient - - -class TestNewsSearchComponent(ComponentTestBaseWithoutClient): - @pytest.fixture - def component_class(self): - return NewsSearchComponent - - @pytest.fixture - def default_kwargs(self): - return {"query": "OpenAI"} - - @pytest.fixture - def file_names_mapping(self): - return [] - - def test_successful_news_search(self): - # Mock Google News RSS feed content - mock_rss_content = """ - - - - - Test News 1 - https://example.com/1 - 2024-03-20 - Summary 1 - - - Test News 2 - https://example.com/2 - 2024-03-21 - Summary 2 - - - - """ - mock_response = Mock() - mock_response.content = mock_rss_content.encode("utf-8") - mock_response.raise_for_status = Mock() - - with patch("requests.get", return_value=mock_response): - component = NewsSearchComponent(query="OpenAI") - result = component.search_news() - assert isinstance(result, DataFrame) - news_results_df = result - assert len(news_results_df) == 2 - assert list(news_results_df.columns) == ["title", "link", "published", "summary"] - assert news_results_df.iloc[0]["title"] == "Test News 1" - assert news_results_df.iloc[1]["title"] == "Test News 2" - - def test_news_search_error(self): - with patch("requests.get", side_effect=requests.RequestException("Network error")): - component = NewsSearchComponent(query="OpenAI") - result = component.search_news() - assert isinstance(result, DataFrame) - news_results_df = result - assert len(news_results_df) == 1 - assert news_results_df.iloc[0]["title"] == "Error" - assert "Network error" in news_results_df.iloc[0]["summary"] - - def test_empty_news_results(self): - # Mock empty RSS feed - mock_rss_content = """ - - - - - - """ - mock_response = Mock() - mock_response.content = mock_rss_content.encode("utf-8") - mock_response.raise_for_status = Mock() - - with patch("requests.get", return_value=mock_response): - component = NewsSearchComponent(query="OpenAI") - result = component.search_news() - assert isinstance(result, DataFrame) - news_results_df = result - assert len(news_results_df) == 1 - assert news_results_df.iloc[0]["title"] == "No articles found" diff --git a/src/backend/tests/unit/components/data/test_rss.py b/src/backend/tests/unit/components/data/test_rss.py deleted file mode 100644 index ae02a4f1ea..0000000000 --- a/src/backend/tests/unit/components/data/test_rss.py +++ /dev/null @@ -1,129 +0,0 @@ -from unittest.mock import Mock, patch - -import pytest -import requests -from lfx.components.data.rss import RSSReaderComponent -from lfx.schema import DataFrame - -from tests.base import ComponentTestBaseWithoutClient - - -class TestRSSReaderComponent(ComponentTestBaseWithoutClient): - @pytest.fixture - def component_class(self): - """Return the component class to test.""" - return RSSReaderComponent - - @pytest.fixture - def default_kwargs(self): - """Return the default kwargs for the component.""" - return { - "rss_url": "https://example.com/feed.xml", - } - - @pytest.fixture - def file_names_mapping(self): - """Return an empty list since this component doesn't have version-specific files.""" - return [] - - def test_successful_rss_fetch(self): - # Mock RSS feed content - mock_rss_content = """ - - - - - Test Article 1 - https://example.com/1 - 2024-03-20 - Test summary 1 - - - Test Article 2 - https://example.com/2 - 2024-03-21 - Test summary 2 - - - - """ - - # Mock the requests.get response - mock_response = Mock() - mock_response.content = mock_rss_content.encode("utf-8") - mock_response.raise_for_status = Mock() - - with patch("requests.get", return_value=mock_response): - component = RSSReaderComponent(rss_url="https://example.com/feed.xml") - result = component.read_rss() - - assert isinstance(result, DataFrame) - assert len(result) == 2 - assert list(result.columns) == ["title", "link", "published", "summary"] - assert result.iloc[0]["title"] == "Test Article 1" - assert result.iloc[1]["title"] == "Test Article 2" - - def test_rss_fetch_with_missing_fields(self): - # Mock RSS feed content with missing fields - mock_rss_content = """ - - - - - Test Article - - 2024-03-20 - - - - - """ - - mock_response = Mock() - mock_response.content = mock_rss_content.encode("utf-8") - mock_response.raise_for_status = Mock() - - with patch("requests.get", return_value=mock_response): - component = RSSReaderComponent(rss_url="https://example.com/feed.xml") - result = component.read_rss() - - assert isinstance(result, DataFrame) - assert len(result) == 1 - assert result.iloc[0]["title"] == "Test Article" - assert result.iloc[0]["link"] == "" - assert result.iloc[0]["summary"] == "" - - def test_rss_fetch_error(self): - # Mock a failed request - with patch("requests.get", side_effect=requests.RequestException("Network error")): - component = RSSReaderComponent(rss_url="https://example.com/feed.xml") - result = component.read_rss() - - assert isinstance(result, DataFrame) - assert len(result) == 1 - assert result.iloc[0]["title"] == "Error" - assert result.iloc[0]["link"] == "" - assert result.iloc[0]["published"] == "" - assert "Network error" in result.iloc[0]["summary"] - - def test_empty_rss_feed(self): - # Mock empty RSS feed - mock_rss_content = """ - - - - - - """ - - mock_response = Mock() - mock_response.content = mock_rss_content.encode("utf-8") - mock_response.raise_for_status = Mock() - - with patch("requests.get", return_value=mock_response): - component = RSSReaderComponent(rss_url="https://example.com/feed.xml") - result = component.read_rss() - - assert isinstance(result, DataFrame) - assert len(result) == 0 - assert list(result.columns) == ["title", "link", "published", "summary"] diff --git a/src/backend/tests/unit/components/data/test_web_search.py b/src/backend/tests/unit/components/data/test_web_search.py index b71187c939..7a54daef6a 100644 --- a/src/backend/tests/unit/components/data/test_web_search.py +++ b/src/backend/tests/unit/components/data/test_web_search.py @@ -1,3 +1,6 @@ +from unittest.mock import Mock, patch + +import pandas as pd import pytest from lfx.components.data.web_search import WebSearchComponent from lfx.schema import DataFrame @@ -15,7 +18,9 @@ class TestWebSearchComponent(ComponentTestBaseWithoutClient): def default_kwargs(self): """Return the default kwargs for the component.""" return { + "search_mode": "Web", "query": "OpenAI GPT-4", + "timeout": 5, } @pytest.fixture @@ -24,19 +29,406 @@ class TestWebSearchComponent(ComponentTestBaseWithoutClient): return [] async def test_invalid_url_handling(self): - # Create a test instance of the component + """Test invalid URL handling.""" component = WebSearchComponent() - # Set an invalid URL + # Test invalid URL invalid_url = "htp://invalid-url" # Ensure the URL is invalid with pytest.raises(ValueError, match="Invalid URL"): component.ensure_url(invalid_url) - def test_successful_web_search(self): + def test_validate_url(self): + """Test URL validation.""" component = WebSearchComponent() - component.query = "OpenAI GPT-4" - result = component.perform_search() + + # Valid URLs + assert component.validate_url("https://example.com") + assert component.validate_url("http://example.com") + assert component.validate_url("www.example.com") + assert component.validate_url("example.com") + assert component.validate_url("https://subdomain.example.co.uk") + + # Invalid URLs + assert not component.validate_url("not a url at all") + assert not component.validate_url("://missing-protocol") + + def test_ensure_url(self): + """Test ensure_url adds protocol if missing.""" + component = WebSearchComponent() + + assert component.ensure_url("https://example.com") == "https://example.com" + assert component.ensure_url("http://example.com") == "http://example.com" + assert component.ensure_url("example.com") == "https://example.com" + assert component.ensure_url("www.example.com") == "https://www.example.com" + + def test_sanitize_query(self): + """Test query sanitization.""" + component = WebSearchComponent() + + # Test removal of dangerous characters + assert component._sanitize_query('') == "scriptalert(test)/script" + assert component._sanitize_query("test'query\"withchars") == "testquerywithdangerouschars" + assert component._sanitize_query(" normal query ") == "normal query" + + def test_clean_html(self): + """Test HTML cleaning.""" + component = WebSearchComponent() + + html = '

This is bold text with link

' + expected = "This is bold text with link" + assert component.clean_html(html) == expected + + # Test with complex HTML + html = "

Title

Paragraph

" + expected = "Title Paragraph" + assert component.clean_html(html) == expected + + def test_update_build_config_web_mode(self): + """Test build config update for Web mode.""" + component = WebSearchComponent() + build_config = {"query": {"info": "", "display_name": ""}} + + result = component.update_build_config(build_config, "Web", "search_mode") + assert result["query"]["info"] == "Keywords to search for" + assert result["query"]["display_name"] == "Search Query" + + def test_update_build_config_news_mode(self): + """Test build config update for News mode.""" + component = WebSearchComponent() + build_config = {"query": {"info": "", "display_name": ""}} + + result = component.update_build_config(build_config, "News", "search_mode") + assert result["query"]["info"] == "Search keywords for news articles." + assert result["query"]["display_name"] == "Search Query" + + def test_update_build_config_rss_mode(self): + """Test build config update for RSS mode.""" + component = WebSearchComponent() + build_config = {"query": {"info": "", "display_name": ""}} + + result = component.update_build_config(build_config, "RSS", "search_mode") + assert result["query"]["info"] == "RSS feed URL to parse" + assert result["query"]["display_name"] == "RSS Feed URL" + + @patch("lfx.components.data.web_search.requests.get") + def test_perform_web_search_success(self, mock_get): + """Test successful web search.""" + component = WebSearchComponent() + component.query = "test query" + component.timeout = 5 + + # Mock DuckDuckGo response + mock_response = Mock() + mock_response.text = """ + +
+ Test Title + Test snippet content +
+ + """ + mock_response.headers = {"content-type": "text/html"} + mock_response.raise_for_status.return_value = None + + # Mock the page fetch + mock_page_response = Mock() + mock_page_response.text = "Page content" + mock_page_response.raise_for_status.return_value = None + + mock_get.side_effect = [mock_response, mock_page_response] + + result = component.perform_web_search() + assert isinstance(result, DataFrame) - assert not result.empty + assert len(result) == 1 + assert result.iloc[0]["title"] == "Test Title" + assert result.iloc[0]["snippet"] == "Test snippet content" + assert "Page content" in result.iloc[0]["content"] + + @patch("lfx.components.data.web_search.requests.get") + def test_perform_web_search_no_results(self, mock_get): + """Test web search with no results.""" + component = WebSearchComponent() + component.query = "test query" + component.timeout = 5 + + mock_response = Mock() + mock_response.text = "" + mock_response.headers = {"content-type": "text/html"} + mock_response.raise_for_status.return_value = None + + mock_get.return_value = mock_response + + result = component.perform_web_search() + + assert isinstance(result, DataFrame) + assert "No results found" in result.iloc[0]["snippet"] + + @patch("lfx.components.data.web_search.requests.get") + def test_perform_web_search_request_error(self, mock_get): + """Test web search with request error.""" + component = WebSearchComponent() + component.query = "test query" + component.timeout = 5 + + from requests import RequestException + + mock_get.side_effect = RequestException("Connection error") + + result = component.perform_web_search() + + assert isinstance(result, DataFrame) + assert "Connection error" in result.iloc[0]["snippet"] + + @patch("lfx.components.data.web_search.requests.get") + def test_perform_news_search_with_query(self, mock_get): + """Test news search with query.""" + component = WebSearchComponent() + component.query = "test news" + component.timeout = 5 + + # Mock RSS response + mock_response = Mock() + mock_response.content = b""" + + + + Test News Title + https://news.example.com + Mon, 01 Jan 2024 00:00:00 GMT + Test news description + + + + """ + mock_response.raise_for_status.return_value = None + mock_get.return_value = mock_response + + result = component.perform_news_search() + + assert isinstance(result, DataFrame) + assert len(result) == 1 + assert result.iloc[0]["title"] == "Test News Title" + assert result.iloc[0]["link"] == "https://news.example.com" + assert result.iloc[0]["summary"] == "Test news description" + + @patch("lfx.components.data.web_search.requests.get") + def test_perform_news_search_with_topic(self, mock_get): + """Test news search with topic.""" + component = WebSearchComponent() + component.topic = "TECHNOLOGY" + component.timeout = 5 + + # Mock RSS response + mock_response = Mock() + mock_response.content = b""" + + + + Tech News + https://tech.example.com + Mon, 01 Jan 2024 00:00:00 GMT + Technology news + + + + """ + mock_response.raise_for_status.return_value = None + mock_get.return_value = mock_response + + result = component.perform_news_search() + + assert isinstance(result, DataFrame) + assert len(result) == 1 + # Check that the URL was constructed with topic + mock_get.assert_called_once() + call_args = mock_get.call_args[0][0] + assert "topic/TECHNOLOGY" in call_args + + @patch("lfx.components.data.web_search.requests.get") + def test_perform_news_search_no_params(self, mock_get): # noqa: ARG002 + """Test news search with no parameters.""" + component = WebSearchComponent() + component.timeout = 5 + + result = component.perform_news_search() + + assert isinstance(result, DataFrame) + assert "No search parameters provided" in result.iloc[0]["summary"] + + @patch("lfx.components.data.web_search.requests.get") + def test_perform_rss_read_success(self, mock_get): + """Test successful RSS feed reading.""" + component = WebSearchComponent() + component.query = "https://example.com/feed.rss" + component.timeout = 5 + + # Mock RSS response + mock_response = Mock() + mock_response.content = b""" + + + + RSS Item 1 + https://example.com/item1 + Mon, 01 Jan 2024 00:00:00 GMT + Description 1 + + + RSS Item 2 + https://example.com/item2 + Tue, 02 Jan 2024 00:00:00 GMT + Description 2 + + + + """ + mock_response.raise_for_status.return_value = None + mock_get.return_value = mock_response + + result = component.perform_rss_read() + + assert isinstance(result, DataFrame) + assert len(result) == 2 + assert result.iloc[0]["title"] == "RSS Item 1" + assert result.iloc[1]["title"] == "RSS Item 2" + + @patch("lfx.components.data.web_search.requests.get") + def test_perform_rss_read_empty_response(self, mock_get): + """Test RSS read with empty response.""" + component = WebSearchComponent() + component.query = "https://example.com/feed.rss" + component.timeout = 5 + + mock_response = Mock() + mock_response.content = b"" + mock_response.raise_for_status.return_value = None + mock_get.return_value = mock_response + + result = component.perform_rss_read() + + assert isinstance(result, DataFrame) + assert "Empty response received" in result.iloc[0]["summary"] + + @patch("lfx.components.data.web_search.requests.get") + def test_perform_rss_read_invalid_xml(self, mock_get): + """Test RSS read with invalid XML - returns empty DataFrame when no items found.""" + component = WebSearchComponent() + component.query = "https://example.com/feed.rss" + component.timeout = 5 + + mock_response = Mock() + mock_response.content = b"This is not valid XML" + mock_response.raise_for_status.return_value = None + mock_get.return_value = mock_response + + result = component.perform_rss_read() + + assert isinstance(result, DataFrame) + # When no RSS items are found, it returns an empty DataFrame + assert len(result) == 0 + + def test_perform_rss_read_no_url(self): + """Test RSS read with no URL provided.""" + component = WebSearchComponent() + component.query = "" + + result = component.perform_rss_read() + + assert isinstance(result, DataFrame) + assert "No RSS URL provided" in result.iloc[0]["summary"] + + @patch.object(WebSearchComponent, "perform_web_search") + def test_perform_search_web_mode(self, mock_web_search): + """Test perform_search routes to web search in Web mode.""" + component = WebSearchComponent() + component.search_mode = "Web" + + mock_web_search.return_value = DataFrame(pd.DataFrame([{"result": "web"}])) + + result = component.perform_search() + + mock_web_search.assert_called_once() + assert result.iloc[0]["result"] == "web" + + @patch.object(WebSearchComponent, "perform_news_search") + def test_perform_search_news_mode(self, mock_news_search): + """Test perform_search routes to news search in News mode.""" + component = WebSearchComponent() + component.search_mode = "News" + + mock_news_search.return_value = DataFrame(pd.DataFrame([{"result": "news"}])) + + result = component.perform_search() + + mock_news_search.assert_called_once() + assert result.iloc[0]["result"] == "news" + + @patch.object(WebSearchComponent, "perform_rss_read") + def test_perform_search_rss_mode(self, mock_rss_read): + """Test perform_search routes to RSS read in RSS mode.""" + component = WebSearchComponent() + component.search_mode = "RSS" + + mock_rss_read.return_value = DataFrame(pd.DataFrame([{"result": "rss"}])) + + result = component.perform_search() + + mock_rss_read.assert_called_once() + assert result.iloc[0]["result"] == "rss" + + @patch.object(WebSearchComponent, "perform_web_search") + def test_perform_search_fallback(self, mock_web_search): + """Test perform_search falls back to web search for unknown mode.""" + component = WebSearchComponent() + component.search_mode = "UnknownMode" + + mock_web_search.return_value = DataFrame(pd.DataFrame([{"result": "fallback"}])) + + result = component.perform_search() + + mock_web_search.assert_called_once() + assert result.iloc[0]["result"] == "fallback" + + def test_empty_query_error(self): + """Test that empty query raises ValueError.""" + component = WebSearchComponent() + component.query = "" + component.timeout = 5 + + with pytest.raises(ValueError, match="Empty search query"): + component.perform_web_search() + + @patch("lfx.components.data.web_search.requests.get") + def test_news_search_with_location(self, mock_get): + """Test news search with location parameter.""" + component = WebSearchComponent() + component.location = "San Francisco" + component.timeout = 5 + + # Mock RSS response + mock_response = Mock() + mock_response.content = b""" + + + + Local News + https://local.example.com + Mon, 01 Jan 2024 00:00:00 GMT + San Francisco news + + + + """ + mock_response.raise_for_status.return_value = None + mock_get.return_value = mock_response + + result = component.perform_news_search() + + assert isinstance(result, DataFrame) + # Check that the URL was constructed with location + mock_get.assert_called_once() + call_args = mock_get.call_args[0][0] + assert "geo/San%20Francisco" in call_args or "geo/San+Francisco" in call_args diff --git a/src/lfx/src/lfx/components/data/__init__.py b/src/lfx/src/lfx/components/data/__init__.py index e75020dc56..cfa7da6a84 100644 --- a/src/lfx/src/lfx/components/data/__init__.py +++ b/src/lfx/src/lfx/components/data/__init__.py @@ -10,9 +10,6 @@ if TYPE_CHECKING: from lfx.components.data.directory import DirectoryComponent from lfx.components.data.file import FileComponent from lfx.components.data.json_to_data import JSONToDataComponent - from lfx.components.data.news_search import NewsSearchComponent - from lfx.components.data.rss import RSSReaderComponent - from lfx.components.data.save_file import SaveToFileComponent from lfx.components.data.sql_executor import SQLComponent from lfx.components.data.url import URLComponent from lfx.components.data.web_search import WebSearchComponent @@ -24,10 +21,7 @@ _dynamic_imports = { "DirectoryComponent": "directory", "FileComponent": "file", "JSONToDataComponent": "json_to_data", - "NewsSearchComponent": "news_search", - "RSSReaderComponent": "rss", "SQLComponent": "sql_executor", - "SaveToFileComponent": "save_file", "URLComponent": "url", "WebSearchComponent": "web_search", "WebhookComponent": "webhook", @@ -39,10 +33,7 @@ __all__ = [ "DirectoryComponent", "FileComponent", "JSONToDataComponent", - "NewsSearchComponent", - "RSSReaderComponent", "SQLComponent", - "SaveToFileComponent", "URLComponent", "WebSearchComponent", "WebhookComponent", diff --git a/src/lfx/src/lfx/components/data/news_search.py b/src/lfx/src/lfx/components/data/news_search.py deleted file mode 100644 index 8f420aa4d2..0000000000 --- a/src/lfx/src/lfx/components/data/news_search.py +++ /dev/null @@ -1,164 +0,0 @@ -from urllib.parse import quote_plus - -import pandas as pd -import requests -from bs4 import BeautifulSoup - -from lfx.custom import Component -from lfx.io import IntInput, MessageTextInput, Output -from lfx.schema import DataFrame - - -class NewsSearchComponent(Component): - display_name = "News Search" - description = "Searches Google News via RSS. Returns clean article data." - documentation: str = "https://docs.langflow.org/components-data#news-search" - icon = "newspaper" - name = "NewsSearch" - - inputs = [ - MessageTextInput( - name="query", - display_name="Search Query", - info="Search keywords for news articles.", - tool_mode=True, - required=True, - ), - MessageTextInput( - name="hl", - display_name="Language (hl)", - info="Language code, e.g. en-US, fr, de. Default: en-US.", - tool_mode=False, - input_types=[], - required=False, - advanced=True, - ), - MessageTextInput( - name="gl", - display_name="Country (gl)", - info="Country code, e.g. US, FR, DE. Default: US.", - tool_mode=False, - input_types=[], - required=False, - advanced=True, - ), - MessageTextInput( - name="ceid", - display_name="Country:Language (ceid)", - info="e.g. US:en, FR:fr. Default: US:en.", - tool_mode=False, - value="US:en", - input_types=[], - required=False, - advanced=True, - ), - MessageTextInput( - name="topic", - display_name="Topic", - info="One of: WORLD, NATION, BUSINESS, TECHNOLOGY, ENTERTAINMENT, SCIENCE, SPORTS, HEALTH.", - tool_mode=False, - input_types=[], - required=False, - advanced=True, - ), - MessageTextInput( - name="location", - display_name="Location (Geo)", - info="City, state, or country for location-based news. Leave blank for keyword search.", - tool_mode=False, - input_types=[], - required=False, - advanced=True, - ), - IntInput( - name="timeout", - display_name="Timeout", - info="Timeout for the request in seconds.", - value=5, - required=False, - advanced=True, - ), - ] - - outputs = [Output(name="articles", display_name="News Articles", method="search_news")] - - def search_news(self) -> DataFrame: - # Defaults - hl = getattr(self, "hl", None) or "en-US" - gl = getattr(self, "gl", None) or "US" - ceid = getattr(self, "ceid", None) or f"{gl}:{hl.split('-')[0]}" - topic = getattr(self, "topic", None) - location = getattr(self, "location", None) - query = getattr(self, "query", None) - - # Build base URL - if topic: - # Topic-based feed - base_url = f"https://news.google.com/rss/headlines/section/topic/{quote_plus(topic.upper())}" - params = f"?hl={hl}&gl={gl}&ceid={ceid}" - rss_url = base_url + params - elif location: - # Location-based feed - base_url = f"https://news.google.com/rss/headlines/section/geo/{quote_plus(location)}" - params = f"?hl={hl}&gl={gl}&ceid={ceid}" - rss_url = base_url + params - elif query: - # Keyword search feed - base_url = "https://news.google.com/rss/search?q=" - query_parts = [query] - query_encoded = quote_plus(" ".join(query_parts)) - params = f"&hl={hl}&gl={gl}&ceid={ceid}" - rss_url = f"{base_url}{query_encoded}{params}" - else: - self.status = "No search query, topic, or location provided." - self.log(self.status) - return DataFrame( - pd.DataFrame( - [ - { - "title": "Error", - "link": "", - "published": "", - "summary": "No search query, topic, or location provided.", - } - ] - ) - ) - - try: - response = requests.get(rss_url, timeout=self.timeout) - response.raise_for_status() - soup = BeautifulSoup(response.content, "xml") - items = soup.find_all("item") - except requests.RequestException as e: - self.status = f"Failed to fetch news: {e}" - self.log(self.status) - return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}])) - except (AttributeError, ValueError, TypeError) as e: - self.status = f"Unexpected error: {e!s}" - self.log(self.status) - return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}])) - - if not items: - self.status = "No news articles found." - self.log(self.status) - return DataFrame(pd.DataFrame([{"title": "No articles found", "link": "", "published": "", "summary": ""}])) - - articles = [] - for item in items: - try: - title = self.clean_html(item.title.text if item.title else "") - link = item.link.text if item.link else "" - published = item.pubDate.text if item.pubDate else "" - summary = self.clean_html(item.description.text if item.description else "") - articles.append({"title": title, "link": link, "published": published, "summary": summary}) - except (AttributeError, ValueError, TypeError) as e: - self.log(f"Error parsing article: {e!s}") - continue - - df_articles = pd.DataFrame(articles) - self.log(f"Found {len(df_articles)} articles.") - return DataFrame(df_articles) - - def clean_html(self, html_string: str) -> str: - return BeautifulSoup(html_string, "html.parser").get_text(separator=" ", strip=True) diff --git a/src/lfx/src/lfx/components/data/rss.py b/src/lfx/src/lfx/components/data/rss.py deleted file mode 100644 index 7735018554..0000000000 --- a/src/lfx/src/lfx/components/data/rss.py +++ /dev/null @@ -1,69 +0,0 @@ -import pandas as pd -import requests -from bs4 import BeautifulSoup - -from lfx.custom import Component -from lfx.io import IntInput, MessageTextInput, Output -from lfx.log.logger import logger -from lfx.schema import DataFrame - - -class RSSReaderComponent(Component): - display_name = "RSS Reader" - description = "Fetches and parses an RSS feed." - documentation: str = "https://docs.langflow.org/components-data#rss-reader" - icon = "rss" - name = "RSSReaderSimple" - - inputs = [ - MessageTextInput( - name="rss_url", - display_name="RSS Feed URL", - info="URL of the RSS feed to parse.", - tool_mode=True, - required=True, - ), - IntInput( - name="timeout", - display_name="Timeout", - info="Timeout for the RSS feed request.", - value=5, - advanced=True, - ), - ] - - outputs = [Output(name="articles", display_name="Articles", method="read_rss")] - - def read_rss(self) -> DataFrame: - try: - response = requests.get(self.rss_url, timeout=self.timeout) - response.raise_for_status() - if not response.content.strip(): - msg = "Empty response received" - raise ValueError(msg) - # Check if the response is valid XML - try: - BeautifulSoup(response.content, "xml") - except Exception as e: - msg = f"Invalid XML response: {e}" - raise ValueError(msg) from e - soup = BeautifulSoup(response.content, "xml") - items = soup.find_all("item") - except (requests.RequestException, ValueError) as e: - self.status = f"Failed to fetch RSS: {e}" - return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}])) - - articles = [ - { - "title": item.title.text if item.title else "", - "link": item.link.text if item.link else "", - "published": item.pubDate.text if item.pubDate else "", - "summary": item.description.text if item.description else "", - } - for item in items - ] - - # Ensure the DataFrame has the correct columns even if empty - df_articles = pd.DataFrame(articles, columns=["title", "link", "published", "summary"]) - logger.info(f"Fetched {len(df_articles)} articles.") - return DataFrame(df_articles) diff --git a/src/lfx/src/lfx/components/data/web_search.py b/src/lfx/src/lfx/components/data/web_search.py index cb4830d8c5..39a6f149fc 100644 --- a/src/lfx/src/lfx/components/data/web_search.py +++ b/src/lfx/src/lfx/components/data/web_search.py @@ -1,43 +1,133 @@ +"""Unified Web Search Component. + +This component consolidates Web Search, News Search, and RSS Reader into a single +component with tabs for different search modes. +""" + import re -from urllib.parse import parse_qs, unquote, urlparse +from typing import Any +from urllib.parse import parse_qs, quote_plus, unquote, urlparse import pandas as pd import requests from bs4 import BeautifulSoup from lfx.custom import Component -from lfx.io import IntInput, MessageTextInput, Output +from lfx.io import IntInput, MessageTextInput, Output, TabInput from lfx.schema import DataFrame from lfx.utils.request_utils import get_user_agent class WebSearchComponent(Component): display_name = "Web Search" - description = "Performs a basic DuckDuckGo search (HTML scraping). May be subject to rate limits." + description = "Search the web, news, or RSS feeds." documentation: str = "https://docs.langflow.org/components-data#web-search" icon = "search" - name = "WebSearchNoAPI" + name = "UnifiedWebSearch" inputs = [ + TabInput( + name="search_mode", + display_name="Search Mode", + options=["Web", "News", "RSS"], + info="Choose search mode: Web (DuckDuckGo), News (Google News), or RSS (Feed Reader)", + value="Web", + real_time_refresh=True, + tool_mode=True, + ), MessageTextInput( name="query", display_name="Search Query", - info="Keywords to search for.", + info="Search keywords for news articles.", tool_mode=True, required=True, ), + MessageTextInput( + name="hl", + display_name="Language (hl)", + info="Language code, e.g. en-US, fr, de. Default: en-US.", + tool_mode=False, + input_types=[], + required=False, + advanced=True, + ), + MessageTextInput( + name="gl", + display_name="Country (gl)", + info="Country code, e.g. US, FR, DE. Default: US.", + tool_mode=False, + input_types=[], + required=False, + advanced=True, + ), + MessageTextInput( + name="ceid", + display_name="Country:Language (ceid)", + info="e.g. US:en, FR:fr. Default: US:en.", + tool_mode=False, + value="US:en", + input_types=[], + required=False, + advanced=True, + ), + MessageTextInput( + name="topic", + display_name="Topic", + info="One of: WORLD, NATION, BUSINESS, TECHNOLOGY, ENTERTAINMENT, SCIENCE, SPORTS, HEALTH.", + tool_mode=False, + input_types=[], + required=False, + advanced=True, + ), + MessageTextInput( + name="location", + display_name="Location (Geo)", + info="City, state, or country for location-based news. Leave blank for keyword search.", + tool_mode=False, + input_types=[], + required=False, + advanced=True, + ), IntInput( name="timeout", display_name="Timeout", - info="Timeout for the web search request.", + info="Timeout for the request in seconds.", value=5, + required=False, advanced=True, ), ] - outputs = [Output(name="results", display_name="Search Results", method="perform_search")] + outputs = [Output(name="results", display_name="Results", method="perform_search")] + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict: + """Update input visibility based on search mode.""" + if field_name == "search_mode": + # Show/hide inputs based on search mode + is_news = field_value == "News" + is_rss = field_value == "RSS" + + # Update query field info based on mode + if is_rss: + build_config["query"]["info"] = "RSS feed URL to parse" + build_config["query"]["display_name"] = "RSS Feed URL" + elif is_news: + build_config["query"]["info"] = "Search keywords for news articles." + build_config["query"]["display_name"] = "Search Query" + else: # Web + build_config["query"]["info"] = "Keywords to search for" + build_config["query"]["display_name"] = "Search Query" + + # Keep news-specific fields as advanced (matching original News Search component) + # They remain advanced=True in all modes, just like in the original component + + return build_config def validate_url(self, string: str) -> bool: + """Validate URL format.""" url_regex = re.compile( r"^(https?:\/\/)?" r"(www\.)?" r"([a-zA-Z0-9.-]+)" r"(\.[a-zA-Z]{2,})?" r"(:\d+)?" r"(\/[^\s]*)?$", re.IGNORECASE, @@ -45,6 +135,7 @@ class WebSearchComponent(Component): return bool(url_regex.match(string)) def ensure_url(self, url: str) -> str: + """Ensure URL has proper protocol.""" if not url.startswith(("http://", "https://")): url = "https://" + url if not self.validate_url(url): @@ -54,14 +145,19 @@ class WebSearchComponent(Component): def _sanitize_query(self, query: str) -> str: """Sanitize search query.""" - # Remove potentially dangerous characters return re.sub(r'[<>"\']', "", query.strip()) - def perform_search(self) -> DataFrame: + def clean_html(self, html_string: str) -> str: + """Remove HTML tags from text.""" + return BeautifulSoup(html_string, "html.parser").get_text(separator=" ", strip=True) + + def perform_web_search(self) -> DataFrame: + """Perform DuckDuckGo web search.""" query = self._sanitize_query(self.query) if not query: msg = "Empty search query" raise ValueError(msg) + headers = {"User-Agent": get_user_agent()} params = {"q": query, "kl": "us-en"} url = "https://html.duckduckgo.com/html/" @@ -78,6 +174,7 @@ class WebSearchComponent(Component): return DataFrame( pd.DataFrame([{"title": "Error", "link": "", "snippet": "No results found", "content": ""}]) ) + soup = BeautifulSoup(response.text, "html.parser") results = [] @@ -108,5 +205,122 @@ class WebSearchComponent(Component): } ) - df_results = pd.DataFrame(results) - return DataFrame(df_results) + return DataFrame(pd.DataFrame(results)) + + def perform_news_search(self) -> DataFrame: + """Perform Google News search.""" + query = getattr(self, "query", "") + hl = getattr(self, "hl", "en-US") or "en-US" + gl = getattr(self, "gl", "US") or "US" + topic = getattr(self, "topic", None) + location = getattr(self, "location", None) + + ceid = f"{gl}:{hl.split('-')[0]}" + + # Build RSS URL based on parameters + if topic: + # Topic-based feed + base_url = f"https://news.google.com/rss/headlines/section/topic/{quote_plus(topic.upper())}" + params = f"?hl={hl}&gl={gl}&ceid={ceid}" + rss_url = base_url + params + elif location: + # Location-based feed + base_url = f"https://news.google.com/rss/headlines/section/geo/{quote_plus(location)}" + params = f"?hl={hl}&gl={gl}&ceid={ceid}" + rss_url = base_url + params + elif query: + # Keyword search feed + base_url = "https://news.google.com/rss/search?q=" + query_encoded = quote_plus(query) + params = f"&hl={hl}&gl={gl}&ceid={ceid}" + rss_url = f"{base_url}{query_encoded}{params}" + else: + self.status = "No search query, topic, or location provided." + return DataFrame( + pd.DataFrame( + [{"title": "Error", "link": "", "published": "", "summary": "No search parameters provided"}] + ) + ) + + try: + response = requests.get(rss_url, timeout=self.timeout) + response.raise_for_status() + soup = BeautifulSoup(response.content, "xml") + items = soup.find_all("item") + except requests.RequestException as e: + self.status = f"Failed to fetch news: {e}" + return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}])) + + if not items: + self.status = "No news articles found." + return DataFrame(pd.DataFrame([{"title": "No articles found", "link": "", "published": "", "summary": ""}])) + + articles = [] + for item in items: + try: + title = self.clean_html(item.title.text if item.title else "") + link = item.link.text if item.link else "" + published = item.pubDate.text if item.pubDate else "" + summary = self.clean_html(item.description.text if item.description else "") + articles.append({"title": title, "link": link, "published": published, "summary": summary}) + except (AttributeError, ValueError, TypeError) as e: + self.log(f"Error parsing article: {e!s}") + continue + + return DataFrame(pd.DataFrame(articles)) + + def perform_rss_read(self) -> DataFrame: + """Read RSS feed.""" + rss_url = getattr(self, "query", "") + if not rss_url: + return DataFrame( + pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": "No RSS URL provided"}]) + ) + + try: + response = requests.get(rss_url, timeout=self.timeout) + response.raise_for_status() + if not response.content.strip(): + msg = "Empty response received" + raise ValueError(msg) + + # Validate XML + try: + BeautifulSoup(response.content, "xml") + except Exception as e: + msg = f"Invalid XML response: {e}" + raise ValueError(msg) from e + + soup = BeautifulSoup(response.content, "xml") + items = soup.find_all("item") + except (requests.RequestException, ValueError) as e: + self.status = f"Failed to fetch RSS: {e}" + return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}])) + + articles = [ + { + "title": item.title.text if item.title else "", + "link": item.link.text if item.link else "", + "published": item.pubDate.text if item.pubDate else "", + "summary": item.description.text if item.description else "", + } + for item in items + ] + + # Ensure DataFrame has correct columns even if empty + df_articles = pd.DataFrame(articles, columns=["title", "link", "published", "summary"]) + self.log(f"Fetched {len(df_articles)} articles.") + return DataFrame(df_articles) + + def perform_search(self) -> DataFrame: + """Main search method that routes to appropriate search function based on mode.""" + search_mode = getattr(self, "search_mode", "Web") + + if search_mode == "Web": + return self.perform_web_search() + if search_mode == "News": + return self.perform_news_search() + if search_mode == "RSS": + return self.perform_rss_read() + # Fallback to web search + return self.perform_web_search()