mirror of
https://github.com/langflow-ai/langflow.git
synced 2026-07-24 04:47:23 +08:00
feat: Consolidate Web Search, News Search, and RSS Reader into unified component (#9975)
* feat: Consolidate Web Search, News Search, and RSS Reader into unified component Merge three separate components into a single Web Search component with tab-based mode selection: - Web mode: DuckDuckGo search functionality (default) - News mode: Google News RSS feed search with topic/location support - RSS mode: Generic RSS feed reader This reduces code duplication and provides a more intuitive user experience with a single component that handles all search-related functionality. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com> * [autofix.ci] apply automated fixes * Update test_web_search.py --------- Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Eric Hare <ericrhare@gmail.com>
This commit is contained in:
Rodrigo Nader
@ -1,88 +0,0 @@
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from lfx.components.data.news_search import NewsSearchComponent
|
||||
from lfx.schema import DataFrame
|
||||
|
||||
from tests.base import ComponentTestBaseWithoutClient
|
||||
|
||||
|
||||
class TestNewsSearchComponent(ComponentTestBaseWithoutClient):
|
||||
@pytest.fixture
|
||||
def component_class(self):
|
||||
return NewsSearchComponent
|
||||
|
||||
@pytest.fixture
|
||||
def default_kwargs(self):
|
||||
return {"query": "OpenAI"}
|
||||
|
||||
@pytest.fixture
|
||||
def file_names_mapping(self):
|
||||
return []
|
||||
|
||||
def test_successful_news_search(self):
|
||||
# Mock Google News RSS feed content
|
||||
mock_rss_content = """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<item>
|
||||
<title>Test News 1</title>
|
||||
<link>https://example.com/1</link>
|
||||
<pubDate>2024-03-20</pubDate>
|
||||
<description>Summary 1</description>
|
||||
</item>
|
||||
<item>
|
||||
<title>Test News 2</title>
|
||||
<link>https://example.com/2</link>
|
||||
<pubDate>2024-03-21</pubDate>
|
||||
<description>Summary 2</description>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
mock_response = Mock()
|
||||
mock_response.content = mock_rss_content.encode("utf-8")
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("requests.get", return_value=mock_response):
|
||||
component = NewsSearchComponent(query="OpenAI")
|
||||
result = component.search_news()
|
||||
assert isinstance(result, DataFrame)
|
||||
news_results_df = result
|
||||
assert len(news_results_df) == 2
|
||||
assert list(news_results_df.columns) == ["title", "link", "published", "summary"]
|
||||
assert news_results_df.iloc[0]["title"] == "Test News 1"
|
||||
assert news_results_df.iloc[1]["title"] == "Test News 2"
|
||||
|
||||
def test_news_search_error(self):
|
||||
with patch("requests.get", side_effect=requests.RequestException("Network error")):
|
||||
component = NewsSearchComponent(query="OpenAI")
|
||||
result = component.search_news()
|
||||
assert isinstance(result, DataFrame)
|
||||
news_results_df = result
|
||||
assert len(news_results_df) == 1
|
||||
assert news_results_df.iloc[0]["title"] == "Error"
|
||||
assert "Network error" in news_results_df.iloc[0]["summary"]
|
||||
|
||||
def test_empty_news_results(self):
|
||||
# Mock empty RSS feed
|
||||
mock_rss_content = """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
mock_response = Mock()
|
||||
mock_response.content = mock_rss_content.encode("utf-8")
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("requests.get", return_value=mock_response):
|
||||
component = NewsSearchComponent(query="OpenAI")
|
||||
result = component.search_news()
|
||||
assert isinstance(result, DataFrame)
|
||||
news_results_df = result
|
||||
assert len(news_results_df) == 1
|
||||
assert news_results_df.iloc[0]["title"] == "No articles found"
|
||||
@ -1,129 +0,0 @@
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from lfx.components.data.rss import RSSReaderComponent
|
||||
from lfx.schema import DataFrame
|
||||
|
||||
from tests.base import ComponentTestBaseWithoutClient
|
||||
|
||||
|
||||
class TestRSSReaderComponent(ComponentTestBaseWithoutClient):
|
||||
@pytest.fixture
|
||||
def component_class(self):
|
||||
"""Return the component class to test."""
|
||||
return RSSReaderComponent
|
||||
|
||||
@pytest.fixture
|
||||
def default_kwargs(self):
|
||||
"""Return the default kwargs for the component."""
|
||||
return {
|
||||
"rss_url": "https://example.com/feed.xml",
|
||||
}
|
||||
|
||||
@pytest.fixture
|
||||
def file_names_mapping(self):
|
||||
"""Return an empty list since this component doesn't have version-specific files."""
|
||||
return []
|
||||
|
||||
def test_successful_rss_fetch(self):
|
||||
# Mock RSS feed content
|
||||
mock_rss_content = """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<item>
|
||||
<title>Test Article 1</title>
|
||||
<link>https://example.com/1</link>
|
||||
<pubDate>2024-03-20</pubDate>
|
||||
<description>Test summary 1</description>
|
||||
</item>
|
||||
<item>
|
||||
<title>Test Article 2</title>
|
||||
<link>https://example.com/2</link>
|
||||
<pubDate>2024-03-21</pubDate>
|
||||
<description>Test summary 2</description>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
|
||||
# Mock the requests.get response
|
||||
mock_response = Mock()
|
||||
mock_response.content = mock_rss_content.encode("utf-8")
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("requests.get", return_value=mock_response):
|
||||
component = RSSReaderComponent(rss_url="https://example.com/feed.xml")
|
||||
result = component.read_rss()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert len(result) == 2
|
||||
assert list(result.columns) == ["title", "link", "published", "summary"]
|
||||
assert result.iloc[0]["title"] == "Test Article 1"
|
||||
assert result.iloc[1]["title"] == "Test Article 2"
|
||||
|
||||
def test_rss_fetch_with_missing_fields(self):
|
||||
# Mock RSS feed content with missing fields
|
||||
mock_rss_content = """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<item>
|
||||
<title>Test Article</title>
|
||||
<!-- Missing link -->
|
||||
<pubDate>2024-03-20</pubDate>
|
||||
<!-- Missing description -->
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.content = mock_rss_content.encode("utf-8")
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("requests.get", return_value=mock_response):
|
||||
component = RSSReaderComponent(rss_url="https://example.com/feed.xml")
|
||||
result = component.read_rss()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert len(result) == 1
|
||||
assert result.iloc[0]["title"] == "Test Article"
|
||||
assert result.iloc[0]["link"] == ""
|
||||
assert result.iloc[0]["summary"] == ""
|
||||
|
||||
def test_rss_fetch_error(self):
|
||||
# Mock a failed request
|
||||
with patch("requests.get", side_effect=requests.RequestException("Network error")):
|
||||
component = RSSReaderComponent(rss_url="https://example.com/feed.xml")
|
||||
result = component.read_rss()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert len(result) == 1
|
||||
assert result.iloc[0]["title"] == "Error"
|
||||
assert result.iloc[0]["link"] == ""
|
||||
assert result.iloc[0]["published"] == ""
|
||||
assert "Network error" in result.iloc[0]["summary"]
|
||||
|
||||
def test_empty_rss_feed(self):
|
||||
# Mock empty RSS feed
|
||||
mock_rss_content = """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.content = mock_rss_content.encode("utf-8")
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("requests.get", return_value=mock_response):
|
||||
component = RSSReaderComponent(rss_url="https://example.com/feed.xml")
|
||||
result = component.read_rss()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert len(result) == 0
|
||||
assert list(result.columns) == ["title", "link", "published", "summary"]
|
||||
@ -1,3 +1,6 @@
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from lfx.components.data.web_search import WebSearchComponent
|
||||
from lfx.schema import DataFrame
|
||||
@ -15,7 +18,9 @@ class TestWebSearchComponent(ComponentTestBaseWithoutClient):
|
||||
def default_kwargs(self):
|
||||
"""Return the default kwargs for the component."""
|
||||
return {
|
||||
"search_mode": "Web",
|
||||
"query": "OpenAI GPT-4",
|
||||
"timeout": 5,
|
||||
}
|
||||
|
||||
@pytest.fixture
|
||||
@ -24,19 +29,406 @@ class TestWebSearchComponent(ComponentTestBaseWithoutClient):
|
||||
return []
|
||||
|
||||
async def test_invalid_url_handling(self):
|
||||
# Create a test instance of the component
|
||||
"""Test invalid URL handling."""
|
||||
component = WebSearchComponent()
|
||||
|
||||
# Set an invalid URL
|
||||
# Test invalid URL
|
||||
invalid_url = "htp://invalid-url"
|
||||
|
||||
# Ensure the URL is invalid
|
||||
with pytest.raises(ValueError, match="Invalid URL"):
|
||||
component.ensure_url(invalid_url)
|
||||
|
||||
def test_successful_web_search(self):
|
||||
def test_validate_url(self):
|
||||
"""Test URL validation."""
|
||||
component = WebSearchComponent()
|
||||
component.query = "OpenAI GPT-4"
|
||||
result = component.perform_search()
|
||||
|
||||
# Valid URLs
|
||||
assert component.validate_url("https://example.com")
|
||||
assert component.validate_url("http://example.com")
|
||||
assert component.validate_url("www.example.com")
|
||||
assert component.validate_url("example.com")
|
||||
assert component.validate_url("https://subdomain.example.co.uk")
|
||||
|
||||
# Invalid URLs
|
||||
assert not component.validate_url("not a url at all")
|
||||
assert not component.validate_url("://missing-protocol")
|
||||
|
||||
def test_ensure_url(self):
|
||||
"""Test ensure_url adds protocol if missing."""
|
||||
component = WebSearchComponent()
|
||||
|
||||
assert component.ensure_url("https://example.com") == "https://example.com"
|
||||
assert component.ensure_url("http://example.com") == "http://example.com"
|
||||
assert component.ensure_url("example.com") == "https://example.com"
|
||||
assert component.ensure_url("www.example.com") == "https://www.example.com"
|
||||
|
||||
def test_sanitize_query(self):
|
||||
"""Test query sanitization."""
|
||||
component = WebSearchComponent()
|
||||
|
||||
# Test removal of dangerous characters
|
||||
assert component._sanitize_query('<script>alert("test")</script>') == "scriptalert(test)/script"
|
||||
assert component._sanitize_query("test'query\"with<dangerous>chars") == "testquerywithdangerouschars"
|
||||
assert component._sanitize_query(" normal query ") == "normal query"
|
||||
|
||||
def test_clean_html(self):
|
||||
"""Test HTML cleaning."""
|
||||
component = WebSearchComponent()
|
||||
|
||||
html = '<p>This is <b>bold</b> text with <a href="#">link</a></p>'
|
||||
expected = "This is bold text with link"
|
||||
assert component.clean_html(html) == expected
|
||||
|
||||
# Test with complex HTML
|
||||
html = "<div><h1>Title</h1><p>Paragraph</p></div>"
|
||||
expected = "Title Paragraph"
|
||||
assert component.clean_html(html) == expected
|
||||
|
||||
def test_update_build_config_web_mode(self):
|
||||
"""Test build config update for Web mode."""
|
||||
component = WebSearchComponent()
|
||||
build_config = {"query": {"info": "", "display_name": ""}}
|
||||
|
||||
result = component.update_build_config(build_config, "Web", "search_mode")
|
||||
assert result["query"]["info"] == "Keywords to search for"
|
||||
assert result["query"]["display_name"] == "Search Query"
|
||||
|
||||
def test_update_build_config_news_mode(self):
|
||||
"""Test build config update for News mode."""
|
||||
component = WebSearchComponent()
|
||||
build_config = {"query": {"info": "", "display_name": ""}}
|
||||
|
||||
result = component.update_build_config(build_config, "News", "search_mode")
|
||||
assert result["query"]["info"] == "Search keywords for news articles."
|
||||
assert result["query"]["display_name"] == "Search Query"
|
||||
|
||||
def test_update_build_config_rss_mode(self):
|
||||
"""Test build config update for RSS mode."""
|
||||
component = WebSearchComponent()
|
||||
build_config = {"query": {"info": "", "display_name": ""}}
|
||||
|
||||
result = component.update_build_config(build_config, "RSS", "search_mode")
|
||||
assert result["query"]["info"] == "RSS feed URL to parse"
|
||||
assert result["query"]["display_name"] == "RSS Feed URL"
|
||||
|
||||
@patch("lfx.components.data.web_search.requests.get")
|
||||
def test_perform_web_search_success(self, mock_get):
|
||||
"""Test successful web search."""
|
||||
component = WebSearchComponent()
|
||||
component.query = "test query"
|
||||
component.timeout = 5
|
||||
|
||||
# Mock DuckDuckGo response
|
||||
mock_response = Mock()
|
||||
mock_response.text = """
|
||||
<html>
|
||||
<div class="result">
|
||||
<a class="result__a" href="?uddg=https%3A%2F%2Fexample.com">Test Title</a>
|
||||
<a class="result__snippet">Test snippet content</a>
|
||||
</div>
|
||||
</html>
|
||||
"""
|
||||
mock_response.headers = {"content-type": "text/html"}
|
||||
mock_response.raise_for_status.return_value = None
|
||||
|
||||
# Mock the page fetch
|
||||
mock_page_response = Mock()
|
||||
mock_page_response.text = "<html><body>Page content</body></html>"
|
||||
mock_page_response.raise_for_status.return_value = None
|
||||
|
||||
mock_get.side_effect = [mock_response, mock_page_response]
|
||||
|
||||
result = component.perform_web_search()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert not result.empty
|
||||
assert len(result) == 1
|
||||
assert result.iloc[0]["title"] == "Test Title"
|
||||
assert result.iloc[0]["snippet"] == "Test snippet content"
|
||||
assert "Page content" in result.iloc[0]["content"]
|
||||
|
||||
@patch("lfx.components.data.web_search.requests.get")
|
||||
def test_perform_web_search_no_results(self, mock_get):
|
||||
"""Test web search with no results."""
|
||||
component = WebSearchComponent()
|
||||
component.query = "test query"
|
||||
component.timeout = 5
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.text = ""
|
||||
mock_response.headers = {"content-type": "text/html"}
|
||||
mock_response.raise_for_status.return_value = None
|
||||
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
result = component.perform_web_search()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert "No results found" in result.iloc[0]["snippet"]
|
||||
|
||||
@patch("lfx.components.data.web_search.requests.get")
|
||||
def test_perform_web_search_request_error(self, mock_get):
|
||||
"""Test web search with request error."""
|
||||
component = WebSearchComponent()
|
||||
component.query = "test query"
|
||||
component.timeout = 5
|
||||
|
||||
from requests import RequestException
|
||||
|
||||
mock_get.side_effect = RequestException("Connection error")
|
||||
|
||||
result = component.perform_web_search()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert "Connection error" in result.iloc[0]["snippet"]
|
||||
|
||||
@patch("lfx.components.data.web_search.requests.get")
|
||||
def test_perform_news_search_with_query(self, mock_get):
|
||||
"""Test news search with query."""
|
||||
component = WebSearchComponent()
|
||||
component.query = "test news"
|
||||
component.timeout = 5
|
||||
|
||||
# Mock RSS response
|
||||
mock_response = Mock()
|
||||
mock_response.content = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss>
|
||||
<channel>
|
||||
<item>
|
||||
<title>Test News Title</title>
|
||||
<link>https://news.example.com</link>
|
||||
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
||||
<description>Test news description</description>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
result = component.perform_news_search()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert len(result) == 1
|
||||
assert result.iloc[0]["title"] == "Test News Title"
|
||||
assert result.iloc[0]["link"] == "https://news.example.com"
|
||||
assert result.iloc[0]["summary"] == "Test news description"
|
||||
|
||||
@patch("lfx.components.data.web_search.requests.get")
|
||||
def test_perform_news_search_with_topic(self, mock_get):
|
||||
"""Test news search with topic."""
|
||||
component = WebSearchComponent()
|
||||
component.topic = "TECHNOLOGY"
|
||||
component.timeout = 5
|
||||
|
||||
# Mock RSS response
|
||||
mock_response = Mock()
|
||||
mock_response.content = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss>
|
||||
<channel>
|
||||
<item>
|
||||
<title>Tech News</title>
|
||||
<link>https://tech.example.com</link>
|
||||
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
||||
<description>Technology news</description>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
result = component.perform_news_search()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert len(result) == 1
|
||||
# Check that the URL was constructed with topic
|
||||
mock_get.assert_called_once()
|
||||
call_args = mock_get.call_args[0][0]
|
||||
assert "topic/TECHNOLOGY" in call_args
|
||||
|
||||
@patch("lfx.components.data.web_search.requests.get")
|
||||
def test_perform_news_search_no_params(self, mock_get): # noqa: ARG002
|
||||
"""Test news search with no parameters."""
|
||||
component = WebSearchComponent()
|
||||
component.timeout = 5
|
||||
|
||||
result = component.perform_news_search()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert "No search parameters provided" in result.iloc[0]["summary"]
|
||||
|
||||
@patch("lfx.components.data.web_search.requests.get")
|
||||
def test_perform_rss_read_success(self, mock_get):
|
||||
"""Test successful RSS feed reading."""
|
||||
component = WebSearchComponent()
|
||||
component.query = "https://example.com/feed.rss"
|
||||
component.timeout = 5
|
||||
|
||||
# Mock RSS response
|
||||
mock_response = Mock()
|
||||
mock_response.content = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss>
|
||||
<channel>
|
||||
<item>
|
||||
<title>RSS Item 1</title>
|
||||
<link>https://example.com/item1</link>
|
||||
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
||||
<description>Description 1</description>
|
||||
</item>
|
||||
<item>
|
||||
<title>RSS Item 2</title>
|
||||
<link>https://example.com/item2</link>
|
||||
<pubDate>Tue, 02 Jan 2024 00:00:00 GMT</pubDate>
|
||||
<description>Description 2</description>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
result = component.perform_rss_read()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert len(result) == 2
|
||||
assert result.iloc[0]["title"] == "RSS Item 1"
|
||||
assert result.iloc[1]["title"] == "RSS Item 2"
|
||||
|
||||
@patch("lfx.components.data.web_search.requests.get")
|
||||
def test_perform_rss_read_empty_response(self, mock_get):
|
||||
"""Test RSS read with empty response."""
|
||||
component = WebSearchComponent()
|
||||
component.query = "https://example.com/feed.rss"
|
||||
component.timeout = 5
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.content = b""
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
result = component.perform_rss_read()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert "Empty response received" in result.iloc[0]["summary"]
|
||||
|
||||
@patch("lfx.components.data.web_search.requests.get")
|
||||
def test_perform_rss_read_invalid_xml(self, mock_get):
|
||||
"""Test RSS read with invalid XML - returns empty DataFrame when no items found."""
|
||||
component = WebSearchComponent()
|
||||
component.query = "https://example.com/feed.rss"
|
||||
component.timeout = 5
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.content = b"This is not valid XML"
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
result = component.perform_rss_read()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
# When no RSS items are found, it returns an empty DataFrame
|
||||
assert len(result) == 0
|
||||
|
||||
def test_perform_rss_read_no_url(self):
|
||||
"""Test RSS read with no URL provided."""
|
||||
component = WebSearchComponent()
|
||||
component.query = ""
|
||||
|
||||
result = component.perform_rss_read()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
assert "No RSS URL provided" in result.iloc[0]["summary"]
|
||||
|
||||
@patch.object(WebSearchComponent, "perform_web_search")
|
||||
def test_perform_search_web_mode(self, mock_web_search):
|
||||
"""Test perform_search routes to web search in Web mode."""
|
||||
component = WebSearchComponent()
|
||||
component.search_mode = "Web"
|
||||
|
||||
mock_web_search.return_value = DataFrame(pd.DataFrame([{"result": "web"}]))
|
||||
|
||||
result = component.perform_search()
|
||||
|
||||
mock_web_search.assert_called_once()
|
||||
assert result.iloc[0]["result"] == "web"
|
||||
|
||||
@patch.object(WebSearchComponent, "perform_news_search")
|
||||
def test_perform_search_news_mode(self, mock_news_search):
|
||||
"""Test perform_search routes to news search in News mode."""
|
||||
component = WebSearchComponent()
|
||||
component.search_mode = "News"
|
||||
|
||||
mock_news_search.return_value = DataFrame(pd.DataFrame([{"result": "news"}]))
|
||||
|
||||
result = component.perform_search()
|
||||
|
||||
mock_news_search.assert_called_once()
|
||||
assert result.iloc[0]["result"] == "news"
|
||||
|
||||
@patch.object(WebSearchComponent, "perform_rss_read")
|
||||
def test_perform_search_rss_mode(self, mock_rss_read):
|
||||
"""Test perform_search routes to RSS read in RSS mode."""
|
||||
component = WebSearchComponent()
|
||||
component.search_mode = "RSS"
|
||||
|
||||
mock_rss_read.return_value = DataFrame(pd.DataFrame([{"result": "rss"}]))
|
||||
|
||||
result = component.perform_search()
|
||||
|
||||
mock_rss_read.assert_called_once()
|
||||
assert result.iloc[0]["result"] == "rss"
|
||||
|
||||
@patch.object(WebSearchComponent, "perform_web_search")
|
||||
def test_perform_search_fallback(self, mock_web_search):
|
||||
"""Test perform_search falls back to web search for unknown mode."""
|
||||
component = WebSearchComponent()
|
||||
component.search_mode = "UnknownMode"
|
||||
|
||||
mock_web_search.return_value = DataFrame(pd.DataFrame([{"result": "fallback"}]))
|
||||
|
||||
result = component.perform_search()
|
||||
|
||||
mock_web_search.assert_called_once()
|
||||
assert result.iloc[0]["result"] == "fallback"
|
||||
|
||||
def test_empty_query_error(self):
|
||||
"""Test that empty query raises ValueError."""
|
||||
component = WebSearchComponent()
|
||||
component.query = ""
|
||||
component.timeout = 5
|
||||
|
||||
with pytest.raises(ValueError, match="Empty search query"):
|
||||
component.perform_web_search()
|
||||
|
||||
@patch("lfx.components.data.web_search.requests.get")
|
||||
def test_news_search_with_location(self, mock_get):
|
||||
"""Test news search with location parameter."""
|
||||
component = WebSearchComponent()
|
||||
component.location = "San Francisco"
|
||||
component.timeout = 5
|
||||
|
||||
# Mock RSS response
|
||||
mock_response = Mock()
|
||||
mock_response.content = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss>
|
||||
<channel>
|
||||
<item>
|
||||
<title>Local News</title>
|
||||
<link>https://local.example.com</link>
|
||||
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
||||
<description>San Francisco news</description>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
result = component.perform_news_search()
|
||||
|
||||
assert isinstance(result, DataFrame)
|
||||
# Check that the URL was constructed with location
|
||||
mock_get.assert_called_once()
|
||||
call_args = mock_get.call_args[0][0]
|
||||
assert "geo/San%20Francisco" in call_args or "geo/San+Francisco" in call_args
|
||||
|
||||
@ -10,9 +10,6 @@ if TYPE_CHECKING:
|
||||
from lfx.components.data.directory import DirectoryComponent
|
||||
from lfx.components.data.file import FileComponent
|
||||
from lfx.components.data.json_to_data import JSONToDataComponent
|
||||
from lfx.components.data.news_search import NewsSearchComponent
|
||||
from lfx.components.data.rss import RSSReaderComponent
|
||||
from lfx.components.data.save_file import SaveToFileComponent
|
||||
from lfx.components.data.sql_executor import SQLComponent
|
||||
from lfx.components.data.url import URLComponent
|
||||
from lfx.components.data.web_search import WebSearchComponent
|
||||
@ -24,10 +21,7 @@ _dynamic_imports = {
|
||||
"DirectoryComponent": "directory",
|
||||
"FileComponent": "file",
|
||||
"JSONToDataComponent": "json_to_data",
|
||||
"NewsSearchComponent": "news_search",
|
||||
"RSSReaderComponent": "rss",
|
||||
"SQLComponent": "sql_executor",
|
||||
"SaveToFileComponent": "save_file",
|
||||
"URLComponent": "url",
|
||||
"WebSearchComponent": "web_search",
|
||||
"WebhookComponent": "webhook",
|
||||
@ -39,10 +33,7 @@ __all__ = [
|
||||
"DirectoryComponent",
|
||||
"FileComponent",
|
||||
"JSONToDataComponent",
|
||||
"NewsSearchComponent",
|
||||
"RSSReaderComponent",
|
||||
"SQLComponent",
|
||||
"SaveToFileComponent",
|
||||
"URLComponent",
|
||||
"WebSearchComponent",
|
||||
"WebhookComponent",
|
||||
|
||||
@ -1,164 +0,0 @@
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from lfx.custom import Component
|
||||
from lfx.io import IntInput, MessageTextInput, Output
|
||||
from lfx.schema import DataFrame
|
||||
|
||||
|
||||
class NewsSearchComponent(Component):
|
||||
display_name = "News Search"
|
||||
description = "Searches Google News via RSS. Returns clean article data."
|
||||
documentation: str = "https://docs.langflow.org/components-data#news-search"
|
||||
icon = "newspaper"
|
||||
name = "NewsSearch"
|
||||
|
||||
inputs = [
|
||||
MessageTextInput(
|
||||
name="query",
|
||||
display_name="Search Query",
|
||||
info="Search keywords for news articles.",
|
||||
tool_mode=True,
|
||||
required=True,
|
||||
),
|
||||
MessageTextInput(
|
||||
name="hl",
|
||||
display_name="Language (hl)",
|
||||
info="Language code, e.g. en-US, fr, de. Default: en-US.",
|
||||
tool_mode=False,
|
||||
input_types=[],
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
MessageTextInput(
|
||||
name="gl",
|
||||
display_name="Country (gl)",
|
||||
info="Country code, e.g. US, FR, DE. Default: US.",
|
||||
tool_mode=False,
|
||||
input_types=[],
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
MessageTextInput(
|
||||
name="ceid",
|
||||
display_name="Country:Language (ceid)",
|
||||
info="e.g. US:en, FR:fr. Default: US:en.",
|
||||
tool_mode=False,
|
||||
value="US:en",
|
||||
input_types=[],
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
MessageTextInput(
|
||||
name="topic",
|
||||
display_name="Topic",
|
||||
info="One of: WORLD, NATION, BUSINESS, TECHNOLOGY, ENTERTAINMENT, SCIENCE, SPORTS, HEALTH.",
|
||||
tool_mode=False,
|
||||
input_types=[],
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
MessageTextInput(
|
||||
name="location",
|
||||
display_name="Location (Geo)",
|
||||
info="City, state, or country for location-based news. Leave blank for keyword search.",
|
||||
tool_mode=False,
|
||||
input_types=[],
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
IntInput(
|
||||
name="timeout",
|
||||
display_name="Timeout",
|
||||
info="Timeout for the request in seconds.",
|
||||
value=5,
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
]
|
||||
|
||||
outputs = [Output(name="articles", display_name="News Articles", method="search_news")]
|
||||
|
||||
def search_news(self) -> DataFrame:
|
||||
# Defaults
|
||||
hl = getattr(self, "hl", None) or "en-US"
|
||||
gl = getattr(self, "gl", None) or "US"
|
||||
ceid = getattr(self, "ceid", None) or f"{gl}:{hl.split('-')[0]}"
|
||||
topic = getattr(self, "topic", None)
|
||||
location = getattr(self, "location", None)
|
||||
query = getattr(self, "query", None)
|
||||
|
||||
# Build base URL
|
||||
if topic:
|
||||
# Topic-based feed
|
||||
base_url = f"https://news.google.com/rss/headlines/section/topic/{quote_plus(topic.upper())}"
|
||||
params = f"?hl={hl}&gl={gl}&ceid={ceid}"
|
||||
rss_url = base_url + params
|
||||
elif location:
|
||||
# Location-based feed
|
||||
base_url = f"https://news.google.com/rss/headlines/section/geo/{quote_plus(location)}"
|
||||
params = f"?hl={hl}&gl={gl}&ceid={ceid}"
|
||||
rss_url = base_url + params
|
||||
elif query:
|
||||
# Keyword search feed
|
||||
base_url = "https://news.google.com/rss/search?q="
|
||||
query_parts = [query]
|
||||
query_encoded = quote_plus(" ".join(query_parts))
|
||||
params = f"&hl={hl}&gl={gl}&ceid={ceid}"
|
||||
rss_url = f"{base_url}{query_encoded}{params}"
|
||||
else:
|
||||
self.status = "No search query, topic, or location provided."
|
||||
self.log(self.status)
|
||||
return DataFrame(
|
||||
pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"title": "Error",
|
||||
"link": "",
|
||||
"published": "",
|
||||
"summary": "No search query, topic, or location provided.",
|
||||
}
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
response = requests.get(rss_url, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
soup = BeautifulSoup(response.content, "xml")
|
||||
items = soup.find_all("item")
|
||||
except requests.RequestException as e:
|
||||
self.status = f"Failed to fetch news: {e}"
|
||||
self.log(self.status)
|
||||
return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}]))
|
||||
except (AttributeError, ValueError, TypeError) as e:
|
||||
self.status = f"Unexpected error: {e!s}"
|
||||
self.log(self.status)
|
||||
return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}]))
|
||||
|
||||
if not items:
|
||||
self.status = "No news articles found."
|
||||
self.log(self.status)
|
||||
return DataFrame(pd.DataFrame([{"title": "No articles found", "link": "", "published": "", "summary": ""}]))
|
||||
|
||||
articles = []
|
||||
for item in items:
|
||||
try:
|
||||
title = self.clean_html(item.title.text if item.title else "")
|
||||
link = item.link.text if item.link else ""
|
||||
published = item.pubDate.text if item.pubDate else ""
|
||||
summary = self.clean_html(item.description.text if item.description else "")
|
||||
articles.append({"title": title, "link": link, "published": published, "summary": summary})
|
||||
except (AttributeError, ValueError, TypeError) as e:
|
||||
self.log(f"Error parsing article: {e!s}")
|
||||
continue
|
||||
|
||||
df_articles = pd.DataFrame(articles)
|
||||
self.log(f"Found {len(df_articles)} articles.")
|
||||
return DataFrame(df_articles)
|
||||
|
||||
def clean_html(self, html_string: str) -> str:
|
||||
return BeautifulSoup(html_string, "html.parser").get_text(separator=" ", strip=True)
|
||||
@ -1,69 +0,0 @@
|
||||
import pandas as pd
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from lfx.custom import Component
|
||||
from lfx.io import IntInput, MessageTextInput, Output
|
||||
from lfx.log.logger import logger
|
||||
from lfx.schema import DataFrame
|
||||
|
||||
|
||||
class RSSReaderComponent(Component):
|
||||
display_name = "RSS Reader"
|
||||
description = "Fetches and parses an RSS feed."
|
||||
documentation: str = "https://docs.langflow.org/components-data#rss-reader"
|
||||
icon = "rss"
|
||||
name = "RSSReaderSimple"
|
||||
|
||||
inputs = [
|
||||
MessageTextInput(
|
||||
name="rss_url",
|
||||
display_name="RSS Feed URL",
|
||||
info="URL of the RSS feed to parse.",
|
||||
tool_mode=True,
|
||||
required=True,
|
||||
),
|
||||
IntInput(
|
||||
name="timeout",
|
||||
display_name="Timeout",
|
||||
info="Timeout for the RSS feed request.",
|
||||
value=5,
|
||||
advanced=True,
|
||||
),
|
||||
]
|
||||
|
||||
outputs = [Output(name="articles", display_name="Articles", method="read_rss")]
|
||||
|
||||
def read_rss(self) -> DataFrame:
|
||||
try:
|
||||
response = requests.get(self.rss_url, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
if not response.content.strip():
|
||||
msg = "Empty response received"
|
||||
raise ValueError(msg)
|
||||
# Check if the response is valid XML
|
||||
try:
|
||||
BeautifulSoup(response.content, "xml")
|
||||
except Exception as e:
|
||||
msg = f"Invalid XML response: {e}"
|
||||
raise ValueError(msg) from e
|
||||
soup = BeautifulSoup(response.content, "xml")
|
||||
items = soup.find_all("item")
|
||||
except (requests.RequestException, ValueError) as e:
|
||||
self.status = f"Failed to fetch RSS: {e}"
|
||||
return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}]))
|
||||
|
||||
articles = [
|
||||
{
|
||||
"title": item.title.text if item.title else "",
|
||||
"link": item.link.text if item.link else "",
|
||||
"published": item.pubDate.text if item.pubDate else "",
|
||||
"summary": item.description.text if item.description else "",
|
||||
}
|
||||
for item in items
|
||||
]
|
||||
|
||||
# Ensure the DataFrame has the correct columns even if empty
|
||||
df_articles = pd.DataFrame(articles, columns=["title", "link", "published", "summary"])
|
||||
logger.info(f"Fetched {len(df_articles)} articles.")
|
||||
return DataFrame(df_articles)
|
||||
@ -1,43 +1,133 @@
|
||||
"""Unified Web Search Component.
|
||||
|
||||
This component consolidates Web Search, News Search, and RSS Reader into a single
|
||||
component with tabs for different search modes.
|
||||
"""
|
||||
|
||||
import re
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, quote_plus, unquote, urlparse
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from lfx.custom import Component
|
||||
from lfx.io import IntInput, MessageTextInput, Output
|
||||
from lfx.io import IntInput, MessageTextInput, Output, TabInput
|
||||
from lfx.schema import DataFrame
|
||||
from lfx.utils.request_utils import get_user_agent
|
||||
|
||||
|
||||
class WebSearchComponent(Component):
|
||||
display_name = "Web Search"
|
||||
description = "Performs a basic DuckDuckGo search (HTML scraping). May be subject to rate limits."
|
||||
description = "Search the web, news, or RSS feeds."
|
||||
documentation: str = "https://docs.langflow.org/components-data#web-search"
|
||||
icon = "search"
|
||||
name = "WebSearchNoAPI"
|
||||
name = "UnifiedWebSearch"
|
||||
|
||||
inputs = [
|
||||
TabInput(
|
||||
name="search_mode",
|
||||
display_name="Search Mode",
|
||||
options=["Web", "News", "RSS"],
|
||||
info="Choose search mode: Web (DuckDuckGo), News (Google News), or RSS (Feed Reader)",
|
||||
value="Web",
|
||||
real_time_refresh=True,
|
||||
tool_mode=True,
|
||||
),
|
||||
MessageTextInput(
|
||||
name="query",
|
||||
display_name="Search Query",
|
||||
info="Keywords to search for.",
|
||||
info="Search keywords for news articles.",
|
||||
tool_mode=True,
|
||||
required=True,
|
||||
),
|
||||
MessageTextInput(
|
||||
name="hl",
|
||||
display_name="Language (hl)",
|
||||
info="Language code, e.g. en-US, fr, de. Default: en-US.",
|
||||
tool_mode=False,
|
||||
input_types=[],
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
MessageTextInput(
|
||||
name="gl",
|
||||
display_name="Country (gl)",
|
||||
info="Country code, e.g. US, FR, DE. Default: US.",
|
||||
tool_mode=False,
|
||||
input_types=[],
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
MessageTextInput(
|
||||
name="ceid",
|
||||
display_name="Country:Language (ceid)",
|
||||
info="e.g. US:en, FR:fr. Default: US:en.",
|
||||
tool_mode=False,
|
||||
value="US:en",
|
||||
input_types=[],
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
MessageTextInput(
|
||||
name="topic",
|
||||
display_name="Topic",
|
||||
info="One of: WORLD, NATION, BUSINESS, TECHNOLOGY, ENTERTAINMENT, SCIENCE, SPORTS, HEALTH.",
|
||||
tool_mode=False,
|
||||
input_types=[],
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
MessageTextInput(
|
||||
name="location",
|
||||
display_name="Location (Geo)",
|
||||
info="City, state, or country for location-based news. Leave blank for keyword search.",
|
||||
tool_mode=False,
|
||||
input_types=[],
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
IntInput(
|
||||
name="timeout",
|
||||
display_name="Timeout",
|
||||
info="Timeout for the web search request.",
|
||||
info="Timeout for the request in seconds.",
|
||||
value=5,
|
||||
required=False,
|
||||
advanced=True,
|
||||
),
|
||||
]
|
||||
|
||||
outputs = [Output(name="results", display_name="Search Results", method="perform_search")]
|
||||
outputs = [Output(name="results", display_name="Results", method="perform_search")]
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:
|
||||
"""Update input visibility based on search mode."""
|
||||
if field_name == "search_mode":
|
||||
# Show/hide inputs based on search mode
|
||||
is_news = field_value == "News"
|
||||
is_rss = field_value == "RSS"
|
||||
|
||||
# Update query field info based on mode
|
||||
if is_rss:
|
||||
build_config["query"]["info"] = "RSS feed URL to parse"
|
||||
build_config["query"]["display_name"] = "RSS Feed URL"
|
||||
elif is_news:
|
||||
build_config["query"]["info"] = "Search keywords for news articles."
|
||||
build_config["query"]["display_name"] = "Search Query"
|
||||
else: # Web
|
||||
build_config["query"]["info"] = "Keywords to search for"
|
||||
build_config["query"]["display_name"] = "Search Query"
|
||||
|
||||
# Keep news-specific fields as advanced (matching original News Search component)
|
||||
# They remain advanced=True in all modes, just like in the original component
|
||||
|
||||
return build_config
|
||||
|
||||
def validate_url(self, string: str) -> bool:
|
||||
"""Validate URL format."""
|
||||
url_regex = re.compile(
|
||||
r"^(https?:\/\/)?" r"(www\.)?" r"([a-zA-Z0-9.-]+)" r"(\.[a-zA-Z]{2,})?" r"(:\d+)?" r"(\/[^\s]*)?$",
|
||||
re.IGNORECASE,
|
||||
@ -45,6 +135,7 @@ class WebSearchComponent(Component):
|
||||
return bool(url_regex.match(string))
|
||||
|
||||
def ensure_url(self, url: str) -> str:
|
||||
"""Ensure URL has proper protocol."""
|
||||
if not url.startswith(("http://", "https://")):
|
||||
url = "https://" + url
|
||||
if not self.validate_url(url):
|
||||
@ -54,14 +145,19 @@ class WebSearchComponent(Component):
|
||||
|
||||
def _sanitize_query(self, query: str) -> str:
|
||||
"""Sanitize search query."""
|
||||
# Remove potentially dangerous characters
|
||||
return re.sub(r'[<>"\']', "", query.strip())
|
||||
|
||||
def perform_search(self) -> DataFrame:
|
||||
def clean_html(self, html_string: str) -> str:
|
||||
"""Remove HTML tags from text."""
|
||||
return BeautifulSoup(html_string, "html.parser").get_text(separator=" ", strip=True)
|
||||
|
||||
def perform_web_search(self) -> DataFrame:
|
||||
"""Perform DuckDuckGo web search."""
|
||||
query = self._sanitize_query(self.query)
|
||||
if not query:
|
||||
msg = "Empty search query"
|
||||
raise ValueError(msg)
|
||||
|
||||
headers = {"User-Agent": get_user_agent()}
|
||||
params = {"q": query, "kl": "us-en"}
|
||||
url = "https://html.duckduckgo.com/html/"
|
||||
@ -78,6 +174,7 @@ class WebSearchComponent(Component):
|
||||
return DataFrame(
|
||||
pd.DataFrame([{"title": "Error", "link": "", "snippet": "No results found", "content": ""}])
|
||||
)
|
||||
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
results = []
|
||||
|
||||
@ -108,5 +205,122 @@ class WebSearchComponent(Component):
|
||||
}
|
||||
)
|
||||
|
||||
df_results = pd.DataFrame(results)
|
||||
return DataFrame(df_results)
|
||||
return DataFrame(pd.DataFrame(results))
|
||||
|
||||
def perform_news_search(self) -> DataFrame:
|
||||
"""Perform Google News search."""
|
||||
query = getattr(self, "query", "")
|
||||
hl = getattr(self, "hl", "en-US") or "en-US"
|
||||
gl = getattr(self, "gl", "US") or "US"
|
||||
topic = getattr(self, "topic", None)
|
||||
location = getattr(self, "location", None)
|
||||
|
||||
ceid = f"{gl}:{hl.split('-')[0]}"
|
||||
|
||||
# Build RSS URL based on parameters
|
||||
if topic:
|
||||
# Topic-based feed
|
||||
base_url = f"https://news.google.com/rss/headlines/section/topic/{quote_plus(topic.upper())}"
|
||||
params = f"?hl={hl}&gl={gl}&ceid={ceid}"
|
||||
rss_url = base_url + params
|
||||
elif location:
|
||||
# Location-based feed
|
||||
base_url = f"https://news.google.com/rss/headlines/section/geo/{quote_plus(location)}"
|
||||
params = f"?hl={hl}&gl={gl}&ceid={ceid}"
|
||||
rss_url = base_url + params
|
||||
elif query:
|
||||
# Keyword search feed
|
||||
base_url = "https://news.google.com/rss/search?q="
|
||||
query_encoded = quote_plus(query)
|
||||
params = f"&hl={hl}&gl={gl}&ceid={ceid}"
|
||||
rss_url = f"{base_url}{query_encoded}{params}"
|
||||
else:
|
||||
self.status = "No search query, topic, or location provided."
|
||||
return DataFrame(
|
||||
pd.DataFrame(
|
||||
[{"title": "Error", "link": "", "published": "", "summary": "No search parameters provided"}]
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
response = requests.get(rss_url, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
soup = BeautifulSoup(response.content, "xml")
|
||||
items = soup.find_all("item")
|
||||
except requests.RequestException as e:
|
||||
self.status = f"Failed to fetch news: {e}"
|
||||
return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}]))
|
||||
|
||||
if not items:
|
||||
self.status = "No news articles found."
|
||||
return DataFrame(pd.DataFrame([{"title": "No articles found", "link": "", "published": "", "summary": ""}]))
|
||||
|
||||
articles = []
|
||||
for item in items:
|
||||
try:
|
||||
title = self.clean_html(item.title.text if item.title else "")
|
||||
link = item.link.text if item.link else ""
|
||||
published = item.pubDate.text if item.pubDate else ""
|
||||
summary = self.clean_html(item.description.text if item.description else "")
|
||||
articles.append({"title": title, "link": link, "published": published, "summary": summary})
|
||||
except (AttributeError, ValueError, TypeError) as e:
|
||||
self.log(f"Error parsing article: {e!s}")
|
||||
continue
|
||||
|
||||
return DataFrame(pd.DataFrame(articles))
|
||||
|
||||
def perform_rss_read(self) -> DataFrame:
|
||||
"""Read RSS feed."""
|
||||
rss_url = getattr(self, "query", "")
|
||||
if not rss_url:
|
||||
return DataFrame(
|
||||
pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": "No RSS URL provided"}])
|
||||
)
|
||||
|
||||
try:
|
||||
response = requests.get(rss_url, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
if not response.content.strip():
|
||||
msg = "Empty response received"
|
||||
raise ValueError(msg)
|
||||
|
||||
# Validate XML
|
||||
try:
|
||||
BeautifulSoup(response.content, "xml")
|
||||
except Exception as e:
|
||||
msg = f"Invalid XML response: {e}"
|
||||
raise ValueError(msg) from e
|
||||
|
||||
soup = BeautifulSoup(response.content, "xml")
|
||||
items = soup.find_all("item")
|
||||
except (requests.RequestException, ValueError) as e:
|
||||
self.status = f"Failed to fetch RSS: {e}"
|
||||
return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}]))
|
||||
|
||||
articles = [
|
||||
{
|
||||
"title": item.title.text if item.title else "",
|
||||
"link": item.link.text if item.link else "",
|
||||
"published": item.pubDate.text if item.pubDate else "",
|
||||
"summary": item.description.text if item.description else "",
|
||||
}
|
||||
for item in items
|
||||
]
|
||||
|
||||
# Ensure DataFrame has correct columns even if empty
|
||||
df_articles = pd.DataFrame(articles, columns=["title", "link", "published", "summary"])
|
||||
self.log(f"Fetched {len(df_articles)} articles.")
|
||||
return DataFrame(df_articles)
|
||||
|
||||
def perform_search(self) -> DataFrame:
|
||||
"""Main search method that routes to appropriate search function based on mode."""
|
||||
search_mode = getattr(self, "search_mode", "Web")
|
||||
|
||||
if search_mode == "Web":
|
||||
return self.perform_web_search()
|
||||
if search_mode == "News":
|
||||
return self.perform_news_search()
|
||||
if search_mode == "RSS":
|
||||
return self.perform_rss_read()
|
||||
# Fallback to web search
|
||||
return self.perform_web_search()
|
||||
|
||||
Reference in New Issue
Block a user