Source code for paper_firehose.processors.feed_processor

"""
RSS feed processing functionality.
Fetches RSS feeds, applies regex filters, and manages entry storage.
"""

import feedparser
import feedparser.http
import io
import re
import time
import datetime
import urllib.parse
from typing import Dict, List, Any
import logging

import requests
import urllib3.exceptions

from ..core.database import DatabaseManager
from ..core.config import ConfigManager

logger = logging.getLogger(__name__)

# Default time window for processing entries (days); can be overridden by config.defaults.time_window_days
DEFAULT_TIME_WINDOW_DAYS = 365

# Per-feed HTTP timeout (seconds); can be overridden by config.defaults.feed_timeout_seconds.
# Bounds each read and the download as a whole.
DEFAULT_FEED_TIMEOUT_SECONDS = 30

# Largest feed body accepted (bytes, after decompression). The biggest
# configured feeds (Wiley, ~500 entries) are a few MB.
FEED_MAX_BYTES = 20 * 1024 * 1024


[docs] class FeedFetchError(Exception): """A feed could not be fetched or parsed."""
[docs] class FeedProcessor: """Processes RSS feeds with regex filtering and database storage.""" def __init__(self, db_manager: DatabaseManager, config_manager: ConfigManager): """Bind database/config managers and derive the time window constraint.""" self.db = db_manager self.config = config_manager # Resolve time window from config (defaults.time_window_days) cfg = self.config.load_config() defaults = cfg.get('defaults') or {} days = int(defaults.get('time_window_days', DEFAULT_TIME_WINDOW_DAYS)) self.time_delta = datetime.timedelta(days=days) self.feed_timeout = float(defaults.get('feed_timeout_seconds', DEFAULT_FEED_TIMEOUT_SECONDS)) self.feed_max_bytes = FEED_MAX_BYTES # One session for the whole run: it keeps cookies across redirects, # which publishers that answer with a cookie check need (Atypon # ``/action/showFeed`` feeds such as ChemRxiv). Same User-Agent and # Accept header as feedparser sends on its own. self.session = requests.Session() self.session.headers.update({ 'User-Agent': feedparser.USER_AGENT, 'Accept': feedparser.http.ACCEPT_HEADER, }) # Feeds that could not be fetched or parsed during this run, so that # callers can tell "nothing new" from "could not read the feed". self.failed_feeds: List[Dict[str, Any]] = [] def _fetch_feed(self, url: str) -> feedparser.FeedParserDict: """Fetch and parse one feed; raise :class:`FeedFetchError` if it cannot be read. HTTP(S) feeds go through ``self.session`` with a timeout; other URLs (``file://``, local paths) are handed to feedparser directly. """ scheme = urllib.parse.urlparse(url).scheme.lower() if scheme in ('http', 'https'): body, status, final_url, headers = self._download(url) # Resolve relative links against the final URL after redirects. headers.setdefault('content-location', final_url) # Wrap the body in a stream: given bare bytes, feedparser first # tries open() on them as a file name, so a server answering with # a path would make us read that local file (issue #90). feed = feedparser.parse(io.BytesIO(body), response_headers=headers) feed['status'] = status feed['href'] = final_url else: feed = feedparser.parse(url) status = feed.get('status') if status is not None and status >= 400: raise FeedFetchError(f"HTTP {status}") if feed.bozo and not feed.entries: raise FeedFetchError(f"could not parse feed: {feed.get('bozo_exception')}") return feed def _download(self, url: str): """GET *url* through the session with bounded size and time. Returns ``(body, status_code, final_url, lowercased_headers)``. The body is streamed and the download is abandoned once it exceeds ``self.feed_max_bytes`` (counted after gzip/deflate decoding) or once ``self.feed_timeout`` seconds have passed in total. The per-read ``timeout`` alone would not stop a server that trickles bytes. Reads with urllib3's ``read1``, which returns whatever has arrived instead of waiting for a full chunk (``iter_content`` would block until 64 KB or the end of a small body), so the deadline is checked as data trickles in. """ deadline = time.monotonic() + self.feed_timeout try: with self.session.get(url, timeout=self.feed_timeout, stream=True) as response: if response.status_code >= 400: raise FeedFetchError(f"HTTP {response.status_code} from {response.url}") # Chained encodings ("gzip, gzip") multiply the expansion # ratio and no feed needs them (issue #91). encodings = [ e.strip().lower() for e in response.headers.get('Content-Encoding', '').split(',') if e.strip() and e.strip().lower() != 'identity' ] if len(encodings) > 1: raise FeedFetchError( f"refused chained Content-Encoding '{', '.join(encodings)}' from {response.url}" ) chunks = [] size = 0 while True: chunk = response.raw.read1(64 * 1024, decode_content=True) if not chunk: break size += len(chunk) if size > self.feed_max_bytes: raise FeedFetchError( f"response larger than {self.feed_max_bytes} bytes from {response.url}" ) if time.monotonic() > deadline: raise FeedFetchError( f"download took longer than {self.feed_timeout:g} s from {response.url}" ) chunks.append(chunk) headers = {k.lower(): v for k, v in response.headers.items()} return b"".join(chunks), response.status_code, response.url, headers except (requests.RequestException, urllib3.exceptions.HTTPError) as e: # urllib3 errors come from reading ``response.raw`` directly # (read timeouts, broken connections, bad gzip data). raise FeedFetchError(f"request failed: {e}") from e def _record_failure(self, feed_key: str, name: str, url: str, error: str) -> None: """Remember a feed that could not be read (once per feed per run).""" if any(f['feed'] == feed_key for f in self.failed_feeds): return self.failed_feeds.append({'feed': feed_key, 'name': name, 'url': url, 'error': error})
[docs] def fetch_feeds(self, topic_name: str) -> Dict[str, List[Dict[str, Any]]]: """ Fetch RSS feeds for a topic and return new entries. Returns: Dict mapping feed names to lists of new entries """ topic_config = self.config.load_topic_config(topic_name) feeds_to_process = topic_config['feeds'] enabled_feeds = self.config.get_enabled_feeds() new_entries_per_feed = {} current_time = datetime.datetime.now() for feed_key in feeds_to_process: if feed_key not in enabled_feeds: logger.warning(f"Feed '{feed_key}' not enabled, skipping") continue feed_config = enabled_feeds[feed_key] feed_url = feed_config['url'] feed_display_name = feed_config.get('name', feed_key) logger.info(f"Processing feed '{feed_display_name}' for topic '{topic_name}'") try: # Fetch and parse RSS feed try: feed = self._fetch_feed(feed_url) except FeedFetchError as e: logger.warning(f"Could not read feed '{feed_display_name}' ({feed_url}): {e}") self._record_failure(feed_key, feed_display_name, feed_url, str(e)) new_entries_per_feed[feed_key] = [] continue if feed.bozo: logger.warning(f"Feed '{feed_display_name}' has parsing issues: {feed.bozo_exception}") feed_entries = feed.entries logger.debug(f"Feed '{feed_display_name}' returned {len(feed_entries)} raw entries") feed_title = getattr(feed.feed, 'title', feed_display_name) # Add feed metadata to each entry for entry in feed_entries: entry['feed_title'] = feed_title candidates = [] for entry in feed_entries: # Check if entry is within time window entry_published = entry.get('published_parsed') or entry.get('updated_parsed') if entry_published: if isinstance(entry_published, time.struct_time): entry_datetime = datetime.datetime(*entry_published[:6]) else: entry_datetime = entry_published else: entry_datetime = current_time # Skip entries older than configured time window if (current_time - entry_datetime) > self.time_delta: continue candidates.append(entry) # Keep entries whose title is not in the dedup DB yet # (one batched lookup per feed). new_titles = self.db.filter_new_titles( [e.get('title', '').strip() for e in candidates] ) new_entries = [e for e in candidates if e.get('title', '').strip() in new_titles] for entry in new_entries: logger.debug(f"New entry found: {entry.get('title', '').strip()[:50]}...") new_entries_per_feed[feed_key] = new_entries logger.info(f"Found {len(new_entries)} new entries in feed '{feed_display_name}'") except Exception as e: logger.error(f"Error processing feed '{feed_display_name}': {e}") self._record_failure(feed_key, feed_display_name, feed_url, f"processing error: {e}") new_entries_per_feed[feed_key] = [] return new_entries_per_feed
[docs] def apply_filters(self, entries_per_feed: Dict[str, List[Dict[str, Any]]], topic_name: str) -> List[Dict[str, Any]]: """ Apply regex filters to entries and return matched entries. Args: entries_per_feed: Dict mapping feed names to entry lists topic_name: Name of the topic to filter for Returns: List of entries that match the topic's regex filter """ topic_config = self.config.load_topic_config(topic_name) filter_config = topic_config['filter'] pattern = filter_config['pattern'] fields = filter_config.get('fields', ['title', 'summary']) # Compile regex pattern try: regex = re.compile(pattern, re.IGNORECASE) except re.error as e: logger.error(f"Invalid regex pattern for topic '{topic_name}': {e}") return [] matched_entries = [] priority_journals = self.config.get_priority_journals() enabled_feeds = self.config.get_enabled_feeds() for feed_key, entries in entries_per_feed.items(): is_priority_feed = feed_key in priority_journals feed_display_name = enabled_feeds.get(feed_key, {}).get('name', feed_key) for entry in entries: entry_id = self.db.compute_entry_id(entry) # Check if entry matches regex pattern matches_regex = self._matches_pattern(entry, regex, fields) # Only include entries that match the regex pattern # Priority status is preserved for future LLM ranking/summarization if matches_regex: # Add metadata entry['entry_id'] = entry_id entry['feed_name'] = feed_display_name entry['topic'] = topic_name entry['is_priority'] = is_priority_feed # Save to matched_entries_history.db if topic has archive: true topic_config = self.config.load_topic_config(topic_name) output_config = topic_config.get('output', {}) if output_config.get('archive', False): self.db.save_matched_entry(entry, feed_display_name, topic_name, entry_id) # Save to papers.db for current run processing self.db.save_current_entry(entry, feed_display_name, topic_name, entry_id) matched_entries.append(entry) logger.debug(f"Entry matched for topic '{topic_name}': {entry.get('title', 'No title')[:50]}... (priority: {is_priority_feed})") logger.info(f"Found {len(matched_entries)} entries matching filters for topic '{topic_name}'") return matched_entries
def _matches_pattern(self, entry: Dict[str, Any], regex: re.Pattern, fields: List[str]) -> bool: """Check if entry matches the regex pattern in specified fields.""" for field in fields: text = "" if field == 'title': text = entry.get('title', '') elif field == 'summary': text = entry.get('summary', entry.get('description', '')) elif field == 'authors': authors = entry.get('authors', []) if authors: text = ', '.join(author.get('name', '') for author in authors) else: text = entry.get('author', '') if text and regex.search(text): return True return False
[docs] def save_all_entries_to_dedup_db(self, all_entries_per_feed: Dict[str, List[Dict[str, Any]]]): """Save ALL processed entries to all_feed_entries.db for deduplication.""" enabled_feeds = self.config.get_enabled_feeds() items = [] for feed_key, entries in all_entries_per_feed.items(): display_name = enabled_feeds.get(feed_key, {}).get('name', feed_key) for entry in entries: items.append((entry, display_name, self.db.compute_entry_id(entry))) self.db.save_feed_entries(items) logger.info(f"Saved all processed entries to deduplication database")
[docs] def prune_old_feed_entries(self) -> int: """Delete entries from all_feed_entries.db older than the configured time window. Keeps the dedup database bounded to the same age horizon used at ingestion (``defaults.time_window_days``, default 365 days), so entries older than that window are removed rather than accumulating forever. Returns the number of rows deleted. """ return self.db.delete_all_feeds_older_than(self.time_delta.days)