aitrade/pkg/app/news/sources.go

package news

import (
	"context"
	"fmt"
	"log/slog"
	"net/http"
	"strings"
	"time"

	"github.com/mmcdole/gofeed"
	"github.com/pheinrich/aitrade/pkg/model"
)

type Source interface {
	Name() string
	Fetch(ctx context.Context) ([]*model.NewsArticle, error)
}

// RSSSource fetches news from RSS feeds
type RSSSource struct {
	name       string
	url        string
	parser     *gofeed.Parser
	logger     *slog.Logger
	httpClient *http.Client
	headers    map[string]string
	username   string
	password   string

	// Rate limiting
	maxPerHour    int
	maxPerDay     int
	hourlyCounter int
	dailyCounter  int
	lastHourReset time.Time
	lastDayReset  time.Time
}

func NewRSSSource(name, url string, logger *slog.Logger) *RSSSource {
	now := time.Now()
	return &RSSSource{
		name:          name,
		url:           url,
		parser:        gofeed.NewParser(),
		logger:        logger,
		httpClient:    &http.Client{Timeout: 30 * time.Second},
		headers:       make(map[string]string),
		lastHourReset: now,
		lastDayReset:  now,
	}
}

func (s *RSSSource) SetRateLimit(maxPerHour, maxPerDay int) {
	s.maxPerHour = maxPerHour
	s.maxPerDay = maxPerDay
}

func (s *RSSSource) SetBasicAuth(username, password string) {
	s.username = username
	s.password = password
}

func (s *RSSSource) AddHeader(key, value string) {
	s.headers[key] = value
}

func (s *RSSSource) Name() string {
	return s.name
}

func (s *RSSSource) Fetch(ctx context.Context) ([]*model.NewsArticle, error) {
	// Check rate limits
	if !s.checkRateLimit() {
		s.logger.Warn("rate limit reached, skipping fetch",
			slog.String("source", s.name),
			slog.Int("hourly", s.hourlyCounter),
			slog.Int("max_per_hour", s.maxPerHour),
			slog.Int("daily", s.dailyCounter),
			slog.Int("max_per_day", s.maxPerDay))
		return nil, nil // Return empty, not an error
	}

	s.logger.Debug("fetching RSS feed", slog.String("source", s.name), slog.String("url", s.url))

	// Create HTTP request with custom headers and auth
	req, err := http.NewRequestWithContext(ctx, "GET", s.url, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request: %w", err)
	}

	// Add custom headers
	for key, value := range s.headers {
		req.Header.Set(key, value)
	}

	// Add basic auth if configured
	if s.username != "" {
		req.SetBasicAuth(s.username, s.password)
	}

	// Fetch the feed
	resp, err := s.httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch feed: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("http error: %s", resp.Status)
	}

	// Parse the feed
	feed, err := s.parser.Parse(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to parse RSS feed: %w", err)
	}

	var articles []*model.NewsArticle
	for _, item := range feed.Items {
		publishedAt := time.Now()
		if item.PublishedParsed != nil {
			publishedAt = *item.PublishedParsed
		} else if item.UpdatedParsed != nil {
			publishedAt = *item.UpdatedParsed
		}

		content := item.Description
		if item.Content != "" {
			content = item.Content
		}

		article := &model.NewsArticle{
			Source:      s.name,
			Title:       item.Title,
			URL:         item.Link,
			Content:     content,
			PublishedAt: publishedAt,
			FetchedAt:   time.Now(),
			Symbols:     extractSymbols(item.Title + " " + content),
		}

		articles = append(articles, article)
	}

	s.logger.Debug("fetched articles",
		slog.String("source", s.name),
		slog.Int("count", len(articles)),
	)

	// Increment rate limit counters
	s.incrementCounters()

	return articles, nil
}

// checkRateLimit checks if we can make a request based on rate limits
func (s *RSSSource) checkRateLimit() bool {
	now := time.Now()

	// Reset hourly counter if an hour has passed
	if now.Sub(s.lastHourReset) >= time.Hour {
		s.hourlyCounter = 0
		s.lastHourReset = now
	}

	// Reset daily counter if a day has passed
	if now.Sub(s.lastDayReset) >= 24*time.Hour {
		s.dailyCounter = 0
		s.lastDayReset = now
	}

	// Check hourly limit (0 means unlimited)
	if s.maxPerHour > 0 && s.hourlyCounter >= s.maxPerHour {
		return false
	}

	// Check daily limit (0 means unlimited)
	if s.maxPerDay > 0 && s.dailyCounter >= s.maxPerDay {
		return false
	}

	return true
}

// incrementCounters increments the rate limit counters after a successful fetch
func (s *RSSSource) incrementCounters() {
	s.hourlyCounter++
	s.dailyCounter++
}

// extractSymbols extracts potential stock symbols from text
// Simple implementation - looks for uppercase words 1-5 chars long
func extractSymbols(text string) string {
	words := strings.Fields(text)
	var symbols []string
	seen := make(map[string]bool)

	for _, word := range words {
		// Clean word
		word = strings.Trim(word, ".,!?;:\"'()")

		// Check if it looks like a stock symbol
		if len(word) >= 1 && len(word) <= 5 && isAllUppercase(word) {
			if !seen[word] {
				symbols = append(symbols, word)
				seen[word] = true
			}
		}
	}

	return strings.Join(symbols, ",")
}

func isAllUppercase(s string) bool {
	for _, r := range s {
		if r < 'A' || r > 'Z' {
			return false
		}
	}
	return len(s) > 0
}