parallel source fetch with timeout

Signed-off-by: kaedwen <kaedwen@heinrich.blue>

pkg/app/app.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"time"
 
 	"github.com/pheinrich/aitrade/pkg/app/client"
 	"github.com/pheinrich/aitrade/pkg/app/news"
@@ -60,7 +61,11 @@ func New(cfg *config.Config, logger *slog.Logger) (*Application, error) {
 	}
 
 	// Create news aggregator
-	newsAgg := news.NewAggregator(newsRepo, cfg.News.PollInterval.Duration, llmScorer, logger)
+	fetchTimeout := cfg.News.FetchTimeout.Duration
+	if fetchTimeout == 0 {
+		fetchTimeout = 5 * time.Minute // Default timeout
+	}
+	newsAgg := news.NewAggregator(newsRepo, cfg.News.PollInterval.Duration, fetchTimeout, llmScorer, logger)
 
 	// Add news sources from config
 	if len(cfg.News.Sources) > 0 {

pkg/app/news/aggregator.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"sync/atomic"
 	"time"
 
 	"github.com/pheinrich/aitrade/pkg/config"
 	"github.com/pheinrich/aitrade/pkg/db"
+	"golang.org/x/sync/errgroup"
 )
 
 type Aggregator struct {
@@ -16,6 +18,7 @@ type Aggregator struct {
 	analyzer      *Analyzer
 	llmScorer     *LLMScorer
 	pollInterval  time.Duration
+	fetchTimeout  time.Duration
 	logger        *slog.Logger
 	onNewsUpdated func() // Callback for SSE notifications
 }
@@ -23,6 +26,7 @@ type Aggregator struct {
 func NewAggregator(
 	newsRepo *db.NewsRepository,
 	pollInterval time.Duration,
+	fetchTimeout time.Duration,
 	llmScorer *LLMScorer,
 	logger *slog.Logger,
 ) *Aggregator {
@@ -32,6 +36,7 @@ func NewAggregator(
 		analyzer:     NewAnalyzer(),
 		llmScorer:    llmScorer,
 		pollInterval: pollInterval,
+		fetchTimeout: fetchTimeout,
 		logger:       logger,
 	}
 
@@ -141,59 +146,79 @@ func (a *Aggregator) Run(ctx context.Context) error {
 func (a *Aggregator) fetchAllSources(ctx context.Context) error {
 	a.logger.Debug("fetching from all news sources")
 
-	totalFetched := 0
-	totalStored := 0
+	var totalFetched, totalStored atomic.Int64
 
+	// Use errgroup for parallel fetching
+	g, gCtx := errgroup.WithContext(ctx)
+
+	// Launch goroutines for each source
 	for _, source := range a.sources {
-		articles, err := source.Fetch(ctx)
-		if err != nil {
-			a.logger.Warn("failed to fetch from source",
-				slog.String("source", source.Name()),
-				slog.Any("error", err),
-			)
-			continue
-		}
+		src := source // Capture loop variable
+		g.Go(func() error {
+			// Create a timeout context for this specific source
+			sourceCtx, cancel := context.WithTimeout(gCtx, a.fetchTimeout)
+			defer cancel()
 
-		totalFetched += len(articles)
-
-		// Analyze sentiment and store
-		for _, article := range articles {
-			// Always use keyword analyzer first (fast)
-			a.analyzer.Analyze(article)
-
-			// If LLM scorer enabled, use it (may fallback to keyword)
-			if a.llmScorer != nil && a.llmScorer.Enabled() {
-				if err := a.llmScorer.Analyze(ctx, article); err != nil {
-					a.logger.Warn("LLM scoring failed",
-						slog.String("url", article.URL),
-						slog.Any("error", err))
-					// Article already has keyword sentiment, continue
-				}
-			} else {
-				// No LLM scorer, mark as keyword-only
-				article.SentimentMethod = "keyword"
-			}
-
-			if err := a.newsRepo.Create(ctx, article); err != nil {
-				a.logger.Error("failed to store article",
-					slog.String("url", article.URL),
+			articles, err := src.Fetch(sourceCtx)
+			if err != nil {
+				a.logger.Warn("failed to fetch from source",
+					slog.String("source", src.Name()),
 					slog.Any("error", err),
 				)
-				continue
+				return nil // Don't fail entire operation on single source error
 			}
 
-			totalStored++
-		}
+			totalFetched.Add(int64(len(articles)))
+
+			// Analyze sentiment and store
+			for _, article := range articles {
+				// Always use keyword analyzer first (fast)
+				a.analyzer.Analyze(article)
+
+				// If LLM scorer enabled, use it (may fallback to keyword)
+				if a.llmScorer != nil && a.llmScorer.Enabled() {
+					if err := a.llmScorer.Analyze(sourceCtx, article); err != nil {
+						a.logger.Warn("LLM scoring failed",
+							slog.String("url", article.URL),
+							slog.Any("error", err))
+						// Article already has keyword sentiment, continue
+					}
+				} else {
+					// No LLM scorer, mark as keyword-only
+					article.SentimentMethod = "keyword"
+				}
+
+				if err := a.newsRepo.Create(sourceCtx, article); err != nil {
+					a.logger.Error("failed to store article",
+						slog.String("url", article.URL),
+						slog.Any("error", err),
+					)
+					continue
+				}
+
+				totalStored.Add(1)
+			}
+
+			return nil
+		})
 	}
 
+	// Wait for all goroutines to complete
+	if err := g.Wait(); err != nil {
+		return err
+	}
+
+	fetched := int(totalFetched.Load())
+	stored := int(totalStored.Load())
+
 	a.logger.Info("news fetch completed",
-		slog.Int("fetched", totalFetched),
-		slog.Int("stored", totalStored),
-		slog.Int("duplicates", totalFetched-totalStored),
+		slog.Int("fetched", fetched),
+		slog.Int("stored", stored),
+		slog.Int("duplicates", fetched-stored),
 	)
 
 	// Notify listeners if new articles were stored
-	if totalStored > 0 && a.onNewsUpdated != nil {
+	if stored > 0 && a.onNewsUpdated != nil {
 		a.onNewsUpdated()
 	}
 

pkg/app/news/json_sources.go

@@ -40,10 +40,22 @@ type alphaVantageResponse struct {
 func NewAlphaVantageSource(name, url string, logger *slog.Logger) *AlphaVantageSource {
 	now := time.Now()
 	return &AlphaVantageSource{
-		name:          name,
-		url:           url,
-		logger:        logger,
-		httpClient:    &http.Client{Timeout: 30 * time.Second},
+		name:   name,
+		url:    url,
+		logger: logger,
+		httpClient: &http.Client{
+			Timeout: 30 * time.Second,
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				if len(via) >= 10 {
+					return fmt.Errorf("stopped after 10 redirects")
+				}
+				logger.Debug("following redirect",
+					slog.String("source", name),
+					slog.String("from", via[len(via)-1].URL.String()),
+					slog.String("to", req.URL.String()))
+				return nil
+			},
+		},
 		lastHourReset: now,
 		lastDayReset:  now,
 	}
@@ -181,10 +193,22 @@ type finnhubArticle struct {
 func NewFinnhubSource(name, url string, logger *slog.Logger) *FinnhubSource {
 	now := time.Now()
 	return &FinnhubSource{
-		name:          name,
-		url:           url,
-		logger:        logger,
-		httpClient:    &http.Client{Timeout: 30 * time.Second},
+		name:   name,
+		url:    url,
+		logger: logger,
+		httpClient: &http.Client{
+			Timeout: 30 * time.Second,
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				if len(via) >= 10 {
+					return fmt.Errorf("stopped after 10 redirects")
+				}
+				logger.Debug("following redirect",
+					slog.String("source", name),
+					slog.String("from", via[len(via)-1].URL.String()),
+					slog.String("to", req.URL.String()))
+				return nil
+			},
+		},
 		lastHourReset: now,
 		lastDayReset:  now,
 	}

pkg/app/news/sources.go

@@ -40,11 +40,24 @@ type RSSSource struct {
 func NewRSSSource(name, url string, logger *slog.Logger) *RSSSource {
 	now := time.Now()
 	return &RSSSource{
-		name:          name,
-		url:           url,
-		parser:        gofeed.NewParser(),
-		logger:        logger,
-		httpClient:    &http.Client{Timeout: 30 * time.Second},
+		name:   name,
+		url:    url,
+		parser: gofeed.NewParser(),
+		logger: logger,
+		httpClient: &http.Client{
+			Timeout: 30 * time.Second,
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				// Follow up to 10 redirects (default behavior, made explicit)
+				if len(via) >= 10 {
+					return fmt.Errorf("stopped after 10 redirects")
+				}
+				logger.Debug("following redirect",
+					slog.String("source", name),
+					slog.String("from", via[len(via)-1].URL.String()),
+					slog.String("to", req.URL.String()))
+				return nil
+			},
+		},
 		headers:       make(map[string]string),
 		lastHourReset: now,
 		lastDayReset:  now,

pkg/config/config.go

@@ -64,6 +64,7 @@ type OIDCConfig struct {
 
 type NewsConfig struct {
 	PollInterval       Duration     `yaml:"poll_interval"`
+	FetchTimeout       Duration     `yaml:"fetch_timeout"`        // Timeout per source (fetch + processing)
 	DefaultRateLimit   *RateLimit   `yaml:"default_rate_limit,omitempty"`
 	Sources            []NewsSource `yaml:"sources"`
 }