Как создать собственные «новости Google» с помощью RSS-каналов

# Categories & Priority Levels
CRITICAL (check hourly):
- Breaking news (Reuters, AP)
- Market data (Bloomberg, WSJ)
- Security alerts (Krebs, BleepingComputer)

HIGH (check 2-4x daily):
- Industry news (TechCrunch, Ars Technica)
- Competitor updates (company blogs)
- Regulatory changes (gov RSS feeds)

MEDIUM (daily digest):
- Long-form analysis (The Atlantic, Vox)
- Research papers (arXiv, Nature)
- Opinion pieces (selected columnists)

LOW (weekly roundup):
- Entertainment
- Sports
- Lifestyle

# Method 1: Common URL patterns
https://reuters.com/rss
https://apnews.com/rss
https://bbc.com/news/rss
https://techcrunch.com/feed/
https://[site].com/feed.xml

# Method 2: View page source, search for:
- type="application/rss+xml"
- type="application/atom+xml"
- <link rel="alternate"

# Method 3: RSS directories
- FeedSpot (feedspot.com)
- RSS.com directory
- GitHub awesome-rss lists

# Method 4: RSS Feed Flow Auto-Detect
Paste any URL → RSS Builder → "Find Feeds"
→ Returns all discoverable feeds from that domain

import { Queue, Worker } from 'bullmq';
import Parser from 'rss-parser';
import Redis from 'ioredis';

const redis = new Redis(process.env.REDIS_URL);
const feedQueue = new Queue('feed-polling', { connection: redis });
const parser = new Parser();

// Add feeds to polling queue with different intervals
export async function scheduleFeedPolling(feeds: Feed[]) {
  for (const feed of feeds) {
    const interval = feed.priority === 'critical' ? 15 :
                     feed.priority === 'high' ? 60 : 240;

    await feedQueue.add('fetch-feed',
      { feedUrl: feed.url, feedId: feed.id },
      { repeat: { every: interval * 60 * 1000 } }
    );
  }
}

// Worker processes feed fetching
const worker = new Worker('feed-polling', async (job) => {
  const { feedUrl, feedId } = job.data;

  try {
    const feed = await parser.parseURL(feedUrl, {
      headers: { 'User-Agent': 'CustomNews/1.0' },
      timeout: 10000,
    });

    await storeFeedItems(feedId, feed.items);
    return { success: true, itemsCount: feed.items.length };
  } catch (error) {
    console.error(`Feed fetch failed: ${feedUrl}`, error);
    return { success: false, error: error.message };
  }
}, { connection: redis });

-- Articles table
CREATE TABLE articles (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  feed_id UUID REFERENCES feeds(id),
  title TEXT NOT NULL,
  url TEXT UNIQUE NOT NULL,
  content_hash TEXT, -- For deduplication
  published_at TIMESTAMP,
  fetched_at TIMESTAMP DEFAULT NOW(),
  summary TEXT, -- AI-generated
  relevance_score FLOAT DEFAULT 0,
  category TEXT[],
  entities TEXT[], -- People, orgs, locations
  sentiment TEXT, -- positive/neutral/negative
  is_duplicate BOOLEAN DEFAULT FALSE,
  duplicate_of UUID REFERENCES articles(id)
);

-- Indexes for performance
CREATE INDEX idx_articles_published ON articles(published_at DESC);
CREATE INDEX idx_articles_relevance ON articles(relevance_score DESC);
CREATE INDEX idx_articles_category ON articles USING GIN(category);
CREATE INDEX idx_content_hash ON articles(content_hash);

-- User preferences
CREATE TABLE user_preferences (
  user_id UUID PRIMARY KEY,
  preferred_categories TEXT[],
  blocked_sources TEXT[],
  keyword_filters JSONB,
  digest_frequency TEXT DEFAULT 'daily'
);

import OpenAI from 'openai';

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

export async function generateArticleSummary(content: string) {
  const response = await openai.chat.completions.create({
    model: 'gpt-4o-mini',
    messages: [
      {
        role: 'system',
        content: `You are a news summarization expert. Create a concise summary with:
          1. One-sentence headline (max 15 words)
          2. Three bullet points of key facts
          3. Why this matters (one sentence)

          Output JSON format only.`
      },
      {
        role: 'user',
        content: content.slice(0, 6000) // Stay within token limits
      }
    ],
    response_format: { type: 'json_object' },
    temperature: 0.3,
  });

  return JSON.parse(response.choices[0].message.content);
}

// Batch processing for efficiency
export async function batchSummarize(articles: Article[]) {
  const results = await Promise.allSettled(
    articles.map(article => generateArticleSummary(article.content))
  );

  return results.map((result, i) => ({
    articleId: articles[i].id,
    summary: result.status === 'fulfilled' ? result.value : null,
    error: result.status === 'rejected' ? result.reason : null
  }));
}

import { createHash } from 'crypto';
import { cosineSimilarity } from 'ml-distance';
import { embed } from 'ai';

export async function findDuplicates(newArticle: Article, existingArticles: Article[]) {
  const newTitleHash = createHash('md5').update(newArticle.title).digest('hex');

  // Fast path: exact title match
  const exactMatch = existingArticles.find(a => a.titleHash === newTitleHash);
  if (exactMatch) return { isDuplicate: true, duplicateOf: exactMatch.id };

  // Slow path: semantic similarity
  const newEmbedding = await embed(newArticle.title + newArticle.content.slice(0, 500));

  for (const existing of existingArticles.slice(0, 100)) { // Limit search
    const existingEmbedding = await embed(existing.title + existing.content.slice(0, 500));
    const similarity = cosineSimilarity(newEmbedding, existingEmbedding);

    if (similarity > 0.85) {
      return { isDuplicate: true, duplicateOf: existing.id, similarity };
    }
  }

  return { isDuplicate: false };
}

export function NewsFeed({ articles, filters }) {
  const [viewMode, setViewMode] = useState('list'); // list | grid | digest
  const [sortBy, setSortBy] = useState('relevance'); // relevance | time | source

  const filteredArticles = useMemo(() => {
    return articles
      .filter(article => {
        if (filters.categories.length &&
            !filters.categories.some(c => article.category.includes(c))) {
          return false;
        }
        if (filters.blockedSources.includes(article.source)) {
          return false;
        }
        if (filters.keywords.exclude.some(k =>
            article.title.toLowerCase().includes(k.toLowerCase()))) {
          return false;
        }
        return true;
      })
      .sort((a, b) => {
        if (sortBy === 'relevance') return b.relevanceScore - a.relevanceScore;
        if (sortBy === 'time') return new Date(b.publishedAt).getTime() - new Date(a.publishedAt).getTime();
        return 0;
      });
  }, [articles, filters, sortBy]);

  return (
    <div className="space-y-4">
      <FilterBar
        viewMode={viewMode}
        setViewMode={setViewMode}
        sortBy={sortBy}
        setSortBy={setSortBy}
      />
      {filteredArticles.map(article => (
        <ArticleCard key={article.id} article={article} />
      ))}
    </div>
  );
}

export async function routeAlert(article: Article, userPreferences: UserPrefs) {
  const alerts: Alert[] = [];

  // Critical breaking news → SMS + Push
  if (article.priority === 'critical' && userPreferences.smsEnabled) {
    alerts.push({
      channel: 'sms',
      to: userPreferences.phoneNumber,
      message: `BREAKING: ${article.title}`
    });
  }

  // High priority → Slack + Email
  if (article.priority === 'high') {
    if (userPreferences.slackWebhook) {
      alerts.push({
        channel: 'slack',
        webhook: userPreferences.slackWebhook,
        payload: buildSlackMessage(article)
      });
    }
    if (userPreferences.emailEnabled) {
      alerts.push({
        channel: 'email',
        to: userPreferences.email,
        subject: article.title,
        body: buildEmailBody(article)
      });
    }
  }

  // Send all alerts in parallel
  await Promise.all(alerts.map(sendAlert));
}