How this was built

// Subreddits searched
const SUBREDDITS = [
  "GoogleWorkspace", "google", "artificial",
  "productivity", "ChatGPT"
];

// Search terms
const SEARCH_TERMS = [
  "gemini workspace", "gemini docs", "gemini gmail",
  "gemini sheets", "google ai workspace", "gemini side panel"
];

// Fetch via Reddit public JSON API (no auth required)
const url = `https://www.reddit.com/r/${sub}/search.json
  ?q=${encodeURIComponent(term)}
  &restrict_sr=1&sort=relevance&t=year&limit=25`;

const res = await fetch(url, {
  headers: { "User-Agent": "workspace-ai-analyzer/1.0" },
});

// Uses Algolia HN search API — no auth required
const queries = [
  "gemini workspace", "google gemini docs",
  "gemini gmail", "google ai productivity"
];

// Fetch stories
fetch(`https://hn.algolia.com/api/v1/search
  ?query=${query}&tags=story&hitsPerPage=30`);

// Also fetch comments (richer feedback signal)
fetch(`https://hn.algolia.com/api/v1/search
  ?query=${query}&tags=comment&hitsPerPage=50`);

// Strip HTML tags from comment text
const text = hit.comment_text
  .replace(/<[^>]*>/g, " ")
  .replace(/\s+/g, " ")
  .trim();

// App Store IDs scraped (5 apps, 5 pages each)
const apps = [
  { id: 422689480, name: "gmail" },
  { id: 842842640, name: "docs" },
  { id: 842849113, name: "sheets" },
  { id: 879478102, name: "slides" },
  { id: 1013161476, name: "meet" },
];

// Only keep reviews mentioning AI-related keywords
const AI_KEYWORDS = [
  "ai", "gemini", "smart", "suggest", "summary",
  "compose", "write", "draft", "autocomplete",
  "hallucin", "context", "side panel", "useless",
  "broken", "buggy", "slow"
];

const reviews = await store.reviews({
  id: app.id,
  sort: store.sort.RECENT,
  page,   // pages 1–5
  country: "us",
});

// Play Store app IDs scraped
const apps = [
  { id: "com.google.android.gm",                      name: "gmail" },
  { id: "com.google.android.apps.docs",               name: "docs" },
  { id: "com.google.android.apps.docs.editors.sheets",name: "sheets" },
  { id: "com.google.android.apps.docs.editors.slides",name: "slides" },
];

// Filter: only reviews mentioning AI/Gemini
const text = (review.text ?? "").toLowerCase();
if (
  !text.includes("ai") &&
  !text.includes("gemini") &&
  !text.includes("smart") &&
  !text.includes("suggest")
) continue;

// Stack Overflow public API v2.3 (no auth)
// Searches both stackoverflow.com and webapps.stackexchange.com
const queries = [
  "gemini+google+workspace", "gemini+docs",
  "gemini+gmail", "gemini+sheets", "google+ai+workspace"
];

fetch(`https://api.stackexchange.com/2.3/search/advanced
  ?order=desc&sort=relevance&q=${query}
  &site=stackoverflow&pagesize=30&filter=withbody`);

// Also webapps.stackexchange.com (more Workspace questions)
fetch(`https://api.stackexchange.com/2.3/search/advanced
  ?q=${query}&site=webapps&pagesize=30&filter=withbody`);

// Step 1: Search YouTube HTML for video IDs
// No API key — parses ytInitialData from page HTML
const url = `https://www.youtube.com/results
  ?search_query=${query}&sp=CAISBAgCEAE`; // filter: this year

const ids = [...html.matchAll(
  /\"videoId\":\"([a-zA-Z0-9_-]{11})\"/g
)].map(m => m[1]);

// Step 2: Fetch transcript via youtube-transcript-api (Python)
// Requires: pip install youtube-transcript-api
execSync(`python3 -c "
from youtube_transcript_api import YouTubeTranscriptApi
api = YouTubeTranscriptApi()
transcript = api.fetch('${videoId}', languages=['en'])
text = ' '.join([s.text for s in transcript.snippets])
print(text[:2000])
"`);

// Each theme has a keyword list
const THEME_KEYWORDS: Record<string, string[]> = {
  "hallucination": [
    "hallucinate", "hallucination", "made up",
    "fabricat", "incorrect", "wrong information",
    "inaccurate", "false", "imagin"
  ],
  "cross-app-memory": [
    "context", "memory", "remember", "forget",
    "loses context", "switch", "cross-app",
    "between apps", "side panel", "persistent"
  ],
  "mobile-voice": [
    "mobile", "phone", "voice", "hands-free",
    "android", "ios", "cramped", "small screen"
  ],
  // ... (see analysis.ts for full list)
};

// Match: simple keyword substring search (case-insensitive)
function analyzeFeedback(raw: RawFeedback[], themes: PainPointTheme[]) {
  for (const feedback of raw) {
    const lower = feedback.text.toLowerCase();
    for (const theme of themes) {
      const keywords = THEME_KEYWORDS[theme.id] ?? [];
      if (keywords.some(kw => lower.includes(kw))) {
        theme.frequency++;
        if (theme.quotes.length < 5) {
          theme.quotes.push({ text, source, url, author, date });
        }
      }
    }
  }
  // Sort by severity × frequency (highest impact first)
  return themes.sort((a, b) =>
    b.severity * b.frequency - a.severity * a.frequency
  );
}

// Full pipeline: scrape → analyze → save to Upstash Redis KV

export async function buildSnapshot() {
  // 1. Scrape all sources in parallel (with timeouts)
  const withTimeout = (p, ms, fallback) =>
    Promise.race([p, new Promise(r => setTimeout(() => r(fallback), ms))]);

  const [reddit, hn, stackoverflow, appstore] = await Promise.all([
    scrapeReddit(),
    scrapeHackerNews(),
    scrapeStackOverflow(),
    withTimeout(scrapeAppStore(), 20_000, []),  // 20s timeout
  ]);
  // YouTube is slower (transcripts), run sequentially after
  const youtube = await withTimeout(scrapeYouTube(), 60_000, []);
  const curated = getCuratedFeedback();

  const allFeedback = [
    ...reddit, ...hn, ...stackoverflow,
    ...appstore, ...youtube, ...curated
  ];

  // 2. Match feedback to themes
  const themes = analyzeFeedback(allFeedback, getDefaultThemes());

  // 3. Aggregate competitor mentions
  const topCompetitors = buildCompetitorRanking(themes);

  // 4. Save to KV (Upstash Redis)
  await saveRawFeedback(allFeedback);  // key: workspace-ai:raw-feedback
  await saveSnapshot({ themes, topCompetitors, ... });  // key: workspace-ai:snapshot
}