insights_dashboard.py

"""
Web Insights Dashboard
A dashboard for collecting, analyzing, and generating Chrome extension ideas from web insights.
"""

import streamlit as st
import os
import json
import time
from datetime import datetime
import traceback
import glob
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

# Set page config
st.set_page_config(
    page_title="Web Insights Dashboard",
    page_icon="🔍",
    layout="wide",
    initial_sidebar_state="expanded"
)

# CSS Styles
st.markdown("""
<style>
    .main-header {
        font-size: 2.5rem;
        margin-bottom: 1rem;
        background: linear-gradient(45deg, #3A7BD5, #00D2FF);
        -webkit-background-clip: text;
        -webkit-text-fill-color: transparent;
        font-weight: 700;
    }
    .sub-header {
        font-size: 1.5rem;
        margin-bottom: 1rem;
        color: #4B5563;
    }
    .card {
        background: white;
        border-radius: 0.75rem;
        padding: 1.5rem;
        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
        margin-bottom: 1rem;
    }
    .metric-card {
        background: white;
        border-radius: 0.5rem;
        padding: 1rem;
        box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
        text-align: center;
    }
    .metric-value {
        font-size: 2rem;
        font-weight: bold;
        margin: 0.5rem 0;
    }
    .metric-label {
        font-size: 0.9rem;
        color: #6B7280;
    }
    .status-indicator {
        display: inline-block;
        width: 12px;
        height: 12px;
        border-radius: 50%;
        margin-right: 8px;
    }
    .status-green {
        background-color: #10B981;
    }
    .status-red {
        background-color: #EF4444;
    }
    .status-yellow {
        background-color: #F59E0B;
    }
    .tag {
        display: inline-block;
        padding: 0.25rem 0.5rem;
        border-radius: 9999px;
        font-size: 0.75rem;
        margin: 0.25rem;
        background-color: #E5E7EB;
    }
    
    /* Responsive styles */
    @media (max-width: 768px) {
        .main-header {
            font-size: 1.8rem !important;
            word-wrap: break-word;
        }
        .sub-header {
            font-size: 1.2rem !important;
        }
        .card {
            padding: 1rem;
        }
        .metric-value {
            font-size: 1.5rem;
        }
        
        /* Make sure columns don't get too squished on mobile */
        [data-testid="column"] {
            width: 100% !important;
            flex: 1 1 100% !important;
            min-width: 100% !important;
        }
        
        /* Adjust stacked columns on mobile */
        [data-testid="stCols"] {
            flex-direction: column;
        }
    }
    
    /* Fix for button width on mobile */
    div.stButton > button {
        width: 100%;
    }
    
    /* Make expanders more mobile-friendly */
    .streamlit-expanderHeader {
        font-size: 1rem;
    }
    
    /* Better readability for code blocks on mobile */
    code {
        word-wrap: break-word;
        white-space: pre-wrap;
    }
</style>
""", unsafe_allow_html=True)

# Constants
DATA_DIR = os.path.join(os.getcwd(), "data")
INSIGHTS_DIR = os.path.join(DATA_DIR, "insights")
IDEAS_DIR = os.path.join(DATA_DIR, "ai_ideas")

# Ensure directories exist
os.makedirs(DATA_DIR, exist_ok=True)
os.makedirs(INSIGHTS_DIR, exist_ok=True)
os.makedirs(IDEAS_DIR, exist_ok=True)

def check_dependencies():
    """Check if required dependencies are installed"""
    missing_deps = []
    
    try:
        import requests
    except ImportError:
        missing_deps.append("requests")
    
    try:
        import beautifulsoup4
    except ImportError:
        missing_deps.append("beautifulsoup4")
    
    try:
        import pandas
    except ImportError:
        missing_deps.append("pandas")
    
    try:
        import matplotlib
    except ImportError:
        missing_deps.append("matplotlib")
    
    try:
        import openai
    except ImportError:
        missing_deps.append("openai")
    
    try:
        import sklearn
    except ImportError:
        missing_deps.append("scikit-learn")
    
    return missing_deps

def check_api_keys():
    """Check if required API keys are set"""
    api_status = {
        "OPENAI_API_KEY": "✅ Set" if os.environ.get("OPENAI_API_KEY") else "❌ Not set",
        "GITHUB_TOKEN": "✅ Set" if os.environ.get("GITHUB_TOKEN") else "❌ Not set",
        "REDDIT_CLIENT_ID": "✅ Set" if os.environ.get("REDDIT_CLIENT_ID") else "❌ Not set",
        "REDDIT_CLIENT_SECRET": "✅ Set" if os.environ.get("REDDIT_CLIENT_SECRET") else "❌ Not set",
        "PRODUCTHUNT_TOKEN": "✅ Set" if os.environ.get("PRODUCTHUNT_TOKEN") else "❌ Not set"
    }
    return api_status

def load_insights_data():
    """Load insights data from files"""
    insights = {
        "github": [],
        "chrome_store": [],
        "producthunt": [],
        "reddit": [],
        "api_changelogs": []
    }
    
    # Find all insight files
    for key in insights.keys():
        files = glob.glob(os.path.join(INSIGHTS_DIR, f"{key}_insights_*.json"))
        for file_path in files:
            try:
                with open(file_path, 'r', encoding='utf-8') as f:
                    data = json.load(f)
                    if isinstance(data, list):
                        insights[key].extend(data)
            except Exception as e:
                st.error(f"Error loading {file_path}: {str(e)}")
    
    return insights

def load_generated_ideas():
    """Load generated ideas from files"""
    ideas = []
    
    # Find all idea files
    files = glob.glob(os.path.join(IDEAS_DIR, "*.json"))
    for file_path in files:
        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                idea = json.load(f)
                if isinstance(idea, dict) and "name" in idea:
                    ideas.append(idea)
        except Exception as e:
            st.error(f"Error loading {file_path}: {str(e)}")
    
    return ideas

def run_web_insights_collector():
    """Run the web insights collector"""
    try:
        from agent.web_insights_collector import run
        with st.spinner("Collecting data from web sources..."):
            result = run(save_to_disk=True, analyze=True)
        return result
    except Exception as e:
        st.error(f"Error running web insights collector: {str(e)}")
        traceback.print_exc()
        return None

def run_insights_processor(count=3):
    """Run the insights processor"""
    try:
        from agent.insights_processor import run
        with st.spinner(f"Processing insights and generating {count} ideas..."):
            ideas = run(count=count, save_to_disk=True)
        return ideas
    except Exception as e:
        st.error(f"Error running insights processor: {str(e)}")
        traceback.print_exc()
        return None

def display_insights_metrics(insights):
    """Display metrics about insights data"""
    total_insights = sum(len(items) for items in insights.values())
    
    col1, col2, col3, col4, col5 = st.columns(5)
    
    with col1:
        st.markdown('<div class="metric-card">', unsafe_allow_html=True)
        st.markdown(f'<div class="metric-value">{total_insights}</div>', unsafe_allow_html=True)
        st.markdown('<div class="metric-label">Total Insights</div>', unsafe_allow_html=True)
        st.markdown('</div>', unsafe_allow_html=True)
    
    with col2:
        st.markdown('<div class="metric-card">', unsafe_allow_html=True)
        st.markdown(f'<div class="metric-value">{len(insights["github"])}</div>', unsafe_allow_html=True)
        st.markdown('<div class="metric-label">GitHub Issues</div>', unsafe_allow_html=True)
        st.markdown('</div>', unsafe_allow_html=True)
    
    with col3:
        st.markdown('<div class="metric-card">', unsafe_allow_html=True)
        st.markdown(f'<div class="metric-value">{len(insights["chrome_store"])}</div>', unsafe_allow_html=True)
        st.markdown('<div class="metric-label">Chrome Extensions</div>', unsafe_allow_html=True)
        st.markdown('</div>', unsafe_allow_html=True)
    
    with col4:
        st.markdown('<div class="metric-card">', unsafe_allow_html=True)
        st.markdown(f'<div class="metric-value">{len(insights["reddit"])}</div>', unsafe_allow_html=True)
        st.markdown('<div class="metric-label">Reddit Posts</div>', unsafe_allow_html=True)
        st.markdown('</div>', unsafe_allow_html=True)
    
    with col5:
        st.markdown('<div class="metric-card">', unsafe_allow_html=True)
        st.markdown(f'<div class="metric-value">{len(insights["producthunt"])}</div>', unsafe_allow_html=True)
        st.markdown('<div class="metric-label">Product Hunt Items</div>', unsafe_allow_html=True)
        st.markdown('</div>', unsafe_allow_html=True)

def display_github_insights(insights):
    """Display GitHub insights"""
    if not insights["github"]:
        st.info("No GitHub insights collected yet")
        return
    
    # Create DataFrame
    df = pd.DataFrame([
        {
            "Title": item.get("title", ""),
            "Repo": item.get("repo", ""),
            "Reactions": item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0,
            "Comments": item.get("comments_count", 0),
            "URL": item.get("url", ""),
            "Created": item.get("created_at", ""),
            "ID": i
        }
        for i, item in enumerate(insights["github"])
    ])
    
    # Sort by reactions and comments
    df["Engagement"] = df["Reactions"] + df["Comments"]
    df = df.sort_values("Engagement", ascending=False).reset_index(drop=True)
    
    # Display table
    st.dataframe(df.drop(columns=["ID", "Engagement"]), use_container_width=True)
    
    # Allow selecting an issue for details
    if not df.empty:
        selected_id = st.selectbox("Select an issue to view details", df["ID"], format_func=lambda x: df[df["ID"] == x]["Title"].iloc[0])
        
        if selected_id is not None:
            item = insights["github"][selected_id]
            
            # Display details
            st.markdown(f"### {item.get('title', '')}")
            st.markdown(f"**Repository:** {item.get('repo', '')}")
            st.markdown(f"**URL:** {item.get('url', '')}")
            
            col1, col2 = st.columns(2)
            
            with col1:
                st.markdown("**Labels:**")
                for label in item.get("labels", []):
                    st.markdown(f"<span class='tag'>{label}</span>", unsafe_allow_html=True)
            
            with col2:
                reactions = item.get("reactions", {})
                if isinstance(reactions, dict):
                    st.markdown("**Reactions:**")
                    reactions_text = []
                    for key, value in reactions.items():
                        if key != "total_count" and value > 0:
                            emoji = ""
                            if key == "plus_one": emoji = "👍"
                            elif key == "minus_one": emoji = "👎"
                            elif key == "laugh": emoji = "😄"
                            elif key == "hooray": emoji = "🎉"
                            elif key == "confused": emoji = "😕"
                            elif key == "heart": emoji = "❤️"
                            elif key == "rocket": emoji = "🚀"
                            elif key == "eyes": emoji = "👀"
                            
                            reactions_text.append(f"{emoji} {value}")
                    
                    st.markdown(" | ".join(reactions_text))
            
            st.markdown("**Issue Body:**")
            st.markdown(item.get("body", ""))

def display_reddit_insights(insights):
    """Display Reddit insights"""
    if not insights["reddit"]:
        st.info("No Reddit insights collected yet")
        return
    
    # Create DataFrame
    df = pd.DataFrame([
        {
            "Title": item.get("title", ""),
            "Subreddit": item.get("subreddit", ""),
            "Score": item.get("score", 0),
            "Comments": item.get("num_comments", 0),
            "URL": item.get("url", ""),
            "Created": item.get("created_at", ""),
            "ID": i
        }
        for i, item in enumerate(insights["reddit"])
    ])
    
    # Sort by score
    df = df.sort_values("Score", ascending=False).reset_index(drop=True)
    
    # Display table
    st.dataframe(df.drop(columns=["ID"]), use_container_width=True)
    
    # Allow selecting a post for details
    if not df.empty:
        selected_id = st.selectbox("Select a post to view details", df["ID"], format_func=lambda x: df[df["ID"] == x]["Title"].iloc[0])
        
        if selected_id is not None:
            item = insights["reddit"][selected_id]
            
            # Display details
            st.markdown(f"### {item.get('title', '')}")
            st.markdown(f"**Subreddit:** r/{item.get('subreddit', '')}")
            st.markdown(f"**URL:** {item.get('url', '')}")
            st.markdown(f"**Author:** u/{item.get('author', '')}")
            st.markdown(f"**Score:** {item.get('score', 0)} (Upvote Ratio: {item.get('upvote_ratio', 0):.2f})")
            st.markdown(f"**Comments:** {item.get('num_comments', 0)}")
            
            st.markdown("**Post Content:**")
            st.markdown(item.get("selftext", ""))

def display_chrome_store_insights(insights):
    """Display Chrome Web Store insights"""
    if not insights["chrome_store"]:
        st.info("No Chrome Web Store insights collected yet")
        return
    
    # Create DataFrame
    df = pd.DataFrame([
        {
            "Title": item.get("title", ""),
            "Category": item.get("category", ""),
            "Rating": item.get("rating", ""),
            "Users": item.get("users", ""),
            "URL": item.get("url", ""),
            "ID": i
        }
        for i, item in enumerate(insights["chrome_store"])
    ])
    
    # Add filtering by category
    categories = ["All"] + sorted(df["Category"].unique().tolist())
    selected_category = st.selectbox("Filter by Category", categories)
    
    if selected_category != "All":
        df = df[df["Category"] == selected_category]
    
    # Display table
    st.dataframe(df.drop(columns=["ID"]), use_container_width=True)
    
    # Allow selecting an extension for details
    if not df.empty:
        selected_id = st.selectbox("Select an extension to view details", df["ID"].tolist(), 
                                 format_func=lambda x: insights["chrome_store"][x]["title"])
        
        if selected_id is not None:
            item = insights["chrome_store"][selected_id]
            
            # Display details
            st.markdown(f"### {item.get('title', '')}")
            st.markdown(f"**Category:** {item.get('category', '')}")
            st.markdown(f"**URL:** {item.get('url', '')}")
            st.markdown(f"**Rating:** {item.get('rating', '')} ({item.get('users', '')} users)")
            
            st.markdown("**Description:**")
            st.markdown(item.get("description", ""))

def display_producthunt_insights(insights):
    """Display Product Hunt insights"""
    if not insights["producthunt"]:
        st.info("No Product Hunt insights collected yet")
        return
    
    # Create DataFrame
    df = pd.DataFrame([
        {
            "Title": item.get("title", ""),
            "Topic": item.get("topic", ""),
            "Upvotes": item.get("upvotes", ""),
            "URL": item.get("url", ""),
            "ID": i
        }
        for i, item in enumerate(insights["producthunt"])
    ])
    
    # Add filtering by topic
    topics = ["All"] + sorted(df["Topic"].unique().tolist())
    selected_topic = st.selectbox("Filter by Topic", topics)
    
    if selected_topic != "All":
        df = df[df["Topic"] == selected_topic]
    
    # Display table
    st.dataframe(df.drop(columns=["ID"]), use_container_width=True)
    
    # Allow selecting a product for details
    if not df.empty:
        selected_id = st.selectbox("Select a product to view details", df["ID"].tolist(), 
                                 format_func=lambda x: insights["producthunt"][x]["title"])
        
        if selected_id is not None:
            item = insights["producthunt"][selected_id]
            
            # Display details
            st.markdown(f"### {item.get('title', '')}")
            st.markdown(f"**Topic:** {item.get('topic', '')}")
            st.markdown(f"**URL:** {item.get('url', '')}")
            st.markdown(f"**Upvotes:** {item.get('upvotes', '')}")
            
            st.markdown("**Description:**")
            st.markdown(item.get("description", ""))

def display_api_changelog_insights(insights):
    """Display API changelog insights"""
    if not insights["api_changelogs"]:
        st.info("No API changelog insights collected yet")
        return
    
    # Create DataFrame
    df = pd.DataFrame([
        {
            "Title": item.get("title", ""),
            "API": item.get("api_name", ""),
            "Date": item.get("date", ""),
            "URL": item.get("url", ""),
            "ID": i
        }
        for i, item in enumerate(insights["api_changelogs"])
    ])
    
    # Add filtering by API
    apis = ["All"] + sorted(df["API"].unique().tolist())
    selected_api = st.selectbox("Filter by API", apis)
    
    if selected_api != "All":
        df = df[df["API"] == selected_api]
    
    # Display table
    st.dataframe(df.drop(columns=["ID"]), use_container_width=True)
    
    # Allow selecting a changelog entry for details
    if not df.empty:
        selected_id = st.selectbox("Select a changelog entry to view details", df["ID"].tolist(), 
                                 format_func=lambda x: f"{insights['api_changelogs'][x]['api_name']} - {insights['api_changelogs'][x]['title']}")
        
        if selected_id is not None:
            item = insights["api_changelogs"][selected_id]
            
            # Display details
            st.markdown(f"### {item.get('title', '')}")
            st.markdown(f"**API:** {item.get('api_name', '')}")
            st.markdown(f"**Date:** {item.get('date', '')}")
            st.markdown(f"**URL:** {item.get('url', '')}")
            
            st.markdown("**Content:**")
            st.markdown(item.get("content", ""))

def display_idea_details(idea):
    """Display detailed view of a generated idea"""
    if not idea:
        return
    
    # Main header for the idea
    st.markdown(f"<h2 style='text-align: center;'>{idea.get('name', 'Unnamed Idea')}</h2>", unsafe_allow_html=True)
    st.markdown(f"<p style='text-align: center; font-style: italic; font-size: 1.1em; margin-bottom: 20px;'>{idea.get('oneLiner', '')}</p>", unsafe_allow_html=True)
    
    # Main content grid
    col1, col2 = st.columns([2, 1])
    
    with col1:
        # Problem section
        st.markdown("### 🔍 Problem")
        problem_text = idea.get('problem', 'No problem specified')
        st.markdown(f"<div style='padding: 15px; background-color: #f8f9fa; border-radius: 5px; margin-bottom: 20px;'>{problem_text}</div>", unsafe_allow_html=True)
        
        # Solution section
        st.markdown("### 💡 Solution")
        solution_text = idea.get('solution', 'No solution specified')
        st.markdown(f"<div style='padding: 15px; background-color: #f8f9fa; border-radius: 5px; margin-bottom: 20px;'>{solution_text}</div>", unsafe_allow_html=True)
        
        # Features section
        st.markdown("### 🌟 Key Features")
        
        features = idea.get('features', [])
        if features:
            # Create a styled list of features
            feature_html = "<div style='padding: 15px; background-color: #f8f9fa; border-radius: 5px;'><ul style='margin-bottom: 0;'>"
            for feature in features:
                feature_html += f"<li style='margin-bottom: 8px;'>{feature}</li>"
            feature_html += "</ul></div>"
            st.markdown(feature_html, unsafe_allow_html=True)
        else:
            st.markdown("No features specified")
        
        # AI Integration
        st.markdown("### 🤖 AI Integration")
        ai_text = idea.get('aiIntegration', 'No AI integration specified')
        st.markdown(f"<div style='padding: 15px; background-color: #f8f9fa; border-radius: 5px; margin-bottom: 20px;'>{ai_text}</div>", unsafe_allow_html=True)
    
    with col2:
        # Source information
        st.markdown("<div style='background-color: white; border-radius: 10px; padding: 15px; box-shadow: 0 4px 6px rgba(0,0,0,0.1);'>", unsafe_allow_html=True)
        
        st.markdown("<h4 style='text-align: center; margin-bottom: 15px;'>📊 Idea Metrics</h4>", unsafe_allow_html=True)
        
        # Metrics
        cols = st.columns(2)
        cols[0].metric("Complexity", f"{idea.get('implementationComplexity', 0)}/10")
        cols[1].metric("Monetization", f"{idea.get('monetizationPotential', 0)}/10")
        
        st.markdown("</div>", unsafe_allow_html=True)
        
        # Source information
        if 'sourceOpportunity' in idea or 'sourceUrl' in idea:
            st.markdown("<div style='background-color: white; border-radius: 10px; padding: 15px; box-shadow: 0 4px 6px rgba(0,0,0,0.1); margin-top: 20px;'>", unsafe_allow_html=True)
            
            st.markdown("<h4 style='text-align: center; margin-bottom: 15px;'>🔍 Source Information</h4>", unsafe_allow_html=True)
            
            if idea.get('sourceOpportunity'):
                st.markdown(f"**Source Type:** {idea.get('sourceOpportunity')}")
            
            if idea.get('sourceUrl'):
                st.markdown(f"**Source URL:** [{idea.get('sourceUrl')}]({idea.get('sourceUrl')})")
            
            st.markdown("</div>", unsafe_allow_html=True)
        
        # Tech Stack
        st.markdown("<div style='background-color: white; border-radius: 10px; padding: 15px; box-shadow: 0 4px 6px rgba(0,0,0,0.1); margin-top: 20px;'>", unsafe_allow_html=True)
        
        st.markdown("<h4 style='text-align: center; margin-bottom: 15px;'>⚙️ Tech Stack</h4>", unsafe_allow_html=True)
        
        tech_stack = idea.get('techStack', [])
        if tech_stack:
            tech_html = "<div>"
            for tech in tech_stack:
                tech_html += f"<span class='tag'>{tech}</span>"
            tech_html += "</div>"
            st.markdown(tech_html, unsafe_allow_html=True)
        
        browser_apis = idea.get('browserAPIs', [])
        if browser_apis:
            st.markdown("**Browser APIs:**")
            apis_html = "<div>"
            for api in browser_apis:
                apis_html += f"<span class='tag'>{api}</span>"
            apis_html += "</div>"
            st.markdown(apis_html, unsafe_allow_html=True)
            
        st.markdown("</div>", unsafe_allow_html=True)
        
        # Tags section
        st.markdown("<div style='background-color: white; border-radius: 10px; padding: 15px; box-shadow: 0 4px 6px rgba(0,0,0,0.1); margin-top: 20px;'>", unsafe_allow_html=True)
        
        st.markdown("<h4 style='text-align: center; margin-bottom: 15px;'>🏷️ Keywords</h4>", unsafe_allow_html=True)
        
        keywords = idea.get('keywordTags', [])
        if keywords:
            keyword_html = "<div style='display: flex; flex-wrap: wrap; gap: 5px;'>"
            for keyword in keywords:
                keyword_html += f"<span style='background-color: #f0f0f0; padding: 5px 10px; border-radius: 15px; font-size: 0.85em;'>{keyword}</span>"
            keyword_html += "</div>"
            st.markdown(keyword_html, unsafe_allow_html=True)
        
        st.markdown("</div>", unsafe_allow_html=True)
    
    # Additional sections as tabs
    tab1, tab2, tab3 = st.tabs(["Monetization", "Target Users", "Launch Strategy"])
    
    with tab1:
        monetization = idea.get('monetization', [])
        if monetization:
            monetization_html = "<ul>"
            for strategy in monetization:
                monetization_html += f"<li>{strategy}</li>"
            monetization_html += "</ul>"
            st.markdown(monetization_html, unsafe_allow_html=True)
        else:
            st.markdown("No monetization strategies specified")
        
        if idea.get('competitiveAdvantage'):
            st.markdown("### Competitive Advantage")
            st.markdown(idea.get('competitiveAdvantage'))
    
    with tab2:
        users = idea.get('targetUsers', [])
        if users:
            users_html = "<ul>"
            for user in users:
                users_html += f"<li>{user}</li>"
            users_html += "</ul>"
            st.markdown(users_html, unsafe_allow_html=True)
        else:
            st.markdown("No target users specified")
        
        selling_points = idea.get('uniqueSellingPoints', [])
        if selling_points:
            st.markdown("### Unique Selling Points")
            selling_html = "<ul>"
            for point in selling_points:
                selling_html += f"<li>{point}</li>"
            selling_html += "</ul>"
            st.markdown(selling_html, unsafe_allow_html=True)
    
    with tab3:
        if idea.get('launchStrategy'):
            st.markdown(idea.get('launchStrategy'))
        else:
            st.markdown("No launch strategy specified")
    
    # Raw JSON view
    with st.expander("View Raw JSON"):
        st.json(idea)

def main():
    # Header
    st.markdown("<h1 class='main-header'>Web Insights Dashboard</h1>", unsafe_allow_html=True)
    st.markdown("<p class='sub-header'>Collect, analyze, and generate Chrome extension ideas from web insights</p>", unsafe_allow_html=True)
    
    # Sidebar
    st.sidebar.title("Navigation")
    page = st.sidebar.radio("Select a page", [
        "Dashboard",
        "Data Collection",
        "Idea Generation",
        "Insights Explorer",
        "Generated Ideas",
        "Settings & Setup"
    ])
    
    # Check dependencies
    missing_deps = check_dependencies()
    if missing_deps:
        st.sidebar.warning(f"Missing dependencies: {', '.join(missing_deps)}")
        st.sidebar.markdown("Run `./install_insights_dependencies.sh` to install")
    
    # Check API keys
    api_status = check_api_keys()
    st.sidebar.markdown("### API Status")
    for key, status in api_status.items():
        st.sidebar.markdown(f"**{key}:** {status}")
    
    # Load data
    insights = load_insights_data()
    generated_ideas = load_generated_ideas()
    
    # Different pages
    if page == "Dashboard":
        # Main dashboard with overview
        st.markdown("## Web Insights Overview")
        
        # Display metrics
        display_insights_metrics(insights)
        
        # Display recent ideas
        st.markdown("## Recent Generated Ideas")
        if generated_ideas:
            recent_ideas = sorted(generated_ideas, key=lambda x: x.get('generatedAt', ''), reverse=True)[:5]
            
            for idea in recent_ideas:
                with st.expander(idea.get('name', 'Unnamed Idea')):
                    st.markdown(f"*{idea.get('oneLiner', '')}*")
                    
                    cols = st.columns(4)
                    cols[0].markdown(f"**Complexity:** {idea.get('implementationComplexity', 0)}/10")
                    cols[1].markdown(f"**Monetization:** {idea.get('monetizationPotential', 0)}/10")
                    cols[2].markdown(f"**Source:** {idea.get('sourceOpportunity', 'N/A')}")
                    cols[3].markdown(f"**Generated:** {idea.get('generatedAt', 'N/A')}")
                    
                    st.markdown("### Key Features")
                    features = idea.get('features', [])
                    if features:
                        feature_html = "<ul>"
                        for feature in features[:3]:  # Show only first 3 features
                            feature_html += f"<li>{feature}</li>"
                        feature_html += "</ul>"
                        st.markdown(feature_html, unsafe_allow_html=True)
                    
                    st.button(f"View Full Details for {idea.get('name', 'Idea')}", key=f"view_{idea.get('name', '')}")
        else:
            st.info("No generated ideas yet. Go to the 'Idea Generation' page to create some.")
        
        # Display source stats
        st.markdown("## Data Sources")
        
        source_counts = {
            "GitHub Issues": len(insights["github"]),
            "Chrome Store Extensions": len(insights["chrome_store"]),
            "Reddit Posts": len(insights["reddit"]),
            "Product Hunt Products": len(insights["producthunt"]),
            "API Changelogs": len(insights["api_changelogs"])
        }
        
        if sum(source_counts.values()) > 0:
            fig, ax = plt.subplots(figsize=(10, 6))
            bars = ax.bar(source_counts.keys(), source_counts.values(), color='#3A7BD5')
            ax.set_ylabel('Number of Items')
            ax.set_title('Data Sources')
            
            # Add count labels on bars
            for i, bar in enumerate(bars):
                height = bar.get_height()
                ax.text(bar.get_x() + bar.get_width()/2., height + 0.5,
                        f'{int(height)}', ha='center', va='bottom')
            
            st.pyplot(fig)
        else:
            st.info("No insights data collected yet. Go to the 'Data Collection' page to get started.")
    
    elif page == "Data Collection":
        # Data collection page
        st.markdown("## Web Insights Collection")
        st.markdown("This page allows you to collect insights from various web sources to inform Chrome extension idea generation.")
        
        col1, col2 = st.columns([2, 1])
        
        with col1:
            st.markdown("### Available Data Sources")
            st.markdown("""
            - **GitHub Issues**: Identifies pain points and feature requests from browser extension repositories
            - **Chrome Web Store**: Analyzes existing extensions for trends and opportunities
            - **Reddit**: Discovers user pain points and discussions about browser experiences
            - **Product Hunt**: Finds trending browser-related products and user feedback
            - **API Changelogs**: Identifies new APIs and features that could enable new extensions
            """)
        
        with col2:
            st.markdown("### Current Data")
            st.markdown(f"GitHub Issues: **{len(insights['github'])}**")
            st.markdown(f"Chrome Store Extensions: **{len(insights['chrome_store'])}**")
            st.markdown(f"Reddit Posts: **{len(insights['reddit'])}**")
            st.markdown(f"Product Hunt Products: **{len(insights['producthunt'])}**")
            st.markdown(f"API Changelogs: **{len(insights['api_changelogs'])}**")
        
        # Collection button
        st.markdown("### Run Data Collection")
        col1, col2 = st.columns([3, 1])
        
        with col1:
            st.markdown("Click the button to start collecting data from all sources. This may take a few minutes.")
        
        with col2:
            if st.button("Collect Web Insights", type="primary"):
                result = run_web_insights_collector()
                if result:
                    st.success(f"Successfully collected {result['total_collected']} insights!")
                    # Reload insights data
                    insights = load_insights_data()
                    display_insights_metrics(insights)
    
    elif page == "Idea Generation":
        # Idea generation page
        st.markdown("## Chrome Extension Idea Generation")
        st.markdown("This page generates innovative Chrome extension ideas based on the collected web insights.")
        
        # Check if we have insights data
        total_insights = sum(len(items) for items in insights.values())
        if total_insights == 0:
            st.warning("No insights data found. Please go to the 'Data Collection' page first to collect data.")
        else:
            st.info(f"Using {total_insights} insights from web sources to generate ideas.")
            
            # Generation options
            st.markdown("### Generation Options")
            
            col1, col2 = st.columns(2)
            with col1:
                num_ideas = st.slider("Number of ideas to generate", 1, 5, 3)
            
            # Generation button
            if st.button("Generate Ideas", type="primary"):
                generated = run_insights_processor(count=num_ideas)
                if generated:
                    st.success(f"Successfully generated {len(generated)} ideas!")
                    # Reload generated ideas
                    generated_ideas = load_generated_ideas()
                    
                    # Display the generated ideas
                    for idea in generated:
                        with st.expander(idea.get("name", "Unnamed Idea")):
                            st.markdown(f"*{idea.get('oneLiner', '')}*")
                            st.markdown(idea.get("problem", ""))
                            
                            if st.button(f"View Full Details for {idea.get('name', 'Idea')}", key=f"gen_{idea.get('name', '')}"):
                                display_idea_details(idea)
    
    elif page == "Insights Explorer":
        # Insights explorer page
        st.markdown("## Web Insights Explorer")
        
        # Create tabs for different sources
        tab1, tab2, tab3, tab4, tab5 = st.tabs([
            f"GitHub ({len(insights['github'])})", 
            f"Chrome Store ({len(insights['chrome_store'])})", 
            f"Reddit ({len(insights['reddit'])})", 
            f"Product Hunt ({len(insights['producthunt'])})",
            f"API Changelogs ({len(insights['api_changelogs'])})"
        ])
        
        with tab1:
            display_github_insights(insights)
        
        with tab2:
            display_chrome_store_insights(insights)
        
        with tab3:
            display_reddit_insights(insights)
        
        with tab4:
            display_producthunt_insights(insights)
        
        with tab5:
            display_api_changelog_insights(insights)
    
    elif page == "Generated Ideas":
        # Generated ideas page
        st.markdown("## Generated Chrome Extension Ideas")
        
        if not generated_ideas:
            st.info("No generated ideas yet. Go to the 'Idea Generation' page to create some.")
        else:
            # Sort ideas by generation time
            sorted_ideas = sorted(generated_ideas, key=lambda x: x.get('generatedAt', ''), reverse=True)
            
            # Create DataFrame for filtering
            df = pd.DataFrame([
                {
                    "Name": idea.get("name", "Unnamed"),
                    "One-Liner": idea.get("oneLiner", ""),
                    "Complexity": idea.get("implementationComplexity", 0),
                    "Monetization": idea.get("monetizationPotential", 0),
                    "Source": idea.get("sourceOpportunity", "Unknown"),
                    "Generated At": idea.get("generatedAt", ""),
                    "ID": i
                }
                for i, idea in enumerate(sorted_ideas)
            ])
            
            # Add filters
            st.markdown("### Filter Ideas")
            
            col1, col2, col3 = st.columns(3)
            
            with col1:
                name_filter = st.text_input("Filter by Name", "")
            
            with col2:
                min_monetization = st.slider("Min Monetization Potential", 0, 10, 0)
            
            with col3:
                source_options = ["All"] + sorted(df["Source"].unique().tolist())
                source_filter = st.selectbox("Filter by Source", source_options)
            
            # Apply filters
            filtered_df = df.copy()
            
            if name_filter:
                filtered_df = filtered_df[filtered_df["Name"].str.contains(name_filter, case=False)]
            
            if min_monetization > 0:
                filtered_df = filtered_df[filtered_df["Monetization"] >= min_monetization]
            
            if source_filter != "All":
                filtered_df = filtered_df[filtered_df["Source"] == source_filter]
            
            # Display the filtered table
            st.markdown(f"### Ideas ({len(filtered_df)} of {len(df)})")
            st.dataframe(filtered_df.drop(columns=["ID"]), use_container_width=True)
            
            # Show detailed view when an idea is selected
            if not filtered_df.empty:
                selected_id = st.selectbox("Select an idea to view details", 
                                         filtered_df["ID"], 
                                         format_func=lambda x: filtered_df[filtered_df["ID"] == x]["Name"].iloc[0])
                
                if selected_id is not None:
                    display_idea_details(sorted_ideas[selected_id])
    
    elif page == "Settings & Setup":
        # Settings page
        st.markdown("## Settings & Setup")
        st.markdown("Configure the Web Insights Dashboard and install required dependencies.")
        
        st.markdown("### Required Dependencies")
        
        if missing_deps:
            st.warning(f"The following dependencies are missing: {', '.join(missing_deps)}")
            st.markdown("Run the following command to install them:")
            st.code("./install_insights_dependencies.sh")
        else:
            st.success("All required dependencies are installed!")
        
        st.markdown("### API Keys")
        st.markdown("Set up the following API keys in your environment variables:")
        
        with st.expander("Required API Keys"):
            st.markdown("""
            - **OPENAI_API_KEY**: Required for AI idea generation ([Get Key](https://platform.openai.com/account/api-keys))
            - **GITHUB_TOKEN**: For GitHub issue mining ([Create Token](https://github.com/settings/tokens))
            - **REDDIT_CLIENT_ID** and **REDDIT_CLIENT_SECRET**: For Reddit data collection ([Create App](https://www.reddit.com/prefs/apps))
            - **PRODUCTHUNT_TOKEN**: For Product Hunt API access ([Get Token](https://api.producthunt.com/v2/docs))
            
            Add these to your `.env` file or export them in your shell session.
            """)
        
        st.markdown("### Directory Structure")
        with st.expander("Data Directory Structure"):
            st.markdown("""
            - `data/insights/`: Raw insights collected from web sources
            - `data/ai_ideas/`: Generated Chrome extension ideas
            """)
        
        # Data management
        st.markdown("### Data Management")
        
        if st.button("Clear All Insights Data"):
            confirm = st.text_input("Type 'DELETE' to confirm clearing all insights data:")
            if confirm == "DELETE":
                try:
                    # Find all insight files
                    insight_files = glob.glob(os.path.join(INSIGHTS_DIR, "*.json"))
                    for file_path in insight_files:
                        os.remove(file_path)
                    st.success("All insights data cleared successfully!")
                except Exception as e:
                    st.error(f"Error clearing data: {str(e)}")
        
        if st.button("Clear All Generated Ideas"):
            confirm = st.text_input("Type 'DELETE IDEAS' to confirm clearing all generated ideas:")
            if confirm == "DELETE IDEAS":
                try:
                    # Find all idea files
                    idea_files = glob.glob(os.path.join(IDEAS_DIR, "*.json"))
                    for file_path in idea_files:
                        os.remove(file_path)
                    st.success("All generated ideas cleared successfully!")
                except Exception as e:
                    st.error(f"Error clearing ideas: {str(e)}")
    
    # Footer
    st.markdown("")
    st.markdown("<div style='text-align: center; color: #6B7280; padding-top: 30px; font-size: 0.8rem;'>Web Insights Dashboard v1.0</div>", unsafe_allow_html=True)

if __name__ == "__main__":
    main()