streamlit_app.py

import streamlit as st
import pandas as pd
import numpy as np
import pyreadstat
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import seaborn as sns
import matplotlib.pyplot as plt
from pathlib import Path
import warnings
from datetime import datetime
import io

# Configure page
st.set_page_config(
    page_title="Afrobarometer Data Explorer",
    page_icon="🌍",
    layout="wide",
    initial_sidebar_state="expanded"
)

# Suppress warnings
warnings.filterwarnings('ignore')

# Custom CSS for better styling
st.markdown("""
<style>
    .main-header {
        font-size: 3rem;
        color: #1f77b4;
        text-align: center;
        margin-bottom: 2rem;
    }
    .metric-card {
        background-color: #f0f2f6;
        padding: 1rem;
        border-radius: 0.5rem;
        border-left: 4px solid #1f77b4;
    }
    .sidebar .sidebar-content {
        background-color: #f8f9fa;
    }
    .stSelectbox > div > div {
        background-color: white;
    }
</style>
""", unsafe_allow_html=True)

@st.cache_data
def load_data():
    """Load and cache the Afrobarometer data"""
    try:
        data_path = "raw_data/R9.Merge_39ctry.20Nov23.final_.release_Updated.4Jun25-3.sav"
        
        if not Path(data_path).exists():
            st.error(f"Data file not found: {data_path}")
            return None, None
        
        # Load data with metadata
        df, meta = pyreadstat.read_sav(data_path)
        
        # Process metadata for easier access
        var_labels = {}
        value_labels = {}
        
        if hasattr(meta, 'column_labels') and meta.column_labels:
            var_labels = meta.column_labels
        
        if hasattr(meta, 'value_labels') and meta.value_labels:
            value_labels = meta.value_labels
        
        return df, {
            'var_labels': var_labels,
            'value_labels': value_labels,
            'original_meta': meta
        }
    
    except Exception as e:
        st.error(f"Error loading data: {str(e)}")
        return None, None

def get_variable_info(var_name, metadata):
    """Get variable label and value labels for a variable"""
    var_label = metadata['var_labels'].get(var_name, var_name)
    value_label_dict = metadata['value_labels'].get(var_name, {})
    return var_label, value_label_dict

def create_country_selector(df, metadata):
    """Create country selector with labels"""
    # Find country variable
    country_vars = [col for col in df.columns if any(keyword in col.lower() for keyword in ['country', 'ctry', 'nation'])]
    
    if not country_vars:
        return None, None
    
    country_var = country_vars[0]
    country_label, country_value_labels = get_variable_info(country_var, metadata)
    
    # Get unique countries with labels
    unique_countries = df[country_var].dropna().unique()
    country_options = {}
    
    for country_code in unique_countries:
        if country_code in country_value_labels:
            country_options[country_value_labels[country_code]] = country_code
        else:
            country_options[f"Country {country_code}"] = country_code
    
    return country_var, country_options

def main():
    # Header
    st.markdown('<h1 class="main-header">🌍 Afrobarometer Data Explorer</h1>', unsafe_allow_html=True)
    st.markdown("### Interactive Dashboard for Afrobarometer Round 9 (39 Countries)")
    
    # Load data
    with st.spinner("Loading Afrobarometer data..."):
        df, metadata = load_data()
    
    if df is None:
        st.stop()
    
    # Sidebar
    st.sidebar.header("📊 Data Filters")
    
    # Basic info
    st.sidebar.markdown("### Dataset Overview")
    st.sidebar.metric("Total Observations", f"{len(df):,}")
    st.sidebar.metric("Total Variables", f"{len(df.columns):,}")
    st.sidebar.metric("Memory Usage", f"{df.memory_usage(deep=True).sum() / 1024**2:.1f} MB")
    
    # Country selector
    country_var, country_options = create_country_selector(df, metadata)
    
    if country_var and country_options:
        st.sidebar.markdown("### 🌍 Country Selection")
        selected_country_label = st.sidebar.selectbox(
            "Select Country:",
            options=list(country_options.keys()),
            index=0
        )
        selected_country_code = country_options[selected_country_label]
        
        # Filter data by country
        df_filtered = df[df[country_var] == selected_country_code].copy()
        st.sidebar.metric("Observations in Selected Country", len(df_filtered))
    else:
        df_filtered = df.copy()
        selected_country_label = "All Countries"
    
    # Variable selection
    st.sidebar.markdown("### 📈 Variable Selection")
    
    # Get numeric and categorical variables
    numeric_vars = df_filtered.select_dtypes(include=[np.number]).columns.tolist()
    categorical_vars = df_filtered.select_dtypes(include=['object', 'category']).columns.tolist()
    
    # Variable type selector
    var_type = st.sidebar.radio("Variable Type:", ["Numeric", "Categorical", "All"])
    
    if var_type == "Numeric":
        available_vars = numeric_vars
    elif var_type == "Categorical":
        available_vars = categorical_vars
    else:
        available_vars = list(df_filtered.columns)
    
    # Variable selector with labels
    var_options = {}
    for var in available_vars[:50]:  # Limit to first 50 for performance
        var_label, _ = get_variable_info(var, metadata)
        var_options[f"{var_label} ({var})"] = var
    
    if var_options:
        selected_var_label = st.sidebar.selectbox(
            "Select Variable:",
            options=list(var_options.keys()),
            index=0
        )
        selected_var = var_options[selected_var_label]
    else:
        selected_var = None
    
    # Main content area
    tab1, tab2, tab3, tab4, tab5 = st.tabs(["📊 Overview", "📈 Visualizations", "🔍 Data Explorer", "📋 Summary Stats", "💾 Export Data"])
    
    with tab1:
        st.header("Dataset Overview")
        
        col1, col2, col3, col4 = st.columns(4)
        
        with col1:
            st.metric("Total Observations", f"{len(df_filtered):,}")
        
        with col2:
            missing_pct = (df_filtered.isnull().sum().sum() / (len(df_filtered) * len(df_filtered.columns)) * 100)
            st.metric("Missing Data %", f"{missing_pct:.1f}%")
        
        with col3:
            numeric_count = len(df_filtered.select_dtypes(include=[np.number]).columns)
            st.metric("Numeric Variables", numeric_count)
        
        with col4:
            categorical_count = len(df_filtered.select_dtypes(include=['object', 'category']).columns)
            st.metric("Categorical Variables", categorical_count)
        
        # Data preview
        st.subheader("Data Preview")
        st.dataframe(df_filtered.head(10), use_container_width=True)
        
        # Missing values heatmap
        if len(df_filtered.columns) > 0:
            st.subheader("Missing Values Pattern")
            
            # Sample columns for heatmap (first 20)
            sample_cols = df_filtered.columns[:20]
            missing_data = df_filtered[sample_cols].isnull()
            
            fig = px.imshow(
                missing_data.T,
                aspect="auto",
                color_continuous_scale="Reds",
                title="Missing Values Heatmap (First 20 Variables)"
            )
            fig.update_layout(height=400)
            st.plotly_chart(fig, use_container_width=True)
    
    with tab2:
        st.header("Data Visualizations")
        
        if selected_var is not None:
            var_label, value_labels = get_variable_info(selected_var, metadata)
            st.subheader(f"Analysis: {var_label}")
            
            col1, col2 = st.columns(2)
            
            with col1:
                # Distribution plot
                if selected_var in numeric_vars:
                    fig = px.histogram(
                        df_filtered, 
                        x=selected_var,
                        title=f"Distribution of {var_label}",
                        nbins=30
                    )
                    fig.update_layout(height=400)
                    st.plotly_chart(fig, use_container_width=True)
                else:
                    # Categorical variable
                    value_counts = df_filtered[selected_var].value_counts().head(15)
                    
                    # Apply value labels if available
                    if value_labels:
                        value_counts.index = [value_labels.get(x, x) for x in value_counts.index]
                    
                    fig = px.bar(
                        x=value_counts.values,
                        y=value_counts.index,
                        orientation='h',
                        title=f"Distribution of {var_label}",
                        labels={'x': 'Count', 'y': var_label}
                    )
                    fig.update_layout(height=400)
                    st.plotly_chart(fig, use_container_width=True)
            
            with col2:
                # Box plot for numeric variables
                if selected_var in numeric_vars:
                    fig = px.box(
                        df_filtered,
                        y=selected_var,
                        title=f"Box Plot: {var_label}"
                    )
                    fig.update_layout(height=400)
                    st.plotly_chart(fig, use_container_width=True)
                else:
                    # Pie chart for categorical
                    value_counts = df_filtered[selected_var].value_counts().head(10)
                    
                    if value_labels:
                        value_counts.index = [value_labels.get(x, x) for x in value_counts.index]
                    
                    fig = px.pie(
                        values=value_counts.values,
                        names=value_counts.index,
                        title=f"Distribution of {var_label}"
                    )
                    fig.update_layout(height=400)
                    st.plotly_chart(fig, use_container_width=True)
            
            # Summary statistics
            st.subheader("Summary Statistics")
            if selected_var in numeric_vars:
                stats = df_filtered[selected_var].describe()
                st.dataframe(stats.to_frame('Statistics'), use_container_width=True)
            else:
                value_counts = df_filtered[selected_var].value_counts()
                if value_labels:
                    value_counts.index = [value_labels.get(x, x) for x in value_counts.index]
                st.dataframe(value_counts.to_frame('Count'), use_container_width=True)
        else:
            st.info("Please select a variable from the sidebar to see visualizations.")
    
    with tab3:
        st.header("Interactive Data Explorer")
        
        # Variable comparison
        st.subheader("Variable Comparison")
        
        # Use same approach as sidebar - limit to first 30 for performance
        filtered_vars = available_vars[:30]
        
        if not filtered_vars:
            st.warning("No variables available for comparison.")
        else:
            col1, col2 = st.columns(2)
            
            with col1:
                # Variable 1 selection - use same approach as sidebar
                selected_var1 = st.selectbox(
                    "Variable 1:",
                    options=filtered_vars,
                    key="streamlit_explorer_var1"
                )
            
            with col2:
                if selected_var1:
                    # Variable 2 selection (exclude var1) - use same approach as sidebar
                    var2_options = [var for var in filtered_vars if var != selected_var1]
                    selected_var2 = st.selectbox(
                        "Variable 2:",
                        options=var2_options,
                        key="streamlit_explorer_var2"
                    )
                else:
                    selected_var2 = None
        
        # Create comparison plot
        if selected_var1 and selected_var2:
            var1_label, _ = get_variable_info(selected_var1, metadata)
            var2_label, _ = get_variable_info(selected_var2, metadata)
            
            if selected_var1 in numeric_vars and selected_var2 in numeric_vars:
                # Scatter plot
                fig = px.scatter(
                    df_filtered,
                    x=selected_var1,
                    y=selected_var2,
                    title=f"{var1_label} vs {var2_label}",
                    opacity=0.6
                )
                st.plotly_chart(fig, use_container_width=True)
                
                # Correlation
                corr = df_filtered[selected_var1].corr(df_filtered[selected_var2])
                st.metric("Correlation Coefficient", f"{corr:.3f}")
            
            elif selected_var1 in categorical_vars and selected_var2 in categorical_vars:
                # Cross-tabulation
                crosstab = pd.crosstab(df_filtered[selected_var1], df_filtered[selected_var2])
                st.subheader("Cross-tabulation")
                st.dataframe(crosstab, use_container_width=True)
            
            else:
                # Mixed types - box plot
                if selected_var1 in numeric_vars:
                    cat_var, num_var = selected_var2, selected_var1
                else:
                    cat_var, num_var = selected_var1, selected_var2
                
                fig = px.box(
                    df_filtered,
                    x=cat_var,
                    y=num_var,
                    title=f"{var2_label} by {var1_label}"
                )
                st.plotly_chart(fig, use_container_width=True)
    
    with tab4:
        st.header("Summary Statistics")
        
        # Overall statistics
        st.subheader("Dataset Summary")
        
        col1, col2 = st.columns(2)
        
        with col1:
            st.write("**Data Types:**")
            dtype_counts = df_filtered.dtypes.value_counts()
            st.dataframe(dtype_counts.to_frame('Count'), use_container_width=True)
        
        with col2:
            st.write("**Missing Values by Variable (Top 10):**")
            missing_summary = df_filtered.isnull().sum().sort_values(ascending=False).head(10)
            missing_pct = (missing_summary / len(df_filtered) * 100).round(2)
            missing_df = pd.DataFrame({
                'Missing Count': missing_summary,
                'Missing %': missing_pct
            })
            st.dataframe(missing_df, use_container_width=True)
        
        # Numeric variables summary
        if len(numeric_vars) > 0:
            st.subheader("Numeric Variables Summary")
            numeric_summary = df_filtered[numeric_vars].describe()
            st.dataframe(numeric_summary, use_container_width=True)
        
        # Categorical variables summary
        if len(categorical_vars) > 0:
            st.subheader("Categorical Variables Summary")
            cat_summary = []
            for var in categorical_vars[:10]:  # Limit to first 10
                var_label, _ = get_variable_info(var, metadata)
                unique_count = df_filtered[var].nunique()
                most_common = df_filtered[var].mode().iloc[0] if len(df_filtered[var].mode()) > 0 else "N/A"
                cat_summary.append({
                    'Variable': var_label,
                    'Unique Values': unique_count,
                    'Most Common': most_common
                })
            
            cat_df = pd.DataFrame(cat_summary)
            st.dataframe(cat_df, use_container_width=True)
    
    with tab5:
        st.header("Export Data")
        
        st.subheader("Download Filtered Dataset")
        
        # Export options
        export_format = st.radio("Export Format:", ["CSV", "Excel", "JSON"])
        
        if st.button("Generate Download Link"):
            if export_format == "CSV":
                csv_data = df_filtered.to_csv(index=False)
                st.download_button(
                    label="Download CSV",
                    data=csv_data,
                    file_name=f"afrobarometer_{selected_country_label.replace(' ', '_')}.csv",
                    mime="text/csv"
                )
            
            elif export_format == "Excel":
                excel_buffer = io.BytesIO()
                with pd.ExcelWriter(excel_buffer, engine='openpyxl') as writer:
                    df_filtered.to_excel(writer, sheet_name='Data', index=False)
                    
                    # Add metadata sheet
                    if metadata['var_labels']:
                        var_labels_df = pd.DataFrame(list(metadata['var_labels'].items()), 
                                                   columns=['Variable', 'Label'])
                        var_labels_df.to_excel(writer, sheet_name='Variable_Labels', index=False)
                
                excel_data = excel_buffer.getvalue()
                st.download_button(
                    label="Download Excel",
                    data=excel_data,
                    file_name=f"afrobarometer_{selected_country_label.replace(' ', '_')}.xlsx",
                    mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
                )
            
            elif export_format == "JSON":
                json_data = df_filtered.to_json(orient='records', indent=2)
                st.download_button(
                    label="Download JSON",
                    data=json_data,
                    file_name=f"afrobarometer_{selected_country_label.replace(' ', '_')}.json",
                    mime="application/json"
                )
        
        # Export summary
        st.subheader("Export Summary Report")
        if st.button("Generate Summary Report"):
            report = f"""
# Afrobarometer Data Summary Report
Generated on: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

## Dataset Information
- **Country**: {selected_country_label}
- **Total Observations**: {len(df_filtered):,}
- **Total Variables**: {len(df_filtered.columns):,}
- **Numeric Variables**: {len(numeric_vars)}
- **Categorical Variables**: {len(categorical_vars)}

## Data Quality
- **Missing Data Percentage**: {(df_filtered.isnull().sum().sum() / (len(df_filtered) * len(df_filtered.columns)) * 100):.1f}%
- **Complete Cases**: {len(df_filtered.dropna()):,}

## Variable Summary
"""
            
            if len(numeric_vars) > 0:
                report += f"\n### Numeric Variables ({len(numeric_vars)})\n"
                for var in numeric_vars[:10]:
                    var_label, _ = get_variable_info(var, metadata)
                    report += f"- {var_label} ({var})\n"
            
            if len(categorical_vars) > 0:
                report += f"\n### Categorical Variables ({len(categorical_vars)})\n"
                for var in categorical_vars[:10]:
                    var_label, _ = get_variable_info(var, metadata)
                    report += f"- {var_label} ({var})\n"
            
            st.download_button(
                label="Download Summary Report",
                data=report,
                file_name=f"afrobarometer_summary_{selected_country_label.replace(' ', '_')}.md",
                mime="text/markdown"
            )
    
    # Footer
    st.markdown("---")
    st.markdown("**Afrobarometer Data Explorer** | Built with Streamlit | Data: Round 9 (39 Countries)")

if __name__ == "__main__":
    main()