CSV download improvements (#1775)

Change endpoint to download-data from download-csv
add endpoint to projects
Adds a Whitelist to allow for specific non-admin users to access individual forecast-level data

Author: lsabor

docs/openapi.yml

@@ -588,15 +588,15 @@
   "boost": "Boost",
   "bury": "Bury",
   "duplicate": "Duplicate",
-  "downloadCSV": "Download CSV",
+  "downloadQuestionData": "Download Question data",
   "partnersUseForecasts": "Our partners use Metaculus forecasts to gain insight into complex challenges and decisions.",
   "learnHowYouCanPartner": "Learn how you can partner with Metaculus to host a forecasting tournament, collaborate on forecasting research, hire our <link>Pro Forecasters</link> for custom projects, or set up a private forecasting space for your organization.",
   "workingWithNonProfits": "We love working with non-profits in our core focus areas. If you’re a non-profit seeking forecasting and modeling capacity or are interested in collaborating on a grant, please reach out!",
   "reachOutToLearnMore": "Please reach out to learn more about working with Metaculus or with any questions, comments or concerns. We’ll get back to you soon.",
   "feelFreeToJustSayHello": "Feel free to just say hello, too - we love hearing from the forecasting community!",
   "contentBoosted": "Content boosted! value {score} activity. Total boost score for the week: {score_total}",
   "contentBuried": "Content buried! value {score} activity. Total boost score for the week: {score_total}",
-  "downloadCSVError": "Error downloading CSV",
+  "downloadQuestionDataError": "Error downloading Question data",
   "cpRevealTime": "CP Reveal Time",
   "openTime": "Open Time",
   "Question": "Question",

front_end/messages/en.json

@@ -351,8 +351,8 @@ export async function changePostSubscriptions(
   return response;
 }
 
-export async function getPostCSVData(postId: number) {
-  const blob = await PostsApi.getPostCSVData(postId);
+export async function getPostZipData(postId: number) {
+  const blob = await PostsApi.getPostZipData(postId);
   const arrayBuffer = await blob.arrayBuffer();
   const base64String = Buffer.from(arrayBuffer).toString("base64");
 

front_end/src/app/(main)/questions/actions.ts

@@ -8,7 +8,7 @@ import toast from "react-hot-toast";
 
 import {
   changePostActivityBoost,
-  getPostCSVData,
+  getPostZipData,
 } from "@/app/(main)/questions/actions";
 import Button from "@/components/ui/button";
 import DropdownMenu, { MenuItemProps } from "@/components/ui/dropdown_menu";
@@ -50,22 +50,22 @@ export const PostDropdownMenu: FC<Props> = ({ post }) => {
     return `/questions/create/question?mode=create&post_id=${post.id}`;
   };
 
-  const handleDownloadCSV = async () => {
+  const handleDownloadQuestionData = async () => {
     try {
-      const base64 = await getPostCSVData(post.id);
+      const base64 = await getPostZipData(post.id);
       const blob = base64ToBlob(base64);
-      const filename = `${post.url_title.replaceAll(" ", "_")}.csv`;
+      const filename = `${post.url_title.replaceAll(" ", "_")}.zip`;
       saveAs(blob, filename);
     } catch (error) {
-      toast.error(t("downloadCSVError") + error);
+      toast.error(t("downloadQuestionDataError") + error);
     }
   };
 
   const menuItems: MenuItemProps[] = [
     {
-      id: "downloadCSV",
-      name: t("downloadCSV"),
-      onClick: handleDownloadCSV,
+      id: "downloadQuestionData",
+      name: t("downloadQuestionData"),
+      onClick: handleDownloadQuestionData,
     },
   ];
   if (user?.is_superuser) {

front_end/src/components/post_actions/post_dropdown_menu.tsx

@@ -221,8 +221,8 @@ class PostsApi {
     return await get<{ id: number; post_slug: string }>("/posts/random/");
   }
 
-  static async getPostCSVData(postId: number): Promise<Blob> {
-    return await get<Blob>(`/posts/${postId}/download-csv/`);
+  static async getPostZipData(postId: number): Promise<Blob> {
+    return await get<Blob>(`/posts/${postId}/download-data/`);
   }
 }
 

front_end/src/services/posts.ts

@@ -74,8 +74,8 @@ const handleResponse = async <T>(response: Response): Promise<T> => {
   // Check the content type to determine how to process the response
   const contentType = response.headers.get("content-type");
 
-  if (contentType && contentType.includes("text/csv")) {
-    // If the response is a CSV, return it as a Blob
+  if (contentType && contentType.includes("application/zip")) {
+    // If the response is a ZIP, return it as a Blob
     return response.blob() as unknown as T;
   }
 

front_end/src/utils/fetch.ts

@@ -0,0 +1,71 @@
+# Generated by Django 5.1.4 on 2024-12-27 18:53
+
+import django.db.models.deletion
+import django.utils.timezone
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("misc", "0002_initial"),
+        ("posts", "0012_alter_post_default_project"),
+        ("projects", "0010_remove_project_add_posts_to_main_feed_and_more"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="WhitelistUser",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created_at",
+                    models.DateTimeField(
+                        default=django.utils.timezone.now, editable=False
+                    ),
+                ),
+                ("edited_at", models.DateTimeField(editable=False, null=True)),
+                (
+                    "post",
+                    models.ForeignKey(
+                        help_text="Optional. If provided, this allows the user to download user-level data for the post. If neither project nor post is set, the user is whitelisted for all data.",
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="whitelists",
+                        to="posts.post",
+                    ),
+                ),
+                (
+                    "project",
+                    models.ForeignKey(
+                        help_text="Optional. If provided, this allows the user to download user-level data for the project. If neither project nor post is set, the user is whitelisted for all data.",
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="whitelists",
+                        to="projects.project",
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="whitelists",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

misc/migrations/0003_whitelistuser.py

@@ -2,6 +2,9 @@
 from pgvector.django import VectorField
 
 from utils.models import TimeStampedModel
+from users.models import User
+from projects.models import Project
+from posts.models import Post
 
 
 class ITNArticle(TimeStampedModel):
@@ -30,9 +33,33 @@ class Bulletin(TimeStampedModel):
 
 class BulletinViewedBy(TimeStampedModel):
     bulletin = models.ForeignKey(Bulletin, on_delete=models.CASCADE)
-    user = models.ForeignKey("users.User", on_delete=models.CASCADE)
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
 
 
 # TODO: index new posts
 # TODO: ensure we sync PostITNArticle new articles only
 # TODO: create a sync command + cron job
+
+
+class WhitelistUser(TimeStampedModel):
+    """Whitelist for users for permission to download user-level data"""
+
+    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="whitelists")
+    project = models.ForeignKey(
+        Project,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="whitelists",
+        help_text="Optional. If provided, this allows the user to download user-level "
+        "data for the project. If neither project nor post is set, the user is "
+        "whitelisted for all data.",
+    )
+    post = models.ForeignKey(
+        Post,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="whitelists",
+        help_text="Optional. If provided, this allows the user to download user-level "
+        "data for the post. If neither project nor post is set, the user is "
+        "whitelisted for all data.",
+    )

misc/models.py

@@ -64,7 +64,7 @@ def export_selected_posts_data(self, request, queryset: QuerySet[Post]):
 
         questions = Question.objects.filter(related_posts__post__in=queryset).distinct()
 
-        data = export_data_for_questions(questions)
+        data = export_data_for_questions(questions, True, True, True)
         if data is None:
             self.message_user(request, "No questions selected.")
             return

posts/admin.py

@@ -29,6 +29,7 @@
     GroupOfQuestionsUpdateSerializer,
 )
 from questions.services import get_aggregated_forecasts_for_questions
+from questions.types import AggregationMethod
 from users.models import User
 from utils.dtypes import flatten
 from utils.serializers import SerializerKeyLookupMixin
@@ -594,3 +595,90 @@ class PostRelatedArticleSerializer(serializers.ModelSerializer):
     class Meta:
         model = ITNArticle
         fields = ("id", "title", "url", "favicon_url", "created_at", "media_label")
+
+
+class DownloadDataSerializer(serializers.Serializer):
+    sub_question = serializers.IntegerField(required=False)
+    aggregation_methods = serializers.CharField(required=False)
+    user_ids = serializers.CharField(required=False, allow_null=True)
+    include_comments = serializers.BooleanField(required=False, default=False)
+    include_scores = serializers.BooleanField(required=False, default=False)
+    include_bots = serializers.BooleanField(required=False, allow_null=True)
+    minimize = serializers.BooleanField(required=False, default=True)
+
+    def validate_aggregation_methods(self, value):
+        if value is None:
+            return
+        user: User = self.context["user"]
+        if value == "all":
+            aggregation_methods = [
+                AggregationMethod.RECENCY_WEIGHTED,
+                AggregationMethod.UNWEIGHTED,
+                AggregationMethod.METACULUS_PREDICTION,
+            ]
+            if user.is_staff:
+                aggregation_methods.append(AggregationMethod.SINGLE_AGGREGATION)
+            return aggregation_methods
+        methods = value.split(",")
+        invalid_methods = [
+            method for method in methods if method not in AggregationMethod.values
+        ]
+        if invalid_methods:
+            raise serializers.ValidationError(
+                f"Invalid aggregation method(s): {', '.join(invalid_methods)}"
+            )
+        if not user.is_staff:
+            methods = [
+                method
+                for method in methods
+                if method != AggregationMethod.SINGLE_AGGREGATION
+            ]
+        return methods
+
+    def validate_user_ids(self, value):
+        if not value:
+            return value
+        user_ids = value.split(",")
+        if not all(user_id.isdigit() for user_id in user_ids):
+            raise serializers.ValidationError(
+                "Invalid user_ids. Must be a comma-separated list of integers."
+            )
+        if not self.context["can_view_private_data"]:
+            raise serializers.ValidationError(
+                "Current user cannot view user-specific data. "
+                "Please remove user_ids parameter."
+            )
+        uids = [int(user_id) for user_id in user_ids]
+        return uids
+
+    def validate(self, attrs):
+        # Check if there are any unexpected fields
+        allowed_fields = {
+            "sub_question",
+            "aggregation_methods",
+            "user_ids",
+            "include_comments",
+            "include_scores",
+            "include_bots",
+            "minimize",
+        }
+        input_fields = set(self.initial_data.keys())
+        unexpected_fields = input_fields - allowed_fields
+        if unexpected_fields:
+            raise ValidationError(f"Unexpected fields: {', '.join(unexpected_fields)}")
+
+        # Aggregation validation logic
+        aggregation_methods = attrs.get("aggregation_methods")
+        user_ids = attrs.get("user_ids")
+        include_bots = attrs.get("include_bots")
+        minimize = attrs.get("minimize", True)
+
+        if not aggregation_methods and (
+            user_ids is not None or include_bots is not None or not minimize
+        ):
+            raise serializers.ValidationError(
+                "If user_ids, include_bots, or minimize is set, "
+                "aggregation_methods must also be set."
+            )
+
+        return attrs