{% blocktrans with grace_period_end=params.grace_period_end %} Please update your forecast by {{ grace_period_end }}. If not, it will be automatically withdrawn at that time. {% endblocktrans %}
+
{% blocktrans %} Learn more in our {% endblocktrans %} FAQ.
{% blocktrans %} You can manage your notification settings on your {% endblocktrans %} {% trans "settings page" %}. {% blocktrans %} You can also unsubscribe from all Metaculus emails here. {% endblocktrans %}
+ {% blocktrans with grace_period_end=params.grace_period_end %}
+ Please update your forecast by {{ grace_period_end }}. If not,
+ it will be automatically withdrawn at that time.
+ {% endblocktrans %}
+
+
+ {% blocktrans %}
+ Learn more in our
+ {% endblocktrans %}
+ FAQ.
+
{% blocktrans with timestep=params.timestep %} This change took effect at {{ timestep }}. Any probability on removed options was moved to the catch-all option. {% endblocktrans %}
{% blocktrans %} You can manage your notification settings on your {% endblocktrans %} {% trans "settings page" %}. {% blocktrans %} You can also unsubscribe from all Metaculus emails here. {% endblocktrans %}
+ {% blocktrans with timestep=params.timestep %}
+ This change took effect at {{ timestep }}. Any probability on removed options
+ was moved to the catch-all option.
+ {% endblocktrans %}
+
+
+
+
+
+
+
+
+ Review the question
+
+
+
+
+
+
+
+
diff --git a/posts/models.py b/posts/models.py
index 275f08a701..757524403f 100644
--- a/posts/models.py
+++ b/posts/models.py
@@ -813,7 +813,11 @@ def update_forecasts_count(self):
Update forecasts count cache
"""
- self.forecasts_count = self.forecasts.filter_within_question_period().count()
+ self.forecasts_count = (
+ self.forecasts.filter_within_question_period()
+ .exclude(source=Forecast.SourceChoices.AUTOMATIC)
+ .count()
+ )
self.save(update_fields=["forecasts_count"])
def update_forecasters_count(self):
diff --git a/questions/admin.py b/questions/admin.py
index 66e662f750..afd8183733 100644
--- a/questions/admin.py
+++ b/questions/admin.py
@@ -1,10 +1,17 @@
from admin_auto_filters.filters import AutocompleteFilterFactory
-from django.contrib import admin
+from datetime import datetime, timedelta
+
+from django import forms
+from django.contrib import admin, messages
+from django.core.exceptions import PermissionDenied
from django.db.models import QuerySet
-from django.http import HttpResponse
-from django.urls import reverse
+from django.http import Http404, HttpResponse, HttpResponseRedirect
+from django.template.response import TemplateResponse
+from django.urls import path, reverse
+from django.utils import timezone
from django.utils.html import format_html
from django_better_admin_arrayfield.admin.mixins import DynamicArrayMixin
+from rest_framework.exceptions import ValidationError as DRFValidationError
from posts.models import Post
from posts.tasks import run_post_generate_history_snapshot
@@ -18,10 +25,402 @@
)
from questions.services.forecasts import build_question_forecasts
from questions.types import AggregationMethod
+from questions.services.multiple_choice_handlers import (
+ MultipleChoiceOptionsUpdateSerializer,
+ get_all_options_from_history,
+ multiple_choice_add_options,
+ multiple_choice_change_grace_period_end,
+ multiple_choice_delete_options,
+ multiple_choice_rename_option,
+ multiple_choice_reorder_options,
+)
from utils.csv_utils import export_all_data_for_questions
from utils.models import CustomTranslationAdmin
+def get_latest_options_history_datetime(options_history):
+ if not options_history:
+ return None
+ raw_timestamp = options_history[-1][0]
+ try:
+ if isinstance(raw_timestamp, datetime):
+ parsed_timestamp = raw_timestamp
+ elif isinstance(raw_timestamp, str):
+ parsed_timestamp = datetime.fromisoformat(raw_timestamp)
+ else:
+ return None
+ except ValueError:
+ return None
+ if timezone.is_naive(parsed_timestamp):
+ parsed_timestamp = timezone.make_aware(parsed_timestamp)
+ return parsed_timestamp
+
+
+def has_active_grace_period(options_history, reference_time=None):
+ reference_time = reference_time or timezone.now()
+ latest_timestamp = get_latest_options_history_datetime(options_history)
+ return bool(latest_timestamp and latest_timestamp > reference_time)
+
+
+class MultipleChoiceOptionsAdminForm(forms.Form):
+ ACTION_RENAME = "rename_options"
+ ACTION_DELETE = "delete_options"
+ ACTION_ADD = "add_options"
+ ACTION_CHANGE_GRACE = "change_grace_period_end" # not ready yet
+ ACTION_REORDER = "reorder_options"
+ ACTION_CHOICES = (
+ (ACTION_RENAME, "Rename options"),
+ (ACTION_DELETE, "Delete options"),
+ (ACTION_ADD, "Add options"),
+ # (ACTION_CHANGE_GRACE, "Change grace period end"),
+ (ACTION_REORDER, "Reorder options"),
+ )
+
+ action = forms.ChoiceField(choices=ACTION_CHOICES, required=True)
+ old_option = forms.ChoiceField(required=False)
+ new_option = forms.CharField(
+ required=False, label="New option text", strip=True, max_length=200
+ )
+ options_to_delete = forms.MultipleChoiceField(
+ required=False, widget=forms.CheckboxSelectMultiple
+ )
+ new_options = forms.CharField(
+ required=False,
+ help_text="Comma-separated options to add before the catch-all option.",
+ )
+ grace_period_end = forms.DateTimeField(
+ required=False,
+ help_text=(
+ "Default value is 2 weeks from now. "
+ "Required when adding options; must be in the future. "
+ "Format: YYYY-MM-DD or YYYY-MM-DD HH:MM (time optional)."
+ ),
+ input_formats=["%Y-%m-%dT%H:%M", "%Y-%m-%d %H:%M", "%Y-%m-%d"],
+ )
+ delete_comment = forms.CharField(
+ required=False,
+ label="Delete options comment",
+ widget=forms.Textarea(attrs={"rows": 3}),
+ help_text="Placeholders will auto-fill; edit as needed."
+ " {removed_options} becomes ['a', 'b'], {timestep} is the time of "
+ "deletion in isoformat.",
+ )
+ add_comment = forms.CharField(
+ required=False,
+ label="Add options comment",
+ widget=forms.Textarea(attrs={"rows": 4}),
+ help_text="Placeholders will auto-fill; edit as needed."
+ " {removed_options} becomes ['a', 'b'], {timestep} is the time of "
+ "deletion in isoformat.",
+ )
+
+ def __init__(self, question: Question, *args, **kwargs):
+ self.question = question
+ super().__init__(*args, **kwargs)
+
+ options_history = question.options_history or []
+ self.options_grace_period_end = get_latest_options_history_datetime(
+ options_history
+ )
+ default_delete_comment = (
+ "Options {removed_options} were removed at {timestep}. "
+ "Forecasts were adjusted to keep remaining probability on the catch-all."
+ )
+ default_add_comment = (
+ "Options {added_options} were added at {timestep}. "
+ "Please update forecasts before {grace_period_end}; "
+ "forecasts that are not updated will auto-withdraw then."
+ )
+
+ active_grace = has_active_grace_period(options_history)
+ action_choices = list(self.ACTION_CHOICES)
+ if active_grace:
+ action_choices = [
+ choice
+ for choice in action_choices
+ if choice[0] in (self.ACTION_RENAME, self.ACTION_CHANGE_GRACE)
+ ]
+ else:
+ action_choices = [
+ choice
+ for choice in action_choices
+ if choice[0] != self.ACTION_CHANGE_GRACE
+ ]
+ if len(options_history) > 1:
+ action_choices = [
+ choice for choice in action_choices if choice[0] != self.ACTION_REORDER
+ ]
+ action = forms.ChoiceField(
+ choices=[("", "Select action")] + action_choices,
+ required=True,
+ initial="",
+ )
+ self.fields["action"] = action
+ all_options = (
+ get_all_options_from_history(options_history) if options_history else []
+ )
+ self.fields["old_option"].choices = [(opt, opt) for opt in all_options]
+
+ current_options = question.options or []
+ self.fields["options_to_delete"].choices = [
+ (opt, opt) for opt in current_options
+ ]
+ self.reorder_field_names: list[tuple[str, str]] = []
+ for index, option in enumerate(current_options):
+ field_name = f"reorder_position_{index}"
+ self.reorder_field_names.append((option, field_name))
+ self.fields[field_name] = forms.IntegerField(
+ required=False,
+ min_value=1,
+ label=f"Order for '{option}'",
+ help_text="Use integers; options will be ordered ascending.",
+ )
+ if current_options:
+ self.fields["options_to_delete"].widget.attrs["data-catch-all"] = (
+ current_options[-1]
+ )
+ self.fields["options_to_delete"].help_text = (
+ "Warning: do not remove all options. The question should have at least "
+ "2 options: the last option you can't delete, and one other."
+ )
+ grace_field = self.fields["grace_period_end"]
+ grace_field.widget = forms.DateTimeInput(attrs={"type": "datetime-local"})
+ grace_initial = self.options_grace_period_end or (
+ timezone.now() + timedelta(days=14)
+ )
+ if grace_initial and timezone.is_naive(grace_initial):
+ grace_initial = timezone.make_aware(grace_initial)
+ grace_field.initial = timezone.localtime(grace_initial).strftime(
+ "%Y-%m-%dT%H:%M"
+ )
+ if self.options_grace_period_end:
+ grace_field.help_text = (
+ f"Current grace period end: "
+ f"{timezone.localtime(self.options_grace_period_end)}. "
+ "Provide a new end to extend or shorten."
+ )
+ self.fields["delete_comment"].initial = default_delete_comment
+ self.fields["add_comment"].initial = default_add_comment
+
+ def is_in_grace_period(self, reference_time=None):
+ reference_time = reference_time or timezone.now()
+ return bool(
+ self.options_grace_period_end
+ and self.options_grace_period_end > reference_time
+ )
+
+ def clean(self):
+ cleaned_data = super().clean()
+ question = self.question
+ action = cleaned_data.get("action")
+ current_options = question.options or []
+ options_history = question.options_history or []
+ now = timezone.now()
+
+ if not question.options or not question.options_history:
+ raise forms.ValidationError(
+ "This question needs options and an options history to update."
+ )
+
+ if not action:
+ return cleaned_data
+
+ if action == self.ACTION_RENAME:
+ old_option = cleaned_data.get("old_option")
+ new_option = cleaned_data.get("new_option", "")
+
+ if not old_option:
+ self.add_error("old_option", "Select an option to rename.")
+ if not new_option or not new_option.strip():
+ self.add_error("new_option", "Enter the new option text.")
+ new_option = (new_option or "").strip()
+
+ if self.errors:
+ return cleaned_data
+
+ if old_option not in current_options:
+ self.add_error(
+ "old_option", "Selected option is not part of the current choices."
+ )
+ return cleaned_data
+
+ new_options = [
+ new_option if opt == old_option else opt for opt in current_options
+ ]
+ if len(set(new_options)) != len(new_options):
+ self.add_error(
+ "new_option", "New option duplicates an existing option."
+ )
+ return cleaned_data
+
+ cleaned_data["target_option"] = old_option
+ cleaned_data["parsed_new_option"] = new_option
+ return cleaned_data
+
+ if action == self.ACTION_DELETE:
+ options_to_delete = cleaned_data.get("options_to_delete") or []
+ catch_all_option = current_options[-1] if current_options else None
+ if not options_to_delete:
+ self.add_error(
+ "options_to_delete", "Select at least one option to delete."
+ )
+ return cleaned_data
+ if catch_all_option and catch_all_option in options_to_delete:
+ self.add_error(
+ "options_to_delete", "The final catch-all option cannot be deleted."
+ )
+
+ new_options = [
+ opt for opt in current_options if opt not in options_to_delete
+ ]
+ if len(new_options) < 2:
+ self.add_error(
+ "options_to_delete",
+ "At least one option in addition to the catch-all must remain.",
+ )
+ if self.is_in_grace_period(now):
+ self.add_error(
+ "options_to_delete",
+ "Options cannot change during an active grace period.",
+ )
+
+ if self.errors:
+ return cleaned_data
+
+ serializer = MultipleChoiceOptionsUpdateSerializer(
+ context={"question": question}
+ )
+ try:
+ serializer.validate_new_options(new_options, options_history, None)
+ except DRFValidationError as exc:
+ raise forms.ValidationError(exc.detail or exc.args)
+
+ cleaned_data["options_to_delete"] = options_to_delete
+ cleaned_data["delete_comment"] = cleaned_data.get("delete_comment", "")
+ return cleaned_data
+
+ if action == self.ACTION_ADD:
+ new_options_raw = cleaned_data.get("new_options") or ""
+ grace_period_end = cleaned_data.get("grace_period_end")
+ if grace_period_end and timezone.is_naive(grace_period_end):
+ grace_period_end = timezone.make_aware(grace_period_end)
+ cleaned_data["grace_period_end"] = grace_period_end
+ new_options_list = [
+ opt.strip() for opt in new_options_raw.split(",") if opt.strip()
+ ]
+ if not new_options_list:
+ self.add_error("new_options", "Enter at least one option to add.")
+ if len(new_options_list) != len(set(new_options_list)):
+ self.add_error("new_options", "New options list includes duplicates.")
+
+ duplicate_existing = set(current_options).intersection(new_options_list)
+ if duplicate_existing:
+ self.add_error(
+ "new_options",
+ f"Options already exist: {', '.join(sorted(duplicate_existing))}",
+ )
+
+ if not grace_period_end:
+ self.add_error(
+ "grace_period_end", "Grace period end is required when adding."
+ )
+ elif grace_period_end <= now:
+ self.add_error(
+ "grace_period_end", "Grace period end must be in the future."
+ )
+ if self.is_in_grace_period(now):
+ self.add_error(
+ "grace_period_end",
+ "Options cannot change during an active grace period.",
+ )
+
+ if self.errors:
+ return cleaned_data
+
+ serializer = MultipleChoiceOptionsUpdateSerializer(
+ context={"question": question}
+ )
+ new_options = current_options[:-1] + new_options_list + current_options[-1:]
+ try:
+ serializer.validate_new_options(
+ new_options, options_history, grace_period_end
+ )
+ except DRFValidationError as exc:
+ raise forms.ValidationError(exc.detail or exc.args)
+
+ cleaned_data["new_options_list"] = new_options_list
+ cleaned_data["grace_period_end"] = grace_period_end
+ cleaned_data["add_comment"] = cleaned_data.get("add_comment", "")
+ return cleaned_data
+
+ if action == self.ACTION_CHANGE_GRACE:
+ new_grace_end = cleaned_data.get("grace_period_end")
+ if new_grace_end and timezone.is_naive(new_grace_end):
+ new_grace_end = timezone.make_aware(new_grace_end)
+ cleaned_data["grace_period_end"] = new_grace_end
+
+ if not new_grace_end:
+ self.add_error(
+ "grace_period_end", "New grace period end is required to change it."
+ )
+ elif new_grace_end <= now:
+ self.add_error(
+ "grace_period_end", "Grace period end must be in the future."
+ )
+
+ if not self.is_in_grace_period(now):
+ self.add_error(
+ "grace_period_end",
+ "There is no active grace period to change.",
+ )
+
+ if self.errors:
+ return cleaned_data
+
+ cleaned_data["new_grace_period_end"] = new_grace_end
+ return cleaned_data
+
+ if action == self.ACTION_REORDER:
+ if len(options_history) > 1:
+ self.add_error(
+ "action",
+ "Options can only be reordered when there is a single options history entry.",
+ )
+ return cleaned_data
+
+ positions: dict[str, int] = {}
+ seen_values: set[int] = set()
+
+ for option, field_name in getattr(self, "reorder_field_names", []):
+ value = cleaned_data.get(field_name)
+ if value is None:
+ self.add_error(field_name, "Enter an order value.")
+ continue
+ if value in seen_values:
+ self.add_error(
+ field_name,
+ "Order value must be unique.",
+ )
+ continue
+ seen_values.add(value)
+ positions[option] = value
+
+ if self.errors:
+ return cleaned_data
+
+ if len(positions) != len(current_options):
+ raise forms.ValidationError("Provide an order value for every option.")
+
+ desired_order = [
+ option
+ for option, _ in sorted(positions.items(), key=lambda item: item[1])
+ ]
+ cleaned_data["new_order"] = desired_order
+ return cleaned_data
+
+ raise forms.ValidationError("Invalid action selected.")
+
+
@admin.register(Question)
class QuestionAdmin(CustomTranslationAdmin, DynamicArrayMixin):
list_display = [
@@ -33,7 +432,13 @@ class QuestionAdmin(CustomTranslationAdmin, DynamicArrayMixin):
"curation_status",
"post_link",
]
- readonly_fields = ["post_link", "view_forecasts"]
+ readonly_fields = [
+ "post_link",
+ "view_forecasts",
+ "options",
+ "options_history",
+ "update_mc_options",
+ ]
search_fields = [
"id",
"title_original",
@@ -84,6 +489,22 @@ def view_forecasts(self, obj):
url = reverse("admin:questions_forecast_changelist") + f"?question={obj.id}"
return format_html('View Forecasts', url)
+ def update_mc_options(self, obj):
+ if not obj:
+ return "Save the question to manage options."
+ if obj.type != Question.QuestionType.MULTIPLE_CHOICE:
+ return "Option updates are available for multiple choice questions only."
+ if not obj.options_history or not obj.options:
+ return "Options and options history are required to update choices."
+ url = reverse("admin:questions_question_update_options", args=[obj.id])
+ return format_html(
+ 'Update multiple choice options'
+ '
Rename, delete, or add options while keeping history.
',
+ url,
+ )
+
+ update_mc_options.short_description = "Multiple choice options"
+
def should_update_translations(self, obj):
post = obj.get_post()
is_private = post.default_project.default_permission is None
@@ -91,12 +512,34 @@ def should_update_translations(self, obj):
return not is_private and is_approved
+ def get_urls(self):
+ urls = super().get_urls()
+ custom_urls = [
+ path(
+ "/update-options/",
+ self.admin_site.admin_view(self.update_options_view),
+ name="questions_question_update_options",
+ ),
+ ]
+ return custom_urls + urls
+
def get_fields(self, request, obj=None):
fields = super().get_fields(request, obj)
+
+ def insert_after(target_field: str, new_field: str):
+ if new_field in fields:
+ fields.remove(new_field)
+ if target_field in fields:
+ fields.insert(fields.index(target_field) + 1, new_field)
+ else:
+ fields.append(new_field)
+
for field in ["post_link", "view_forecasts"]:
if field in fields:
fields.remove(field)
fields.insert(0, field)
+ if obj:
+ insert_after("options_history", "update_mc_options")
return fields
def save_model(self, request, obj, form, change):
@@ -139,6 +582,122 @@ def export_selected_questions_data_anonymized(
):
return self.export_selected_questions_data(request, queryset, anonymized=True)
+ def update_options_view(self, request, question_id: int):
+ question = Question.objects.filter(pk=question_id).first()
+ if not question:
+ raise Http404("Question not found.")
+ if not self.has_change_permission(request, question):
+ raise PermissionDenied
+
+ change_url = reverse("admin:questions_question_change", args=[question.id])
+ if question.type != Question.QuestionType.MULTIPLE_CHOICE:
+ messages.error(
+ request, "Option updates are available for multiple choice questions."
+ )
+ return HttpResponseRedirect(change_url)
+ if not question.options or not question.options_history:
+ messages.error(
+ request,
+ "Options and options history are required before updating choices.",
+ )
+ return HttpResponseRedirect(change_url)
+
+ form = MultipleChoiceOptionsAdminForm(
+ question, data=request.POST or None, prefix="options"
+ )
+ if request.method == "POST" and form.is_valid():
+ action = form.cleaned_data["action"]
+ if action == form.ACTION_RENAME:
+ old_option = form.cleaned_data["target_option"]
+ new_option = form.cleaned_data["parsed_new_option"]
+ multiple_choice_rename_option(question, old_option, new_option)
+ question.save(update_fields=["options", "options_history"])
+ self.message_user(
+ request, f"Renamed option '{old_option}' to '{new_option}'."
+ )
+ elif action == form.ACTION_REORDER:
+ new_order = form.cleaned_data["new_order"]
+ multiple_choice_reorder_options(question, new_order)
+ question.save(update_fields=["options", "options_history"])
+ self.message_user(
+ request,
+ "Reordered options.",
+ )
+ elif action == form.ACTION_DELETE:
+ options_to_delete = form.cleaned_data["options_to_delete"]
+ delete_comment = form.cleaned_data.get("delete_comment", "")
+ multiple_choice_delete_options(
+ question,
+ options_to_delete,
+ comment_author=request.user,
+ timestep=timezone.now(),
+ comment_text=delete_comment,
+ )
+ question.save(update_fields=["options", "options_history"])
+ self.message_user(
+ request,
+ f"Deleted {len(options_to_delete)} option"
+ f"{'' if len(options_to_delete) == 1 else 's'}.",
+ )
+ elif action == form.ACTION_ADD:
+ new_options = form.cleaned_data["new_options_list"]
+ grace_period_end = form.cleaned_data["grace_period_end"]
+ add_comment = form.cleaned_data.get("add_comment", "")
+ if timezone.is_naive(grace_period_end):
+ grace_period_end = timezone.make_aware(grace_period_end)
+ multiple_choice_add_options(
+ question,
+ new_options,
+ grace_period_end=grace_period_end,
+ comment_author=request.user,
+ timestep=timezone.now(),
+ comment_text=add_comment,
+ )
+ question.save(update_fields=["options", "options_history"])
+ self.message_user(
+ request,
+ f"Added {len(new_options)} option"
+ f"{'' if len(new_options) == 1 else 's'}.",
+ )
+ elif action == form.ACTION_CHANGE_GRACE:
+ new_grace_period_end = form.cleaned_data["new_grace_period_end"]
+ if timezone.is_naive(new_grace_period_end):
+ new_grace_period_end = timezone.make_aware(new_grace_period_end)
+ multiple_choice_change_grace_period_end(
+ question,
+ new_grace_period_end,
+ comment_author=request.user,
+ timestep=timezone.now(),
+ )
+ question.save(update_fields=["options_history"])
+ self.message_user(
+ request,
+ f"Grace period end updated to {timezone.localtime(new_grace_period_end)}.",
+ )
+ return HttpResponseRedirect(change_url)
+
+ grace_period_end = form.options_grace_period_end
+ in_grace_period = form.is_in_grace_period()
+
+ context = {
+ **self.admin_site.each_context(request),
+ "opts": self.model._meta,
+ "app_label": self.model._meta.app_label,
+ "original": question,
+ "question": question,
+ "title": f"Update options for {question}",
+ "form": form,
+ "media": self.media + form.media,
+ "change_url": change_url,
+ "current_options": question.options or [],
+ "all_history_options": get_all_options_from_history(
+ question.options_history
+ ),
+ "grace_period_end": grace_period_end,
+ "in_grace_period": in_grace_period,
+ }
+ return TemplateResponse(request, "admin/questions/update_options.html", context)
+
def rebuild_aggregation_history(self, request, queryset: QuerySet[Question]):
for question in queryset:
build_question_forecasts(question)
diff --git a/questions/migrations/0013_forecast_source.py b/questions/migrations/0013_forecast_source.py
index ccd11208eb..4230d216bf 100644
--- a/questions/migrations/0013_forecast_source.py
+++ b/questions/migrations/0013_forecast_source.py
@@ -15,7 +15,7 @@ class Migration(migrations.Migration):
name="source",
field=models.CharField(
blank=True,
- choices=[("api", "Api"), ("ui", "Ui")],
+ choices=[("api", "Api"), ("ui", "Ui"), ("automatic", "Automatic")],
default="",
max_length=30,
null=True,
diff --git a/questions/migrations/0033_question_options_history.py b/questions/migrations/0033_question_options_history.py
new file mode 100644
index 0000000000..7c4b69a97b
--- /dev/null
+++ b/questions/migrations/0033_question_options_history.py
@@ -0,0 +1,50 @@
+# Generated by Django 5.1.13 on 2025-11-15 19:35
+from datetime import datetime
+
+
+import questions.models
+from django.db import migrations, models
+
+
+def initialize_options_history(apps, schema_editor):
+ Question = apps.get_model("questions", "Question")
+ questions = Question.objects.filter(options__isnull=False)
+ for question in questions:
+ if question.options:
+ question.options_history = [(datetime.min.isoformat(), question.options)]
+ Question.objects.bulk_update(questions, ["options_history"])
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("questions", "0032_alter_aggregateforecast_forecast_values_and_more"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="forecast",
+ name="source",
+ field=models.CharField(
+ blank=True,
+ choices=[("api", "Api"), ("ui", "Ui"), ("automatic", "Automatic")],
+ db_index=True,
+ default="",
+ max_length=30,
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="question",
+ name="options_history",
+ field=models.JSONField(
+ blank=True,
+ help_text="For Multiple Choice only.\n list of tuples: (isoformat_datetime, options_list). (json stores them as lists)\n Records the history of options over time.\n Initialized with (datetime.min.isoformat(), self.options) upon question creation.\n Updated whenever options are changed.",
+ null=True,
+ validators=[questions.models.validate_options_history],
+ ),
+ ),
+ migrations.RunPython(
+ initialize_options_history, reverse_code=migrations.RunPython.noop
+ ),
+ ]
diff --git a/questions/models.py b/questions/models.py
index 78d1252853..a7916777dc 100644
--- a/questions/models.py
+++ b/questions/models.py
@@ -9,7 +9,7 @@
from sql_util.aggregates import SubqueryAggregate
from questions.constants import QuestionStatus
-from questions.types import AggregationMethod
+from questions.types import AggregationMethod, OptionsHistoryType
from scoring.constants import ScoreTypes
from users.models import User
from utils.models import TimeStampedModel, TranslatedModel
@@ -21,6 +21,27 @@
DEFAULT_INBOUND_OUTCOME_COUNT = 200
+def validate_options_history(value):
+ # Expect: [ (float, [str, ...]), ... ] or equivalent
+ if not isinstance(value, list):
+ raise ValidationError("Must be a list.")
+ for i, item in enumerate(value):
+ if (
+ not isinstance(item, (list, tuple))
+ or len(item) != 2
+ or not isinstance(item[0], str)
+ or not isinstance(item[1], list)
+ or not all(isinstance(s, str) for s in item[1])
+ ):
+ raise ValidationError(f"Bad item at index {i}: {item!r}")
+ try:
+ datetime.fromisoformat(item[0])
+ except ValueError:
+ raise ValidationError(
+ f"Bad datetime format at index {i}: {item[0]!r}, must be isoformat string"
+ )
+
+
class QuestionQuerySet(QuerySet):
def annotate_forecasts_count(self):
return self.annotate(
@@ -198,8 +219,20 @@ class QuestionType(models.TextChoices):
)
unit = models.CharField(max_length=25, blank=True)
- # list of multiple choice option labels
- options = ArrayField(models.CharField(max_length=200), blank=True, null=True)
+ # multiple choice fields
+ options: list[str] | None = ArrayField(
+ models.CharField(max_length=200), blank=True, null=True
+ )
+ options_history: OptionsHistoryType | None = models.JSONField(
+ null=True,
+ blank=True,
+ validators=[validate_options_history],
+ help_text="""For Multiple Choice only.
+ list of tuples: (isoformat_datetime, options_list). (json stores them as lists)
+ Records the history of options over time.
+ Initialized with (datetime.min.isoformat(), self.options) upon question creation.
+ Updated whenever options are changed.""",
+ )
# Legacy field that will be removed
possibilities = models.JSONField(null=True, blank=True)
@@ -271,6 +304,9 @@ def save(self, **kwargs):
self.zero_point = None
if self.type != self.QuestionType.MULTIPLE_CHOICE:
self.options = None
+ if self.type == self.QuestionType.MULTIPLE_CHOICE and not self.options_history:
+ # initialize options history on first save
+ self.options_history = [(datetime.min.isoformat(), self.options or [])]
return super().save(**kwargs)
@@ -570,8 +606,11 @@ class Forecast(models.Model):
)
class SourceChoices(models.TextChoices):
- API = "api"
- UI = "ui"
+ API = "api" # made via the api
+ UI = "ui" # made using the api
+ # an automatically assigned forecast
+ # usually this means a regular forecast was split
+ AUTOMATIC = "automatic"
# logging the source of the forecast for data purposes
source = models.CharField(
@@ -580,6 +619,7 @@ class SourceChoices(models.TextChoices):
null=True,
choices=SourceChoices.choices,
default="",
+ db_index=True,
)
distribution_input = models.JSONField(
@@ -621,15 +661,17 @@ def get_prediction_values(self) -> list[float | None]:
return self.probability_yes_per_category
return self.continuous_cdf
- def get_pmf(self) -> list[float]:
+ def get_pmf(self, replace_none: bool = False) -> list[float]:
"""
- gets the PMF for this forecast, replacing None values with 0.0
- Not for serialization use (keep None values in that case)
+ gets the PMF for this forecast
+ replaces None values with 0.0 if replace_none is True
"""
# TODO: return a numpy array with NaNs instead of 0.0s
if self.probability_yes:
return [1 - self.probability_yes, self.probability_yes]
if self.probability_yes_per_category:
+ if not replace_none:
+ return self.probability_yes_per_category
return [
v or 0.0 for v in self.probability_yes_per_category
] # replace None with 0.0
@@ -704,19 +746,21 @@ def get_cdf(self) -> list[float | None] | None:
return self.forecast_values
return None
- def get_pmf(self) -> list[float]:
+ def get_pmf(self, replace_none: bool = False) -> list[float | None]:
"""
- gets the PMF for this forecast, replacing None values with 0.0
- Not for serialization use (keep None values in that case)
+ gets the PMF for this forecast
+ replacing None values with 0.0 if replace_none is True
"""
# TODO: return a numpy array with NaNs instead of 0.0s
# grab annotation if it exists for efficiency
question_type = getattr(self, "question_type", self.question.type)
- forecast_values = [
- v or 0.0 for v in self.forecast_values
- ] # replace None with 0.0
+ forecast_values = self.forecast_values
+ if question_type == Question.QuestionType.MULTIPLE_CHOICE:
+ if not replace_none:
+ return forecast_values
+ return [v or 0.0 for v in forecast_values] # replace None with 0.0
if question_type in QUESTION_CONTINUOUS_TYPES:
- cdf: list[float] = forecast_values
+ cdf: list[float] = forecast_values # type: ignore
pmf = [cdf[0]]
for i in range(1, len(cdf)):
pmf.append(cdf[i] - cdf[i - 1])
diff --git a/questions/serializers/common.py b/questions/serializers/common.py
index 7d579e97eb..4777d7fcd7 100644
--- a/questions/serializers/common.py
+++ b/questions/serializers/common.py
@@ -17,10 +17,9 @@
AggregateForecast,
Forecast,
)
-from questions.serializers.aggregate_forecasts import (
- serialize_question_aggregations,
-)
-from questions.types import QuestionMovement
+from questions.serializers.aggregate_forecasts import serialize_question_aggregations
+from questions.services.multiple_choice_handlers import get_all_options_from_history
+from questions.types import OptionsHistoryType, QuestionMovement
from users.models import User
from utils.the_math.formulas import (
get_scaled_quartiles_from_cdf,
@@ -40,6 +39,7 @@ class QuestionSerializer(serializers.ModelSerializer):
actual_close_time = serializers.SerializerMethodField()
resolution = serializers.SerializerMethodField()
spot_scoring_time = serializers.SerializerMethodField()
+ all_options_ever = serializers.SerializerMethodField()
class Meta:
model = Question
@@ -58,6 +58,8 @@ class Meta:
"type",
# Multiple-choice Questions only
"options",
+ "all_options_ever",
+ "options_history",
"group_variable",
# Used for Group Of Questions to determine
# whether question is eligible for forecasting
@@ -122,6 +124,10 @@ def get_actual_close_time(self, question: Question):
return min(question.scheduled_close_time, question.actual_resolve_time)
return question.scheduled_close_time
+ def get_all_options_ever(self, question: Question):
+ if question.options_history:
+ return get_all_options_from_history(question.options_history)
+
def get_resolution(self, question: Question):
resolution = question.resolution
@@ -226,6 +232,23 @@ class Meta(QuestionWriteSerializer.Meta):
"cp_reveal_time",
)
+ def validate(self, data: dict):
+ data = super().validate(data)
+
+ if qid := data.get("id"):
+ question = Question.objects.get(id=qid)
+ if data.get("options") != question.options:
+ # if there are user forecasts, we can't update options this way
+ if question.user_forecasts.exists():
+ ValidationError(
+ "Cannot update options through this endpoint while there are "
+ "user forecasts. "
+ "Instead, use /api/questions/update-mc-options/ or the UI on "
+ "the question detail page."
+ )
+
+ return data
+
# TODO: add validation for updating continuous question bounds
@@ -394,7 +417,7 @@ class ForecastWriteSerializer(serializers.ModelSerializer):
probability_yes = serializers.FloatField(allow_null=True, required=False)
probability_yes_per_category = serializers.DictField(
- child=serializers.FloatField(), allow_null=True, required=False
+ child=serializers.FloatField(allow_null=True), allow_null=True, required=False
)
continuous_cdf = serializers.ListField(
child=serializers.FloatField(),
@@ -435,21 +458,47 @@ def binary_validation(self, probability_yes):
)
return probability_yes
- def multiple_choice_validation(self, probability_yes_per_category, options):
+ def multiple_choice_validation(
+ self,
+ probability_yes_per_category: dict[str, float | None],
+ current_options: list[str],
+ options_history: OptionsHistoryType | None,
+ ):
if probability_yes_per_category is None:
raise serializers.ValidationError(
"probability_yes_per_category is required"
)
if not isinstance(probability_yes_per_category, dict):
raise serializers.ValidationError("Forecast must be a dictionary")
- if set(probability_yes_per_category.keys()) != set(options):
- raise serializers.ValidationError("Forecast must include all options")
- values = [float(probability_yes_per_category[option]) for option in options]
- if not all([0.001 <= v <= 0.999 for v in values]) or not np.isclose(
- sum(values), 1
- ):
+ if not set(current_options).issubset(set(probability_yes_per_category.keys())):
+ raise serializers.ValidationError(
+ f"Forecast must reflect current options: {current_options}"
+ )
+ all_options = get_all_options_from_history(options_history)
+ if not set(probability_yes_per_category.keys()).issubset(set(all_options)):
+ raise serializers.ValidationError(
+ "Forecast contains probabilities for unknown options"
+ )
+
+ values: list[float | None] = []
+ for option in all_options:
+ value = probability_yes_per_category.get(option, None)
+ if option in current_options:
+ if (value is None) or (not (0.001 <= value <= 0.999)):
+ raise serializers.ValidationError(
+ "Probabilities for current options must be between 0.001 and 0.999"
+ )
+ elif value is not None:
+ raise serializers.ValidationError(
+ f"Probability for inactivate option '{option}' must be null or absent"
+ )
+ values.append(value)
+ if not np.isclose(sum(filter(None, values)), 1):
raise serializers.ValidationError(
- "All probabilities must be between 0.001 and 0.999 and sum to 1.0"
+ "Forecast values must sum to 1.0. "
+ f"Received {probability_yes_per_category} which is interpreted as "
+ f"values: {values} representing {all_options} "
+ f"with current options {current_options}"
)
return values
@@ -555,7 +604,7 @@ def validate(self, data):
"provided for multiple choice questions"
)
data["probability_yes_per_category"] = self.multiple_choice_validation(
- probability_yes_per_category, question.options
+ probability_yes_per_category, question.options, question.options_history
)
else: # Continuous question
if probability_yes or probability_yes_per_category:
@@ -633,6 +682,21 @@ def serialize_question(
archived_scores = question.user_archived_scores
user_forecasts = question.request_user_forecasts
last_forecast = user_forecasts[-1] if user_forecasts else None
+ # if the user has a pre-registered forecast,
+ # replace the current forecast and anything after it
+ if question.type == Question.QuestionType.MULTIPLE_CHOICE:
+ # Right now, Multiple Choice is the only type that can have pre-registered
+ # forecasts
+ if last_forecast and last_forecast.start_time > timezone.now():
+ user_forecasts = [
+ f for f in user_forecasts if f.start_time < timezone.now()
+ ]
+ if user_forecasts:
+ last_forecast.start_time = user_forecasts[-1].start_time
+ user_forecasts[-1] = last_forecast
+ else:
+ last_forecast.start_time = timezone.now()
+ user_forecasts = [last_forecast]
if (
last_forecast
and last_forecast.end_time
@@ -647,11 +711,7 @@ def serialize_question(
many=True,
).data,
"latest": (
- MyForecastSerializer(
- user_forecasts[-1],
- ).data
- if user_forecasts
- else None
+ MyForecastSerializer(last_forecast).data if last_forecast else None
),
"score_data": dict(),
}
diff --git a/questions/services/forecasts.py b/questions/services/forecasts.py
index 15aba16fa3..2616dc7f09 100644
--- a/questions/services/forecasts.py
+++ b/questions/services/forecasts.py
@@ -1,7 +1,7 @@
import logging
from collections import defaultdict
-from datetime import timedelta
-from typing import cast, Iterable
+from datetime import datetime, timedelta, timezone as dt_timezone
+from typing import cast, Iterable, Literal
import sentry_sdk
from django.db import transaction
@@ -13,6 +13,7 @@
from posts.models import PostUserSnapshot, PostSubscription
from posts.services.subscriptions import create_subscription_cp_change
from posts.tasks import run_on_post_forecast
+from questions.services.multiple_choice_handlers import get_all_options_from_history
from scoring.models import Score
from users.models import User
from utils.cache import cache_per_object
@@ -34,21 +35,67 @@
def create_forecast(
*,
- question: Question = None,
- user: User = None,
- continuous_cdf: list[float] = None,
- probability_yes: float = None,
- probability_yes_per_category: list[float] = None,
- distribution_input=None,
+ question: Question,
+ user: User,
+ continuous_cdf: list[float] | None = None,
+ probability_yes: float | None = None,
+ probability_yes_per_category: list[float | None] | None = None,
+ distribution_input: dict | None = None,
+ end_time: datetime | None = None,
+ source: Forecast.SourceChoices | Literal[""] | None = None,
**kwargs,
):
now = timezone.now()
post = question.get_post()
+ source = source or ""
+
+ # delete all future-dated predictions, as this one will override them
+ Forecast.objects.filter(question=question, author=user, start_time__gt=now).delete()
+
+ # if the forecast to be created is for a multiple choice question during a grace
+ # period, we need to agument the forecast accordingly (possibly preregister)
+ if question.type == Question.QuestionType.MULTIPLE_CHOICE:
+ if not probability_yes_per_category:
+ raise ValueError("probability_yes_per_category required for MC questions")
+ options_history = question.options_history
+ if options_history and len(options_history) > 1:
+ period_end = datetime.fromisoformat(options_history[-1][0]).replace(
+ tzinfo=dt_timezone.utc
+ )
+ if period_end > now:
+ all_options = get_all_options_from_history(question.options_history)
+ prior_options = options_history[-2][1]
+ if end_time is None or end_time > period_end:
+ # create a pre-registration for the given forecast
+ Forecast.objects.create(
+ question=question,
+ author=user,
+ start_time=period_end,
+ end_time=end_time,
+ probability_yes_per_category=probability_yes_per_category,
+ post=post,
+ source=Forecast.SourceChoices.AUTOMATIC,
+ **kwargs,
+ )
+ end_time = period_end
+
+ prior_pmf: list[float | None] = [None] * len(all_options)
+ for i, (option, value) in enumerate(
+ zip(all_options, probability_yes_per_category)
+ ):
+ if value is None:
+ continue
+ if option in prior_options:
+ prior_pmf[i] = (prior_pmf[i] or 0.0) + value
+ else:
+ prior_pmf[-1] = (prior_pmf[-1] or 0.0) + value
+ probability_yes_per_category = prior_pmf
forecast = Forecast.objects.create(
question=question,
author=user,
start_time=now,
+ end_time=end_time,
continuous_cdf=continuous_cdf,
probability_yes=probability_yes,
probability_yes_per_category=probability_yes_per_category,
@@ -56,6 +103,7 @@ def create_forecast(
distribution_input if question.type in QUESTION_CONTINUOUS_TYPES else None
),
post=post,
+ source=source,
**kwargs,
)
# tidy up all forecasts
diff --git a/questions/services/multiple_choice_handlers.py b/questions/services/multiple_choice_handlers.py
new file mode 100644
index 0000000000..d88d7e1532
--- /dev/null
+++ b/questions/services/multiple_choice_handlers.py
@@ -0,0 +1,376 @@
+from datetime import datetime, timezone as dt_timezone
+
+from django.db import transaction
+from django.db.models import Q
+from django.utils import timezone
+
+from questions.models import Question, Forecast
+from questions.types import OptionsHistoryType
+
+# MOVE THIS serializer imports
+from rest_framework import serializers
+from collections import Counter
+from rest_framework.exceptions import ValidationError
+from users.models import User
+
+
+class MultipleChoiceOptionsUpdateSerializer(serializers.Serializer):
+ options = serializers.ListField(child=serializers.CharField(), required=True)
+ grace_period_end = serializers.DateTimeField(required=False)
+
+ def validate_new_options(
+ self,
+ new_options: list[str],
+ options_history: OptionsHistoryType,
+ grace_period_end: datetime | None = None,
+ ):
+ datetime_str, current_options = options_history[-1]
+ ts = (
+ datetime.fromisoformat(datetime_str)
+ .replace(tzinfo=dt_timezone.utc)
+ .timestamp()
+ )
+ if new_options == current_options: # no change
+ return
+ if len(new_options) == len(current_options): # renaming
+ if any(v > 1 for v in Counter(new_options).values()):
+ ValidationError("new_options includes duplicate labels")
+ elif timezone.now().timestamp() < ts:
+ raise ValidationError("options cannot change during a grace period")
+ elif len(new_options) < len(current_options): # deletion
+ if len(new_options) < 2:
+ raise ValidationError("Must have 2 or more options")
+ if new_options[-1] != current_options[-1]:
+ raise ValidationError("Cannot delete last option")
+ if [o for o in new_options if o not in current_options]:
+ raise ValidationError(
+ "options cannot change name while some are being deleted"
+ )
+ elif len(new_options) > len(current_options): # addition
+ if not grace_period_end or grace_period_end <= timezone.now():
+ raise ValidationError(
+ "grace_period_end must be in the future if adding options"
+ )
+ if new_options[-1] != current_options[-1]:
+ raise ValidationError("Cannot add option after last option")
+ if [o for o in current_options if o not in new_options]:
+ raise ValidationError(
+ "options cannot change name while some are being added"
+ )
+
+ def validate(self, data: dict) -> dict:
+ question: Question = self.context.get("question")
+ if not question:
+ raise ValidationError("question must be provided in context")
+
+ if question.type != Question.QuestionType.MULTIPLE_CHOICE:
+ raise ValidationError("question must be of multiple choice type")
+
+ options = data.get("options")
+ options_history = question.options_history
+ if not options or not options_history:
+ raise ValidationError(
+ "updating multiple choice questions requires options "
+ "and question must already have options_history"
+ )
+
+ grace_period_end = data.get("grace_period_end")
+ self.validate_new_options(options, options_history, grace_period_end)
+
+ return data
+
+
+def get_all_options_from_history(
+ options_history: OptionsHistoryType | None,
+) -> list[str]:
+ """Returns the list of all options ever available. The last value in the list
+ is always the "catch-all" option.
+
+ example:
+ options_history = [
+ ("2020-01-01", ["a", "b", "other"]),
+ ("2020-01-02", ["a", "b", "c", "other"]),
+ ("2020-01-03", ["a", "c", "other"]),
+ ]
+ return ["a", "b", "c", "other"]
+ """
+ if not options_history:
+ raise ValueError("Cannot make master list from empty history")
+ designated_other_label = options_history[0][1][-1]
+ all_labels: list[str] = []
+ for _, options in options_history:
+ for label in options[:-1]:
+ if label not in all_labels:
+ all_labels.append(label)
+ return all_labels + [designated_other_label]
+
+
+def multiple_choice_rename_option(
+ question: Question,
+ old_option: str,
+ new_option: str,
+) -> Question:
+ """
+ Modifies question in place and returns it.
+ Renames multiple choice option in question options and options history.
+ """
+ if question.type != Question.QuestionType.MULTIPLE_CHOICE:
+ raise ValueError("Question must be multiple choice")
+ if not question.options or old_option not in question.options:
+ raise ValueError("Old option not found")
+ if new_option in question.options:
+ raise ValueError("New option already exists")
+ if not question.options_history:
+ raise ValueError("Options history is empty")
+
+ question.options = [
+ new_option if opt == old_option else opt for opt in question.options
+ ]
+ for i, (timestr, options) in enumerate(question.options_history):
+ question.options_history[i] = (
+ timestr,
+ [new_option if opt == old_option else opt for opt in options],
+ )
+
+ return question
+
+
+def multiple_choice_reorder_options(
+ question: Question,
+ new_options_order: list[str],
+) -> Question:
+ """
+ Modifies question in place and returns it.
+ Reorders multiple choice options in question options and options history.
+ Requires all options ever to be present in new_options_order.
+
+ For now, only supports reordering if options have never changed.
+ """
+ current_options = question.options
+ all_options_ever = get_all_options_from_history(question.options_history)
+ if question.type != Question.QuestionType.MULTIPLE_CHOICE:
+ raise ValueError("Question must be multiple choice")
+ if not current_options:
+ raise ValueError("Question has no options")
+ if set(new_options_order) != set(all_options_ever):
+ raise ValueError("New order does not match existing options")
+ if not question.options_history:
+ raise ValueError("Options history is empty")
+
+ if len(question.options_history) != 1:
+ # TODO: support reordering options with history changes
+ raise ValueError("Cannot reorder options that have changed")
+
+ # update options history (it is only one entry long)
+ question.options_history[0] = (question.options_history[0][0], new_options_order)
+ question.options = new_options_order
+ question.save()
+
+ # update user forecasts
+ # example forecast remap: all_options_ever = [a,b,c], new_options_order = [c,a,b]
+ # remap = [2,0,1]
+ # if a forecast is [0.2,0.3,0.5], then the new one is [0.5,0.2,0.3]
+ remap = [all_options_ever.index(option) for option in new_options_order]
+ for forecast in question.user_forecasts.all():
+ forecast.probability_yes_per_category = [
+ forecast.probability_yes_per_category[i] for i in remap
+ ]
+ forecast.save()
+
+ # trigger recalculation of aggregates
+ from questions.services.forecasts import build_question_forecasts
+
+ build_question_forecasts(question)
+
+ return question
+
+
+def multiple_choice_change_grace_period_end(*args, **kwargs):
+ raise NotImplementedError("multiple_choice_change_grace_period_end")
+
+
+def multiple_choice_delete_options(
+ question: Question,
+ options_to_delete: list[str],
+ comment_author: User,
+ timestep: datetime | None = None,
+ comment_text: str | None = None,
+) -> Question:
+ """
+ Modifies question in place and returns it.
+ Deletes multiple choice options in question options.
+ Adds a new entry to options_history.
+ Slices all user forecasts at timestep.
+ Triggers recalculation of aggregates.
+ """
+ if not options_to_delete:
+ return question
+ timestep = timestep or timezone.now()
+ if question.type != Question.QuestionType.MULTIPLE_CHOICE:
+ raise ValueError("Question must be multiple choice")
+ if not question.options or not all(
+ [opt in question.options for opt in options_to_delete]
+ ):
+ raise ValueError("Option to delete not found")
+ if not question.options_history:
+ raise ValueError("Options history is empty")
+
+ if (
+ datetime.fromisoformat(question.options_history[-1][0]).replace(
+ tzinfo=dt_timezone.utc
+ )
+ > timestep
+ ):
+ raise ValueError("timestep is before the last options history entry")
+
+ # update question
+ new_options = [opt for opt in question.options if opt not in options_to_delete]
+ all_options = get_all_options_from_history(question.options_history)
+
+ question.options = new_options
+ question.options_history.append((timestep.isoformat(), new_options))
+ question.save()
+
+ # update user forecasts
+ user_forecasts = question.user_forecasts.filter(
+ Q(end_time__isnull=True) | Q(end_time__gt=timestep),
+ start_time__lt=timestep,
+ )
+ forecasts_to_create: list[Forecast] = []
+ for forecast in user_forecasts:
+ # get new PMF
+ previous_pmf = forecast.probability_yes_per_category
+ if len(previous_pmf) != len(all_options):
+ raise ValueError(
+ f"Forecast {forecast.id} PMF length does not match "
+ f"all options {all_options}"
+ )
+ new_pmf: list[float | None] = [None] * len(all_options)
+ for value, label in zip(previous_pmf, all_options):
+ if value is None:
+ continue
+ if label in new_options:
+ new_pmf[all_options.index(label)] = (
+ new_pmf[all_options.index(label)] or 0.0
+ ) + value
+ else:
+ new_pmf[-1] = (
+ new_pmf[-1] or 0.0
+ ) + value # add to catch-all last option
+
+ # slice forecast
+ if forecast.start_time >= timestep:
+ # forecast is completely after timestep, just update PMF
+ forecast.probability_yes_per_category = new_pmf
+ continue
+ forecasts_to_create.append(
+ Forecast(
+ question=question,
+ author=forecast.author,
+ start_time=timestep,
+ end_time=forecast.end_time,
+ probability_yes_per_category=new_pmf,
+ post=forecast.post,
+ source=Forecast.SourceChoices.AUTOMATIC, # mark as automatic forecast
+ )
+ )
+ forecast.end_time = timestep
+
+ with transaction.atomic():
+ Forecast.objects.bulk_update(
+ user_forecasts, ["end_time", "probability_yes_per_category"]
+ )
+ Forecast.objects.bulk_create(forecasts_to_create)
+
+ # trigger recalculation of aggregates
+ from questions.services.forecasts import build_question_forecasts
+
+ build_question_forecasts(question)
+
+ # notify users that about the change
+ from questions.tasks import multiple_choice_delete_option_notificiations
+
+ multiple_choice_delete_option_notificiations(
+ question_id=question.id,
+ timestep=timestep,
+ comment_author_id=comment_author.id,
+ comment_text=comment_text,
+ )
+
+ return question
+
+
+def multiple_choice_add_options(
+ question: Question,
+ options_to_add: list[str],
+ grace_period_end: datetime,
+ comment_author: User,
+ timestep: datetime | None = None,
+ comment_text: str | None = None,
+) -> Question:
+ """
+ Modifies question in place and returns it.
+ Adds multiple choice options in question options.
+ Adds a new entry to options_history.
+ Terminates all user forecasts at grace_period_end.
+ Triggers recalculation of aggregates.
+ """
+ if not options_to_add:
+ return question
+ timestep = timestep or timezone.now()
+ if question.type != Question.QuestionType.MULTIPLE_CHOICE:
+ raise ValueError("Question must be multiple choice")
+ if not question.options or any([opt in question.options for opt in options_to_add]):
+ raise ValueError("Option to add already found")
+ if not question.options_history:
+ raise ValueError("Options history is empty")
+
+ if timestep > grace_period_end:
+ raise ValueError("grace_period_end must end after timestep")
+ if (
+ datetime.fromisoformat(question.options_history[-1][0]).replace(
+ tzinfo=dt_timezone.utc
+ )
+ > timestep
+ ):
+ raise ValueError("timestep is before the last options history entry")
+
+ # update question
+ new_options = question.options[:-1] + options_to_add + question.options[-1:]
+ question.options = new_options
+ question.options_history.append((grace_period_end.isoformat(), new_options))
+ question.save()
+
+ # update user forecasts
+ user_forecasts = question.user_forecasts.all()
+ for forecast in user_forecasts:
+ pmf = forecast.probability_yes_per_category
+ forecast.probability_yes_per_category = (
+ pmf[:-1] + [None] * len(options_to_add) + [pmf[-1]]
+ )
+ if forecast.start_time < grace_period_end and (
+ forecast.end_time is None or forecast.end_time > grace_period_end
+ ):
+ forecast.end_time = grace_period_end
+ with transaction.atomic():
+ Forecast.objects.bulk_update(
+ user_forecasts, ["probability_yes_per_category", "end_time"]
+ )
+
+ # trigger recalculation of aggregates
+ from questions.services.forecasts import build_question_forecasts
+
+ build_question_forecasts(question)
+
+ # notify users that about the change
+ from questions.tasks import multiple_choice_add_option_notificiations
+
+ multiple_choice_add_option_notificiations(
+ question_id=question.id,
+ grace_period_end=grace_period_end,
+ timestep=timestep,
+ comment_author_id=comment_author.id,
+ comment_text=comment_text,
+ )
+
+ return question
diff --git a/questions/tasks.py b/questions/tasks.py
index 988b7e0fbe..74caf247f1 100644
--- a/questions/tasks.py
+++ b/questions/tasks.py
@@ -1,10 +1,12 @@
import logging
-from datetime import timedelta
+from datetime import datetime, timedelta
import dramatiq
+from django.conf import settings
from django.db.models import Q
from django.utils import timezone
+from comments.services.common import create_comment
from notifications.constants import MailingTags
from notifications.services import (
NotificationPredictedQuestionResolved,
@@ -15,17 +17,18 @@
)
from posts.models import Post
from posts.services.subscriptions import notify_post_status_change
+from questions.models import Forecast, Question, UserForecastNotification
+from questions.services.common import get_outbound_question_links
+from questions.services.forecasts import (
+ build_question_forecasts,
+ get_forecasts_per_user,
+)
from scoring.constants import ScoreTypes
from scoring.utils import score_question
from users.models import User
from utils.dramatiq import concurrency_retries, task_concurrent_limit
+from utils.email import send_email_with_template
from utils.frontend import build_frontend_account_settings_url, build_post_url
-from .models import Question, UserForecastNotification
-from .services.common import get_outbound_question_links
-from .services.forecasts import (
- build_question_forecasts,
- get_forecasts_per_user,
-)
@dramatiq.actor(max_backoff=10_000, retry_when=concurrency_retries(max_retries=20))
@@ -255,3 +258,160 @@ def format_time_remaining(time_remaining: timedelta):
return f"{minutes} minute{'s' if minutes != 1 else ''}"
else:
return f"{total_seconds} second{'s' if total_seconds != 1 else ''}"
+
+
+@dramatiq.actor
+def multiple_choice_delete_option_notificiations(
+ question_id: int,
+ timestep: datetime,
+ comment_author_id: int,
+ comment_text: str | None = None,
+):
+ question = Question.objects.get(id=question_id)
+ post = question.get_post()
+ options_history = question.options_history
+ removed_options = list(set(options_history[-2][1]) - set(options_history[-1][1]))
+
+ # send out a comment
+ comment_author = User.objects.get(id=comment_author_id)
+ default_text = (
+ "Options {removed_options} were removed at {timestep}. "
+ "Forecasts were adjusted to keep remaining probability on the catch-all."
+ )
+ template = comment_text or default_text
+ try:
+ text = template.format(removed_options=removed_options, timestep=timestep)
+ except Exception:
+ text = f"{template} (removed options: {removed_options}, at {timestep})"
+
+ create_comment(comment_author, post, text=text)
+
+ forecasters = (
+ User.objects.filter(
+ forecast__in=question.user_forecasts.filter(
+ Q(end_time__isnull=True) | Q(end_time__gt=timestep)
+ )
+ )
+ .exclude(
+ unsubscribed_mailing_tags__contains=[
+ MailingTags.BEFORE_PREDICTION_AUTO_WITHDRAWAL # seems most reasonable
+ ]
+ )
+ .exclude(email__isnull=True)
+ .exclude(email="")
+ .distinct("id")
+ .order_by("id")
+ )
+ # send out an immediate email
+ for forecaster in forecasters:
+ send_email_with_template(
+ to=forecaster.email,
+ subject="Multiple choice option removed",
+ template_name="emails/multiple_choice_option_deletion.html",
+ context={
+ "recipient": forecaster,
+ "email_subject_display": "Multiple choice option removed",
+ "similar_posts": [],
+ "params": {
+ "post": NotificationPostParams.from_post(post),
+ "removed_options": removed_options,
+ "timestep": timestep,
+ },
+ },
+ use_async=False,
+ from_email=settings.EMAIL_NOTIFICATIONS_USER,
+ )
+
+
+@dramatiq.actor
+def multiple_choice_add_option_notificiations(
+ question_id: int,
+ grace_period_end: datetime,
+ timestep: datetime,
+ comment_author_id: int,
+ comment_text: str | None = None,
+):
+ question = Question.objects.get(id=question_id)
+ post = question.get_post()
+ options_history = question.options_history
+ added_options = list(set(options_history[-1][1]) - set(options_history[-2][1]))
+
+ # send out a comment
+ comment_author = User.objects.get(id=comment_author_id)
+ default_text = (
+ "Options {added_options} were added at {timestep}. "
+ "Please update forecasts before {grace_period_end}, when existing "
+ "forecasts will auto-withdraw."
+ )
+ template = comment_text or default_text
+ try:
+ text = template.format(
+ added_options=added_options,
+ timestep=timestep,
+ grace_period_end=grace_period_end,
+ )
+ except Exception:
+ text = (
+ f"{template} (added options: {added_options}, at {timestep}, "
+ f"grace ends: {grace_period_end})"
+ )
+
+ create_comment(comment_author, post, text=text)
+
+ forecasters = (
+ User.objects.filter(
+ forecast__in=question.user_forecasts.filter(
+ end_time=grace_period_end
+ ) # all effected forecasts have their end_time set to grace_period_end
+ )
+ .exclude(
+ unsubscribed_mailing_tags__contains=[
+ MailingTags.BEFORE_PREDICTION_AUTO_WITHDRAWAL # seems most reasonable
+ ]
+ )
+ .exclude(email__isnull=True)
+ .exclude(email="")
+ .distinct("id")
+ .order_by("id")
+ )
+ # send out an immediate email
+ for forecaster in forecasters:
+ send_email_with_template(
+ to=forecaster.email,
+ subject="Multiple choice options added",
+ template_name="emails/multiple_choice_option_addition.html",
+ context={
+ "recipient": forecaster,
+ "email_subject_display": "Multiple choice options added",
+ "similar_posts": [],
+ "params": {
+ "post": NotificationPostParams.from_post(post),
+ "added_options": added_options,
+ "grace_period_end": grace_period_end,
+ "timestep": timestep,
+ },
+ },
+ use_async=False,
+ from_email=settings.EMAIL_NOTIFICATIONS_USER,
+ )
+
+ # schedule a followup email for 1 day before grace period
+ # (if grace period is more than 1 day away)
+ if grace_period_end - timedelta(days=1) > timestep:
+ for forecaster in forecasters:
+ UserForecastNotification.objects.filter(
+ user=forecaster, question=question
+ ).delete() # is this necessary?
+ UserForecastNotification.objects.update_or_create(
+ user=forecaster,
+ question=question,
+ defaults={
+ "trigger_time": grace_period_end - timedelta(days=1),
+ "email_sent": False,
+ "forecast": Forecast.objects.filter(
+ question=question, author=forecaster
+ )
+ .order_by("-start_time")
+ .first(),
+ },
+ )
diff --git a/questions/types.py b/questions/types.py
index 9556806b41..f87735e520 100644
--- a/questions/types.py
+++ b/questions/types.py
@@ -3,6 +3,8 @@
from django.db import models
from django.db.models import TextChoices
+OptionsHistoryType = list[tuple[str, list[str]]]
+
class Direction(TextChoices):
UNCHANGED = "unchanged"
diff --git a/scoring/score_math.py b/scoring/score_math.py
index fada04f0d1..546b19d310 100644
--- a/scoring/score_math.py
+++ b/scoring/score_math.py
@@ -20,7 +20,7 @@
@dataclass
class AggregationEntry:
- pmf: np.ndarray | list[float]
+ pmf: np.ndarray | list[float | None]
num_forecasters: int
timestamp: float
@@ -36,7 +36,7 @@ def get_geometric_means(
timesteps.add(forecast.end_time.timestamp())
for timestep in sorted(timesteps):
prediction_values = [
- f.get_pmf()
+ f.get_pmf(replace_none=True)
for f in forecasts
if f.start_time.timestamp() <= timestep
and (f.end_time is None or f.end_time.timestamp() > timestep)
@@ -84,9 +84,12 @@ def evaluate_forecasts_baseline_accuracy(
forecast_coverage = forecast_duration / total_duration
pmf = forecast.get_pmf()
if question_type in ["binary", "multiple_choice"]:
- forecast_score = (
- 100 * np.log(pmf[resolution_bucket] * len(pmf)) / np.log(len(pmf))
- )
+ # forecasts always have `None` assigned to MC options that aren't
+ # available at the time. Detecting these allows us to avoid trying to
+ # follow the question's options_history.
+ options_at_time = len([p for p in pmf if p is not None])
+ p = pmf[resolution_bucket] or pmf[-1] # if None, read from Other
+ forecast_score = 100 * np.log(p * options_at_time) / np.log(options_at_time)
else:
if resolution_bucket in [0, len(pmf) - 1]:
baseline = 0.05
@@ -116,8 +119,13 @@ def evaluate_forecasts_baseline_spot_forecast(
if start <= spot_forecast_timestamp < end:
pmf = forecast.get_pmf()
if question_type in ["binary", "multiple_choice"]:
+ # forecasts always have `None` assigned to MC options that aren't
+ # available at the time. Detecting these allows us to avoid trying to
+ # follow the question's options_history.
+ options_at_time = len([p for p in pmf if p is not None])
+ p = pmf[resolution_bucket] or pmf[-1] # if None, read from Other
forecast_score = (
- 100 * np.log(pmf[resolution_bucket] * len(pmf)) / np.log(len(pmf))
+ 100 * np.log(p * options_at_time) / np.log(options_at_time)
)
else:
if resolution_bucket in [0, len(pmf) - 1]:
@@ -159,17 +167,21 @@ def evaluate_forecasts_peer_accuracy(
continue
pmf = forecast.get_pmf()
+ p = pmf[resolution_bucket] or pmf[-1] # if None, read from Other
interval_scores: list[float | None] = []
for gm in geometric_mean_forecasts:
if forecast_start <= gm.timestamp < forecast_end:
- score = (
+ gmp = (
+ gm.pmf[resolution_bucket] or gm.pmf[-1]
+ ) # if None, read from Other
+ interval_score = (
100
* (gm.num_forecasters / (gm.num_forecasters - 1))
- * np.log(pmf[resolution_bucket] / gm.pmf[resolution_bucket])
+ * np.log(p / gmp)
)
if question_type in QUESTION_CONTINUOUS_TYPES:
- score /= 2
- interval_scores.append(score)
+ interval_score /= 2
+ interval_scores.append(interval_score)
else:
interval_scores.append(None)
@@ -218,10 +230,10 @@ def evaluate_forecasts_peer_spot_forecast(
)
if start <= spot_forecast_timestamp < end:
pmf = forecast.get_pmf()
+ p = pmf[resolution_bucket] or pmf[-1] # if None, read from Other
+ gmp = gm.pmf[resolution_bucket] or gm.pmf[-1] # if None, read from Other
forecast_score = (
- 100
- * (gm.num_forecasters / (gm.num_forecasters - 1))
- * np.log(pmf[resolution_bucket] / gm.pmf[resolution_bucket])
+ 100 * (gm.num_forecasters / (gm.num_forecasters - 1)) * np.log(p / gmp)
)
if question_type in QUESTION_CONTINUOUS_TYPES:
forecast_score /= 2
@@ -260,11 +272,15 @@ def evaluate_forecasts_legacy_relative(
continue
pmf = forecast.get_pmf()
+ p = pmf[resolution_bucket] or pmf[-1] # if None, read from Other
interval_scores: list[float | None] = []
for bf in baseline_forecasts:
if forecast_start <= bf.timestamp < forecast_end:
- score = np.log2(pmf[resolution_bucket] / bf.pmf[resolution_bucket])
- interval_scores.append(score)
+ bfp = (
+ bf.pmf[resolution_bucket] or bf.pmf[-1]
+ ) # if None, read from Other
+ interval_score = np.log2(p / bfp)
+ interval_scores.append(interval_score)
else:
interval_scores.append(None)
@@ -316,7 +332,7 @@ def evaluate_question(
if spot_forecast_time:
spot_forecast_timestamp = min(spot_forecast_time.timestamp(), actual_close_time)
- # We need all user forecasts to calculated GeoMean even
+ # We need all user forecasts to calculate GeoMean even
# if we're only scoring some or none of the users
user_forecasts = question.user_forecasts.all()
if only_include_user_ids:
diff --git a/templates/admin/questions/update_options.html b/templates/admin/questions/update_options.html
new file mode 100644
index 0000000000..7d7a426248
--- /dev/null
+++ b/templates/admin/questions/update_options.html
@@ -0,0 +1,182 @@
+{% extends "admin/base_site.html" %}
+{% load i18n admin_urls %}
+
+{% block extrahead %}
+{{ block.super }}
+{{ media }}
+{% endblock %}
+
+{% block breadcrumbs %}
+
+ This question is in an active grace period until {{ grace_period_end|date:"DATETIME_FORMAT" }}.
+ You can rename options, change the grace period end, but adding or deleting options is temporarily disabled.
+
+