feat: add script to export TA data (#548)

* feat: add script to export TA data

* get rid of unnecessary comments

* address cursor comments

* get rid of enumerate since it's redundant

* add grouping logic

Author: adrianviquez

apps/codecov-api/api/sentry/tests/test_views.py

@@ -8,13 +8,19 @@
 from rest_framework.test import APIClient
 
 from codecov_auth.models import Account
-from shared.django_apps.codecov_auth.models import GithubAppInstallation, Owner
+from shared.django_apps.codecov_auth.models import (
+    GithubAppInstallation,
+    Owner,
+    Service,
+)
 from shared.django_apps.codecov_auth.tests.factories import (
     AccountFactory,
     OwnerFactory,
     PlanFactory,
     TierFactory,
 )
+from shared.django_apps.core.tests.factories import RepositoryFactory
+from shared.django_apps.ta_timeseries.tests.factories import TestrunFactory
 from shared.plan.constants import PlanName, TierName
 
 
@@ -823,3 +829,229 @@ def test_account_unlink_authentication_failure(self):
         )
 
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+
+class TestAnalyticsEuViewTests(TestCase):
+    databases = ["default", "ta_timeseries"]
+
+    def setUp(self):
+        self.client = APIClient()
+        self.url = reverse("test-analytics-eu")
+
+    def _make_authenticated_request(self, data, jwt_payload=None):
+        """Helper method to make an authenticated request with JWT payload"""
+        with patch(
+            "codecov_auth.permissions.get_sentry_jwt_payload"
+        ) as mock_get_payload:
+            mock_get_payload.return_value = jwt_payload or {
+                "g_p": "github",
+                "g_o": "test-org",
+            }
+            return self.client.post(
+                self.url, data=json.dumps(data), content_type="application/json"
+            )
+
+    def test_test_analytics_eu_empty_integration_names(self):
+        """Test that empty integration_names list fails validation"""
+        data = {"integration_names": []}
+
+        response = self._make_authenticated_request(data=data)
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertIn("integration_names", response.data)
+
+    def test_test_analytics_eu_missing_integration_names(self):
+        """Test that missing integration_names fails validation"""
+        data = {}
+
+        response = self._make_authenticated_request(data=data)
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertIn("integration_names", response.data)
+
+    @patch("api.sentry.views.log")
+    def test_test_analytics_eu_owner_not_found(self, mock_log):
+        """Test that non-existent owner is skipped with warning log"""
+        data = {"integration_names": ["non-existent-org"]}
+
+        response = self._make_authenticated_request(data=data)
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["test_runs_per_integration"], {})
+
+        mock_log.warning.assert_called_once()
+        warning_call = mock_log.warning.call_args[0][0]
+        self.assertIn("non-existent-org", warning_call)
+        self.assertIn("not found", warning_call)
+
+    def test_test_analytics_eu_owner_without_repositories(self):
+        """Test that owner without repositories returns empty dict"""
+        OwnerFactory(name="org-no-repos", service=Service.GITHUB)
+
+        data = {"integration_names": ["org-no-repos"]}
+
+        response = self._make_authenticated_request(data=data)
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            response.data["test_runs_per_integration"], {"org-no-repos": {}}
+        )
+
+    @patch("api.sentry.views.log")
+    def test_test_analytics_eu_mixed_owners_found_and_not_found(self, mock_log):
+        """Test mix of existing and non-existing owners"""
+        owner = OwnerFactory(name="org-exists", service=Service.GITHUB)
+        repo = RepositoryFactory(
+            author=owner, name="test-repo", test_analytics_enabled=True
+        )
+        TestrunFactory(
+            repo_id=repo.repoid,
+            commit_sha="abc123",
+            outcome="pass",
+            name="test_example",
+        )
+
+        data = {"integration_names": ["org-exists", "org-not-exists"]}
+
+        response = self._make_authenticated_request(data=data)
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertIn("org-exists", response.data["test_runs_per_integration"])
+        self.assertNotIn("org-not-exists", response.data["test_runs_per_integration"])
+
+        # Verify warning log was called for non-existent owner
+        mock_log.warning.assert_called_once()
+        warning_call = mock_log.warning.call_args[0][0]
+        self.assertIn("org-not-exists", warning_call)
+
+    def test_test_analytics_eu_filters_by_test_analytics_enabled(self):
+        """Test that only repositories with test_analytics_enabled=True are included"""
+        owner = OwnerFactory(name="org-with-repos", service=Service.GITHUB)
+
+        repo_enabled = RepositoryFactory(
+            author=owner,
+            name="repo-enabled",
+            test_analytics_enabled=True,
+        )
+        TestrunFactory(
+            repo_id=repo_enabled.repoid,
+            commit_sha="abc123",
+            outcome="pass",
+            name="test_enabled",
+        )
+
+        repo_disabled = RepositoryFactory(
+            author=owner,
+            name="repo-disabled",
+            test_analytics_enabled=False,
+        )
+
+        data = {"integration_names": ["org-with-repos"]}
+
+        response = self._make_authenticated_request(data=data)
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        test_runs_data = response.data["test_runs_per_integration"]["org-with-repos"]
+
+        # Only repo-enabled should be in the response
+        self.assertIn("repo-enabled", test_runs_data)
+        self.assertNotIn("repo-disabled", test_runs_data)
+
+        test_runs_list = test_runs_data["repo-enabled"]
+        self.assertEqual(len(test_runs_list), 1)
+        self.assertEqual(test_runs_list[0]["name"], "test_enabled")
+
+    def test_test_analytics_eu_multiple_owners_with_multiple_repos_and_testruns(self):
+        """Test complex scenario with 2 owners, different repositories and test runs"""
+        owner1 = OwnerFactory(name="org-one", service=Service.GITHUB)
+        repo1 = RepositoryFactory(
+            author=owner1,
+            name="repo-one",
+            test_analytics_enabled=True,
+        )
+        TestrunFactory(
+            repo_id=repo1.repoid,
+            commit_sha="commit1",
+            outcome="pass",
+            name="test_one_first",
+            classname="TestClass1",
+        )
+        TestrunFactory(
+            repo_id=repo1.repoid,
+            commit_sha="commit1",
+            outcome="failure",
+            name="test_one_second",
+            classname="TestClass2",
+        )
+
+        owner2 = OwnerFactory(name="org-two", service=Service.GITHUB)
+        repo2_1 = RepositoryFactory(
+            author=owner2,
+            name="repo-two-first",
+            test_analytics_enabled=True,
+        )
+        TestrunFactory(
+            repo_id=repo2_1.repoid,
+            commit_sha="commit2",
+            outcome="pass",
+            name="test_two_first",
+            classname="TestClassA",
+        )
+
+        repo2_2 = RepositoryFactory(
+            author=owner2,
+            name="repo-two-second",
+            test_analytics_enabled=True,
+        )
+        TestrunFactory(
+            repo_id=repo2_2.repoid,
+            commit_sha="commit3",
+            outcome="skip",
+            name="test_two_second",
+            classname="TestClassB",
+        )
+
+        data = {"integration_names": ["org-one", "org-two"]}
+
+        response = self._make_authenticated_request(data=data)
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        test_runs_per_integration = response.data["test_runs_per_integration"]
+
+        # Verify org-one data
+        self.assertIn("org-one", test_runs_per_integration)
+        org_one_data = test_runs_per_integration["org-one"]
+        self.assertIn("repo-one", org_one_data)
+        self.assertEqual(len(org_one_data), 1)
+
+        repo_one_testruns = org_one_data["repo-one"]
+        self.assertEqual(len(repo_one_testruns), 2)
+        testrun_names = [tr["name"] for tr in repo_one_testruns]
+        self.assertIn("test_one_first", testrun_names)
+        self.assertIn("test_one_second", testrun_names)
+
+        self.assertIn("org-two", test_runs_per_integration)
+        org_two_data = test_runs_per_integration["org-two"]
+        self.assertIn("repo-two-first", org_two_data)
+        self.assertIn("repo-two-second", org_two_data)
+        self.assertEqual(len(org_two_data), 2)
+
+        repo_two_first_testruns = org_two_data["repo-two-first"]
+        self.assertEqual(len(repo_two_first_testruns), 1)
+        self.assertEqual(repo_two_first_testruns[0]["name"], "test_two_first")
+        self.assertEqual(repo_two_first_testruns[0]["outcome"], "pass")
+
+        repo_two_second_testruns = org_two_data["repo-two-second"]
+        self.assertEqual(len(repo_two_second_testruns), 1)
+        self.assertEqual(repo_two_second_testruns[0]["name"], "test_two_second")
+        self.assertEqual(repo_two_second_testruns[0]["outcome"], "skip")
+
+    def test_test_analytics_eu_authentication_failure(self):
+        """Test that the endpoint requires authentication"""
+        data = {"integration_names": ["test-org"]}
+
+        response = self.client.post(
+            self.url, data=json.dumps(data), content_type="application/json"
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)

apps/codecov-api/api/sentry/urls.py

@@ -1,8 +1,9 @@
 from django.urls import path
 
-from .views import account_link, account_unlink
+from .views import account_link, account_unlink, test_analytics_eu
 
 urlpatterns = [
     path("internal/account/link/", account_link, name="account-link"),
     path("internal/account/unlink/", account_unlink, name="account-unlink"),
+    path("internal/test-analytics/eu/", test_analytics_eu, name="test-analytics-eu"),
 ]

apps/codecov-api/api/sentry/views.py

@@ -1,7 +1,9 @@
 import logging
+from itertools import groupby
 
 import sentry_sdk
 from django.conf import settings
+from django.views.decorators.gzip import gzip_page
 from rest_framework import serializers, status
 from rest_framework.decorators import (
     api_view,
@@ -10,6 +12,7 @@
 )
 from rest_framework.response import Response
 
+from api.public.v2.test_results.serializers import TestrunSerializer
 from codecov_auth.models import Account
 from codecov_auth.permissions import JWTAuthenticationPermission
 from shared.django_apps.codecov_auth.models import (
@@ -18,6 +21,8 @@
     Plan,
     Service,
 )
+from shared.django_apps.core.models import Repository
+from shared.django_apps.ta_timeseries.models import Testrun
 from shared.plan.constants import PlanName
 
 log = logging.getLogger(__name__)
@@ -243,3 +248,69 @@ def account_unlink(request, *args, **kwargs):
             "total_requested": total_requested,
         }
     )
+
+
+class SentryTestAnalyticsEuSerializer(serializers.Serializer):
+    """Serializer for test analytics EU endpoint"""
+
+    integration_names = serializers.ListField(
+        child=serializers.CharField(),
+        help_text="The Sentry integration names",
+        min_length=1,
+    )
+
+
+@gzip_page
+@api_view(["POST"])
+@authentication_classes([])
+@permission_classes([JWTAuthenticationPermission])
+def test_analytics_eu(request, *args, **kwargs):
+    serializer = SentryTestAnalyticsEuSerializer(data=request.data)
+    serializer.is_valid(raise_exception=True)
+
+    integration_names = serializer.validated_data["integration_names"]
+
+    # For every integration name, determine if an Owner record exist by filtering by name and service=github
+    test_runs_per_integration = {}
+    for name in integration_names:
+        try:
+            owner = Owner.objects.get(name=name, service=Service.GITHUB)
+        except Owner.DoesNotExist:
+            log.warning(
+                f"Owner with name {name} and service {Service.GITHUB} not found"
+            )
+            continue
+
+        # Only fetch name and repoid fields
+        repo_id_to_name = dict(
+            Repository.objects.filter(
+                author=owner, test_analytics_enabled=True
+            ).values_list("repoid", "name")
+        )
+
+        if not repo_id_to_name:
+            test_runs_per_integration[name] = {}
+            continue
+
+        # Fetch all test runs for all repositories in a single query
+        test_runs = Testrun.objects.filter(repo_id__in=repo_id_to_name.keys()).order_by(
+            "repo_id", "-timestamp"
+        )
+
+        # Group by repo_id (data is already ordered by repo_id) and serialize each group
+        test_runs_per_repository = {}
+        for repo_id, group in groupby(test_runs, key=lambda tr: tr.repo_id):
+            repo_name = repo_id_to_name[repo_id]  # Safe: we only fetch these repo_ids
+            test_runs_list = list(group)
+            test_runs_per_repository[repo_name] = TestrunSerializer(
+                test_runs_list, many=True
+            ).data
+
+        # Store each test_runs_per_repository in a dictionary
+        test_runs_per_integration[name] = test_runs_per_repository
+
+    return Response(
+        {
+            "test_runs_per_integration": test_runs_per_integration,
+        }
+    )