[Entity Analytics] Remove legacy scripted metric calculation for risk scoring (#244470)

## Summary

This PR removes the legacy scripted metric calculation code from Entity
Analytics Risk Scoring, leaving only the ESQL-based implementation as
the default.

Closes elastic/security-team#14204

## Changes

### Core Implementation
- **Moved shared utilities**: Extracted `processScores`,
`filterFromRange`, and `getGlobalWeightForIdentifierType` functions from
`calculate_risk_scores.ts` to `helpers.ts` for reuse
- **Deleted legacy implementation**: Removed `calculate_risk_scores.ts`
and all related test/mock files
- **Deleted painless scripts**: Removed entire `painless/` folder
containing 6 files (index.ts, index.test.ts, and 4 .painless script
files)
- **Simplified service logic**: Updated `risk_score_service.ts` and
`calculate_and_persist_risk_scores.ts` to always use
`calculateScoresWithESQL` without conditional logic

### Feature Flags & Settings
- **Removed experimental flag**: Deleted `disableESQLRiskScoring` from
`experimental_features.ts`
- **Removed UI setting**: Deleted `ENABLE_ESQL_RISK_SCORING` constant
and its UI setting configuration
- **Updated imports**: Cleaned up all references to removed constants

### Tests
- **Unit tests**: Updated `calculate_and_persist_risk_scores.test.ts` to
remove scripted metric test blocks
- **Integration tests**: Removed "Scripted metric" describe blocks from
4 test files:
  - `task_execution.ts`
  - `task_execution_nondefault_spaces.ts`
  - `risk_score_preview.ts`
  - `risk_score_entity_calculation.ts`

### Documentation & Schema
- **Documentation**: Removed `securitySolution:enableEsqlRiskScoring`
entry from advanced settings documentation
- **Telemetry schema**: Removed references from:
  - `src/platform/plugins/shared/telemetry/schema/oss_platform.json`
-
`src/platform/plugins/private/kibana_usage_collection/server/collectors/management/types.ts`
-
`src/platform/plugins/private/kibana_usage_collection/server/collectors/management/schema.ts`

## Files Changed

### Modified (14 files)
-
`x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/helpers.ts`
- Added shared utility functions
-
`x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/reset_to_zero.ts`
- Updated import
-
`x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.ts`
- Updated import
-
`x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/apply_score_modifiers.ts`
- Updated import
-
`x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts`
- Removed flag
-
`x-pack/solutions/security/plugins/security_solution/common/constants.ts`
- Removed constant
-
`x-pack/solutions/security/plugins/security_solution/server/ui_settings.ts`
- Removed setting
-
`x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_service.ts`
- Simplified logic
-
`x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.ts`
- Simplified logic
-
`x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.test.ts`
- Removed tests
- 4 integration test files - Removed scripted metric test blocks

### Deleted (9 files)
- `calculate_risk_scores.ts`
- `calculate_risk_scores.test.ts`
- `calculate_risk_scores.mock.ts`
- `painless/index.ts`
- `painless/index.test.ts`
- `painless/risk_scoring_init.painless`
- `painless/risk_scoring_map.painless`
- `painless/risk_scoring_combine.painless`
- `painless/risk_scoring_reduce.painless`

Author: abhishekbhatia1710

docs/reference/advanced-settings.md

@@ -590,7 +590,7 @@ $$$security-solution-excluded-data-tiers-for-rule-execution$$$`securitySolution:
 $$$security-solution-enable-privileged-user-monitoring$$$`securitySolution:enablePrivilegedUserMonitoring` {applies_to}`stack: preview` {applies_to}`serverless: unavailable`
 :   Enables the privileged user monitoring dashboard and onboarding experience, which are in technical preview. `true` by default.
 
-$$$security-solution-enable-esql-risk-scoring$$$`securitySolution:enableEsqlRiskScoring` {applies_to}`stack: preview` {applies_to}`serverless: unavailable`
+$$$security-solution-enable-esql-risk-scoring$$$`securitySolution:enableEsqlRiskScoring` {applies_to}`stack: preview 9.0, removed 9.3` {applies_to}`serverless: unavailable`
 :   Enables risk scoring based on {{esql}} queries. Disabling this reverts to using scripted metrics. `true` by default.
 
 $$$security-solution-default-ai-connector$$$`securitySolution:defaultAIConnector` {applies_to}`stack: unavailable` {applies_to}`security: ga`

src/platform/plugins/private/kibana_usage_collection/server/collectors/management/schema.ts

@@ -122,10 +122,6 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'boolean',
     _meta: { description: 'Non-default value of setting.' },
   },
-  'securitySolution:enableEsqlRiskScoring': {
-    type: 'boolean',
-    _meta: { description: 'Non-default value of setting.' },
-  },
   'securitySolution:defaultAnomalyScore': {
     type: 'long',
     _meta: { description: 'Non-default value of setting.' },

src/platform/plugins/private/kibana_usage_collection/server/collectors/management/types.ts

@@ -75,7 +75,6 @@ export interface UsageStats {
   'securitySolution:enableGraphVisualization': boolean;
   'securitySolution:enableAssetInventory': boolean;
   'securitySolution:enablePrivilegedUserMonitoring': boolean;
-  'securitySolution:enableEsqlRiskScoring': boolean;
   'securitySolution:enableCloudConnector': boolean;
   'securitySolution:suppressionBehaviorOnAlertClosure': string;
   'securitySolution:defaultValueReportMinutes': string;

src/platform/plugins/shared/telemetry/schema/oss_platform.json

@@ -10229,12 +10229,6 @@
             "description": "Non-default value of setting."
           }
         },
-        "securitySolution:enableEsqlRiskScoring": {
-          "type": "boolean",
-          "_meta": {
-            "description": "Non-default value of setting."
-          }
-        },
         "securitySolution:defaultAnomalyScore": {
           "type": "long",
           "_meta": {

x-pack/solutions/security/plugins/security_solution/common/constants.ts

@@ -266,9 +266,6 @@ export const ENABLE_SIEM_READINESS_SETTING = 'securitySolution:enableSiemReadine
 export const ENABLE_PRIVILEGED_USER_MONITORING_SETTING =
   'securitySolution:enablePrivilegedUserMonitoring' as const;
 
-/** This Kibana Advanced Setting allows users to enable/disable ESQL-based risk scoring */
-export const ENABLE_ESQL_RISK_SCORING = 'securitySolution:enableEsqlRiskScoring' as const;
-
 /**
  * Id for the notifications alerting type
  * @deprecated Once we are confident all rules relying on side-car actions SO's have been migrated to SO references we should remove this function

x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts

@@ -60,11 +60,6 @@ export const allowedExperimentalValues = Object.freeze({
    */
   assistantModelEvaluation: false,
 
-  /**
-   * Disables ESQL-based risk scoring
-   */
-  disableESQLRiskScoring: false,
-
   /**
    * Enable resetting risk scores to zero for outdated entities
    */

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/apply_score_modifiers.ts

@@ -20,12 +20,11 @@ import type { PrivmonUserCrudService } from '../privilege_monitoring/users/privi
 
 import type { RiskScoreBucket } from '../types';
 import { RIEMANN_ZETA_VALUE } from './constants';
-import { getGlobalWeightForIdentifierType } from './calculate_risk_scores';
+import { getGlobalWeightForIdentifierType, max10DecimalPlaces } from './helpers';
 import type { AssetCriticalityRiskFields } from './modifiers/asset_criticality';
 import { applyCriticalityModifier } from './modifiers/asset_criticality';
 import type { PrivmonRiskFields } from './modifiers/privileged_users';
 import { applyPrivmonModifier } from './modifiers/privileged_users';
-import { max10DecimalPlaces } from './helpers';
 import type { ExperimentalFeatures } from '../../../../common';
 interface ModifiersUpdateParams {
   now: string;

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.test.ts

@@ -11,23 +11,19 @@ import { assetCriticalityServiceMock } from '../asset_criticality/asset_critical
 import { privmonUserCrudServiceMock } from '../privilege_monitoring/users/privileged_users_crud.mock';
 
 import { calculateAndPersistRiskScores } from './calculate_and_persist_risk_scores';
-import { calculateRiskScores } from './calculate_risk_scores';
-import { calculateRiskScoresMock } from './calculate_risk_scores.mock';
 import { calculateScoresWithESQL } from './calculate_esql_risk_scores';
 import { calculateScoresWithESQLMock } from './calculate_esql_risk_scores.mock';
 import { riskScoreDataClientMock } from './risk_score_data_client.mock';
 import type { RiskScoreDataClient } from './risk_score_data_client';
 import type { ExperimentalFeatures } from '../../../../common';
 import { EntityType } from '../../../../common/search_strategy';
 
-jest.mock('./calculate_risk_scores');
 jest.mock('./calculate_esql_risk_scores');
 
 const calculateAndPersistRecentHostRiskScores = (
   esClient: ElasticsearchClient,
   logger: Logger,
-  riskScoreDataClient: RiskScoreDataClient,
-  esql: boolean = false
+  riskScoreDataClient: RiskScoreDataClient
 ) => {
   return calculateAndPersistRiskScores({
     afterKeys: {},
@@ -42,9 +38,7 @@ const calculateAndPersistRecentHostRiskScores = (
     assetCriticalityService: assetCriticalityServiceMock.create(),
     privmonUserCrudService: privmonUserCrudServiceMock.create(),
     runtimeMappings: {},
-    experimentalFeatures: {
-      disableESQLRiskScoring: !esql,
-    } as ExperimentalFeatures,
+    experimentalFeatures: {} as ExperimentalFeatures,
   });
 };
 
@@ -53,100 +47,52 @@ describe('calculateAndPersistRiskScores', () => {
   let logger: Logger;
   let riskScoreDataClient: RiskScoreDataClient;
 
-  describe('scripted metric', () => {
-    const calculate = () =>
-      calculateAndPersistRecentHostRiskScores(esClient, logger, riskScoreDataClient, false);
+  const calculate = () =>
+    calculateAndPersistRecentHostRiskScores(esClient, logger, riskScoreDataClient);
 
+  beforeEach(() => {
+    esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser;
+    logger = loggingSystemMock.createLogger();
+    riskScoreDataClient = riskScoreDataClientMock.create();
+  });
+
+  describe('with no risk scores to persist', () => {
     beforeEach(() => {
-      esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser;
-      logger = loggingSystemMock.createLogger();
-      riskScoreDataClient = riskScoreDataClientMock.create();
+      (calculateScoresWithESQL as jest.Mock).mockResolvedValueOnce(
+        calculateScoresWithESQLMock.buildResponse({ scores: { host: [] } })
+      );
     });
 
-    describe('with no risk scores to persist', () => {
-      beforeEach(() => {
-        (calculateRiskScores as jest.Mock).mockResolvedValueOnce(
-          calculateRiskScoresMock.buildResponse({ scores: { host: [] } })
-        );
-      });
-
-      it('does not upgrade configurations', async () => {
-        await calculate();
+    it('does not upgrade configurations', async () => {
+      await calculate();
 
-        expect(riskScoreDataClient.upgradeIfNeeded).not.toHaveBeenCalled();
-      });
-
-      it('returns an appropriate response', async () => {
-        const results = await calculate();
-
-        const entities = {
-          host: [],
-          user: [],
-          service: [],
-          generic: [],
-        };
-        expect(results).toEqual({ after_keys: {}, errors: [], scores_written: 0, entities });
-      });
+      expect(riskScoreDataClient.upgradeIfNeeded).not.toHaveBeenCalled();
     });
-    describe('with risk scores to persist', () => {
-      beforeEach(() => {
-        (calculateRiskScores as jest.Mock).mockResolvedValueOnce(
-          calculateRiskScoresMock.buildResponseWithOneScore()
-        );
-      });
-      it('upgrades configurations when persisting risk scores', async () => {
-        await calculate();
 
-        expect(riskScoreDataClient.upgradeIfNeeded).toHaveBeenCalled();
-      });
+    it('returns an appropriate response', async () => {
+      const results = await calculate();
+
+      const entities = {
+        host: [],
+        user: [],
+        service: [],
+        generic: [],
+      };
+      expect(results).toEqual({ after_keys: {}, errors: [], scores_written: 0, entities });
     });
   });
 
-  describe('ESQL', () => {
-    const calculate = () =>
-      calculateAndPersistRecentHostRiskScores(esClient, logger, riskScoreDataClient, true);
+  describe('with risk scores to persist', () => {
     beforeEach(() => {
-      esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser;
-      logger = loggingSystemMock.createLogger();
-      riskScoreDataClient = riskScoreDataClientMock.create();
+      (calculateScoresWithESQL as jest.Mock).mockResolvedValueOnce(
+        calculateScoresWithESQLMock.buildResponseWithOneScore()
+      );
     });
 
-    describe('with no risk scores to persist', () => {
-      beforeEach(() => {
-        (calculateScoresWithESQL as jest.Mock).mockResolvedValueOnce(
-          calculateScoresWithESQLMock.buildResponse({ scores: { host: [] } })
-        );
-      });
-
-      it('does not upgrade configurations', async () => {
-        await calculate();
-
-        expect(riskScoreDataClient.upgradeIfNeeded).not.toHaveBeenCalled();
-      });
-
-      it('returns an appropriate response', async () => {
-        const results = await calculate();
-
-        const entities = {
-          host: [],
-          user: [],
-          service: [],
-          generic: [],
-        };
-        expect(results).toEqual({ after_keys: {}, errors: [], scores_written: 0, entities });
-      });
-    });
-    describe('with risk scores to persist', () => {
-      beforeEach(() => {
-        (calculateScoresWithESQL as jest.Mock).mockResolvedValueOnce(
-          calculateScoresWithESQLMock.buildResponseWithOneScore()
-        );
-      });
-      it('upgrades configurations when persisting risk scores', async () => {
-        await calculate();
+    it('upgrades configurations when persisting risk scores', async () => {
+      await calculate();
 
-        expect(riskScoreDataClient.upgradeIfNeeded).toHaveBeenCalled();
-      });
+      expect(riskScoreDataClient.upgradeIfNeeded).toHaveBeenCalled();
     });
   });
 });

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.ts

@@ -11,7 +11,6 @@ import type { ExperimentalFeatures } from '../../../../common';
 import type { EntityType } from '../../../../common/search_strategy';
 import type { RiskScoreDataClient } from './risk_score_data_client';
 import type { AssetCriticalityService } from '../asset_criticality/asset_criticality_service';
-import { calculateRiskScores } from './calculate_risk_scores';
 import type { CalculateAndPersistScoresParams } from '../types';
 import { calculateScoresWithESQL } from './calculate_esql_risk_scores';
 import type { RiskScoresCalculationResponse } from '../../../../common/api/entity_analytics';
@@ -38,10 +37,7 @@ export const calculateAndPersistRiskScores = async (
     namespace: spaceId,
   });
 
-  const calculate = params.experimentalFeatures.disableESQLRiskScoring
-    ? calculateRiskScores
-    : calculateScoresWithESQL;
-  const { after_keys: afterKeys, scores } = await calculate(rest);
+  const { after_keys: afterKeys, scores } = await calculateScoresWithESQL(rest);
 
   // Extract entity IDs from scores for reset-to-zero functionality
   const entities: Record<EntityType, string[]> = {

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.ts

@@ -32,7 +32,7 @@ import type { AssetCriticalityService } from '../asset_criticality/asset_critica
 import type { RiskScoresPreviewResponse } from '../../../../common/api/entity_analytics';
 import type { CalculateScoresParams, RiskScoreBucket, RiskScoreCompositeBuckets } from '../types';
 import { RIEMANN_ZETA_S_VALUE, RIEMANN_ZETA_VALUE } from './constants';
-import { filterFromRange } from './calculate_risk_scores';
+import { filterFromRange } from './helpers';
 import { applyScoreModifiers } from './apply_score_modifiers';
 import type { PrivmonUserCrudService } from '../privilege_monitoring/users/privileged_users_crud';