Skip to content

Commit ee1a104

Browse files
navnit-elasticnavnit-elastic
authored
crowdstrike: improvements and fixes to ingest pipelines for all data streams (#16730)
fixes all the issues outlined in issue #15973.
1 parent 7c2a0a8 commit ee1a104

33 files changed

+167
-129
lines changed

‎packages/crowdstrike/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "3.2.0"
3+
changes:
4+
- description: Improvements and fixes to ingest pipelines for all data streams.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/16730
27
- version: "3.1.0"
38
changes:
49
- description: Improve ingest pipeline maintainability.

‎packages/crowdstrike/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,7 @@
299299
"related": {
300300
"hash": [
301301
"b07f77fd3f9828b2c9d61f8a36609741",
302+
"caef4ae19056eeb122a0540508fa8984cea960173ada0dc648cb846d6ef5dd33",
302303
"cdf9cfebb400ce89d5b6032bfcdc693b",
303304
"b26a6791b72753d2317efd5e1363d93fdd33e611c8b9e08a3b24ea4d755b81fd",
304305
"8cc3fcdd7d52d2d5221303c213e044ae",
@@ -1035,6 +1036,7 @@
10351036
"related": {
10361037
"hash": [
10371038
"b07f77fd3f9828b2c9d61f8a36609741",
1039+
"caef4ae19056eeb122a0540508fa8984cea960173ada0dc648cb846d6ef5dd33",
10381040
"cdf9cfebb400ce89d5b6032bfcdc693b",
10391041
"b26a6791b72753d2317efd5e1363d93fdd33e611c8b9e08a3b24ea4d755b81fd",
10401042
"8cc3fcdd7d52d2d5221303c213e044ae",
@@ -1374,6 +1376,7 @@
13741376
"related": {
13751377
"hash": [
13761378
"b07f77fd3f9828b2c9d61f8a36609741",
1379+
"caef4ae19056eeb122a0540508fa8984cea960173ada0dc648cb846d6ef5dd33",
13771380
"cdf9cfebb400ce89d5b6032bfcdc693b",
13781381
"b26a6791b72753d2317efd5e1363d93fdd33e611c8b9e08a3b24ea4d755b81fd",
13791382
"8cc3fcdd7d52d2d5221303c213e044ae",

‎packages/crowdstrike/data_stream/alert/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ processors:
3333
ignore_missing: true
3434
description: The `message` field is no longer required if the document has an `event.original` field.
3535
if: ctx.event?.original != null
36+
- terminate:
37+
tag: data_collection_error
38+
if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null
39+
description: error message set and no data to process.
3640
- json:
3741
field: event.original
3842
tag: json_event_original
@@ -158,11 +162,6 @@ processors:
158162
tag: rename_child_process_ids
159163
target_field: crowdstrike.alert.child_process_ids
160164
ignore_missing: true
161-
- rename:
162-
field: json.aggregate_id
163-
tag: rename_aggregate_id
164-
target_field: crowdstrike.alert.aggregate_id
165-
ignore_missing: true
166165
- rename:
167166
field: json.cid
168167
tag: rename_cid
@@ -844,9 +843,9 @@ processors:
844843
ignore_missing: true
845844
- append:
846845
field: related.hash
847-
value: '{{{crowdstrike.alert.parent_details.sha256}}}'
846+
value: '{{{crowdstrike.alert.grandparent_details.sha256}}}'
848847
allow_duplicates: false
849-
if: ctx.crowdstrike?.alert?.parent_details?.sha256 != null
848+
if: ctx.crowdstrike?.alert?.grandparent_details?.sha256 != null
850849
- date:
851850
field: json.grandparent_details.timestamp
852851
tag: date_grandparent_details_timestamp
@@ -1147,7 +1146,7 @@ processors:
11471146
ignore_missing: true
11481147
- append:
11491148
field: crowdstrike.alert.ioc_values
1150-
tag: append_ioc_context_ioc_value_to_ioc_values
1149+
tag: append_ioc_value_to_ioc_values
11511150
value: '{{{crowdstrike.alert.ioc_value}}}'
11521151
if: ctx.crowdstrike?.alert?.ioc_value != null
11531152
allow_duplicates: false
@@ -2117,19 +2116,19 @@ processors:
21172116
- script:
21182117
lang: painless
21192118
description: Script to set event.severity.
2120-
tag: set_event_severity
2119+
tag: set_event_severity_from_severity
21212120
if: ctx.crowdstrike?.alert?.severity instanceof long && ctx.crowdstrike.alert.severity_name == null
21222121
source: |-
21232122
long severity = ctx.crowdstrike.alert.severity;
21242123
if (0 <= severity && severity < 20) {
21252124
ctx.crowdstrike.alert.severity_name = "info";
2126-
} if (20 <= severity && severity < 40) {
2125+
} else if (20 <= severity && severity < 40) {
21272126
ctx.crowdstrike.alert.severity_name = "low";
2128-
} if (40 <= severity && severity < 60) {
2127+
} else if (40 <= severity && severity < 60) {
21292128
ctx.crowdstrike.alert.severity_name = "medium";
2130-
} if (60 <= severity && severity < 80) {
2129+
} else if (60 <= severity && severity < 80) {
21312130
ctx.crowdstrike.alert.severity_name = "high";
2132-
} if (80 <= severity && severity <= 100) {
2131+
} else if (80 <= severity && severity <= 100) {
21332132
ctx.crowdstrike.alert.severity_name = "critical";
21342133
}
21352134
on_failure:
@@ -2139,7 +2138,7 @@ processors:
21392138
- script:
21402139
lang: painless
21412140
description: Script to set event.severity.
2142-
tag: set_event_severity
2141+
tag: set_event_severity_from_severity_name
21432142
if: ctx.crowdstrike?.alert?.severity_name instanceof String
21442143
source: |-
21452144
ctx.event = ctx.event ?: [:];
@@ -2808,7 +2807,7 @@ processors:
28082807
drop(ctx);
28092808
- set:
28102809
field: event.kind
2811-
tag: set_pipeline_error_to_event_kind
2810+
tag: set_pipeline_error_to_event_kind_processor
28122811
value: pipeline_error
28132812
if: ctx.error?.message != null
28142813
- append:

‎packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-event-stream.log-expected.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,18 @@
2828
}
2929
},
3030
"destination": {
31+
"geo": {
32+
"city_name": "London",
33+
"continent_name": "Europe",
34+
"country_iso_code": "GB",
35+
"country_name": "United Kingdom",
36+
"location": {
37+
"lat": 51.5142,
38+
"lon": -0.0931
39+
},
40+
"region_iso_code": "GB-ENG",
41+
"region_name": "England"
42+
},
3143
"ip": "81.2.69.142",
3244
"port": 445
3345
},
@@ -228,6 +240,7 @@
228240
"iam"
229241
],
230242
"created": "2023-11-02T11:12:03.065Z",
243+
"end": "2023-11-02T11:09:00.000Z",
231244
"id": "7a79b2de-4e10-41fb-818f-2bdf53c1625a",
232245
"kind": "event",
233246
"original": "{\"metadata\":{\"customerIDString\":\"abcabcabc22222\",\"offset\":8694126,\"eventType\":\"IdentityProtectionEvent\",\"eventCreationTime\":1698923523065,\"version\":\"1.0\"},\"event\":{\"IncidentType\":\"Unusuallogintoanendpoint\",\"IncidentDescription\":\"Unusuallogintoanendpoint\",\"Severity\":4,\"SeverityName\":\"LOW\",\"StartTime\":1698923340000,\"EndTime\":1698923340000,\"IdentityProtectionIncidentId\":\"7a79b2de-4e10-41fb-818f-2bdf53c1625a\",\"UserName\":\"NA.NET.ABC.com\\\\abc.service\",\"EndpointName\":\"itreg1d3wp8vh3.eu.net.ABC.com\",\"EndpointIp\":\"\",\"Category\":\"Detections\",\"NumbersOfAlerts\":1,\"NumberOfCompromisedEntities\":2,\"State\":\"NEW\",\"FalconHostLink\":\"https://falcon.crowdstrike.com/identity-protection/detections/abcabcabc22222:ind:abcabcabc22222:7A79B2DE-4E10-41FB-818F-2BDF53C1625A\"}}",
@@ -652,6 +665,7 @@
652665
"sha1": "0000000000000000000000000000000000000000",
653666
"sha256": "d64d27be4c907b0a2a0c32e1b1a4a44a1333a936fe63127d69df41c859da8c9e"
654667
},
668+
"name": "rundll32.exe",
655669
"path": "\\Device\\HarddiskVolume4\\Windows\\System32"
656670
},
657671
"host": {

‎packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-data-protection-detection-summary.log-expected.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@
167167
]
168168
},
169169
"file": {
170-
"extension": ".txt",
170+
"extension": "txt",
171171
"hash": {
172172
"sha256": "21cdfbea0d48077b93b4c4cccb23b7aadfea6f4e7fbb1237fdfd0caebf6efd08"
173173
},

‎packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-detection-summary.log-expected.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@
9696
"sha1": "7e54a24d4df5b7fe4a75d7ce1c027705c368666b",
9797
"sha256": "0b2cde5b355bda69aeb15159fa98b5554053f0936259a53c6dfb0934389238a2"
9898
},
99+
"name": "TESTFILE.exe",
99100
"path": "\\Device\\HarddiskVolume3\\Users\\ELASTICUSER\\Software\\TESTSOFTWARE"
100101
},
101102
"host": {

‎packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-epp-detection-summary.log-expected.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
"sha1": "0000000000000000000000000000000000000000",
8686
"sha256": "faf0c5e4ef7dbcfd863377c55a4d1d87a3f6a58c13a8a9882e11012066f31976"
8787
},
88+
"name": "rfusclient.exe",
8889
"path": "\\Device\\HarddiskVolume2\\Program Files (x86)\\Remote Utilities - Host\\rfusclient.exe"
8990
},
9091
"host": {
@@ -314,6 +315,7 @@
314315
"sha1": "0000000000000000000000000000000000000000",
315316
"sha256": "29aa689f38158d2e8941fa54e436f0260890af31cecad1e8799e5c2df7bc1ecc"
316317
},
318+
"name": "git",
317319
"path": "/usr/bin/git"
318320
},
319321
"host": {
@@ -431,6 +433,7 @@
431433
"hash": {
432434
"sha256": "774f50830a645392a94338815913e281096f1594ce5f4d992cf3f167fde509a1"
433435
},
436+
"name": "testfile.vmx",
434437
"path": "D:\\RECYCLER\\testpath\\testfile.vmx"
435438
},
436439
"host": {
@@ -623,6 +626,7 @@
623626
"sha1": "0000000000000000000000000000000000000000",
624627
"sha256": "29aa689f38158d2e8941fa54e436f0260890af31cecad1e8799e5c2df7bc1ecc"
625628
},
629+
"name": "git",
626630
"path": "/usr/bin/git"
627631
},
628632
"host": {
@@ -777,6 +781,7 @@
777781
"sha1": "0000000000000000000000000000000000000000",
778782
"sha256": "0000000000000000000000000000000000000000000000000000000000000000"
779783
},
784+
"name": "choice.exe",
780785
"path": "\\Device\\HarddiskVolume4\\Windows\\System32\\choice.exe"
781786
},
782787
"host": {
@@ -941,6 +946,7 @@
941946
"sha1": "0000000000000000000000000000000000000000",
942947
"sha256": "0000000000000000000000000000000000000000000000000000000000000000"
943948
},
949+
"name": "atera.exe.exe",
944950
"path": "\\Device\\HarddiskVolume4\\Users\\Administrator\\Desktop\\atera.exe.exe"
945951
},
946952
"host": {
@@ -1095,6 +1101,7 @@
10951101
"sha1": "0000000000000000000000000000000000000000",
10961102
"sha256": "0cb73775cb5eff70ee9189030eb00bfeb03b7245f2e12edc7d59119e86fd7490"
10971103
},
1104+
"name": "whoami.rtf",
10981105
"path": "/home/devuser/whoami.rtf"
10991106
},
11001107
"host": {
@@ -1249,6 +1256,7 @@
12491256
"sha1": "0000000000000000000000000000000000000000",
12501257
"sha256": "b2088f2a4a4648dd659ad60169369184a98528aeb7c0766273870b96dd4fa682"
12511258
},
1259+
"name": "cmd.exe",
12521260
"path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe"
12531261
},
12541262
"host": {
@@ -1406,6 +1414,7 @@
14061414
"sha1": "0000000000000000000000000000000000000000",
14071415
"sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06"
14081416
},
1417+
"name": "cmd.exe",
14091418
"path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe"
14101419
},
14111420
"host": {
@@ -1563,6 +1572,7 @@
15631572
"sha1": "0000000000000000000000000000000000000000",
15641573
"sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06"
15651574
},
1575+
"name": "cmd.exe",
15661576
"path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe"
15671577
},
15681578
"host": {
@@ -1720,6 +1730,7 @@
17201730
"sha1": "0000000000000000000000000000000000000000",
17211731
"sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06"
17221732
},
1733+
"name": "cmd.exe",
17231734
"path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe"
17241735
},
17251736
"host": {
@@ -1876,6 +1887,7 @@
18761887
"sha1": "0000000000000000000000000000000000000000",
18771888
"sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06"
18781889
},
1890+
"name": "cmd.exe",
18791891
"path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe"
18801892
},
18811893
"host": {
@@ -2033,6 +2045,7 @@
20332045
"sha1": "0000000000000000000000000000000000000000",
20342046
"sha256": "a1eb4e497f40111f44a1dca4674ead06dce7ea37d6b8df5a84b5022bf4072930"
20352047
},
2048+
"name": "choice.exe",
20362049
"path": "\\Device\\HarddiskVolume4\\Windows\\System32\\choice.exe"
20372050
},
20382051
"host": {
@@ -2190,6 +2203,7 @@
21902203
"sha1": "0000000000000000000000000000000000000000",
21912204
"sha256": "c43e9716c26e88801364e80e997f986760f5d82b5649f9e149b164396dd08c3d"
21922205
},
2206+
"name": "bash",
21932207
"path": "/usr/bin/bash"
21942208
},
21952209
"host": {
@@ -2745,6 +2759,7 @@
27452759
"sha1": "0000000000000000000000000000000000000000",
27462760
"sha256": "a9cc4e666b1a0cb0b6349fbe076e2bd7d2ac5b10f40d6466eb369e9e596978f7"
27472761
},
2762+
"name": "explorer.exe",
27482763
"path": "\\Device\\HarddiskVolume4\\Windows\\explorer.exe"
27492764
},
27502765
"host": {

‎packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-events.log-expected.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
"md5": "ac4c51eb24aa95b77f705ab159189e24",
5656
"sha256": "6a671b92a69755de6fd063fcbe4ba926d83b49f78c42dbaeed8cdb6bbc57576a"
5757
},
58+
"name": "explorer.exe",
5859
"path": "\\Device\\HarddiskVolume1\\Windows"
5960
},
6061
"host": {

‎packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-identity-protection-incident.log-expected.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"expected": [
33
{
4-
"@timestamp": "2023-06-16T05:03:56.553Z",
4+
"@timestamp": "2023-06-16T05:03:56.383Z",
55
"crowdstrike": {
66
"event": {
77
"Category": "Incidents",
@@ -27,12 +27,13 @@
2727
"iam"
2828
],
2929
"created": "2023-06-16T05:03:56.581Z",
30+
"end": "2023-06-16T05:03:56.553Z",
3031
"id": "INC-123",
3132
"kind": "event",
3233
"original": "{\n \"metadata\": {\n \"customerIDString\": \"3426316453426234\",\n \"offset\": 2664264,\n \"eventType\": \"IdentityProtectionEvent\",\n \"eventCreationTime\": 1686891836581,\n \"version\": \"1.0\"\n },\n \"event\": {\n \"IncidentType\": \"UNUSUAL_ENDPOINT_USE\",\n \"IncidentDescription\": \"User seen coming from a location that deviates from their baseline.\",\n \"Severity\": 1,\n \"SeverityName\": \"INFO\",\n \"StartTime\": 1686891836383,\n \"EndTime\": 1686891836553,\n \"IdentityProtectionIncidentId\": \"INC-123\",\n \"UserName\": \"TESTUSER\",\n \"EndpointName\": \"TESTHOSTNAME\",\n \"EndpointIp\": \"89.160.20.112\",\n \"Category\": \"Incidents\",\n \"NumbersOfAlerts\": 1,\n \"NumberOfCompromisedEntities\": 1,\n \"State\": \"NEW\",\n \"FalconHostLink\": \"https://falcon.crowdstrike.com/identity-protection/incidents/INC-123\"\n }\n}",
3334
"reference": "https://falcon.crowdstrike.com/identity-protection/incidents/INC-123",
3435
"severity": 21,
35-
"start": "2023-06-16T05:03:56.553Z",
36+
"start": "2023-06-16T05:03:56.383Z",
3637
"type": [
3738
"info"
3839
]

‎packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-recon-notification.log-expected.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
"version": "8.17.0"
2424
},
2525
"event": {
26-
"action": "recon-notification-",
26+
"action": "recon-notification-post",
2727
"category": [
2828
"threat"
2929
],
@@ -40,6 +40,7 @@
4040
"vendor": "Crowdstrike"
4141
},
4242
"rule": {
43+
"description": "TestRuleTopic",
4344
"id": "123543-0cb1-4806-9ccd-a543123",
4445
"name": "TestRuleName",
4546
"ruleset": "TestRuleTopic"

0 commit comments

Comments
 (0)