Skip to content

Commit de773e4

Browse files
desherdesher
committed
feat: Improve FrozenVideo and VideoDecoderCPU detectors
1 parent 66d8553 commit de773e4

13 files changed

+185
-165
lines changed

‎.eslintrc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
"max-len": ["error", { "code": 120 }],
1717
"import/extensions": "off",
1818
"import/no-cycle": "off",
19-
"no-continue": "off"
19+
"no-continue": "off",
20+
"import/prefer-default-export": "off"
2021
}
2122
}

‎src/WebRTCIssueDetector.ts‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
IssueDetector,
88
IssuePayload,
99
Logger,
10+
NetworkScores,
1011
StatsReportItem,
1112
WebRTCIssueDetectorConstructorParams,
1213
WebRTCStatsParsed,
@@ -88,11 +89,8 @@ class WebRTCIssueDetector {
8889
}
8990

9091
this.statsReporter.on(PeriodicWebRTCStatsReporter.STATS_REPORT_READY_EVENT, (report: StatsReportItem) => {
91-
this.detectIssues({
92-
data: report.stats,
93-
});
94-
95-
this.calculateNetworkScores(report.stats);
92+
const networkScores = this.calculateNetworkScores(report.stats);
93+
this.detectIssues({ data: report.stats }, networkScores);
9694
});
9795

9896
this.statsReporter.on(PeriodicWebRTCStatsReporter.STATS_REPORTS_PARSED, (data: {
@@ -161,16 +159,19 @@ class WebRTCIssueDetector {
161159
this.eventEmitter.emit(EventType.Issue, issues);
162160
}
163161

164-
private detectIssues({ data }: DetectIssuesPayload): void {
165-
const issues = this.detectors.reduce<IssuePayload[]>((acc, detector) => [...acc, ...detector.detect(data)], []);
162+
private detectIssues({ data }: DetectIssuesPayload, networkScores: NetworkScores): void {
163+
const issues = this.detectors
164+
.reduce<IssuePayload[]>((acc, detector) => [...acc, ...detector.detect(data, networkScores)], []);
165+
166166
if (issues.length > 0) {
167167
this.emitIssues(issues);
168168
}
169169
}
170170

171-
private calculateNetworkScores(data: WebRTCStatsParsed): void {
171+
private calculateNetworkScores(data: WebRTCStatsParsed): NetworkScores {
172172
const networkScores = this.networkScoresCalculator.calculate(data);
173173
this.eventEmitter.emit(EventType.NetworkScoresUpdated, networkScores);
174+
return networkScores;
174175
}
175176

176177
private wrapRTCPeerConnection(): void {

‎src/detectors/BaseIssueDetector.ts‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
1-
import { IssueDetector, IssueDetectorResult, WebRTCStatsParsed } from '../types';
1+
import {
2+
IssueDetector,
3+
IssueDetectorResult,
4+
NetworkScores,
5+
WebRTCStatsParsed,
6+
WebRTCStatsParsedWithNetworkScores,
7+
} from '../types';
28
import { scheduleTask } from '../utils/tasks';
39
import { CLEANUP_PREV_STATS_TTL_MS, MAX_PARSED_STATS_STORAGE_SIZE } from '../utils/constants';
410

@@ -13,7 +19,7 @@ export interface BaseIssueDetectorParams {
1319
}
1420

1521
abstract class BaseIssueDetector implements IssueDetector {
16-
readonly #parsedStatsStorage: Map<string, WebRTCStatsParsed[]> = new Map();
22+
readonly #parsedStatsStorage: Map<string, WebRTCStatsParsedWithNetworkScores[]> = new Map();
1723

1824
readonly #statsCleanupDelayMs: number;
1925

@@ -24,11 +30,19 @@ abstract class BaseIssueDetector implements IssueDetector {
2430
this.#maxParsedStatsStorageSize = params.maxParsedStatsStorageSize ?? MAX_PARSED_STATS_STORAGE_SIZE;
2531
}
2632

27-
abstract performDetection(data: WebRTCStatsParsed): IssueDetectorResult;
33+
abstract performDetection(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult;
2834

29-
detect(data: WebRTCStatsParsed): IssueDetectorResult {
30-
const result = this.performDetection(data);
35+
detect(data: WebRTCStatsParsed, networkScores?: NetworkScores): IssueDetectorResult {
36+
const parsedStatsWithNetworkScores = {
37+
...data,
38+
networkScores: {
39+
...networkScores,
40+
statsSamples: networkScores?.statsSamples || {},
41+
},
42+
};
43+
const result = this.performDetection(parsedStatsWithNetworkScores);
3144

45+
this.setLastProcessedStats(data.connection.id, parsedStatsWithNetworkScores);
3246
this.performPrevStatsCleanup({
3347
connectionId: data.connection.id,
3448
});
@@ -56,7 +70,7 @@ abstract class BaseIssueDetector implements IssueDetector {
5670
});
5771
}
5872

59-
protected setLastProcessedStats(connectionId: string, parsedStats: WebRTCStatsParsed): void {
73+
protected setLastProcessedStats(connectionId: string, parsedStats: WebRTCStatsParsedWithNetworkScores): void {
6074
if (!connectionId || parsedStats.connection.id !== connectionId) {
6175
return;
6276
}
@@ -71,16 +85,16 @@ abstract class BaseIssueDetector implements IssueDetector {
7185
this.#parsedStatsStorage.set(connectionId, connectionStats);
7286
}
7387

74-
protected getLastProcessedStats(connectionId: string): WebRTCStatsParsed | undefined {
88+
protected getLastProcessedStats(connectionId: string): WebRTCStatsParsedWithNetworkScores | undefined {
7589
const connectionStats = this.#parsedStatsStorage.get(connectionId);
7690
return connectionStats?.[connectionStats.length - 1];
7791
}
7892

79-
protected getAllLastProcessedStats(connectionId: string): WebRTCStatsParsed[] {
93+
protected getAllLastProcessedStats(connectionId: string): WebRTCStatsParsedWithNetworkScores[] {
8094
return this.#parsedStatsStorage.get(connectionId) ?? [];
8195
}
8296

83-
private deleteLastProcessedStats(connectionId: string): void {
97+
protected deleteLastProcessedStats(connectionId: string): void {
8498
this.#parsedStatsStorage.delete(connectionId);
8599
}
86100
}

‎src/detectors/FramesEncodedSentIssueDetector.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,7 @@ class FramesEncodedSentIssueDetector extends BaseIssueDetector {
1919
}
2020

2121
performDetection(data: WebRTCStatsParsed): IssueDetectorResult {
22-
const { connection: { id: connectionId } } = data;
23-
const issues = this.processData(data);
24-
this.setLastProcessedStats(connectionId, data);
25-
return issues;
22+
return this.processData(data);
2623
}
2724

2825
private processData(data: WebRTCStatsParsed): IssueDetectorResult {

‎src/detectors/FrozenVideoTrackDetector.ts‎

Lines changed: 76 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -2,134 +2,108 @@ import {
22
IssueDetectorResult,
33
IssueReason,
44
IssueType,
5-
ParsedInboundVideoStreamStats,
5+
MosQuality,
66
WebRTCStatsParsed,
7+
WebRTCStatsParsedWithNetworkScores,
78
} from '../types';
9+
import { isSvcSpatialLayerChanged } from '../utils/video';
810
import BaseIssueDetector from './BaseIssueDetector';
911

1012
interface FrozenVideoTrackDetectorParams {
11-
timeoutMs?: number;
12-
framesDroppedThreshold?: number;
13+
avgFreezeDurationThresholdMs?: number;
14+
frozenDurationThresholdPct?: number;
1315
}
1416

15-
class FrozenVideoTrackDetector extends BaseIssueDetector {
16-
readonly #lastMarkedAt = new Map<string, number>();
17+
interface FrozenStream {
18+
ssrc: number;
19+
avgFreezeDurationMs: number;
20+
frozenDurationPct: number;
21+
}
1722

18-
readonly #timeoutMs: number;
23+
class FrozenVideoTrackDetector extends BaseIssueDetector {
24+
#avgFreezeDurationThresholdMs: number;
1925

20-
readonly #framesDroppedThreshold: number;
26+
#frozenDurationThresholdPct: number;
2127

2228
constructor(params: FrozenVideoTrackDetectorParams = {}) {
2329
super();
24-
this.#timeoutMs = params.timeoutMs ?? 10_000;
25-
this.#framesDroppedThreshold = params.framesDroppedThreshold ?? 0.5;
30+
this.#avgFreezeDurationThresholdMs = params.avgFreezeDurationThresholdMs ?? 1_000;
31+
this.#frozenDurationThresholdPct = params.frozenDurationThresholdPct ?? 30;
2632
}
2733

28-
performDetection(data: WebRTCStatsParsed): IssueDetectorResult {
29-
const { connection: { id: connectionId } } = data;
30-
const issues = this.processData(data);
31-
this.setLastProcessedStats(connectionId, data);
32-
return issues;
34+
performDetection(data: WebRTCStatsParsedWithNetworkScores): IssueDetectorResult {
35+
const inboundScore = data.networkScores.inbound;
36+
if (inboundScore !== undefined && inboundScore <= MosQuality.BAD) {
37+
// do not execute detection on stats based on poor network quality
38+
// to avoid false positives
39+
return [];
40+
}
41+
42+
return this.processData(data);
3343
}
3444

3545
private processData(data: WebRTCStatsParsed): IssueDetectorResult {
36-
const { connection: { id: connectionId } } = data;
37-
const previousStats = this.getLastProcessedStats(connectionId);
3846
const issues: IssueDetectorResult = [];
39-
40-
if (!previousStats) {
41-
return issues;
47+
const allLastProcessedStats = this.getAllLastProcessedStats(data.connection.id);
48+
if (allLastProcessedStats.length === 0) {
49+
return [];
4250
}
4351

44-
const { video: { inbound: newInbound } } = data;
45-
const { video: { inbound: prevInbound } } = previousStats;
46-
47-
const mapByTrackId = (items: ParsedInboundVideoStreamStats[]) => new Map<string, ParsedInboundVideoStreamStats>(
48-
items.map((item) => [item.track.trackIdentifier, item] as const),
49-
);
50-
51-
const newInboundByTrackId = mapByTrackId(newInbound);
52-
const prevInboundByTrackId = mapByTrackId(prevInbound);
53-
const unvisitedTrackIds = new Set(this.#lastMarkedAt.keys());
54-
55-
Array.from(newInboundByTrackId.entries()).forEach(([trackId, newInboundItem]) => {
56-
unvisitedTrackIds.delete(trackId);
57-
58-
const prevInboundItem = prevInboundByTrackId.get(trackId);
59-
if (!prevInboundItem) {
60-
return;
61-
}
62-
63-
const deltaFramesReceived = newInboundItem.framesReceived - prevInboundItem.framesReceived;
64-
const deltaFramesDropped = newInboundItem.framesDropped - prevInboundItem.framesDropped;
65-
const deltaFramesDecoded = newInboundItem.framesDecoded - prevInboundItem.framesDecoded;
66-
const ratioFramesDropped = deltaFramesDropped / deltaFramesReceived;
67-
68-
if (deltaFramesReceived === 0) {
69-
return;
70-
}
71-
72-
// We skip it when ratio is too low because it should be handled by VideoDecoderIssueDetector
73-
if (ratioFramesDropped >= this.#framesDroppedThreshold) {
74-
return;
75-
}
76-
77-
// It seems that track is alive and we can remove mark if it was marked
78-
if (deltaFramesDecoded > 0) {
79-
this.removeMarkIssue(trackId);
80-
return;
81-
}
82-
83-
const hasIssue = this.markIssue(trackId);
84-
85-
if (!hasIssue) {
86-
return;
87-
}
88-
89-
const statsSample = {
90-
framesReceived: newInboundItem.framesReceived,
91-
framesDropped: newInboundItem.framesDropped,
92-
framesDecoded: newInboundItem.framesDecoded,
93-
deltaFramesReceived,
94-
deltaFramesDropped,
95-
deltaFramesDecoded,
96-
};
97-
52+
const frozenStreams = data.video.inbound
53+
.map((videoStream): FrozenStream | undefined => {
54+
const prevStat = allLastProcessedStats[allLastProcessedStats.length - 1]
55+
.video.inbound.find((stream) => stream.ssrc === videoStream.ssrc);
56+
57+
if (!prevStat) {
58+
return undefined;
59+
}
60+
61+
const isSpatialLayerChanged = isSvcSpatialLayerChanged(videoStream.ssrc, [
62+
allLastProcessedStats[allLastProcessedStats.length - 1],
63+
data,
64+
]);
65+
66+
if (isSpatialLayerChanged) {
67+
return undefined;
68+
}
69+
70+
const deltaFreezeCount = videoStream.freezeCount - (prevStat.freezeCount ?? 0);
71+
const deltaFreezesTimeMs = (videoStream.totalFreezesDuration - (prevStat.totalFreezesDuration ?? 0)) * 1000;
72+
const avgFreezeDurationMs = deltaFreezeCount > 0 ? deltaFreezesTimeMs / deltaFreezeCount : 0;
73+
74+
const statsTimeDiff = videoStream.timestamp - prevStat.timestamp;
75+
const frozenDurationPct = (deltaFreezesTimeMs / statsTimeDiff) * 100;
76+
if (frozenDurationPct > this.#frozenDurationThresholdPct) {
77+
return {
78+
ssrc: videoStream.ssrc,
79+
avgFreezeDurationMs,
80+
frozenDurationPct,
81+
};
82+
}
83+
84+
if (avgFreezeDurationMs > this.#avgFreezeDurationThresholdMs) {
85+
return {
86+
ssrc: videoStream.ssrc,
87+
avgFreezeDurationMs,
88+
frozenDurationPct,
89+
};
90+
}
91+
92+
return undefined;
93+
})
94+
.filter((stream) => stream !== undefined) as FrozenStream[];
95+
96+
if (frozenStreams.length > 0) {
9897
issues.push({
99-
statsSample,
10098
type: IssueType.Stream,
10199
reason: IssueReason.FrozenVideoTrack,
102-
trackIdentifier: trackId,
100+
statsSample: {
101+
ssrcs: frozenStreams.map((stream) => stream.ssrc),
102+
},
103103
});
104-
});
105-
106-
// just clear unvisited tracks from memory
107-
unvisitedTrackIds.forEach((trackId) => {
108-
this.removeMarkIssue(trackId);
109-
});
110-
111-
return issues;
112-
}
113-
114-
private markIssue(trackId: string): boolean {
115-
const now = Date.now();
116-
117-
const lastMarkedAt = this.#lastMarkedAt.get(trackId);
118-
119-
if (!lastMarkedAt) {
120-
this.#lastMarkedAt.set(trackId, now);
121-
return false;
122-
}
123-
124-
if (now - lastMarkedAt < this.#timeoutMs) {
125-
return false;
126104
}
127105

128-
return true;
129-
}
130-
131-
private removeMarkIssue(trackId: string): void {
132-
this.#lastMarkedAt.delete(trackId);
106+
return issues;
133107
}
134108
}
135109

‎src/detectors/InboundNetworkIssueDetector.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,7 @@ class InboundNetworkIssueDetector extends BaseIssueDetector {
3131
}
3232

3333
performDetection(data: WebRTCStatsParsed): IssueDetectorResult {
34-
const { connection: { id: connectionId } } = data;
35-
const issues = this.processData(data);
36-
this.setLastProcessedStats(connectionId, data);
37-
return issues;
34+
return this.processData(data);
3835
}
3936

4037
private processData(data: WebRTCStatsParsed): IssueDetectorResult {

‎src/detectors/NetworkMediaSyncIssueDetector.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,7 @@ class NetworkMediaSyncIssueDetector extends BaseIssueDetector {
1919
}
2020

2121
performDetection(data: WebRTCStatsParsed): IssueDetectorResult {
22-
const { connection: { id: connectionId } } = data;
23-
const issues = this.processData(data);
24-
this.setLastProcessedStats(connectionId, data);
25-
return issues;
22+
return this.processData(data);
2623
}
2724

2825
private processData(data: WebRTCStatsParsed): IssueDetectorResult {

‎src/detectors/OutboundNetworkIssueDetector.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,7 @@ class OutboundNetworkIssueDetector extends BaseIssueDetector {
2323
}
2424

2525
performDetection(data: WebRTCStatsParsed): IssueDetectorResult {
26-
const { connection: { id: connectionId } } = data;
27-
const issues = this.processData(data);
28-
this.setLastProcessedStats(connectionId, data);
29-
return issues;
26+
return this.processData(data);
3027
}
3128

3229
private processData(data: WebRTCStatsParsed): IssueDetectorResult {

‎src/detectors/QualityLimitationsIssueDetector.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,7 @@ import BaseIssueDetector from './BaseIssueDetector';
88

99
class QualityLimitationsIssueDetector extends BaseIssueDetector {
1010
performDetection(data: WebRTCStatsParsed): IssueDetectorResult {
11-
const { connection: { id: connectionId } } = data;
12-
const issues = this.processData(data);
13-
this.setLastProcessedStats(connectionId, data);
14-
return issues;
11+
return this.processData(data);
1512
}
1613

1714
private processData(data: WebRTCStatsParsed): IssueDetectorResult {

0 commit comments

Comments
 (0)