Add per-frame encoding time in video encoding CPU connection health p…

…olicy

CHANGELOG.md

@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Add per-frame encoding time in video encoding CPU connection health policy
+
 ### Fixed
 
 - Prevent DataMessage callback errors from killing a meeting

docs/classes/connectionhealthpolicyconfiguration.html

@@ -95,6 +95,7 @@ <h3>Properties</h3>
 								<li class="tsd-kind-property tsd-parent-kind-class"><a href="connectionhealthpolicyconfiguration.html#fourbarstimems" class="tsd-kind-icon">four<wbr>Bars<wbr>Time<wbr>Ms</a></li>
 								<li class="tsd-kind-property tsd-parent-kind-class"><a href="connectionhealthpolicyconfiguration.html#fractionalloss" class="tsd-kind-icon">fractional<wbr>Loss</a></li>
 								<li class="tsd-kind-property tsd-parent-kind-class"><a href="connectionhealthpolicyconfiguration.html#goodsignaltimems" class="tsd-kind-icon">good<wbr>Signal<wbr>Time<wbr>Ms</a></li>
+								<li class="tsd-kind-property tsd-parent-kind-class"><a href="connectionhealthpolicyconfiguration.html#highencodecpumsperframethreshold" class="tsd-kind-icon">high<wbr>Encode<wbr>Cpu<wbr>MsPer<wbr>Frame<wbr>Threshold</a></li>
 								<li class="tsd-kind-property tsd-parent-kind-class"><a href="connectionhealthpolicyconfiguration.html#highencodecpumsthreshold" class="tsd-kind-icon">high<wbr>Encode<wbr>Cpu<wbr>MsThreshold</a></li>
 								<li class="tsd-kind-property tsd-parent-kind-class"><a href="connectionhealthpolicyconfiguration.html#initialhealth" class="tsd-kind-icon">initial<wbr>Health</a></li>
 								<li class="tsd-kind-property tsd-parent-kind-class"><a href="connectionhealthpolicyconfiguration.html#maxhealth" class="tsd-kind-icon">max<wbr>Health</a></li>
@@ -180,7 +181,7 @@ <h3>consecutive<wbr>Video<wbr>Encoding<wbr>Failure<wbr>Threshold</h3>
 					<div class="tsd-signature tsd-kind-icon">consecutive<wbr>Video<wbr>Encoding<wbr>Failure<wbr>Threshold<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> = 5</span></div>
 					<aside class="tsd-sources">
 						<ul>
-							<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/connectionhealthpolicy/ConnectionHealthPolicyConfiguration.ts#L71">src/connectionhealthpolicy/ConnectionHealthPolicyConfiguration.ts:71</a></li>
+							<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/connectionhealthpolicy/ConnectionHealthPolicyConfiguration.ts#L79">src/connectionhealthpolicy/ConnectionHealthPolicyConfiguration.ts:79</a></li>
 						</ul>
 					</aside>
 					<div class="tsd-comment tsd-typography">
@@ -240,6 +241,24 @@ <h3>good<wbr>Signal<wbr>Time<wbr>Ms</h3>
 						</ul>
 					</aside>
 				</section>
+				<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-class">
+					<a name="highencodecpumsperframethreshold" class="tsd-anchor"></a>
+					<h3>high<wbr>Encode<wbr>Cpu<wbr>MsPer<wbr>Frame<wbr>Threshold</h3>
+					<div class="tsd-signature tsd-kind-icon">high<wbr>Encode<wbr>Cpu<wbr>MsPer<wbr>Frame<wbr>Threshold<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> = 15</span></div>
+					<aside class="tsd-sources">
+						<ul>
+							<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/connectionhealthpolicy/ConnectionHealthPolicyConfiguration.ts#L73">src/connectionhealthpolicy/ConnectionHealthPolicyConfiguration.ts:73</a></li>
+						</ul>
+					</aside>
+					<div class="tsd-comment tsd-typography">
+						<div class="lead">
+							<p>Encode time per frame threshold to determine high CPU usage of software encoders in video encoding health
+								monitoring. Recuding the value results in video codec degradation due to high CPU usage software encoder to
+								be triggered at a lower CPU usage. Note that encoder counts each SVC spatial layer of a frame as an encoded
+							frame. The default value is set with the consideration of this logic.</p>
+						</div>
+					</div>
+				</section>
 				<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-class">
 					<a name="highencodecpumsthreshold" class="tsd-anchor"></a>
 					<h3>high<wbr>Encode<wbr>Cpu<wbr>MsThreshold</h3>
@@ -503,6 +522,9 @@ <h3>zero<wbr>Bars<wbr>NoSignal<wbr>Time<wbr>Ms</h3>
 							<li class=" tsd-kind-property tsd-parent-kind-class">
 								<a href="connectionhealthpolicyconfiguration.html#goodsignaltimems" class="tsd-kind-icon">good<wbr>Signal<wbr>Time<wbr>Ms</a>
 							</li>
+							<li class=" tsd-kind-property tsd-parent-kind-class">
+								<a href="connectionhealthpolicyconfiguration.html#highencodecpumsperframethreshold" class="tsd-kind-icon">high<wbr>Encode<wbr>Cpu<wbr>MsPer<wbr>Frame<wbr>Threshold</a>
+							</li>
 							<li class=" tsd-kind-property tsd-parent-kind-class">
 								<a href="connectionhealthpolicyconfiguration.html#highencodecpumsthreshold" class="tsd-kind-icon">high<wbr>Encode<wbr>Cpu<wbr>MsThreshold</a>
 							</li>

docs/classes/videoencodingcpuconnectionhealthpolicy.html

@@ -132,7 +132,7 @@ <h3>constructor</h3>
 							<aside class="tsd-sources">
 								<p>Overrides <a href="baseconnectionhealthpolicy.html">BaseConnectionHealthPolicy</a>.<a href="baseconnectionhealthpolicy.html#constructor">constructor</a></p>
 								<ul>
-									<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/connectionhealthpolicy/VideoEncodingCpuConnectionHealthPolicy.ts#L15">src/connectionhealthpolicy/VideoEncodingCpuConnectionHealthPolicy.ts:15</a></li>
+									<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/connectionhealthpolicy/VideoEncodingCpuConnectionHealthPolicy.ts#L16">src/connectionhealthpolicy/VideoEncodingCpuConnectionHealthPolicy.ts:16</a></li>
 								</ul>
 							</aside>
 							<h4 class="tsd-parameters-title">Parameters</h4>
@@ -245,7 +245,7 @@ <h3>health</h3>
 								<p>Implementation of <a href="../interfaces/connectionhealthpolicy.html">ConnectionHealthPolicy</a>.<a href="../interfaces/connectionhealthpolicy.html#health">health</a></p>
 								<p>Overrides <a href="baseconnectionhealthpolicy.html">BaseConnectionHealthPolicy</a>.<a href="baseconnectionhealthpolicy.html#health">health</a></p>
 								<ul>
-									<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/connectionhealthpolicy/VideoEncodingCpuConnectionHealthPolicy.ts#L23">src/connectionhealthpolicy/VideoEncodingCpuConnectionHealthPolicy.ts:23</a></li>
+									<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/connectionhealthpolicy/VideoEncodingCpuConnectionHealthPolicy.ts#L25">src/connectionhealthpolicy/VideoEncodingCpuConnectionHealthPolicy.ts:25</a></li>
 								</ul>
 							</aside>
 							<div class="tsd-comment tsd-typography">

src/connectionhealthpolicy/ConnectionHealthData.ts

@@ -15,6 +15,7 @@ export default class ConnectionHealthData {
   audioSpeakerDelayMs = 0;
   isVideoEncoderHardware = false;
   videoEncodingTimeInMs = 0;
+  videoEncodingTimePerFrameInMs = 0;
   cpuLimitationDuration = 0;
   videoInputFps = 0;
   videoEncodeFps = 0;
@@ -48,6 +49,7 @@ export default class ConnectionHealthData {
     this.lastGoodSignalTimestampMs = Date.now();
     this.isVideoEncoderHardware = false;
     this.videoEncodingTimeInMs = 0;
+    this.videoEncodingTimePerFrameInMs = 0;
     this.cpuLimitationDuration = 0;
     this.videoInputFps = 0;
     this.videoEncodeFps = 0;
@@ -91,6 +93,7 @@ export default class ConnectionHealthData {
     cloned.audioSpeakerDelayMs = this.audioSpeakerDelayMs;
     cloned.isVideoEncoderHardware = this.isVideoEncoderHardware;
     cloned.videoEncodingTimeInMs = this.videoEncodingTimeInMs;
+    cloned.videoEncodingTimePerFrameInMs = this.videoEncodingTimePerFrameInMs;
     cloned.cpuLimitationDuration = this.cpuLimitationDuration;
     cloned.videoInputFps = this.videoInputFps;
     cloned.videoEncodeFps = this.videoEncodeFps;
@@ -126,6 +129,9 @@ export default class ConnectionHealthData {
   setVideoEncodingTimeInMs(stats: number): void {
     this.videoEncodingTimeInMs = stats;
   }
+  setVideoEncodingTimePerFrameInMs(stats: number): void {
+    this.videoEncodingTimePerFrameInMs = stats;
+  }
   setCpuLimitationDuration(stats: number): void {
     this.cpuLimitationDuration = stats;
   }

src/connectionhealthpolicy/ConnectionHealthPolicyConfiguration.ts

@@ -64,6 +64,14 @@ export default class ConnectionHealthPolicyConfiguration {
    */
   highEncodeCpuMsThreshold = 500;
 
+  /**
+   * Encode time per frame threshold to determine high CPU usage of software encoders in video encoding health
+   * monitoring. Recuding the value results in video codec degradation due to high CPU usage software encoder to
+   * be triggered at a lower CPU usage. Note that encoder counts each SVC spatial layer of a frame as an encoded
+   * frame. The magnification in framerate should be considered when configuring this parameter with SVC enabled.
+   */
+  highEncodeCpuMsPerFrameThreshold = 15;
+
   /**
    * Consecutive seconds of zero encoded framerate to trigger video codec degradation in video encoding health monitoring.
    * Increasing the value results in less sensitive video codec degradaion and vice versa.

src/connectionhealthpolicy/VideoEncodingCpuConnectionHealthPolicy.ts

@@ -12,19 +12,23 @@ export default class VideoEncodingCpuConnectionHealthPolicy
   implements ConnectionHealthPolicy {
   private readonly consecutiveHighEncodeCpuThreshold: number;
   private readonly highEncodeCpuMsThreshold: number;
+  private readonly highEncodeCpuMsPerFrameThreshold: number;
   private consecutiveHighEncodeCpuCnt = 0;
 
   constructor(configuration: ConnectionHealthPolicyConfiguration, data: ConnectionHealthData) {
     super(configuration, data, VideoEncodingConnectionHealthPolicyName.VideoEncodingCpuHealth);
     this.consecutiveHighEncodeCpuThreshold = configuration.consecutiveHighEncodeCpuThreshold;
     this.highEncodeCpuMsThreshold = configuration.highEncodeCpuMsThreshold;
+    this.highEncodeCpuMsPerFrameThreshold = configuration.highEncodeCpuMsPerFrameThreshold;
   }
 
   health(): number {
+    const videoEncodingTimeIsHigh =
+      this.currentData.videoEncodingTimeInMs >= this.highEncodeCpuMsThreshold &&
+      this.currentData.videoEncodingTimePerFrameInMs >= this.highEncodeCpuMsPerFrameThreshold;
     const cpuUsageIsHigh =
       !this.currentData.isVideoEncoderHardware &&
-      (this.currentData.videoEncodingTimeInMs >= this.highEncodeCpuMsThreshold ||
-        this.currentData.cpuLimitationDuration > 0);
+      (videoEncodingTimeIsHigh || this.currentData.cpuLimitationDuration > 0);
     if (cpuUsageIsHigh) {
       this.consecutiveHighEncodeCpuCnt++;
       if (this.consecutiveHighEncodeCpuCnt > this.consecutiveHighEncodeCpuThreshold) {

src/connectionmonitor/SignalingAndMetricsConnectionMonitor.ts

@@ -179,7 +179,7 @@ export default class SignalingAndMetricsConnectionMonitor
     const ssrc = clientMetricReport.getVideoUpstreamSsrc();
     if (!isLocalVideoTileStarted || ssrc === null) {
       this.connectionHealthData.setIsVideoEncoderHardware(false);
-      this.connectionHealthData.setVideoEncodingTimeInMs(0);
+      this.connectionHealthData.setVideoEncodingTimePerFrameInMs(0);
       this.connectionHealthData.setCpuLimitationDuration(0);
       this.connectionHealthData.setVideoInputFps(0);
       this.connectionHealthData.setVideoEncodeFps(0);
@@ -206,9 +206,12 @@ export default class SignalingAndMetricsConnectionMonitor
       'videoUpstreamFramesEncodedPerSecond',
       ssrc
     );
+    const videoEncodingTimePerFrameInMs =
+      videoEncodeFps > 0 ? videoEncodingTimeInMs / videoEncodeFps : 0;
 
     this.connectionHealthData.setIsVideoEncoderHardware(Boolean(isHardwareEncoder));
     this.connectionHealthData.setVideoEncodingTimeInMs(videoEncodingTimeInMs);
+    this.connectionHealthData.setVideoEncodingTimePerFrameInMs(videoEncodingTimePerFrameInMs);
     this.connectionHealthData.setCpuLimitationDuration(cpuLimitationDuration);
     this.connectionHealthData.setVideoInputFps(videoInputFps);
     this.connectionHealthData.setVideoEncodeFps(videoEncodeFps);

test/connectionmonitor/SignalingAndMetricsConnectionMonitor.test.ts

@@ -472,6 +472,7 @@ describe('SignalingAndMetricsConnectionMonitor', () => {
     sendClientMetricReport(clientMetricReport);
     expect(connectionHealthData.isVideoEncoderHardware).to.be.false;
     expect(connectionHealthData.videoEncodingTimeInMs).to.equal(0);
+    expect(connectionHealthData.videoEncodingTimePerFrameInMs).to.equal(0);
     expect(connectionHealthData.cpuLimitationDuration).to.equal(0);
     expect(connectionHealthData.videoInputFps).to.equal(0);
     expect(connectionHealthData.videoEncodeFps).to.equal(0);
@@ -500,6 +501,7 @@ describe('SignalingAndMetricsConnectionMonitor', () => {
     sendClientMetricReport(clientMetricReport);
     expect(connectionHealthData.isVideoEncoderHardware).to.be.false;
     expect(connectionHealthData.videoEncodingTimeInMs).to.equal(0);
+    expect(connectionHealthData.videoEncodingTimePerFrameInMs).to.equal(0);
     expect(connectionHealthData.cpuLimitationDuration).to.equal(0);
     expect(connectionHealthData.videoInputFps).to.equal(0);
     expect(connectionHealthData.videoEncodeFps).to.equal(0);
@@ -515,7 +517,7 @@ describe('SignalingAndMetricsConnectionMonitor', () => {
     upstreamReport.mediaType = MediaType.VIDEO;
     upstreamReport.direction = Direction.UPSTREAM;
     upstreamReport.previousMetrics['totalEncodeTime'] = 1.0;
-    upstreamReport.currentMetrics['totalEncodeTime'] = 1.1;
+    upstreamReport.currentMetrics['totalEncodeTime'] = 1.3;
     upstreamReport.currentMetrics['framesPerSecond'] = 15;
     upstreamReport.previousMetrics['framesEncoded'] = 0;
     upstreamReport.currentMetrics['framesEncoded'] = 15;
@@ -543,9 +545,55 @@ describe('SignalingAndMetricsConnectionMonitor', () => {
     audioVideoController.videoTileController.startLocalVideoTile();
     sendClientMetricReport(clientMetricReport);
     expect(connectionHealthData.isVideoEncoderHardware).to.be.true;
-    expect(Math.trunc(connectionHealthData.videoEncodingTimeInMs)).to.equal(100);
+    expect(Math.trunc(connectionHealthData.videoEncodingTimeInMs)).to.equal(300);
+    expect(Math.trunc(connectionHealthData.videoEncodingTimePerFrameInMs)).to.equal(20);
     expect(connectionHealthData.cpuLimitationDuration).to.equal(0);
     expect(connectionHealthData.videoInputFps).to.equal(15);
     expect(connectionHealthData.videoEncodeFps).to.equal(15);
   });
+
+  it('does translate data when upstream metric is available with 0 encoded frame', async () => {
+    const index = prepareIndex([1, 2]);
+    const clientMetricReport = new ClientMetricReport(new NoOpDebugLogger(), index, 'attendee-1');
+    clientMetricReport.currentTimestampMs = 2000;
+    clientMetricReport.previousTimestampMs = 1000;
+    const upstreamSsrc = 1;
+    const upstreamReport = new StreamMetricReport();
+    upstreamReport.mediaType = MediaType.VIDEO;
+    upstreamReport.direction = Direction.UPSTREAM;
+    upstreamReport.previousMetrics['totalEncodeTime'] = 1.0;
+    upstreamReport.currentMetrics['totalEncodeTime'] = 1.0;
+    upstreamReport.currentMetrics['framesPerSecond'] = 0;
+    upstreamReport.previousMetrics['framesEncoded'] = 0;
+    upstreamReport.currentMetrics['framesEncoded'] = 0;
+    upstreamReport.currentStringMetrics['encoderImplementation'] = 'ExternalEncoder';
+    upstreamReport.currentObjectMetrics['qualityLimitationDurations'] = {
+      cpu: 0.0,
+      other: 0.0,
+    };
+    upstreamReport.previousObjectMetrics['qualityLimitationDurations'] = {
+      cpu: 0.0,
+      other: 0.0,
+    };
+    clientMetricReport.streamMetricReports[upstreamSsrc] = upstreamReport;
+    clientMetricReport.rtcStatsReport = new Map<string, RawMetrics>([
+      [
+        'candidatePairId1',
+        {
+          type: 'candidate-pair',
+          ...{
+            packetsReceived: 0,
+          },
+        },
+      ],
+    ]);
+    audioVideoController.videoTileController.startLocalVideoTile();
+    sendClientMetricReport(clientMetricReport);
+    expect(connectionHealthData.isVideoEncoderHardware).to.be.true;
+    expect(Math.trunc(connectionHealthData.videoEncodingTimeInMs)).to.equal(0);
+    expect(Math.trunc(connectionHealthData.videoEncodingTimePerFrameInMs)).to.equal(0);
+    expect(connectionHealthData.cpuLimitationDuration).to.equal(0);
+    expect(connectionHealthData.videoInputFps).to.equal(0);
+    expect(connectionHealthData.videoEncodeFps).to.equal(0);
+  });
 });

test/task/MonitorTask.test.ts

@@ -972,7 +972,8 @@ describe('MonitorTask', () => {
 
       it('does not degrade video codec when metrics are normal with software encoder', () => {
         connectionHealthData.setIsVideoEncoderHardware(false);
-        connectionHealthData.setVideoEncodingTimeInMs(20);
+        connectionHealthData.setVideoEncodingTimeInMs(300);
+        connectionHealthData.setVideoEncodingTimePerFrameInMs(10);
         connectionHealthData.setCpuLimitationDuration(0);
         connectionHealthData.setVideoInputFps(15);
         connectionHealthData.setVideoEncodeFps(15);
@@ -987,7 +988,8 @@ describe('MonitorTask', () => {
 
       it('does not degrade video codec when metrics are normal with hardware encoder', () => {
         connectionHealthData.setIsVideoEncoderHardware(true);
-        connectionHealthData.setVideoEncodingTimeInMs(30);
+        connectionHealthData.setVideoEncodingTimeInMs(450);
+        connectionHealthData.setVideoEncodingTimePerFrameInMs(10);
         connectionHealthData.setCpuLimitationDuration(0);
         connectionHealthData.setVideoInputFps(15);
         connectionHealthData.setVideoEncodeFps(15);
@@ -1002,8 +1004,11 @@ describe('MonitorTask', () => {
 
       it('does degrade video codec when software encoding CPU is constantly high', () => {
         connectionHealthData.setIsVideoEncoderHardware(false);
-        connectionHealthData.setVideoEncodingTimeInMs(600);
+        connectionHealthData.setVideoEncodingTimeInMs(900);
+        connectionHealthData.setVideoEncodingTimePerFrameInMs(60);
         connectionHealthData.setCpuLimitationDuration(0);
+        connectionHealthData.setVideoInputFps(15);
+        connectionHealthData.setVideoEncodeFps(15);
         for (let i = 0; i < 15; i++) {
           task.connectionHealthDidChange(connectionHealthData);
         }
@@ -1015,6 +1020,7 @@ describe('MonitorTask', () => {
 
       it('does degrade video codec when video quality is limited due to CPU', () => {
         connectionHealthData.setIsVideoEncoderHardware(false);
+        connectionHealthData.setVideoEncodingTimePerFrameInMs(0);
         connectionHealthData.setVideoEncodingTimeInMs(0);
         connectionHealthData.setCpuLimitationDuration(1);
         for (let i = 0; i < 15; i++) {
@@ -1028,6 +1034,7 @@ describe('MonitorTask', () => {
 
       it('does degrade video codec when video encoding fails', () => {
         connectionHealthData.setIsVideoEncoderHardware(true);
+        connectionHealthData.setVideoEncodingTimePerFrameInMs(0);
         connectionHealthData.setVideoEncodingTimeInMs(0);
         connectionHealthData.setCpuLimitationDuration(1);
         connectionHealthData.setVideoInputFps(15);