feat: Consider object-fit when selecting playlist by player size (#1051)

* Make simple playlist selecor easier to extend

Refactor from positional parameters to a single `settings`
argument. This makes it clearer what each argument means in calls of
the function. Additional parameters can now be added without making
the argument list overly long.

Key names were chosen to match those of
`minRebufferMaxBandwidthSelector` to align the signatures.

* Make simpleSelector test easier to understand

Inline the passed bandwidth value instead of referencing the config
constant since the concrete value is needed to understand why the
expected playlist is chosen. Also if the config constant should ever
change the test will fail for no good reason.

* Consider object-fit when selecting playlist by player size

So far, when `limitRenditionByPlayerDimensions` is `true`,
`simpleSelector` tried to either find a rendition with a resolution
that matches the size of the player exactly or, if that does not
exist, a rendition with the smallest resolution that has either
greater width or greater height than the player. This makes sense
since by default the video will be scaled to fit inside the media
element. So every resolution that exceeds player size in at least one
dimension will be scaled down.

Most browsers support [1] customizing this scaling behavior via the
`object-fit` CSS property [2]. If it set to `cover`, the video will
instead be scaled up if video and player aspect ratio do not match.

The previous behavior caused renditions with low resolution to be
selected for players with small width (e.g. portrait phone aspect
ratio) even when videos were then scaled up to cover the whole player.

We therefore detect if `object-fit` is set to `cover` and instead
select the smallest rendition with a resolution that exceeds player
dimensions in both width and height.

[1] https://caniuse.com/?search=object-fit
[2] https://developer.mozilla.org/de/docs/Web/CSS/object-fit

* Add usePlayerObjectFit option

Only consider `object-fit` CSS property when selecting playlists based
on play size when `usePlayerObjectFit` option is `true` to make new
behavior an opt-in.

* chore: add object-fit option to the demo page

---------

Co-authored-by: Dzianis Dashkevich <98566601+dzianis-dashkevich@users.noreply.github.com>

README.md

@@ -49,6 +49,7 @@ Video.js Compatibility: 7.x, 8.x
       - [enableLowInitialPlaylist](#enablelowinitialplaylist)
       - [limitRenditionByPlayerDimensions](#limitrenditionbyplayerdimensions)
       - [useDevicePixelRatio](#usedevicepixelratio)
+      - [usePlayerObjectFit](#useplayerobjectfit)
       - [customPixelRatio](#custompixelratio)
       - [allowSeeksWithinUnsafeLiveWindow](#allowseekswithinunsafelivewindow)
       - [customTagParsers](#customtagparsers)
@@ -414,6 +415,17 @@ This setting is `true` by default.
 If true, this will take the device pixel ratio into account when doing rendition switching. This means that if you have a player with the width of `540px` in a high density display with a device pixel ratio of 2, a rendition of `1080p` will be allowed.
 This setting is `false` by default.
 
+
+##### usePlayerObjectFit
+* Type: `boolean`
+* can be used as an initialization option.
+
+If true, the video element's `object-fit` CSS property will be taken
+into account when doing rendition switching. This ensures that a
+suitable rendition is selected for videos that are scaled up to cover
+the media element. This setting is `false` by default.
+
+
 ##### customPixelRatio
 * Type: `number`
 * can be used as an initialization option.
@@ -426,6 +438,7 @@ It is worth noting that if the player dimension multiplied by the custom pixel r
 
 If `useDevicePixelRatio` is set to `true`, the custom pixel ratio will be prioritized and overwrite any previous pixel ratio.
 
+
 ##### allowSeeksWithinUnsafeLiveWindow
 * Type: `boolean`
 * can be used as a source option

index.html

@@ -172,6 +172,11 @@
           <label class="form-check-label" for="pixel-diff-selector">[EXPERIMENTAL] Use the Pixel difference resolution selector (reloads player)</label>
         </div>
 
+        <div class="form-check">
+          <input id=object-fit type="checkbox" class="form-check-input">
+          <label class="form-check-label" for="object-fit">Account Object-fit for resolution selection (reloads player)</label>
+        </div>
+
         <div class="form-check">
           <input id=override-native type="checkbox" class="form-check-input" checked>
           <label class="form-check-label" for="override-native">Override Native (reloads player)</label>

scripts/index.js

@@ -471,6 +471,7 @@
     'network-info',
     'dts-offset',
     'override-native',
+    'object-fit',
     'use-mms',
     'preload',
     'mirror-source',
@@ -523,6 +524,7 @@
       'llhls',
       'buffer-water',
       'override-native',
+      'object-fit',
       'use-mms',
       'liveui',
       'pixel-diff-selector',
@@ -615,7 +617,8 @@
               leastPixelDiffSelector: getInputValue(stateEls['pixel-diff-selector']),
               useNetworkInformationApi: getInputValue(stateEls['network-info']),
               useDtsForTimestampOffset: getInputValue(stateEls['dts-offset']),
-              useForcedSubtitles: getInputValue(stateEls['forced-subtitles'])
+              useForcedSubtitles: getInputValue(stateEls['forced-subtitles']),
+              usePlayerObjectFit: getInputValue(stateEls['object-fit'])
             }
           }
         });

src/playlist-selectors.js

@@ -133,30 +133,38 @@ export const comparePlaylistResolution = function(left, right) {
 /**
  * Chooses the appropriate media playlist based on bandwidth and player size
  *
- * @param {Object} main
+ * @param {Object} settings
+ *        Object of information required to use this selector
+ * @param {Object} settings.main
  *        Object representation of the main manifest
- * @param {number} playerBandwidth
+ * @param {number} settings.bandwidth
  *        Current calculated bandwidth of the player
- * @param {number} playerWidth
+ * @param {number} settings.playerWidth
  *        Current width of the player element (should account for the device pixel ratio)
- * @param {number} playerHeight
+ * @param {number} settings.playerHeight
  *        Current height of the player element (should account for the device pixel ratio)
- * @param {boolean} limitRenditionByPlayerDimensions
+ * @param {number} settings.playerObjectFit
+ *        Current value of the video element's object-fit CSS property. Allows taking into
+ *        account that the video might be scaled up to cover the media element when selecting
+ *        media playlists based on player size.
+ * @param {boolean} settings.limitRenditionByPlayerDimensions
  *        True if the player width and height should be used during the selection, false otherwise
- * @param {Object} playlistController
+ * @param {Object} settings.playlistController
  *        the current playlistController object
  * @return {Playlist} the highest bitrate playlist less than the
  * currently detected bandwidth, accounting for some amount of
  * bandwidth variance
  */
-export let simpleSelector = function(
-  main,
-  playerBandwidth,
-  playerWidth,
-  playerHeight,
-  limitRenditionByPlayerDimensions,
-  playlistController
-) {
+export let simpleSelector = function(settings) {
+  const {
+    main,
+    bandwidth: playerBandwidth,
+    playerWidth,
+    playerHeight,
+    playerObjectFit,
+    limitRenditionByPlayerDimensions,
+    playlistController
+  } = settings;
 
   // If we end up getting called before `main` is available, exit early
   if (!main) {
@@ -271,7 +279,18 @@ export let simpleSelector = function(
   // find the smallest variant that is larger than the player
   // if there is no match of exact resolution
   if (!resolutionBestRep) {
-    resolutionPlusOneList = haveResolution.filter((rep) => rep.width > playerWidth || rep.height > playerHeight);
+    resolutionPlusOneList = haveResolution.filter((rep) => {
+      if (playerObjectFit === 'cover') {
+        // video will be scaled up to cover the player. We need to
+        // make sure rendition is at least as wide and as high as the
+        // player.
+        return rep.width > playerWidth && rep.height > playerHeight;
+      }
+
+      // video will be scaled down to fit inside the player soon as
+      // its resolution exceeds player size in at least one dimension.
+      return rep.width > playerWidth || rep.height > playerHeight;
+    });
 
     // find all the variants have the same smallest resolution
     resolutionPlusOneSmallest = resolutionPlusOneList.filter((rep) => rep.width === resolutionPlusOneList[0].width &&
@@ -370,14 +389,15 @@ export const lastBandwidthSelector = function() {
     pixelRatio = this.customPixelRatio;
   }
 
-  return simpleSelector(
-    this.playlists.main,
-    this.systemBandwidth,
-    parseInt(safeGetComputedStyle(this.tech_.el(), 'width'), 10) * pixelRatio,
-    parseInt(safeGetComputedStyle(this.tech_.el(), 'height'), 10) * pixelRatio,
-    this.limitRenditionByPlayerDimensions,
-    this.playlistController_
-  );
+  return simpleSelector({
+    main: this.playlists.main,
+    bandwidth: this.systemBandwidth,
+    playerWidth: parseInt(safeGetComputedStyle(this.tech_.el(), 'width'), 10) * pixelRatio,
+    playerHeight: parseInt(safeGetComputedStyle(this.tech_.el(), 'height'), 10) * pixelRatio,
+    playerObjectFit: this.usePlayerObjectFit ? safeGetComputedStyle(this.tech_.el(), 'objectFit') : '',
+    limitRenditionByPlayerDimensions: this.limitRenditionByPlayerDimensions,
+    playlistController: this.playlistController_
+  });
 };
 
 /**
@@ -425,14 +445,15 @@ export const movingAverageBandwidthSelector = function(decay) {
       lastSystemBandwidth = this.systemBandwidth;
     }
 
-    return simpleSelector(
-      this.playlists.main,
-      average,
-      parseInt(safeGetComputedStyle(this.tech_.el(), 'width'), 10) * pixelRatio,
-      parseInt(safeGetComputedStyle(this.tech_.el(), 'height'), 10) * pixelRatio,
-      this.limitRenditionByPlayerDimensions,
-      this.playlistController_
-    );
+    return simpleSelector({
+      main: this.playlists.main,
+      bandwidth: average,
+      playerWidth: parseInt(safeGetComputedStyle(this.tech_.el(), 'width'), 10) * pixelRatio,
+      playerHeight: parseInt(safeGetComputedStyle(this.tech_.el(), 'height'), 10) * pixelRatio,
+      playerObjectFit: this.usePlayerObjectFit ? safeGetComputedStyle(this.tech_.el(), 'objectFit') : '',
+      limitRenditionByPlayerDimensions: this.limitRenditionByPlayerDimensions,
+      playlistController: this.playlistController_
+    });
   };
 };
 

src/videojs-http-streaming.js

@@ -691,6 +691,7 @@ class VhsHandler extends Component {
     this.options_.withCredentials = this.options_.withCredentials || false;
     this.options_.limitRenditionByPlayerDimensions = this.options_.limitRenditionByPlayerDimensions === false ? false : true;
     this.options_.useDevicePixelRatio = this.options_.useDevicePixelRatio || false;
+    this.options_.usePlayerObjectFit = this.options_.usePlayerObjectFit || false;
     this.options_.useBandwidthFromLocalStorage =
       typeof this.source_.useBandwidthFromLocalStorage !== 'undefined' ?
         this.source_.useBandwidthFromLocalStorage :
@@ -739,6 +740,7 @@ class VhsHandler extends Component {
     [
       'withCredentials',
       'useDevicePixelRatio',
+      'usePlayerObjectFit',
       'customPixelRatio',
       'limitRenditionByPlayerDimensions',
       'bandwidth',
@@ -763,6 +765,7 @@ class VhsHandler extends Component {
 
     this.limitRenditionByPlayerDimensions = this.options_.limitRenditionByPlayerDimensions;
     this.useDevicePixelRatio = this.options_.useDevicePixelRatio;
+    this.usePlayerObjectFit = this.options_.usePlayerObjectFit;
 
     const customPixelRatio = this.options_.customPixelRatio;
 

test/configuration.test.js

@@ -31,6 +31,12 @@ const options = [{
   default: false,
   test: true,
   alt: false
+}, {
+  name: 'usePlayerObjectFit',
+  default: false,
+  test: true,
+  alt: false
+}, {
 },
 {
   name: 'customPixelRatio',