From 3519ecbd6200fa4ea7bd52850703099468694c8d Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Fri, 1 Mar 2024 11:52:03 -0500 Subject: [PATCH 01/15] feat: Command and Control Intents --- .../intents/command-and-control.md | 783 ++++++++++++++++++ 1 file changed, 783 insertions(+) create mode 100644 requirements/specifications/intents/command-and-control.md diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md new file mode 100644 index 000000000..e44ee6148 --- /dev/null +++ b/requirements/specifications/intents/command-and-control.md @@ -0,0 +1,783 @@ +# Command and Control Intents + +Document Status: Proposed Specification + +See [Firebolt Requirements Governance](../../governance.md) for more info. + +| Contributor | Organization | +| ---------------- | ------------ | +| Saras Arveti | Comcast | +| Eileen Bengston | Comcast | +| Michael Driscoll | Comcast | +| Simon Grist | Sky | +| Jeremy LaCivita | Comcast | + +## 1. Overview + +This document outlines several basic Intents for controlling a Firebolt +compliant device. + +### 1.1. Message.type + +Message.type should be a useful grouping to bucket related intents +together for easier forwarding to appropriate components. + +## 2. Table of Contents +- [1. Overview](#1-overview) + - [1.1. Message.type](#11-messagetype) +- [2. Table of Contents](#2-table-of-contents) +- [3. Control Intents](#3-control-intents) + - [3.1. Power Intent](#31-power-intent) + - [3.2. Volume Intents](#32-volume-intents) + - [3.2.1. Volume Intent](#321-volume-intent) + - [3.2.2. Mute Intent](#322-mute-intent) + - [3.3. Channel Intent](#33-channel-intent) + - [3.4. Media Control Intents](#34-media-control-intents) + - [3.4.1. Pause, Resume, Replay, and Stop Intents](#341-pause-resume-replay-and-stop-intents) + - [3.4.2. Seek Intent](#342-seek-intent) + - [3.4.3. Trick Play Intent](#343-trick-play-intent) + - [3.5. Accessibility Intents](#35-accessibility-intents) + - [3.5.1. Closed Captions Intent](#351-closed-captions-intent) + - [3.5.2. Voice Guidance Intent](#352-voice-guidance-intent) + - [3.5.3. Audio Descritions Intent](#353-audio-descritions-intent) + - [3.6. Interaction Intents](#36-interaction-intents) + - [3.6.1. Focus Intent](#361-focus-intent) + - [3.6.2. Select Intent](#362-select-intent) + - [3.6.3. Scroll Intent](#363-scroll-intent) + - [3.6.4. Back Intent](#364-back-intent) + - [3.6.5. Exit Intent](#365-exit-intent) +- [4. Section Intents](#4-section-intents) + - [4.1. Content Discovery Section Intents](#41-content-discovery-section-intents) + - [4.2. Device Settings Launch Intent](#42-device-settings-launch-intent) +- [5. Volume Notes](#5-volume-notes) + - [5.1. Jump Notes](#51-jump-notes) + - [5.2. Core SDK APIs](#52-core-sdk-apis) + +## 3. Control Intents + +Control intents are for user intentions that will be needed regardless +of whether there are any apps installed. + +For example, these intents are all useful even if only using your TV +with a single HDMI input, and not for apps. + +### 3.1. Power Intent + +This intent allows a user to turn the device on or off. + +```json +{ + "type": "xrn:firebolt:intent:platform:power", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "power", + "context": { + "source": "voice" + }, + "data": { + "value": true \| false + } + } +} +``` + +Additionally, this intent allows a user to set a timer for turning off +the power, aka a "sleep timer." + +This is handled by the optional field delay, which is measured in whole +seconds: + +```json +{ + "type": "xrn:firebolt:intent:platform:power", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "power", + "context": { + "source": "voice" + }, + "data": { + "value": true \| false, + "delay": 3600 + } + } +} + +``` + +While it may not be implemented by all platforms, this could also be +used to turn on the TV with a timer. + +### 3.2. Volume Intents + +Volume Intents control the audio level of the device. + +#### 3.2.1. Volume Intent + +This intent allows setting the volume to an absolute or relative value. + +```json +{ + "type": "xrn:firebolt:intent:platform:volume", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "volume", + "context": { + "source": "VOICE" + }, + "data": { + "value": 70 + } + } +} + +``` + +The value is an integer value from 0 to 100. + +This intent also supports relative volume changes, by providing the +optional relative field: + +```json +{ + "type": "xrn:firebolt:intent:platform:volume", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "volume", + "context": { + "source": "VOICE" + }, + "data": { + "value": -10, + "relative": true + } + } +} + +``` + +The value is a positive or negative integer that is relative to a scale +of 0-100. + +Firebolt will not support complicated relative changes, e.g. "Set the +volume to 50% *of what it currently is\...*" + +**Note**: Both Google & Alexa support either 0-10 or 0% to 100%. + +Firebolt uses a scale of 0-100 for this intent. It\'s up to each voice +integration if it wants to convert "5" to "50%" before generating +the intent, but convenience transformations like this are recommended. + +#### 3.2.2. Mute Intent + +This intent allows the user to mute or unmute the device. + +```json +{ + "type": "xrn:firebolt:intent:platform:volume", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "mute", + "context": { + "source": "VOICE" + }, + "data": { + "value": true \| false + } + } +} +``` + +### 3.3. Channel Intent + +For tuning to a specific channel, either OTA or in-app, see [Tune +Intents](./tune.md). + +The intents in this section are for relative next/previous channel user +intentions and are a separate type of Intent. This allows each app to +decide what "channel" means. For example, an App might simply take you +to the next section/genre if it doesn\'t have linear streams in it\'s +catalog. + +The goal of the action property to is tell the client how to parse the +Intent, so overloading the tune intent with a different structure is not +desirable. + +Also, +"tune" inherently means to zero in on a specific part of a +scale, e.g. tuning a harp. + +For relative "channel surfing" we\'ll use the more content-centric +action "channel" which will also align with non-linear apps that want +to leverage the channel up/down intent. + +The Channel Intent allows a user to scan "channels" in an app (or +actual OTA channels if not in an app). + +Users can scan to the next or previous channel. For scanning to the most +recent, i.e. "Last" channel, see [Interaction +Intent +](#interaction-intents). + +```json +{ + "type": "xrn:firebolt:intent:platform:channel", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "channel", + "context": { + "source": "voice" + }, + "data": { + "value": "next" \| "previous" + } + } +} + +``` + +The value property MUST always be "next" or "previous". These are +chosen over up/down since not all use cases will be numeric. + +Since this intent is always relative to the current app, there is no +need for an appId. + +If this Intent needs to be passed to the current app, it can be passed +as-is, via the Discovery. onNavigateTo API, or a simulated RCU press of +one of the channel up/down buttons. + +### 3.4. Media Control Intents + +#### 3.4.1. Pause, Resume, Replay, and Stop Intents + +These intents allow the user to pause and resume playback of the current +Media: + +```json +{ + "type": "xrn:firebolt:intent:platform:media-control", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "pause" \| "resume" \| "replay" \| "stop", + "context": { + "source": "voice" + } + } +} +``` + +**TODO**: this is different than what we officially published. We need +to track down if anyone implemented the old Pause Intent. + +If the action is pause, then the currently playing media should be +paused, with the frames on-screen and the video decoder ready to resume. + +If the action is resume, then the currently paused media should resume. + +If the action is replay, then the currently paused media should restart +from the beginning. This should work even if the decoder has finished, +and its resources have been released. + +If the action is stop, then the currently playing media should be +stopped, frames removed from the screen, and any decoder resources +should be released. + +#### 3.4.2. Seek Intent + +The seek intent allows users to jump to a relative or absolute position +in the currently playing media. + +```json +{ + "type": "xrn:firebolt:intent:platform:media-control", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "seek", + "data": { + "seconds": 3600 + }, + "context": { + "source": "voice" + } + } +} + +``` + +The seconds value is a positive integer representing where to seek. + +This intent also supports relative seeking, by providing the optional +relative field: + +```json +{ + "type": "xrn:firebolt:intent:platform:media-control", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "seek", + "data": { + "seconds": -30, + "relative": true + },s + +"context": { + "source": "voice" + } + } +} + +``` + +For relative seeking, the seconds value may be a positive or negative +value. + +#### 3.4.3. Trick Play Intent + +The Trick Play Intent allows users to fast-forward or rewind: + +```json +{ + "type": "xrn:firebolt:intent:platform:media-control", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "trickplay", + "context": { + "source": "voice" + }, + "data": { + "speed": 2.5 + } + } +} + +``` + +Speed is a float in the range of -10 to 10, with negative values +denoting backwards/rewind and values between -1 and 1 denoting slow +motion. + +It is a device-level decision how to implement different speeds, however +actual fast playback (with audio) should be used where possible and +reasonable, e.g. a speed of 1.5 should actually be playing the video w/ +sync\'d audio, while a speed of 10 will likely be using iframes and not +have audio. For speeds less than zero it is not important, and likely +undesirable, to provide audio. + +### 3.5. Accessibility Intents + +These intents manipulate accessibility features on the device. + +#### 3.5.1. Closed Captions Intent + +This intent allows a user to turn closed captions on or off. + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "closedCaptions", + "context": { + "source": "voice" + }, + "data": { + "value": true \| false + } + } +} + +``` + +Additionally, this intent may specify a toggle: + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "closedCaptions", + "context": { + "source": "voice" + }, + "data": { + "toggle": true + } + } +} +``` + +#### 3.5.2. Voice Guidance Intent + +This intent allows a user to turn voice guidance on or off. + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "voiceGuidance", + "context": { + "source": "voice" + }, + "data": { + "value": true \| false + } + } +} + +``` + +Additionally, this intent may specify a toggle: + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "voiceGuidance", + "context": { + "source": "voice" + }, + "data": { + "toggle": true + } + } +} +``` + +#### 3.5.3. Audio Descritions Intent + +This intent allows a user to turn voice guidance on or off. + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "audioDescription", + "context": { + "source": "voice" + }, + "data": { + "value": true \| false + } + } +} + +``` + +Additionally, this intent may specify a toggle: + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "audioDescription", + "context": { + "source": "voice" + }, + "data": { + "toggle": true + } + } +} +``` + +### 3.6. Interaction Intents + +Interaction Intents allow for voice (or other upstream intent service) +to control an on-screen UI without need for a keyboard or remote. + +#### 3.6.1. Focus Intent + +The Focus Intent allows users to move the focus / cursor +up/down/left/right: + +```json +{ + "type": "xrn:firebolt:intent:platform:interaction", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "focus", + "context": { + "source": "voice" + }, + "data": { + "direction": "up" \| "down" \| "left" \| "right" + } + } +} + +``` + +Note that this does not give focus to a particular app, which is handled +by the "launch" action. + +These Intents will generate appropriate HTML browser keyCode events to +facilitate up/down/left/right key presses. + +#### 3.6.2. Select Intent + +The select intent allows users to tell an app select, e.g., +"click" on +whatever is focused. This is a platform-level intent that effectively +sends the "Ok" or "Select" key to the current app. + +**NOTE**: i don\'t like using "click" here, but i can\'t think of a +good word that isn\'t super vague. "Select" means highlighting, not +actually hitting okay, so we can\'t use that. I considered "activate" +but that seems odd\... Open to discussing this. + +```json +{ + "type": "xrn:firebolt:intent:platform:interaction", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "select", + "context": { + "source": "voice" + } + } +} +``` + +#### 3.6.3. Scroll Intent + +The Scroll Intent allows users to move the current view port +up/down/left/right: + +```json +{ + "type": "xrn:firebolt:intent:platform:interaction", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "scroll", + "context": { + "source": "voice" + }, + "data": { + "direction": "up" \| "down" \| "left" \| "right", + "unit": "page" \| "line" \| "percent", + "distance": 2.5 + } + } +} + +``` + +The distance is a float that represents how many of units to scroll. All +three data properties are required. + +These Intents will generate appropriate browser / DOM scrolling +operations that don\'t require custom APIs. + +#### 3.6.4. Back Intent + +The back intent allows users to tell an app go to "back" like a +browser. This is a platform-level intent and will initiate a browser +back flow for web apps. For native apps, this will be converted to an +app Navigation Intent by the client and surfaced through the navigateTo +API. + +```json +{ + "type": "xrn:firebolt:intent:platform:interaction", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "back", + "context": { + "source": "voice" + } + } +} +``` + +#### 3.6.5. Exit Intent + +The exit intent allows users to tell an app close. This is a +platform-level intent and will simply move the current app into the +inactive state. + +```json +{ + "type": "xrn:firebolt:intent:platform:interaction", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "exit", + "context": { + "source": "voice" + } + } +} +``` + +## 4. Section Intents + +The section intent is already defined as part of Firebolt. + +### 4.1. Content Discovery Section Intents + +The following section IDs will be used, with the Firebolt application +type: + +xrn:firebolt:application-type:main + +| Section | Description | +| ---------- | ------------------------------------------------------------------- | +| guide | Launches a program guide, typically linear, in the main experience. | +| menu | Launches a top-level menu, or home page, of the main experience. | +| recordings | Launches a list of user recordings in the main experience. | +| favorites | Launches a list of user favorites in the main experience. | +| purchases | Launches a list of user purchased content in the main experience. | +| playlist | Launches a list of content the user has saved for future playback. | +| recent | Launches a list of recently watched content. | + +Note that not every Firebolt Distributor will have all of these +features. For unsupported features, these intents should simply launch +the main aggregated experience UI. + +### 4.2. Device Settings Launch Intent + +To launch the settings UI, a Launch Intent will be used, with the +Firebolt application type: + +xrn:firebolt:application-type:settings + +Currently there are no standardized sections for the settings App. + +## 5. Volume Notes + +Whether or not a TV uses logarithmic or linear scale is irrelevant to +the Intent schema. + +### 5.1. Jump Notes + +For the feature of instantly moving forward/backwards by \ seconds I +prefer *not* to call this "skip" because in the AVOD world, +"skip" +means to completely bypass an add or chapter (like next track of a CD +player). I\'ve used "jump" in the past, but I\'m open to other names. +We could go with seek, and have a relative and absolute version? + +### 5.2. Core SDK APIs + +We need SDK APIs for some of these, since they\'re potentially surfaced +to apps, e.g. channel scanning. + +Need App-facing SDK APIs for: + +- Seek (otherwise how to pass the value) - New API, maybe + MediaControl.onSeek + +- TrickPlay (how to pass the speed) - New API, maybe + MediaControl.onTrickPlay + +- Channel (how to pass direction) - Discovery.onNavigateTo + +Everything else is handled by the OS, i think. From 313daaa78c5b7148b8f86349183f6e888f1af6a6 Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Fri, 1 Mar 2024 12:45:01 -0500 Subject: [PATCH 02/15] fix: Clean up JSON examples --- .../intents/command-and-control.md | 103 +++++++++--------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index e44ee6148..e217c6b77 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -76,11 +76,11 @@ This intent allows a user to turn the device on or off. }, "intent": { "action": "power", + "data": { + "value": true | false + }, "context": { "source": "voice" - }, - "data": { - "value": true \| false } } } @@ -103,12 +103,12 @@ seconds: }, "intent": { "action": "power", - "context": { - "source": "voice" - }, "data": { - "value": true \| false, + "value": true | false, "delay": 3600 + }, + "context": { + "source": "voice" } } } @@ -137,11 +137,11 @@ This intent allows setting the volume to an absolute or relative value. }, "intent": { "action": "volume", - "context": { - "source": "VOICE" - }, "data": { "value": 70 + }, + "context": { + "source": "VOICE" } } } @@ -164,12 +164,12 @@ optional relative field: }, "intent": { "action": "volume", - "context": { - "source": "VOICE" - }, "data": { "value": -10, "relative": true + }, + "context": { + "source": "VOICE" } } } @@ -203,11 +203,11 @@ This intent allows the user to mute or unmute the device. }, "intent": { "action": "mute", + "data": { + "value": true | false + }, "context": { "source": "VOICE" - }, - "data": { - "value": true \| false } } } @@ -255,11 +255,11 @@ Intent }, "intent": { "action": "channel", + "data": { + "value": "next" | "previous" + }, "context": { "source": "voice" - }, - "data": { - "value": "next" \| "previous" } } } @@ -293,7 +293,7 @@ Media: "micType": "NEAR_FIELD" }, "intent": { - "action": "pause" \| "resume" \| "replay" \| "stop", + "action": "pause" | "resume" | "replay" | "stop", "context": { "source": "voice" } @@ -363,9 +363,8 @@ relative field: "data": { "seconds": -30, "relative": true - },s - -"context": { + }, + "context": { "source": "voice" } } @@ -391,11 +390,11 @@ The Trick Play Intent allows users to fast-forward or rewind: }, "intent": { "action": "trickplay", - "context": { - "source": "voice" - }, "data": { "speed": 2.5 + }, + "context": { + "source": "voice" } } } @@ -432,11 +431,11 @@ This intent allows a user to turn closed captions on or off. }, "intent": { "action": "closedCaptions", + "data": { + "value": true | false + }, "context": { "source": "voice" - }, - "data": { - "value": true \| false } } } @@ -456,11 +455,11 @@ Additionally, this intent may specify a toggle: }, "intent": { "action": "closedCaptions", - "context": { - "source": "voice" - }, "data": { "toggle": true + }, + "context": { + "source": "voice" } } } @@ -481,11 +480,11 @@ This intent allows a user to turn voice guidance on or off. }, "intent": { "action": "voiceGuidance", + "data": { + "value": true | false + }, "context": { "source": "voice" - }, - "data": { - "value": true \| false } } } @@ -505,11 +504,11 @@ Additionally, this intent may specify a toggle: }, "intent": { "action": "voiceGuidance", - "context": { - "source": "voice" - }, "data": { "toggle": true + }, + "context": { + "source": "voice" } } } @@ -530,11 +529,11 @@ This intent allows a user to turn voice guidance on or off. }, "intent": { "action": "audioDescription", + "data": { + "value": true | false + }, "context": { "source": "voice" - }, - "data": { - "value": true \| false } } } @@ -554,11 +553,11 @@ Additionally, this intent may specify a toggle: }, "intent": { "action": "audioDescription", - "context": { - "source": "voice" - }, "data": { "toggle": true + }, + "context": { + "source": "voice" } } } @@ -585,11 +584,11 @@ up/down/left/right: }, "intent": { "action": "focus", + "data": { + "direction": "up" | "down" | "left" | "right" + }, "context": { "source": "voice" - }, - "data": { - "direction": "up" \| "down" \| "left" \| "right" } } } @@ -648,13 +647,13 @@ up/down/left/right: }, "intent": { "action": "scroll", - "context": { - "source": "voice" - }, "data": { - "direction": "up" \| "down" \| "left" \| "right", - "unit": "page" \| "line" \| "percent", + "direction": "up" | "down" | "left" | "right", + "unit": "page" | "line" | "percent", "distance": 2.5 + }, + "context": { + "source": "voice" } } } From 8f04700d739fb1b2ecb8330b9136f6d9ffd801d1 Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Fri, 1 Mar 2024 12:45:29 -0500 Subject: [PATCH 03/15] feat: Adding Text Magnification Intent --- .../intents/command-and-control.md | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index e217c6b77..50a098ff1 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -40,6 +40,7 @@ together for easier forwarding to appropriate components. - [3.5.1. Closed Captions Intent](#351-closed-captions-intent) - [3.5.2. Voice Guidance Intent](#352-voice-guidance-intent) - [3.5.3. Audio Descritions Intent](#353-audio-descritions-intent) + - [3.5.4. Text Magnification Intent](#354-text-magnification-intent) - [3.6. Interaction Intents](#36-interaction-intents) - [3.6.1. Focus Intent](#361-focus-intent) - [3.6.2. Select Intent](#362-select-intent) @@ -563,6 +564,55 @@ Additionally, this intent may specify a toggle: } ``` +#### 3.5.4. Text Magnification Intent + +This intent allows a user to turn text magnification on or off. + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "textMagnification", + "data": { + "value": true | false + }, + "context": { + "source": "voice" + } + } +} + +``` + +Additionally, this intent may specify a toggle: + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "textMagnification", + "data": { + "toggle": true + }, + "context": { + "source": "voice" + } + } +} +``` + ### 3.6. Interaction Intents Interaction Intents allow for voice (or other upstream intent service) From d4636b8293d505784fd0170c468078a9250ac1fa Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Fri, 1 Mar 2024 12:55:57 -0500 Subject: [PATCH 04/15] fix: Adding scale to magnification --- .../intents/command-and-control.md | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index 50a098ff1..8f92f7342 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -185,7 +185,7 @@ volume to 50% *of what it currently is\...*" **Note**: Both Google & Alexa support either 0-10 or 0% to 100%. -Firebolt uses a scale of 0-100 for this intent. It\'s up to each voice +Firebolt uses a size of 0-100 for this intent. It\'s up to each voice integration if it wants to convert "5" to "50%" before generating the intent, but convenience transformations like this are recommended. @@ -613,6 +613,35 @@ Additionally, this intent may specify a toggle: } ``` +Finally, this intent may specify a magnification scale as a number: + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "textMagnification", + "data": { + "scale": 2.5 + }, + "context": { + "source": "voice" + } + } +} +``` + +Setting the scale to `1` turns off magnification. Setting the scale to a value greater than 1 turns on magnification. + +Even if a Firebolt platform does not support specifying the numeric scale, it **MUST** turn magnifacation on and off based on them. + +The magnification intent **MUST** have only one property, `scale`, `toggle`, `value` and **MUST NOT** comebine them in a single intent. + ### 3.6. Interaction Intents Interaction Intents allow for voice (or other upstream intent service) From 77a0797b3e6d00d69d0ff325e99d6f8c903514e0 Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Fri, 1 Mar 2024 12:57:03 -0500 Subject: [PATCH 05/15] feat: Added High Contrast --- .../intents/command-and-control.md | 54 ++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index 8f92f7342..f32c98b02 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -40,7 +40,8 @@ together for easier forwarding to appropriate components. - [3.5.1. Closed Captions Intent](#351-closed-captions-intent) - [3.5.2. Voice Guidance Intent](#352-voice-guidance-intent) - [3.5.3. Audio Descritions Intent](#353-audio-descritions-intent) - - [3.5.4. Text Magnification Intent](#354-text-magnification-intent) + - [3.5.4. High Contrast Intent](#354-high-contrast-intent) + - [3.5.5. Text Magnification Intent](#355-text-magnification-intent) - [3.6. Interaction Intents](#36-interaction-intents) - [3.6.1. Focus Intent](#361-focus-intent) - [3.6.2. Select Intent](#362-select-intent) @@ -564,7 +565,56 @@ Additionally, this intent may specify a toggle: } ``` -#### 3.5.4. Text Magnification Intent +#### 3.5.4. High Contrast Intent + +This intent allows a user to turn high contrast mode on or off. + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "highContrast", + "data": { + "value": true | false + }, + "context": { + "source": "voice" + } + } +} + +``` + +Additionally, this intent may specify a toggle: + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "highContrast", + "data": { + "toggle": true + }, + "context": { + "source": "voice" + } + } +} +``` + +#### 3.5.5. Text Magnification Intent This intent allows a user to turn text magnification on or off. From bfff702b69b436713b2a0055bac107c815ce9230 Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Tue, 12 Mar 2024 11:15:56 -0400 Subject: [PATCH 06/15] fix: Rationalize existing intents --- .../intents/command-and-control.md | 141 +++++++++++------- 1 file changed, 85 insertions(+), 56 deletions(-) diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index f32c98b02..df0d50e34 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -41,7 +41,7 @@ together for easier forwarding to appropriate components. - [3.5.2. Voice Guidance Intent](#352-voice-guidance-intent) - [3.5.3. Audio Descritions Intent](#353-audio-descritions-intent) - [3.5.4. High Contrast Intent](#354-high-contrast-intent) - - [3.5.5. Text Magnification Intent](#355-text-magnification-intent) + - [3.5.5. Screen Magnification Intent](#355-screen-magnification-intent) - [3.6. Interaction Intents](#36-interaction-intents) - [3.6.1. Focus Intent](#361-focus-intent) - [3.6.2. Select Intent](#362-select-intent) @@ -51,9 +51,6 @@ together for easier forwarding to appropriate components. - [4. Section Intents](#4-section-intents) - [4.1. Content Discovery Section Intents](#41-content-discovery-section-intents) - [4.2. Device Settings Launch Intent](#42-device-settings-launch-intent) -- [5. Volume Notes](#5-volume-notes) - - [5.1. Jump Notes](#51-jump-notes) - - [5.2. Core SDK APIs](#52-core-sdk-apis) ## 3. Control Intents @@ -120,6 +117,8 @@ seconds: While it may not be implemented by all platforms, this could also be used to turn on the TV with a timer. +**TODO**: Current spec supports toggle. i think we can drop this, though. + ### 3.2. Volume Intents Volume Intents control the audio level of the device. @@ -181,15 +180,18 @@ optional relative field: The value is a positive or negative integer that is relative to a scale of 0-100. +**TODO**: Firebolt currently specifies the value as 0-1 + Firebolt will not support complicated relative changes, e.g. "Set the volume to 50% *of what it currently is\...*" -**Note**: Both Google & Alexa support either 0-10 or 0% to 100%. - Firebolt uses a size of 0-100 for this intent. It\'s up to each voice integration if it wants to convert "5" to "50%" before generating the intent, but convenience transformations like this are recommended. +Whether or not a TV uses logarithmic or linear scale is irrelevant to +the VolumeIntent schema. + #### 3.2.2. Mute Intent This intent allows the user to mute or unmute the device. @@ -303,8 +305,7 @@ Media: } ``` -**TODO**: this is different than what we officially published. We need -to track down if anyone implemented the old Pause Intent. +**TODO**: this is different than what we officially published. We need to track down if anyone implemented the old Pause Intent. If so, we need to support pause w/ value: false to mean resume. If the action is pause, then the currently playing media should be paused, with the frames on-screen and the video decoder ready to resume. @@ -343,7 +344,6 @@ in the currently playing media. } } } - ``` The seconds value is a positive integer representing where to seek. @@ -371,12 +371,13 @@ relative field: } } } - ``` For relative seeking, the seconds value may be a positive or negative value. +**TODO**: currently published spec has seconds always positive, and a `direction` enum... can we drop that? + #### 3.4.3. Trick Play Intent The Trick Play Intent allows users to fast-forward or rewind: @@ -422,6 +423,8 @@ These intents manipulate accessibility features on the device. This intent allows a user to turn closed captions on or off. +**TODO**: there is no hyphen in closed captions because it was already spec'd w/out one previously. Need to check if any platforms implemented it or not. + ```json { "type": "xrn:firebolt:intent:platform:accessibility", @@ -432,7 +435,7 @@ This intent allows a user to turn closed captions on or off. "micType": "NEAR_FIELD" }, "intent": { - "action": "closedCaptions", + "action": "closedcaptions", "data": { "value": true | false }, @@ -456,7 +459,7 @@ Additionally, this intent may specify a toggle: "micType": "NEAR_FIELD" }, "intent": { - "action": "closedCaptions", + "action": "closedcaptions", "data": { "toggle": true }, @@ -481,7 +484,7 @@ This intent allows a user to turn voice guidance on or off. "micType": "NEAR_FIELD" }, "intent": { - "action": "voiceGuidance", + "action": "voice-guidance", "data": { "value": true | false }, @@ -505,7 +508,7 @@ Additionally, this intent may specify a toggle: "micType": "NEAR_FIELD" }, "intent": { - "action": "voiceGuidance", + "action": "voice-guidance", "data": { "toggle": true }, @@ -516,9 +519,64 @@ Additionally, this intent may specify a toggle: } ``` +The intent **MAY** specify `speed` `number` property that specifies a speed from 0 to 10: + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "voice-guidance", + "data": { + "speed": 2 + }, + "context": { + "source": "voice" + } + } +} +``` + +The voice guidance intent **MUST** have only one property, `speed`, `toggle`, `value` and **MUST NOT** comebine them in a single intent. + +Finally, the intent **MAY** specify a `verbosity` property, which **MUST** use one of the following values is provided: + +- `low` +- `high` + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "voice-guidance", + "data": { + "value": true, + "verbosity": "low" + }, + "context": { + "source": "voice" + } + } +} +``` + #### 3.5.3. Audio Descritions Intent -This intent allows a user to turn voice guidance on or off. +This intent allows a user to turn audio descriptions of content on or off. + +**TODO**: there is no hyphen in closed captions because it was already spec'd w/out one previously. Need to check if any platforms implemented it or not. + ```json { @@ -530,7 +588,7 @@ This intent allows a user to turn voice guidance on or off. "micType": "NEAR_FIELD" }, "intent": { - "action": "audioDescription", + "action": "audiodescriptions", "data": { "value": true | false }, @@ -554,7 +612,7 @@ Additionally, this intent may specify a toggle: "micType": "NEAR_FIELD" }, "intent": { - "action": "audioDescription", + "action": "audiodescriptions", "data": { "toggle": true }, @@ -579,7 +637,7 @@ This intent allows a user to turn high contrast mode on or off. "micType": "NEAR_FIELD" }, "intent": { - "action": "highContrast", + "action": "high-contrast", "data": { "value": true | false }, @@ -603,7 +661,7 @@ Additionally, this intent may specify a toggle: "micType": "NEAR_FIELD" }, "intent": { - "action": "highContrast", + "action": "high-contrast", "data": { "toggle": true }, @@ -614,9 +672,9 @@ Additionally, this intent may specify a toggle: } ``` -#### 3.5.5. Text Magnification Intent +#### 3.5.5. Screen Magnification Intent -This intent allows a user to turn text magnification on or off. +This intent allows a user to turn screen magnification on or off. ```json { @@ -628,7 +686,7 @@ This intent allows a user to turn text magnification on or off. "micType": "NEAR_FIELD" }, "intent": { - "action": "textMagnification", + "action": "screen-magnification", "data": { "value": true | false }, @@ -652,7 +710,7 @@ Additionally, this intent may specify a toggle: "micType": "NEAR_FIELD" }, "intent": { - "action": "textMagnification", + "action": "screen-magnification", "data": { "toggle": true }, @@ -675,7 +733,7 @@ Finally, this intent may specify a magnification scale as a number: "micType": "NEAR_FIELD" }, "intent": { - "action": "textMagnification", + "action": "screen-magnification", "data": { "scale": 2.5 }, @@ -852,9 +910,9 @@ The section intent is already defined as part of Firebolt. ### 4.1. Content Discovery Section Intents The following section IDs will be used, with the Firebolt application -type: +type as the target App ID: -xrn:firebolt:application-type:main +`xrn:firebolt:application-type:main` | Section | Description | | ---------- | ------------------------------------------------------------------- | @@ -875,37 +933,8 @@ the main aggregated experience UI. To launch the settings UI, a Launch Intent will be used, with the Firebolt application type: -xrn:firebolt:application-type:settings +`xrn:firebolt:application-type:settings` Currently there are no standardized sections for the settings App. -## 5. Volume Notes - -Whether or not a TV uses logarithmic or linear scale is irrelevant to -the Intent schema. - -### 5.1. Jump Notes - -For the feature of instantly moving forward/backwards by \ seconds I -prefer *not* to call this "skip" because in the AVOD world, -"skip" -means to completely bypass an add or chapter (like next track of a CD -player). I\'ve used "jump" in the past, but I\'m open to other names. -We could go with seek, and have a relative and absolute version? - -### 5.2. Core SDK APIs - -We need SDK APIs for some of these, since they\'re potentially surfaced -to apps, e.g. channel scanning. - -Need App-facing SDK APIs for: - -- Seek (otherwise how to pass the value) - New API, maybe - MediaControl.onSeek - -- TrickPlay (how to pass the speed) - New API, maybe - MediaControl.onTrickPlay - -- Channel (how to pass direction) - Discovery.onNavigateTo -Everything else is handled by the OS, i think. From a93e7a3eb8be8311abc4309817c0f2611fed623b Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Tue, 12 Mar 2024 13:56:33 -0400 Subject: [PATCH 07/15] fix: Adding descriptions to high and low --- requirements/specifications/intents/command-and-control.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index df0d50e34..dbf0b35ae 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -546,8 +546,10 @@ The voice guidance intent **MUST** have only one property, `speed`, `toggle`, `v Finally, the intent **MAY** specify a `verbosity` property, which **MUST** use one of the following values is provided: -- `low` -- `high` +| Value | Description | +|--------|-------------| +| `low` | to select shorter response, less context, and less detail; can use abbreviations and can selectively skip words | +| `high` | to select longer response, more context, and more detail; full comprehensive readout and explicit reflectION of what is seen on screen | ```json { From c3c2122ee532ce63c70446b2c10195134a0d745c Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Tue, 12 Mar 2024 14:05:11 -0400 Subject: [PATCH 08/15] fix: Feedback from todays working group --- .../intents/command-and-control.md | 89 ++++++++++++------- 1 file changed, 58 insertions(+), 31 deletions(-) diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index dbf0b35ae..9c7429d73 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -48,8 +48,8 @@ together for easier forwarding to appropriate components. - [3.6.3. Scroll Intent](#363-scroll-intent) - [3.6.4. Back Intent](#364-back-intent) - [3.6.5. Exit Intent](#365-exit-intent) -- [4. Section Intents](#4-section-intents) - - [4.1. Content Discovery Section Intents](#41-content-discovery-section-intents) +- [4. Launch Intents](#4-launch-intents) + - [4.1. Content Discovery Launch Intents](#41-content-discovery-launch-intents) - [4.2. Device Settings Launch Intent](#42-device-settings-launch-intent) ## 3. Control Intents @@ -85,6 +85,29 @@ This intent allows a user to turn the device on or off. } ``` +Additionally, this intent may specify a toggle: + +```json +{ + "type": "xrn:firebolt:intent:platform:power", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "power", + "data": { + "toggle": true + }, + "context": { + "source": "voice" + } + } +} +``` + Additionally, this intent allows a user to set a timer for turning off the power, aka a "sleep timer." @@ -111,14 +134,11 @@ seconds: } } } - ``` While it may not be implemented by all platforms, this could also be used to turn on the TV with a timer. -**TODO**: Current spec supports toggle. i think we can drop this, though. - ### 3.2. Volume Intents Volume Intents control the audio level of the device. @@ -373,10 +393,9 @@ relative field: } ``` -For relative seeking, the seconds value may be a positive or negative -value. +For relative seeking, the seconds value may be a positive or negative value. -**TODO**: currently published spec has seconds always positive, and a `direction` enum... can we drop that? +If a relative seek intent with a seconds value of `0` is received, the platform **SHOULD** ignore it, rather than rebuffering at the current position. #### 3.4.3. Trick Play Intent @@ -435,7 +454,7 @@ This intent allows a user to turn closed captions on or off. "micType": "NEAR_FIELD" }, "intent": { - "action": "closedcaptions", + "action": "closed-captions", "data": { "value": true | false }, @@ -459,7 +478,7 @@ Additionally, this intent may specify a toggle: "micType": "NEAR_FIELD" }, "intent": { - "action": "closedcaptions", + "action": "closed-captions", "data": { "toggle": true }, @@ -542,6 +561,30 @@ The intent **MAY** specify `speed` `number` property that specifies a speed from } ``` +When providing a `speed` this intent **MAY** also set the `relative` property to `true` denoting an increase or decrease in speed. The speed value may be between -5 and 5 inclusive: + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "voice-guidance", + "data": { + "speed": -1, + "relative": true + }, + "context": { + "source": "voice" + } + } +} +``` + The voice guidance intent **MUST** have only one property, `speed`, `toggle`, `value` and **MUST NOT** comebine them in a single intent. Finally, the intent **MAY** specify a `verbosity` property, which **MUST** use one of the following values is provided: @@ -590,7 +633,7 @@ This intent allows a user to turn audio descriptions of content on or off. "micType": "NEAR_FIELD" }, "intent": { - "action": "audiodescriptions", + "action": "audio-descriptions", "data": { "value": true | false }, @@ -614,7 +657,7 @@ Additionally, this intent may specify a toggle: "micType": "NEAR_FIELD" }, "intent": { - "action": "audiodescriptions", + "action": "audio-descriptions", "data": { "toggle": true }, @@ -905,31 +948,17 @@ inactive state. } ``` -## 4. Section Intents +## 4. Launch Intents -The section intent is already defined as part of Firebolt. +If a Firebolt app wants to launch the main or settings experience of the device, it can use one of the following abstract appIds with the `launch` intent. -### 4.1. Content Discovery Section Intents +### 4.1. Content Discovery Launch Intents The following section IDs will be used, with the Firebolt application type as the target App ID: `xrn:firebolt:application-type:main` -| Section | Description | -| ---------- | ------------------------------------------------------------------- | -| guide | Launches a program guide, typically linear, in the main experience. | -| menu | Launches a top-level menu, or home page, of the main experience. | -| recordings | Launches a list of user recordings in the main experience. | -| favorites | Launches a list of user favorites in the main experience. | -| purchases | Launches a list of user purchased content in the main experience. | -| playlist | Launches a list of content the user has saved for future playback. | -| recent | Launches a list of recently watched content. | - -Note that not every Firebolt Distributor will have all of these -features. For unsupported features, these intents should simply launch -the main aggregated experience UI. - ### 4.2. Device Settings Launch Intent To launch the settings UI, a Launch Intent will be used, with the @@ -937,6 +966,4 @@ Firebolt application type: `xrn:firebolt:application-type:settings` -Currently there are no standardized sections for the settings App. - From dc434bb06b474a55531c0fa691301911d3981367 Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Tue, 12 Mar 2024 14:06:28 -0400 Subject: [PATCH 09/15] fix: Remove todos --- .../specifications/intents/command-and-control.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index 9c7429d73..8b9ef9293 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -200,8 +200,6 @@ optional relative field: The value is a positive or negative integer that is relative to a scale of 0-100. -**TODO**: Firebolt currently specifies the value as 0-1 - Firebolt will not support complicated relative changes, e.g. "Set the volume to 50% *of what it currently is\...*" @@ -325,8 +323,6 @@ Media: } ``` -**TODO**: this is different than what we officially published. We need to track down if anyone implemented the old Pause Intent. If so, we need to support pause w/ value: false to mean resume. - If the action is pause, then the currently playing media should be paused, with the frames on-screen and the video decoder ready to resume. @@ -442,8 +438,6 @@ These intents manipulate accessibility features on the device. This intent allows a user to turn closed captions on or off. -**TODO**: there is no hyphen in closed captions because it was already spec'd w/out one previously. Need to check if any platforms implemented it or not. - ```json { "type": "xrn:firebolt:intent:platform:accessibility", @@ -620,8 +614,6 @@ Finally, the intent **MAY** specify a `verbosity` property, which **MUST** use o This intent allows a user to turn audio descriptions of content on or off. -**TODO**: there is no hyphen in closed captions because it was already spec'd w/out one previously. Need to check if any platforms implemented it or not. - ```json { From df41cdcec89c36bfcbc402e40bf09fda0fcfa792 Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Thu, 21 Mar 2024 08:44:00 -0400 Subject: [PATCH 10/15] fix: Typo --- requirements/specifications/intents/command-and-control.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index 8b9ef9293..83530eddb 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -586,7 +586,7 @@ Finally, the intent **MAY** specify a `verbosity` property, which **MUST** use o | Value | Description | |--------|-------------| | `low` | to select shorter response, less context, and less detail; can use abbreviations and can selectively skip words | -| `high` | to select longer response, more context, and more detail; full comprehensive readout and explicit reflectION of what is seen on screen | +| `high` | to select longer response, more context, and more detail; full comprehensive readout and explicit reflection of what is seen on screen | ```json { From 29b2a73869b1fd0389a17d2b7d92baae692c73e7 Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Tue, 7 May 2024 10:00:20 -0400 Subject: [PATCH 11/15] fix: Simon's feedback --- .../intents/command-and-control.md | 66 ++++++++++++++----- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index 83530eddb..00c986468 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -33,9 +33,9 @@ together for easier forwarding to appropriate components. - [3.2.2. Mute Intent](#322-mute-intent) - [3.3. Channel Intent](#33-channel-intent) - [3.4. Media Control Intents](#34-media-control-intents) - - [3.4.1. Pause, Resume, Replay, and Stop Intents](#341-pause-resume-replay-and-stop-intents) + - [3.4.1. Pause, Play, Replay, and Stop Intents](#341-pause-play-replay-and-stop-intents) - [3.4.2. Seek Intent](#342-seek-intent) - - [3.4.3. Trick Play Intent](#343-trick-play-intent) + - [3.4.3. Fast-forward and Rewind Intents](#343-fast-forward-and-rewind-intents) - [3.5. Accessibility Intents](#35-accessibility-intents) - [3.5.1. Closed Captions Intent](#351-closed-captions-intent) - [3.5.2. Voice Guidance Intent](#352-voice-guidance-intent) @@ -136,6 +136,8 @@ seconds: } ``` +To cancel a sleep timer, send a new intent without a delay. + While it may not be implemented by all platforms, this could also be used to turn on the TV with a timer. @@ -300,7 +302,7 @@ one of the channel up/down buttons. ### 3.4. Media Control Intents -#### 3.4.1. Pause, Resume, Replay, and Stop Intents +#### 3.4.1. Pause, Play, Replay, and Stop Intents These intents allow the user to pause and resume playback of the current Media: @@ -315,7 +317,7 @@ Media: "micType": "NEAR_FIELD" }, "intent": { - "action": "pause" | "resume" | "replay" | "stop", + "action": "pause" | "play" | "replay" | "stop", "context": { "source": "voice" } @@ -326,7 +328,11 @@ Media: If the action is pause, then the currently playing media should be paused, with the frames on-screen and the video decoder ready to resume. -If the action is resume, then the currently paused media should resume. +If the action is play, and the current media is paused, then the +currently paused media should resume. + +If the action is play, and there is something playbable selected, then +playback of the selected asset should be initiated. If the action is replay, then the currently paused media should restart from the beginning. This should work even if the decoder has finished, @@ -393,9 +399,9 @@ For relative seeking, the seconds value may be a positive or negative value. If a relative seek intent with a seconds value of `0` is received, the platform **SHOULD** ignore it, rather than rebuffering at the current position. -#### 3.4.3. Trick Play Intent +#### 3.4.3. Fast-forward and Rewind Intents -The Trick Play Intent allows users to fast-forward or rewind: +These intents allow users to fast-forward or rewind: ```json { @@ -407,7 +413,7 @@ The Trick Play Intent allows users to fast-forward or rewind: "micType": "NEAR_FIELD" }, "intent": { - "action": "trickplay", + "action": "fast-forward" | "rewind", "data": { "speed": 2.5 }, @@ -416,19 +422,23 @@ The Trick Play Intent allows users to fast-forward or rewind: } } } - ``` -Speed is a float in the range of -10 to 10, with negative values -denoting backwards/rewind and values between -1 and 1 denoting slow -motion. +TODO: send closed captions vs subtitles + +Speed is a float in the range of 0 (non-includsive) to 10 (inclusive), +with values between 0 and 1 denoting slow motion. It is a device-level decision how to implement different speeds, however actual fast playback (with audio) should be used where possible and reasonable, e.g. a speed of 1.5 should actually be playing the video w/ sync\'d audio, while a speed of 10 will likely be using iframes and not -have audio. For speeds less than zero it is not important, and likely -undesirable, to provide audio. +have audio. For rewind it is not important, and likely undesirable, to +provide audio. + +If speed is not provided then the device should cycle through a range +of speeds defined by the device. This range of speeds **COULD** include +the value `1` so that users can get back to normal speed if desired. ### 3.5. Accessibility Intents @@ -637,6 +647,32 @@ This intent allows a user to turn audio descriptions of content on or off. ``` +This intent may specify a language: + +```json +{ + "type": "xrn:firebolt:intent:platform:accessibility", + "target": "client", + "metadata": { + "assistant": "XFINITY", + "lang": "eng-USA", + "micType": "NEAR_FIELD" + }, + "intent": { + "action": "audio-descriptions", + "data": { + "value": true, + "language": "eng" + }, + "context": { + "source": "voice" + } + } +} +``` + +The `language` must be a three character ISO 639 1/2 code, e.g. `eng`. + Additionally, this intent may specify a toggle: ```json @@ -957,5 +993,3 @@ To launch the settings UI, a Launch Intent will be used, with the Firebolt application type: `xrn:firebolt:application-type:settings` - - From 8f38c56f7bffa3cfffdae30f9a2595c2db484242 Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Tue, 21 May 2024 11:16:56 -0400 Subject: [PATCH 12/15] feat: Add intent schemas from requirements spec --- package-lock.json | 4 +- .../intents/command-and-control.md | 15 +- src/schemas/intents.json | 1135 +++++++++++++---- 3 files changed, 908 insertions(+), 246 deletions(-) diff --git a/package-lock.json b/package-lock.json index e5af29a6b..421c709b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16951,7 +16951,7 @@ }, "src/sdks/core": { "name": "@firebolt-js/sdk", - "version": "1.1.0-next.3", + "version": "1.1.1-next.1", "license": "Apache-2.0", "devDependencies": { "jest": "^28.1.0", @@ -16962,7 +16962,7 @@ }, "src/sdks/manage": { "name": "@firebolt-js/manage-sdk", - "version": "1.1.0-next.3", + "version": "1.1.1-next.1", "license": "Apache-2.0", "devDependencies": { "jest": "^28.1.0", diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index 00c986468..0fc2b171f 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -334,7 +334,7 @@ currently paused media should resume. If the action is play, and there is something playbable selected, then playback of the selected asset should be initiated. -If the action is replay, then the currently paused media should restart +If the action is replay, then the currently paused or playing media should restart from the beginning. This should work even if the decoder has finished, and its resources have been released. @@ -424,8 +424,6 @@ These intents allow users to fast-forward or rewind: } ``` -TODO: send closed captions vs subtitles - Speed is a float in the range of 0 (non-includsive) to 10 (inclusive), with values between 0 and 1 denoting slow motion. @@ -467,7 +465,6 @@ This intent allows a user to turn closed captions on or off. } } } - ``` Additionally, this intent may specify a toggle: @@ -589,8 +586,6 @@ When providing a `speed` this intent **MAY** also set the `relative` property to } ``` -The voice guidance intent **MUST** have only one property, `speed`, `toggle`, `value` and **MUST NOT** comebine them in a single intent. - Finally, the intent **MAY** specify a `verbosity` property, which **MUST** use one of the following values is provided: | Value | Description | @@ -821,7 +816,7 @@ Setting the scale to `1` turns off magnification. Setting the scale to a value g Even if a Firebolt platform does not support specifying the numeric scale, it **MUST** turn magnifacation on and off based on them. -The magnification intent **MUST** have only one property, `scale`, `toggle`, `value` and **MUST NOT** comebine them in a single intent. +The magnification intent **MUST** have only one property, `scale`, `toggle`, `value` and **MUST NOT** combine them in a single intent. ### 3.6. Interaction Intents @@ -868,11 +863,6 @@ The select intent allows users to tell an app select, e.g., whatever is focused. This is a platform-level intent that effectively sends the "Ok" or "Select" key to the current app. -**NOTE**: i don\'t like using "click" here, but i can\'t think of a -good word that isn\'t super vague. "Select" means highlighting, not -actually hitting okay, so we can\'t use that. I considered "activate" -but that seems odd\... Open to discussing this. - ```json { "type": "xrn:firebolt:intent:platform:interaction", @@ -917,7 +907,6 @@ up/down/left/right: } } } - ``` The distance is a float that represents how many of units to scroll. All diff --git a/src/schemas/intents.json b/src/schemas/intents.json index 041018322..6f1195c5d 100644 --- a/src/schemas/intents.json +++ b/src/schemas/intents.json @@ -273,7 +273,10 @@ "$ref": "#/definitions/SkipIntent" }, { - "$ref": "#/definitions/TrickPlayIntent" + "$ref": "#/definitions/FastForwardIntent" + }, + { + "$ref": "#/definitions/RewindIntent" }, { "$ref": "#/definitions/ClosedCaptionsIntent" @@ -307,6 +310,9 @@ "action": { "const": "launch" } + }, + "not": { + "required": [ "data" ] } } ], @@ -338,6 +344,9 @@ "action": { "const": "home" } + }, + "not": { + "required": [ "data" ] } } ], @@ -803,6 +812,34 @@ } ] }, + "ChannelIntent": { + "description": "A Firebolt compliant representation of a user intent to 'surf' to the next or previous channel.", + "title": "ChannelIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "required": [ "data" ], + "properties": { + "action": { + "const": "channel" + }, + "data": { + "type": "string", + "enum": [ + "next", + "previous" + ] + } + } + } + ] + }, "TuneIntent": { "description": "A Firebolt compliant representation of a user intention to 'tune' to a traditional over-the-air broadcast, or an OTT Stream from an OTT or vMVPD App.", "title": "TuneIntent", @@ -991,7 +1028,8 @@ }, "additionalProperties": false } - } + }, + "required": [ "data" ] } ], "examples": [ @@ -1484,9 +1522,9 @@ } ] }, - "ButtonIntent": { - "description": "A Firebolt compliant representation of a user intention to interact with their device in a way analogous to pressing one of the remote buttons.", - "title": "ButtonIntent", + "FocusIntent": { + "description": "A Firebolt compliant representation of a user intention to interact with their device in a way analogous to pressing remote directional pad buttons.", + "title": "FocusIntent", "allOf": [ { "$ref": "#/definitions/Intent" @@ -1496,30 +1534,24 @@ }, { "type": "object", + "required": [ "data" ], "properties": { "action": { - "const": "button" + "const": "focus" }, "data": { "type": "object", "required": [ - "operation" + "direction" ], "properties": { - "operation": { + "direction": { "type": "string", "enum": [ - "down", "up", - "prev", - "next", - "enter", - "exit", - "info", - "menu", - "back", - "cancel", - "record" + "down", + "left", + "right" ] } }, @@ -1530,9 +1562,9 @@ ], "examples": [ { - "action": "button", + "action": "focus", "data": { - "operation": "menu" + "direction": "up" }, "context": { "source": "voice" @@ -1540,9 +1572,9 @@ } ] }, - "VolumeIntent": { - "description": "A Firebolt compliant representation of a user intention to change the device volume.", - "title": "VolumeIntent", + "SelectIntent": { + "description": "A Firebolt compliant representation of a user intention to interact with their device in a way analogous to pressing the remote 'select' button.", + "title": "SelectIntent", "allOf": [ { "$ref": "#/definitions/Intent" @@ -1554,79 +1586,26 @@ "type": "object", "properties": { "action": { - "const": "volume" - }, - "data": { - "anyOf": [ - { - "type": "object", - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, - "toggle": { - "const": true - } - }, - "minProperties": 1, - "maxProperties": 1, - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, - "relative": { - "const": true - } - }, - "additionalProperties": false - } - ] + "const": "select" } + }, + "not": { + "required": [ "data" ] } } ], "examples": [ { - "action": "volume", - "data": { - "toggle": true - }, - "context": { - "source": "voice" - } - }, - { - "action": "volume", - "data": { - "value": 0.7 - }, - "context": { - "source": "voice" - } - }, - { - "action": "volume", - "data": { - "value": 0.1, - "relative": true - }, + "action": "select", "context": { "source": "voice" } } ] }, - "PowerIntent": { - "description": "A Firebolt compliant representation of a user intention to turn their device on or off.", - "title": "PowerIntent", + "BackIntent": { + "description": "A Firebolt compliant representation of a user intention to interact with their device in a way analogous to pressing the remote 'back' button.", + "title": "BackIntent", "allOf": [ { "$ref": "#/definitions/Intent" @@ -1638,38 +1617,26 @@ "type": "object", "properties": { "action": { - "const": "power" - }, - "data": { - "$ref": "#/definitions/BooleanToggle" + "const": "back" } + }, + "not": { + "required": [ "data" ] } } ], "examples": [ { - "action": "power", - "data": { - "value": false - }, - "context": { - "source": "voice" - } - }, - { - "action": "power", - "data": { - "toggle": true - }, + "action": "back", "context": { "source": "voice" } } ] }, - "MicrophoneIntent": { - "description": "A Firebolt compliant representation of a user intention to turn their microphone on or off.", - "title": "MicrophoneIntent", + "ExitIntent": { + "description": "A Firebolt compliant representation of a user intention to interact with their device in a way analogous to pressing the remote 'back' button.", + "title": "ExitIntent", "allOf": [ { "$ref": "#/definitions/Intent" @@ -1681,38 +1648,26 @@ "type": "object", "properties": { "action": { - "const": "microphone" - }, - "data": { - "$ref": "#/definitions/BooleanToggle" + "const": "exit" } + }, + "not": { + "required": [ "data" ] } } ], "examples": [ { - "action": "microphone", - "data": { - "value": false - }, - "context": { - "source": "voice" - } - }, - { - "action": "microphone", - "data": { - "toggle": true - }, + "action": "exit", "context": { "source": "voice" } } ] - }, - "InputIntent": { - "description": "A Firebolt compliant representation of a user intention to change which video input is active.", - "title": "InputIntent", + }, + "ScrollIntent": { + "description": "A Firebolt compliant representation of a user intention to interact with their device in a way analogous to pressing remote directional pad buttons.", + "title": "ScrollIntent", "allOf": [ { "$ref": "#/definitions/Intent" @@ -1722,29 +1677,37 @@ }, { "type": "object", + "required": [ "data" ], "properties": { "action": { - "const": "input" + "const": "scroll" }, "data": { "type": "object", "required": [ - "interface" + "direction" ], "properties": { - "interface": { + "direction": { "type": "string", "enum": [ - "hdmi", - "rca", - "vga", - "etc..." + "up", + "down", + "left", + "right" ] }, - "number": { - "type": "integer", - "minimum": 1, - "maximum": 100 + "unit": { + "type": "string", + "enum": [ + "line", + "page", + "percent" + ] + }, + "distance": { + "type": "number", + "minimum": 0 } }, "additionalProperties": false @@ -1754,19 +1717,9 @@ ], "examples": [ { - "action": "input", - "data": { - "interface": "hdmi" - }, - "context": { - "source": "voice" - } - }, - { - "action": "input", + "action": "scroll", "data": { - "interface": "hdmi", - "number": 1 + "direction": "up" }, "context": { "source": "voice" @@ -1774,9 +1727,9 @@ } ] }, - "PauseIntent": { - "description": "A Firebolt compliant representation of a user intention to pause/unpause in-progress playback.", - "title": "PauseIntent", + "ButtonIntent": { + "description": "A Firebolt compliant representation of a user intention to interact with their device in a way analogous to pressing one of the remote buttons.", + "title": "ButtonIntent", "allOf": [ { "$ref": "#/definitions/Intent" @@ -1788,28 +1741,41 @@ "type": "object", "properties": { "action": { - "const": "pause" + "const": "button" }, "data": { - "$ref": "#/definitions/BooleanToggle" + "type": "object", + "required": [ + "operation" + ], + "properties": { + "operation": { + "type": "string", + "enum": [ + "down", + "up", + "prev", + "next", + "enter", + "exit", + "info", + "menu", + "back", + "cancel", + "record" + ] + } + }, + "additionalProperties": false } } } ], "examples": [ { - "action": "pause", - "data": { - "value": false - }, - "context": { - "source": "voice" - } - }, - { - "action": "pause", + "action": "button", "data": { - "toggle": true + "operation": "menu" }, "context": { "source": "voice" @@ -1817,9 +1783,9 @@ } ] }, - "PlaybackSpeedIntent": { - "description": "A Firebolt compliant representation of a user intention to change the speed of in-progress playback.", - "title": "PlaybackSpeedIntent", + "VolumeIntent": { + "description": "A Firebolt compliant representation of a user intention to change the device volume.", + "title": "VolumeIntent", "allOf": [ { "$ref": "#/definitions/Intent" @@ -1831,22 +1797,30 @@ "type": "object", "properties": { "action": { - "const": "speed" + "const": "volume" }, "data": { "type": "object", "properties": { "value": { - "type": "number", - "exclusiveMinimum": 0, - "maximum": 4 + "type": "number" }, - "toggle": { - "type": "boolean" + "relative": { + "const": true } }, - "minProperties": 1, - "maxProperties": 1, + "required": [ "value" ], + "if": { + "required": [ "relative" ] + }, + "then": { + "minimum": 0, + "maximum": 100 + }, + "else": { + "minimum": -50, + "maximum": 50 + }, "additionalProperties": false } } @@ -1854,18 +1828,19 @@ ], "examples": [ { - "action": "speed", + "action": "volume", "data": { - "value": 2 + "value": 70 }, "context": { "source": "voice" } }, { - "action": "speed", + "action": "volume", "data": { - "toggle": true + "value": 10, + "relative": true }, "context": { "source": "voice" @@ -1873,9 +1848,9 @@ } ] }, - "TrickPlayIntent": { - "description": "A Firebolt compliant representation of a user intention to fast-forward or rewind in-progress playback.", - "title": "TrickPlayIntent", + "MuteIntent": { + "description": "A Firebolt compliant representation of a user intention to mute or unmute the device.", + "title": "MuteIntent", "allOf": [ { "$ref": "#/definitions/Intent" @@ -1887,39 +1862,476 @@ "type": "object", "properties": { "action": { - "const": "trickplay" + "const": "mute" }, "data": { - "allOf": [ - { - "$ref": "#/definitions/DirectionalOperation" - }, - { - "type": "object", - "properties": { - "speed": { - "type": "number", - "exclusiveMinimum": 0, - "maximum": 10 - } - }, - "propertyNames": { - "enum": [ - "direction", - "speed" - ] - } + "$ref": "#/definitions/BooleanToggle" + } + } + } + ], + "examples": [ + { + "action": "mute", + "data": { + "value": false + }, + "context": { + "source": "voice" + } + }, + { + "action": "mute", + "data": { + "toggle": true + }, + "context": { + "source": "voice" + } + } + ] + }, + "PowerIntent": { + "description": "A Firebolt compliant representation of a user intention to turn their device on or off.", + "title": "PowerIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "power" + }, + "data": { + "type": "object", + "properties": { + "value": { + "type": "boolean" + }, + "toggle": { + "const": true + }, + "delay": { + "type": "integer", + "minimum": 0 + } + }, + "if": { + "required": [ "value" ] + }, + "then": { + "not": { + "required": [ "toggle" ] + } + }, + "else": { + "required": [ "toggle" ] + }, + "additionalProperties": false + } + } + } + ], + "examples": [ + { + "action": "power", + "data": { + "value": false + }, + "context": { + "source": "voice" + } + }, + { + "action": "power", + "data": { + "toggle": true + }, + "context": { + "source": "voice" + } + }, + { + "action": "power", + "data": { + "value": false, + "delay": "900" + }, + "context": { + "source": "voice" + } + } + ] + }, + "MicrophoneIntent": { + "description": "A Firebolt compliant representation of a user intention to turn their microphone on or off.", + "title": "MicrophoneIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "microphone" + }, + "data": { + "$ref": "#/definitions/BooleanToggle" + } + } + } + ], + "examples": [ + { + "action": "microphone", + "data": { + "value": false + }, + "context": { + "source": "voice" + } + }, + { + "action": "microphone", + "data": { + "toggle": true + }, + "context": { + "source": "voice" + } + } + ] + }, + "InputIntent": { + "description": "A Firebolt compliant representation of a user intention to change which video input is active.", + "title": "InputIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "input" + }, + "data": { + "type": "object", + "required": [ + "interface" + ], + "properties": { + "interface": { + "type": "string", + "enum": [ + "hdmi", + "rca", + "vga", + "etc..." + ] + }, + "number": { + "type": "integer", + "minimum": 1, + "maximum": 100 + } + }, + "additionalProperties": false + } + } + } + ], + "examples": [ + { + "action": "input", + "data": { + "interface": "hdmi" + }, + "context": { + "source": "voice" + } + }, + { + "action": "input", + "data": { + "interface": "hdmi", + "number": 1 + }, + "context": { + "source": "voice" + } + } + ] + }, + "PauseIntent": { + "description": "A Firebolt compliant representation of a user intention to pause in-progress playback.", + "title": "PauseIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "pause" + } + } + } + ], + "examples": [ + { + "action": "pause", + "context": { + "source": "voice" + } + } + ] + }, + "PlayIntent": { + "description": "A Firebolt compliant representation of a user intention to play/resume content.", + "title": "PlayIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "play" + } + } + } + ], + "examples": [ + { + "action": "play", + "context": { + "source": "voice" + } + } + ] + }, + "ReplayIntent": { + "description": "A Firebolt compliant representation of a user intention to replay content.", + "title": "ReplayIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "replay" + } + } + } + ], + "examples": [ + { + "action": "replay", + "context": { + "source": "voice" + } + } + ] + }, + "StopIntent": { + "description": "A Firebolt compliant representation of a user intention to stop content.", + "title": "StopIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "stop" + } + } + } + ], + "examples": [ + { + "action": "stop", + "context": { + "source": "voice" + } + } + ] + }, + "PlaybackSpeedIntent": { + "description": "A Firebolt compliant representation of a user intention to change the speed of in-progress playback.", + "title": "PlaybackSpeedIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "speed" + }, + "data": { + "type": "object", + "properties": { + "value": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 4 + }, + "toggle": { + "type": "boolean" + } + }, + "minProperties": 1, + "maxProperties": 1, + "additionalProperties": false + } + } + } + ], + "examples": [ + { + "action": "speed", + "data": { + "value": 2 + }, + "context": { + "source": "voice" + } + }, + { + "action": "speed", + "data": { + "toggle": true + }, + "context": { + "source": "voice" + } + } + ] + }, + "FastForwardIntent": { + "description": "A Firebolt compliant representation of a user intention to fast-forward in-progress playback.", + "title": "FastForwardIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "fast-forward" + }, + "data": { + "type": "object", + "properties": { + "speed": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 10 + } + } + } + } + } + ], + "examples": [ + { + "action": "fast-forward", + "data": { + "speed": 2 + }, + "context": { + "source": "voice" + } + }, + { + "action": "fast-forward", + "data": { + "speed": 0.5 + }, + "context": { + "source": "voice" + } + }, + { + "action": "fast-forward", + "context": { + "source": "voice" + } + } + ] + }, + "RewindIntent": { + "description": "A Firebolt compliant representation of a user intention to rewind in-progress playback.", + "title": "RewindIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "rewind" + }, + "data": { + "type": "object", + "properties": { + "speed": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 10 } - ] + } } } } ], "examples": [ { - "action": "trickplay", + "action": "rewind", "data": { - "direction": "forward", "speed": 2 }, "context": { @@ -1927,17 +2339,22 @@ } }, { - "action": "trickplay", + "action": "rewind", "data": { - "direction": "backward", - "speed": 2 + "speed": 0.5 }, "context": { "source": "voice" } - } + }, + { + "action": "rewind", + "context": { + "source": "voice" + } + } ] - }, + }, "SeekIntent": { "description": "A Firebolt compliant representation of a user intention to seek to a different time for in-progress playback.", "title": "SeekIntent", @@ -1966,13 +2383,23 @@ "type": "number", "minimum": 0, "maximum": 1800 + }, + "relative": { + "const": true } }, - "propertyNames": { - "enum": [ - "direction", - "seconds" - ] + "required": [ "seconds" ], + "if": { + "not": { + "required": [ "relative" ] + } + }, + "then": { + "properties": { + "seconds": { + "minimum": 0 + } + } } } ] @@ -1993,18 +2420,8 @@ { "action": "seek", "data": { - "direction": "forward", - "seconds": 30 - }, - "context": { - "source": "voice" - } - }, - { - "action": "seek", - "data": { - "direction": "backward", - "seconds": 30 + "relative": true, + "seconds": -30 }, "context": { "source": "voice" @@ -2091,7 +2508,7 @@ "type": "object", "properties": { "action": { - "const": "closedcaptions" + "const": "closed-captions" }, "data": { "$ref": "#/definitions/BooleanToggle" @@ -2120,6 +2537,117 @@ } ] }, + "VoiceGuideanceIntent": { + "description": "A Firebolt compliant representation of a user intention to enable/disable voice guidance.", + "title": "VoiceGuideanceIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "voice-guidance" + }, + "data": { + "type": "object", + "properties": { + "value": { + "type": "boolean" + }, + "toggle": { + "const": true + }, + "speed": { + "type": "integer" + }, + "relative": { + "const": true + }, + "verbosity": { + "type": "string", + "enum": [ + "low", + "hight" + ] + } + }, + "if": { + "required": [ "value" ] + }, + "then": { + "not": { + "required": [ "toggle" ] + }, + "if": { + "required": [ "relative" ] + }, + "then": { + "properties": { + "speed": { + "minimum": -10, + "maximum": 10 + } + } + }, + "else": { + "properties": { + "speed": { + "exclusiveMinimum": 0, + "maximum": 10 + } + } + } + }, + "else": { + "required": [ "toggle" ], + "not": { + "required": [ + "speed" + ] + } + }, + "additionalProperties": false + } + } + } + ], + "examples": [ + { + "action": "voice-guidance", + "data": { + "value": true, + "verbosity": "low" + }, + "context": { + "source": "voice" + } + }, + { + "action": "voice-guidance", + "data": { + "speed": -1, + "relative": true + }, + "context": { + "source": "voice" + } + }, + { + "action": "voice-guidance", + "data": { + "toggle": true + }, + "context": { + "source": "voice" + } + } + ] + }, "AudioDescriptionIntent": { "description": "A Firebolt compliant representation of a user intention to enable/disable audio descriptions.", "title": "AudioDescriptionIntent", @@ -2134,7 +2662,83 @@ "type": "object", "properties": { "action": { - "const": "audiodescriptions" + "const": "audio-descriptions" + }, + "data": { + "type": "object", + "properties": { + "value": { + "type": "boolean" + }, + "toggle": { + "const": true + }, + "language": { + "$ref": "https://meta.comcast.com/firebolt/localization#/definitions/ISO639_2Language" + } + }, + "if": { + "required": [ "value" ] + }, + "then": { + "not": { + "required": [ "toggle" ] + } + }, + "else": { + "required": [ "toggle" ] + }, + "additionalProperties": false + } + } + } + ], + "examples": [ + { + "action": "audio-descriptions", + "data": { + "value": false + }, + "context": { + "source": "voice" + } + }, + { + "action": "audio-descriptions", + "data": { + "toggle": true + }, + "context": { + "source": "voice" + } + }, + { + "action": "audio-descriptions", + "data": { + "value": true, + "language": "eng" + }, + "context": { + "source": "voice" + } + } + ] + }, + "HighContrastIntent": { + "description": "A Firebolt compliant representation of a user intention to enable or disable high contrast mode.", + "title": "HighContrastIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "high-contrast" }, "data": { "$ref": "#/definitions/BooleanToggle" @@ -2144,7 +2748,7 @@ ], "examples": [ { - "action": "audiodescriptions", + "action": "high-contrast", "data": { "value": false }, @@ -2153,7 +2757,7 @@ } }, { - "action": "audiodescriptions", + "action": "high-contrast", "data": { "toggle": true }, @@ -2162,6 +2766,75 @@ } } ] + }, + "ScreenMagnificationIntent": { + "description": "A Firebolt compliant representation of a user intention to turn screen magnification on or off.", + "title": "ScreenMagnificationIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "properties": { + "action": { + "const": "screen-magnification" + }, + "data": { + "type": "object", + "properties": { + "value": { + "type": "boolean" + }, + "toggle": { + "const": true + }, + "scale": { + "type": "number", + "minimum": 1, + "maximum": 10 + } + }, + "maxProperties": 1, + "minProperties": 1, + "additionalProperties": false + } + } + } + ], + "examples": [ + { + "action": "power", + "data": { + "value": false + }, + "context": { + "source": "voice" + } + }, + { + "action": "power", + "data": { + "toggle": true + }, + "context": { + "source": "voice" + } + }, + { + "action": "power", + "data": { + "value": false, + "delay": "900" + }, + "context": { + "source": "voice" + } + } + ] }, "MessageIntent": { "description": "A Firebolt compliant representation of a platform intention to display a message on the device.", From 11bfcbdef49159931b606b7069c839fe809ba69c Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Tue, 21 May 2024 11:47:33 -0400 Subject: [PATCH 13/15] fix: Cleaned up examples to match schemas --- package-lock.json | 16 ++-- .../intents/command-and-control.md | 10 +- src/schemas/intents.json | 95 +++++++++++++------ 3 files changed, 79 insertions(+), 42 deletions(-) diff --git a/package-lock.json b/package-lock.json index 421c709b5..173940cab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@firebolt-js/sdks", - "version": "1.1.1-next.1", + "version": "1.2.0-next.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@firebolt-js/sdks", - "version": "1.1.1-next.1", + "version": "1.2.0-next.2", "license": "Apache-2.0", "workspaces": [ "src/sdks/core", @@ -18,7 +18,7 @@ "devDependencies": { "@commitlint/cli": "^17.0.3", "@commitlint/config-conventional": "^17.0.3", - "@firebolt-js/openrpc": "2.3.0", + "@firebolt-js/openrpc": "3.0.0-next.3", "@firebolt-js/schemas": "2.0.0", "@saithodev/semantic-release-backmerge": "^3.2.0", "@semantic-release/changelog": "^6.0.1", @@ -1070,9 +1070,9 @@ "link": true }, "node_modules/@firebolt-js/openrpc": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@firebolt-js/openrpc/-/openrpc-2.3.0.tgz", - "integrity": "sha512-JlhFy4r1dkIUHESVi72LaYgbdjP3hG04OfaLEtqLx5sLYDOQ8Y+KHiD2h81u8fvRgLEc6HiX5HgCFdapoUg8Wg==", + "version": "3.0.0-next.3", + "resolved": "https://registry.npmjs.org/@firebolt-js/openrpc/-/openrpc-3.0.0-next.3.tgz", + "integrity": "sha512-hhNtHIpNwwN99Zd4ZABnQ3MIEMPYDN5rSZPNad4ah4R5rmLypWv79vfHSPYeCrTTmoQCwwCFhpObjhJk0EVX/g==", "dev": true, "dependencies": { "ajv": "^8.3.0", @@ -16951,7 +16951,7 @@ }, "src/sdks/core": { "name": "@firebolt-js/sdk", - "version": "1.1.1-next.1", + "version": "1.2.0-next.2", "license": "Apache-2.0", "devDependencies": { "jest": "^28.1.0", @@ -16962,7 +16962,7 @@ }, "src/sdks/manage": { "name": "@firebolt-js/manage-sdk", - "version": "1.1.1-next.1", + "version": "1.2.0-next.2", "license": "Apache-2.0", "devDependencies": { "jest": "^28.1.0", diff --git a/requirements/specifications/intents/command-and-control.md b/requirements/specifications/intents/command-and-control.md index 0fc2b171f..db799cde9 100644 --- a/requirements/specifications/intents/command-and-control.md +++ b/requirements/specifications/intents/command-and-control.md @@ -816,7 +816,9 @@ Setting the scale to `1` turns off magnification. Setting the scale to a value g Even if a Firebolt platform does not support specifying the numeric scale, it **MUST** turn magnifacation on and off based on them. -The magnification intent **MUST** have only one property, `scale`, `toggle`, `value` and **MUST NOT** combine them in a single intent. +If the intent has the `toggle` property, then it **MUST NOT** have the `scale` or `value` property. + +If the intent has the `value` property, then it **MUST NOT** have the `toggle`. ### 3.6. Interaction Intents @@ -899,8 +901,7 @@ up/down/left/right: "action": "scroll", "data": { "direction": "up" | "down" | "left" | "right", - "unit": "page" | "line" | "percent", - "distance": 2.5 + "unit": "page" | "line" | "percent" }, "context": { "source": "voice" @@ -909,8 +910,7 @@ up/down/left/right: } ``` -The distance is a float that represents how many of units to scroll. All -three data properties are required. +Both `direction` and `unit` are required. These Intents will generate appropriate browser / DOM scrolling operations that don\'t require custom APIs. diff --git a/src/schemas/intents.json b/src/schemas/intents.json index 459520ab0..e86e9ad1b 100644 --- a/src/schemas/intents.json +++ b/src/schemas/intents.json @@ -1114,6 +1114,7 @@ "type": "object", "properties": { "options": { + "type": "object", "maxProperties": 1 } } @@ -1122,6 +1123,7 @@ "type": "object", "properties": { "options": { + "type": "object", "maxProperties": 0 } } @@ -1707,10 +1709,6 @@ "page", "percent" ] - }, - "distance": { - "type": "number", - "minimum": 0 } }, "additionalProperties": false @@ -1817,12 +1815,22 @@ "required": [ "relative" ] }, "then": { - "minimum": 0, - "maximum": 100 + "properties": { + "value": { + "type": "number", + "minimum": -50, + "maximum": 50 + } + } }, "else": { - "minimum": -50, - "maximum": 50 + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 100 + } + } }, "additionalProperties": false } @@ -1963,7 +1971,7 @@ "action": "power", "data": { "value": false, - "delay": "900" + "delay": 900 }, "context": { "source": "voice" @@ -2383,9 +2391,7 @@ "type": "object", "properties": { "seconds": { - "type": "number", - "minimum": 0, - "maximum": 1800 + "type": "number" }, "relative": { "const": true @@ -2400,7 +2406,18 @@ "then": { "properties": { "seconds": { - "minimum": 0 + "type": "number", + "minimum": 0, + "maximum": 86400 + } + } + }, + "else": { + "properties": { + "seconds": { + "type": "number", + "minimum": -43200, + "maximum": 43200 } } } @@ -2521,7 +2538,7 @@ ], "examples": [ { - "action": "closedcaptions", + "action": "closed-captions", "data": { "value": false }, @@ -2530,7 +2547,7 @@ } }, { - "action": "closedcaptions", + "action": "closed-captions", "data": { "toggle": true }, @@ -2540,9 +2557,9 @@ } ] }, - "VoiceGuideanceIntent": { + "VoiceGuidanceIntent": { "description": "A Firebolt compliant representation of a user intention to enable/disable voice guidance.", - "title": "VoiceGuideanceIntent", + "title": "VoiceGuidanceIntent", "allOf": [ { "$ref": "#/definitions/Intent" @@ -2592,6 +2609,7 @@ "then": { "properties": { "speed": { + "type": "integer", "minimum": -10, "maximum": 10 } @@ -2600,6 +2618,7 @@ "else": { "properties": { "speed": { + "type": "integer", "exclusiveMinimum": 0, "maximum": 10 } @@ -2607,11 +2626,13 @@ } }, "else": { - "required": [ "toggle" ], - "not": { - "required": [ - "speed" - ] + "if": { + "required": [ "toggle" ] + }, + "then": { + "not": { + "required": [ "value" ] + } } }, "additionalProperties": false @@ -2801,8 +2822,15 @@ "maximum": 10 } }, - "maxProperties": 1, - "minProperties": 1, + "if": { + "required": [ + "toggle" + ] + }, + "then": { + "type": "object", + "maxProperties": 1 + }, "additionalProperties": false } } @@ -2810,7 +2838,7 @@ ], "examples": [ { - "action": "power", + "action": "screen-magnification", "data": { "value": false }, @@ -2819,7 +2847,17 @@ } }, { - "action": "power", + "action": "screen-magnification", + "data": { + "value": true, + "scale": 2.5 + }, + "context": { + "source": "voice" + } + }, + { + "action": "screen-magnification", "data": { "toggle": true }, @@ -2828,10 +2866,9 @@ } }, { - "action": "power", + "action": "screen-magnification", "data": { - "value": false, - "delay": "900" + "value": false }, "context": { "source": "voice" From f593631d2a62e6ecb6f39f712a5c8045a9655c5a Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Tue, 21 May 2024 12:02:35 -0400 Subject: [PATCH 14/15] fix: Added all new intents to top-level schema --- src/schemas/intents.json | 56 +++++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/src/schemas/intents.json b/src/schemas/intents.json index e86e9ad1b..a7119e372 100644 --- a/src/schemas/intents.json +++ b/src/schemas/intents.json @@ -243,12 +243,36 @@ { "$ref": "#/definitions/ButtonIntent" }, + { + "$ref": "#/definitions/ChannelIntent" + }, + { + "$ref": "#/definitions/FocusIntent" + }, + { + "$ref": "#/definitions/SelectIntent" + }, + { + "$ref": "#/definitions/BackIntent" + }, + { + "$ref": "#/definitions/ExitIntent" + }, + { + "$ref": "#/definitions/ChannelIntent" + }, + { + "$ref": "#/definitions/ScrollIntent" + }, { "$ref": "#/definitions/PowerIntent" }, { "$ref": "#/definitions/VolumeIntent" }, + { + "$ref": "#/definitions/MuteIntent" + }, { "$ref": "#/definitions/MicrophoneIntent" }, @@ -257,15 +281,36 @@ }, { "$ref": "#/definitions/TuneIntent" + }, + { + "$ref": "#/definitions/VoiceGuidanceIntent" + }, + { + "$ref": "#/definitions/HighContrastIntent" + }, + { + "$ref": "#/definitions/ScreenMagnificationIntent" } ] }, "PlaybackControlIntent": { "description": "A Firebolt compliant representation of a user intention to control some aspect of in-progress playback.", "anyOf": [ + { + "$ref": "#/definitions/PlayIntent" + }, { "$ref": "#/definitions/PauseIntent" }, + { + "$ref": "#/definitions/ReplayIntent" + }, + { + "$ref": "#/definitions/StopIntent" + }, + { + "$ref": "#/definitions/PlaybackSpeedIntent" + }, { "$ref": "#/definitions/SeekIntent" }, @@ -282,7 +327,7 @@ "$ref": "#/definitions/ClosedCaptionsIntent" }, { - "$ref": "#/definitions/AudioDescriptionIntent" + "$ref": "#/definitions/AudioDescriptionsIntent" } ] }, @@ -2047,10 +2092,7 @@ "interface": { "type": "string", "enum": [ - "hdmi", - "rca", - "vga", - "etc..." + "hdmi" ] }, "number": { @@ -2672,9 +2714,9 @@ } ] }, - "AudioDescriptionIntent": { + "AudioDescriptionsIntent": { "description": "A Firebolt compliant representation of a user intention to enable/disable audio descriptions.", - "title": "AudioDescriptionIntent", + "title": "AudioDescriptionsIntent", "allOf": [ { "$ref": "#/definitions/Intent" From 782f35988a0929d67ae5739599abf0834f5ea7fd Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Thu, 6 Jun 2024 16:54:45 -0400 Subject: [PATCH 15/15] fix: Missing ChannelIntent --- package-lock.json | 4 ++-- src/schemas/intents.json | 31 ++++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5dc4afb85..d8d00eb1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16956,7 +16956,7 @@ }, "src/sdks/core": { "name": "@firebolt-js/sdk", - "version": "1.2.0-next.2", + "version": "1.2.0-next.3", "license": "Apache-2.0", "devDependencies": { "jest": "^28.1.0", @@ -16978,7 +16978,7 @@ }, "src/sdks/manage": { "name": "@firebolt-js/manage-sdk", - "version": "1.2.0-next.2", + "version": "1.2.0-next.3", "license": "Apache-2.0", "devDependencies": { "jest": "^28.1.0", diff --git a/src/schemas/intents.json b/src/schemas/intents.json index 649084ceb..8f5a86c38 100644 --- a/src/schemas/intents.json +++ b/src/schemas/intents.json @@ -243,9 +243,6 @@ { "$ref": "#/definitions/ButtonIntent" }, - { - "$ref": "#/definitions/ChannelIntent" - }, { "$ref": "#/definitions/FocusIntent" }, @@ -443,6 +440,34 @@ } ] }, + "ChannelIntent": { + "description": "A Firebolt compliant representation of a user intent to 'surf' to the next or previous channel.", + "title": "ChannelIntent", + "allOf": [ + { + "$ref": "#/definitions/Intent" + }, + { + "$ref": "#/definitions/IntentProperties" + }, + { + "type": "object", + "required": [ "data" ], + "properties": { + "action": { + "const": "channel" + }, + "data": { + "type": "string", + "enum": [ + "next", + "previous" + ] + } + } + } + ] + }, "TuneIntent": { "description": "A Firebolt compliant representation of a user intention to 'tune' to a traditional over-the-air broadcast, or an OTT Stream from an OTT or vMVPD App.", "title": "TuneIntent",