diff --git a/ts/a11y/SpeechUtil.ts b/ts/a11y/SpeechUtil.ts
new file mode 100644
index 000000000..2b66bbee1
--- /dev/null
+++ b/ts/a11y/SpeechUtil.ts
@@ -0,0 +1,232 @@
+/*************************************************************
+ *
+ * Copyright (c) 2018-2023 The MathJax Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @fileoverview Provides utility functions for speech handling.
+ *
+ * @author v.sorge@mathjax.org (Volker Sorge)
+ */
+
+import {MmlNode} from '../core/MmlTree/MmlNode.js';
+import Sre from './sre.js';
+
+const ProsodyKeys = [ 'pitch', 'rate', 'volume' ];
+
+interface ProsodyElement {
+ [propName: string]: string | boolean | number;
+ pitch?: number;
+ rate?: number;
+ volume?: number;
+}
+
+export interface SsmlElement extends ProsodyElement {
+ pause?: string;
+ text?: string;
+ mark?: string;
+ character?: boolean;
+ kind?: string;
+}
+
+/**
+ * Parses a string containing an ssml structure into a list of text strings
+ * with associated ssml annotation elements.
+ *
+ * @param {string} speech The speech string.
+ * @return {[string, SsmlElement[]]} The annotation structure.
+ */
+export function ssmlParsing(speech: string): [string, SsmlElement[]] {
+ let xml = Sre.parseDOM(speech);
+ let instr: SsmlElement[] = [];
+ let text: String[] = [];
+ recurseSsml(Array.from(xml.childNodes), instr, text);
+ return [text.join(' '), instr];
+}
+
+/**
+ * Tail recursive combination of SSML components.
+ *
+ * @param {Node[]} nodes A list of SSML nodes.
+ * @param {SsmlElement[]} instr Accumulator for collating Ssml annotation
+ * elements.
+ * @param {String[]} text A list of text elements.
+ * @param {ProsodyElement?} prosody The currently active prosody elements.
+ */
+function recurseSsml(nodes: Node[], instr: SsmlElement[], text: String[],
+ prosody: ProsodyElement = {}) {
+ for (let node of nodes) {
+ if (node.nodeType === 3) {
+ let content = node.textContent.trim();
+ if (content) {
+ text.push(content);
+ instr.push(Object.assign({text: content}, prosody));
+ }
+ continue;
+ }
+ if (node.nodeType === 1) {
+ let element = node as Element;
+ let tag = element.tagName;
+ if (tag === 'speak') {
+ continue;
+ }
+ if (tag === 'prosody') {
+ recurseSsml(
+ Array.from(node.childNodes), instr, text,
+ getProsody(element, prosody));
+ continue;
+ }
+ switch (tag) {
+ case 'break':
+ instr.push({pause: element.getAttribute('time')});
+ break;
+ case 'mark':
+ instr.push({mark: element.getAttribute('name')});
+ break;
+ case 'say-as':
+ let txt = element.textContent;
+ instr.push(Object.assign({text: txt, character: true}, prosody));
+ text.push(txt);
+ break;
+ }
+ }
+ }
+}
+
+/**
+ * Maps prosody types to scaling functions.
+ */
+// TODO: These should be tweaked after more testing.
+const combinePros: {[key: string]: (x: number, sign: string) => number} = {
+ pitch: (x: number, _sign: string) => 1 * (x / 100),
+ volume: (x: number, _sign: string) => .5 * (x / 100),
+ rate: (x: number, _sign: string) => 1 * (x / 100)
+};
+
+/**
+ * Retrieves prosody annotations from and SSML node.
+ * @param {Element} element The SSML node.
+ * @param {ProsodyElement} prosody The prosody annotation.
+ */
+function getProsody(element: Element, prosody: ProsodyElement) {
+ let combine: ProsodyElement = {};
+ for (let pros of ProsodyKeys) {
+ if (element.hasAttribute(pros)) {
+ let [sign, value] = extractProsody(element.getAttribute(pros));
+ if (!sign) {
+ // TODO: Sort out the base value. It is .5 for volume!
+ combine[pros] = (pros === 'volume') ? .5 : 1;
+ continue;
+ }
+ let orig = prosody[pros] as number;
+ orig = orig ? orig : ((pros === 'volume') ? .5 : 1);
+ let relative = combinePros[pros](parseInt(value, 10), sign);
+ combine[pros] = (sign === '-') ? orig - relative : orig + relative;
+ }
+ }
+ return combine;
+}
+
+/**
+ * Extracts the prosody value from an attribute.
+ */
+const prosodyRegexp = /([\+-]?)([0-9]+)%/;
+
+/**
+ * Extracts the prosody value from an attribute.
+ * @param {string} attr
+ */
+function extractProsody(attr: string) {
+ let match = attr.match(prosodyRegexp);
+ if (!match) {
+ console.warn('Something went wrong with the prosody matching.');
+ return ['', '100'];
+ }
+ return [match[1], match[2]];
+}
+
+/**
+ * Computes the aria-label from the node.
+ * @param {MmlNode} node The Math element.
+ * @param {string=} sep The speech separator. Defaults to space.
+ */
+function getLabel(node: MmlNode, sep: string = ' ') {
+ const attributes = node.attributes;
+ const speech = attributes.getExplicit('data-semantic-speech') as string;
+ if (!speech) {
+ return '';
+ }
+ const label = [speech];
+ const prefix = attributes.getExplicit('data-semantic-prefix') as string;
+ if (prefix) {
+ label.unshift(prefix);
+ }
+ // TODO: check if we need this or if it is automatic by the screen readers.
+ const postfix = attributes.getExplicit('data-semantic-postfix') as string;
+ if (postfix) {
+ label.push(postfix);
+ }
+ // TODO: Do we need to merge wrt. locale in SRE.
+ return label.join(sep);
+}
+
+/**
+ * Builds speechs from SSML markup strings.
+ *
+ * @param {string} speech The speech string.
+ * @param {string=} locale An optional locale.
+ * @param {string=} rate The base speech rate.
+ * @return {[string, SsmlElement[]]} The speech with the ssml annotation structure
+ */
+export function buildSpeech(speech: string, locale: string = 'en',
+ rate: string = '100'): [string, SsmlElement[]] {
+ return ssmlParsing('` +
+ `${speech}`+
+ '');
+}
+
+/**
+ * Retrieve and sets aria and braille labels recursively.
+ * @param {MmlNode} node The root node to search from.
+ */
+export function setAria(node: MmlNode, locale: string) {
+ const attributes = node.attributes;
+ if (!attributes) return;
+ const speech = getLabel(node);
+ if (speech) {
+ attributes.set('aria-label', buildSpeech(speech, locale)[0]);
+ }
+ const braille = node.attributes.getExplicit('data-semantic-braille') as string;
+ if (braille) {
+ attributes.set('aria-braillelabel', braille);
+ }
+ for (let child of node.childNodes) {
+ setAria(child, locale);
+ }
+}
+
+/**
+ * Creates a honking sound.
+ */
+export function honk() {
+ let ac = new AudioContext();
+ let os = ac.createOscillator();
+ os.frequency.value = 300;
+ os.connect(ac.destination);
+ os.start(ac.currentTime);
+ os.stop(ac.currentTime + .05);
+}
diff --git a/ts/a11y/explorer.ts b/ts/a11y/explorer.ts
index 0ebf0f425..f3b44457a 100644
--- a/ts/a11y/explorer.ts
+++ b/ts/a11y/explorer.ts
@@ -123,7 +123,7 @@ export function ExplorerMathItemMixin>(
if (!this.explorers) {
this.explorers = new ExplorerPool();
}
- this.explorers.init(document, node, mml);
+ this.explorers.init(document, node, mml, this);
}
this.state(STATE.EXPLORER);
}
@@ -199,7 +199,9 @@ export function ExplorerMathDocumentMixin {
let explorer = ke.SpeechExplorer.create(
- doc, pool, doc.explorerRegions.speechRegion, node, ...rest) as ke.SpeechExplorer;
- explorer.speechGenerator.setOptions({
- automark: true as any, markup: 'ssml',
- locale: doc.options.sre.locale, domain: doc.options.sre.domain,
- style: doc.options.sre.style, modality: 'speech'});
- // This weeds out the case of providing a non-existent locale option.
- let locale = explorer.speechGenerator.getOptions().locale;
- if (locale !== Sre.engineSetup().locale) {
- doc.options.sre.locale = Sre.engineSetup().locale;
- explorer.speechGenerator.setOptions({locale: doc.options.sre.locale});
- }
+ doc, pool, doc.explorerRegions.speechRegion, node,
+ doc.explorerRegions.brailleRegion, doc.explorerRegions.magnifier, rest[0], rest[1]) as ke.SpeechExplorer;
explorer.sound = true;
- explorer.showRegion = 'subtitles';
- return explorer;
- },
- braille: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => {
- let explorer = ke.SpeechExplorer.create(
- doc, pool, doc.explorerRegions.brailleRegion, node, ...rest) as ke.SpeechExplorer;
- explorer.speechGenerator.setOptions({automark: false as any, markup: 'none',
- locale: 'nemeth', domain: 'default',
- style: 'default', modality: 'braille'});
- explorer.showRegion = 'viewBraille';
return explorer;
},
- keyMagnifier: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) =>
- ke.Magnifier.create(doc, pool, doc.explorerRegions.magnifier, node, ...rest),
mouseMagnifier: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ..._rest: any[]) =>
me.ContentHoverer.create(doc, pool, doc.explorerRegions.magnifier, node,
(x: HTMLElement) => x.hasAttribute('data-semantic-type'),
@@ -212,13 +191,14 @@ export class ExplorerPool {
* @param mml The corresponding Mathml node as a string.
*/
public init(document: ExplorerMathDocument,
- node: HTMLElement, mml: string) {
+ node: HTMLElement, mml: string,
+ item: ExplorerMathItem) {
this.document = document;
this.mml = mml;
this.node = node;
this.setPrimaryHighlighter();
for (let key of Object.keys(allExplorers)) {
- this.explorers[key] = allExplorers[key](this.document, this, this.node, this.mml);
+ this.explorers[key] = allExplorers[key](this.document, this, this.node, this.mml, item);
}
this.setSecondaryHighlighter();
this.attach();
@@ -233,7 +213,7 @@ export class ExplorerPool {
let keyExplorers = [];
for (let key of Object.keys(this.explorers)) {
let explorer = this.explorers[key];
- if (explorer instanceof ke.AbstractKeyExplorer) {
+ if (explorer instanceof ke.SpeechExplorer) {
explorer.AddEvents();
explorer.stoppable = false;
keyExplorers.unshift(explorer);
diff --git a/ts/a11y/explorer/KeyExplorer.ts b/ts/a11y/explorer/KeyExplorer.ts
index 93e353e42..0bb46e1f0 100644
--- a/ts/a11y/explorer/KeyExplorer.ts
+++ b/ts/a11y/explorer/KeyExplorer.ts
@@ -23,11 +23,16 @@
*/
-import {A11yDocument, Region} from './Region.js';
+import {A11yDocument, HoverRegion, SpeechRegion, LiveRegion} from './Region.js';
+import type { ExplorerMathItem } from '../explorer.js';
import {Explorer, AbstractExplorer} from './Explorer.js';
import {ExplorerPool} from './ExplorerPool.js';
+import {MmlNode} from '../../core/MmlTree/MmlNode.js';
+import { honk } from '../SpeechUtil.js';
import {Sre} from '../sre.js';
+// import { Walker } from './Walker.js';
+
/**
* Interface for keyboard explorers. Adds the necessary keyboard events.
@@ -58,7 +63,7 @@ export interface KeyExplorer extends Explorer {
* Move made on keypress.
* @param key The key code of the pressed key.
*/
- Move(key: number): void;
+ Move(event: KeyboardEvent): void;
/**
* A method that is executed if no move is executed.
@@ -68,13 +73,22 @@ export interface KeyExplorer extends Explorer {
}
+const codeSelector = 'mjx-container';
+const roles = ['tree', 'group', 'treeitem'];
+const nav = roles.map(x => `[role="${x}"]`).join(',');
+const prevNav = roles.map(x => `[tabindex="0"][role="${x}"]`).join(',');
+
+function isCodeBlock(el: HTMLElement) {
+ return el.matches(codeSelector);
+}
+
/**
* @constructor
* @extends {AbstractExplorer}
*
* @template T The type that is consumed by the Region of this explorer.
*/
-export abstract class AbstractKeyExplorer extends AbstractExplorer implements KeyExplorer {
+export class SpeechExplorer extends AbstractExplorer implements KeyExplorer {
/**
* Flag indicating if the explorer is attached to an object.
@@ -94,14 +108,50 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme
private eventsAttached: boolean = false;
+ protected current: HTMLElement = null;
+
+ private move = false;
+
+ private mousedown = false;
+
/**
* @override
*/
protected events: [string, (x: Event) => void][] =
super.Events().concat(
- [['keydown', this.KeyDown.bind(this)],
- ['focusin', this.FocusIn.bind(this)],
- ['focusout', this.FocusOut.bind(this)]]);
+ [
+ ['keydown', this.KeyDown.bind(this)],
+ ['mousedown', this.MouseDown.bind(this)],
+ ['click', this.Click.bind(this)],
+ ['focusin', this.FocusIn.bind(this)],
+ ['focusout', this.FocusOut.bind(this)]
+ ]);
+
+ /**
+ * Records a mouse down event on the element. This ensures that focus events
+ * only fire if they were not triggered by a mouse click.
+ *
+ * @param e The mouse event.
+ */
+ private MouseDown(e: MouseEvent) {
+ this.mousedown = true;
+ e.preventDefault();
+ }
+
+ public Click(e: MouseEvent) {
+ const clicked = (e.target as HTMLElement).closest(nav) as HTMLElement;
+ if (this.node.contains(clicked)) {
+ const prev = this.node.querySelector(prevNav);
+ if (prev) {
+ prev.removeAttribute('tabindex');
+ }
+ this.current = clicked;
+ if (!this.triggerLinkMouse()) {
+ this.Start()
+ }
+ e.preventDefault();
+ }
+ }
/**
* The original tabindex value before explorer was attached.
@@ -112,33 +162,23 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme
/**
* @override
*/
- public abstract KeyDown(event: KeyboardEvent): void;
-
- /**
- * @override
- */
- public FocusIn(_event: FocusEvent) {
+ public FocusIn(event: FocusEvent) {
+ if (this.mousedown) {
+ this.mousedown = false;
+ return;
+ }
+ this.current = this.current || this.node.querySelector('[role="tree"]');
+ this.Start();
+ event.preventDefault();
}
/**
* @override
*/
public FocusOut(_event: FocusEvent) {
- this.Stop();
- }
-
- /**
- * @override
- */
- public Update(force: boolean = false) {
- if (!this.active && !force) return;
- this.pool.unhighlight();
- let nodes = this.walker.getFocus(true).getNodes();
- if (!nodes.length) {
- this.walker.refocus();
- nodes = this.walker.getFocus().getNodes();
+ if (!this.move) {
+ this.Stop();
}
- this.pool.highlight(nodes as HTMLElement[]);
}
/**
@@ -148,8 +188,7 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme
super.Attach();
this.attached = true;
this.oldIndex = this.node.tabIndex;
- this.node.tabIndex = 1;
- this.node.setAttribute('role', 'tree');
+ this.node.tabIndex = 0;
}
/**
@@ -174,53 +213,77 @@ export abstract class AbstractKeyExplorer extends AbstractExplorer impleme
this.attached = false;
}
- /**
- * @override
- */
- public Stop() {
- if (this.active) {
- this.walker.deactivate();
- this.pool.unhighlight();
+ protected nextSibling(el: HTMLElement): HTMLElement {
+ const sib = el.nextElementSibling as HTMLElement;
+ if (sib) {
+ if (sib.matches(nav)) {
+ return sib;
+ }
+ const sibChild = sib.querySelector(nav) as HTMLElement;
+ return sibChild ?? this.nextSibling(sib);
}
- super.Stop();
+ if (!isCodeBlock(el) && !el.parentElement.matches(nav)) {
+ return this.nextSibling(el.parentElement);
+ }
+ return null;
+ }
+
+ protected prevSibling(el: HTMLElement): HTMLElement {
+ const sib = el.previousElementSibling as HTMLElement;
+ if (sib) {
+ if (sib.matches(nav)) {
+ return sib;
+ }
+ const sibChild = sib.querySelector(nav) as HTMLElement;
+ return sibChild ?? this.prevSibling(sib);
+ }
+ if (!isCodeBlock(el) && !el.parentElement.matches(nav)) {
+ return this.prevSibling(el.parentElement);
+ }
+ return null;
}
+ protected moves: Map HTMLElement | null> = new Map([
+ ['ArrowDown', (node: HTMLElement) => node.querySelector(nav)],
+ ['ArrowUp', (node: HTMLElement) => node.parentElement.closest(nav)],
+ ['ArrowLeft', this.prevSibling.bind(this)],
+ ['ArrowRight', this.nextSibling.bind(this)],
+ ['>', (_node: HTMLElement) => {
+ return null;
+ }],
+ ]);
+
/**
* @override
*/
- public Move(key: number) {
- let result = this.walker.move(key);
- if (result) {
- this.Update();
- return;
+ public Move(e: KeyboardEvent) {
+ this.move = true;
+ const target = e.target as HTMLElement;
+ const move = this.moves.get(e.key);
+ let next = null;
+ if (move) {
+ e.preventDefault();
+ next = move(target);
}
- if (this.sound) {
- this.NoMove();
+ if (next) {
+ target.removeAttribute('tabindex');
+ next.setAttribute('tabindex', '0');
+ next.focus();
+ this.current = next;
+ this.move = false;
+ return true;
}
+ this.move = false;
+ return false;
}
/**
* @override
*/
public NoMove() {
- let ac = new AudioContext();
- let os = ac.createOscillator();
- os.frequency.value = 300;
- os.connect(ac.destination);
- os.start(ac.currentTime);
- os.stop(ac.currentTime + .05);
+ honk();
}
-}
-
-
-/**
- * Explorer that pushes speech to live region.
- * @constructor
- * @extends {AbstractKeyExplorer}
- */
-export class SpeechExplorer extends AbstractKeyExplorer {
-
private static updatePromise = Promise.resolve();
/**
@@ -235,7 +298,7 @@ export class SpeechExplorer extends AbstractKeyExplorer {
*/
public showRegion: string = 'subtitles';
- private init: boolean = false;
+ // private init: boolean = false;
/**
* Flag in case the start method is triggered before the walker is fully
@@ -251,11 +314,14 @@ export class SpeechExplorer extends AbstractKeyExplorer {
*/
constructor(public document: A11yDocument,
public pool: ExplorerPool,
- public region: Region,
+ public region: SpeechRegion,
protected node: HTMLElement,
- private mml: string) {
- super(document, pool, region, node);
- this.initWalker();
+ public brailleRegion: LiveRegion,
+ public magnifyRegion: HoverRegion,
+ _mml: MmlNode,
+ public item: ExplorerMathItem
+ ) {
+ super(document, pool, null, node);
}
@@ -264,34 +330,52 @@ export class SpeechExplorer extends AbstractKeyExplorer {
*/
public Start() {
if (!this.attached) return;
- let options = this.getOptions();
- if (!this.init) {
- this.init = true;
- SpeechExplorer.updatePromise = SpeechExplorer.updatePromise.then(async () => {
- return Sre.sreReady()
- .then(() => Sre.setupEngine({locale: options.locale}))
- .then(() => {
- // Important that both are in the same block so speech explorers
- // are restarted sequentially.
- this.Speech(this.walker);
- this.Start();
- });
- })
- .catch((error: Error) => console.log(error.message));
- return;
- }
+ if (this.active) return;
+ this.current.setAttribute('tabindex', '0');
+ this.current.focus();
super.Start();
- this.speechGenerator = Sre.getSpeechGenerator('Direct');
- this.speechGenerator.setOptions(options);
- this.walker = Sre.getWalker(
- 'table', this.node, this.speechGenerator, this.highlighter, this.mml);
- this.walker.activate();
- this.Update();
- if (this.document.options.a11y[this.showRegion]) {
+ // let options = this.getOptions();
+ // if (!this.init) {
+ // this.init = true;
+ // SpeechExplorer.updatePromise = SpeechExplorer.updatePromise.then(async () => {
+ // return Sre.sreReady()
+ // .then(() => Sre.setupEngine({locale: options.locale}))
+ // .then(() => {
+ // // Important that both are in the same block so speech explorers
+ // // are restarted sequentially.
+ // this.Speech(this.walker);
+ // })
+ // .then(() => Sre.setupEngine({automark: false as any, markup: 'none',
+ // locale: 'nemeth', domain: 'default',
+ // style: 'default', modality: 'braille'}))
+ // .then(() => {
+ // this.speechGenerator.setOptions({automark: false as any, markup: 'none',
+ // locale: 'nemeth', domain: 'default',
+ // style: 'default', modality: 'braille'});
+ // this.Speech(this.walker);
+ // this.Start();
+ // });
+ // })
+ // return;
+ // }
+ // this.speechGenerator = Sre.getSpeechGenerator('Direct');
+ // this.speechGenerator.setOptions(options);
+ // this.walker = Sre.getWalker(
+ // 'table', this.node, this.speechGenerator, this.highlighter, this.mml);
+ // this.walker.activate();
+ if (this.document.options.a11y.subtitles) {
+ SpeechExplorer.updatePromise.then(
+ () => this.region.Show(this.node, this.highlighter))
+ }
+ if (this.document.options.a11y.viewBraille) {
SpeechExplorer.updatePromise.then(
- () => this.region.Show(this.node, this.highlighter));
+ () => this.brailleRegion.Show(this.node, this.highlighter))
}
- this.restarted = true;
+ if (this.document.options.a11y.keyMagnifier) {
+ this.magnifyRegion.Show(this.node, this.highlighter);
+ }
+ this.Update();
+ // this.restarted = true;
}
@@ -301,30 +385,41 @@ export class SpeechExplorer extends AbstractKeyExplorer {
public Update(force: boolean = false) {
// TODO (v4): This is a hack to avoid double voicing on initial startup!
// Make that cleaner and remove force as it is not really used!
- let noUpdate = force;
- force = false;
- super.Update(force);
- let options = this.speechGenerator.getOptions();
+ // let noUpdate = force;
+ if (!this.active && !force) return;
+ this.pool.unhighlight();
+ // let nodes = this.walker.getFocus(true).getNodes();
+ // if (!nodes.length) {
+ // this.walker.refocus();
+ // nodes = this.walker.getFocus().getNodes();
+ // }
+ this.pool.highlight([this.current]);
+ this.region.node = this.node;
+ this.region.Update(this.current.getAttribute('data-semantic-speech'));
+ this.brailleRegion.Update(this.current.getAttribute('aria-braillelabel'));
+ this.magnifyRegion.Update(this.current);
+ // let options = this.speechGenerator.getOptions();
// This is a necessary in case speech options have changed via keypress
// during walking.
- if (options.modality === 'speech') {
- this.document.options.sre.domain = options.domain;
- this.document.options.sre.style = options.style;
- this.document.options.a11y.speechRules =
- options.domain + '-' + options.style;
- }
- SpeechExplorer.updatePromise = SpeechExplorer.updatePromise.then(async () => {
- return Sre.sreReady()
- .then(() => Sre.setupEngine({markup: options.markup,
- modality: options.modality,
- locale: options.locale}))
- .then(() => {
- if (!noUpdate) {
- let speech = this.walker.speech();
- this.region.Update(speech);
- }
- });
- });
+ // if (options.modality === 'speech') {
+ // this.document.options.sre.domain = options.domain;
+ // this.document.options.sre.style = options.style;
+ // this.document.options.a11y.speechRules =
+ // options.domain + '-' + options.style;
+ // }
+ // Ensure this autovoicing is retained later:
+ // SpeechExplorer.updatePromise = SpeechExplorer.updatePromise.then(async () => {
+ // return Sre.sreReady()
+ // .then(() => Sre.setupEngine({markup: options.markup,
+ // modality: options.modality,
+ // locale: options.locale}))
+ // .then(() => {
+ // if (!noUpdate) {
+ // let speech = this.walker.speech();
+ // this.region.Update(speech);
+ // }
+ // });
+ // });
}
@@ -348,38 +443,71 @@ export class SpeechExplorer extends AbstractKeyExplorer {
* @override
*/
public KeyDown(event: KeyboardEvent) {
- const code = event.keyCode;
- this.walker.modifier = event.shiftKey;
- if (code === 17) {
+ const code = event.key;
+ // this.walker.modifier = event.shiftKey;
+ if (code === 'Control') {
speechSynthesis.cancel();
return;
}
- if (code === 27) {
+ if (code === 'Escape') {
this.Stop();
this.stopEvent(event);
return;
}
- if (this.active) {
- this.Move(code);
- if (this.triggerLink(code)) return;
- this.stopEvent(event);
- return;
+ if (code === 'Enter') {
+ if (!this.active && event.target instanceof HTMLAnchorElement) {
+ event.target.dispatchEvent(new MouseEvent('click'));
+ this.stopEvent(event);
+ return;
+ }
+ if (this.active && this.triggerLinkKeyboard(event)) {
+ this.Stop()
+ this.stopEvent(event);
+ return;
+ }
+ if (!this.active) {
+ if (!this.current) {
+ this.current = this.node.querySelector('[role="tree"]');
+ }
+ this.Start();
+ this.stopEvent(event);
+ return;
+ }
}
- if (code === 32 && event.shiftKey || code === 13) {
- this.Start();
+ if (this.active) {
this.stopEvent(event);
+ if (this.Move(event)) {
+ this.Update();
+ return;
+ }
+ if (event.getModifierState(code)) {
+ return;
+ }
+ if (this.sound) {
+ this.NoMove();
+ }
}
}
/**
* Programmatically triggers a link if the focused node contains one.
- * @param {number} code The keycode of the last key pressed.
+ * @param {KeyboardEvent} event The keyboard event for the last keydown event.
*/
- protected triggerLink(code: number) {
- if (code !== 13) {
+ protected triggerLinkKeyboard(event: KeyboardEvent) {
+ if (event.code !== 'Enter') {
return false;
}
- let node = this.walker.getFocus().getNodes()?.[0];
+ if (!this.current) {
+ if (event.target instanceof HTMLAnchorElement) {
+ event.target.dispatchEvent(new MouseEvent('click'));
+ return true;
+ }
+ return false;
+ }
+ return this.triggerLink(this.current);
+ }
+
+ protected triggerLink(node: HTMLElement) {
let focus = node?.
getAttribute('data-semantic-postfix')?.
match(/(^| )link($| )/);
@@ -390,14 +518,19 @@ export class SpeechExplorer extends AbstractKeyExplorer {
return false;
}
+
/**
- * Initialises the Sre walker.
+ * Programmatically triggers a link if the clicked mouse contains one.
*/
- private initWalker() {
- this.speechGenerator = Sre.getSpeechGenerator('Tree');
- let dummy = Sre.getWalker(
- 'dummy', this.node, this.speechGenerator, this.highlighter, this.mml);
- this.walker = dummy;
+ protected triggerLinkMouse() {
+ let node = this.current;
+ while (node && node !== this.node) {
+ if (this.triggerLink(node)) {
+ return true;
+ }
+ node = node.parentNode as HTMLElement;
+ }
+ return false;
}
/**
@@ -405,7 +538,7 @@ export class SpeechExplorer extends AbstractKeyExplorer {
* @return {{[key: string]: string}} The options settings for the speech
* generator.
*/
- private getOptions(): {[key: string]: string} {
+ protected getOptions(): {[key: string]: string} {
let options = this.speechGenerator.getOptions();
let sreOptions = this.document.options.sre;
if (options.modality === 'speech' &&
@@ -420,78 +553,20 @@ export class SpeechExplorer extends AbstractKeyExplorer {
return options;
}
-}
-
-
-/**
- * Explorer that magnifies what is currently explored. Uses a hover region.
- * @constructor
- * @extends {AbstractKeyExplorer}
- */
-export class Magnifier extends AbstractKeyExplorer {
-
- /**
- * @constructor
- * @extends {AbstractKeyExplorer}
- */
- constructor(public document: A11yDocument,
- public pool: ExplorerPool,
- public region: Region,
- protected node: HTMLElement,
- private mml: string) {
- super(document, pool, region, node);
- this.walker = Sre.getWalker(
- 'table', this.node, Sre.getSpeechGenerator('Dummy'),
- this.highlighter, this.mml);
- }
-
/**
* @override
*/
- public Update(force: boolean = false) {
- super.Update(force);
- this.showFocus();
+ public Stop() {
+ if (this.active) {
+ this.current.removeAttribute('tabindex');
+ this.pool.unhighlight();
+ this.magnifyRegion.Hide();
+ this.region.Hide();
+ this.brailleRegion.Hide();
+ }
+ super.Stop();
}
- /**
- * @override
- */
- public Start() {
- super.Start();
- if (!this.attached) return;
- this.region.Show(this.node, this.highlighter);
- this.walker.activate();
- this.Update();
- }
- /**
- * Shows the nodes that are currently focused.
- */
- private showFocus() {
- let node = this.walker.getFocus().getNodes()[0] as HTMLElement;
- this.region.Show(node, this.highlighter);
- }
-
- /**
- * @override
- */
- public KeyDown(event: KeyboardEvent) {
- const code = event.keyCode;
- this.walker.modifier = event.shiftKey;
- if (code === 27) {
- this.Stop();
- this.stopEvent(event);
- return;
- }
- if (this.active && code !== 13) {
- this.Move(code);
- this.stopEvent(event);
- return;
- }
- if (code === 32 && event.shiftKey || code === 13) {
- this.Start();
- this.stopEvent(event);
- }
- }
}
diff --git a/ts/a11y/explorer/Region.ts b/ts/a11y/explorer/Region.ts
index de0fcbc26..aeafde224 100644
--- a/ts/a11y/explorer/Region.ts
+++ b/ts/a11y/explorer/Region.ts
@@ -26,6 +26,7 @@
import {MathDocument} from '../../core/MathDocument.js';
import {CssStyles} from '../../util/StyleList.js';
import {Sre} from '../sre.js';
+import {SsmlElement, buildSpeech} from '../SpeechUtil.js';
export type A11yDocument = MathDocument;
@@ -90,7 +91,7 @@ export abstract class AbstractRegion implements Region {
* The outer div node.
* @type {HTMLElement}
*/
- protected div: HTMLElement;
+ public div: HTMLElement;
/**
* The inner node.
@@ -210,8 +211,9 @@ export abstract class AbstractRegion implements Region {
baseLeft = Math.min(region.getBoundingClientRect().left, baseLeft);
}
}
- const bot = (baseBottom ? baseBottom : rect.bottom + 10) + window.pageYOffset;
- const left = (baseLeft < Number.POSITIVE_INFINITY ? baseLeft : rect.left) + window.pageXOffset;
+
+ const bot = (baseBottom ? baseBottom : rect.bottom + 10) + window.scrollY;
+ const left = (baseLeft < Number.POSITIVE_INFINITY ? baseLeft : rect.left) + window.scrollX;
this.div.style.top = bot + 'px';
this.div.style.left = left + 'px';
}
@@ -354,36 +356,8 @@ export class LiveRegion extends StringRegion {
}
});
-
- /**
- * @constructor
- * @param {A11yDocument} document The document the live region is added to.
- */
- constructor(public document: A11yDocument) {
- super(document);
- this.div.setAttribute('aria-live', 'assertive');
- }
-
-}
-
-
-const ProsodyKeys = [ 'pitch', 'rate', 'volume' ];
-
-interface ProsodyElement {
- [propName: string]: string | boolean | number;
- pitch?: number;
- rate?: number;
- volume?: number;
}
-interface SsmlElement extends ProsodyElement {
- [propName: string]: string | boolean | number;
- pause?: string;
- text?: string;
- mark?: string;
- character?: boolean;
- kind?: string;
-}
/**
* Region class that enables auto voicing of content via SSML markup.
@@ -424,15 +398,43 @@ export class SpeechRegion extends LiveRegion {
super.Show(node, highlighter);
}
+ /**
+ * Have we already requested voices from the browser?
+ */
+ private voiceRequest = false;
+
/**
* @override
*/
public Update(speech: string) {
+ if (this.voiceRequest) {
+ this.makeVoice(speech);
+ return;
+ }
+ speechSynthesis.onvoiceschanged = (() => this.voiceRequest = true).bind(this);
+ super.Update('\u00a0'); // Ensures region shown and cannot be overwritten.
+ const promise = new Promise((resolve) => {
+ setTimeout(() => {
+ if (this.voiceRequest) {
+ resolve(true);
+ }
+ }, 100);
+ });
+ promise.then(
+ () => this.makeVoice(speech)
+ );
+ }
+
+ private makeVoice(speech: string) {
this.active = this.document.options.a11y.voicing &&
!!speechSynthesis.getVoices().length;
speechSynthesis.cancel();
this.clear = true;
- let [text, ssml] = this.ssmlParsing(speech);
+ let [text, ssml] = buildSpeech(
+ speech,
+ this.document.options.sre.locale,
+ this.document.options.sre.rate
+ );
super.Update(text);
if (this.active && text) {
this.makeUtterances(ssml, this.document.options.sre.locale);
@@ -444,11 +446,12 @@ export class SpeechRegion extends LiveRegion {
* @param {SsmlElement[]} ssml The list of ssml annotations.
* @param {string} locale The locale to use.
*/
- private makeUtterances(ssml: SsmlElement[], locale: string) {
+ protected makeUtterances(ssml: SsmlElement[], locale: string) {
let utterance = null;
for (let utter of ssml) {
if (utter.mark) {
if (!utterance) {
+ // First utterance, call with init = true.
this.highlightNode(utter.mark, true);
continue;
}
@@ -504,125 +507,6 @@ export class SpeechRegion extends LiveRegion {
}
- /**
- * Parses a string containing an ssml structure into a list of text strings
- * with associated ssml annotation elements.
- *
- * @param {string} speech The speech string.
- * @return {[string, SsmlElement[]]} The annotation structure.
- */
- private ssmlParsing(speech: string): [string, SsmlElement[]] {
- let dp = new DOMParser();
- let xml = dp.parseFromString(speech, 'text/xml');
- let instr: SsmlElement[] = [];
- let text: String[] = [];
- this.recurseSsml(Array.from(xml.documentElement.childNodes), instr, text);
- return [text.join(' '), instr];
- }
-
- /**
- * Tail recursive combination of SSML components.
- *
- * @param {Node[]} nodes A list of SSML nodes.
- * @param {SsmlElement[]} instr Accumulator for collating Ssml annotation
- * elements.
- * @param {String[]} text A list of text elements.
- * @param {ProsodyElement?} prosody The currently active prosody elements.
- */
- private recurseSsml(nodes: Node[], instr: SsmlElement[], text: String[],
- prosody: ProsodyElement = {}) {
- for (let node of nodes) {
- if (node.nodeType === 3) {
- let content = node.textContent.trim();
- if (content) {
- text.push(content);
- instr.push(Object.assign({text: content}, prosody));
- }
- continue;
- }
- if (node.nodeType === 1) {
- let element = node as Element;
- let tag = element.tagName;
- if (tag === 'speak') {
- continue;
- }
- if (tag === 'prosody') {
- this.recurseSsml(
- Array.from(node.childNodes), instr, text,
- this.getProsody(element, prosody));
- continue;
- }
- switch (tag) {
- case 'break':
- instr.push({pause: element.getAttribute('time')});
- break;
- case 'mark':
- instr.push({mark: element.getAttribute('name')});
- break;
- case 'say-as':
- let txt = element.textContent;
- instr.push(Object.assign({text: txt, character: true}, prosody));
- text.push(txt);
- break;
- default:
- break;
- }
- }
- }
- }
-
- /**
- * Maps prosody types to scaling functions.
- */
- // TODO: These should be tweaked after more testing.
- private static combinePros: {[key: string]: (x: number, sign: string) => number} = {
- pitch: (x: number, _sign: string) => 1 * (x / 100),
- volume: (x: number, _sign: string) => .5 * (x / 100),
- rate: (x: number, _sign: string) => 1 * (x / 100)
- };
-
- /**
- * Retrieves prosody annotations from and SSML node.
- * @param {Element} element The SSML node.
- * @param {ProsodyElement} prosody The prosody annotation.
- */
- private getProsody(element: Element, prosody: ProsodyElement) {
- let combine: ProsodyElement = {};
- for (let pros of ProsodyKeys) {
- if (element.hasAttribute(pros)) {
- let [sign, value] = SpeechRegion.extractProsody(element.getAttribute(pros));
- if (!sign) {
- // TODO: Sort out the base value. It is .5 for volume!
- combine[pros] = (pros === 'volume') ? .5 : 1;
- continue;
- }
- let orig = prosody[pros] as number;
- orig = orig ? orig : ((pros === 'volume') ? .5 : 1);
- let relative = SpeechRegion.combinePros[pros](parseInt(value, 10), sign);
- combine[pros] = (sign === '-') ? orig - relative : orig + relative;
- }
- }
- return combine;
- }
-
- /**
- * Extracts the prosody value from an attribute.
- */
- private static prosodyRegexp = /([\+|-]*)([0-9]+)%/;
-
- /**
- * Extracts the prosody value from an attribute.
- * @param {string} attr
- */
- private static extractProsody(attr: string) {
- let match = attr.match(SpeechRegion.prosodyRegexp);
- if (!match) {
- console.warn('Something went wrong with the prosody matching.');
- return ['', '100'];
- }
- return [match[1], match[2]];
- }
-
}
@@ -674,7 +558,7 @@ export class HoverRegion extends AbstractRegion {
const xCenter = nodeRect.left + (nodeRect.width / 2);
let left = xCenter - (divRect.width / 2);
left = (left < 0) ? 0 : left;
- left = left + window.pageXOffset;
+ left = left + window.scrollX;
let top;
switch (this.document.options.a11y.align) {
case 'top':
@@ -688,7 +572,7 @@ export class HoverRegion extends AbstractRegion {
const yCenter = nodeRect.top + (nodeRect.height / 2);
top = yCenter - (divRect.height / 2);
}
- top = top + window.pageYOffset;
+ top = top + window.scrollY;
top = (top < 0) ? 0 : top;
this.div.style.top = top + 'px';
this.div.style.left = left + 'px';
diff --git a/ts/a11y/semantic-enrich.ts b/ts/a11y/semantic-enrich.ts
index 6dcfa748d..31e62438c 100644
--- a/ts/a11y/semantic-enrich.ts
+++ b/ts/a11y/semantic-enrich.ts
@@ -30,13 +30,15 @@ import {MathML} from '../input/mathml.js';
import {SerializedMmlVisitor} from '../core/MmlTree/SerializedMmlVisitor.js';
import {OptionList, expandable} from '../util/Options.js';
import {Sre} from './sre.js';
+import { buildSpeech, setAria } from './SpeechUtil.js';
/*==========================================================================*/
/**
* The current speech setting for Sre
*/
-let currentSpeech = 'none';
+let currentLocale = 'none';
+let currentBraille = 'none';
/**
* Generic constructor for Mixins
@@ -134,6 +136,11 @@ export function EnrichedMathItemMixin, force: boolean = false) {
if (this.state() >= STATE.ENRICHED) return;
if (!this.isEscaped && (document.options.enableEnrichment || force)) {
- if (document.options.sre.speech !== currentSpeech) {
- currentSpeech = document.options.sre.speech;
- mathjax.retryAfter(
- Sre.setupEngine(document.options.sre).then(
- () => Sre.sreReady()));
+ // TODO: Sort out the loading of the locales better
+ if (document.options.enableSpeech) {
+ if (document.options.sre.locale !== currentLocale) {
+ currentLocale = document.options.sre.locale;
+ // TODO: Sort out the loading of the locales better
+ mathjax.retryAfter(
+ Sre.setupEngine({locale: document.options.sre.locale})
+ .then(() => Sre.sreReady()));
+ }
+ if (document.options.sre.braille !== currentBraille) {
+ currentBraille = document.options.sre.braille;
+ mathjax.retryAfter(
+ Sre.setupEngine({locale: document.options.sre.braille})
+ .then(() => Sre.sreReady()));
+ }
}
const math = new document.options.MathItem('', MmlJax);
try {
@@ -177,7 +194,28 @@ export function EnrichedMathItemMixin) {
if (this.state() >= STATE.ATTACHSPEECH) return;
const attributes = this.root.attributes;
- const speech = (attributes.get('aria-label') ||
- this.getSpeech(this.root)) as string;
+ const speech = (attributes.get('aria-label') || this.outputData.speech);
+ const braille = (attributes.get('aria-braillelabel') || this.outputData.braille);
+ if (!speech && !braille) {
+ this.state(STATE.ATTACHSPEECH);
+ return;
+ }
+ const adaptor = document.adaptor;
+ const node = this.typesetRoot;
if (speech) {
- const adaptor = document.adaptor;
- const node = this.typesetRoot;
- adaptor.setAttribute(node, 'aria-label', speech);
- for (const child of adaptor.childNodes(node) as N[]) {
- adaptor.setAttribute(child, 'aria-hidden', 'true');
- }
- this.outputData.speech = speech;
+ adaptor.setAttribute(node, 'aria-label', speech as string);
+ }
+ if (braille) {
+ adaptor.setAttribute(node, 'aria-braillelabel', braille as string);
+ }
+ for (const child of adaptor.childNodes(node) as N[]) {
+ adaptor.setAttribute(child, 'aria-hidden', 'true');
}
+ this.outputData.speech = speech;
+ this.outputData.braille = braille;
this.state(STATE.ATTACHSPEECH);
}
@@ -311,6 +360,7 @@ export function EnrichedMathDocumentMixin,
math: EnrichedMathItem,
err: Error) => doc.enrichError(doc, math, err),
@@ -320,11 +370,12 @@ export function EnrichedMathDocumentMixin svg a': {
fill: 'blue', stroke: 'blue'
+ },
+ 'rect[sre-highlighter-added]': {
+ stroke: 'black',
+ 'stroke-width': '40px'
}
};