diff --git a/app/main/appium-method-handler.js b/app/main/appium-method-handler.js index f4ac4db61..5dfe7f426 100644 --- a/app/main/appium-method-handler.js +++ b/app/main/appium-method-handler.js @@ -1,5 +1,6 @@ import Bluebird from 'bluebird'; import _ from 'lodash'; +import wd from 'wd'; export default class AppiumMethodHandler { constructor (driver) { @@ -80,7 +81,20 @@ export default class AppiumMethodHandler { async executeMethod (methodName, args = []) { let res = {}; - if (methodName !== 'source' && methodName !== 'screenshot') { + + // Specially handle the tap and swipe method + if (methodName === 'tap') { + res = await (new wd.TouchAction(this.driver)) + .tap({x: args[0], y: args[1]}) + .perform(); + } else if (methodName === 'swipe') { + const [startX, startY, endX, endY] = args; + res = await (new wd.TouchAction(this.driver)) + .press({x: startX, y: startY}) + .moveTo({x: endX, y: endY}) + .release() + .perform(); + } else if (methodName !== 'source' && methodName !== 'screenshot') { res = await this.driver[methodName].apply(this.driver, args); } @@ -94,7 +108,7 @@ export default class AppiumMethodHandler { res, }; } - + async _getSourceAndScreenshot () { let source, sourceError, screenshot, screenshotError; try { diff --git a/app/main/appium.js b/app/main/appium.js index d6ef9c1ed..70d71856e 100644 --- a/app/main/appium.js +++ b/app/main/appium.js @@ -278,7 +278,7 @@ function connectClientMethodListener () { } = data; let renderer = evt.sender; - let driver = appiumHandlers[renderer.id]; + let methodHandler = appiumHandlers[renderer.id]; try { if (methodName === 'quit') { @@ -296,18 +296,18 @@ function connectClientMethodListener () { if (methodName) { if (elementId) { console.log(`Handling client method request with method '${methodName}', args ${JSON.stringify(args)} and elementId ${elementId}`); - res = await driver.executeElementCommand(elementId, methodName, args); + res = await methodHandler.executeElementCommand(elementId, methodName, args); } else { console.log(`Handling client method request with method '${methodName}' and args ${JSON.stringify(args)}`); - res = await driver.executeMethod(methodName, args); + res = await methodHandler.executeMethod(methodName, args); } } else if (strategy && selector) { if (fetchArray) { console.log(`Fetching elements with selector '${selector}' and strategy ${strategy}`); - res = await driver.fetchElements(strategy, selector); + res = await methodHandler.fetchElements(strategy, selector); } else { console.log(`Fetching an element with selector '${selector}' and strategy ${strategy}`); - res = await driver.fetchElement(strategy, selector); + res = await methodHandler.fetchElement(strategy, selector); } } @@ -324,7 +324,7 @@ function connectClientMethodListener () { renderer.send('appium-session-done', e); } console.log('Caught an exception: ', e); - renderer.send('appium-client-command-response-error', {e, uuid}); + renderer.send('appium-client-command-response-error', {e: e.message, uuid}); } }); } diff --git a/app/renderer/actions/Inspector.js b/app/renderer/actions/Inspector.js index 4d3e25122..44397f0ea 100644 --- a/app/renderer/actions/Inspector.js +++ b/app/renderer/actions/Inspector.js @@ -47,6 +47,10 @@ export const ADD_ASSIGNED_VAR_CACHE = 'ADD_ASSIGNED_VAR_CACHE'; export const CLEAR_ASSIGNED_VAR_CACHE = 'CLEAR_ASSIGNED_VAR_CACHE'; export const SET_SCREENSHOT_INTERACTION_MODE = 'SET_SCREENSHOT_INTERACTION_MODE'; +export const SET_SWIPE_START = 'SET_SWIPE_START'; +export const SET_SWIPE_END = 'SET_SWIPE_END'; +export const CLEAR_SWIPE_ACTION = 'CLEAR_SWIPE_ACTION'; + // Attributes on nodes that we know are unique to the node const uniqueAttributes = [ 'name', @@ -375,4 +379,22 @@ export function selectScreenshotInteractionMode (screenshotInteractionMode) { return (dispatch) => { dispatch({type: SET_SCREENSHOT_INTERACTION_MODE, screenshotInteractionMode }); }; +} + +export function setSwipeStart (swipeStartX, swipeStartY) { + return (dispatch) => { + dispatch({type: SET_SWIPE_START, swipeStartX, swipeStartY}); + }; +} + +export function setSwipeEnd (swipeEndX, swipeEndY) { + return (dispatch) => { + dispatch({type: SET_SWIPE_END, swipeEndX, swipeEndY}); + }; +} + +export function clearSwipeAction () { + return (dispatch) => { + dispatch({type: CLEAR_SWIPE_ACTION}); + }; } \ No newline at end of file diff --git a/app/renderer/actions/shared.js b/app/renderer/actions/shared.js index 622ecec99..4da6dc8ae 100644 --- a/app/renderer/actions/shared.js +++ b/app/renderer/actions/shared.js @@ -27,7 +27,7 @@ if (ipcRenderer) { const {e, uuid} = resp; let promise = clientMethodPromises[uuid]; if (promise) { - promise.reject(e); + promise.reject(new Error(e)); delete clientMethodPromises[uuid]; } }); diff --git a/app/renderer/components/Inspector/Actions.js b/app/renderer/components/Inspector/Actions.js index 13992d9d9..aadbfd1a6 100644 --- a/app/renderer/components/Inspector/Actions.js +++ b/app/renderer/components/Inspector/Actions.js @@ -1,5 +1,5 @@ import React, { Component } from 'react'; -import { Button, Radio, Icon } from 'antd'; +import { Radio } from 'antd'; /** @@ -8,8 +8,8 @@ import { Button, Radio, Icon } from 'antd'; export default class Actions extends Component { handleScreenshotInteractionChange (e) { - const {selectScreenshotInteractionMode} = this.props; - + const {selectScreenshotInteractionMode, clearSwipeAction} = this.props; + clearSwipeAction(); // When the action changes, reset the swipe action selectScreenshotInteractionMode(e.target.value); } diff --git a/app/renderer/components/Inspector/HighlighterRects.js b/app/renderer/components/Inspector/HighlighterRects.js new file mode 100644 index 000000000..35d98a728 --- /dev/null +++ b/app/renderer/components/Inspector/HighlighterRects.js @@ -0,0 +1,142 @@ +import React, { Component } from 'react'; +import ReactDOM from 'react-dom'; +import { debounce } from 'lodash'; +import HighlighterRect from './HighlighterRect'; +import B from 'bluebird'; +import { parseCoordinates } from './shared'; + +/** + * Shows screenshot of running application and divs that highlight the elements' bounding boxes + */ +export default class HighlighterRects extends Component { + + constructor (props) { + super(props); + this.state = { + scaleRatio: 1, + }; + this.updateScaleRatio = debounce(this.updateScaleRatio.bind(this), 1000); + } + + /** + * Calculates the ratio that the image is being scaled by + */ + updateScaleRatio () { + const screenshotEl = this.props.containerEl.querySelector('img'); + + // now update scale ratio + const {x1, x2} = parseCoordinates(this.props.source.children[0].children[0]); + this.setState({ + scaleRatio: (x2 - x1) / screenshotEl.offsetWidth + }); + + } + + async handleScreenshotClick () { + const {screenshotInteractionMode, applyClientMethod, + swipeStart, swipeEnd, setSwipeStart, setSwipeEnd} = this.props; + const {x, y} = this.state; + + if (screenshotInteractionMode === 'tap') { + applyClientMethod({ + methodName: 'tap', + args: [x, y], + }); + } else if (screenshotInteractionMode === 'swipe') { + if (!swipeStart) { + setSwipeStart(x, y); + } else if (!swipeEnd) { + setSwipeEnd(x, y); + await B.delay(500); // Wait a second to do the swipe so user can see the SVG line + await this.handleDoSwipe(); + } + } + } + + handleMouseMove (e) { + const {screenshotInteractionMode} = this.props; + const {scaleRatio} = this.state; + + if (screenshotInteractionMode !== 'select') { + const offsetX = e.nativeEvent.offsetX; + const offsetY = e.nativeEvent.offsetY; + const x = offsetX * scaleRatio; + const y = offsetY * scaleRatio; + this.setState({ + ...this.state, + x: Math.round(x), + y: Math.round(y), + }); + } + } + + handleMouseOut () { + this.setState({ + ...this.state, + x: null, + y: null, + }); + } + + async handleDoSwipe () { + const {swipeStart, swipeEnd, clearSwipeAction, applyClientMethod} = this.props; + await applyClientMethod({ + methodName: 'swipe', + args: [swipeStart.x, swipeStart.y, swipeEnd.x - swipeStart.x, swipeEnd.y - swipeStart.y], + }); + clearSwipeAction(); + } + + componentDidMount () { + // When DOM is ready, calculate the image scale ratio and re-calculate it whenever the window is resized + this.updateScaleRatio(); + window.addEventListener('resize', this.updateScaleRatio); + } + + componentWillUnmount () { + window.removeEventListener('resize', this.updateScaleRatio); + } + + render () { + const {source, screenshotInteractionMode, containerEl} = this.props; + const {scaleRatio} = this.state; + + // Recurse through the 'source' JSON and render a highlighter rect for each element + const highlighterRects = []; + + let highlighterXOffset = 0; + if (containerEl) { + const screenshotEl = containerEl.querySelector('img'); + highlighterXOffset = screenshotEl.getBoundingClientRect().left - + containerEl.getBoundingClientRect().left; + } + + // TODO: Refactor this into a separate component + let recursive = (element, zIndex = 0) => { + if (!element) { + return; + } + highlighterRects.push(); + + for (let childEl of element.children) { + recursive(childEl, zIndex + 1); + } + }; + + // If we're tapping or swiping, show the 'crosshair' cursor style + const screenshotStyle = {}; + if (screenshotInteractionMode === 'tap' || screenshotInteractionMode === 'swipe') { + screenshotStyle.cursor = 'crosshair'; + } + + recursive(source); + + return
{ highlighterRects }
; + } +} diff --git a/app/renderer/components/Inspector/Inspector.css b/app/renderer/components/Inspector/Inspector.css index 2e499d650..e01bac940 100644 --- a/app/renderer/components/Inspector/Inspector.css +++ b/app/renderer/components/Inspector/Inspector.css @@ -234,3 +234,47 @@ width: 100%; margin-bottom: 1em; } +.coordinatesContainer { + position: absolute; + background:rgba(255,250,205,0.8); + z-index: 10000000; + padding: 2px; +} +.swipeInstructions { + position: absolute; + top: 0; + right: 0; + z-index: 100000000; + background-color: white; + padding: 4px; + width: 10em; +} +.swipeSvg { + position: absolute; + z-index: 100000000; + top: 0; + height: 100%; + width: 100%; +} + +.swipeSvg line { + stroke-width: 15; + stroke: rgba(255,153,153,0.8); + stroke-linecap: round; +} + +.swipeSvg circle { + r: 10; + fill: rgba(255,153,153,0.8); +} + + .innerScreenshotContainer { + display: flex; + flex-direction: column; + height: 100%; + } + + .innerScreenshotContainer .screenshotActionsPanel { + margin-bottom: 2px; + margin: 0 auto 2px auto; + } \ No newline at end of file diff --git a/app/renderer/components/Inspector/Screenshot.js b/app/renderer/components/Inspector/Screenshot.js index 3cc94f0cd..4a5b8492e 100644 --- a/app/renderer/components/Inspector/Screenshot.js +++ b/app/renderer/components/Inspector/Screenshot.js @@ -1,9 +1,10 @@ import React, { Component } from 'react'; import ReactDOM from 'react-dom'; import { debounce } from 'lodash'; -import HighlighterRect from './HighlighterRect'; +import HighlighterRects from './HighlighterRects'; import Actions from './Actions'; import { Spin } from 'antd'; +import B from 'bluebird'; import styles from './Inspector.css'; import { parseCoordinates } from './shared'; @@ -35,15 +36,61 @@ export default class Screenshot extends Component { } - handleScreenshotClick (e) { - const {screenshotInteractionMode} = this.props; + async handleScreenshotClick () { + const {screenshotInteractionMode, applyClientMethod, + swipeStart, swipeEnd, setSwipeStart, setSwipeEnd} = this.props; + const {x, y} = this.state; + if (screenshotInteractionMode === 'tap') { + applyClientMethod({ + methodName: 'tap', + args: [x, y], + }); + } else if (screenshotInteractionMode === 'swipe') { + if (!swipeStart) { + setSwipeStart(x, y); + } else if (!swipeEnd) { + setSwipeEnd(x, y); + await B.delay(500); // Wait a second to do the swipe so user can see the SVG line + await this.handleDoSwipe(); + } + } + } + + handleMouseMove (e) { + const {screenshotInteractionMode} = this.props; + const {scaleRatio} = this.state; + + if (screenshotInteractionMode !== 'select') { const offsetX = e.nativeEvent.offsetX; const offsetY = e.nativeEvent.offsetY; - console.log('!!!!', offsetX, offsetY); + const x = offsetX * scaleRatio; + const y = offsetY * scaleRatio; + this.setState({ + ...this.state, + x: Math.round(x), + y: Math.round(y), + }); } } + handleMouseOut () { + this.setState({ + ...this.state, + x: null, + y: null, + }); + } + + async handleDoSwipe () { + const {swipeStart, swipeEnd, clearSwipeAction, applyClientMethod} = this.props; + await applyClientMethod({ + methodName: 'swipe', + args: [swipeStart.x, swipeStart.y, swipeEnd.x - swipeStart.x, swipeEnd.y - swipeStart.y], + }); + clearSwipeAction(); + } + componentDidMount () { // When DOM is ready, calculate the image scale ratio and re-calculate it whenever the window is resized this.updateScaleRatio(); @@ -55,54 +102,53 @@ export default class Screenshot extends Component { } render () { - const {source, screenshot, methodCallInProgress, screenshotInteractionMode} = this.props; - const {scaleRatio} = this.state; - - // Recurse through the 'source' JSON and render a highlighter rect for each element - const highlighterRects = []; - - let highlighterXOffset = 0; - if (this.containerEl) { - const screenshotEl = this.containerEl.querySelector('img'); - highlighterXOffset = screenshotEl.getBoundingClientRect().left - - this.containerEl.getBoundingClientRect().left; - } - - // TODO: Refactor this into a separate component - let recursive = (element, zIndex = 0) => { - if (!element) { - return; - } - highlighterRects.push(); - - for (let childEl of element.children) { - recursive(childEl, zIndex + 1); - } - }; + const {screenshot, methodCallInProgress, screenshotInteractionMode, + swipeStart, swipeEnd} = this.props; + const {scaleRatio, x, y} = this.state; - // If we're tapping or swiping, show the 'touch' cursor style + // If we're tapping or swiping, show the 'crosshair' cursor style const screenshotStyle = {}; if (screenshotInteractionMode === 'tap' || screenshotInteractionMode === 'swipe') { - screenshotStyle.cursor = 'crosshair'; // TODO: Change this to touch + screenshotStyle.cursor = 'crosshair'; } - recursive(source); - // Show the screenshot and highlighter rects. Show loading indicator if a method call is in progress. return - -
{ this.containerEl = containerEl; }} - style={screenshotStyle} - onClick={this.handleScreenshotClick.bind(this)} - className={styles.screenshotBox}> - - {screenshotInteractionMode === 'select' && highlighterRects} +
+
+ +
+
{ this.containerEl = containerEl; }} + style={screenshotStyle} + onClick={this.handleScreenshotClick.bind(this)} + onMouseMove={this.handleMouseMove.bind(this)} + onMouseOut={this.handleMouseOut.bind(this)} + className={styles.screenshotBox}> + {x !== null &&
+

X: {x}

+

Y: {y}

+
} + + {screenshotInteractionMode === 'select' && this.containerEl && } + {screenshotInteractionMode === 'swipe' &&
+ {(!swipeStart || !swipeEnd) &&
+ {!swipeStart &&

Click swipe start

} + {swipeStart && !swipeEnd &&

Click swipe end

} +
} + + {swipeStart && !swipeEnd && } + {swipeStart && swipeEnd && } + +
} +
; } diff --git a/app/renderer/lib/client-frameworks/framework.js b/app/renderer/lib/client-frameworks/framework.js index bb6706324..eb7e6f9d3 100644 --- a/app/renderer/lib/client-frameworks/framework.js +++ b/app/renderer/lib/client-frameworks/framework.js @@ -95,7 +95,11 @@ export default class Framework { } - codeFor_clickElement () { - return ''; + codeFor_tap () { + throw new Error("Need to implement codeFor_tap"); + } + + codeFor_swipe () { + throw new Error("Need to implement codeFor_tap"); } } diff --git a/app/renderer/lib/client-frameworks/java.js b/app/renderer/lib/client-frameworks/java.js index 985637a4e..9411d7a7f 100644 --- a/app/renderer/lib/client-frameworks/java.js +++ b/app/renderer/lib/client-frameworks/java.js @@ -97,6 +97,19 @@ ${this.indent(code, 4)} codeFor_back () { return `driver.navigate().back();`; } + + codeFor_tap (varNameIgnore, varIndexIgnore, x, y) { + return `(new TouchAction(driver)).tap(${x}, ${y}).perform()`; + } + + codeFor_swipe (varNameIgnore, varIndexIgnore, x1, y1, x2, y2) { + return `(new TouchAction(driver)) + .press({x: ${x1}, y: ${y1}}) + .moveTo({x: ${x2}: y: ${y2}}) + .release() + .perform() + `; + } } JavaFramework.readableName = "Java - JUnit"; diff --git a/app/renderer/lib/client-frameworks/js-wd.js b/app/renderer/lib/client-frameworks/js-wd.js index ae9bc2e72..46bc904ee 100644 --- a/app/renderer/lib/client-frameworks/js-wd.js +++ b/app/renderer/lib/client-frameworks/js-wd.js @@ -64,6 +64,22 @@ main().catch(console.log); return `await driver.back();`; } + codeFor_tap (varNameIgnore, varIndexIgnore, x, y) { + return `await (new TouchAction(driver)) + .tap({x: ${x}, y: ${y}}) + .perform() + `; + } + + codeFor_swipe (varNameIgnore, varIndexIgnore, x1, y1, x2, y2) { + return `await (new TouchAction(driver)) + .press({x: ${x1}, y: ${y1}}) + .moveTo({x: ${x2}: y: ${y2}}) + .release() + .perform() + `; + } + } JsWdFramework.readableName = "JS - WD (Promise)"; diff --git a/app/renderer/lib/client-frameworks/js-wdio.js b/app/renderer/lib/client-frameworks/js-wdio.js index 7b93026e8..c40bc1381 100644 --- a/app/renderer/lib/client-frameworks/js-wdio.js +++ b/app/renderer/lib/client-frameworks/js-wdio.js @@ -70,6 +70,18 @@ ${this.indent(this.chainifyCode(code), 2)} codeFor_back () { return `driver.back();`; } + + codeFor_tap (varNameIgnore, varIndexIgnore, x, y) { + return `driver.touchAction({actions: 'tap', x: ${x}, y: ${y}})`; + } + + codeFor_swipe (varNameIgnore, varIndexIgnore, x1, y1, x2, y2) { + return `driver.touchAction([ + {action: 'press', x: ${x1}, y: ${y1}}, + {action: 'moveTo', x: ${x2}, y: ${y2}}, + 'release' +];`; + } } JsWdIoFramework.readableName = "JS - Webdriver.io"; diff --git a/app/renderer/lib/client-frameworks/python.js b/app/renderer/lib/client-frameworks/python.js index ed6ae3bfe..58adf408b 100644 --- a/app/renderer/lib/client-frameworks/python.js +++ b/app/renderer/lib/client-frameworks/python.js @@ -68,6 +68,18 @@ driver.quit()`; codeFor_back () { return `driver.back()`; } + + codeFor_tap (varNameIgnore, varIndexIgnore, x, y) { + return `TouchAction(driver).tap([(${x}, ${y})])`; + } + + codeFor_swipe (varNameIgnore, varIndexIgnore, x1, y1, x2, y2) { + return `TouchAction(driver) + .press({x: ${x1}, y: ${y1}}) + .moveTo({x: ${x2}: y: ${y2}}) + .release() + `; + } } PythonFramework.readableName = "Python"; diff --git a/app/renderer/lib/client-frameworks/ruby.js b/app/renderer/lib/client-frameworks/ruby.js index 524a799d5..e46209a10 100644 --- a/app/renderer/lib/client-frameworks/ruby.js +++ b/app/renderer/lib/client-frameworks/ruby.js @@ -65,6 +65,24 @@ driver.quit`; codeFor_back () { return `driver.back`; } + + codeFor_tap (varNameIgnore, varIndexIgnore, x, y) { + return `TouchAction + .new + .tap(x: ${x}, y: ${y}) + .perform + `; + } + + codeFor_swipe (varNameIgnore, varIndexIgnore, x1, y1, x2, y2) { + return `TouchAction + .new + .press({x: ${x1}, y: ${y1}}) + .moveTo({x: ${x2}: y: ${y2}}) + .release + .perform + `; + } } RubyFramework.readableName = "Ruby"; diff --git a/app/renderer/reducers/Inspector.js b/app/renderer/reducers/Inspector.js index 79c412551..dc2ba8dbc 100644 --- a/app/renderer/reducers/Inspector.js +++ b/app/renderer/reducers/Inspector.js @@ -9,6 +9,7 @@ import { SET_SOURCE_AND_SCREENSHOT, QUIT_SESSION_REQUESTED, QUIT_SESSION_DONE, SHOW_LOCATOR_TEST_MODAL, HIDE_LOCATOR_TEST_MODAL, SET_LOCATOR_TEST_STRATEGY, SET_LOCATOR_TEST_VALUE, SEARCHING_FOR_ELEMENTS, SEARCHING_FOR_ELEMENTS_COMPLETED, SET_LOCATOR_TEST_ELEMENT, CLEAR_SEARCH_RESULTS, ADD_ASSIGNED_VAR_CACHE, CLEAR_ASSIGNED_VAR_CACHE, SET_SCREENSHOT_INTERACTION_MODE, + SET_SWIPE_START, SET_SWIPE_END, CLEAR_SWIPE_ACTION } from '../actions/Inspector'; const DEFAULT_FRAMEWORK = 'java'; @@ -261,6 +262,31 @@ export default function inspector (state=INITIAL_STATE, action) { screenshotInteractionMode: action.screenshotInteractionMode, }; + case SET_SWIPE_START: + return { + ...state, + swipeStart: { + x: action.swipeStartX, + y: action.swipeStartY, + }, + }; + + case SET_SWIPE_END: + return { + ...state, + swipeEnd: { + x: action.swipeEndX, + y: action.swipeEndY, + }, + }; + + case CLEAR_SWIPE_ACTION: + return { + ...state, + swipeStart: null, + swipeEnd: null, + }; + default: return {...state}; }