Can swipe and tap on screenshot

app/main/appium-method-handler.js

@@ -1,5 +1,6 @@
 import Bluebird from 'bluebird';
 import _ from 'lodash';
+import wd from 'wd';
 
 export default class AppiumMethodHandler {
   constructor (driver) {
@@ -80,7 +81,14 @@ export default class AppiumMethodHandler {
 
   async executeMethod (methodName, args = []) {
     let res = {};
-    if (methodName !== 'source' && methodName !== 'screenshot') {
+
+    // Specially handle the tap method
+    if (methodName === 'tap') {
+      res = await (new wd.TouchAction(this.driver)).tap({x: args[0], y: args[1]}).perform();
+    } else if (methodName === 'swipe') {
+      const [startX, startY, endX, endY]  = args;
+      res = await (new wd.TouchAction(this.driver)).press({x: startX, y: startY}).moveTo({x: endX, y: endY}).release().perform();
+    } else if (methodName !== 'source' && methodName !== 'screenshot') {
       res = await this.driver[methodName].apply(this.driver, args);
     }
 
@@ -94,7 +102,7 @@ export default class AppiumMethodHandler {
       res,
     };
   }
-
+  
   async _getSourceAndScreenshot () {
     let source, sourceError, screenshot, screenshotError;
     try {

app/main/appium.js

@@ -278,7 +278,7 @@ function connectClientMethodListener () {
     } = data;
 
     let renderer = evt.sender;
-    let driver = appiumHandlers[renderer.id];
+    let methodHandler = appiumHandlers[renderer.id];
 
     try {
       if (methodName === 'quit') {
@@ -296,18 +296,18 @@ function connectClientMethodListener () {
         if (methodName) {
           if (elementId) {
             console.log(`Handling client method request with method '${methodName}', args ${JSON.stringify(args)} and elementId ${elementId}`);
-            res = await driver.executeElementCommand(elementId, methodName, args);
+            res = await methodHandler.executeElementCommand(elementId, methodName, args);
           } else {
             console.log(`Handling client method request with method '${methodName}' and args ${JSON.stringify(args)}`);
-            res = await driver.executeMethod(methodName, args);
+            res = await methodHandler.executeMethod(methodName, args);
           }
         } else  if (strategy && selector) {
           if (fetchArray) {
             console.log(`Fetching elements with selector '${selector}' and strategy ${strategy}`);
-            res = await driver.fetchElements(strategy, selector);
+            res = await methodHandler.fetchElements(strategy, selector);
           } else {
             console.log(`Fetching an element with selector '${selector}' and strategy ${strategy}`);
-            res = await driver.fetchElement(strategy, selector);
+            res = await methodHandler.fetchElement(strategy, selector);
           }
         }
 
@@ -324,7 +324,7 @@ function connectClientMethodListener () {
         renderer.send('appium-session-done', e);
       }
       console.log('Caught an exception: ', e);
-      renderer.send('appium-client-command-response-error', {e, uuid});
+      renderer.send('appium-client-command-response-error', {e: e.message, uuid});
     }
   });
 }

app/renderer/actions/Inspector.js

@@ -47,6 +47,10 @@ export const ADD_ASSIGNED_VAR_CACHE = 'ADD_ASSIGNED_VAR_CACHE';
 export const CLEAR_ASSIGNED_VAR_CACHE = 'CLEAR_ASSIGNED_VAR_CACHE';
 export const SET_SCREENSHOT_INTERACTION_MODE = 'SET_SCREENSHOT_INTERACTION_MODE';
 
+export const SET_SWIPE_START = 'SET_SWIPE_START';
+export const SET_SWIPE_END = 'SET_SWIPE_END';
+export const CLEAR_SWIPE_ACTION = 'CLEAR_SWIPE_ACTION';
+
 // Attributes on nodes that we know are unique to the node
 const uniqueAttributes = [
   'name',
@@ -375,4 +379,22 @@ export function selectScreenshotInteractionMode (screenshotInteractionMode) {
   return (dispatch) => {
     dispatch({type: SET_SCREENSHOT_INTERACTION_MODE, screenshotInteractionMode });
   };
+}
+
+export function setSwipeStart (swipeStartX, swipeStartY) {
+  return (dispatch) => {
+    dispatch({type: SET_SWIPE_START, swipeStartX, swipeStartY});
+  };
+}
+
+export function setSwipeEnd (swipeEndX, swipeEndY) {
+  return (dispatch) => {
+    dispatch({type: SET_SWIPE_END, swipeEndX, swipeEndY});
+  };
+}
+
+export function clearSwipeAction () {
+  return (dispatch) => {
+    dispatch({type: CLEAR_SWIPE_ACTION});
+  };
 }

app/renderer/actions/shared.js

@@ -27,7 +27,7 @@ if (ipcRenderer) {
     const {e, uuid} = resp;
     let promise = clientMethodPromises[uuid];
     if (promise) {
-      promise.reject(e);
+      promise.reject(new Error(e));
       delete clientMethodPromises[uuid];
     }
   });

app/renderer/components/Inspector/Actions.js

@@ -1,5 +1,5 @@
 import React, { Component } from 'react';
-import { Button, Radio, Icon } from 'antd';
+import { Radio } from 'antd';
 
 
 /**
@@ -8,8 +8,8 @@ import { Button, Radio, Icon } from 'antd';
 export default class Actions extends Component {
 
   handleScreenshotInteractionChange (e) {
-    const {selectScreenshotInteractionMode} = this.props;
-
+    const {selectScreenshotInteractionMode, clearSwipeAction} = this.props;
+    clearSwipeAction(); // When the action changes, reset the swipe action
     selectScreenshotInteractionMode(e.target.value);
   }
 

app/renderer/components/Inspector/Inspector.css

@@ -234,3 +234,46 @@
     width: 100%;
     margin-bottom: 1em;
 }
+.coordinatesContainer {
+    position: absolute;
+    background:rgba(255,250,205,0.8);
+    z-index: 10000000;
+    padding: 2px;
+}
+.swipeInstructions {
+    position: absolute;
+    top: 0;
+    right: 0;
+    z-index: 100000000;
+    background-color: white;
+    padding: 4px;
+    width: 10em;
+}
+.swipeSvg {
+    position: absolute;
+    z-index: 100000000;
+    top: 0;
+    height: 100%;
+    width: 100%;
+}
+
+.swipeSvg line {
+    stroke-width: 15;
+    stroke: rgba(255,153,153,0.8);
+    stroke-linecap: round;
+}
+
+.swipeSvg circle {
+    r: 10;
+    fill: rgba(255,153,153,0.8);
+}
+
+ .innerScreenshotContainer {
+    display: flex; 
+    flex-direction: column;
+    height: 100%;
+ }
+
+ .innerScreenshotContainer .screenshotActionsPanel {
+    margin-bottom: 2px;
+ }

app/renderer/components/Inspector/Screenshot.js

@@ -4,6 +4,7 @@ import { debounce } from 'lodash';
 import HighlighterRect from './HighlighterRect';
 import Actions from './Actions';
 import { Spin } from 'antd';
+import B from 'bluebird';
 import styles from './Inspector.css';
 import { parseCoordinates } from './shared';
 
@@ -35,15 +36,61 @@ export default class Screenshot extends Component {
 
   }
 
-  handleScreenshotClick (e) {
-    const {screenshotInteractionMode} = this.props;
+  async handleScreenshotClick () {
+    const {screenshotInteractionMode, applyClientMethod, 
+      swipeStart, swipeEnd, setSwipeStart, setSwipeEnd} = this.props;
+    const {x, y} = this.state;
+
     if (screenshotInteractionMode === 'tap') {
+      applyClientMethod({
+        methodName: 'tap',
+        args: [x, y],
+      });
+    } else if (screenshotInteractionMode === 'swipe') {
+      if (!swipeStart) {
+        setSwipeStart(x, y);
+      } else if (!swipeEnd) {
+        setSwipeEnd(x, y);
+        await B.delay(500); // Wait a second to do the swipe so user can see the SVG line
+        await this.handleDoSwipe();
+      }
+    }
+  }
+
+  handleMouseMove (e) {
+    const {screenshotInteractionMode} = this.props;
+    const {scaleRatio} = this.state;
+
+    if (screenshotInteractionMode !== 'select') {
       const offsetX = e.nativeEvent.offsetX;
       const offsetY = e.nativeEvent.offsetY;
-      console.log('!!!!', offsetX, offsetY);
+      const x = offsetX * scaleRatio;
+      const y = offsetY * scaleRatio;
+      this.setState({
+        ...this.state,
+        x: Math.round(x),
+        y: Math.round(y),
+      });
     }
   }
 
+  handleMouseOut () {
+    this.setState({
+      ...this.state,
+      x: undefined,
+      y: undefined,
+    });
+  }
+
+  async handleDoSwipe () {
+    const {swipeStart, swipeEnd, clearSwipeAction, applyClientMethod} = this.props;
+    await applyClientMethod({
+      methodName: 'swipe',
+      args: [swipeStart.x, swipeStart.y, swipeEnd.x - swipeStart.x, swipeEnd.y - swipeStart.y],
+    });
+    clearSwipeAction();
+  }
+
   componentDidMount () {
     // When DOM is ready, calculate the image scale ratio and re-calculate it whenever the window is resized
     this.updateScaleRatio();
@@ -55,8 +102,9 @@ export default class Screenshot extends Component {
   }
 
   render () {
-    const {source, screenshot, methodCallInProgress, screenshotInteractionMode} = this.props;
-    const {scaleRatio} = this.state;
+    const {source, screenshot, methodCallInProgress, screenshotInteractionMode, 
+      swipeStart, swipeEnd, clearSwipeAction} = this.props;
+    const {scaleRatio, x, y} = this.state;
 
     // Recurse through the 'source' JSON and render a highlighter rect for each element
     const highlighterRects = [];
@@ -89,20 +137,48 @@ export default class Screenshot extends Component {
     // If we're tapping or swiping, show the 'touch' cursor style
     const screenshotStyle = {};
     if (screenshotInteractionMode === 'tap' || screenshotInteractionMode === 'swipe') {
-      screenshotStyle.cursor = 'crosshair'; // TODO: Change this to touch
+      screenshotStyle.cursor = 'crosshair';
     }
 
     recursive(source);
 
     // Show the screenshot and highlighter rects. Show loading indicator if a method call is in progress.
     return <Spin size='large' spinning={!!methodCallInProgress}>
-      <Actions {...this.props} />
-      <div ref={(containerEl) => { this.containerEl = containerEl; }}
-        style={screenshotStyle} 
-        onClick={this.handleScreenshotClick.bind(this)}
-        className={styles.screenshotBox}>
-        <img src={`data:image/gif;base64,${screenshot}`} id="screenshot" />
-        {screenshotInteractionMode === 'select' && highlighterRects}
+      <div className={styles.innerScreenshotContainer}>
+        <div className={styles.screenshotActionsPanel}>
+          <Actions {...this.props} />
+        </div>
+        <div ref={(containerEl) => { this.containerEl = containerEl; }}
+          style={screenshotStyle} 
+          onClick={this.handleScreenshotClick.bind(this)}
+          onMouseMove={this.handleMouseMove.bind(this)}
+          onMouseOut={this.handleMouseOut.bind(this)}
+          className={styles.screenshotBox}>
+          {x !== undefined ? <div className={styles.coordinatesContainer}>
+            <p>X: {x}</p>
+            <p>Y: {y}</p>
+          </div> : null}
+          <img src={`data:image/gif;base64,${screenshot}`} id="screenshot" />
+          {screenshotInteractionMode === 'select' && highlighterRects}
+          {screenshotInteractionMode === 'swipe' && <div>
+            {(!swipeStart || !swipeEnd) && <div className={styles.swipeInstructions}>
+              {!swipeStart && <p>Click swipe start</p>}
+              {swipeStart && !swipeEnd && <p>Click swipe end</p>}
+            </div>}
+            <svg className={styles.swipeSvg}>
+              {swipeStart && !swipeEnd && <circle 
+                cx={swipeStart.x / scaleRatio} 
+                cy={swipeStart.y / scaleRatio} 
+              />}
+              {swipeStart && swipeEnd && <line
+                x1={swipeStart.x / scaleRatio}
+                y1={swipeStart.y / scaleRatio}
+                x2={swipeEnd.x / scaleRatio}
+                y2={swipeEnd.y / scaleRatio}
+              />}
+            </svg>
+          </div>}
+        </div>
       </div>
     </Spin>;
   }

app/renderer/lib/client-frameworks/framework.js

@@ -95,7 +95,11 @@ export default class Framework {
 
   }
 
-  codeFor_clickElement () {
-    return '';
+  codeFor_tap () {
+    throw new Error("Need to implement codeFor_tap");
+  }
+
+  codeFor_swipe () {
+    throw new Error("Need to implement codeFor_tap");
   }
 }

app/renderer/lib/client-frameworks/java.js

@@ -97,6 +97,14 @@ ${this.indent(code, 4)}
   codeFor_back () {
     return `driver.navigate().back();`;
   }
+
+  codeFor_tap (varNameIgnore, varIndexIgnore, x, y) {
+    return `(new TouchAction(driver)).tap(${x}, ${y}).perform()`;
+  }
+
+  codeFor_swipe (varNameIgnore, varIndexIgnore, x1, y1, x2, y2) {
+    return `(new TouchAction(driver)).press({x: ${x1}, y: ${y1}}).moveTo({x: ${x2}: y: ${y2}}).release().perform()`;
+  }
 }
 
 JavaFramework.readableName = "Java - JUnit";

app/renderer/lib/client-frameworks/js-wd.js

@@ -64,6 +64,14 @@ main().catch(console.log);
     return `await driver.back();`;
   }
 
+  codeFor_tap (varNameIgnore, varIndexIgnore, x, y) {
+    return `await (new TouchAction(driver)).tap({x: ${x}, y: ${y}}).perform()`;
+  }
+
+  codeFor_swipe (varNameIgnore, varIndexIgnore, x1, y1, x2, y2) {
+    return `await (new TouchAction(driver)).press({x: ${x1}, y: ${y1}}).moveTo({x: ${x2}: y: ${y2}}).release().perform()`;
+  }
+
 }
 
 JsWdFramework.readableName = "JS - WD (Promise)";

app/renderer/lib/client-frameworks/js-wdio.js

@@ -70,6 +70,18 @@ ${this.indent(this.chainifyCode(code), 2)}
   codeFor_back () {
     return `driver.back();`;
   }
+
+  codeFor_tap (varNameIgnore, varIndexIgnore, x, y) {
+    return `driver.touchAction({actions: 'tap', x: ${x}, y: ${y}})`;
+  }
+
+  codeFor_swipe (varNameIgnore, varIndexIgnore, x1, y1, x2, y2) {
+    return `driver.touchAction([
+  {action: 'press', x: ${x1}, y: ${y1}},
+  {action: 'moveTo', x: ${x2}, y: ${y2}},
+  'release'
+];`;
+  }
 }
 
 JsWdIoFramework.readableName = "JS - Webdriver.io";