Support for Bitfinex Importing (take 2)

docs/introduction/supported_exchanges.md

@@ -11,11 +11,11 @@ Gekko is able to directly communicate with the APIs of a number of exchanges. Ho
 | [Poloniex](https://poloniex.com/)      | V | V | V | |
 | [GDAX](https://gdax.com/)      | V | V | V | |
 | [BTCC](https://btcc.com/)      | V | V | V | (=BTCChina) |
-| [Bitstamp](https://bitstamp.com/)      | V | V | X | |
-| [Kraken](https://kraken.com/)      | V | V | X | |
-| [Bitfinex](https://bitfinex.com/)      | V | V | X | |
+| [Bitstamp](https://bitstamp.com/)      | V | V | V | |
+| [Kraken](https://kraken.com/)      | V | V | V | |
+| [Bitfinex](https://bitfinex.com/)      | V | V | V | |
 | [Bittrex](https://bittrex.com/)      | V | V | X | |
-| [BTC-e](https://btc-e.com/)      | V | V | X | |
+| [wex.nz](https://wex.nz/)      | V | V | X | |
 | [Okcoin.cn](https://www.okcoin.cn/)      | V | V | X | (China, see [#352](https://github.com/askmike/gekko/pull/352)) |
 | [Cex.io](https://cex.io/)      | V | X | X | |
 | [BTC Markets](https://btcmarkets.net)      | V | V | X | |

docs/strategies/creating_a_strategy.md

@@ -35,6 +35,15 @@ A strategy is a combination of functions that get market data in the form of can
       // your code!
     }
 
+    // Optional for executing code
+    // after completion of a backtest.
+    // This block will not execute in
+    // live use as a live gekko is
+    // never ending.
+    strat.end = function() {
+      // your code!
+    }
+
     module.exports = strat;
 
 In the boilerplate we define four functions you have to write yourself. The functions are executed like so:

exchanges/bitfinex.js

@@ -17,7 +17,7 @@ var Trader = function(config) {
   this.asset = config.asset;
   this.currency = config.currency;
   this.pair = this.asset + this.currency;
-  this.bitfinex = new Bitfinex(this.key, this.secret).rest;
+  this.bitfinex = new Bitfinex(this.key, this.secret, { version: 1 }).rest;
 }
 
 // if the exchange errors we try the same call again after
@@ -186,19 +186,19 @@ Trader.prototype.cancelOrder = function(order_id, callback) {
 Trader.prototype.getTrades = function(since, callback, descending) {
   var args = _.toArray(arguments);
 
-  var path = this.pair;
-  if(since)
-    path += '?limit_trades=2000';
+  var path = this.pair; 
+  if(since) 
+    path += '?limit_trades=2000'; 
 
-  this.bitfinex.trades(path, (err, data) => {
+  this.bitfinex.trades(path, (err, data) => {  
     if (err)
       return this.retry(this.getTrades, args);
 
     var trades = _.map(data, function(trade) {
       return {
-        tid: trade.tid,
-        date:  trade.timestamp,
-        price: +trade.price,
+        tid: trade.tid, 
+        date:  trade.timestamp, 
+        price: +trade.price, 
         amount: +trade.amount
       }
     });
@@ -212,7 +212,7 @@ Trader.getCapabilities = function () {
     name: 'Bitfinex',
     slug: 'bitfinex',
     currencies: ['USD', 'BTC', 'ETH'],
-    assets: ['BTC', 'LTC', 'ETH', 'SAN', 'IOT', 'BCH', 'OMG', 'XMR', 'DSH', 'ZEC', 'EOS', 'ETC', 'XRP', 'NEO'],
+    assets: ['BTC', 'LTC', 'ETH', 'SAN', 'IOT', 'BCH', 'OMG', 'XMR', 'DSH', 'ZEC', 'EOS', 'ETC', 'XRP', 'NEO', 'ETP'],
     markets: [
 
         //Tradeable Pairs to USD
@@ -230,6 +230,7 @@ Trader.getCapabilities = function () {
         { pair: ['USD', 'ETC'], minimalOrder: { amount: 0.01, unit: 'asset' } },
         { pair: ['USD', 'XRP'], minimalOrder: { amount: 0.01, unit: 'asset' } },
         { pair: ['USD', 'NEO'], minimalOrder: { amount: 0.01, unit: 'asset' } },
+        { pair: ['USD', 'ETP'], minimalOrder: { amount: 0.01, unit: 'asset' } },
 
         //Tradeable Pairs to BTC
         { pair: ['BTC', 'ETH'], minimalOrder: { amount: 0.01, unit: 'asset' } },
@@ -245,6 +246,7 @@ Trader.getCapabilities = function () {
         { pair: ['BTC', 'ETC'], minimalOrder: { amount: 0.01, unit: 'asset' } },
         { pair: ['BTC', 'XRP'], minimalOrder: { amount: 0.01, unit: 'asset' } },
         { pair: ['BTC', 'NEO'], minimalOrder: { amount: 0.01, unit: 'asset' } },
+        { pair: ['BTC', 'ETP'], minimalOrder: { amount: 0.01, unit: 'asset' } },
 
         //Tradeable Pairs to ETH
         { pair: ['ETH', 'BCH'], minimalOrder: { amount: 0.01, unit: 'asset' } },
@@ -253,10 +255,12 @@ Trader.getCapabilities = function () {
         { pair: ['ETH', 'SAN'], minimalOrder: { amount: 0.01, unit: 'asset' } },
         { pair: ['ETH', 'EOS'], minimalOrder: { amount: 0.01, unit: 'asset' } },
         { pair: ['ETH', 'NEO'], minimalOrder: { amount: 0.01, unit: 'asset' } },
+        { pair: ['ETH', 'ETP'], minimalOrder: { amount: 0.01, unit: 'asset' } },
 
     ],
     requires: ['key', 'secret'],
     tid: 'tid',
+    providesFullHistory: true,
     tradable: true
   };
 }

exchanges/kraken.js

@@ -20,7 +20,8 @@ var crypto_currencies = [
   "XLM",
   "XMR",
   "XRP",
-  "ZEC"
+  "ZEC",
+  "BCH",
 ];
 
 var fiat_currencies = [
@@ -29,9 +30,15 @@ var fiat_currencies = [
   "USD",
   "JPY",
   "CAD",
-
 ];
 
+var assets_without_prefix = [
+  'BCH',
+  'DASH',
+  'EOS',
+  'GNO',
+]
+
 // Method to check if asset/currency is a crypto currency
 var isCrypto = function(value) {
   return _.contains(crypto_currencies, value);
@@ -43,7 +50,6 @@ var isFiat = function(value) {
 };
 
 var addPrefix = function(value) {
-
   var fiatPrefix = "Z";
   var cryptoPrefix = "X";
 
@@ -55,6 +61,14 @@ var addPrefix = function(value) {
     return value;
 }
 
+// Some currencies in Kraken don't use the prefix, not clearly documented
+var getAssetPair = function(asset, currency) {
+  if (_.contains(assets_without_prefix, asset))
+    return asset + currency;
+  else
+    return addPrefix(asset) + addPrefix(currency);
+}
+
 var Trader = function(config) {
   _.bindAll(this);
 
@@ -65,13 +79,7 @@ var Trader = function(config) {
     this.asset = config.asset.toUpperCase();
   }
 
-  // We need to prefix the asset and currency
-  // with either Z or X on all markets.
-  // EXCEPT for BCH markets..
-  if(this.asset === 'BCH')
-    this.pair = this.asset + this.currency;
-  else
-    this.pair = addPrefix(this.asset) + addPrefix(this.currency);
+  this.pair = getAssetPair(this.asset, this.currency);
   this.name = 'kraken';
   this.since = null;
 
@@ -105,19 +113,26 @@ Trader.prototype.retry = function(method, args) {
 
 Trader.prototype.getTrades = function(since, callback, descending) {
   var args = _.toArray(arguments);
+  var startTs = !_.isNull(since) ? moment(since).valueOf() : null;
+
   var process = function(err, trades) {
     if (err || !trades || trades.length === 0) {
       log.error('error getting trades', err);
       return this.retry(this.getTrades, args);
     }
 
+
     var parsedTrades = [];
     _.each(trades.result[this.pair], function(trade) {
-      parsedTrades.push({
-        date: parseInt(Math.round(trade[2]), 10),
-        price: parseFloat(trade[0]),
-        amount: parseFloat(trade[1])
-      });
+      // Even when you supply 'since' you can still get more trades than you asked for, it needs to be filtered
+      if (_.isNull(startTs) || startTs < moment.unix(trade[2]).valueOf()) {
+        parsedTrades.push({
+          tid: moment.unix(trade[2]).valueOf() * 1000000,
+          date: parseInt(Math.round(trade[2]), 10),
+          price: parseFloat(trade[0]),
+          amount: parseFloat(trade[1])
+        });
+      }
     }, this);
 
     if(descending)
@@ -129,12 +144,12 @@ Trader.prototype.getTrades = function(since, callback, descending) {
   var reqData = {
     pair: this.pair
   };
-  // This appears to not work correctly
-  // skipping for now so we have the same
-  // behaviour cross exchange.
-  //
-  // if(!_.isNull(this.since))
-  //   reqData.since = this.since;
+
+  if(!_.isNull(since)) {
+    // Kraken wants a tid, which is found to be timestamp_ms * 1000000 in practice. No clear documentation on this though
+    reqData.since = startTs * 1000000;
+  }
+
   this.kraken.api('Trades', reqData, _.bind(process, this));
 };
 
@@ -153,7 +168,10 @@ Trader.prototype.getPortfolio = function(callback) {
       return this.retry(this.getPortfolio, args);
     }
 
-    var assetAmount = parseFloat( data.result[addPrefix(this.asset)] );
+    // When using the prefix-less assets, you remove the prefix from the assset but leave
+    // it on the curreny in this case. An undocumented Kraken quirk.
+    var assetId = _.contains(assets_without_prefix, this.asset) ? this.asset : addPrefix(this.asset);
+    var assetAmount = parseFloat( data.result[addPrefix(assetId)] );
     var currencyAmount = parseFloat( data.result[addPrefix(this.currency)] );
 
     if(!_.isNumber(assetAmount) || _.isNaN(assetAmount)) {
@@ -328,7 +346,7 @@ Trader.getCapabilities = function () {
   return {
     name: 'Kraken',
     slug: 'kraken',
-    currencies: ['CAD', 'EUR', 'GBP', 'JPY', 'USD', 'XBT'],
+    currencies: ['CAD', 'EUR', 'GBP', 'JPY', 'USD', 'XBT', 'ETH'],
     assets: ['XBT', 'LTC', 'GNO', 'ICN', 'MLN', 'REP', 'XDG', 'XLM', 'XMR', 'XRP', 'ZEC', 'ETH', 'BCH', 'DASH', 'EOS', 'ETC'],
     markets: [
       //Tradeable againt ETH
@@ -338,19 +356,12 @@ Trader.getCapabilities = function () {
       { pair: ['GBP', 'ETH'], minimalOrder: { amount: 0.01, unit: 'asset' } },
       { pair: ['JPY', 'ETH'], minimalOrder: { amount: 1, unit: 'asset' }, precision: 0 },
       { pair: ['USD', 'ETH'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 2 },
-      { pair: ['EOS', 'ETH'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 6 },
-      { pair: ['ETC', 'ETH'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 5 },
-      { pair: ['GNO', 'ETH'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 4 },
-      { pair: ['ICN', 'ETH'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 6 },
-      { pair: ['MLN', 'ETH'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 5 },
-      { pair: ['REP', 'ETH'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 5 },
 
       //Tradeable against LTC
       { pair: ['XBT', 'LTC'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 6 },
       { pair: ['EUR', 'LTC'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 2 },
       { pair: ['USD', 'LTC'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 2 },
 
-
       //Tradeable against BCH
       { pair: ['USD', 'BCH'], minimalOrder: { amount: 0.1, unit: 'asset' }, precision: 1 },
       { pair: ['EUR', 'BCH'], minimalOrder: { amount: 0.1, unit: 'asset' }, precision: 1 },
@@ -393,7 +404,6 @@ Trader.getCapabilities = function () {
       { pair: ['XBT', 'REP'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 6 },
       { pair: ['ETH', 'REP'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 5 },
 
-
       //Tradeable against XDG
       { pair: ['XBT', 'XDG'], minimalOrder: { amount: 0.01, unit: 'asset' } },
 
@@ -407,7 +417,6 @@ Trader.getCapabilities = function () {
       { pair: ['EUR', 'XMR'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 2 },
       { pair: ['XBT', 'XMR'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 6 },
 
-
       //Tradeable against XRP
       { pair: ['USD', 'XRP'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 5 },
       { pair: ['EUR', 'XRP'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 5 },
@@ -422,32 +431,15 @@ Trader.getCapabilities = function () {
 
       //Tradeable against XBT
       { pair: ['BCH', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 5 },
-      { pair: ['LTC', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 6 },
-      { pair: ['XDG', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' } },
-      { pair: ['XLM', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' } },
-      { pair: ['XRP', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' } },
       { pair: ['CAD', 'XBT'], minimalOrder: { amount: 0.1, unit: 'asset' }, precision: 1 },
       { pair: ['EUR', 'XBT'], minimalOrder: { amount: 0.1, unit: 'asset' }, precision: 1 },
       { pair: ['GBP', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' } },
       { pair: ['JPY', 'XBT'], minimalOrder: { amount: 1, unit: 'asset' }, precision: 0 },
       { pair: ['USD', 'XBT'], minimalOrder: { amount: 0.1, unit: 'asset' }, precision: 1 },
-      { pair: ['DASH', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 5 },
-      { pair: ['EOS', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 7 },
-      { pair: ['ETC', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 6 },
-      { pair: ['ETH', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 5 },
-      { pair: ['GNO', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 5 },
-      { pair: ['ICN', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 7 },
-      { pair: ['MLN', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 6 },
-      { pair: ['REP', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' } },
-      { pair: ['XDG', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' } },
-      { pair: ['XLM', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 8 },
-      { pair: ['XMR', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 6 },
-      { pair: ['XRP', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 8 },
-      { pair: ['ZEC', 'XBT'], minimalOrder: { amount: 0.01, unit: 'asset' }, precision: 5 },
-
     ],
     requires: ['key', 'secret'],
-    providesHistory: false,
+    providesHistory: 'date',
+    providesFullHistory: true,
     tid: 'date',
     tradable: true
   };