Merge pull request #2372 from xmatthias/kraken_ohlcv_emulate

download tick-based data to emulate candles

config_kraken.json.example

@@ -70,5 +70,6 @@
     "forcebuy_enable": false,
     "internals": {
         "process_throttle_secs": 5
-    }
+    },
+    "download_trades": true
 }

docs/data-download.md

@@ -38,7 +38,7 @@ Mixing different stake-currencies is allowed for this file, since it's only used
 ]
 ```
 
-### start download
+### Start download
 
 Then run:
 
@@ -57,6 +57,32 @@ This will download ticker data for all the currency pairs you defined in `pairs.
 - Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers.
 - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options.
 
+### Trades (tick) data
+
+By default, `download-data` subcommand downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
+This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes.
+
+Since this data is large by default, the files use gzip by default. They are stored in your data-directory with the naming convention of `<pair>-trades.json.gz` (`ETH_BTC-trades.json.gz`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.
+
+To use this mode, simply add `--dl-trades` to your call. This will swap the download method to download trades, and resamples the data locally.
+
+Example call:
+
+```bash
+freqtrade download-data --exchange binance --pairs XRP/ETH ETH/BTC --days 20 --dl-trades
+```
+
+!!! Note
+    While this method uses async calls, it will be slow, since it requires the result of the previous call to generate the next request to the exchange.
+
+!!! Warning
+    The historic trades are not available during Freqtrade dry-run and live trade modes because all exchanges tested provide this data with a delay of few 100 candles, so it's not suitable for real-time trading.
+
+### Historic Kraken data
+
+The Kraken API does only provide 720 historic candles, which is sufficient for FreqTrade dry-run and live trade modes, but is a problem for backtesting.
+To download data for the Kraken exchange, using `--dl-trades` is mandatory, otherwise the bot will download the same 720 candles over and over, and you'll not have enough backtest data.
+
 ## Next step
 
 Great, you now have backtest data downloaded, so you can now start [backtesting](backtesting.md) your strategy.

freqtrade/configuration/arguments.py

@@ -35,7 +35,8 @@ ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"]
 
 ARGS_CREATE_USERDIR = ["user_data_dir"]
 
-ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"]
+ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",
+                      "timeframes", "erase"]
 
 ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url",
                        "trade_source", "export", "exportfilename", "timerange", "ticker_interval"]

freqtrade/configuration/cli_options.py

@@ -273,6 +273,12 @@ AVAILABLE_CLI_OPTIONS = {
         type=check_int_positive,
         metavar='INT',
     ),
+    "download_trades": Arg(
+        '--dl-trades',
+        help='Download trades instead of OHLCV data. The bot will resample trades to the '
+             'desired timeframe as specified as --timeframes/-t.',
+        action='store_true',
+    ),
     "exchange": Arg(
         '--exchange',
         help=f'Exchange name (default: `{constants.DEFAULT_EXCHANGE}`). '

freqtrade/configuration/configuration.py

@@ -312,6 +312,8 @@ class Configuration:
 
         self._args_to_config(config, argname='days',
                              logstring='Detected --days: {}')
+        self._args_to_config(config, argname='download_trades',
+                             logstring='Detected --dl-trades: {}')
 
     def _process_runmode(self, config: Dict[str, Any]) -> None:
 

freqtrade/data/converter.py

@@ -114,3 +114,25 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
                       keys=['b_sum', 'b_size', 'bids', 'asks', 'a_size', 'a_sum'])
     # logger.info('order book %s', frame )
     return frame
+
+
+def trades_to_ohlcv(trades: list, timeframe: str) -> list:
+    """
+    Converts trades list to ohlcv list
+    :param trades: List of trades, as returned by ccxt.fetch_trades.
+    :param timeframe: Ticker timeframe to resample data to
+    :return: ohlcv timeframe as list (as returned by ccxt.fetch_ohlcv)
+    """
+    from freqtrade.exchange import timeframe_to_minutes
+    ticker_minutes = timeframe_to_minutes(timeframe)
+    df = pd.DataFrame(trades)
+    df['datetime'] = pd.to_datetime(df['datetime'])
+    df = df.set_index('datetime')
+
+    df_new = df['price'].resample(f'{ticker_minutes}min').ohlc()
+    df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum()
+    df_new['date'] = df_new.index.astype("int64") // 10 ** 6
+    # Drop 0 volume rows
+    df_new = df_new.dropna()
+    columns = ["date", "open", "high", "low", "close", "volume"]
+    return list(zip(*[df_new[x].values.tolist() for x in columns]))

freqtrade/data/history.py

@@ -17,7 +17,7 @@ from pandas import DataFrame
 
 from freqtrade import OperationalException, misc
 from freqtrade.configuration import TimeRange
-from freqtrade.data.converter import parse_ticker_dataframe
+from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv
 from freqtrade.exchange import Exchange, timeframe_to_minutes
 
 logger = logging.getLogger(__name__)
@@ -82,6 +82,29 @@ def store_tickerdata_file(datadir: Path, pair: str,
     misc.file_dump_json(filename, data, is_zip=is_zip)
 
 
+def load_trades_file(datadir: Path, pair: str,
+                     timerange: Optional[TimeRange] = None) -> List[Dict]:
+    """
+    Load a pair from file, either .json.gz or .json
+    :return: tradelist or empty list if unsuccesful
+    """
+    filename = pair_trades_filename(datadir, pair)
+    tradesdata = misc.file_load_json(filename)
+    if not tradesdata:
+        return []
+
+    return tradesdata
+
+
+def store_trades_file(datadir: Path, pair: str,
+                      data: list, is_zip: bool = True):
+    """
+    Stores tickerdata to file
+    """
+    filename = pair_trades_filename(datadir, pair)
+    misc.file_dump_json(filename, data, is_zip=is_zip)
+
+
 def _validate_pairdata(pair, pairdata, timerange: TimeRange):
     if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000:
         logger.warning('Missing data at start for pair %s, data starts at %s',
@@ -173,6 +196,12 @@ def pair_data_filename(datadir: Path, pair: str, ticker_interval: str) -> Path:
     return filename
 
 
+def pair_trades_filename(datadir: Path, pair: str) -> Path:
+    pair_s = pair.replace("/", "_")
+    filename = datadir.joinpath(f'{pair_s}-trades.json.gz')
+    return filename
+
+
 def _load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str,
                                    timerange: Optional[TimeRange]) -> Tuple[List[Any],
                                                                             Optional[int]]:
@@ -299,6 +328,92 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
     return pairs_not_available
 
 
+def download_trades_history(datadir: Path,
+                            exchange: Exchange,
+                            pair: str,
+                            timerange: Optional[TimeRange] = None) -> bool:
+    """
+    Download trade history from the exchange.
+    Appends to previously downloaded trades data.
+    """
+    try:
+
+        since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None
+
+        trades = load_trades_file(datadir, pair)
+
+        from_id = trades[-1]['id'] if trades else None
+
+        logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None')
+        logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None')
+
+        new_trades = exchange.get_historic_trades(pair=pair,
+                                                  since=since if since else
+                                                  int(arrow.utcnow().shift(
+                                                      days=-30).float_timestamp) * 1000,
+                                                  #  until=xxx,
+                                                  from_id=from_id,
+                                                  )
+        trades.extend(new_trades[1])
+        store_trades_file(datadir, pair, trades)
+
+        logger.debug("New Start: %s", trades[0]['datetime'])
+        logger.debug("New End: %s", trades[-1]['datetime'])
+        logger.info(f"New Amount of trades: {len(trades)}")
+        return True
+
+    except Exception as e:
+        logger.error(
+            f'Failed to download historic trades for pair: "{pair}". '
+            f'Error: {e}'
+        )
+        return False
+
+
+def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
+                                 timerange: TimeRange, erase=False) -> List[str]:
+    """
+    Refresh stored trades data.
+    Used by freqtrade download-data
+    :return: Pairs not available
+    """
+    pairs_not_available = []
+    for pair in pairs:
+        if pair not in exchange.markets:
+            pairs_not_available.append(pair)
+            logger.info(f"Skipping pair {pair}...")
+            continue
+
+        dl_file = pair_trades_filename(datadir, pair)
+        if erase and dl_file.exists():
+            logger.info(
+                f'Deleting existing data for pair {pair}.')
+            dl_file.unlink()
+
+        logger.info(f'Downloading trades for pair {pair}.')
+        download_trades_history(datadir=datadir, exchange=exchange,
+                                pair=pair,
+                                timerange=timerange)
+    return pairs_not_available
+
+
+def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
+                            datadir: Path, timerange: TimeRange, erase=False) -> None:
+    """
+    Convert stored trades data to ohlcv data
+    """
+    for pair in pairs:
+        trades = load_trades_file(datadir, pair)
+        for timeframe in timeframes:
+            ohlcv_file = pair_data_filename(datadir, pair, timeframe)
+            if erase and ohlcv_file.exists():
+                logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.')
+                ohlcv_file.unlink()
+            ohlcv = trades_to_ohlcv(trades, timeframe)
+            # Store ohlcv
+            store_tickerdata_file(datadir, pair, timeframe, data=ohlcv)
+
+
 def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
     """
     Get the maximum timeframe for the given backtest data

freqtrade/exchange/binance.py

@@ -16,6 +16,8 @@ class Binance(Exchange):
     _ft_has: Dict = {
         "stoploss_on_exchange": True,
         "order_time_in_force": ['gtc', 'fok', 'ioc'],
+        "trades_pagination": "id",
+        "trades_pagination_arg": "fromId",
     }
 
     def get_order_book(self, pair: str, limit: int = 100) -> dict:

freqtrade/exchange/exchange.py

@@ -147,6 +147,8 @@ def retrier(f):
 class Exchange:
 
     _config: Dict = {}
+
+    # Parameters to add directly to buy/sell calls (like agreeing to trading agreement)
     _params: Dict = {}
 
     # Dict to specify which options each exchange implements
@@ -157,6 +159,9 @@ class Exchange:
         "order_time_in_force": ["gtc"],
         "ohlcv_candle_limit": 500,
         "ohlcv_partial_candle": True,
+        "trades_pagination": "time",  # Possible are "time" or "id"
+        "trades_pagination_arg": "since",
+
     }
     _ft_has: Dict = {}
 
@@ -200,6 +205,9 @@ class Exchange:
         self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit']
         self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle']
 
+        self._trades_pagination = self._ft_has['trades_pagination']
+        self._trades_pagination_arg = self._ft_has['trades_pagination_arg']
+
         # Initialize ccxt objects
         self._api = self._init_ccxt(
             exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config'))
@@ -742,6 +750,154 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(f'Could not fetch ticker data. Msg: {e}') from e
 
+    @retrier_async
+    async def _async_fetch_trades(self, pair: str,
+                                  since: Optional[int] = None,
+                                  params: Optional[dict] = None) -> List[Dict]:
+        """
+        Asyncronously gets trade history using fetch_trades.
+        Handles exchange errors, does one call to the exchange.
+        :param pair: Pair to fetch trade data for
+        :param since: Since as integer timestamp in milliseconds
+        returns: List of dicts containing trades
+        """
+        try:
+            # fetch trades asynchronously
+            if params:
+                logger.debug("Fetching trades for pair %s, params: %s ", pair, params)
+                trades = await self._api_async.fetch_trades(pair, params=params, limit=1000)
+            else:
+                logger.debug(
+                    "Fetching trades for pair %s, since %s %s...",
+                    pair,  since,
+                    '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else ''
+                )
+                trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
+            return trades
+        except ccxt.NotSupported as e:
+            raise OperationalException(
+                f'Exchange {self._api.name} does not support fetching historical trade data.'
+                f'Message: {e}') from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(f'Could not load trade history due to {e.__class__.__name__}. '
+                                 f'Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(f'Could not fetch trade data. Msg: {e}') from e
+
+    async def _async_get_trade_history_id(self, pair: str,
+                                          until: int,
+                                          since: Optional[int] = None,
+                                          from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
+        """
+        Asyncronously gets trade history using fetch_trades
+        use this when exchange uses id-based iteration (check `self._trades_pagination`)
+        :param pair: Pair to fetch trade data for
+        :param since: Since as integer timestamp in milliseconds
+        :param until: Until as integer timestamp in milliseconds
+        :param from_id: Download data starting with ID (if id is known). Ignores "since" if set.
+        returns tuple: (pair, trades-list)
+        """
+
+        trades: List[Dict] = []
+
+        if not from_id:
+            # Fetch first elements using timebased method to get an ID to paginate on
+            # Depending on the Exchange, this can introduce a drift at the start of the interval
+            # of up to an hour.
+            # e.g. Binance returns the "last 1000" candles within a 1h time interval
+            # - so we will miss the first trades.
+            t = await self._async_fetch_trades(pair, since=since)
+            from_id = t[-1]['id']
+            trades.extend(t[:-1])
+        while True:
+            t = await self._async_fetch_trades(pair,
+                                               params={self._trades_pagination_arg: from_id})
+            if len(t):
+                # Skip last id since its the key for the next call
+                trades.extend(t[:-1])
+                if from_id == t[-1]['id'] or t[-1]['timestamp'] > until:
+                    logger.debug(f"Stopping because from_id did not change. "
+                                 f"Reached {t[-1]['timestamp']} > {until}")
+                    # Reached the end of the defined-download period - add last trade as well.
+                    trades.extend(t[-1:])
+                    break
+
+                from_id = t[-1]['id']
+            else:
+                break
+
+        return (pair, trades)
+
+    async def _async_get_trade_history_time(self, pair: str, until: int,
+                                            since: Optional[int] = None) -> Tuple[str, List]:
+        """
+        Asyncronously gets trade history using fetch_trades,
+        when the exchange uses time-based iteration (check `self._trades_pagination`)
+        :param pair: Pair to fetch trade data for
+        :param since: Since as integer timestamp in milliseconds
+        :param until: Until as integer timestamp in milliseconds
+        returns tuple: (pair, trades-list)
+        """
+
+        trades: List[Dict] = []
+        while True:
+            t = await self._async_fetch_trades(pair, since=since)
+            if len(t):
+                since = t[-1]['timestamp']
+                trades.extend(t)
+                # Reached the end of the defined-download period
+                if until and t[-1]['timestamp'] > until:
+                    logger.debug(
+                        f"Stopping because until was reached. {t[-1]['timestamp']} > {until}")
+                    break
+            else:
+                break
+
+        return (pair, trades)
+
+    async def _async_get_trade_history(self, pair: str,
+                                       since: Optional[int] = None,
+                                       until: Optional[int] = None,
+                                       from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
+        """
+        Async wrapper handling downloading trades using either time or id based methods.
+        """
+
+        if self._trades_pagination == 'time':
+            return await self._async_get_trade_history_time(
+                pair=pair, since=since,
+                until=until or ccxt.Exchange.milliseconds())
+        elif self._trades_pagination == 'id':
+            return await self._async_get_trade_history_id(
+                pair=pair, since=since,
+                until=until or ccxt.Exchange.milliseconds(), from_id=from_id
+            )
+        else:
+            raise OperationalException(f"Exchange {self.name} does use neither time, "
+                                       f"nor id based pagination")
+
+    def get_historic_trades(self, pair: str,
+                            since: Optional[int] = None,
+                            until: Optional[int] = None,
+                            from_id: Optional[str] = None) -> Tuple[str, List]:
+        """
+        Gets candle history using asyncio and returns the list of candles.
+        Handles all async doing.
+        Async over one pair, assuming we get `_ohlcv_candle_limit` candles per call.
+        :param pair: Pair to download
+        :param ticker_interval: Interval to get
+        :param since: Timestamp in milliseconds to get history from
+        :param until: Timestamp in milliseconds. Defaults to current timestamp if not defined.
+        :param from_id: Download data starting with ID (if id is known)
+        :returns List of tickers
+        """
+        if not self.exchange_has("fetchTrades"):
+            raise OperationalException("This exchange does not suport downloading Trades.")
+
+        return asyncio.get_event_loop().run_until_complete(
+            self._async_get_trade_history(pair=pair, since=since,
+                                          until=until, from_id=from_id))
+
     @retrier
     def cancel_order(self, order_id: str, pair: str) -> None:
         if self._config['dry_run']:

freqtrade/exchange/kraken.py

@@ -14,6 +14,10 @@ logger = logging.getLogger(__name__)
 class Kraken(Exchange):
 
     _params: Dict = {"trading_agreement": "agree"}
+    _ft_has: Dict = {
+        "trades_pagination": "id",
+        "trades_pagination_arg": "since",
+    }
 
     @retrier
     def get_balances(self) -> dict:

freqtrade/misc.py

@@ -72,8 +72,10 @@ def json_load(datafile: IO):
 
 def file_load_json(file):
 
+    if file.suffix != ".gz":
         gzipfile = file.with_suffix(file.suffix + '.gz')
-
+    else:
+        gzipfile = file
     # Try gzip file first, otherwise regular json file.
     if gzipfile.is_file():
         logger.debug('Loading ticker data from file %s', gzipfile)

freqtrade/utils.py

@@ -8,7 +8,9 @@ import arrow
 from freqtrade import OperationalException
 from freqtrade.configuration import Configuration, TimeRange
 from freqtrade.configuration.directory_operations import create_userdata_dir
-from freqtrade.data.history import refresh_backtest_ohlcv_data
+from freqtrade.data.history import (convert_trades_to_ohlcv,
+                                    refresh_backtest_ohlcv_data,
+                                    refresh_backtest_trades_data)
 from freqtrade.exchange import available_exchanges, ccxt_exchanges
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.state import RunMode
@@ -88,6 +90,16 @@ def start_download_data(args: Dict[str, Any]) -> None:
         # Init exchange
         exchange = ExchangeResolver(config['exchange']['name'], config).exchange
 
+        if config.get('download_trades'):
+            pairs_not_available = refresh_backtest_trades_data(
+                exchange, pairs=config["pairs"], datadir=Path(config['datadir']),
+                timerange=timerange, erase=config.get("erase"))
+
+            # Convert downloaded trade data to different timeframes
+            convert_trades_to_ohlcv(
+                pairs=config["pairs"], timeframes=config["timeframes"],
+                datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
+        else:
             pairs_not_available = refresh_backtest_ohlcv_data(
                 exchange, pairs=config["pairs"], timeframes=config["timeframes"],
                 dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))

tests/conftest.py

@@ -930,6 +930,110 @@ def trades_for_order():
              'fee': {'cost': 0.008, 'currency': 'LTC'}}]
 
 
+@pytest.fixture(scope="function")
+def trades_history():
+    return [{'info': {'a': 126181329,
+                      'p': '0.01962700',
+                      'q': '0.04000000',
+                      'f': 138604155,
+                      'l': 138604155,
+                      'T': 1565798399463,
+                      'm': False,
+                      'M': True},
+             'timestamp': 1565798399463,
+             'datetime': '2019-08-14T15:59:59.463Z',
+             'symbol': 'ETH/BTC',
+             'id': '126181329',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'buy',
+             'price': 0.019627,
+             'amount': 0.04,
+             'cost': 0.00078508,
+             'fee': None},
+            {'info': {'a': 126181330,
+                      'p': '0.01962700',
+                      'q': '0.24400000',
+                      'f': 138604156,
+                      'l': 138604156,
+                      'T': 1565798399629,
+                      'm': False,
+                      'M': True},
+             'timestamp': 1565798399629,
+             'datetime': '2019-08-14T15:59:59.629Z',
+             'symbol': 'ETH/BTC',
+             'id': '126181330',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'buy',
+             'price': 0.019627,
+             'amount': 0.244,
+             'cost': 0.004788987999999999,
+             'fee': None},
+            {'info': {'a': 126181331,
+                      'p': '0.01962600',
+                      'q': '0.01100000',
+                      'f': 138604157,
+                      'l': 138604157,
+                      'T': 1565798399752,
+                      'm': True,
+                      'M': True},
+             'timestamp': 1565798399752,
+             'datetime': '2019-08-14T15:59:59.752Z',
+             'symbol': 'ETH/BTC',
+             'id': '126181331',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'sell',
+             'price': 0.019626,
+             'amount': 0.011,
+             'cost': 0.00021588599999999999,
+             'fee': None},
+            {'info': {'a': 126181332,
+                      'p': '0.01962600',
+                      'q': '0.01100000',
+                      'f': 138604158,
+                      'l': 138604158,
+                      'T': 1565798399862,
+                      'm': True,
+                      'M': True},
+             'timestamp': 1565798399862,
+             'datetime': '2019-08-14T15:59:59.862Z',
+             'symbol': 'ETH/BTC',
+             'id': '126181332',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'sell',
+             'price': 0.019626,
+             'amount': 0.011,
+             'cost': 0.00021588599999999999,
+             'fee': None},
+            {'info': {'a': 126181333,
+                      'p': '0.01952600',
+                      'q': '0.01200000',
+                      'f': 138604158,
+                      'l': 138604158,
+                      'T': 1565798399872,
+                      'm': True,
+                      'M': True},
+             'timestamp': 1565798399872,
+             'datetime': '2019-08-14T15:59:59.872Z',
+             'symbol': 'ETH/BTC',
+             'id': '126181333',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'sell',
+             'price': 0.019626,
+             'amount': 0.011,
+             'cost': 0.00021588599999999999,
+             'fee': None}]
+
+
 @pytest.fixture(scope="function")
 def trades_for_order2():
     return [{'info': {'id': 34567,

tests/data/test_history.py

@@ -13,15 +13,20 @@ from pandas import DataFrame
 from freqtrade import OperationalException
 from freqtrade.configuration import TimeRange
 from freqtrade.data import history
-from freqtrade.data.history import (download_pair_history,
+from freqtrade.data.history import (_load_cached_data_for_updating,
-                                    _load_cached_data_for_updating,
+                                    convert_trades_to_ohlcv,
-                                    load_tickerdata_file,
+                                    download_pair_history,
+                                    download_trades_history,
+                                    load_tickerdata_file, pair_data_filename,
+                                    pair_trades_filename,
                                     refresh_backtest_ohlcv_data,
+                                    refresh_backtest_trades_data,
                                     trim_tickerlist)
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
 from freqtrade.strategy.default_strategy import DefaultStrategy
-from tests.conftest import get_patched_exchange, log_has, log_has_re, patch_exchange
+from tests.conftest import (get_patched_exchange, log_has, log_has_re,
+                            patch_exchange)
 
 # Change this if modifying UNITTEST/BTC testdatafile
 _BTC_UNITTEST_LENGTH = 13681
@@ -134,6 +139,18 @@ def test_testdata_path(testdatadir) -> None:
     assert str(Path('tests') / 'testdata') in str(testdatadir)
 
 
+def test_pair_data_filename():
+    fn = pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m')
+    assert isinstance(fn, Path)
+    assert fn == Path('freqtrade/hello/world/ETH_BTC-5m.json')
+
+
+def test_pair_trades_filename():
+    fn = pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC')
+    assert isinstance(fn, Path)
+    assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json.gz')
+
+
 def test_load_cached_data_for_updating(mocker) -> None:
     datadir = Path(__file__).parent.parent.joinpath('testdata')
 
@@ -569,3 +586,92 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
     assert "ETH/BTC" in unav_pairs
     assert "XRP/BTC" in unav_pairs
     assert log_has("Skipping pair ETH/BTC...", caplog)
+
+
+def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir):
+    dl_mock = mocker.patch('freqtrade.data.history.download_trades_history', MagicMock())
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
+    )
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    mocker.patch.object(Path, "unlink", MagicMock())
+
+    ex = get_patched_exchange(mocker, default_conf)
+    timerange = TimeRange.parse_timerange("20190101-20190102")
+    unavailable_pairs = refresh_backtest_trades_data(exchange=ex,
+                                                     pairs=["ETH/BTC", "XRP/BTC", "XRP/ETH"],
+                                                     datadir=testdatadir,
+                                                     timerange=timerange, erase=True
+                                                     )
+
+    assert dl_mock.call_count == 2
+    assert dl_mock.call_args[1]['timerange'].starttype == 'date'
+
+    assert log_has("Downloading trades for pair ETH/BTC.", caplog)
+    assert unavailable_pairs == ["XRP/ETH"]
+    assert log_has("Skipping pair XRP/ETH...", caplog)
+
+
+def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog) -> None:
+
+    ght_mock = MagicMock(side_effect=lambda pair, *args, **kwargs: (pair, trades_history))
+    mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
+                 ght_mock)
+    exchange = get_patched_exchange(mocker, default_conf)
+    file1 = testdatadir / 'ETH_BTC-trades.json.gz'
+
+    _backup_file(file1)
+
+    assert not file1.is_file()
+
+    assert download_trades_history(datadir=testdatadir, exchange=exchange,
+                                   pair='ETH/BTC')
+    assert log_has("New Amount of trades: 5", caplog)
+    assert file1.is_file()
+
+    # clean files freshly downloaded
+    _clean_test_file(file1)
+
+    mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
+                 MagicMock(side_effect=ValueError))
+
+    assert not download_trades_history(datadir=testdatadir, exchange=exchange,
+                                       pair='ETH/BTC')
+    assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)
+
+
+def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog):
+
+    pair = 'XRP/ETH'
+    file1 = testdatadir / 'XRP_ETH-1m.json'
+    file5 = testdatadir / 'XRP_ETH-5m.json'
+    # Compare downloaded dataset with converted dataset
+    dfbak_1m = history.load_pair_history(datadir=testdatadir,
+                                         ticker_interval="1m",
+                                         pair=pair)
+    dfbak_5m = history.load_pair_history(datadir=testdatadir,
+                                         ticker_interval="5m",
+                                         pair=pair)
+
+    _backup_file(file1, copy_file=True)
+    _backup_file(file5)
+
+    tr = TimeRange.parse_timerange('20191011-20191012')
+
+    convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'],
+                            datadir=testdatadir, timerange=tr, erase=True)
+
+    assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog)
+    # Load new data
+    df_1m = history.load_pair_history(datadir=testdatadir,
+                                      ticker_interval="1m",
+                                      pair=pair)
+    df_5m = history.load_pair_history(datadir=testdatadir,
+                                      ticker_interval="5m",
+                                      pair=pair)
+
+    assert df_1m.equals(dfbak_1m)
+    assert df_5m.equals(dfbak_5m)
+
+    _clean_test_file(file1)
+    _clean_test_file(file5)

tests/exchange/test_exchange.py

@@ -1142,6 +1142,13 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_
         await exchange._async_get_candle_history(pair, "5m",
                                                  (arrow.utcnow().timestamp - 2000) * 1000)
 
+    with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching '
+                                                   r'historical candlestick data\..*'):
+        api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported("Not supported"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+        await exchange._async_get_candle_history(pair, "5m",
+                                                 (arrow.utcnow().timestamp - 2000) * 1000)
+
 
 @pytest.mark.asyncio
 async def test__async_get_candle_history_empty(default_conf, mocker, caplog):
@@ -1313,6 +1320,196 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na
     assert ticks[9][5] == 2.31452783
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name,
+                                   trades_history):
+
+    caplog.set_level(logging.DEBUG)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    # Monkey-patch async function
+    exchange._api_async.fetch_trades = get_mock_coro(trades_history)
+
+    pair = 'ETH/BTC'
+    res = await exchange._async_fetch_trades(pair, since=None, params=None)
+    assert type(res) is list
+    assert isinstance(res[0], dict)
+    assert isinstance(res[1], dict)
+
+    assert exchange._api_async.fetch_trades.call_count == 1
+    assert exchange._api_async.fetch_trades.call_args[0][0] == pair
+    assert exchange._api_async.fetch_trades.call_args[1]['limit'] == 1000
+
+    assert log_has_re(f"Fetching trades for pair {pair}, since .*", caplog)
+    caplog.clear()
+    exchange._api_async.fetch_trades.reset_mock()
+    res = await exchange._async_fetch_trades(pair, since=None, params={'from': '123'})
+    assert exchange._api_async.fetch_trades.call_count == 1
+    assert exchange._api_async.fetch_trades.call_args[0][0] == pair
+    assert exchange._api_async.fetch_trades.call_args[1]['limit'] == 1000
+    assert exchange._api_async.fetch_trades.call_args[1]['params'] == {'from': '123'}
+    assert log_has_re(f"Fetching trades for pair {pair}, params: .*", caplog)
+
+    exchange = Exchange(default_conf)
+    await async_ccxt_exception(mocker, default_conf, MagicMock(),
+                               "_async_fetch_trades", "fetch_trades",
+                               pair='ABCD/BTC', since=None)
+
+    api_mock = MagicMock()
+    with pytest.raises(OperationalException, match=r'Could not fetch trade data*'):
+        api_mock.fetch_trades = MagicMock(side_effect=ccxt.BaseError("Unknown error"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+        await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000)
+
+    with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching '
+                                                   r'historical trade data\..*'):
+        api_mock.fetch_trades = MagicMock(side_effect=ccxt.NotSupported("Not supported"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+        await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchange_name,
+                                           trades_history):
+
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    pagination_arg = exchange._trades_pagination_arg
+
+    async def mock_get_trade_hist(pair, *args, **kwargs):
+        if 'since' in kwargs:
+            # Return first 3
+            return trades_history[:-2]
+        elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3]['id']:
+            # Return 2
+            return trades_history[-3:-1]
+        else:
+            # Return last 2
+            return trades_history[-2:]
+    # Monkey-patch async function
+    exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
+
+    pair = 'ETH/BTC'
+    ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0]["timestamp"],
+                                                     until=trades_history[-1]["timestamp"]-1)
+    assert type(ret) is tuple
+    assert ret[0] == pair
+    assert type(ret[1]) is list
+    assert len(ret[1]) == len(trades_history)
+    assert exchange._async_fetch_trades.call_count == 3
+    fetch_trades_cal = exchange._async_fetch_trades.call_args_list
+    # first call (using since, not fromId)
+    assert fetch_trades_cal[0][0][0] == pair
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+
+    # 2nd call
+    assert fetch_trades_cal[1][0][0] == pair
+    assert 'params' in fetch_trades_cal[1][1]
+    assert exchange._ft_has['trades_pagination_arg'] in fetch_trades_cal[1][1]['params']
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+async def test__async_get_trade_history_time(default_conf, mocker, caplog, exchange_name,
+                                             trades_history):
+
+    caplog.set_level(logging.DEBUG)
+
+    async def mock_get_trade_hist(pair, *args, **kwargs):
+        if kwargs['since'] == trades_history[0]["timestamp"]:
+            return trades_history[:-1]
+        else:
+            return trades_history[-1:]
+
+    caplog.set_level(logging.DEBUG)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    # Monkey-patch async function
+    exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
+    pair = 'ETH/BTC'
+    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
+                                                       until=trades_history[-1]["timestamp"]-1)
+    assert type(ret) is tuple
+    assert ret[0] == pair
+    assert type(ret[1]) is list
+    assert len(ret[1]) == len(trades_history)
+    assert exchange._async_fetch_trades.call_count == 2
+    fetch_trades_cal = exchange._async_fetch_trades.call_args_list
+    # first call (using since, not fromId)
+    assert fetch_trades_cal[0][0][0] == pair
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+
+    # 2nd call
+    assert fetch_trades_cal[1][0][0] == pair
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+    assert log_has_re(r"Stopping because until was reached.*", caplog)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, exchange_name,
+                                                   trades_history):
+
+    caplog.set_level(logging.DEBUG)
+
+    async def mock_get_trade_hist(pair, *args, **kwargs):
+        if kwargs['since'] == trades_history[0]["timestamp"]:
+            return trades_history[:-1]
+        else:
+            return []
+
+    caplog.set_level(logging.DEBUG)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    # Monkey-patch async function
+    exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
+    pair = 'ETH/BTC'
+    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
+                                                       until=trades_history[-1]["timestamp"]-1)
+    assert type(ret) is tuple
+    assert ret[0] == pair
+    assert type(ret[1]) is list
+    assert len(ret[1]) == len(trades_history) - 1
+    assert exchange._async_fetch_trades.call_count == 2
+    fetch_trades_cal = exchange._async_fetch_trades.call_args_list
+    # first call (using since, not fromId)
+    assert fetch_trades_cal[0][0][0] == pair
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+
+
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test_get_historic_trades(default_conf, mocker, caplog, exchange_name, trades_history):
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+
+    pair = 'ETH/BTC'
+
+    exchange._async_get_trade_history_id = get_mock_coro((pair, trades_history))
+    exchange._async_get_trade_history_time = get_mock_coro((pair, trades_history))
+    ret = exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
+                                       until=trades_history[-1]["timestamp"])
+
+    # Depending on the exchange, one or the other method should be called
+    assert sum([exchange._async_get_trade_history_id.call_count,
+                exchange._async_get_trade_history_time.call_count]) == 1
+
+    assert len(ret) == 2
+    assert ret[0] == pair
+    assert len(ret[1]) == len(trades_history)
+
+
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange_name,
+                                          trades_history):
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=False)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+
+    pair = 'ETH/BTC'
+
+    with pytest.raises(OperationalException,
+                       match="This exchange does not suport downloading Trades."):
+        exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
+                                     until=trades_history[-1]["timestamp"])
+
+
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 def test_cancel_order_dry_run(default_conf, mocker, exchange_name):
     default_conf['dry_run'] = True
@@ -1459,13 +1656,17 @@ def test_merge_ft_has_dict(default_conf, mocker):
     assert ex._ft_has == Exchange._ft_has_default
 
     ex = Kraken(default_conf)
-    assert ex._ft_has == Exchange._ft_has_default
+    assert ex._ft_has != Exchange._ft_has_default
+    assert ex._ft_has['trades_pagination'] == 'id'
+    assert ex._ft_has['trades_pagination_arg'] == 'since'
 
     # Binance defines different values
     ex = Binance(default_conf)
     assert ex._ft_has != Exchange._ft_has_default
     assert ex._ft_has['stoploss_on_exchange']
     assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc']
+    assert ex._ft_has['trades_pagination'] == 'id'
+    assert ex._ft_has['trades_pagination_arg'] == 'fromId'
 
     conf = copy.deepcopy(default_conf)
     conf['exchange']['_ft_has_params'] = {"DeadBeef": 20,

tests/test_utils.py

@@ -260,3 +260,25 @@ def test_download_data_no_pairs(mocker, caplog):
     with pytest.raises(OperationalException,
                        match=r"Downloading data requires a list of pairs\..*"):
         start_download_data(pargs)
+
+
+def test_download_data_trades(mocker, caplog):
+    dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_trades_data',
+                           MagicMock(return_value=[]))
+    convert_mock = mocker.patch('freqtrade.utils.convert_trades_to_ohlcv',
+                                MagicMock(return_value=[]))
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    args = [
+        "download-data",
+        "--exchange", "kraken",
+        "--pairs", "ETH/BTC", "XRP/BTC",
+        "--days", "20",
+        "--dl-trades"
+    ]
+    start_download_data(get_args(args))
+    assert dl_mock.call_args[1]['timerange'].starttype == "date"
+    assert dl_mock.call_count == 1
+    assert convert_mock.call_count == 1