adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1404 from freqtrade/feat/pass_df

Browse Source 
keep DF instead of list
This commit is contained in:
Matthias 2018-12-13 20:14:32 +01:00 committed by GitHub
commit 04c330f10b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 81 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
freqtrade/exchange/__init__.py
View File

@ -7,12 +7,14 @@ from typing import List, Dict, Tuple, Any, Optional
from datetime import datetime
from math import floor, ceil
import arrow
import asyncio
import ccxt
import ccxt.async_support as ccxt_async
import arrow
from pandas import DataFrame
from freqtrade import constants, OperationalException, DependencyException, TemporaryError
from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe
logger = logging.getLogger(__name__)
@ -81,7 +83,7 @@ class Exchange(object):
self._pairs_last_refresh_time: Dict[str, int] = {}
# Holds candles
self.klines: Dict[str, Any] = {}
self._klines: Dict[str, DataFrame] = {}
# Holds all open sell orders for dry_run
self._dry_run_open_orders: Dict[str, Any] = {}
@ -155,6 +157,12 @@ class Exchange(object):
"""exchange ccxt id"""
return self._api.id
def klines(self, pair: str) -> DataFrame:
if pair in self._klines:
return self._klines[pair].copy()
else:
return None
def set_sandbox(self, api, exchange_config: dict, name: str):
if exchange_config.get('sandbox'):
if api.urls.get('test'):
@ -499,7 +507,7 @@ class Exchange(object):
def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None:
"""
Refresh tickers asyncronously and set `klines` of this object with the result
Refresh tickers asyncronously and set `_klines` of this object with the result
"""
logger.debug("Refreshing klines for %d pairs", len(pair_list))
asyncio.get_event_loop().run_until_complete(
@ -515,7 +523,7 @@ class Exchange(object):
# Gather corotines to run
for pair in pairs:
if not (self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >=
arrow.utcnow().timestamp and pair in self.klines):
arrow.utcnow().timestamp and pair in self._klines):
input_coroutines.append(self._async_get_candle_history(pair, tick_interval))
else:
logger.debug("Using cached klines data for %s ...", pair)
@ -528,7 +536,7 @@ class Exchange(object):
if ticks:
self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000
# keeping parsed dataframe in cache
self.klines[pair] = ticks
self._klines[pair] = parse_ticker_dataframe(ticks)
return tickers
@retrier_async

1
freqtrade/exchange/exchange_helpers.py
View File

@ -14,6 +14,7 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame:
:param ticker: ticker list, as returned by exchange.async_get_candle_history
:return: DataFrame
"""
logger.debug("Parsing tickerlist to dataframe")
cols = ['date', 'open', 'high', 'low', 'close', 'volume']
frame = DataFrame(ticker, columns=cols)

5
freqtrade/freqtradebot.py
View File

@ -317,7 +317,7 @@ class FreqtradeBot(object):
# running get_signal on historical data fetched
for _pair in whitelist:
(buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair))
(buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines(_pair))
if buy and not sell:
stake_amount = self._get_trade_stake_amount(_pair)
if not stake_amount:
@ -540,9 +540,8 @@ class FreqtradeBot(object):
(buy, sell) = (False, False)
experimental = self.config.get('experimental', {})
if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'):
ticker = self.exchange.klines.get(trade.pair)
(buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval,
ticker)
self.exchange.klines(trade.pair))
config_ask_strategy = self.config.get('ask_strategy', {})
if config_ask_strategy.get('use_order_book', False):

2
freqtrade/optimize/backtesting.py
View File

@ -362,7 +362,7 @@ class Backtesting(object):
if self.config.get('live'):
logger.info('Downloading data for all pairs in whitelist ...')
self.exchange.refresh_tickers(pairs, self.ticker_interval)
data = self.exchange.klines
data = self.exchange._klines
else:
logger.info('Using local backtesting data (using whitelist in given config) ...')

15
freqtrade/strategy/interface.py
View File

@ -6,7 +6,7 @@ import logging
from abc import ABC, abstractmethod
from datetime import datetime
from enum import Enum
from typing import Dict, List, NamedTuple, Optional, Tuple
from typing import Dict, List, NamedTuple, Tuple
import warnings
import arrow
@ -122,19 +122,17 @@ class IStrategy(ABC):
"""
return self.__class__.__name__
def analyze_ticker(self, ticker_history: List[Dict], metadata: dict) -> DataFrame:
def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Parses the given ticker history and returns a populated DataFrame
add several TA indicators and buy signal to it
:return DataFrame with ticker data and indicator data
"""
dataframe = parse_ticker_dataframe(ticker_history)
pair = str(metadata.get('pair'))
# Test if seen this pair and last candle before.
# always run if process_only_new_candles is set to true
# always run if process_only_new_candles is set to false
if (not self.process_only_new_candles or
self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']):
# Defs that only make change on new candle data.
@ -155,19 +153,20 @@ class IStrategy(ABC):
return dataframe
def get_signal(self, pair: str, interval: str,
ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]:
dataframe: DataFrame) -> Tuple[bool, bool]:
"""
Calculates current signal based several technical analysis indicators
:param pair: pair in format ANT/BTC
:param interval: Interval to use (in min)
:param dataframe: Dataframe to analyze
:return: (Buy, Sell) A bool-tuple indicating buy/sell signal
"""
if not ticker_hist:
if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning('Empty ticker history for pair %s', pair)
return False, False
try:
dataframe = self.analyze_ticker(ticker_hist, {'pair': pair})
dataframe = self.analyze_ticker(dataframe, {'pair': pair})
except ValueError as error:
logger.warning(
'Unable to analyze ticker for pair %s: %s',

7
freqtrade/tests/conftest.py
View File

@ -481,7 +481,7 @@ def order_book_l2():
@pytest.fixture
def ticker_history():
def ticker_history_list():
return [
[
1511686200000, # unix timestamp ms
@ -510,6 +510,11 @@ def ticker_history():
]
@pytest.fixture
def ticker_history(ticker_history_list):
return parse_ticker_dataframe(ticker_history_list)
@pytest.fixture
def tickers():
return MagicMock(return_value={

18
freqtrade/tests/exchange/test_exchange.py
View File

@ -9,6 +9,7 @@ from unittest.mock import Mock, MagicMock, PropertyMock
import arrow
import ccxt
import pytest
from pandas import DataFrame
from freqtrade import DependencyException, OperationalException, TemporaryError
from freqtrade.exchange import API_RETRY_COUNT, Exchange
@ -737,12 +738,20 @@ def test_get_history(default_conf, mocker, caplog):
def test_refresh_tickers(mocker, default_conf, caplog) -> None:
tick = [
[
arrow.utcnow().timestamp * 1000, # unix timestamp ms
(arrow.utcnow().timestamp - 1) * 1000, # unix timestamp ms
1, # open
2, # high
3, # low
4, # close
5, # volume (in quote currency)
],
[
arrow.utcnow().timestamp * 1000, # unix timestamp ms
3, # open
1, # high
4, # low
6, # close
5, # volume (in quote currency)
]
]
@ -752,14 +761,15 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None:
pairs = ['IOTA/ETH', 'XRP/ETH']
# empty dicts
assert not exchange.klines
assert not exchange._klines
exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m')
assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples)
assert exchange.klines
assert exchange._klines
assert exchange._api_async.fetch_ohlcv.call_count == 2
for pair in pairs:
assert exchange.klines[pair]
assert isinstance(exchange.klines(pair), DataFrame)
assert len(exchange.klines(pair)) > 0
# test caching
exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m')

6
freqtrade/tests/exchange/test_exchange_helpers.py
View File

@ -1,6 +1,8 @@
# pragma pylint: disable=missing-docstring, C0103
import logging
from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe
from freqtrade.tests.conftest import log_has
def test_dataframe_correct_length(result):
@ -13,9 +15,11 @@ def test_dataframe_correct_columns(result):
['date', 'open', 'high', 'low', 'close', 'volume']
def test_parse_ticker_dataframe(ticker_history):
def test_parse_ticker_dataframe(ticker_history, caplog):
columns = ['date', 'open', 'high', 'low', 'close', 'volume']
caplog.set_level(logging.DEBUG)
# Test file with BV data
dataframe = parse_ticker_dataframe(ticker_history)
assert dataframe.columns.tolist() == columns
assert log_has('Parsing tickerlist to dataframe', caplog.record_tuples)

4
freqtrade/tests/optimize/test_backtesting.py
View File

@ -840,7 +840,7 @@ def test_backtest_start_live(default_conf, mocker, caplog):
'Using stake_currency: BTC ...',
'Using stake_amount: 0.001 ...',
'Downloading data for all pairs in whitelist ...',
'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..',
'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:57:00+00:00 (0 days)..',
'Parameter --enable-position-stacking detected ...'
]
@ -899,7 +899,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog):
'Using stake_currency: BTC ...',
'Using stake_amount: 0.001 ...',
'Downloading data for all pairs in whitelist ...',
'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..',
'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:57:00+00:00 (0 days)..',
'Parameter --enable-position-stacking detected ...',
'Running backtesting for Strategy DefaultStrategy',
'Running backtesting for Strategy TestStrategy',

12
freqtrade/tests/optimize/test_optimize.py
View File

@ -85,11 +85,11 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None:
_clean_test_file(file)
def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_conf) -> None:
def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, default_conf) -> None:
"""
Test load_data() with 1 min ticker
"""
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list)
exchange = get_patched_exchange(mocker, default_conf)
file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
@ -119,8 +119,8 @@ def test_testdata_path() -> None:
assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None)
def test_download_pairs(ticker_history, mocker, default_conf) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
def test_download_pairs(ticker_history_list, mocker, default_conf) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list)
exchange = get_patched_exchange(mocker, default_conf)
file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json')
@ -280,8 +280,8 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf)
assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples)
def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
def test_download_backtesting_testdata(ticker_history_list, mocker, default_conf) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list)
exchange = get_patched_exchange(mocker, default_conf)
# Tst that pairs-cached is not touched.
assert not exchange._pairs_last_refresh_time

34
freqtrade/tests/strategy/test_interface.py
View File

@ -16,62 +16,69 @@ from freqtrade.strategy.default_strategy import DefaultStrategy
_STRATEGY = DefaultStrategy(config={})
def test_returns_latest_buy_signal(mocker, default_conf):
def test_returns_latest_buy_signal(mocker, default_conf, ticker_history):
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}])
)
assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (True, False)
assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (True, False)
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}])
)
assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (False, True)
assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (False, True)
def test_returns_latest_sell_signal(mocker, default_conf):
def test_returns_latest_sell_signal(mocker, default_conf, ticker_history):
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}])
)
assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (False, True)
assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (False, True)
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}])
)
assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (True, False)
assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (True, False)
def test_get_signal_empty(default_conf, mocker, caplog):
assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'],
None)
DataFrame())
assert log_has('Empty ticker history for pair foo', caplog.record_tuples)
caplog.clear()
assert (False, False) == _STRATEGY.get_signal('bar', default_conf['ticker_interval'],
[])
assert log_has('Empty ticker history for pair bar', caplog.record_tuples)
def test_get_signal_exception_valueerror(default_conf, mocker, caplog):
def test_get_signal_exception_valueerror(default_conf, mocker, caplog, ticker_history):
caplog.set_level(logging.INFO)
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
side_effect=ValueError('xyz')
)
assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'], 1)
assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'],
ticker_history)
assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples)
def test_get_signal_empty_dataframe(default_conf, mocker, caplog):
def test_get_signal_empty_dataframe(default_conf, mocker, caplog, ticker_history):
caplog.set_level(logging.INFO)
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
return_value=DataFrame([])
)
assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], 1)
assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
ticker_history)
assert log_has('Empty dataframe for pair xyz', caplog.record_tuples)
def test_get_signal_old_dataframe(default_conf, mocker, caplog):
def test_get_signal_old_dataframe(default_conf, mocker, caplog, ticker_history):
caplog.set_level(logging.INFO)
# default_conf defines a 5m interval. we check interval * 2 + 5m
# this is necessary as the last candle is removed (partial candles) by default
@ -81,7 +88,8 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog):
_STRATEGY, 'analyze_ticker',
return_value=DataFrame(ticks)
)
assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], 1)
assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
ticker_history)
assert log_has(
'Outdated history for pair xyz. Last tick is 16 minutes old',
caplog.record_tuples

4
freqtrade/tests/test_misc.py
View File

@ -16,8 +16,8 @@ def test_shorten_date() -> None:
assert shorten_date(str_data) == str_shorten_data
def test_datesarray_to_datetimearray(ticker_history):
dataframes = parse_ticker_dataframe(ticker_history)
def test_datesarray_to_datetimearray(ticker_history_list):
dataframes = parse_ticker_dataframe(ticker_history_list)
dates = datesarray_to_datetimearray(dataframes['date'])
assert isinstance(dates[0], datetime.datetime)

2
scripts/plot_dataframe.py
View File

@ -139,7 +139,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
if args.live:
logger.info('Downloading pair.')
exchange.refresh_tickers([pair], tick_interval)
tickers[pair] = exchange.klines[pair]
tickers[pair] = exchange.klines(pair)
else:
tickers = optimize.load_data(
datadir=_CONF.get("datadir"),