adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into align_userdata

Browse Source
This commit is contained in:
Matthias
2019-08-05 06:55:51 +02:00
parent 39e8e507d9 eeecdd4e5a
commit 383b24ab84
18 changed files with 267 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
freqtrade/data/btanalysis.py
View File

@@ -81,19 +81,30 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
"""
trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS)
persistence.init(db_url, clean_open_orders=False)
columns = ["pair", "profit", "open_time", "close_time",
"open_rate", "close_rate", "duration", "sell_reason",
"max_rate", "min_rate"]
trades = pd.DataFrame([(t.pair, t.calc_profit(),
columns = ["pair", "open_time", "close_time", "profit", "profitperc",
"open_rate", "close_rate", "amount", "duration", "sell_reason",
"fee_open", "fee_close", "open_rate_requested", "close_rate_requested",
"stake_amount", "max_rate", "min_rate", "id", "exchange",
"stop_loss", "initial_stop_loss", "strategy", "ticker_interval"]
trades = pd.DataFrame([(t.pair,
t.open_date.replace(tzinfo=pytz.UTC),
t.close_date.replace(tzinfo=pytz.UTC) if t.close_date else None,
t.open_rate, t.close_rate,
t.close_date.timestamp() - t.open_date.timestamp()
if t.close_date else None,
t.calc_profit(), t.calc_profit_percent(),
t.open_rate, t.close_rate, t.amount,
(t.close_date.timestamp() - t.open_date.timestamp()
if t.close_date else None),
t.sell_reason,
t.fee_open, t.fee_close,
t.open_rate_requested,
t.close_rate_requested,
t.stake_amount,
t.max_rate,
t.min_rate,
t.id, t.exchange,
t.stop_loss, t.initial_stop_loss,
t.strategy, t.ticker_interval
)
for t in Trade.query.all()],
columns=columns)

8
freqtrade/exchange/exchange.py
View File

@@ -260,7 +260,7 @@ class Exchange(object):
if not self.markets:
logger.warning('Unable to validate pairs (assuming they are correct).')
# return
return
for pair in pairs:
# Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
@@ -269,6 +269,12 @@ class Exchange(object):
raise OperationalException(
f'Pair {pair} is not available on {self.name}. '
f'Please remove {pair} from your whitelist.')
elif self.markets[pair].get('info', {}).get('IsRestricted', False):
# Warn users about restricted pairs in whitelist.
# We cannot determine reliably if Users are affected.
logger.warning(f"Pair {pair} is restricted for some users on this exchange."
f"Please check if you are impacted by this restriction "
f"on the exchange and eventually remove {pair} from your whitelist.")
def get_valid_pair_combination(self, curr_1, curr_2) -> str:
"""

32
freqtrade/optimize/hyperopt.py
View File

@@ -10,7 +10,7 @@ import sys
from operator import itemgetter
from pathlib import Path
from pprint import pprint
from typing import Any, Dict, List
from typing import Any, Dict, List, Optional
from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects, cpu_count
from pandas import DataFrame
@@ -70,7 +70,7 @@ class Hyperopt(Backtesting):
if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
self.advise_sell = self.custom_hyperopt.populate_sell_trend # type: ignore
# Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set
# Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set
if self.config.get('use_max_market_positions', True):
self.max_open_trades = self.config['max_open_trades']
else:
@@ -138,11 +138,20 @@ class Hyperopt(Backtesting):
params = best_result['params']
log_str = self.format_results_logstring(best_result)
print(f"\nBest result:\n{log_str}\nwith values:")
pprint(params, indent=4)
print(f"\nBest result:\n\n{log_str}\n")
if self.has_space('buy'):
print('Buy hyperspace params:')
pprint({p.name: params.get(p.name) for p in self.hyperopt_space('buy')},
indent=4)
if self.has_space('sell'):
print('Sell hyperspace params:')
pprint({p.name: params.get(p.name) for p in self.hyperopt_space('sell')},
indent=4)
if self.has_space('roi'):
print("ROI table:")
pprint(self.custom_hyperopt.generate_roi_table(params), indent=4)
if self.has_space('stoploss'):
print(f"Stoploss: {params.get('stoploss')}")
def log_results(self, results) -> None:
"""
@@ -176,21 +185,24 @@ class Hyperopt(Backtesting):
"""
return any(s in self.config['spaces'] for s in [space, 'all'])
def hyperopt_space(self) -> List[Dimension]:
def hyperopt_space(self, space: Optional[str] = None) -> List[Dimension]:
"""
Return the space to use during Hyperopt
Return the dimensions in the hyperoptimization space.
:param space: Defines hyperspace to return dimensions for.
If None, then the self.has_space() will be used to return dimensions
for all hyperspaces used.
"""
spaces: List[Dimension] = []
if self.has_space('buy'):
if space == 'buy' or (space is None and self.has_space('buy')):
logger.debug("Hyperopt has 'buy' space")
spaces += self.custom_hyperopt.indicator_space()
if self.has_space('sell'):
if space == 'sell' or (space is None and self.has_space('sell')):
logger.debug("Hyperopt has 'sell' space")
spaces += self.custom_hyperopt.sell_indicator_space()
if self.has_space('roi'):
if space == 'roi' or (space is None and self.has_space('roi')):
logger.debug("Hyperopt has 'roi' space")
spaces += self.custom_hyperopt.roi_space()
if self.has_space('stoploss'):
if space == 'stoploss' or (space is None and self.has_space('stoploss')):
logger.debug("Hyperopt has 'stoploss' space")
spaces += self.custom_hyperopt.stoploss_space()
return spaces

5
freqtrade/plot/plotting.py
View File

@@ -316,8 +316,9 @@ def store_plot_file(fig, filename: str, directory: Path, auto_open: bool = False
:param ticker_interval: Used as part of the filename
:return: None
"""
directory.mkdir(parents=True, exist_ok=True)
plot(fig, filename=str(directory.joinpath(filename)),
_filename = directory.joinpath(filename)
plot(fig, filename=str(_filename),
auto_open=auto_open)
logger.info(f"Stored plot as {_filename}")

24
freqtrade/strategy/interface.py
View File

@@ -158,6 +158,23 @@ class IStrategy(ABC):
"""
Parses the given ticker history and returns a populated DataFrame
add several TA indicators and buy signal to it
:param dataframe: Dataframe containing ticker data
:param metadata: Metadata dictionary with additional data (e.g. 'pair')
:return: DataFrame with ticker data and indicator data
"""
logger.debug("TA Analysis Launched")
dataframe = self.advise_indicators(dataframe, metadata)
dataframe = self.advise_buy(dataframe, metadata)
dataframe = self.advise_sell(dataframe, metadata)
return dataframe
def _analyze_ticker_internal(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Parses the given ticker history and returns a populated DataFrame
add several TA indicators and buy signal to it
WARNING: Used internally only, may skip analysis if `process_only_new_candles` is set.
:param dataframe: Dataframe containing ticker data
:param metadata: Metadata dictionary with additional data (e.g. 'pair')
:return: DataFrame with ticker data and indicator data
"""
@@ -168,10 +185,7 @@ class IStrategy(ABC):
if (not self.process_only_new_candles or
self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']):
# Defs that only make change on new candle data.
logger.debug("TA Analysis Launched")
dataframe = self.advise_indicators(dataframe, metadata)
dataframe = self.advise_buy(dataframe, metadata)
dataframe = self.advise_sell(dataframe, metadata)
dataframe = self.analyze_ticker(dataframe, metadata)
self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date']
else:
logger.debug("Skipping TA Analysis for already analyzed candle")
@@ -198,7 +212,7 @@ class IStrategy(ABC):
return False, False
try:
dataframe = self.analyze_ticker(dataframe, {'pair': pair})
dataframe = self._analyze_ticker_internal(dataframe, {'pair': pair})
except ValueError as error:
logger.warning(
'Unable to analyze ticker for pair %s: %s',

18
freqtrade/tests/conftest.py
View File

@@ -305,7 +305,7 @@ def markets():
'max': 500000,
},
},
'info': '',
'info': {},
},
'TKN/BTC': {
'id': 'tknbtc',
@@ -330,7 +330,7 @@ def markets():
'max': 500000,
},
},
'info': '',
'info': {},
},
'BLK/BTC': {
'id': 'blkbtc',
@@ -355,7 +355,7 @@ def markets():
'max': 500000,
},
},
'info': '',
'info': {},
},
'LTC/BTC': {
'id': 'ltcbtc',
@@ -380,7 +380,7 @@ def markets():
'max': 500000,
},
},
'info': '',
'info': {},
},
'XRP/BTC': {
'id': 'xrpbtc',
@@ -405,7 +405,7 @@ def markets():
'max': 500000,
},
},
'info': '',
'info': {},
},
'NEO/BTC': {
'id': 'neobtc',
@@ -430,7 +430,7 @@ def markets():
'max': 500000,
},
},
'info': '',
'info': {},
},
'BTT/BTC': {
'id': 'BTTBTC',
@@ -458,7 +458,7 @@ def markets():
'max': None
}
},
'info': "",
'info': {},
},
'ETH/USDT': {
'id': 'USDT-ETH',
@@ -480,7 +480,7 @@ def markets():
}
},
'active': True,
'info': ""
'info': {},
},
'LTC/USDT': {
'id': 'USDT-LTC',
@@ -502,7 +502,7 @@ def markets():
'max': None
}
},
'info': ""
'info': {},
}
}

5
freqtrade/tests/data/test_btanalysis.py
View File

@@ -45,6 +45,11 @@ def test_load_trades_db(default_conf, fee, mocker):
assert isinstance(trades, DataFrame)
assert "pair" in trades.columns
assert "open_time" in trades.columns
assert "profitperc" in trades.columns
for col in BT_DATA_COLUMNS:
if col not in ['index', 'open_at_end']:
assert col in trades.columns
def test_extract_trades_of_period():

21
freqtrade/tests/exchange/test_exchange.py
View File

@@ -318,7 +318,7 @@ def test__reload_markets_exception(default_conf, mocker, caplog):
def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs directly
api_mock = MagicMock()
type(api_mock).markets = PropertyMock(return_value={
'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': ''
'ETH/BTC': {}, 'LTC/BTC': {}, 'XRP/BTC': {}, 'NEO/BTC': {}
})
id_mock = PropertyMock(return_value='test_exchange')
type(api_mock).id = id_mock
@@ -332,7 +332,7 @@ def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs d
def test_validate_pairs_not_available(default_conf, mocker):
api_mock = MagicMock()
type(api_mock).markets = PropertyMock(return_value={
'XRP/BTC': 'inactive'
'XRP/BTC': {'inactive': True}
})
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
@@ -361,6 +361,23 @@ def test_validate_pairs_exception(default_conf, mocker, caplog):
caplog.record_tuples)
def test_validate_pairs_restricted(default_conf, mocker, caplog):
api_mock = MagicMock()
type(api_mock).markets = PropertyMock(return_value={
'ETH/BTC': {}, 'LTC/BTC': {}, 'NEO/BTC': {},
'XRP/BTC': {'info': {'IsRestricted': True}}
})
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
Exchange(default_conf)
assert log_has(f"Pair XRP/BTC is restricted for some users on this exchange."
f"Please check if you are impacted by this restriction "
f"on the exchange and eventually remove XRP/BTC from your whitelist.",
caplog.record_tuples)
def test_validate_timeframes(default_conf, mocker):
default_conf["ticker_interval"] = "5m"
api_mock = MagicMock()

3
freqtrade/tests/optimize/test_backtesting.py
View File

@@ -202,6 +202,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
assert config['runmode'] == RunMode.BACKTEST
@pytest.mark.filterwarnings("ignore:DEPRECATED")
def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> None:
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch(
@@ -812,6 +813,7 @@ def test_backtest_record(default_conf, fee, mocker):
assert dur > 0
@pytest.mark.filterwarnings("ignore:DEPRECATED")
def test_backtest_start_live(default_conf, mocker, caplog):
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
@@ -858,6 +860,7 @@ def test_backtest_start_live(default_conf, mocker, caplog):
assert log_has(line, caplog.record_tuples)
@pytest.mark.filterwarnings("ignore:DEPRECATED")
def test_backtest_start_multi_strat(default_conf, mocker, caplog):
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']

2
freqtrade/tests/optimize/test_hyperopt.py
View File

@@ -466,7 +466,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None:
parallel.assert_called_once()
out, err = capsys.readouterr()
assert 'Best result:\n* 1/1: foo result Objective: 1.00000\nwith values:\n' in out
assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out
assert dumper.called
# Should be called twice, once for tickerdata, once to save evaluations
assert dumper.call_count == 2

20
freqtrade/tests/strategy/test_interface.py
View File

@@ -19,13 +19,13 @@ _STRATEGY = DefaultStrategy(config={})
def test_returns_latest_buy_signal(mocker, default_conf, ticker_history):
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
_STRATEGY, '_analyze_ticker_internal',
return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}])
)
assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (True, False)
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
_STRATEGY, '_analyze_ticker_internal',
return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}])
)
assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (False, True)
@@ -33,14 +33,14 @@ def test_returns_latest_buy_signal(mocker, default_conf, ticker_history):
def test_returns_latest_sell_signal(mocker, default_conf, ticker_history):
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
_STRATEGY, '_analyze_ticker_internal',
return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}])
)
assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (False, True)
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
_STRATEGY, '_analyze_ticker_internal',
return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}])
)
assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (True, False)
@@ -60,7 +60,7 @@ def test_get_signal_empty(default_conf, mocker, caplog):
def test_get_signal_exception_valueerror(default_conf, mocker, caplog, ticker_history):
caplog.set_level(logging.INFO)
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
_STRATEGY, '_analyze_ticker_internal',
side_effect=ValueError('xyz')
)
assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'],
@@ -71,7 +71,7 @@ def test_get_signal_exception_valueerror(default_conf, mocker, caplog, ticker_hi
def test_get_signal_empty_dataframe(default_conf, mocker, caplog, ticker_history):
caplog.set_level(logging.INFO)
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
_STRATEGY, '_analyze_ticker_internal',
return_value=DataFrame([])
)
assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
@@ -86,7 +86,7 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog, ticker_history):
oldtime = arrow.utcnow().shift(minutes=-16)
ticks = DataFrame([{'buy': 1, 'date': oldtime}])
mocker.patch.object(
_STRATEGY, 'analyze_ticker',
_STRATEGY, '_analyze_ticker_internal',
return_value=DataFrame(ticks)
)
assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
@@ -252,7 +252,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None:
caplog.record_tuples)
def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None:
def test__analyze_ticker_internal_skip_analyze(ticker_history, mocker, caplog) -> None:
caplog.set_level(logging.DEBUG)
ind_mock = MagicMock(side_effect=lambda x, meta: x)
buy_mock = MagicMock(side_effect=lambda x, meta: x)
@@ -267,7 +267,7 @@ def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None:
strategy = DefaultStrategy({})
strategy.process_only_new_candles = True
ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'})
ret = strategy._analyze_ticker_internal(ticker_history, {'pair': 'ETH/BTC'})
assert 'high' in ret.columns
assert 'low' in ret.columns
assert 'close' in ret.columns
@@ -280,7 +280,7 @@ def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None:
caplog.record_tuples)
caplog.clear()
ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'})
ret = strategy._analyze_ticker_internal(ticker_history, {'pair': 'ETH/BTC'})
# No analysis happens as process_only_new_candles is true
assert ind_mock.call_count == 1
assert buy_mock.call_count == 1

1
freqtrade/tests/test_configuration.py
View File

@@ -327,6 +327,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
assert 'export' not in config
@pytest.mark.filterwarnings("ignore:DEPRECATED")
def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None:
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch(

2
freqtrade/tests/test_plotting.py
View File

@@ -217,6 +217,8 @@ def test_generate_plot_file(mocker, caplog):
assert plot_mock.call_args[0][0] == fig
assert (plot_mock.call_args_list[0][1]['filename']
== "user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html")
assert log_has("Stored plot as user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html",
caplog.record_tuples)
def test_add_profit():