adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2278 from freqtrade/remove_refresh

Browse Source 
Remove refresh-pairs-cached
This commit is contained in:
hroff-1902 2019-09-24 09:07:25 +03:00 committed by GitHub
commit d066ab2620
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 27 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/backtesting.md
View File

@ -7,7 +7,7 @@ Backtesting.
To download data (candles / OHLCV) needed for backtesting and hyperoptimization use the `freqtrade download-data` command.
If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes.
If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes for 30 days.
Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory.
Alternatively, a `pairs.json` file can be used.
@ -37,6 +37,10 @@ This will download ticker data for all the currency pairs you defined in `pairs.
- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers.
- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options.
!!! Tip Updating existing data
If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data.
Be carefull though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded.
## Test your strategy with Backtesting
Now you have good Buy and Sell strategies and some historic data, you want to test it against

12
docs/bot-usage.md
View File

@ -184,10 +184,6 @@ optional arguments:
Specify max_open_trades to use.
--stake_amount STAKE_AMOUNT
Specify stake_amount.
-r, --refresh-pairs-cached
Refresh the pairs files in tests/testdata with the
latest data from the exchange. Use it if you want to
run your optimization commands with up-to-date data.
--eps, --enable-position-stacking
Allow buying the same pair multiple times (position
stacking).
@ -245,10 +241,6 @@ optional arguments:
Specify max_open_trades to use.
--stake_amount STAKE_AMOUNT
Specify stake_amount.
-r, --refresh-pairs-cached
Refresh the pairs files in tests/testdata with the
latest data from the exchange. Use it if you want to
run your optimization commands with up-to-date data.
--customhyperopt NAME
Specify hyperopt class name (default:
`DefaultHyperOpts`).
@ -310,10 +302,6 @@ optional arguments:
Specify max_open_trades to use.
--stake_amount STAKE_AMOUNT
Specify stake_amount.
-r, --refresh-pairs-cached
Refresh the pairs files in tests/testdata with the
latest data from the exchange. Use it if you want to
run your optimization commands with up-to-date data.
--stoplosses STOPLOSS_RANGE
Defines a range of stoploss against which edge will
assess the strategy the format is "min,max,step"

6
docs/deprecated.md
View File

@ -4,7 +4,7 @@ This page contains description of the command line arguments, configuration para
and the bot features that were declared as DEPRECATED by the bot development team
and are no longer supported. Please avoid their usage in your configuration.
## Deprecated
## Removed features
### the `--refresh-pairs-cached` command line option
@ -12,9 +12,7 @@ and are no longer supported. Please avoid their usage in your configuration.
Since this leads to much confusion, and slows down backtesting (while not being part of backtesting) this has been singled out
as a seperate freqtrade subcommand `freqtrade download-data`.
This command line option was deprecated in `2019.7-dev` and will be removed after the next release.
## Removed features
This command line option was deprecated in 2019.7-dev (develop branch) and removed in 2019.9 (master branch).
### The **--dynamic-whitelist** command line option

2
freqtrade/configuration/arguments.py
View File

@ -15,7 +15,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
ARGS_MAIN = ARGS_COMMON + ARGS_STRATEGY + ["db_url", "sd_notify"]
ARGS_COMMON_OPTIMIZE = ["ticker_interval", "timerange",
"max_open_trades", "stake_amount", "refresh_pairs"]
"max_open_trades", "stake_amount"]
ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
"strategy_list", "export", "exportfilename"]

7
freqtrade/configuration/cli_options.py
View File

@ -107,13 +107,6 @@ AVAILABLE_CLI_OPTIONS = {
help='Specify stake_amount.',
type=float,
),
"refresh_pairs": Arg(
'-r', '--refresh-pairs-cached',
help='Refresh the pairs files in tests/testdata with the latest data from the '
'exchange. Use it if you want to run your optimization commands with '
'up-to-date data.',
action='store_true',
),
# Backtesting
"position_stacking": Arg(
'--eps', '--enable-position-stacking',

4
freqtrade/configuration/configuration.py
View File

@ -215,10 +215,6 @@ class Configuration:
self._process_datadir_options(config)
self._args_to_config(config, argname='refresh_pairs',
logstring='Parameter -r/--refresh-pairs-cached detected ...',
deprecated_msg='-r/--refresh-pairs-cached will be removed soon.')
self._args_to_config(config, argname='strategy_list',
logstring='Using strategy list of {} strategies', logfun=len)

1
freqtrade/data/dataprovider.py
View File

@ -65,7 +65,6 @@ class DataProvider:
"""
return load_pair_history(pair=pair,
ticker_interval=ticker_interval or self._config['ticker_interval'],
refresh_pairs=False,
datadir=Path(self._config['datadir'])
)

37
freqtrade/data/history.py
View File

@ -129,8 +129,7 @@ def load_pair_history(pair: str,
else:
logger.warning(
f'No history data for pair: "{pair}", interval: {ticker_interval}. '
'Use --refresh-pairs-cached option or `freqtrade download-data` '
'script to download the data'
'Use `freqtrade download-data` to download the data'
)
return None
@ -142,33 +141,25 @@ def load_data(datadir: Path,
exchange: Optional[Exchange] = None,
timerange: TimeRange = TimeRange(None, None, 0, 0),
fill_up_missing: bool = True,
live: bool = False
) -> Dict[str, DataFrame]:
"""
Loads ticker history data for a list of pairs the given parameters
Loads ticker history data for a list of pairs
:return: dict(<pair>:<tickerlist>)
TODO: refresh_pairs is still used by edge to keep the data uptodate.
This should be replaced in the future. Instead, writing the current candles to disk
from dataprovider should be implemented, as this would avoid loading ohlcv data twice.
exchange and refresh_pairs are then not needed here nor in load_pair_history.
"""
result: Dict[str, DataFrame] = {}
if live:
if exchange:
logger.info('Live: Downloading data for all defined pairs ...')
exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs])
result = {key[0]: value for key, value in exchange._klines.items() if value is not None}
else:
raise OperationalException(
"Exchange needs to be initialized when using live data."
)
else:
logger.info('Using local backtesting data ...')
for pair in pairs:
hist = load_pair_history(pair=pair, ticker_interval=ticker_interval,
datadir=datadir, timerange=timerange,
refresh_pairs=refresh_pairs,
exchange=exchange,
fill_up_missing=fill_up_missing)
if hist is not None:
result[pair] = hist
for pair in pairs:
hist = load_pair_history(pair=pair, ticker_interval=ticker_interval,
datadir=datadir, timerange=timerange,
refresh_pairs=refresh_pairs,
exchange=exchange,
fill_up_missing=fill_up_missing)
if hist is not None:
result[pair] = hist
return result

2
freqtrade/optimize/backtesting.py
View File

@ -414,8 +414,6 @@ class Backtesting:
datadir=Path(self.config['datadir']),
pairs=pairs,
ticker_interval=self.ticker_interval,
refresh_pairs=self.config.get('refresh_pairs', False),
exchange=self.exchange,
timerange=timerange,
)

3
freqtrade/optimize/edge_cli.py
View File

@ -39,7 +39,8 @@ class EdgeCli:
self.strategy = StrategyResolver(self.config).strategy
self.edge = Edge(config, self.exchange, self.strategy)
self.edge._refresh_pairs = self.config.get('refresh_pairs', False)
# Set refresh_pairs to false for edge-cli (it must be true for edge)
self.edge._refresh_pairs = False
self.timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))

2
freqtrade/optimize/hyperopt.py
View File

@ -362,8 +362,6 @@ class Hyperopt:
datadir=Path(self.config['datadir']),
pairs=self.config['exchange']['pair_whitelist'],
ticker_interval=self.backtesting.ticker_interval,
refresh_pairs=self.config.get('refresh_pairs', False),
exchange=self.backtesting.exchange,
timerange=timerange
)

1
tests/data/test_converter.py
View File

@ -24,7 +24,6 @@ def test_parse_ticker_dataframe(ticker_history_list, caplog):
def test_ohlcv_fill_up_missing_data(testdatadir, caplog):
data = load_pair_history(datadir=testdatadir,
ticker_interval='1m',
refresh_pairs=False,
pair='UNITTEST/BTC',
fill_up_missing=False)
caplog.set_level(logging.DEBUG)

1
tests/data/test_dataprovider.py
View File

@ -45,7 +45,6 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history):
data = dp.historic_ohlcv("UNITTEST/BTC", "5m")
assert isinstance(data, DataFrame)
assert historymock.call_count == 1
assert historymock.call_args_list[0][1]["refresh_pairs"] is False
assert historymock.call_args_list[0][1]["ticker_interval"] == "5m"

35
tests/data/test_history.py
View File

@ -74,8 +74,7 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf, testdatadir) -> Non
assert ld is None
assert log_has(
'No history data for pair: "UNITTEST/BTC", interval: 7m. '
'Use --refresh-pairs-cached option or `freqtrade download-data` '
'script to download the data', caplog
'Use `freqtrade download-data` to download the data', caplog
)
@ -105,13 +104,11 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog,
# do not download a new pair if refresh_pairs isn't set
history.load_pair_history(datadir=testdatadir,
ticker_interval='1m',
refresh_pairs=False,
pair='MEME/BTC')
assert os.path.isfile(file) is False
assert log_has(
'No history data for pair: "MEME/BTC", interval: 1m. '
'Use --refresh-pairs-cached option or `freqtrade download-data` '
'script to download the data', caplog
'Use `freqtrade download-data` to download the data', caplog
)
# download a new pair if refresh_pairs is set
@ -134,31 +131,6 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog,
_clean_test_file(file)
def test_load_data_live(default_conf, mocker, caplog, testdatadir) -> None:
refresh_mock = MagicMock()
mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock)
exchange = get_patched_exchange(mocker, default_conf)
history.load_data(datadir=testdatadir, ticker_interval='5m',
pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'],
live=True,
exchange=exchange)
assert refresh_mock.call_count == 1
assert len(refresh_mock.call_args_list[0][0][0]) == 2
assert log_has('Live: Downloading data for all defined pairs ...', caplog)
def test_load_data_live_noexchange(default_conf, mocker, caplog, testdatadir) -> None:
with pytest.raises(OperationalException,
match=r'Exchange needs to be initialized when using live data.'):
history.load_data(datadir=testdatadir, ticker_interval='5m',
pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'],
exchange=None,
live=True,
)
def test_testdata_path(testdatadir) -> None:
assert str(Path('tests') / 'testdata') in str(testdatadir)
@ -349,7 +321,6 @@ def test_load_partial_missing(testdatadir, caplog) -> None:
start = arrow.get('2018-01-01T00:00:00')
end = arrow.get('2018-01-11T00:00:00')
tickerdata = history.load_data(testdatadir, '5m', ['UNITTEST/BTC'],
refresh_pairs=False,
timerange=TimeRange('date', 'date',
start.timestamp, end.timestamp))
# timedifference in 5 minutes
@ -364,7 +335,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None:
start = arrow.get('2018-01-10T00:00:00')
end = arrow.get('2018-02-20T00:00:00')
tickerdata = history.load_data(datadir=testdatadir, ticker_interval='5m',
pairs=['UNITTEST/BTC'], refresh_pairs=False,
pairs=['UNITTEST/BTC'],
timerange=TimeRange('date', 'date',
start.timestamp, end.timestamp))
# timedifference in 5 minutes

8
tests/optimize/test_backtesting.py
View File

@ -188,16 +188,12 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
assert 'position_stacking' not in config
assert not log_has('Parameter --enable-position-stacking detected ...', caplog)
assert 'refresh_pairs' not in config
assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog)
assert 'timerange' not in config
assert 'export' not in config
assert 'runmode' in config
assert config['runmode'] == RunMode.BACKTEST
@pytest.mark.filterwarnings("ignore:DEPRECATED")
def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> None:
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch(
@ -213,7 +209,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
'--ticker-interval', '1m',
'--enable-position-stacking',
'--disable-max-market-positions',
'--refresh-pairs-cached',
'--timerange', ':100',
'--export', '/bar/foo',
'--export-filename', 'foo_bar.json'
@ -240,9 +235,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
assert log_has('Parameter --disable-max-market-positions detected ...', caplog)
assert log_has('max_open_trades set to unlimited ...', caplog)
assert 'refresh_pairs' in config
assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog)
assert 'timerange' in config
assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog)

9
tests/optimize/test_edge_cli.py
View File

@ -3,8 +3,6 @@
from unittest.mock import MagicMock
import pytest
from freqtrade.edge import PairInfo
from freqtrade.optimize import setup_configuration, start_edge
from freqtrade.optimize.edge_cli import EdgeCli
@ -35,14 +33,10 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
assert 'ticker_interval' in config
assert not log_has_re('Parameter -i/--ticker-interval detected .*', caplog)
assert 'refresh_pairs' not in config
assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog)
assert 'timerange' not in config
assert 'stoploss_range' not in config
@pytest.mark.filterwarnings("ignore:DEPRECATED")
def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> None:
patched_configuration_load_config_file(mocker, edge_conf)
mocker.patch(
@ -56,7 +50,6 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N
'--datadir', '/foo/bar',
'edge',
'--ticker-interval', '1m',
'--refresh-pairs-cached',
'--timerange', ':100',
'--stoplosses=-0.01,-0.10,-0.001'
]
@ -74,8 +67,6 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N
assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
caplog)
assert 'refresh_pairs' in config
assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog)
assert 'timerange' in config
assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog)

8
tests/optimize/test_hyperopt.py
View File

@ -86,15 +86,11 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca
assert 'position_stacking' not in config
assert not log_has('Parameter --enable-position-stacking detected ...', caplog)
assert 'refresh_pairs' not in config
assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog)
assert 'timerange' not in config
assert 'runmode' in config
assert config['runmode'] == RunMode.HYPEROPT
@pytest.mark.filterwarnings("ignore:DEPRECATED")
def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplog) -> None:
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch(
@ -108,7 +104,6 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo
'hyperopt',
'--ticker-interval', '1m',
'--timerange', ':100',
'--refresh-pairs-cached',
'--enable-position-stacking',
'--disable-max-market-positions',
'--epochs', '1000',
@ -137,9 +132,6 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo
assert log_has('Parameter --disable-max-market-positions detected ...', caplog)
assert log_has('max_open_trades set to unlimited ...', caplog)
assert 'refresh_pairs' in config
assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog)
assert 'timerange' in config
assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog)

2
tests/test_arguments.py
View File

@ -99,7 +99,6 @@ def test_parse_args_backtesting_custom() -> None:
'-c', 'test_conf.json',
'backtesting',
'--ticker-interval', '1m',
'--refresh-pairs-cached',
'--strategy-list',
'DefaultStrategy',
'SampleStrategy'
@ -110,7 +109,6 @@ def test_parse_args_backtesting_custom() -> None:
assert call_args["subparser"] == 'backtesting'
assert call_args["func"] is not None
assert call_args["ticker_interval"] == '1m'
assert call_args["refresh_pairs"] is True
assert type(call_args["strategy_list"]) is list
assert len(call_args["strategy_list"]) == 2

7
tests/test_configuration.py
View File

@ -358,14 +358,10 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
assert 'position_stacking' not in config
assert not log_has('Parameter --enable-position-stacking detected ...', caplog)
assert 'refresh_pairs' not in config
assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog)
assert 'timerange' not in config
assert 'export' not in config
@pytest.mark.filterwarnings("ignore:DEPRECATED")
def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None:
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch(
@ -385,7 +381,6 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
'--ticker-interval', '1m',
'--enable-position-stacking',
'--disable-max-market-positions',
'--refresh-pairs-cached',
'--timerange', ':100',
'--export', '/bar/foo'
]
@ -415,8 +410,6 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
assert log_has('Parameter --disable-max-market-positions detected ...', caplog)
assert log_has('max_open_trades set to unlimited ...', caplog)
assert 'refresh_pairs'in config
assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog)
assert 'timerange' in config
assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog)