From 9d2b7c1fc0b1209ce9491930e61485ef98d25010 Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Mon, 26 Mar 2018 20:18:14 +0200 Subject: [PATCH 01/13] Add convert script --- scripts/convert_backtestdata.py | 181 ++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100755 scripts/convert_backtestdata.py diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py new file mode 100755 index 000000000..c96579e46 --- /dev/null +++ b/scripts/convert_backtestdata.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +""" +Script to display when the bot will buy a specific pair + +Mandatory Cli parameters: +-p / --pair: pair to examine + +Optional Cli parameters +-d / --datadir: path to pair backtest data +--timerange: specify what timerange of data to use. +-l / --live: Live, to download the latest ticker for the pair +""" + +import sys +from argparse import Namespace +from os import path +import glob +import json +import re +from typing import List, Dict + +from freqtrade.arguments import Arguments +from freqtrade import misc +from freqtrade.logger import Logger +from pandas import DataFrame + +import dateutil.parser + +logger = Logger(name="Convert data").get_logger() + + +def load_old_file(filename) -> List[Dict]: + if not path.isfile(filename): + logger.warning("filename %s does not exist", filename) + return None + logger.debug('Loading ticker data from file %s', filename) + # as in optimize/__init__.py::load_tickerdata_file + # if os.path.isfile(gzipfile): + # logger.debug('Loading ticker data from file %s', gzipfile) + # with gzip.open(gzipfile) as tickerdata: + # pairdata = json.load(tickerdata) + + pairdata = None + with open(filename) as tickerdata: + pairdata = json.load(tickerdata) + return pairdata + + +def parse_old_backtest_data(ticker) -> DataFrame: + """ + Reads old backtest data + Format: "O": 8.794e-05, + "H": 8.948e-05, + "L": 8.794e-05, + "C": 8.88e-05, + "V": 991.09056638, + "T": "2017-11-26T08:50:00", + "BV": 0.0877869 + """ + + columns = {'C': 'close', 'V': 'volume', 'O': 'open', + 'H': 'high', 'L': 'low', 'T': 'date'} + + frame = DataFrame(ticker) \ + .rename(columns=columns) + if 'BV' in frame: + frame.drop('BV', 1, inplace=True) + + frame.sort_values('date', inplace=True) + return frame + + +def convert_dataframe(frame: DataFrame): + """Convert dataframe to new format""" + # reorder columns: + cols = ['date', 'open', 'high', 'low', 'close', 'volume'] + frame = frame[cols] + + frame['date'] = frame['date'].apply( + lambda d: int(dateutil.parser.parse(d).timestamp()) * 1000) + frame['date'] = frame['date'].astype(int) + # Convert columns one by one to preserve type. + by_column = [frame[x].values.tolist() for x in frame.columns] + return list(list(x) for x in zip(*by_column)) + + + +def convert_file(filename: str, filename_new: str): + """Converts a file following the old format to the new format""" + pairdata = load_old_file(filename) + if pairdata and type(pairdata) is list and len(pairdata) > 0: + if type(pairdata[0]) is list: + logger.error("pairdata for %s already in new format", filename) + return 1 + + frame = parse_old_backtest_data(pairdata) + # Convert frame to new format + frame1 = convert_dataframe(frame) + + misc.file_dump_json(filename_new, frame1) + + +def convert_main(args: Namespace) -> None: + """ + converts a folder given in --datadir from old to new format to support ccxt + """ + + workdir = args.datadir if args.datadir.endswith("/") else args.datadir + "/" + print(workdir) + + for filename in glob.glob(workdir + "*.json"): + # swap currency names + ret = re.search(r'[A-Z_]{7,}', path.basename(filename)) + if args.norename: + filename_new = filename + else: + if not ret: + logger.warning("file %s could not be converted, could not extract currencies", + filename) + continue + pair = ret.group(0) + currencies = pair.split("_") + if len(currencies) != 2: + logger.warning("file %s could not be converted, could not extract currencies", + filename) + continue + + ret = re.search(r'\d+(?=\.json)', path.basename(filename)) + if not ret: + logger.warning("file %s could not be converted, interval not found", filename) + continue + interval = ret.group(0) + # filename = "user_data/data/BTC_ADA-5.json" + # filename_new = "user_data/data/ADA_BTC-5.json" + filename_new = "{}/{}_{}-{}.json".format(path.dirname(filename), + currencies[1], currencies[0], interval) + logger.debug("Converting and renaming %s to %s", filename, filename_new) + convert_file(filename, filename_new) + + +def convert_parse_args(args: List[str]) -> Namespace: + """ + Parse args passed to the script + :param args: Cli arguments + :return: args: Array with all arguments + """ + arguments = Arguments(args, 'Convert datafiles') + arguments.parser.add_argument( + '-d', '--datadir', + help='path to backtest data (default: %(default)s', + dest='datadir', + default=path.join('freqtrade', 'tests', 'testdata'), + type=str, + metavar='PATH', + ) + arguments.parser.add_argument( + '-n', '--norename', + help='don''t rename files from BTC_ to _BTC - ' + 'Note that not renaming will overwrite source files', + dest='norename', + default=False, + action='store_true' + ) + + return arguments.parse_args() + + +def main(sysargv: List[str]) -> None: + """ + This function will initiate the bot and start the trading loop. + :return: None + """ + logger.info('Starting Dataframe conversation') + convert_main(convert_parse_args(sysargv)) + + + + + +if __name__ == '__main__': + main(sys.argv[1:]) From 756bd63e1de1f9039d51493f52e272dce724e0ae Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Mon, 26 Mar 2018 23:16:41 +0200 Subject: [PATCH 02/13] whitespace fix --- scripts/convert_backtestdata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py index c96579e46..051ff24cc 100755 --- a/scripts/convert_backtestdata.py +++ b/scripts/convert_backtestdata.py @@ -84,9 +84,8 @@ def convert_dataframe(frame: DataFrame): return list(list(x) for x in zip(*by_column)) - def convert_file(filename: str, filename_new: str): - """Converts a file following the old format to the new format""" + """Converts a file from old format to ccxt format""" pairdata = load_old_file(filename) if pairdata and type(pairdata) is list and len(pairdata) > 0: if type(pairdata[0]) is list: From 96b2210c0fb1946a16456f084576005d3cefa0e8 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 30 Mar 2018 12:11:06 -0700 Subject: [PATCH 03/13] Change deprecated logger.warn by warning --- freqtrade/exchange/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 908820466..2c4136cab 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -30,11 +30,11 @@ def retrier(f): return f(*args, **kwargs) # TODO dont be a gotta-catch-them-all pokemon collector except Exception as ex: - logger.warn('%s returned exception: "%s"', f, ex) + logger.warning('%s returned exception: "%s"', f, ex) if count > 0: count -= 1 kwargs.update({'count': count}) - logger.warn('retrying %s still for %s times', f, count) + logger.warning('retrying %s still for %s times', f, count) return wrapper(*args, **kwargs) else: raise OperationalException('Giving up retrying: %s', f) From 052404ffbd36d9754d824af3211e3f9247997396 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 30 Mar 2018 13:14:35 -0700 Subject: [PATCH 04/13] Check if the exchange is supported --- freqtrade/configuration.py | 26 ++++++++++++++++++++- freqtrade/exchange/__init__.py | 6 ++--- freqtrade/tests/test_configuration.py | 33 +++++++++++++++++++++++++-- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 1f6cea4e6..10768db55 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -5,10 +5,11 @@ This module contains the configuration class import json from argparse import Namespace from typing import Dict, Any - from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match +import ccxt +from freqtrade import OperationalException from freqtrade.constants import Constants from freqtrade.logger import Logger @@ -100,6 +101,9 @@ class Configuration(object): else: self.logger.info('Dry run is disabled. (--dry_run_db ignored)') + # Check if the exchange set by the user is supported + self.check_exchange() + return config def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: @@ -198,3 +202,23 @@ class Configuration(object): self.config = self.load_config() return self.config + + def check_exchange(self) -> bool: + """ + Check if the exchange name in the config file is supported by Freqtrade + :return: True or raised an exception if the exchange if not supported + """ + exchange = self.config.get('exchange', {}).get('name').lower() + if exchange not in ccxt.exchanges: + + exception_msg = 'Exchange "{}" not supported.\n' \ + 'The following exchanges are supported: {}'\ + .format(exchange, ', '.join(ccxt.exchanges)) + + self.logger.critical(exception_msg) + raise OperationalException( + exception_msg + ) + + self.logger.debug('Exchange "%s" supported', exchange) + return True diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 2c4136cab..20ea4f43e 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -74,16 +74,14 @@ def init(config: dict) -> None: # Find matching class for the given exchange name name = exchange_config['name'] - # TODO add check for a list of supported exchanges - + # Init the exchange if the exchange name passed is supported try: - # exchange_class = Exchanges[name.upper()].value _API = getattr(ccxt, name.lower())({ 'apiKey': exchange_config.get('key'), 'secret': exchange_config.get('secret'), }) logger.info('Using Exchange %s', name.capitalize()) - except KeyError: + except (KeyError, AttributeError): raise OperationalException('Exchange {} is not supported'.format(name)) # we need load api markets diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 002eac722..f43c29b43 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -13,6 +13,7 @@ from jsonschema import ValidationError from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.tests.conftest import log_has +from freqtrade import OperationalException def test_configuration_object() -> None: @@ -28,7 +29,7 @@ def test_configuration_object() -> None: assert hasattr(Configuration, 'get_config') -def test_load_config_invalid_pair(default_conf, mocker) -> None: +def test_load_config_invalid_pair(default_conf) -> None: """ Test the configuration validator with an invalid PAIR format """ @@ -40,7 +41,7 @@ def test_load_config_invalid_pair(default_conf, mocker) -> None: configuration._validate_config(conf) -def test_load_config_missing_attributes(default_conf, mocker) -> None: +def test_load_config_missing_attributes(default_conf) -> None: """ Test the configuration validator with a missing attribute """ @@ -314,3 +315,31 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: assert 'spaces' in config assert config['spaces'] == ['all'] assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples) + + +def test_check_exchange(default_conf) -> None: + """ + Test the configuration validator with a missing attribute + """ + conf = deepcopy(default_conf) + configuration = Configuration([]) + + # Test a valid exchange + conf.get('exchange').update({'name': 'BITTREX'}) + configuration.config = conf + assert configuration.check_exchange() + + # Test a valid exchange + conf.get('exchange').update({'name': 'binance'}) + configuration.config = conf + assert configuration.check_exchange() + + # Test a invalid exchange + conf.get('exchange').update({'name': 'unknown_exchange'}) + configuration.config = conf + + with pytest.raises( + OperationalException, + match=r'.*Exchange "unknown_exchange" not supported.*' + ): + configuration.check_exchange() From 3d2c6a22a377ab970ef53af0b7e5ba7ea5ef948e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 30 Mar 2018 13:31:13 -0700 Subject: [PATCH 05/13] Fix test_validate_pairs() --- freqtrade/tests/exchange/test_exchange.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 6a423cefd..f5f20a265 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -42,9 +42,7 @@ def test_init_exception(default_conf): def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() - api_mock.get_markets = MagicMock(return_value=[ - 'BTC_ETH', 'BTC_TKN', 'BTC_TRST', 'BTC_SWT', 'BTC_BCC', - ]) + api_mock.markets = ["ETH/BTC", "NEO/BTC", "LTC/BTC", "XRP/BTC"] mocker.patch('freqtrade.exchange._API', api_mock) mocker.patch.dict('freqtrade.exchange._CONF', default_conf) validate_pairs(default_conf['exchange']['pair_whitelist']) From 7cafd1f17e89e88b3fe835167796374321f7a945 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 30 Mar 2018 13:52:25 -0700 Subject: [PATCH 06/13] Update exchange unit tests --- freqtrade/exchange/__init__.py | 7 +--- freqtrade/tests/exchange/test_exchange.py | 51 +++++++++++++++-------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 20ea4f43e..088128dab 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -102,12 +102,7 @@ def validate_pairs(pairs: List[str]) -> None: if not _API.markets: _API.load_markets() - try: - markets = _API.markets - except requests.exceptions.RequestException as e: - logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) - return - + markets = _API.markets stake_cur = _CONF['stake_currency'] for pair in pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index f5f20a265..7a4d0483e 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1,11 +1,11 @@ # pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement # pragma pylint: disable=protected-access import logging +from copy import deepcopy from random import randint from unittest.mock import MagicMock import pytest -from requests.exceptions import RequestException import freqtrade.exchange as exchange from freqtrade import OperationalException @@ -60,31 +60,46 @@ def test_validate_pairs_not_available(default_conf, mocker): def test_validate_pairs_not_compatible(default_conf, mocker): api_mock = MagicMock() api_mock.get_markets = MagicMock( - return_value=['BTC_ETH', 'BTC_TKN', 'BTC_TRST', 'BTC_SWT']) - default_conf['stake_currency'] = 'ETH' + return_value=['BTC/ETH', 'BTC/TKN', 'BTC/TRST', 'BTC/SWT']) + conf = deepcopy(default_conf) + conf['stake_currency'] = 'ETH' mocker.patch('freqtrade.exchange._API', api_mock) - mocker.patch.dict('freqtrade.exchange._CONF', default_conf) + mocker.patch.dict('freqtrade.exchange._CONF', conf) with pytest.raises(OperationalException, match=r'not compatible'): - validate_pairs(default_conf['exchange']['pair_whitelist']) + validate_pairs(conf['exchange']['pair_whitelist']) def test_validate_pairs_exception(default_conf, mocker, caplog): caplog.set_level(logging.INFO) api_mock = MagicMock() - api_mock.get_markets = MagicMock(side_effect=RequestException()) + api_mock.name = 'binance' mocker.patch('freqtrade.exchange._API', api_mock) + mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - # with pytest.raises(RequestException, match=r'Unable to validate pairs'): - validate_pairs(default_conf['exchange']['pair_whitelist']) - assert log_has('Unable to validate pairs (assuming they are correct). Reason: ', - caplog.record_tuples) + with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at binance'): + validate_pairs(default_conf['exchange']['pair_whitelist']) +def test_validate_pairs_stake_exception(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + conf = deepcopy(default_conf) + conf['stake_currency'] = 'ETH' + api_mock = MagicMock() + api_mock.name = 'binance' + mocker.patch('freqtrade.exchange._API', api_mock) + mocker.patch.dict('freqtrade.exchange._CONF', conf) + + with pytest.raises( + OperationalException, + match=r'Pair ETH/BTC not compatible with stake_currency: ETH' + ): + validate_pairs(default_conf['exchange']['pair_whitelist']) + def test_buy_dry_run(default_conf, mocker): default_conf['dry_run'] = True mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - assert 'dry_run_buy_' in buy(pair='BTC_ETH', rate=200, amount=1) + assert 'dry_run_buy_' in buy(pair='BTC/ETH', rate=200, amount=1) def test_buy_prod(default_conf, mocker): @@ -96,14 +111,14 @@ def test_buy_prod(default_conf, mocker): default_conf['dry_run'] = False mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - assert 'dry_run_buy_' in buy(pair='BTC_ETH', rate=200, amount=1) + assert 'dry_run_buy_' in buy(pair='BTC/ETH', rate=200, amount=1) def test_sell_dry_run(default_conf, mocker): default_conf['dry_run'] = True mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - assert 'dry_run_sell_' in sell(pair='BTC_ETH', rate=200, amount=1) + assert 'dry_run_sell_' in sell(pair='BTC/ETH', rate=200, amount=1) def test_sell_prod(default_conf, mocker): @@ -115,7 +130,7 @@ def test_sell_prod(default_conf, mocker): default_conf['dry_run'] = False mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - assert 'dry_run_sell_' in sell(pair='BTC_ETH', rate=200, amount=1) + assert 'dry_run_sell_' in sell(pair='BTC/ETH', rate=200, amount=1) def test_get_balance_dry_run(default_conf, mocker): @@ -177,7 +192,7 @@ def test_get_ticker(default_conf, mocker): mocker.patch('freqtrade.exchange.bittrex._API', api_mock) # retrieve original ticker - ticker = get_ticker(pair='BTC_ETH') + ticker = get_ticker(pair='BTC/ETH') assert ticker['bid'] == 0.00001098 assert ticker['ask'] == 0.00001099 @@ -188,12 +203,12 @@ def test_get_ticker(default_conf, mocker): # if not caching the result we should get the same ticker # if not fetching a new result we should get the cached ticker - ticker = get_ticker(pair='BTC_ETH', refresh=False) + ticker = get_ticker(pair='BTC/ETH', refresh=False) assert ticker['bid'] == 0.00001098 assert ticker['ask'] == 0.00001099 # force ticker refresh - ticker = get_ticker(pair='BTC_ETH', refresh=True) + ticker = get_ticker(pair='BTC/ETH', refresh=True) assert ticker['bid'] == 0.5 assert ticker['ask'] == 1 @@ -215,7 +230,7 @@ def test_get_ticker_history(default_conf, mocker): mocker.patch('freqtrade.exchange._API', api_mock) # ensure caching will still return the original ticker - ticks = get_ticker_history('BTC_ETH', int(default_conf['ticker_interval'])) + ticks = get_ticker_history('BTC/ETH', int(default_conf['ticker_interval'])) assert ticks == 123 From a4906c477edf44db9070e731ebb3476c6291ba8e Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Fri, 30 Mar 2018 23:30:23 +0200 Subject: [PATCH 07/13] Add handling for gzip files --- freqtrade/misc.py | 13 +++++++++--- freqtrade/tests/test_misc.py | 5 +++++ scripts/convert_backtestdata.py | 36 ++++++++++++++++----------------- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index bc04d6b88..fac379c18 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -5,6 +5,7 @@ Various tool function for Freqtrade and scripts import json import logging import re +import gzip from datetime import datetime from typing import Dict @@ -63,15 +64,21 @@ def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray: return np.sort(arr, axis=0) -def file_dump_json(filename, data) -> None: +def file_dump_json(filename, data, is_zip=False) -> None: """ Dump JSON data into a file :param filename: file to create :param data: JSON Data to save :return: """ - with open(filename, 'w') as fp: - json.dump(data, fp, default=str) + if not is_zip: + with open(filename, 'w') as fp: + json.dump(data, fp, default=str) + else: + if not filename.endswith('.gz'): + filename = filename + '.gz' + with gzip.open(filename, 'w') as fp: + json.dump(data, fp, default=str) def format_ms_time(date: str) -> str: diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 3560b2db1..91c34b620 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -71,3 +71,8 @@ def test_file_dump_json(mocker) -> None: file_dump_json('somefile', [1, 2, 3]) assert file_open.call_count == 1 assert json_dump.call_count == 1 + file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock()) + json_dump = mocker.patch('json.dump', MagicMock()) + file_dump_json('somefile', [1, 2, 3], True) + assert file_open.call_count == 1 + assert json_dump.call_count == 1 diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py index 051ff24cc..1ec553666 100755 --- a/scripts/convert_backtestdata.py +++ b/scripts/convert_backtestdata.py @@ -18,6 +18,7 @@ import glob import json import re from typing import List, Dict +import gzip from freqtrade.arguments import Arguments from freqtrade import misc @@ -26,24 +27,27 @@ from pandas import DataFrame import dateutil.parser -logger = Logger(name="Convert data").get_logger() +logger = Logger(name="Convert data", level=10).get_logger() -def load_old_file(filename) -> List[Dict]: +def load_old_file(filename) -> (List[Dict], bool): if not path.isfile(filename): logger.warning("filename %s does not exist", filename) - return None + return (None, False) logger.debug('Loading ticker data from file %s', filename) - # as in optimize/__init__.py::load_tickerdata_file - # if os.path.isfile(gzipfile): - # logger.debug('Loading ticker data from file %s', gzipfile) - # with gzip.open(gzipfile) as tickerdata: - # pairdata = json.load(tickerdata) pairdata = None - with open(filename) as tickerdata: - pairdata = json.load(tickerdata) - return pairdata + + if filename.endswith('.gz'): + logger.debug('Loading ticker data from file %s', filename) + is_zip = True + with gzip.open(filename) as tickerdata: + pairdata = json.load(tickerdata) + else: + is_zip = False + with open(filename) as tickerdata: + pairdata = json.load(tickerdata) + return (pairdata, is_zip) def parse_old_backtest_data(ticker) -> DataFrame: @@ -86,7 +90,7 @@ def convert_dataframe(frame: DataFrame): def convert_file(filename: str, filename_new: str): """Converts a file from old format to ccxt format""" - pairdata = load_old_file(filename) + (pairdata, is_zip) = load_old_file(filename) if pairdata and type(pairdata) is list and len(pairdata) > 0: if type(pairdata[0]) is list: logger.error("pairdata for %s already in new format", filename) @@ -95,8 +99,7 @@ def convert_file(filename: str, filename_new: str): frame = parse_old_backtest_data(pairdata) # Convert frame to new format frame1 = convert_dataframe(frame) - - misc.file_dump_json(filename_new, frame1) + misc.file_dump_json(filename_new, frame1, is_zip) def convert_main(args: Namespace) -> None: @@ -105,7 +108,7 @@ def convert_main(args: Namespace) -> None: """ workdir = args.datadir if args.datadir.endswith("/") else args.datadir + "/" - print(workdir) + logger.info("Workdir: %s", workdir) for filename in glob.glob(workdir + "*.json"): # swap currency names @@ -173,8 +176,5 @@ def main(sysargv: List[str]) -> None: convert_main(convert_parse_args(sysargv)) - - - if __name__ == '__main__': main(sys.argv[1:]) From a972b8768d9ab55a5cd451113d60485a2387bc3f Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Fri, 30 Mar 2018 23:34:22 +0200 Subject: [PATCH 08/13] Improve errorhandling for json files which are not ticker data --- scripts/convert_backtestdata.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py index 1ec553666..42bc0b4ad 100755 --- a/scripts/convert_backtestdata.py +++ b/scripts/convert_backtestdata.py @@ -69,7 +69,9 @@ def parse_old_backtest_data(ticker) -> DataFrame: .rename(columns=columns) if 'BV' in frame: frame.drop('BV', 1, inplace=True) - + if not 'date' in frame: + logger.warning("Date not in frame - probably not a Ticker file") + return None frame.sort_values('date', inplace=True) return frame @@ -98,8 +100,9 @@ def convert_file(filename: str, filename_new: str): frame = parse_old_backtest_data(pairdata) # Convert frame to new format - frame1 = convert_dataframe(frame) - misc.file_dump_json(filename_new, frame1, is_zip) + if frame is not None: + frame1 = convert_dataframe(frame) + misc.file_dump_json(filename_new, frame1, is_zip) def convert_main(args: Namespace) -> None: From 8a83e050d0b23ac0347440e12089e4a6d6e71afa Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 31 Mar 2018 17:24:25 +0200 Subject: [PATCH 09/13] use path to handle filenames --- scripts/convert_backtestdata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py index 42bc0b4ad..3c6c47f0d 100755 --- a/scripts/convert_backtestdata.py +++ b/scripts/convert_backtestdata.py @@ -110,7 +110,7 @@ def convert_main(args: Namespace) -> None: converts a folder given in --datadir from old to new format to support ccxt """ - workdir = args.datadir if args.datadir.endswith("/") else args.datadir + "/" + workdir = path.join(args.datadir, "") logger.info("Workdir: %s", workdir) for filename in glob.glob(workdir + "*.json"): @@ -135,10 +135,10 @@ def convert_main(args: Namespace) -> None: logger.warning("file %s could not be converted, interval not found", filename) continue interval = ret.group(0) - # filename = "user_data/data/BTC_ADA-5.json" - # filename_new = "user_data/data/ADA_BTC-5.json" - filename_new = "{}/{}_{}-{}.json".format(path.dirname(filename), - currencies[1], currencies[0], interval) + + filename_new = path.join(path.dirname(filename), + "{}_{}-{}.json".format(currencies[1], + currencies[0], interval)) logger.debug("Converting and renaming %s to %s", filename, filename_new) convert_file(filename, filename_new) From 2f40e23dcc870dbbe309f606c731bd88694b5d56 Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 31 Mar 2018 17:28:54 +0200 Subject: [PATCH 10/13] don't check negated if both trees are handled --- freqtrade/misc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index fac379c18..7546dba8f 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -71,14 +71,14 @@ def file_dump_json(filename, data, is_zip=False) -> None: :param data: JSON Data to save :return: """ - if not is_zip: - with open(filename, 'w') as fp: - json.dump(data, fp, default=str) - else: + if is_zip: if not filename.endswith('.gz'): filename = filename + '.gz' with gzip.open(filename, 'w') as fp: json.dump(data, fp, default=str) + else: + with open(filename, 'w') as fp: + json.dump(data, fp, default=str) def format_ms_time(date: str) -> str: From 18f8686cdbf8dd3a3abfb2b81d1512478c3ee598 Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 31 Mar 2018 17:29:52 +0200 Subject: [PATCH 11/13] fix returncode for convert_file --- scripts/convert_backtestdata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py index 3c6c47f0d..024b3c9af 100755 --- a/scripts/convert_backtestdata.py +++ b/scripts/convert_backtestdata.py @@ -90,13 +90,13 @@ def convert_dataframe(frame: DataFrame): return list(list(x) for x in zip(*by_column)) -def convert_file(filename: str, filename_new: str): +def convert_file(filename: str, filename_new: str) -> None: """Converts a file from old format to ccxt format""" (pairdata, is_zip) = load_old_file(filename) - if pairdata and type(pairdata) is list and len(pairdata) > 0: + if pairdata and type(pairdata) is list: if type(pairdata[0]) is list: logger.error("pairdata for %s already in new format", filename) - return 1 + return frame = parse_old_backtest_data(pairdata) # Convert frame to new format From 4ac591b0763325c5b06004b40c7acc19f4abd45b Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sat, 31 Mar 2018 17:30:11 +0200 Subject: [PATCH 12/13] rename logging to freqtrade --- scripts/convert_backtestdata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py index 024b3c9af..a6f6ccc60 100755 --- a/scripts/convert_backtestdata.py +++ b/scripts/convert_backtestdata.py @@ -27,7 +27,7 @@ from pandas import DataFrame import dateutil.parser -logger = Logger(name="Convert data", level=10).get_logger() +logger = Logger(name="freqtrade").get_logger() def load_old_file(filename) -> (List[Dict], bool): From 0203a48f3e02c691628c00d827857d57cac54d7b Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Wed, 4 Apr 2018 22:05:17 +0200 Subject: [PATCH 13/13] use local config-object for check_exchange fix AttributeError: 'NoneType' object has no attribute 'get' when starting the bot. --- freqtrade/configuration.py | 6 +++--- freqtrade/tests/test_configuration.py | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 10768db55..6df73e6a0 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -102,7 +102,7 @@ class Configuration(object): self.logger.info('Dry run is disabled. (--dry_run_db ignored)') # Check if the exchange set by the user is supported - self.check_exchange() + self.check_exchange(config) return config @@ -203,12 +203,12 @@ class Configuration(object): return self.config - def check_exchange(self) -> bool: + def check_exchange(self, config: Dict[str, Any]) -> bool: """ Check if the exchange name in the config file is supported by Freqtrade :return: True or raised an exception if the exchange if not supported """ - exchange = self.config.get('exchange', {}).get('name').lower() + exchange = config.get('exchange', {}).get('name').lower() if exchange not in ccxt.exchanges: exception_msg = 'Exchange "{}" not supported.\n' \ diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index f43c29b43..aee329d6b 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -326,13 +326,11 @@ def test_check_exchange(default_conf) -> None: # Test a valid exchange conf.get('exchange').update({'name': 'BITTREX'}) - configuration.config = conf - assert configuration.check_exchange() + assert configuration.check_exchange(conf) # Test a valid exchange conf.get('exchange').update({'name': 'binance'}) - configuration.config = conf - assert configuration.check_exchange() + assert configuration.check_exchange(conf) # Test a invalid exchange conf.get('exchange').update({'name': 'unknown_exchange'}) @@ -342,4 +340,4 @@ def test_check_exchange(default_conf) -> None: OperationalException, match=r'.*Exchange "unknown_exchange" not supported.*' ): - configuration.check_exchange() + configuration.check_exchange(conf)