diff --git a/.gitignore b/.gitignore index 7c7102874..8c7a6e257 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ config.json logfile.txt hyperopt_trials.pickle user_data/ +freqtrade-plot.html # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/docs/backtesting.md b/docs/backtesting.md index 8b12b04e8..a704f1cb9 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -51,6 +51,12 @@ python3 ./freqtrade/main.py backtesting --realistic-simulation --live python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101 ``` +**With a (custom) strategy file** +```bash +python3.6 ./freqtrade/main.py -s currentstrategy backtesting +``` +Where `-s currentstrategy` refers to a filename `currentstrategy.py` in `freqtrade/user_data/strategies` + **Exporting trades to file** ```bash freqtrade backtesting --export trades diff --git a/docs/installation.md b/docs/installation.md index c48f170c6..f2a32e8b2 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -297,6 +297,7 @@ cp config.json.example config.json ```bash python3.6 -m venv .env source .env/bin/activate +pip3.6 install --upgrade pip pip3.6 install -r requirements.txt pip3.6 install -e . ``` diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 904723b30..7de9809ec 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -15,7 +15,7 @@ official commands. You can ask at any moment for help with `/help`. | Command | Default | Description | |----------|---------|-------------| | `/start` | | Starts the trader -| `/stop` | | Starts the trader +| `/stop` | | Stops the trader | `/status` | | Lists all open trades | `/status table` | | List all open trades in a table format | `/count` | | Displays number of trades used and available @@ -126,4 +126,4 @@ Day Profit BTC Profit USD ``` ## /version -> **Version:** `0.14.3` \ No newline at end of file +> **Version:** `0.14.3` diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index fe92c1ba2..70bf40936 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -112,7 +112,7 @@ def get_signal(pair: str, interval: int) -> (bool, bool): # Check if dataframe is out of date signal_date = arrow.get(latest['date']) - if signal_date < arrow.now() - timedelta(minutes=10): + if signal_date < arrow.now() - timedelta(minutes=(interval + 5)): logger.warning('Too old dataframe for pair %s', pair) return (False, False) # return False ? diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 7a1631737..cf2db1004 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -60,7 +60,7 @@ def common_datearray(dfs): return np.sort(arr, axis=0) -def file_dump_json(filename, data): +def file_dump_json(filename, data) -> None: with open(filename, 'w') as fp: json.dump(data, fp) @@ -287,27 +287,27 @@ def hyperopt_options(parser: argparse.ArgumentParser) -> None: def parse_timerange(text): if text is None: return None - syntax = [('^-(\d{8})$', (None, 'date')), - ('^(\d{8})-$', ('date', None)), - ('^(\d{8})-(\d{8})$', ('date', 'date')), - ('^(-\d+)$', (None, 'line')), - ('^(\d+)-$', ('line', None)), - ('^(\d+)-(\d+)$', ('index', 'index'))] + syntax = [(r'^-(\d{8})$', (None, 'date')), + (r'^(\d{8})-$', ('date', None)), + (r'^(\d{8})-(\d{8})$', ('date', 'date')), + (r'^(-\d+)$', (None, 'line')), + (r'^(\d+)-$', ('line', None)), + (r'^(\d+)-(\d+)$', ('index', 'index'))] for rex, stype in syntax: # Apply the regular expression to text - m = re.match(rex, text) - if m: # Regex has matched - rvals = m.groups() - n = 0 + match = re.match(rex, text) + if match: # Regex has matched + rvals = match.groups() + index = 0 start = None stop = None if stype[0]: - start = rvals[n] + start = rvals[index] if stype[0] != 'date': start = int(start) - n += 1 + index += 1 if stype[1]: - stop = rvals[n] + stop = rvals[index] if stype[1] != 'date': stop = int(stop) return (stype, start, stop) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index a0eb5b8a2..52ea55853 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -10,6 +10,7 @@ from freqtrade.analyze import populate_indicators, parse_ticker_dataframe from freqtrade import misc from user_data.hyperopt_conf import hyperopt_optimize_conf +import gzip logger = logging.getLogger(__name__) @@ -26,8 +27,7 @@ def trim_tickerlist(tickerlist, timerange): return tickerlist -def load_tickerdata_file(datadir, pair, ticker_interval, - timerange=None): +def load_tickerdata_file(datadir, pair, ticker_interval, timerange=None): """ Load a pair from file, :return dict OR empty if unsuccesful @@ -38,13 +38,19 @@ def load_tickerdata_file(datadir, pair, ticker_interval, pair=pair, ticker_interval=ticker_interval, ) - # The file does not exist we download it - if not os.path.isfile(file): + gzipfile = file + '.gz' + + # If the file does not exist we download it when None is returned. + # If file exists, read the file, load the json + if os.path.isfile(gzipfile): + with gzip.open(gzipfile) as tickerdata: + pairdata = json.load(tickerdata) + elif os.path.isfile(file): + with open(file) as tickerdata: + pairdata = json.load(tickerdata) + else: return None - # Read the file, load the json - with open(file) as tickerdata: - pairdata = json.load(tickerdata) if timerange: pairdata = trim_tickerlist(pairdata, timerange) return pairdata diff --git a/freqtrade/rpc/__init__.py b/freqtrade/rpc/__init__.py index 7c331c552..0eb23cd9f 100644 --- a/freqtrade/rpc/__init__.py +++ b/freqtrade/rpc/__init__.py @@ -322,7 +322,10 @@ def rpc_balance(fiat_display_currency): if coin == 'BTC': currency["Rate"] = 1.0 else: - currency["Rate"] = exchange.get_ticker('BTC_' + coin, False)['bid'] + if coin == 'USDT': + currency["Rate"] = 1.0 / exchange.get_ticker('USDT_BTC', False)['bid'] + else: + currency["Rate"] = exchange.get_ticker('BTC_' + coin, False)['bid'] currency['BTC'] = currency["Rate"] * currency["Balance"] total = total + currency['BTC'] output.append({'currency': currency['Currency'], diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 1b39c96ff..bdb5f7ed1 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -244,7 +244,6 @@ def _profit(bot: Bot, update: Update) -> None: def _balance(bot: Bot, update: Update) -> None: """ Handler for /balance - Returns current account balance per crypto """ (error, result) = rpc_balance(_CONF['fiat_display_currency']) if error: @@ -259,7 +258,6 @@ def _balance(bot: Bot, update: Update) -> None: *Balance*: {balance} *Pending*: {pending} *Est. BTC*: {est_btc: .8f} - """.format(**currency) output += """*Estimated Value*: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9281e72ca..dc9f33244 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -17,13 +17,6 @@ class IStrategy(ABC): stoploss -> float: optimal stoploss designed for the strategy ticker_interval -> int: value of the ticker interval to use for the strategy """ - @property - def name(self) -> str: - """ - Name of the strategy. - :return: str representation of the class name - """ - return self.__class__.__name__ @abstractmethod def populate_indicators(self, dataframe: DataFrame) -> DataFrame: diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py index 97e260ee8..427c24fc6 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/strategy.py @@ -1,3 +1,5 @@ +# pragma pylint: disable=attribute-defined-outside-init + """ This module load custom strategies """ @@ -21,7 +23,7 @@ class Strategy(object): DEFAULT_STRATEGY = 'default_strategy' - def __new__(cls): + def __new__(cls) -> object: """ Used to create the Singleton :return: Strategy object @@ -30,15 +32,7 @@ class Strategy(object): Strategy.__instance = object.__new__(cls) return Strategy.__instance - def __init__(self): - if Strategy.__instance is None: - self.logger = None - self.minimal_roi = None - self.stoploss = None - self.ticker_interval = None - self.custom_strategy = None - - def init(self, config): + def init(self, config: dict) -> None: """ Load the custom class from config parameter :param config: diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 646959669..e968084bc 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -1,12 +1,14 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import os +import json import logging +import uuid from shutil import copyfile from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ - download_backtesting_testdata, load_tickerdata_file + download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, file_dump_json # Change this if modifying BTC_UNITEST testdatafile _BTC_UNITTEST_LENGTH = 13681 @@ -202,9 +204,14 @@ def test_download_backtesting_testdata2(mocker): def test_load_tickerdata_file(): + # 7 does not exist in either format. assert not load_tickerdata_file(None, 'BTC_UNITEST', 7) + # 1 exists only as a .json tickerdata = load_tickerdata_file(None, 'BTC_UNITEST', 1) assert _BTC_UNITTEST_LENGTH == len(tickerdata) + # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json + tickerdata = load_tickerdata_file(None, 'BTC_UNITEST', 8) + assert _BTC_UNITTEST_LENGTH == len(tickerdata) def test_init(default_conf, mocker): @@ -220,3 +227,73 @@ def test_tickerdata_to_dataframe(): tickerlist = {'BTC_UNITEST': tick} data = optimize.tickerdata_to_dataframe(tickerlist) assert len(data['BTC_UNITEST']) == 100 + + +def test_trim_tickerlist(): + with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: + ticker_list = json.load(data_file) + ticker_list_len = len(ticker_list) + + # Test the pattern ^(-\d+)$ + # This pattern remove X element from the beginning + timerange = ((None, 'line'), None, 5) + ticker = trim_tickerlist(ticker_list, timerange) + ticker_len = len(ticker) + + assert ticker_list_len == ticker_len + 5 + assert ticker_list[0] is not ticker[0] # The first element should be different + assert ticker_list[-1] is ticker[-1] # The last element must be the same + + # Test the pattern ^(\d+)-$ + # This pattern keep X element from the end + timerange = (('line', None), 5, None) + ticker = trim_tickerlist(ticker_list, timerange) + ticker_len = len(ticker) + + assert ticker_len == 5 + assert ticker_list[0] is ticker[0] # The first element must be the same + assert ticker_list[-1] is not ticker[-1] # The last element should be different + + # Test the pattern ^(\d+)-(\d+)$ + # This pattern extract a window + timerange = (('index', 'index'), 5, 10) + ticker = trim_tickerlist(ticker_list, timerange) + ticker_len = len(ticker) + + assert ticker_len == 5 + assert ticker_list[0] is not ticker[0] # The first element should be different + assert ticker_list[5] is ticker[0] # The list starts at the index 5 + assert ticker_list[9] is ticker[-1] # The list ends at the index 9 (5 elements) + + # Test a wrong pattern + # This pattern must return the list unchanged + timerange = ((None, None), None, 5) + ticker = trim_tickerlist(ticker_list, timerange) + ticker_len = len(ticker) + + assert ticker_list_len == ticker_len + + +def test_file_dump_json(): + + file = 'freqtrade/tests/testdata/test_{id}.json'.format(id=str(uuid.uuid4())) + data = {'bar': 'foo'} + + # check the file we will create does not exist + assert os.path.isfile(file) is False + + # Create the Json file + file_dump_json(file, data) + + # Check the file was create + assert os.path.isfile(file) is True + + # Open the Json file created and test the data is in it + with open(file) as data_file: + json_from_file = json.load(data_file) + + assert 'bar' in json_from_file + assert json_from_file['bar'] == 'foo' + + # Remove the file + _clean_test_file(file) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 35c59f4d7..fe2f02a10 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -381,8 +381,7 @@ def test_performance_handle( assert 'BTC_ETH\t6.20% (1)' in msg_mock.call_args_list[0][0][0] -def test_daily_handle( - default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): +def test_daily_handle(default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) msg_mock = MagicMock() @@ -445,6 +444,25 @@ def test_daily_handle( assert str(' 2.798 USD') in msg_mock.call_args_list[0][0][0] assert str(' 3 trades') in msg_mock.call_args_list[0][0][0] + +def test_daily_wrong_input(default_conf, update, ticker, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) + msg_mock = MagicMock() + mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) + mocker.patch.multiple('freqtrade.rpc.telegram', + _CONF=default_conf, + init=MagicMock(), + send_msg=msg_mock) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker) + mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', + ticker=MagicMock(return_value={'price_usd': 15000.0}), + _cache_symbols=MagicMock(return_value={'BTC': 1})) + mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + init(default_conf, create_engine('sqlite://')) + # Try invalid data msg_mock.reset_mock() update_state(State.RUNNING) @@ -453,6 +471,13 @@ def test_daily_handle( assert msg_mock.call_count == 1 assert 'must be an integer greater than 0' in msg_mock.call_args_list[0][0][0] + # Try invalid data + msg_mock.reset_mock() + update_state(State.RUNNING) + update.message.text = '/daily today' + _daily(bot=MagicMock(), update=update) + assert str('Daily Profit over the last 7 days') in msg_mock.call_args_list[0][0][0] + def test_count_handle(default_conf, update, ticker, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -593,7 +618,34 @@ def test_telegram_balance_handle(default_conf, update, mocker): 'Available': 0.0, 'Pending': 0.0, 'CryptoAddress': 'XXXX', + }, { + 'Currency': 'USDT', + 'Balance': 10000.0, + 'Available': 0.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', + }, { + 'Currency': 'LTC', + 'Balance': 10.0, + 'Available': 10.0, + 'Pending': 0.0, + 'CryptoAddress': 'XXXX', }] + + def mock_ticker(symbol, refresh): + if symbol == 'USDT_BTC': + return { + 'bid': 10000.00, + 'ask': 10000.00, + 'last': 10000.00, + } + else: + return { + 'bid': 0.1, + 'ask': 0.1, + 'last': 0.1, + } + mocker.patch.dict('freqtrade.main._CONF', default_conf) msg_mock = MagicMock() mocker.patch.multiple('freqtrade.rpc.telegram', @@ -605,12 +657,32 @@ def test_telegram_balance_handle(default_conf, update, mocker): mocker.patch.multiple('freqtrade.fiat_convert.Pymarketcap', ticker=MagicMock(return_value={'price_usd': 15000.0}), _cache_symbols=MagicMock(return_value={'BTC': 1})) + mocker.patch('freqtrade.main.exchange.get_ticker', side_effect=mock_ticker) _balance(bot=MagicMock(), update=update) + result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 - assert '*Currency*: BTC' in msg_mock.call_args_list[0][0][0] - assert 'Balance' in msg_mock.call_args_list[0][0][0] - assert 'Est. BTC' in msg_mock.call_args_list[0][0][0] + assert '*Currency*: BTC' in result + assert '*Currency*: ETH' not in result + assert '*Currency*: USDT' in result + assert 'Balance' in result + assert 'Est. BTC' in result + assert '*BTC*: 12.00000000' in result + + +def test_zero_balance_handle(default_conf, update, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + msg_mock = MagicMock() + mocker.patch.multiple('freqtrade.rpc.telegram', + _CONF=default_conf, + init=MagicMock(), + send_msg=msg_mock) + mocker.patch.multiple('freqtrade.main.exchange', + get_balances=MagicMock(return_value=[])) + _balance(bot=MagicMock(), update=update) + result = msg_mock.call_args_list[0][0][0] + assert msg_mock.call_count == 1 + assert '`All balances are zero.`' in result def test_help_handle(default_conf, update, mocker): @@ -666,3 +738,18 @@ def test_send_msg_network_error(default_conf, mocker): # Bot should've tried to send it twice assert len(bot.method_calls) == 2 + + +def test_init(default_conf, update, ticker, limit_buy_order, limit_sell_order, mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) + msg_mock = MagicMock() + mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) + mocker.patch.multiple('freqtrade.rpc.telegram', + _CONF=default_conf, + init=MagicMock(), + send_msg=msg_mock) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker) + init(default_conf, create_engine('sqlite://')) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 8a2a21f41..dd772f784 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -16,6 +16,7 @@ def test_sanitize_module_name(): def test_search_strategy(): assert Strategy._search_strategy('default_strategy') == '.' + assert Strategy._search_strategy('test_strategy') == 'user_data.strategies.' assert Strategy._search_strategy('super_duper') is None diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 96743dfab..c0d4db7a1 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -5,11 +5,12 @@ import time from copy import deepcopy from unittest.mock import MagicMock +import datetime import pytest from jsonschema import ValidationError - +from freqtrade.analyze import parse_ticker_dataframe from freqtrade.misc import (common_args_parser, file_dump_json, load_config, - parse_args, parse_timerange, throttle) + parse_args, parse_timerange, throttle, datesarray_to_datetimearray) def test_throttle(): @@ -178,3 +179,18 @@ def test_load_config_missing_attributes(default_conf, mocker): read_data=json.dumps(conf))) with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): load_config('somefile') + + +def test_datesarray_to_datetimearray(ticker_history): + dataframes = parse_ticker_dataframe(ticker_history) + dates = datesarray_to_datetimearray(dataframes['date']) + + assert isinstance(dates[0], datetime.datetime) + assert dates[0].year == 2017 + assert dates[0].month == 11 + assert dates[0].day == 26 + assert dates[0].hour == 8 + assert dates[0].minute == 50 + + date_len = len(dates) + assert date_len == 3 diff --git a/freqtrade/tests/testdata/BTC_UNITEST-8.json b/freqtrade/tests/testdata/BTC_UNITEST-8.json new file mode 100644 index 000000000..e4766ad4f --- /dev/null +++ b/freqtrade/tests/testdata/BTC_UNITEST-8.json @@ -0,0 +1,3 @@ +[ + {"O": 0.00162008, "H": 0.00162008, "L": 0.00162008, "C": 0.00162008, "V": 108.14853839, "T": "2017-11-04T23:02:00", "BV": 0.17520927} +] \ No newline at end of file diff --git a/freqtrade/tests/testdata/BTC_UNITEST-8.json.gz b/freqtrade/tests/testdata/BTC_UNITEST-8.json.gz new file mode 100644 index 000000000..38194582c Binary files /dev/null and b/freqtrade/tests/testdata/BTC_UNITEST-8.json.gz differ