From a98fcee4f950448495f160324738e929705929bf Mon Sep 17 00:00:00 2001 From: Pan Long Date: Fri, 25 May 2018 22:29:06 +0800 Subject: [PATCH 01/17] Sell filled amount or an open limit buy order in forcesell. Currently forcesell only cancels an open limit buy order and doesn't sell the filled amount. After this change, forcesell will also update trade's amount to filled amount and sell the filled amount. --- freqtrade/rpc/rpc.py | 6 ++++-- freqtrade/tests/rpc/test_rpc.py | 34 ++++++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index dd3eed001..f972e64cc 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -316,8 +316,10 @@ class RPC(object): and order['side'] == 'buy': exchange.cancel_order(trade.open_order_id, trade.pair) trade.close(order.get('price') or trade.open_rate) - # TODO: sell amount which has been bought already - return + # Do the best effort, if we don't know 'filled' amount, don't try selling + if order['filled'] is None: + return + trade.amount = order['filled'] # Ignore trades with an attached LIMIT_SELL order if order and order['status'] == 'open' \ diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 1d1b3b39c..1cf374b6b 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -449,20 +449,44 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: freqtradebot.state = State.RUNNING assert cancel_order_mock.call_count == 0 # make an limit-buy open trade + trade = Trade.query.filter(Trade.id == '1').first() + filled_amount = trade.amount / 2 mocker.patch( 'freqtrade.freqtradebot.exchange.get_order', return_value={ 'status': 'open', 'type': 'limit', - 'side': 'buy' + 'side': 'buy', + 'filled': filled_amount } ) - # check that the trade is called, which is done - # by ensuring exchange.cancel_order is called + # check that the trade is called, which is done by ensuring exchange.cancel_order is called + # and trade amount is updated (error, res) = rpc.rpc_forcesell('1') assert not error assert res == '' assert cancel_order_mock.call_count == 1 + assert trade.amount == filled_amount + + freqtradebot.create_trade() + trade = Trade.query.filter(Trade.id == '2').first() + amount = trade.amount + # make an limit-buy open trade, if there is no 'filled', don't sell it + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_order', + return_value={ + 'status': 'open', + 'type': 'limit', + 'side': 'buy', + 'filled': None + } + ) + # check that the trade is called, which is done by ensuring exchange.cancel_order is called + (error, res) = rpc.rpc_forcesell('2') + assert not error + assert res == '' + assert cancel_order_mock.call_count == 2 + assert trade.amount == amount freqtradebot.create_trade() # make an limit-sell open trade @@ -474,11 +498,11 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: 'side': 'sell' } ) - (error, res) = rpc.rpc_forcesell('2') + (error, res) = rpc.rpc_forcesell('3') assert not error assert res == '' # status quo, no exchange calls - assert cancel_order_mock.call_count == 1 + assert cancel_order_mock.call_count == 2 def test_performance_handle(default_conf, ticker, limit_buy_order, fee, From 5a4eb2cbf29e421230bba6e7763cd2c792c100f7 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Tue, 29 May 2018 20:48:34 -0700 Subject: [PATCH 02/17] Setup.sh: make message format consistent --- setup.sh | 56 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/setup.sh b/setup.sh index 1d06e8208..8568d1a99 100755 --- a/setup.sh +++ b/setup.sh @@ -2,16 +2,17 @@ #encoding=utf8 function updateenv () { - echo " - ------------------------- - Update your virtual env - ------------------------- - " + echo "-------------------------" + echo "Update your virtual env" + echo "-------------------------" source .env/bin/activate - pip3.6 install --upgrade pip - pip3 install -r requirements.txt --upgrade - pip3 install -r requirements.txt - pip3 install -e . + echo "pip3 install in-progress. Please wait..." + pip3.6 install --quiet --upgrade pip + pip3 install --quiet -r requirements.txt --upgrade + pip3 install --quiet -r requirements.txt + pip3 install --quiet -e . + echo "pip3 install completed" + echo } # Install tab lib @@ -29,7 +30,6 @@ function install_macos () { echo "-------------------------" echo "Install Brew" echo "-------------------------" - echo /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" fi brew install python3 wget ta-lib @@ -54,7 +54,6 @@ function reset () { echo "----------------------------" echo "Reset branch and virtual env" echo "----------------------------" - echo if [ "1" == $(git branch -vv |grep -cE "\* develop|\* master") ] then if [ -d ".env" ]; then @@ -77,6 +76,7 @@ function reset () { echo "Reset ignored because you are not on 'master' or 'develop'." fi + echo python3.6 -m venv .env updateenv } @@ -84,11 +84,9 @@ function reset () { function config_generator () { echo "Starting to generate config.json" - - echo "-------------------------" + echo echo "General configuration" echo "-------------------------" - echo default_max_trades=3 read -p "Max open trades: (Default: $default_max_trades) " max_trades max_trades=${max_trades:-$default_max_trades} @@ -105,14 +103,13 @@ function config_generator () { read -p "Fiat currency: (Default: $default_fiat_currency) " fiat_currency fiat_currency=${fiat_currency:-$default_fiat_currency} - echo "------------------------" - echo "Bittrex config generator" - echo "------------------------" echo + echo "Exchange config generator" + echo "------------------------" read -p "Exchange API key: " api_key read -p "Exchange API Secret: " api_secret - echo "-------------------------" + echo echo "Telegram config generator" echo "-------------------------" read -p "Telegram Token: " token @@ -131,6 +128,10 @@ function config_generator () { } function config () { + + echo "-------------------------" + echo "Config file generator" + echo "-------------------------" if [ -f config.json ] then read -p "A config file already exist, do you want to override it [Y/N]? " @@ -144,22 +145,26 @@ function config () { config_generator fi + echo + echo "-------------------------" + echo "Config file generated" + echo "-------------------------" echo "Edit ./config.json to modify Pair and other configurations." + echo } function install () { echo "-------------------------" echo "Install mandatory dependencies" echo "-------------------------" - echo if [ "$(uname -s)" == "Darwin" ] then - echo "- You are on macOS" + echo "macOS detected. Setup for this system in-progress" install_macos elif [ -x "$(command -v apt-get)" ] then - echo "- You are on Debian/Ubuntu" + echo "Debian/Ubuntu detected. Setup for this system in-progress" install_debian else echo "This script does not support your OS." @@ -167,12 +172,13 @@ function install () { echo "Wait 10 seconds to continue the next install steps or use ctrl+c to interrupt this shell." sleep 10 fi + echo reset - echo " - - Install complete. - " config - echo "You can now use the bot by executing 'source .env/bin/activate; python3 freqtrade/main.py'." + echo "-------------------------" + echo "Run the bot" + echo "-------------------------" + echo "You can now use the bot by executing 'source .env/bin/activate; python3.6 freqtrade/main.py'." } function plot () { From f59f534c6461edd90d93f66f89f2bdc7c175209e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Tue, 29 May 2018 20:49:37 -0700 Subject: [PATCH 03/17] Setup.sh: fix Python3.6 when broken on macOS --- setup.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/setup.sh b/setup.sh index 8568d1a99..a825ca41f 100755 --- a/setup.sh +++ b/setup.sh @@ -33,6 +33,8 @@ function install_macos () { /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" fi brew install python3 wget ta-lib + + test_and_fix_python_on_mac } # Install bot Debian_ubuntu @@ -81,6 +83,19 @@ function reset () { updateenv } +function test_and_fix_python_on_mac() { + + if ! [ -x "$(command -v python3.6)" ] + then + echo "-------------------------" + echo "Fixing Python" + echo "-------------------------" + echo "Python 3.6 is not linked in your system. Fixing it..." + brew link --overwrite python + echo + fi +} + function config_generator () { echo "Starting to generate config.json" From b7e0466d7c3377aabb6e66b83ef5ca6e4b0b8fa4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 30 May 2018 18:42:00 +0200 Subject: [PATCH 04/17] Update ccxt from 1.14.73 to 1.14.96 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 37b6a09f1..7000f0eb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.73 +ccxt==1.14.96 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From b731a65c7502a2b317d5cedb3fd76e4bf1979470 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 2 Jun 2018 04:27:04 +0200 Subject: [PATCH 05/17] Update ccxt from 1.14.96 to 1.14.119 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7000f0eb1..ad64e58fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.96 +ccxt==1.14.119 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 792dd556a1b1faefabdaacdd5ec8e1d3ce63398e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 1 Jun 2018 19:37:39 -0700 Subject: [PATCH 06/17] Fix wrong hint '--update-pairs-cached' from Backtesting/Hyperopt --- freqtrade/optimize/__init__.py | 5 ++++- freqtrade/tests/optimize/test_optimize.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index f6f1ba47a..68ba5622e 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -103,7 +103,10 @@ def load_data(datadir: str, if pairdata: result[pair] = pairdata else: - logger.warn('No data for pair %s, use --update-pairs-cached to download the data', pair) + logger.warning( + 'No data for pair %s, use --refresh-pairs-cached to download the data', + pair + ) return result diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 8624b500d..765d88cd5 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -105,7 +105,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None: refresh_pairs=False, pairs=['MEME/BTC']) assert os.path.isfile(file) is False - assert log_has('No data for pair MEME/BTC, use --update-pairs-cached to download the data', + assert log_has('No data for pair MEME/BTC, use --refresh-pairs-cached to download the data', caplog.record_tuples) # download a new pair if refresh_pairs is set From a82a31341bc1d26f08023f3260471460cfa16620 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 2 Jun 2018 11:32:05 +0300 Subject: [PATCH 07/17] change misleading logging for datadir --- freqtrade/configuration.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 6 +++--- freqtrade/tests/test_configuration.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 43d0a0bf9..03a25efb3 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -145,7 +145,7 @@ class Configuration(object): # If --datadir is used we add it to the configuration if 'datadir' in self.args and self.args.datadir: config.update({'datadir': self.args.datadir}) - logger.info('Parameter --datadir detected: %s ...', self.args.datadir) + logger.info('Using data folder: %s ...', self.args.datadir) # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f17a0115e..bfb82f3ec 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -181,7 +181,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config @@ -229,7 +229,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config @@ -632,7 +632,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Parameter -l/--live detected ...', 'Using max_open_trades: 1 ...', 'Parameter --timerange detected: -100 ..', - 'Parameter --datadir detected: freqtrade/tests/testdata ...', + 'Using data folder: freqtrade/tests/testdata ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Downloading data for all pairs in whitelist ...', diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index d27405d91..dcf725e83 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -235,7 +235,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config @@ -286,7 +286,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config From 6ca375a39783eea32aafe271c5672ea85215c11c Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Sat, 2 Jun 2018 19:45:08 +0300 Subject: [PATCH 08/17] Extend timerange to accept unix timestamps. This gives greater granularity over backtest, parsing tickerfiles. Example runs using date and unix time below. /usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/bin/python3.6 /Users/creslin/PycharmProjects/freqtrade/scripts/report_correlation.py --timerange=20180528-20180529 2018-06-02 18:44:58,829 - freqtrade.configuration - INFO - Log level set to INFO 2018-06-02 18:44:58,830 - freqtrade.configuration - INFO - Using max_open_trades: 200 ... 2018-06-02 18:44:58,831 - freqtrade.configuration - INFO - Parameter --timerange detected: 20180528-20180529 ... 2018-06-02 18:44:58,831 - freqtrade.configuration - INFO - Parameter --datadir detected: freqtrade/tests/testdata ... BasePair Pair Correlation BTC % Change Pair % USD Ch Pair % BTC Ch Gain % on BTC Start Stop BTC Volume 1 BTC_USDT ETC_USD 0.965 -2.942 -4.070 -1.163 -1.128585 05-28 00:00 05-29 00:00 335.19 0 BTC_USDT SNT_USD 0.943 -2.942 -5.857 -3.004 -2.915585 05-28 00:00 05-29 00:00 496.01 3 BTC_USDT DASH_USD 0.902 -2.942 -9.034 -6.277 -6.092432 05-28 00:00 05-29 00:00 751.41 2 BTC_USDT MTH_USD 0.954 -2.942 -9.290 -6.541 -6.348708 05-28 00:00 05-29 00:00 23.00 4 BTC_USDT TRX_USD 0.951 -2.942 -13.647 -11.029 -10.704957 05-28 00:00 05-29 00:00 14544.57 Process finished with exit code 0 /usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/bin/python3.6 /Users/creslin/PycharmProjects/freqtrade/scripts/report_correlation.py --timerange=1527595200-1527618600 2018-06-02 18:47:40,382 - freqtrade.configuration - INFO - Log level set to INFO 2018-06-02 18:47:40,382 - freqtrade.configuration - INFO - Using max_open_trades: 200 ... 2018-06-02 18:47:40,383 - freqtrade.configuration - INFO - Parameter --timerange detected: 1527595200-1527618600 ... 2018-06-02 18:47:40,383 - freqtrade.configuration - INFO - Parameter --datadir detected: freqtrade/tests/testdata ... BasePair Pair Correlation BTC % Change Pair % USD Ch Pair % BTC Ch Gain % on BTC Start Stop BTC Volume 0 BTC_USDT SNT_USD 0.680 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 68866.30 1 BTC_USDT ETC_USD 0.857 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 227514.17 2 BTC_USDT MTH_USD 0.790 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 12103.96 3 BTC_USDT DASH_USD 0.862 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 72982.78 4 BTC_USDT TRX_USD 0.178 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 1258316.95 Process finished with exit code 0 --- freqtrade/arguments.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index afcb05511..00869f974 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -222,6 +222,9 @@ class Arguments(object): syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), + (r'^-(\d{10})$', (None, 'timestamp')), + (r'^(\d{10})-$', ('timestamp', None)), + (r'^(\d{10})-(\d{10})$', ('timestamp', 'timestamp')), (r'^(-\d+)$', (None, 'line')), (r'^(\d+)-$', ('line', None)), (r'^(\d+)-(\d+)$', ('index', 'index'))] @@ -237,6 +240,8 @@ class Arguments(object): start = rvals[index] if stype[0] == 'date': start = arrow.get(start, 'YYYYMMDD').timestamp + elif stype[0] == 'timestamp': + start = arrow.get(start).timestamp else: start = int(start) index += 1 @@ -244,6 +249,8 @@ class Arguments(object): stop = rvals[index] if stype[1] == 'date': stop = arrow.get(stop, 'YYYYMMDD').timestamp + elif stype[1] == 'timestamp': + stop = arrow.get(stop).timestamp else: stop = int(stop) return stype, start, stop From 9dbe5fdb852e1fc32bab6792bb37610fab508d04 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Sat, 2 Jun 2018 19:49:23 +0300 Subject: [PATCH 09/17] Update back testing document to include example using Posix timestamps as timerange e.g --timerange=1527595200-1527618600 --- docs/backtesting.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/backtesting.md b/docs/backtesting.md index df105bd77..8c4c4180d 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -83,6 +83,8 @@ The full timerange specification: - Use tickframes till 2018/01/31: `--timerange=-20180131` - Use tickframes since 2018/01/31: `--timerange=20180131-` - Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301` +- Use tickframes between POSIX timestamps 1527595200 1527618600: + `--timerange=1527595200-1527618600` **Update testdata directory** From 2791d543ea38cf15224c8bf8dc7ae5a7f8e0cbb0 Mon Sep 17 00:00:00 2001 From: Raymond Luo Date: Fri, 18 May 2018 19:02:38 +0800 Subject: [PATCH 10/17] Make backtesting report markdown shareable Small tweak to make the backtesting report markdown ready and much easier to share reports on many markdown publishing tools and editors that already support Markdown Extra with just a copy and paste Example: ![Example](https://i.imgur.com/HXlNkfm.png) --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 376730d0f..ab45b7754 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -106,7 +106,7 @@ class Backtesting(object): len(results[results.profit_BTC > 0]), len(results[results.profit_BTC < 0]) ]) - return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, From 9537f17dd41ea0f5ba6e87f06bc7cc32e0d7ccad Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 20:06:29 +0200 Subject: [PATCH 11/17] Fix test --- freqtrade/tests/optimize/test_backtesting.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index bfb82f3ec..786568f98 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -374,16 +374,15 @@ def test_generate_text_table(default_conf, mocker): ) result_str = ( - 'pair buy count avg profit % ' - 'total profit BTC avg duration profit loss\n' - '------- ----------- -------------- ' - '------------------ -------------- -------- ------\n' - 'ETH/BTC 2 15.00 ' - '0.60000000 20.0 2 0\n' - 'TOTAL 2 15.00 ' - '0.60000000 20.0 2 0' + '| pair | buy count | avg profit % | ' + 'total profit BTC | avg duration | profit | loss |\n' + '|:--------|------------:|---------------:|' + '-------------------:|---------------:|---------:|-------:|\n' + '| ETH/BTC | 2 | 15.00 | ' + '0.60000000 | 20.0 | 2 | 0 |\n' + '| TOTAL | 2 | 15.00 | ' + '0.60000000 | 20.0 | 2 | 0 |' ) - assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str From 43ba02afc63e106925b47d3ee9e25a87809f02ae Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Sat, 2 Jun 2018 21:59:18 +0300 Subject: [PATCH 12/17] Per feed back, kept the stype as date. Use a tuple to keep as epoch int or process via arrow to timestamp. I'll look at the test file also. --- freqtrade/arguments.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 00869f974..b61324ccc 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -222,9 +222,9 @@ class Arguments(object): syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), - (r'^-(\d{10})$', (None, 'timestamp')), - (r'^(\d{10})-$', ('timestamp', None)), - (r'^(\d{10})-(\d{10})$', ('timestamp', 'timestamp')), + (r'^-(\d{10})$', (None, 'date')), + (r'^(\d{10})-$', ('date', None)), + (r'^(\d{10})-(\d{10})$', ('date', 'date')), (r'^(-\d+)$', (None, 'line')), (r'^(\d+)-$', ('line', None)), (r'^(\d+)-(\d+)$', ('index', 'index'))] @@ -239,18 +239,16 @@ class Arguments(object): if stype[0]: start = rvals[index] if stype[0] == 'date': - start = arrow.get(start, 'YYYYMMDD').timestamp - elif stype[0] == 'timestamp': - start = arrow.get(start).timestamp + start = int(start) if len(start) == 10 \ + else arrow.get(start, 'YYYYMMDD').timestamp else: start = int(start) index += 1 if stype[1]: stop = rvals[index] if stype[1] == 'date': - stop = arrow.get(stop, 'YYYYMMDD').timestamp - elif stype[1] == 'timestamp': - stop = arrow.get(stop).timestamp + stop = int(stop) if len(stop) == 10 \ + else arrow.get(stop, 'YYYYMMDD').timestamp else: stop = int(stop) return stype, start, stop From dc65753a641a4c0de2bf9052eddc3f26e7470e67 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 2 Jun 2018 12:35:07 -0700 Subject: [PATCH 13/17] Fix the in-progress dot that does not show up during a Hyperopt run --- freqtrade/optimize/hyperopt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 20fa5380d..3317a6c6f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -455,6 +455,7 @@ class Hyperopt(Backtesting): if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: print('.', end='') + sys.stdout.flush() return { 'status': STATUS_FAIL, 'loss': float('inf') From 94e586c049211e2267726e3b0509ece14b95bef3 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 2 Jun 2018 22:46:54 +0300 Subject: [PATCH 14/17] Added unit test to check posix time arguments passed to timerange Here is the pass report: freqtrade_new creslin$ pytest freqtrade/tests/test_arguments.py ==================================================================== test session starts ===================================================================== platform darwin -- Python 3.6.5, pytest-3.6.0, py-1.5.3, pluggy-0.6.0 rootdir: /Users/creslin/PycharmProjects/freqtrade_new, inifile: plugins: mock-1.10.0, cov-2.5.1 collected 19 items freqtrade/tests/test_arguments.py ................... [100%] ================================================================= 19 passed in 2.37 seconds ================================================================== --- freqtrade/tests/test_arguments.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 279ace0dc..474aa2507 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -116,6 +116,12 @@ def test_parse_timerange_incorrect() -> None: timerange = Arguments.parse_timerange('20100522-20150730') assert timerange == (('date', 'date'), 1274486400, 1438214400) + # Added test for unix timestamp - BTC genesis date + assert (('date', None), 1231006505, None) == Arguments.parse_timerange('1231006505-') + assert ((None, 'date'), None, 1233360000) == Arguments.parse_timerange('-1233360000') + timerange = Arguments.parse_timerange('1231006505-1233360000') + assert timerange == (('date', 'date'), 1231006505, 1233360000) + with pytest.raises(Exception, match=r'Incorrect syntax.*'): Arguments.parse_timerange('-') From 127cf5d6192ce3c277798c6982887f688f9c55ac Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 2 Jun 2018 13:55:05 -0700 Subject: [PATCH 15/17] Backtesting: Add the Interval required when data is missing Change the message: "No data for pair ETH/BTC, use --refresh-pairs-cached to download the data" for: "No data for pair: "ETH/BTC", Interval: 5m. Use --refresh-pairs-cached to download the data" The message structure is unified with the download message: "Download the pair: "ETH/BTC", Interval: 5m" --- freqtrade/optimize/__init__.py | 6 ++++-- freqtrade/tests/optimize/test_optimize.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 68ba5622e..19e235a7a 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -104,8 +104,10 @@ def load_data(datadir: str, result[pair] = pairdata else: logger.warning( - 'No data for pair %s, use --refresh-pairs-cached to download the data', - pair + 'No data for pair: "%s", Interval: %s. ' + 'Use --refresh-pairs-cached to download the data', + pair, + ticker_interval ) return result diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 765d88cd5..349fa3be3 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -105,7 +105,8 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None: refresh_pairs=False, pairs=['MEME/BTC']) assert os.path.isfile(file) is False - assert log_has('No data for pair MEME/BTC, use --refresh-pairs-cached to download the data', + assert log_has('No data for pair: "MEME/BTC", Interval: 1m. ' + 'Use --refresh-pairs-cached to download the data', caplog.record_tuples) # download a new pair if refresh_pairs is set From fe8ff1b929ed361197e459d2ff8568c9d6c2f7ac Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 2 Jun 2018 14:07:31 -0700 Subject: [PATCH 16/17] Fix stake_currency return by Hyperopt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hyperopt had BTC hard coded in the result. This commit will display the real stake_currency used. If you used `"stake_currency": "USDT",` in your config file. Before this commit you saw a message like: "2 trades. Avg profit 0.13%. Total profit 0.00002651 BTC (0.0027Σ%). Avg duration 142.5 mins." Now with the commit, we fix the wrong BTC currency: "2 trades. Avg profit 0.13%. Total profit 0.00002651 USDT (0.0027Σ%). Avg duration 142.5 mins." --- freqtrade/optimize/hyperopt.py | 6 +++--- freqtrade/tests/optimize/test_hyperopt.py | 21 ++++++++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 20fa5380d..111da9b5c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -479,16 +479,16 @@ class Hyperopt(Backtesting): 'result': result_explanation, } - @staticmethod - def format_results(results: DataFrame) -> str: + def format_results(self, results: DataFrame) -> str: """ Return the format result in a string """ return ('{:6d} trades. Avg profit {: 5.2f}%. ' - 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( + 'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( len(results.index), results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), + self.config['stake_currency'], results.profit_percent.sum(), results.duration.mean(), ) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index f8fa66b2e..3edfe4393 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -389,10 +389,12 @@ def test_start_uses_mongotrials(mocker, init_hyperopt, default_conf) -> None: # test buy_strategy_generator def populate_buy_trend # test optimizer if 'ro_t1' in params -def test_format_results(): +def test_format_results(init_hyperopt): """ Test Hyperopt.format_results() """ + + # Test with BTC as stake_currency trades = [ ('ETH/BTC', 2, 2, 123), ('LTC/BTC', 1, 1, 123), @@ -400,8 +402,21 @@ def test_format_results(): ] labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] df = pd.DataFrame.from_records(trades, columns=labels) - x = Hyperopt.format_results(df) - assert x.find(' 66.67%') + + result = _HYPEROPT.format_results(df) + assert result.find(' 66.67%') + assert result.find('Total profit 1.00000000 BTC') + assert result.find('2.0000Σ %') + + # Test with EUR as stake_currency + trades = [ + ('ETH/EUR', 2, 2, 123), + ('LTC/EUR', 1, 1, 123), + ('XPR/EUR', -1, -2, -246) + ] + df = pd.DataFrame.from_records(trades, columns=labels) + result = _HYPEROPT.format_results(df) + assert result.find('Total profit 1.00000000 EUR') def test_signal_handler(mocker, init_hyperopt): From cfb06ceb58243618a52a00035cdaacd7c9e857be Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 3 Jun 2018 10:12:07 +0200 Subject: [PATCH 17/17] Update ccxt from 1.14.119 to 1.14.120 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ad64e58fd..d96084348 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.119 +ccxt==1.14.120 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1