From 56e697acf5987811e62f09ee737f95de521ae078 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 21 May 2018 20:01:41 +0200 Subject: [PATCH 01/72] Fix error initializing coinmarketcap --- freqtrade/fiat_convert.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 7e74adcd2..17882f51a 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -8,6 +8,7 @@ import time from typing import Dict from coinmarketcap import Market +from requests.exceptions import RequestException logger = logging.getLogger(__name__) @@ -94,8 +95,8 @@ class CryptoToFiatConverter(object): coinlistings = self._coinmarketcap.listings() self._cryptomap = dict(map(lambda coin: (coin["symbol"], str(coin["id"])), coinlistings["data"])) - except ValueError: - logger.error("Could not load FIAT Cryptocurrency map") + except (ValueError, RequestException) as e: + logger.error("Could not load FIAT Cryptocurrency map for the following problem: %s", e) def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float: """ From e2efd7c6ec12dcef187864053c71d088bd34b7e8 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Mon, 21 May 2018 20:03:25 +0200 Subject: [PATCH 02/72] add test to verify network exception is cought on init of coinmarketcap --- freqtrade/tests/test_fiat_convert.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index f5be9daf0..b37ca0f5c 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -6,6 +6,8 @@ from unittest.mock import MagicMock import pytest +from requests.exceptions import RequestException + from freqtrade.fiat_convert import CryptoFiat, CryptoToFiatConverter from freqtrade.tests.conftest import patch_coinmarketcap @@ -133,6 +135,21 @@ def test_loadcryptomap(mocker): assert fiat_convert._cryptomap["BTC"] == "1" +def test_fiat_init_network_exception(mocker): + # Because CryptoToFiatConverter is a Singleton we reset the listings + listmock = MagicMock(side_effect=RequestException) + mocker.patch.multiple( + 'freqtrade.fiat_convert.Market', + listings=listmock, + ) + # with pytest.raises(RequestEsxception): + fiat_convert = CryptoToFiatConverter() + fiat_convert._cryptomap = {} + fiat_convert._load_cryptomap() + + assert len(fiat_convert._cryptomap) == 0 + + def test_fiat_convert_without_network(): # Because CryptoToFiatConverter is a Singleton we reset the value of _coinmarketcap From e1cb0dbf28cd0ba1c19ccfa33e040d47a6783116 Mon Sep 17 00:00:00 2001 From: Anton Date: Mon, 21 May 2018 22:31:08 +0300 Subject: [PATCH 03/72] Do not try to redownload pair data if --refresh-pairs-cached is not set --- freqtrade/optimize/__init__.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 168481920..ab7a6be43 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -29,7 +29,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) - if stype[0] == 'index': start_index = start elif stype[0] == 'date': - while tickerlist[start_index][0] < start * 1000: + while start_index < len(tickerlist) and tickerlist[start_index][0] < start * 1000: start_index += 1 if stype[1] == 'line': @@ -37,7 +37,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) - if stype[1] == 'index': stop_index = stop elif stype[1] == 'date': - while tickerlist[stop_index-1][0] > stop * 1000: + while stop_index > 0 and tickerlist[stop_index-1][0] > stop * 1000: stop_index -= 1 if start_index > stop_index: @@ -100,15 +100,11 @@ def load_data(datadir: str, for pair in _pairs: pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) - if not pairdata: - # download the tickerdata from exchange - download_backtesting_testdata(datadir, - pair=pair, - tick_interval=ticker_interval, - timerange=timerange) - # and retry reading the pair - pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) - result[pair] = pairdata + if pairdata: + result[pair] = pairdata + else: + logger.warn('No data for pair %s, use --update-pairs-cached to download the data', pair) + return result From 8c22cfce37cb945ef0c31804cc8056c69d7f1786 Mon Sep 17 00:00:00 2001 From: Anton Date: Mon, 21 May 2018 23:15:01 +0300 Subject: [PATCH 04/72] Fix tests; fix codestyle --- freqtrade/optimize/__init__.py | 2 +- freqtrade/tests/optimize/test_optimize.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index ab7a6be43..f6f1ba47a 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -104,7 +104,7 @@ def load_data(datadir: str, result[pair] = pairdata else: logger.warn('No data for pair %s, use --update-pairs-cached to download the data', pair) - + return result diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index e7f3b18fd..8624b500d 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -99,7 +99,20 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None: file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') _backup_file(file) - optimize.load_data(None, ticker_interval='1m', pairs=['MEME/BTC']) + # do not download a new pair if refresh_pairs isn't set + optimize.load_data(None, + ticker_interval='1m', + refresh_pairs=False, + pairs=['MEME/BTC']) + assert os.path.isfile(file) is False + assert log_has('No data for pair MEME/BTC, use --update-pairs-cached to download the data', + caplog.record_tuples) + + # download a new pair if refresh_pairs is set + optimize.load_data(None, + ticker_interval='1m', + refresh_pairs=True, + pairs=['MEME/BTC']) assert os.path.isfile(file) is True assert log_has('Download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) _clean_test_file(file) From c7ef69f4eb63777f1ea1d331201523c4237fcf1d Mon Sep 17 00:00:00 2001 From: Pan Long Date: Tue, 22 May 2018 22:05:08 +0800 Subject: [PATCH 05/72] Auto apply default values in setup. Before this commit, during setup, even a default value is displayed for some config, if user doesn't enter anything, an empty value is applied. After this commit, if user doesn't enter anything for a config with default value, the default value will be applied. --- setup.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/setup.sh b/setup.sh index bdcec7186..1d06e8208 100755 --- a/setup.sh +++ b/setup.sh @@ -89,13 +89,21 @@ function config_generator () { echo "General configuration" echo "-------------------------" echo - read -p "Max open trades: (Default: 3) " max_trades + default_max_trades=3 + read -p "Max open trades: (Default: $default_max_trades) " max_trades + max_trades=${max_trades:-$default_max_trades} - read -p "Stake amount: (Default: 0.05) " stake_amount + default_stake_amount=0.05 + read -p "Stake amount: (Default: $default_stake_amount) " stake_amount + stake_amount=${stake_amount:-$default_stake_amount} - read -p "Stake currency: (Default: BTC) " stake_currency + default_stake_currency="BTC" + read -p "Stake currency: (Default: $default_stake_currency) " stake_currency + stake_currency=${stake_currency:-$default_stake_currency} - read -p "Fiat currency: (Default: USD) " fiat_currency + default_fiat_currency="USD" + read -p "Fiat currency: (Default: $default_fiat_currency) " fiat_currency + fiat_currency=${fiat_currency:-$default_fiat_currency} echo "------------------------" echo "Bittrex config generator" From 34e78a7400a618cb15282da7557fd05a1aef9f7d Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Wed, 23 May 2018 13:17:35 +0300 Subject: [PATCH 06/72] OSX docker start cmd updated New versions of Docker will not start in OSX using the cmd in these instructions as /etc/localtime cannot be mounted. The change provides an alternate command that does work. `docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade` More info is in this thread: https://github.com/docker/for-mac/issues/2396 --- docs/installation.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/installation.md b/docs/installation.md index a7e61bbe5..bba3ff66c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -132,6 +132,12 @@ You can run a one-off container that is immediately deleted upon exiting with th docker run --rm -v /etc/localtime:/etc/localtime:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade ``` +There is known issue in OSX Docker versions after 17.09.1, whereby /etc/localtime cannot be shared causing Docker to not start. A work-around for this is to start with the following cmd. + +```bash +docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade +``` + In this example, the database will be created inside the docker instance and will be lost when you will refresh your image. From 318c9734616eabda33c517cbf21c6a00e28c42cb Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Wed, 23 May 2018 15:20:16 +0300 Subject: [PATCH 07/72] Update to installation.md Added link to Docker issue on OSX with greater detail of the problem and work-around. --- docs/installation.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/installation.md b/docs/installation.md index bba3ff66c..be8e2e501 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -137,6 +137,7 @@ There is known issue in OSX Docker versions after 17.09.1, whereby /etc/localtim ```bash docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade ``` +More information on this docker issue and work-around can be read here: https://github.com/docker/for-mac/issues/2396 In this example, the database will be created inside the docker instance and will be lost when you will refresh your image. From af0b1e806f56a93f21a5becba6c120978150901c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 23 May 2018 15:06:26 +0200 Subject: [PATCH 08/72] Update pytest from 3.5.1 to 3.6.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 32fd1f5f2..e951e7411 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.14.3 TA-Lib==0.4.17 -pytest==3.5.1 +pytest==3.6.0 pytest-mock==1.10.0 pytest-cov==2.5.1 hyperopt==0.1 From bad5d57d712613b0377b1c0a847d4a95a7566389 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 24 May 2018 08:26:46 +0200 Subject: [PATCH 09/72] Update ccxt from 1.14.27 to 1.14.62 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e951e7411..34987823a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.27 +ccxt==1.14.62 SQLAlchemy==1.2.7 python-telegram-bot==10.1.0 arrow==0.12.1 From 1ba5c5d9c6cf6140317eca8a1011b3d52c8fcc58 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Fri, 25 May 2018 21:23:15 +0200 Subject: [PATCH 10/72] set ccxt loglevel to info --- freqtrade/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/main.py b/freqtrade/main.py index 9639922f9..973ed031d 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -61,6 +61,7 @@ def set_loggers() -> None: :return: None """ logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) + logging.getLogger('ccxt.base.exchange').setLevel(logging.INFO) logging.getLogger('telegram').setLevel(logging.INFO) From a98fcee4f950448495f160324738e929705929bf Mon Sep 17 00:00:00 2001 From: Pan Long Date: Fri, 25 May 2018 22:29:06 +0800 Subject: [PATCH 11/72] Sell filled amount or an open limit buy order in forcesell. Currently forcesell only cancels an open limit buy order and doesn't sell the filled amount. After this change, forcesell will also update trade's amount to filled amount and sell the filled amount. --- freqtrade/rpc/rpc.py | 6 ++++-- freqtrade/tests/rpc/test_rpc.py | 34 ++++++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index dd3eed001..f972e64cc 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -316,8 +316,10 @@ class RPC(object): and order['side'] == 'buy': exchange.cancel_order(trade.open_order_id, trade.pair) trade.close(order.get('price') or trade.open_rate) - # TODO: sell amount which has been bought already - return + # Do the best effort, if we don't know 'filled' amount, don't try selling + if order['filled'] is None: + return + trade.amount = order['filled'] # Ignore trades with an attached LIMIT_SELL order if order and order['status'] == 'open' \ diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 1d1b3b39c..1cf374b6b 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -449,20 +449,44 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: freqtradebot.state = State.RUNNING assert cancel_order_mock.call_count == 0 # make an limit-buy open trade + trade = Trade.query.filter(Trade.id == '1').first() + filled_amount = trade.amount / 2 mocker.patch( 'freqtrade.freqtradebot.exchange.get_order', return_value={ 'status': 'open', 'type': 'limit', - 'side': 'buy' + 'side': 'buy', + 'filled': filled_amount } ) - # check that the trade is called, which is done - # by ensuring exchange.cancel_order is called + # check that the trade is called, which is done by ensuring exchange.cancel_order is called + # and trade amount is updated (error, res) = rpc.rpc_forcesell('1') assert not error assert res == '' assert cancel_order_mock.call_count == 1 + assert trade.amount == filled_amount + + freqtradebot.create_trade() + trade = Trade.query.filter(Trade.id == '2').first() + amount = trade.amount + # make an limit-buy open trade, if there is no 'filled', don't sell it + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_order', + return_value={ + 'status': 'open', + 'type': 'limit', + 'side': 'buy', + 'filled': None + } + ) + # check that the trade is called, which is done by ensuring exchange.cancel_order is called + (error, res) = rpc.rpc_forcesell('2') + assert not error + assert res == '' + assert cancel_order_mock.call_count == 2 + assert trade.amount == amount freqtradebot.create_trade() # make an limit-sell open trade @@ -474,11 +498,11 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: 'side': 'sell' } ) - (error, res) = rpc.rpc_forcesell('2') + (error, res) = rpc.rpc_forcesell('3') assert not error assert res == '' # status quo, no exchange calls - assert cancel_order_mock.call_count == 1 + assert cancel_order_mock.call_count == 2 def test_performance_handle(default_conf, ticker, limit_buy_order, fee, From 607c895065a2b0f2860cab5a96a99c30b5ba6890 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Sat, 26 May 2018 20:09:20 +0300 Subject: [PATCH 12/72] Update backtesting.md: how to call a custom strat Corrected instructions, to paraphrase the PR prior - to call a custom strategy -s the custom strategy file name in user_data/strategies after - to call a custom strategy -s the class name within the custom strategy file name in user_data/strategies --- docs/backtesting.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index b3783a665..9a0bb6d5c 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -53,9 +53,9 @@ python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180 **With a (custom) strategy file** ```bash -python3 ./freqtrade/main.py -s currentstrategy backtesting +python3 ./freqtrade/main.py -s TestStrategy backtesting ``` -Where `-s currentstrategy` refers to a filename `currentstrategy.py` in `freqtrade/user_data/strategies` +Where `-s TestStrategy` refers to the class name within the stratgey file `test_strategy.py` found in the `freqtrade/user_data/strategies` directory **Exporting trades to file** ```bash From 280e8b32089ad03f3c9d6d3396a7a7ea7b75b69e Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Sat, 26 May 2018 20:14:33 +0300 Subject: [PATCH 13/72] Update backtesting.md - correct instructions Correct instructions for calling a custom strategy file To paraphrase the change: Prior - to call a custom strategy -s the strategy file name within users_data/strategies/ directory After - to call a custom strategy -s the class name within the strategy within users_data/strategies/ directory --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 9a0bb6d5c..df105bd77 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -55,7 +55,7 @@ python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180 ```bash python3 ./freqtrade/main.py -s TestStrategy backtesting ``` -Where `-s TestStrategy` refers to the class name within the stratgey file `test_strategy.py` found in the `freqtrade/user_data/strategies` directory +Where `-s TestStrategy` refers to the class name within the strategy file `test_strategy.py` found in the `freqtrade/user_data/strategies` directory **Exporting trades to file** ```bash From 94c1a6f2a607d75d1e12c258c45e7676b1dc2998 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 26 May 2018 23:41:52 +0200 Subject: [PATCH 14/72] Update ccxt from 1.14.62 to 1.14.73 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 34987823a..575cdff9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.62 +ccxt==1.14.73 SQLAlchemy==1.2.7 python-telegram-bot==10.1.0 arrow==0.12.1 From 9cd774986792f4dc3b19941f41395efe78a22000 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 28 May 2018 22:14:50 +0200 Subject: [PATCH 15/72] Update sqlalchemy from 1.2.7 to 1.2.8 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 575cdff9b..37b6a09f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.14.73 -SQLAlchemy==1.2.7 +SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 From 5a4eb2cbf29e421230bba6e7763cd2c792c100f7 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Tue, 29 May 2018 20:48:34 -0700 Subject: [PATCH 16/72] Setup.sh: make message format consistent --- setup.sh | 56 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/setup.sh b/setup.sh index 1d06e8208..8568d1a99 100755 --- a/setup.sh +++ b/setup.sh @@ -2,16 +2,17 @@ #encoding=utf8 function updateenv () { - echo " - ------------------------- - Update your virtual env - ------------------------- - " + echo "-------------------------" + echo "Update your virtual env" + echo "-------------------------" source .env/bin/activate - pip3.6 install --upgrade pip - pip3 install -r requirements.txt --upgrade - pip3 install -r requirements.txt - pip3 install -e . + echo "pip3 install in-progress. Please wait..." + pip3.6 install --quiet --upgrade pip + pip3 install --quiet -r requirements.txt --upgrade + pip3 install --quiet -r requirements.txt + pip3 install --quiet -e . + echo "pip3 install completed" + echo } # Install tab lib @@ -29,7 +30,6 @@ function install_macos () { echo "-------------------------" echo "Install Brew" echo "-------------------------" - echo /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" fi brew install python3 wget ta-lib @@ -54,7 +54,6 @@ function reset () { echo "----------------------------" echo "Reset branch and virtual env" echo "----------------------------" - echo if [ "1" == $(git branch -vv |grep -cE "\* develop|\* master") ] then if [ -d ".env" ]; then @@ -77,6 +76,7 @@ function reset () { echo "Reset ignored because you are not on 'master' or 'develop'." fi + echo python3.6 -m venv .env updateenv } @@ -84,11 +84,9 @@ function reset () { function config_generator () { echo "Starting to generate config.json" - - echo "-------------------------" + echo echo "General configuration" echo "-------------------------" - echo default_max_trades=3 read -p "Max open trades: (Default: $default_max_trades) " max_trades max_trades=${max_trades:-$default_max_trades} @@ -105,14 +103,13 @@ function config_generator () { read -p "Fiat currency: (Default: $default_fiat_currency) " fiat_currency fiat_currency=${fiat_currency:-$default_fiat_currency} - echo "------------------------" - echo "Bittrex config generator" - echo "------------------------" echo + echo "Exchange config generator" + echo "------------------------" read -p "Exchange API key: " api_key read -p "Exchange API Secret: " api_secret - echo "-------------------------" + echo echo "Telegram config generator" echo "-------------------------" read -p "Telegram Token: " token @@ -131,6 +128,10 @@ function config_generator () { } function config () { + + echo "-------------------------" + echo "Config file generator" + echo "-------------------------" if [ -f config.json ] then read -p "A config file already exist, do you want to override it [Y/N]? " @@ -144,22 +145,26 @@ function config () { config_generator fi + echo + echo "-------------------------" + echo "Config file generated" + echo "-------------------------" echo "Edit ./config.json to modify Pair and other configurations." + echo } function install () { echo "-------------------------" echo "Install mandatory dependencies" echo "-------------------------" - echo if [ "$(uname -s)" == "Darwin" ] then - echo "- You are on macOS" + echo "macOS detected. Setup for this system in-progress" install_macos elif [ -x "$(command -v apt-get)" ] then - echo "- You are on Debian/Ubuntu" + echo "Debian/Ubuntu detected. Setup for this system in-progress" install_debian else echo "This script does not support your OS." @@ -167,12 +172,13 @@ function install () { echo "Wait 10 seconds to continue the next install steps or use ctrl+c to interrupt this shell." sleep 10 fi + echo reset - echo " - - Install complete. - " config - echo "You can now use the bot by executing 'source .env/bin/activate; python3 freqtrade/main.py'." + echo "-------------------------" + echo "Run the bot" + echo "-------------------------" + echo "You can now use the bot by executing 'source .env/bin/activate; python3.6 freqtrade/main.py'." } function plot () { From f59f534c6461edd90d93f66f89f2bdc7c175209e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Tue, 29 May 2018 20:49:37 -0700 Subject: [PATCH 17/72] Setup.sh: fix Python3.6 when broken on macOS --- setup.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/setup.sh b/setup.sh index 8568d1a99..a825ca41f 100755 --- a/setup.sh +++ b/setup.sh @@ -33,6 +33,8 @@ function install_macos () { /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" fi brew install python3 wget ta-lib + + test_and_fix_python_on_mac } # Install bot Debian_ubuntu @@ -81,6 +83,19 @@ function reset () { updateenv } +function test_and_fix_python_on_mac() { + + if ! [ -x "$(command -v python3.6)" ] + then + echo "-------------------------" + echo "Fixing Python" + echo "-------------------------" + echo "Python 3.6 is not linked in your system. Fixing it..." + brew link --overwrite python + echo + fi +} + function config_generator () { echo "Starting to generate config.json" From d9eddfb1ee9eb976689d4e64ae00b31097be5a05 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Tue, 29 May 2018 22:21:29 -0700 Subject: [PATCH 18/72] Doc: Update the exchanges supported --- README.md | 8 ++++---- docs/configuration.md | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1caa34a85..3546416df 100644 --- a/README.md +++ b/README.md @@ -70,10 +70,10 @@ want to avoid. - [x] **Performance status report**: Provide a performance status of your current trades. -### Exchange supported -- [x] Bittrex -- [ ] Binance -- [ ] Others +### Exchange marketplaces supported +- [X] [Bittrex](https://bittrex.com/) +- [X] [Binance](https://www.binance.com/) +- [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ## Quick start This quick start section is a very short explanation on how to test the diff --git a/docs/configuration.md b/docs/configuration.md index 3d36947c3..a2df3f2fe 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -24,7 +24,7 @@ The table below will list all configuration parameters. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `unfilledtimeout` | 0 | No | How long (in minutes) the bot will wait for an unfilled order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. -| `exchange.name` | bittrex | Yes | Name of the exchange class to use. +| `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. | `exchange.pair_whitelist` | [] | No | List of currency to use by the bot. Can be overrided with `--dynamic-whitelist` param. @@ -79,6 +79,18 @@ use the `last` price and values between those interpolate between ask and last price. Using `ask` price will guarantee quick success in bid, but bot will also end up paying more then would probably have been necessary. +### What values for exchange.name? +Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports 115 cryptocurrency +exchange markets and trading APIs. The complete up-to-date list can be found in the +[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested +with only Bittrex and Binance. + +The bot was tested with the following exchanges: +- [Bittrex](https://bittrex.com/): "bittrex" +- [Binance](https://www.binance.com/): "binance" + +Feel free to test other exchanges and submit your PR to improve the bot. + ### What values for fiat_display_currency? `fiat_display_currency` set the fiat to use for the conversion form coin to fiat in Telegram. The valid value are: "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD". @@ -96,7 +108,7 @@ creating trades. "dry_run": true, ``` -3. Remove your Bittrex API key (change them by fake api credentials) +3. Remove your Exchange API key (change them by fake api credentials) ```json "exchange": { "name": "bittrex", @@ -122,7 +134,7 @@ you run it in production mode. "dry_run": false, ``` -3. Insert your Bittrex API key (change them by fake api keys) +3. Insert your Exchange API key (change them by fake api keys) ```json "exchange": { "name": "bittrex", From 963d2a8368b671c04e390ce24a818778846368d4 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Tue, 29 May 2018 22:24:13 -0700 Subject: [PATCH 19/72] Doc: update bot usage --- README.md | 18 ++++++++++++------ docs/bot-usage.md | 14 ++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3546416df..3d90a0a8c 100644 --- a/README.md +++ b/README.md @@ -144,8 +144,9 @@ to understand the requirements before sending your pull-requests. ### Bot commands ```bash -usage: main.py [-h] [-v] [--version] [-c PATH] [--dry-run-db] [--datadir PATH] - [--dynamic-whitelist [INT]] +usage: main.py [-h] [-v] [--version] [-c PATH] [-d PATH] [-s NAME] + [--strategy-path PATH] [--dynamic-whitelist [INT]] + [--dry-run-db] {backtesting,hyperopt} ... Simple High Frequency Trading Bot for crypto currencies @@ -161,13 +162,18 @@ optional arguments: --version show program's version number and exit -c PATH, --config PATH specify configuration file (default: config.json) - --dry-run-db Force dry run to use a local DB - "tradesv3.dry_run.sqlite" instead of memory DB. Work - only if dry_run is enabled. - --datadir PATH path to backtest data (default freqdata/tests/testdata + -d PATH, --datadir PATH + path to backtest data (default: + freqtrade/tests/testdata + -s NAME, --strategy NAME + specify strategy class name (default: DefaultStrategy) + --strategy-path PATH specify additional strategy lookup path --dynamic-whitelist [INT] dynamically generate and update whitelist based on 24h BaseVolume (Default 20 currencies) + --dry-run-db Force dry run to use a local DB + "tradesv3.dry_run.sqlite" instead of memory DB. Work + only if dry_run is enabled. ``` More details on: - [How to run the bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 075413b21..b42df3ba3 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -9,7 +9,8 @@ it. ## Bot commands ``` -usage: main.py [-h] [-c PATH] [-v] [--version] [--dynamic-whitelist [INT]] +usage: main.py [-h] [-v] [--version] [-c PATH] [-d PATH] [-s NAME] + [--strategy-path PATH] [--dynamic-whitelist [INT]] [--dry-run-db] {backtesting,hyperopt} ... @@ -26,17 +27,18 @@ optional arguments: --version show program's version number and exit -c PATH, --config PATH specify configuration file (default: config.json) + -d PATH, --datadir PATH + path to backtest data (default: + freqtrade/tests/testdata -s NAME, --strategy NAME specify strategy class name (default: DefaultStrategy) --strategy-path PATH specify additional strategy lookup path - --dry-run-db Force dry run to use a local DB - "tradesv3.dry_run.sqlite" instead of memory DB. Work - only if dry_run is enabled. - --datadir PATH - path to backtest data (default freqdata/tests/testdata --dynamic-whitelist [INT] dynamically generate and update whitelist based on 24h BaseVolume (Default 20 currencies) + --dry-run-db Force dry run to use a local DB + "tradesv3.dry_run.sqlite" instead of memory DB. Work + only if dry_run is enabled. ``` ### How to use a different config file? From 4329c15a9b5ab493768173e7e7e91e686e031cd2 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Tue, 29 May 2018 22:38:48 -0700 Subject: [PATCH 20/72] Doc: Add Buzz/trendy word --- README.md | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3d90a0a8c..3ded0099a 100644 --- a/README.md +++ b/README.md @@ -56,19 +56,14 @@ Windows, macOS and Linux - [x] **Persistence**: Persistence is achieved through sqlite - [x] **Dry-run**: Run the bot without playing money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. -- [x] **Strategy Optimization**: Optimize your buy/sell strategy -parameters with Hyperopts. -- [x] **Whitelist crypto-currencies**: Select which crypto-currency you -want to trade. -- [x] **Blacklist crypto-currencies**: Select which crypto-currency you -want to avoid. +- [x] **Strategy Optimization by machine learning**: Use machine learning to optimize your buy/sell +strategy parameters with real exchange data. +- [x] **Whitelist crypto-currencies**: Select which crypto-currency you want to trade. +- [x] **Blacklist crypto-currencies**: Select which crypto-currency you want to avoid. - [x] **Manageable via Telegram**: Manage the bot with Telegram -- [x] **Display profit/loss in fiat**: Display your profit/loss in -33 fiat. -- [x] **Daily summary of profit/loss**: Provide a daily summary - of your profit/loss. -- [x] **Performance status report**: Provide a performance status of -your current trades. +- [x] **Display profit/loss in fiat**: Display your profit/loss in 33 fiat. +- [x] **Daily summary of profit/loss**: Provide a daily summary of your profit/loss. +- [x] **Performance status report**: Provide a performance status of your current trades. ### Exchange marketplaces supported - [X] [Bittrex](https://bittrex.com/) From b7e0466d7c3377aabb6e66b83ef5ca6e4b0b8fa4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 30 May 2018 18:42:00 +0200 Subject: [PATCH 21/72] Update ccxt from 1.14.73 to 1.14.96 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 37b6a09f1..7000f0eb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.73 +ccxt==1.14.96 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 9aa468adda4387106143bf0e2565f1ca24221edc Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 30 May 2018 22:01:29 +0200 Subject: [PATCH 22/72] fix for typehint --- freqtrade/fiat_convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 17882f51a..3d268fdbe 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -34,7 +34,7 @@ class CryptoFiat(object): self.price = 0.0 # Private attributes - self._expiration = 0 + self._expiration = 0.0 self.crypto_symbol = crypto_symbol.upper() self.fiat_symbol = fiat_symbol.upper() From 0d6dffdc7ecfa71f51816b46ef1c39239c9741b2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 30 May 2018 22:09:03 +0200 Subject: [PATCH 23/72] fix typehinting --- freqtrade/fiat_convert.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 3d268fdbe..88eb702c9 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -5,7 +5,7 @@ e.g BTC to USD import logging import time -from typing import Dict +from typing import Dict, List from coinmarketcap import Market from requests.exceptions import RequestException @@ -65,7 +65,7 @@ class CryptoToFiatConverter(object): This object is also a Singleton """ __instance = None - _coinmarketcap = None + _coinmarketcap: Market = None # Constants SUPPORTED_FIAT = [ @@ -87,7 +87,7 @@ class CryptoToFiatConverter(object): return CryptoToFiatConverter.__instance def __init__(self) -> None: - self._pairs = [] + self._pairs: List[CryptoFiat] = [] self._load_cryptomap() def _load_cryptomap(self) -> None: From 88755fcded808c2c094b6ee317affbcd1894bf7c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 30 May 2018 22:09:20 +0200 Subject: [PATCH 24/72] fix typing --- freqtrade/indicator_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/indicator_helpers.py b/freqtrade/indicator_helpers.py index 14519d7a2..50586578a 100644 --- a/freqtrade/indicator_helpers.py +++ b/freqtrade/indicator_helpers.py @@ -13,7 +13,7 @@ def went_down(series: Series) -> bool: return series < series.shift(1) -def ehlers_super_smoother(series: Series, smoothing: float = 6) -> type(Series): +def ehlers_super_smoother(series: Series, smoothing: float = 6) -> Series: magic = pi * sqrt(2) / smoothing a1 = exp(-magic) coeff2 = 2 * a1 * cos(magic) From 45909af7e003770d903460e40051d8b79d49655b Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 30 May 2018 22:38:09 +0200 Subject: [PATCH 25/72] type anotation fixes --- freqtrade/configuration.py | 6 +++--- freqtrade/misc.py | 2 +- freqtrade/optimize/__init__.py | 6 ++++-- freqtrade/persistence.py | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 43d0a0bf9..14c29e8fd 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -5,7 +5,7 @@ This module contains the configuration class import json import logging from argparse import Namespace -from typing import Dict, Any +from typing import Optional, Dict, Any from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match import ccxt @@ -23,7 +23,7 @@ class Configuration(object): """ def __init__(self, args: Namespace) -> None: self.args = args - self.config = None + self.config: Optional[Dict[str, Any]] = None def load_config(self) -> Dict[str, Any]: """ @@ -192,7 +192,7 @@ class Configuration(object): validate(conf, constants.CONF_SCHEMA) return conf except ValidationError as exception: - logger.fatal( + logger.critical( 'Invalid configuration. See config.json.example. Reason: %s', exception ) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 225fb32df..90a1db42b 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -83,7 +83,7 @@ def file_dump_json(filename, data, is_zip=False) -> None: json.dump(data, fp, default=str) -def format_ms_time(date: str) -> str: +def format_ms_time(date: int) -> str: """ convert MS date to readable format. : epoch-string in ms diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index f6f1ba47a..58a4b07fe 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -4,8 +4,8 @@ import gzip import json import logging import os +from typing import Optional, List, Dict, Tuple, Any import arrow -from typing import Optional, List, Dict, Tuple from freqtrade import misc, constants from freqtrade.exchange import get_ticker_history @@ -139,7 +139,9 @@ def download_pairs(datadir, pairs: List[str], def load_cached_data_for_updating(filename: str, tick_interval: str, - timerange: Optional[Tuple[Tuple, int, int]]) -> Tuple[list, int]: + timerange: Optional[Tuple[Tuple, int, int]]) -> Tuple[ + List[Any], + Optional[int]]: """ Load cached data and choose what part of the data should be updated """ diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 2d497662e..c10599b3c 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -5,7 +5,7 @@ This module contains the class to persist trades into SQLite import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Dict, Optional +from typing import Dict, Optional, Any import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, @@ -21,7 +21,7 @@ from sqlalchemy import inspect logger = logging.getLogger(__name__) _CONF = {} -_DECL_BASE = declarative_base() +_DECL_BASE = declarative_base() # type: Any def init(config: dict, engine: Optional[Engine] = None) -> None: From 48516e6e1e4a608ff3a0d898e0d6c286b2ecf1a0 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 20:41:05 +0200 Subject: [PATCH 26/72] Add typehint --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index afcb05511..cf7db5901 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -17,7 +17,7 @@ class Arguments(object): Arguments Class. Manage the arguments received by the cli """ - def __init__(self, args: List[str], description: str): + def __init__(self, args: List[str], description: str) -> None: self.args = args self.parsed_arg = None self.parser = argparse.ArgumentParser(description=description) From 4733aad7ff30220d1a135f3fce1c813b523ff6c6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 20:54:37 +0200 Subject: [PATCH 27/72] mypy_typing --- freqtrade/rpc/rpc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index dd3eed001..df809cc2f 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -4,7 +4,7 @@ This module contains class to define a RPC communications import logging from datetime import datetime, timedelta from decimal import Decimal -from typing import Tuple, Any +from typing import Dict, Tuple, Any, Optional import arrow import sqlalchemy as sql @@ -114,7 +114,7 @@ class RPC(object): self, timescale: int, stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: today = datetime.utcnow().date() - profit_days = {} + profit_days: Dict[int, Dict] = {} if not (isinstance(timescale, int) and timescale > 0): return True, '*Daily [n]:* `must be an integer greater than 0`' @@ -172,7 +172,7 @@ class RPC(object): durations = [] for trade in trades: - current_rate = None + current_rate: Optional[float] = None if not trade.open_rate: continue @@ -278,7 +278,7 @@ class RPC(object): value = fiat.convert_amount(total, 'BTC', symbol) return False, (output, total, symbol, value) - def rpc_start(self) -> (bool, str): + def rpc_start(self) -> Tuple[bool, str]: """ Handler for start. """ @@ -288,7 +288,7 @@ class RPC(object): self.freqtrade.state = State.RUNNING return False, '`Starting trader ...`' - def rpc_stop(self) -> (bool, str): + def rpc_stop(self) -> Tuple[bool, str]: """ Handler for stop. """ From 0d251cbfdd497279b93e134cb8fc4b5534273895 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 20:55:26 +0200 Subject: [PATCH 28/72] rpc type hints --- freqtrade/rpc/rpc_manager.py | 5 +++-- freqtrade/rpc/telegram.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 0299c793a..bedd8fdd4 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,6 +1,7 @@ """ This module contains class to manage RPC communications (Telegram, Slack, ...) """ +from typing import Any, Optional, List import logging from freqtrade.rpc.telegram import Telegram @@ -21,8 +22,8 @@ class RPCManager(object): """ self.freqtrade = freqtrade - self.registered_modules = [] - self.telegram = None + self.registered_modules: List[str] = [] + self.telegram: Any = None self._init() def _init(self) -> None: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c640fc77b..c110b9627 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -18,7 +18,7 @@ from freqtrade.rpc.rpc import RPC logger = logging.getLogger(__name__) -def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[..., Any]: +def authorized_only(command_handler: Callable[[Any, Bot, Update], None]) -> Callable[..., Any]: """ Decorator to check if the message comes from the correct chat_id :param command_handler: Telegram CommandHandler @@ -65,7 +65,7 @@ class Telegram(RPC): """ super().__init__(freqtrade) - self._updater = None + self._updater: Updater = None self._config = freqtrade.config self._init() From 1352f135d058df6be20c404a9fe81202d7da5945 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 20:55:45 +0200 Subject: [PATCH 29/72] typing --- freqtrade/analyze.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index dcb5376ce..5756b845c 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -12,7 +12,7 @@ from pandas import DataFrame, to_datetime from freqtrade import constants from freqtrade.exchange import get_ticker_history from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import StrategyResolver +from freqtrade.strategy.resolver import StrategyResolver, IStrategy logger = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class Analyze(object): :param config: Bot configuration (use the one from Configuration()) """ self.config = config - self.strategy = StrategyResolver(self.config).strategy + self.strategy: IStrategy = StrategyResolver(self.config).strategy @staticmethod def parse_ticker_dataframe(ticker: list) -> DataFrame: From 4eb55acdbcea4cf53b8fc2405de2d6154d5a338d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:04:10 +0200 Subject: [PATCH 30/72] fix typing --- freqtrade/tests/test_configuration.py | 37 ++++++++++++++------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index d27405d91..8ba8f8289 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -6,6 +6,7 @@ Unit test file for configuration.py import json from copy import deepcopy from unittest.mock import MagicMock +from argparse import Namespace import pytest from jsonschema import ValidationError @@ -37,7 +38,7 @@ def test_load_config_invalid_pair(default_conf) -> None: conf['exchange']['pair_whitelist'].append('ETH-BTC') with pytest.raises(ValidationError, match=r'.*does not match.*'): - configuration = Configuration([]) + configuration = Configuration(Namespace()) configuration._validate_config(conf) @@ -49,7 +50,7 @@ def test_load_config_missing_attributes(default_conf) -> None: conf.pop('exchange') with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): - configuration = Configuration([]) + configuration = Configuration(Namespace()) configuration._validate_config(conf) @@ -61,7 +62,7 @@ def test_load_config_file(default_conf, mocker, caplog) -> None: read_data=json.dumps(default_conf) )) - configuration = Configuration([]) + configuration = Configuration(Namespace()) validated_conf = configuration._load_config_file('somefile') assert file_mock.call_count == 1 assert validated_conf.items() >= default_conf.items() @@ -79,7 +80,7 @@ def test_load_config_max_open_trades_zero(default_conf, mocker, caplog) -> None: read_data=json.dumps(conf) )) - Configuration([])._load_config_file('somefile') + Configuration(Namespace())._load_config_file('somefile') assert file_mock.call_count == 1 assert log_has('Validating configuration ...', caplog.record_tuples) @@ -92,7 +93,7 @@ def test_load_config_file_exception(mocker, caplog) -> None: 'freqtrade.configuration.open', MagicMock(side_effect=FileNotFoundError('File not found')) ) - configuration = Configuration([]) + configuration = Configuration(Namespace()) with pytest.raises(SystemExit): configuration._load_config_file('somefile') @@ -128,13 +129,13 @@ def test_load_config_with_params(default_conf, mocker) -> None: read_data=json.dumps(default_conf) )) - args = [ + arglist = [ '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', '--strategy-path', '/some/path', '--dry-run-db', ] - args = Arguments(args, '').get_parsed_arg() + args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) validated_conf = configuration.load_config() @@ -174,12 +175,12 @@ def test_show_info(default_conf, mocker, caplog) -> None: read_data=json.dumps(default_conf) )) - args = [ + arglist = [ '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', '--dry-run-db' ] - args = Arguments(args, '').get_parsed_arg() + args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) configuration.get_config() @@ -202,8 +203,8 @@ def test_show_info(default_conf, mocker, caplog) -> None: ) # Test the Dry run condition - configuration.config.update({'dry_run': False}) - configuration._load_common_config(configuration.config) + configuration.config.update({'dry_run': False}) # type: ignore + configuration._load_common_config(configuration.config) # type: ignore assert log_has( 'Dry run is disabled. (--dry_run_db ignored)', caplog.record_tuples @@ -218,13 +219,13 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> read_data=json.dumps(default_conf) )) - args = [ + arglist = [ '--config', 'config.json', '--strategy', 'DefaultStrategy', 'backtesting' ] - args = Arguments(args, '').get_parsed_arg() + args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) config = configuration.get_config() @@ -262,7 +263,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non read_data=json.dumps(default_conf) )) - args = [ + arglist = [ '--config', 'config.json', '--strategy', 'DefaultStrategy', '--datadir', '/foo/bar', @@ -275,7 +276,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--export', '/bar/foo' ] - args = Arguments(args, '').get_parsed_arg() + args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) config = configuration.get_config() @@ -326,14 +327,14 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: read_data=json.dumps(default_conf) )) - args = [ + arglist = [ 'hyperopt', '--epochs', '10', '--use-mongodb', '--spaces', 'all', ] - args = Arguments(args, '').get_parsed_arg() + args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) config = configuration.get_config() @@ -357,7 +358,7 @@ def test_check_exchange(default_conf) -> None: Test the configuration validator with a missing attribute """ conf = deepcopy(default_conf) - configuration = Configuration([]) + configuration = Configuration(Namespace()) # Test a valid exchange conf.get('exchange').update({'name': 'BITTREX'}) From 69006b8fe821948f191839762f9246cacaba9dc8 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:08:11 +0200 Subject: [PATCH 31/72] flake8 --- freqtrade/persistence.py | 2 +- freqtrade/rpc/rpc_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c10599b3c..47a7ac4ab 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -5,7 +5,7 @@ This module contains the class to persist trades into SQLite import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Dict, Optional, Any +from typing import Dict, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index bedd8fdd4..58e9bf2b9 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,7 +1,7 @@ """ This module contains class to manage RPC communications (Telegram, Slack, ...) """ -from typing import Any, Optional, List +from typing import Any, List import logging from freqtrade.rpc.telegram import Telegram From 2976a50c58a1ad9e43a6f12fc9ff9ebc0c3f7f77 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:10:15 +0200 Subject: [PATCH 32/72] fix typing --- freqtrade/persistence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 47a7ac4ab..f9a7d1e3c 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -5,7 +5,7 @@ This module contains the class to persist trades into SQLite import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Dict, Optional +from typing import Dict, Optional, Any import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, @@ -21,7 +21,7 @@ from sqlalchemy import inspect logger = logging.getLogger(__name__) _CONF = {} -_DECL_BASE = declarative_base() # type: Any +_DECL_BASE: Any = declarative_base() def init(config: dict, engine: Optional[Engine] = None) -> None: From c0cef7250d6bc394769b647b32405597cd5ce4e3 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:22:46 +0200 Subject: [PATCH 33/72] typing - avoid variable reuse with differen ttype --- freqtrade/arguments.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index cf7db5901..2421abc9f 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -19,7 +19,7 @@ class Arguments(object): def __init__(self, args: List[str], description: str) -> None: self.args = args - self.parsed_arg = None + self.parsed_arg: Optional[argparse.Namespace] = None self.parser = argparse.ArgumentParser(description=description) def _load_args(self) -> None: @@ -211,7 +211,7 @@ class Arguments(object): self.hyperopt_options(hyperopt_cmd) @staticmethod - def parse_timerange(text: str) -> Optional[Tuple[List, int, int]]: + def parse_timerange(text: str) -> Optional[Tuple[Tuple, Optional[int], Optional[int]]]: """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange @@ -231,21 +231,21 @@ class Arguments(object): if match: # Regex has matched rvals = match.groups() index = 0 - start = None - stop = None + start: Optional[int] = None + stop: Optional[int] = None if stype[0]: - start = rvals[index] + starts = rvals[index] if stype[0] == 'date': - start = arrow.get(start, 'YYYYMMDD').timestamp + start = arrow.get(starts, 'YYYYMMDD').timestamp else: - start = int(start) + start = int(starts) index += 1 if stype[1]: - stop = rvals[index] + stops = rvals[index] if stype[1] == 'date': - stop = arrow.get(stop, 'YYYYMMDD').timestamp + stop = arrow.get(stops, 'YYYYMMDD').timestamp else: - stop = int(stop) + stop = int(stops) return stype, start, stop raise Exception('Incorrect syntax for timerange "%s"' % text) From f4f821e88eeeda2b56d03004fe72a2764f6e68dc Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:44:18 +0200 Subject: [PATCH 34/72] add typehints --- freqtrade/strategy/resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 8f4972919..23380dad9 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -33,7 +33,7 @@ class StrategyResolver(object): # Verify the strategy is in the configuration, otherwise fallback to the default strategy strategy_name = config.get('strategy') or constants.DEFAULT_STRATEGY - self.strategy = self._load_strategy(strategy_name, extra_dir=config.get('strategy_path')) + self.strategy: IStrategy = self._load_strategy(strategy_name, extra_dir=config.get('strategy_path')) # Set attributes # Check if we need to override configuration @@ -61,7 +61,7 @@ class StrategyResolver(object): self.strategy.stoploss = float(self.strategy.stoploss) def _load_strategy( - self, strategy_name: str, extra_dir: Optional[str] = None) -> Optional[IStrategy]: + self, strategy_name: str, extra_dir: Optional[str] = None) -> IStrategy: """ Search and loads the specified strategy. :param strategy_name: name of the module to import From cf34b84cf1c67a9bcc60f89b886e514d615d22f2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 21:59:22 +0200 Subject: [PATCH 35/72] add attributes with typehints --- freqtrade/strategy/interface.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dcf665a02..4ae358c6f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -2,7 +2,7 @@ IStrategy interface This module defines the interface to apply for strategies """ - +from typing import Dict from abc import ABC, abstractmethod from pandas import DataFrame @@ -16,9 +16,13 @@ class IStrategy(ABC): Attributes you can use: minimal_roi -> Dict: Minimal ROI designed for the strategy stoploss -> float: optimal stoploss designed for the strategy - ticker_interval -> int: value of the ticker interval to use for the strategy + ticker_interval -> str: value of the ticker interval to use for the strategy """ + minimal_roi: Dict + stoploss: float + ticker_interval: str + @abstractmethod def populate_indicators(self, dataframe: DataFrame) -> DataFrame: """ From 3fb1dd02f1204996da2c5cfc326cd719b64839a0 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 22:00:46 +0200 Subject: [PATCH 36/72] add typehints and type: ignores --- freqtrade/analyze.py | 6 +++--- freqtrade/exchange/__init__.py | 2 +- freqtrade/optimize/hyperopt.py | 8 ++++---- freqtrade/strategy/resolver.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 5756b845c..6334fd846 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -95,7 +95,7 @@ class Analyze(object): Return ticker interval to use :return: Ticker interval value to use """ - return self.strategy.ticker_interval + return self.strategy.ticker_interval # type: ignore def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ @@ -195,13 +195,13 @@ class Analyze(object): :return True if bot should sell at current rate """ current_profit = trade.calc_profit_percent(current_rate) - if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: + if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: # type: ignore logger.debug('Stop loss hit.') return True # Check if time matches and current rate is above threshold time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 - for duration, threshold in self.strategy.minimal_roi.items(): + for duration, threshold in self.strategy.minimal_roi.items(): # type: ignore if time_diff <= duration: return False if current_profit > threshold: diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 109e3f7b2..b186f3611 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -290,7 +290,7 @@ def get_ticker_history(pair: str, tick_interval: str, since_ms: Optional[int] = # chached data was already downloaded till_time_ms = min(till_time_ms, arrow.utcnow().shift(minutes=-10).timestamp * 1000) - data = [] + data: List[Dict[Any, Any]] = [] while not since_ms or since_ms < till_time_ms: data_part = _API.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 20fa5380d..b4f534d7c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,7 +14,7 @@ from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable +from typing import Dict, Any, Callable, Optional import numpy import talib.abstract as ta @@ -60,7 +60,7 @@ class Hyperopt(Backtesting): self.expected_max_profit = 3.0 # Configuration and data used by hyperopt - self.processed = None + self.processed: Optional[Dict[str, Any]] = None # Hyperopt Trials self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') @@ -344,7 +344,7 @@ class Hyperopt(Backtesting): """ Return the space to use during Hyperopt """ - spaces = {} + spaces: Dict = {} if self.has_space('buy'): spaces = {**spaces, **Hyperopt.indicator_space()} if self.has_space('roi'): @@ -503,7 +503,7 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.analyze.populate_indicators = Hyperopt.populate_indicators + self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) if self.config.get('mongodb'): diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 23380dad9..28465210c 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -101,7 +101,7 @@ class StrategyResolver(object): # Generate spec based on absolute path spec = importlib.util.spec_from_file_location('user_data.strategies', module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # type: ignore valid_strategies_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) From 41a47df93f81ef868f29f5cf6d9cc66d728e22bd Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 22:09:31 +0200 Subject: [PATCH 37/72] setup travis to check mypy --- .travis.yml | 3 ++- setup.cfg | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6554f2095..1cff5c04b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ addons: install: - ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -- pip install --upgrade flake8 coveralls pytest-random-order +- pip install --upgrade flake8 coveralls pytest-random-order mypy - pip install -r requirements.txt - pip install -e . jobs: @@ -26,6 +26,7 @@ jobs: - cp config.json.example config.json - python freqtrade/main.py hyperopt -e 5 - script: flake8 freqtrade + - script: mypy freqtrade after_success: - coveralls notifications: diff --git a/setup.cfg b/setup.cfg index ba065a7c2..6ffad0824 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,3 +2,6 @@ #ignore = max-line-length = 100 max-complexity = 12 + +[mypy] +ignore_missing_imports = True From 633620a5e925f08e8b7c5e8f5e29317066aba45b Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 22:15:18 +0200 Subject: [PATCH 38/72] exclude .mypy_cache --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e5ac932f8..219a9fb40 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,4 @@ target/ .vscode .pytest_cache/ +.mypy_cache/ From e28973c50a04dabd386342c9ff7b170616e2a6df Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 31 May 2018 22:17:46 +0200 Subject: [PATCH 39/72] fix flake8 --- freqtrade/analyze.py | 3 ++- freqtrade/strategy/resolver.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 6334fd846..0a560c86e 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -195,7 +195,8 @@ class Analyze(object): :return True if bot should sell at current rate """ current_profit = trade.calc_profit_percent(current_rate) - if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: # type: ignore + if self.strategy.stoploss is not None \ + and current_profit < self.strategy.stoploss: # type: ignore logger.debug('Stop loss hit.') return True diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 28465210c..60427bcf4 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -33,7 +33,8 @@ class StrategyResolver(object): # Verify the strategy is in the configuration, otherwise fallback to the default strategy strategy_name = config.get('strategy') or constants.DEFAULT_STRATEGY - self.strategy: IStrategy = self._load_strategy(strategy_name, extra_dir=config.get('strategy_path')) + self.strategy: IStrategy = self._load_strategy(strategy_name, + extra_dir=config.get('strategy_path')) # Set attributes # Check if we need to override configuration From b731a65c7502a2b317d5cedb3fd76e4bf1979470 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 2 Jun 2018 04:27:04 +0200 Subject: [PATCH 40/72] Update ccxt from 1.14.96 to 1.14.119 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7000f0eb1..ad64e58fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.96 +ccxt==1.14.119 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 792dd556a1b1faefabdaacdd5ec8e1d3ce63398e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 1 Jun 2018 19:37:39 -0700 Subject: [PATCH 41/72] Fix wrong hint '--update-pairs-cached' from Backtesting/Hyperopt --- freqtrade/optimize/__init__.py | 5 ++++- freqtrade/tests/optimize/test_optimize.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index f6f1ba47a..68ba5622e 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -103,7 +103,10 @@ def load_data(datadir: str, if pairdata: result[pair] = pairdata else: - logger.warn('No data for pair %s, use --update-pairs-cached to download the data', pair) + logger.warning( + 'No data for pair %s, use --refresh-pairs-cached to download the data', + pair + ) return result diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 8624b500d..765d88cd5 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -105,7 +105,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None: refresh_pairs=False, pairs=['MEME/BTC']) assert os.path.isfile(file) is False - assert log_has('No data for pair MEME/BTC, use --update-pairs-cached to download the data', + assert log_has('No data for pair MEME/BTC, use --refresh-pairs-cached to download the data', caplog.record_tuples) # download a new pair if refresh_pairs is set From a82a31341bc1d26f08023f3260471460cfa16620 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 2 Jun 2018 11:32:05 +0300 Subject: [PATCH 42/72] change misleading logging for datadir --- freqtrade/configuration.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 6 +++--- freqtrade/tests/test_configuration.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 43d0a0bf9..03a25efb3 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -145,7 +145,7 @@ class Configuration(object): # If --datadir is used we add it to the configuration if 'datadir' in self.args and self.args.datadir: config.update({'datadir': self.args.datadir}) - logger.info('Parameter --datadir detected: %s ...', self.args.datadir) + logger.info('Using data folder: %s ...', self.args.datadir) # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f17a0115e..bfb82f3ec 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -181,7 +181,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config @@ -229,7 +229,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config @@ -632,7 +632,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Parameter -l/--live detected ...', 'Using max_open_trades: 1 ...', 'Parameter --timerange detected: -100 ..', - 'Parameter --datadir detected: freqtrade/tests/testdata ...', + 'Using data folder: freqtrade/tests/testdata ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Downloading data for all pairs in whitelist ...', diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index d27405d91..dcf725e83 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -235,7 +235,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config @@ -286,7 +286,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config From 4a322abd4d0736502b78d68b5d1611cb9b402931 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:43:51 +0200 Subject: [PATCH 43/72] Typecheck improvements --- freqtrade/optimize/backtesting.py | 6 +++--- freqtrade/optimize/hyperopt.py | 5 +++-- freqtrade/rpc/rpc.py | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 376730d0f..5a3b790c9 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -78,7 +78,7 @@ class Backtesting(object): Generates and returns a text table for the given backtest data and the results dataframe :return: pretty printed table with tabulate as str """ - stake_currency = self.config.get('stake_currency') + stake_currency = str(self.config.get('stake_currency')) floatfmt = ('s', 'd', '.2f', '.8f', '.1f') tabular_data = [] @@ -168,7 +168,7 @@ class Backtesting(object): record = args.get('record', None) records = [] trades = [] - trade_count_lock = {} + trade_count_lock: Dict = {} for pair, pair_data in processed.items(): pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run @@ -230,7 +230,7 @@ class Backtesting(object): else: logger.info('Using local backtesting data (using whitelist in given config) ...') - timerange = Arguments.parse_timerange(self.config.get('timerange')) + timerange = Arguments.parse_timerange(str(self.config.get('timerange'))) data = optimize.load_data( self.config['datadir'], pairs=pairs, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b4f534d7c..f6cbb270b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -494,9 +494,10 @@ class Hyperopt(Backtesting): ) def start(self) -> None: - timerange = Arguments.parse_timerange(self.config.get('timerange')) + timerange = Arguments.parse_timerange(self.config.get('timerange') if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) data = load_data( - datadir=self.config.get('datadir'), + datadir=str(self.config.get('datadir')), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, timerange=timerange diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index df809cc2f..3e39657f2 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -2,7 +2,7 @@ This module contains class to define a RPC communications """ import logging -from datetime import datetime, timedelta +from datetime import datetime, timedelta, date from decimal import Decimal from typing import Dict, Tuple, Any, Optional @@ -114,7 +114,7 @@ class RPC(object): self, timescale: int, stake_currency: str, fiat_display_currency: str) -> Tuple[bool, Any]: today = datetime.utcnow().date() - profit_days: Dict[int, Dict] = {} + profit_days: Dict[date, Dict] = {} if not (isinstance(timescale, int) and timescale > 0): return True, '*Daily [n]:* `must be an integer greater than 0`' @@ -172,7 +172,7 @@ class RPC(object): durations = [] for trade in trades: - current_rate: Optional[float] = None + current_rate: float = 0.0 if not trade.open_rate: continue From 6106822d102ef696ef46831d1e5ef6bc8870a49e Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:44:41 +0200 Subject: [PATCH 44/72] typing --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7c955d423..50fe10667 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -33,7 +33,7 @@ class FreqtradeBot(object): This is from here the bot start its logic. """ - def __init__(self, config: Dict[str, Any], db_url: Optional[str] = None): + def __init__(self, config: Dict[str, Any], db_url: Optional[str] = None)-> None: """ Init all variables and object the bot need to work :param config: configuration dict, you can use the Configuration.get_config() @@ -93,7 +93,7 @@ class FreqtradeBot(object): persistence.cleanup() return True - def worker(self, old_state: None) -> State: + def worker(self, old_state: State = None) -> State: """ Trading routine that must be run at each loop :param old_state: the previous service state from the previous call From 6fc21e30e5388b67921a47fd389fc58764220dce Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:52:55 +0200 Subject: [PATCH 45/72] remove unused import --- freqtrade/rpc/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 3e39657f2..715d38f95 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -4,7 +4,7 @@ This module contains class to define a RPC communications import logging from datetime import datetime, timedelta, date from decimal import Decimal -from typing import Dict, Tuple, Any, Optional +from typing import Dict, Tuple, Any import arrow import sqlalchemy as sql From d9e951447fc5bc7a43ea1d00c08fe040df0f4763 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:54:22 +0200 Subject: [PATCH 46/72] remove _init function in backtesting (and according test) --- freqtrade/optimize/backtesting.py | 12 ------------ freqtrade/tests/optimize/test_backtesting.py | 17 ----------------- 2 files changed, 29 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5a3b790c9..0e77572a5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -33,18 +33,6 @@ class Backtesting(object): """ def __init__(self, config: Dict[str, Any]) -> None: self.config = config - self.analyze = None - self.ticker_interval = None - self.tickerdata_to_dataframe = None - self.populate_buy_trend = None - self.populate_sell_trend = None - self._init() - - def _init(self) -> None: - """ - Init objects required for backtesting - :return: None - """ self.analyze = Analyze(self.config) self.ticker_interval = self.analyze.strategy.ticker_interval self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f17a0115e..5cc41845d 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -286,23 +286,6 @@ def test_start(mocker, fee, default_conf, caplog) -> None: assert start_mock.call_count == 1 -def test_backtesting__init__(mocker, default_conf) -> None: - """ - Test Backtesting.__init__() method - """ - init_mock = MagicMock() - mocker.patch('freqtrade.optimize.backtesting.Backtesting._init', init_mock) - - backtesting = Backtesting(default_conf) - assert backtesting.config == default_conf - assert backtesting.analyze is None - assert backtesting.ticker_interval is None - assert backtesting.tickerdata_to_dataframe is None - assert backtesting.populate_buy_trend is None - assert backtesting.populate_sell_trend is None - assert init_mock.call_count == 1 - - def test_backtesting_init(mocker, default_conf) -> None: """ Test Backtesting._init() method From 32300f6d5fe46dce5b29f9a934f568e765942840 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:55:06 +0200 Subject: [PATCH 47/72] don't initialize with None where it's not necessary --- freqtrade/freqtradebot.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 50fe10667..41841e911 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -51,9 +51,9 @@ class FreqtradeBot(object): # Init objects self.config = config - self.analyze = None - self.fiat_converter = None - self.rpc = None + self.analyze = Analyze(self.config) + self.fiat_converter = CryptoToFiatConverter() + self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = None @@ -66,9 +66,6 @@ class FreqtradeBot(object): :return: None """ # Initialize all modules - self.analyze = Analyze(self.config) - self.fiat_converter = CryptoToFiatConverter() - self.rpc = RPCManager(self) persistence.init(self.config, db_url) exchange.init(self.config) From 0a595190a3e622915defb62153ec3886ce5c4c54 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 13:59:35 +0200 Subject: [PATCH 48/72] fix last typechecks --- freqtrade/arguments.py | 3 ++- freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/hyperopt.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 2421abc9f..fc917394e 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -211,7 +211,8 @@ class Arguments(object): self.hyperopt_options(hyperopt_cmd) @staticmethod - def parse_timerange(text: str) -> Optional[Tuple[Tuple, Optional[int], Optional[int]]]: + def parse_timerange(text: Optional[str]) -> Optional[Tuple[Tuple, + Optional[int], Optional[int]]]: """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0e77572a5..78ab72988 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -219,7 +219,7 @@ class Backtesting(object): logger.info('Using local backtesting data (using whitelist in given config) ...') timerange = Arguments.parse_timerange(str(self.config.get('timerange'))) - data = optimize.load_data( + data = optimize.load_data( # type: ignore self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f6cbb270b..8b8e9576b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -494,9 +494,9 @@ class Hyperopt(Backtesting): ) def start(self) -> None: - timerange = Arguments.parse_timerange(self.config.get('timerange') if self.config.get( + timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = load_data( + data = load_data( # type: ignore datadir=str(self.config.get('datadir')), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, From 0007002c80822cba5085655a10ca22dd1678e251 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 14:07:54 +0200 Subject: [PATCH 49/72] fix test failure --- freqtrade/optimize/backtesting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 78ab72988..ef69539db 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -218,7 +218,8 @@ class Backtesting(object): else: logger.info('Using local backtesting data (using whitelist in given config) ...') - timerange = Arguments.parse_timerange(str(self.config.get('timerange'))) + timerange = Arguments.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) data = optimize.load_data( # type: ignore self.config['datadir'], pairs=pairs, From 884395415fc4e46221f0b0bac205744c8f9a4f16 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 14:10:15 +0200 Subject: [PATCH 50/72] remove type:ignore --- freqtrade/analyze.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 0a560c86e..5756b845c 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -95,7 +95,7 @@ class Analyze(object): Return ticker interval to use :return: Ticker interval value to use """ - return self.strategy.ticker_interval # type: ignore + return self.strategy.ticker_interval def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ @@ -195,14 +195,13 @@ class Analyze(object): :return True if bot should sell at current rate """ current_profit = trade.calc_profit_percent(current_rate) - if self.strategy.stoploss is not None \ - and current_profit < self.strategy.stoploss: # type: ignore + if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: logger.debug('Stop loss hit.') return True # Check if time matches and current rate is above threshold time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 - for duration, threshold in self.strategy.minimal_roi.items(): # type: ignore + for duration, threshold in self.strategy.minimal_roi.items(): if time_diff <= duration: return False if current_profit > threshold: From 3447e4bb9729d8f2c43240cc27ba2947471e7464 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 14:13:17 +0200 Subject: [PATCH 51/72] comment on ignore hint --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8b8e9576b..2497d6752 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -496,7 +496,7 @@ class Hyperopt(Backtesting): def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = load_data( # type: ignore + data = load_data( # type: ignore # timerange will be refactored datadir=str(self.config.get('datadir')), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, From f88729f0e855ef167027f64b6b85f07c94169f2d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 14:14:28 +0200 Subject: [PATCH 52/72] add ignore comment --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ef69539db..a0aee78b1 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -220,7 +220,7 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = optimize.load_data( # type: ignore + data = optimize.load_data( # type: ignore # timerange will be refactored self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, From a8bf5092e86aaa84b28815ae3670ab852c037761 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 14:18:57 +0200 Subject: [PATCH 53/72] add ignore explanation --- freqtrade/strategy/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 60427bcf4..3fd39bca3 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -102,7 +102,7 @@ class StrategyResolver(object): # Generate spec based on absolute path spec = importlib.util.spec_from_file_location('user_data.strategies', module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) # type: ignore + spec.loader.exec_module(module) # type: ignore # importlib does not use typehints valid_strategies_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) From 6ca375a39783eea32aafe271c5672ea85215c11c Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Sat, 2 Jun 2018 19:45:08 +0300 Subject: [PATCH 54/72] Extend timerange to accept unix timestamps. This gives greater granularity over backtest, parsing tickerfiles. Example runs using date and unix time below. /usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/bin/python3.6 /Users/creslin/PycharmProjects/freqtrade/scripts/report_correlation.py --timerange=20180528-20180529 2018-06-02 18:44:58,829 - freqtrade.configuration - INFO - Log level set to INFO 2018-06-02 18:44:58,830 - freqtrade.configuration - INFO - Using max_open_trades: 200 ... 2018-06-02 18:44:58,831 - freqtrade.configuration - INFO - Parameter --timerange detected: 20180528-20180529 ... 2018-06-02 18:44:58,831 - freqtrade.configuration - INFO - Parameter --datadir detected: freqtrade/tests/testdata ... BasePair Pair Correlation BTC % Change Pair % USD Ch Pair % BTC Ch Gain % on BTC Start Stop BTC Volume 1 BTC_USDT ETC_USD 0.965 -2.942 -4.070 -1.163 -1.128585 05-28 00:00 05-29 00:00 335.19 0 BTC_USDT SNT_USD 0.943 -2.942 -5.857 -3.004 -2.915585 05-28 00:00 05-29 00:00 496.01 3 BTC_USDT DASH_USD 0.902 -2.942 -9.034 -6.277 -6.092432 05-28 00:00 05-29 00:00 751.41 2 BTC_USDT MTH_USD 0.954 -2.942 -9.290 -6.541 -6.348708 05-28 00:00 05-29 00:00 23.00 4 BTC_USDT TRX_USD 0.951 -2.942 -13.647 -11.029 -10.704957 05-28 00:00 05-29 00:00 14544.57 Process finished with exit code 0 /usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/bin/python3.6 /Users/creslin/PycharmProjects/freqtrade/scripts/report_correlation.py --timerange=1527595200-1527618600 2018-06-02 18:47:40,382 - freqtrade.configuration - INFO - Log level set to INFO 2018-06-02 18:47:40,382 - freqtrade.configuration - INFO - Using max_open_trades: 200 ... 2018-06-02 18:47:40,383 - freqtrade.configuration - INFO - Parameter --timerange detected: 1527595200-1527618600 ... 2018-06-02 18:47:40,383 - freqtrade.configuration - INFO - Parameter --datadir detected: freqtrade/tests/testdata ... BasePair Pair Correlation BTC % Change Pair % USD Ch Pair % BTC Ch Gain % on BTC Start Stop BTC Volume 0 BTC_USDT SNT_USD 0.680 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 68866.30 1 BTC_USDT ETC_USD 0.857 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 227514.17 2 BTC_USDT MTH_USD 0.790 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 12103.96 3 BTC_USDT DASH_USD 0.862 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 72982.78 4 BTC_USDT TRX_USD 0.178 NaN NaN NaN NaN 05-29 12:00 05-29 18:30 1258316.95 Process finished with exit code 0 --- freqtrade/arguments.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index afcb05511..00869f974 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -222,6 +222,9 @@ class Arguments(object): syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), + (r'^-(\d{10})$', (None, 'timestamp')), + (r'^(\d{10})-$', ('timestamp', None)), + (r'^(\d{10})-(\d{10})$', ('timestamp', 'timestamp')), (r'^(-\d+)$', (None, 'line')), (r'^(\d+)-$', ('line', None)), (r'^(\d+)-(\d+)$', ('index', 'index'))] @@ -237,6 +240,8 @@ class Arguments(object): start = rvals[index] if stype[0] == 'date': start = arrow.get(start, 'YYYYMMDD').timestamp + elif stype[0] == 'timestamp': + start = arrow.get(start).timestamp else: start = int(start) index += 1 @@ -244,6 +249,8 @@ class Arguments(object): stop = rvals[index] if stype[1] == 'date': stop = arrow.get(stop, 'YYYYMMDD').timestamp + elif stype[1] == 'timestamp': + stop = arrow.get(stop).timestamp else: stop = int(stop) return stype, start, stop From 9dbe5fdb852e1fc32bab6792bb37610fab508d04 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Sat, 2 Jun 2018 19:49:23 +0300 Subject: [PATCH 55/72] Update back testing document to include example using Posix timestamps as timerange e.g --timerange=1527595200-1527618600 --- docs/backtesting.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/backtesting.md b/docs/backtesting.md index df105bd77..8c4c4180d 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -83,6 +83,8 @@ The full timerange specification: - Use tickframes till 2018/01/31: `--timerange=-20180131` - Use tickframes since 2018/01/31: `--timerange=20180131-` - Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301` +- Use tickframes between POSIX timestamps 1527595200 1527618600: + `--timerange=1527595200-1527618600` **Update testdata directory** From 2791d543ea38cf15224c8bf8dc7ae5a7f8e0cbb0 Mon Sep 17 00:00:00 2001 From: Raymond Luo Date: Fri, 18 May 2018 19:02:38 +0800 Subject: [PATCH 56/72] Make backtesting report markdown shareable Small tweak to make the backtesting report markdown ready and much easier to share reports on many markdown publishing tools and editors that already support Markdown Extra with just a copy and paste Example: ![Example](https://i.imgur.com/HXlNkfm.png) --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 376730d0f..ab45b7754 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -106,7 +106,7 @@ class Backtesting(object): len(results[results.profit_BTC > 0]), len(results[results.profit_BTC < 0]) ]) - return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, From 9537f17dd41ea0f5ba6e87f06bc7cc32e0d7ccad Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 2 Jun 2018 20:06:29 +0200 Subject: [PATCH 57/72] Fix test --- freqtrade/tests/optimize/test_backtesting.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index bfb82f3ec..786568f98 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -374,16 +374,15 @@ def test_generate_text_table(default_conf, mocker): ) result_str = ( - 'pair buy count avg profit % ' - 'total profit BTC avg duration profit loss\n' - '------- ----------- -------------- ' - '------------------ -------------- -------- ------\n' - 'ETH/BTC 2 15.00 ' - '0.60000000 20.0 2 0\n' - 'TOTAL 2 15.00 ' - '0.60000000 20.0 2 0' + '| pair | buy count | avg profit % | ' + 'total profit BTC | avg duration | profit | loss |\n' + '|:--------|------------:|---------------:|' + '-------------------:|---------------:|---------:|-------:|\n' + '| ETH/BTC | 2 | 15.00 | ' + '0.60000000 | 20.0 | 2 | 0 |\n' + '| TOTAL | 2 | 15.00 | ' + '0.60000000 | 20.0 | 2 | 0 |' ) - assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str From 43ba02afc63e106925b47d3ee9e25a87809f02ae Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Sat, 2 Jun 2018 21:59:18 +0300 Subject: [PATCH 58/72] Per feed back, kept the stype as date. Use a tuple to keep as epoch int or process via arrow to timestamp. I'll look at the test file also. --- freqtrade/arguments.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 00869f974..b61324ccc 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -222,9 +222,9 @@ class Arguments(object): syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), - (r'^-(\d{10})$', (None, 'timestamp')), - (r'^(\d{10})-$', ('timestamp', None)), - (r'^(\d{10})-(\d{10})$', ('timestamp', 'timestamp')), + (r'^-(\d{10})$', (None, 'date')), + (r'^(\d{10})-$', ('date', None)), + (r'^(\d{10})-(\d{10})$', ('date', 'date')), (r'^(-\d+)$', (None, 'line')), (r'^(\d+)-$', ('line', None)), (r'^(\d+)-(\d+)$', ('index', 'index'))] @@ -239,18 +239,16 @@ class Arguments(object): if stype[0]: start = rvals[index] if stype[0] == 'date': - start = arrow.get(start, 'YYYYMMDD').timestamp - elif stype[0] == 'timestamp': - start = arrow.get(start).timestamp + start = int(start) if len(start) == 10 \ + else arrow.get(start, 'YYYYMMDD').timestamp else: start = int(start) index += 1 if stype[1]: stop = rvals[index] if stype[1] == 'date': - stop = arrow.get(stop, 'YYYYMMDD').timestamp - elif stype[1] == 'timestamp': - stop = arrow.get(stop).timestamp + stop = int(stop) if len(stop) == 10 \ + else arrow.get(stop, 'YYYYMMDD').timestamp else: stop = int(stop) return stype, start, stop From dc65753a641a4c0de2bf9052eddc3f26e7470e67 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 2 Jun 2018 12:35:07 -0700 Subject: [PATCH 59/72] Fix the in-progress dot that does not show up during a Hyperopt run --- freqtrade/optimize/hyperopt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 20fa5380d..3317a6c6f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -455,6 +455,7 @@ class Hyperopt(Backtesting): if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: print('.', end='') + sys.stdout.flush() return { 'status': STATUS_FAIL, 'loss': float('inf') From 94e586c049211e2267726e3b0509ece14b95bef3 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 2 Jun 2018 22:46:54 +0300 Subject: [PATCH 60/72] Added unit test to check posix time arguments passed to timerange Here is the pass report: freqtrade_new creslin$ pytest freqtrade/tests/test_arguments.py ==================================================================== test session starts ===================================================================== platform darwin -- Python 3.6.5, pytest-3.6.0, py-1.5.3, pluggy-0.6.0 rootdir: /Users/creslin/PycharmProjects/freqtrade_new, inifile: plugins: mock-1.10.0, cov-2.5.1 collected 19 items freqtrade/tests/test_arguments.py ................... [100%] ================================================================= 19 passed in 2.37 seconds ================================================================== --- freqtrade/tests/test_arguments.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 279ace0dc..474aa2507 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -116,6 +116,12 @@ def test_parse_timerange_incorrect() -> None: timerange = Arguments.parse_timerange('20100522-20150730') assert timerange == (('date', 'date'), 1274486400, 1438214400) + # Added test for unix timestamp - BTC genesis date + assert (('date', None), 1231006505, None) == Arguments.parse_timerange('1231006505-') + assert ((None, 'date'), None, 1233360000) == Arguments.parse_timerange('-1233360000') + timerange = Arguments.parse_timerange('1231006505-1233360000') + assert timerange == (('date', 'date'), 1231006505, 1233360000) + with pytest.raises(Exception, match=r'Incorrect syntax.*'): Arguments.parse_timerange('-') From 127cf5d6192ce3c277798c6982887f688f9c55ac Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 2 Jun 2018 13:55:05 -0700 Subject: [PATCH 61/72] Backtesting: Add the Interval required when data is missing Change the message: "No data for pair ETH/BTC, use --refresh-pairs-cached to download the data" for: "No data for pair: "ETH/BTC", Interval: 5m. Use --refresh-pairs-cached to download the data" The message structure is unified with the download message: "Download the pair: "ETH/BTC", Interval: 5m" --- freqtrade/optimize/__init__.py | 6 ++++-- freqtrade/tests/optimize/test_optimize.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 68ba5622e..19e235a7a 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -104,8 +104,10 @@ def load_data(datadir: str, result[pair] = pairdata else: logger.warning( - 'No data for pair %s, use --refresh-pairs-cached to download the data', - pair + 'No data for pair: "%s", Interval: %s. ' + 'Use --refresh-pairs-cached to download the data', + pair, + ticker_interval ) return result diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 765d88cd5..349fa3be3 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -105,7 +105,8 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None: refresh_pairs=False, pairs=['MEME/BTC']) assert os.path.isfile(file) is False - assert log_has('No data for pair MEME/BTC, use --refresh-pairs-cached to download the data', + assert log_has('No data for pair: "MEME/BTC", Interval: 1m. ' + 'Use --refresh-pairs-cached to download the data', caplog.record_tuples) # download a new pair if refresh_pairs is set From fe8ff1b929ed361197e459d2ff8568c9d6c2f7ac Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 2 Jun 2018 14:07:31 -0700 Subject: [PATCH 62/72] Fix stake_currency return by Hyperopt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hyperopt had BTC hard coded in the result. This commit will display the real stake_currency used. If you used `"stake_currency": "USDT",` in your config file. Before this commit you saw a message like: "2 trades. Avg profit 0.13%. Total profit 0.00002651 BTC (0.0027Σ%). Avg duration 142.5 mins." Now with the commit, we fix the wrong BTC currency: "2 trades. Avg profit 0.13%. Total profit 0.00002651 USDT (0.0027Σ%). Avg duration 142.5 mins." --- freqtrade/optimize/hyperopt.py | 6 +++--- freqtrade/tests/optimize/test_hyperopt.py | 21 ++++++++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 20fa5380d..111da9b5c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -479,16 +479,16 @@ class Hyperopt(Backtesting): 'result': result_explanation, } - @staticmethod - def format_results(results: DataFrame) -> str: + def format_results(self, results: DataFrame) -> str: """ Return the format result in a string """ return ('{:6d} trades. Avg profit {: 5.2f}%. ' - 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( + 'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( len(results.index), results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), + self.config['stake_currency'], results.profit_percent.sum(), results.duration.mean(), ) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index f8fa66b2e..3edfe4393 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -389,10 +389,12 @@ def test_start_uses_mongotrials(mocker, init_hyperopt, default_conf) -> None: # test buy_strategy_generator def populate_buy_trend # test optimizer if 'ro_t1' in params -def test_format_results(): +def test_format_results(init_hyperopt): """ Test Hyperopt.format_results() """ + + # Test with BTC as stake_currency trades = [ ('ETH/BTC', 2, 2, 123), ('LTC/BTC', 1, 1, 123), @@ -400,8 +402,21 @@ def test_format_results(): ] labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] df = pd.DataFrame.from_records(trades, columns=labels) - x = Hyperopt.format_results(df) - assert x.find(' 66.67%') + + result = _HYPEROPT.format_results(df) + assert result.find(' 66.67%') + assert result.find('Total profit 1.00000000 BTC') + assert result.find('2.0000Σ %') + + # Test with EUR as stake_currency + trades = [ + ('ETH/EUR', 2, 2, 123), + ('LTC/EUR', 1, 1, 123), + ('XPR/EUR', -1, -2, -246) + ] + df = pd.DataFrame.from_records(trades, columns=labels) + result = _HYPEROPT.format_results(df) + assert result.find('Total profit 1.00000000 EUR') def test_signal_handler(mocker, init_hyperopt): From acbfe91f131ffa5854436bff275d92d2fce2eb21 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Thu, 31 May 2018 23:44:17 -0700 Subject: [PATCH 63/72] Allow EUR / USD as stake_currency It will enable to trade with FIAT on exhanges like GDAX or Kraken. --- freqtrade/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 7731ea610..82794774b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -31,7 +31,7 @@ CONF_SCHEMA = { 'properties': { 'max_open_trades': {'type': 'integer', 'minimum': 0}, 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, - 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, + 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT', 'EUR', 'USD']}, 'stake_amount': {'type': 'number', 'minimum': 0.0005}, 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', 'CLP', 'CNY', 'CZK', 'DKK', From c9e49ed7b40a541eb9d2e385d586c73eb6f22b3c Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Thu, 31 May 2018 23:45:35 -0700 Subject: [PATCH 64/72] Sort ticker_history CCXT does not sort the ticker history from exchanges. Bittrex and Binance are sorted ASC (oldest first, newest last) when GDAX is sorted DESC (newest first, oldest last). Because of that the get_ticker_history() fall in a very long loop when the tickers are sorted DESC. Means it downloads more than needed. This commit enable exhanges like GDAX and unify the ticker_history list across all exchanges. --- freqtrade/exchange/__init__.py | 5 ++ freqtrade/tests/exchange/test_exchange.py | 72 +++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 109e3f7b2..7be323329 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -294,6 +294,11 @@ def get_ticker_history(pair: str, tick_interval: str, since_ms: Optional[int] = while not since_ms or since_ms < till_time_ms: data_part = _API.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) + # Because some exchange sort Tickers ASC and other DESC. + # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) + # when GDAX returns a list of tickers DESC (newest first, oldest last) + data_part = sorted(data_part, key=lambda x: x[0]) + if not data_part: break diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index a506a8416..56812c75e 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -393,6 +393,78 @@ def test_get_ticker_history(default_conf, mocker): get_ticker_history('EFGH/BTC', default_conf['ticker_interval']) +def test_get_ticker_history_sort(default_conf, mocker): + api_mock = MagicMock() + + # GDAX use-case (real data from GDAX) + # This ticker history is ordered DESC (newest first, oldest last) + tick = [ + [1527833100000, 0.07666, 0.07671, 0.07666, 0.07668, 16.65244264], + [1527832800000, 0.07662, 0.07666, 0.07662, 0.07666, 1.30051526], + [1527832500000, 0.07656, 0.07661, 0.07656, 0.07661, 12.034778840000001], + [1527832200000, 0.07658, 0.07658, 0.07655, 0.07656, 0.59780186], + [1527831900000, 0.07658, 0.07658, 0.07658, 0.07658, 1.76278136], + [1527831600000, 0.07658, 0.07658, 0.07658, 0.07658, 2.22646521], + [1527831300000, 0.07655, 0.07657, 0.07655, 0.07657, 1.1753], + [1527831000000, 0.07654, 0.07654, 0.07651, 0.07651, 0.8073060299999999], + [1527830700000, 0.07652, 0.07652, 0.07651, 0.07652, 10.04822687], + [1527830400000, 0.07649, 0.07651, 0.07649, 0.07651, 2.5734867] + ] + type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) + api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) + mocker.patch('freqtrade.exchange._API', api_mock) + + # Test the ticker history sort + ticks = get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + assert ticks[0][0] == 1527830400000 + assert ticks[0][1] == 0.07649 + assert ticks[0][2] == 0.07651 + assert ticks[0][3] == 0.07649 + assert ticks[0][4] == 0.07651 + assert ticks[0][5] == 2.5734867 + + assert ticks[9][0] == 1527833100000 + assert ticks[9][1] == 0.07666 + assert ticks[9][2] == 0.07671 + assert ticks[9][3] == 0.07666 + assert ticks[9][4] == 0.07668 + assert ticks[9][5] == 16.65244264 + + # Bittrex use-case (real data from Bittrex) + # This ticker history is ordered ASC (oldest first, newest last) + tick = [ + [1527827700000, 0.07659999, 0.0766, 0.07627, 0.07657998, 1.85216924], + [1527828000000, 0.07657995, 0.07657995, 0.0763, 0.0763, 26.04051037], + [1527828300000, 0.0763, 0.07659998, 0.0763, 0.0764, 10.36434124], + [1527828600000, 0.0764, 0.0766, 0.0764, 0.0766, 5.71044773], + [1527828900000, 0.0764, 0.07666998, 0.0764, 0.07666998, 47.48888565], + [1527829200000, 0.0765, 0.07672999, 0.0765, 0.07672999, 3.37640326], + [1527829500000, 0.0766, 0.07675, 0.0765, 0.07675, 8.36203831], + [1527829800000, 0.07675, 0.07677999, 0.07620002, 0.076695, 119.22963884], + [1527830100000, 0.076695, 0.07671, 0.07624171, 0.07671, 1.80689244], + [1527830400000, 0.07671, 0.07674399, 0.07629216, 0.07655213, 2.31452783] + ] + type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) + api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) + mocker.patch('freqtrade.exchange._API', api_mock) + + # Test the ticker history sort + ticks = get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + assert ticks[0][0] == 1527827700000 + assert ticks[0][1] == 0.07659999 + assert ticks[0][2] == 0.0766 + assert ticks[0][3] == 0.07627 + assert ticks[0][4] == 0.07657998 + assert ticks[0][5] == 1.85216924 + + assert ticks[9][0] == 1527830400000 + assert ticks[9][1] == 0.07671 + assert ticks[9][2] == 0.07674399 + assert ticks[9][3] == 0.07629216 + assert ticks[9][4] == 0.07655213 + assert ticks[9][5] == 2.31452783 + + def test_cancel_order_dry_run(default_conf, mocker): default_conf['dry_run'] = True mocker.patch.dict('freqtrade.exchange._CONF', default_conf) From 638d98735f237b94ecca0e7c999284c6aee85d3e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 1 Jun 2018 20:58:07 -0700 Subject: [PATCH 65/72] Allow fiat_convert to use same symbol for Crypto and FIAT --- freqtrade/fiat_convert.py | 15 +++++++++++---- freqtrade/tests/test_fiat_convert.py | 7 +++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 17882f51a..7e61a3f2f 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -95,8 +95,11 @@ class CryptoToFiatConverter(object): coinlistings = self._coinmarketcap.listings() self._cryptomap = dict(map(lambda coin: (coin["symbol"], str(coin["id"])), coinlistings["data"])) - except (ValueError, RequestException) as e: - logger.error("Could not load FIAT Cryptocurrency map for the following problem: %s", e) + except (ValueError, RequestException) as exception: + logger.error( + "Could not load FIAT Cryptocurrency map for the following problem: %s", + exception + ) def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float: """ @@ -188,6 +191,10 @@ class CryptoToFiatConverter(object): if not self._is_supported_fiat(fiat=fiat_symbol): raise ValueError('The fiat {} is not supported.'.format(fiat_symbol)) + # No need to convert if both crypto and fiat are the same + if crypto_symbol == fiat_symbol: + return 1.0 + if crypto_symbol not in self._cryptomap: # return 0 for unsupported stake currencies (fiat-convert should not break the bot) logger.warning("unsupported crypto-symbol %s - returning 0.0", crypto_symbol) @@ -199,6 +206,6 @@ class CryptoToFiatConverter(object): convert=fiat_symbol )['data']['quotes'][fiat_symbol.upper()]['price'] ) - except BaseException as ex: - logger.error("Error in _find_price: %s", ex) + except BaseException as exception: + logger.error("Error in _find_price: %s", exception) return 0.0 diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index b37ca0f5c..2da427d27 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -126,6 +126,13 @@ def test_fiat_convert_get_price(mocker): assert fiat_convert._pairs[0]._expiration is not expiration +def test_fiat_convert_same_currencies(mocker): + patch_coinmarketcap(mocker) + fiat_convert = CryptoToFiatConverter() + + assert fiat_convert.get_price(crypto_symbol='USD', fiat_symbol='USD') == 1.0 + + def test_loadcryptomap(mocker): patch_coinmarketcap(mocker) From e8a59f4c20f135cb3b977324c574f694d27a1ac0 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 2 Jun 2018 13:26:24 -0700 Subject: [PATCH 66/72] Add a test to check the behavior when converting two FIAT --- freqtrade/tests/test_fiat_convert.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 2da427d27..faf7462c0 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -133,6 +133,13 @@ def test_fiat_convert_same_currencies(mocker): assert fiat_convert.get_price(crypto_symbol='USD', fiat_symbol='USD') == 1.0 +def test_fiat_convert_two_FIAT(mocker): + patch_coinmarketcap(mocker) + fiat_convert = CryptoToFiatConverter() + + assert fiat_convert.get_price(crypto_symbol='USD', fiat_symbol='EUR') == 0.0 + + def test_loadcryptomap(mocker): patch_coinmarketcap(mocker) From cfb06ceb58243618a52a00035cdaacd7c9e857be Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 3 Jun 2018 10:12:07 +0200 Subject: [PATCH 67/72] Update ccxt from 1.14.119 to 1.14.120 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ad64e58fd..d96084348 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.119 +ccxt==1.14.120 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 3a158faa30cdfb30d768d6b1e38644f8c46d69fe Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 3 Jun 2018 13:47:36 +0200 Subject: [PATCH 68/72] Refactor fiat-list to constants --- freqtrade/constants.py | 15 +++++++-------- freqtrade/fiat_convert.py | 11 ++--------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 7731ea610..f1cb03d26 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -24,6 +24,12 @@ TICKER_INTERVAL_MINUTES = { '1w': 10080, } +SUPPORTED_FIAT = [ + "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", + "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", + "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", + "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD" + ] # Required json-schema for user specified config CONF_SCHEMA = { @@ -33,14 +39,7 @@ CONF_SCHEMA = { 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, 'stake_amount': {'type': 'number', 'minimum': 0.0005}, - 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', - 'CLP', 'CNY', 'CZK', 'DKK', - 'EUR', 'GBP', 'HKD', 'HUF', - 'IDR', 'ILS', 'INR', 'JPY', - 'KRW', 'MXN', 'MYR', 'NOK', - 'NZD', 'PHP', 'PKR', 'PLN', - 'RUB', 'SEK', 'SGD', 'THB', - 'TRY', 'TWD', 'ZAR', 'USD']}, + 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, 'dry_run': {'type': 'boolean'}, 'minimal_roi': { 'type': 'object', diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 88eb702c9..a653ad76e 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -9,6 +9,7 @@ from typing import Dict, List from coinmarketcap import Market from requests.exceptions import RequestException +from freqtrade.constants import SUPPORTED_FIAT logger = logging.getLogger(__name__) @@ -67,14 +68,6 @@ class CryptoToFiatConverter(object): __instance = None _coinmarketcap: Market = None - # Constants - SUPPORTED_FIAT = [ - "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", - "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", - "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", - "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD" - ] - _cryptomap: Dict = {} def __new__(cls): @@ -175,7 +168,7 @@ class CryptoToFiatConverter(object): fiat = fiat.upper() - return fiat in self.SUPPORTED_FIAT + return fiat in SUPPORTED_FIAT def _find_price(self, crypto_symbol: str, fiat_symbol: str) -> float: """ From 0f352a4b5cb0f41afa948a8883b437af08b2eaf8 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 3 Jun 2018 15:14:51 +0200 Subject: [PATCH 69/72] update contributing document to include mypy --- CONTRIBUTING.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 967f57b65..93089495b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,4 +42,16 @@ pip3.6 install flake8 coveralls flake8 freqtrade ``` +## 3. Test if all type-hints are correct +**Install packages** (If not already installed) + +``` bash +pip3.6 install mypy +``` + +**Run mypy** + +``` bash +mypy freqtrade +``` From d3d62e90d321ab02673c79309f458cb49b497728 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 3 Jun 2018 08:36:01 -0700 Subject: [PATCH 70/72] Update Backtesting/Hyperopt usage documentation --- docs/bot-usage.md | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index b42df3ba3..e2c18473c 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -118,21 +118,25 @@ python3 ./freqtrade/main.py -c config.json --dry-run-db Backtesting also uses the config specified via `-c/--config`. ``` -usage: freqtrade backtesting [-h] [-l] [-i INT] [--realistic-simulation] - [-r] +usage: main.py backtesting [-h] [-i TICKER_INTERVAL] [--realistic-simulation] + [--timerange TIMERANGE] [-l] [-r] [--export EXPORT] optional arguments: -h, --help show this help message and exit - -l, --live using live data - -i INT, --ticker-interval INT - specify ticker interval (default: '5m') + -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL + specify ticker interval (1m, 5m, 30m, 1h, 1d) --realistic-simulation uses max_open_trades from config to simulate real world limitations + --timerange TIMERANGE + specify what timerange of data to use. + -l, --live using live data -r, --refresh-pairs-cached - refresh the pairs files in tests/testdata with - the latest data from the exchange. Use it if you want - to run your backtesting with up-to-date data. + refresh the pairs files in tests/testdata with the + latest data from the exchange. Use it if you want to + run your backtesting with up-to-date data. + --export EXPORT export backtest results, argument are: trades Example + --export=trades ``` ### How to use --refresh-pairs-cached parameter? @@ -155,14 +159,25 @@ Hyperopt uses an internal json config return by `hyperopt_optimize_conf()` located in `freqtrade/optimize/hyperopt_conf.py`. ``` -usage: freqtrade hyperopt [-h] [-e INT] [--use-mongodb] +usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--realistic-simulation] + [--timerange TIMERANGE] [-e INT] [--use-mongodb] + [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] optional arguments: -h, --help show this help message and exit + -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL + specify ticker interval (1m, 5m, 30m, 1h, 1d) + --realistic-simulation + uses max_open_trades from config to simulate real + world limitations + --timerange TIMERANGE + specify what timerange of data to use. -e INT, --epochs INT specify number of epochs (default: 100) --use-mongodb parallelize evaluations with mongodb (requires mongod in PATH) - + -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] + Specify which parameters to hyperopt. Space separate + list. Default: all ``` ## A parameter missing in the configuration? From 43696eff5cfc950074ac221ffca7c3e12d0548f5 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sun, 3 Jun 2018 08:57:13 -0700 Subject: [PATCH 71/72] Add __main__.py to improve how to launch the bot --- freqtrade/__main__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 freqtrade/__main__.py diff --git a/freqtrade/__main__.py b/freqtrade/__main__.py new file mode 100644 index 000000000..fe1318a35 --- /dev/null +++ b/freqtrade/__main__.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +""" +__main__.py for Freqtrade +To launch Freqtrade as a module + +> python -m freqtrade (with Python >= 3.6) +""" + +import sys +from freqtrade import main + + +if __name__ == '__main__': + main.set_loggers() + main.main(sys.argv[1:]) From 4eb82959554fba43d3c1ce0362e8efe895304f46 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 3 Jun 2018 19:27:08 +0200 Subject: [PATCH 72/72] Update ccxt from 1.14.120 to 1.14.121 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d96084348..6e7550515 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.120 +ccxt==1.14.121 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1