From dd2ccea6e5794d1b3bbc7a29d296b01e462f6279 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 8 Jan 2018 21:59:46 +0200 Subject: [PATCH 01/33] fix wrong range in stoploss search space --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7ef537ba1..cf46b96ad 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -95,7 +95,7 @@ SPACE = { {'type': 'stochf_cross'}, {'type': 'ht_sine'}, ]), - 'stoploss': hp.quniform('stoploss', -30, -2, 1), + 'stoploss': hp.uniform('stoploss', -0.5, -0.02), } From f7dd5e6396ba1a127232af11ad9fd574e91816ae Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 8 Jan 2018 22:00:10 +0200 Subject: [PATCH 02/33] use sensible value for stoploss in test --- freqtrade/tests/optimize/test_hyperopt.py | 30 +++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 02cf31ec3..a309af7fe 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -87,20 +87,20 @@ def test_no_log_if_loss_does_not_improve(mocker): def test_fmin_best_results(mocker, caplog): fmin_result = { - "adx": 1, - "adx-value": 15.0, - "fastd": 1, - "fastd-value": 40.0, - "green_candle": 1, - "mfi": 0, - "over_sar": 0, - "rsi": 1, - "rsi-value": 37.0, - "trigger": 2, - "uptrend_long_ema": 1, - "uptrend_short_ema": 0, - "uptrend_sma": 0, - "stoploss": -10, + "adx": 1, + "adx-value": 15.0, + "fastd": 1, + "fastd-value": 40.0, + "green_candle": 1, + "mfi": 0, + "over_sar": 0, + "rsi": 1, + "rsi-value": 37.0, + "trigger": 2, + "uptrend_long_ema": 1, + "uptrend_short_ema": 0, + "uptrend_sma": 0, + "stoploss": -0.1, } mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', return_value=create_trials(mocker)) @@ -117,7 +117,7 @@ def test_fmin_best_results(mocker, caplog): '"green_candle": {\n "enabled": true\n },', '"mfi": {\n "enabled": false\n },', '"trigger": {\n "type": "ao_cross_zero"\n },', - '"stoploss": -10.0', + '"stoploss": -0.1', ] for line in exists: From fbf9bfe8972a75b66770d136629e1686831947e7 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 9 Jan 2018 07:24:00 +0200 Subject: [PATCH 03/33] Update installation.md it seems that ta-lib requires python3.6-dev package to be installed --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 9ea6265ba..585ffeb24 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -139,7 +139,7 @@ This bot require Python 3.6 and TA-LIB ```bash sudo add-apt-repository ppa:jonathonf/python-3.6 sudo apt-get update -sudo apt-get install python3.6 python3.6-venv build-essential autoconf libtool pkg-config make wget git +sudo apt-get install python3.6 python3.6-venv python3.6-dev build-essential autoconf libtool pkg-config make wget git ``` **2.1.2. Install TA-LIB** From ca8cab0ce9e59abd4dc3aa9b02a640a035e01f19 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 6 Jan 2018 11:44:41 +0200 Subject: [PATCH 04/33] Hyperopt to handle SIGINT by saving/reading the trials file --- .gitignore | 2 ++ freqtrade/optimize/hyperopt.py | 47 +++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 672dd1f6d..c81b55222 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,5 @@ target/ .venv .idea .vscode + +hyperopt_trials.pickle diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index cf46b96ad..febaa196d 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,6 +4,9 @@ import json import logging import sys +import pickle +import signal +import os from functools import reduce from math import exp from operator import itemgetter @@ -43,6 +46,10 @@ EXPECTED_MAX_PROFIT = 3.85 PROCESSED = None # optimize.preprocess(optimize.load_data()) OPTIMIZE_CONFIG = hyperopt_optimize_conf() +# Hyperopt Trials +TRIALS_FILE = 'freqtrade/optimize/hyperopt_trials.pickle' +TRIALS = Trials() + # Monkey patch config from freqtrade import main # noqa main._CONF = OPTIMIZE_CONFIG @@ -99,6 +106,19 @@ SPACE = { } +def save_trials(trials, trials_path=TRIALS_FILE): + "Save hyperopt trials to file" + logger.info('Saving Trials to \'{}\''.format(trials_path)) + pickle.dump(trials, open(trials_path, 'wb')) + + +def read_trials(trials_path=TRIALS_FILE): + "Read hyperopt trials file" + logger.info('Reading Trials from \'{}\''.format(trials_path)) + trials = pickle.load(open(trials_path, 'rb')) + # os.remove(trials_path) + return trials + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -216,7 +236,12 @@ def buy_strategy_generator(params): def start(args): +<<<<<<< HEAD global TOTAL_TRIES, PROCESSED, SPACE +======= + global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES + +>>>>>>> Hyperopt to handle SIGINT by saving/reading the trials file TOTAL_TRIES = args.epochs exchange._API = Bittrex({'key': '', 'secret': ''}) @@ -238,9 +263,14 @@ def start(args): logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!') db_name = 'freqtrade_hyperopt' - trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') + TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') else: - trials = Trials() + logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, signal_handler) + # read trials file if we have one + if os.path.exists(TRIALS_FILE): + TRIALS = read_trials() + _CURRENT_TRIES = len([result for result in TRIALS.results if result['status'] == 'ok']) try: best_parameters = fmin( @@ -248,7 +278,7 @@ def start(args): space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, - trials=trials + trials=TRIALS ) results = sorted(trials.results, key=itemgetter('loss')) @@ -265,3 +295,14 @@ def start(args): logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) logger.info('Best Result:\n%s', best_result) + + # Store trials result to file to resume next time + save_trials(TRIALS) + + +def signal_handler(sig, frame): + "Hyperopt SIGINT handler" + logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) + + save_trials(TRIALS) + sys.exit(0) From a48840509b4d168e44750e96bf6277dbbe53bb5d Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 6 Jan 2018 15:34:21 +0200 Subject: [PATCH 05/33] Hyperopt: use results from previous runs --- freqtrade/optimize/hyperopt.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index febaa196d..c520ed81d 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -119,6 +119,7 @@ def read_trials(trials_path=TRIALS_FILE): # os.remove(trials_path) return trials + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -270,7 +271,11 @@ def start(args): # read trials file if we have one if os.path.exists(TRIALS_FILE): TRIALS = read_trials() - _CURRENT_TRIES = len([result for result in TRIALS.results if result['status'] == 'ok']) + _CURRENT_TRIES = len(TRIALS.results) + TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES + logger.info( + 'Continuing with trials. Current: {}, Total: {}' + .format(_CURRENT_TRIES, TOTAL_TRIES)) try: best_parameters = fmin( From b35fa4c9f6ba29c4c2333251df77276301cf449c Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 6 Jan 2018 15:53:22 +0200 Subject: [PATCH 06/33] hyperopt: show the best results so far --- freqtrade/optimize/hyperopt.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c520ed81d..dc611fcbc 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -120,6 +120,11 @@ def read_trials(trials_path=TRIALS_FILE): return trials +def log_trials_result(trials): + vals = trials.best_trial['misc']['vals'] + results = trials.best_trial['result']['result'] + logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -310,4 +315,5 @@ def signal_handler(sig, frame): logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) save_trials(TRIALS) + log_trials_result(TRIALS) sys.exit(0) From 1647e7a0c1357fc663d52386381563687f45554c Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 9 Jan 2018 11:37:27 +0200 Subject: [PATCH 07/33] update fix failing tests, unitest that resume hyperopt functionality works --- freqtrade/optimize/hyperopt.py | 12 ++--- freqtrade/tests/optimize/test_hyperopt.py | 66 +++++++++++++++++++++-- 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index dc611fcbc..4162e0758 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data TARGET_TRADES = 1100 -TOTAL_TRIES = None +TOTAL_TRIES = 0 _CURRENT_TRIES = 0 CURRENT_BEST_LOSS = 100 @@ -116,7 +116,7 @@ def read_trials(trials_path=TRIALS_FILE): "Read hyperopt trials file" logger.info('Reading Trials from \'{}\''.format(trials_path)) trials = pickle.load(open(trials_path, 'rb')) - # os.remove(trials_path) + os.remove(trials_path) return trials @@ -125,6 +125,7 @@ def log_trials_result(trials): results = trials.best_trial['result']['result'] logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -242,12 +243,8 @@ def buy_strategy_generator(params): def start(args): -<<<<<<< HEAD - global TOTAL_TRIES, PROCESSED, SPACE -======= - global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES + global TOTAL_TRIES, PROCESSED, SPACE, TRIALS, _CURRENT_TRIES ->>>>>>> Hyperopt to handle SIGINT by saving/reading the trials file TOTAL_TRIES = args.epochs exchange._API = Bittrex({'key': '', 'secret': ''}) @@ -276,6 +273,7 @@ def start(args): # read trials file if we have one if os.path.exists(TRIALS_FILE): TRIALS = read_trials() + _CURRENT_TRIES = len(TRIALS.results) TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES logger.info( diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index a309af7fe..99a1f4a3c 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,5 +1,8 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 - +import pickle +import os +import pytest +import freqtrade.optimize.hyperopt from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ log_results @@ -27,16 +30,35 @@ def test_loss_calculation_has_limited_profit(): def create_trials(mocker): + """ + When creating trials, mock the hyperopt Trials so that *by default* + - we don't create any pickle'd files in the filesystem + - we might have a pickle'd file so make sure that we return + false when looking for it + """ + mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', + return_value='freqtrade/tests/optimize/ut_trials.pickle') + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', + return_value=False) + mocker.patch('freqtrade.optimize.hyperopt.save_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.read_trials', + return_value=None) return mocker.Mock( results=[{ 'loss': 1, - 'result': 'foo' - }] + 'result': 'foo', + 'status': 'ok' + }], + best_trial={'misc': {'vals': {'adx': 999}}} ) def test_start_calls_fmin(mocker): - mocker.patch('freqtrade.optimize.hyperopt.Trials', return_value=create_trials(mocker)) + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.TRIALS', return_value=trials) + mocker.patch('freqtrade.optimize.hyperopt.sorted', + return_value=trials.results) mocker.patch('freqtrade.optimize.preprocess') mocker.patch('freqtrade.optimize.load_data') mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) @@ -141,3 +163,39 @@ def test_fmin_throw_value_error(mocker, caplog): for line in exists: assert line in caplog.text + + +def test_resuming_previous_hyperopt_results_succeeds(mocker): + import freqtrade.optimize.hyperopt as hyperopt + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.TRIALS', + return_value=trials) + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', + return_value=True) + mocker.patch('freqtrade.optimize.hyperopt.len', + return_value=len(trials.results)) + mock_read = mocker.patch('freqtrade.optimize.hyperopt.read_trials', + return_value=trials) + mock_save = mocker.patch('freqtrade.optimize.hyperopt.save_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.sorted', + return_value=trials.results) + mocker.patch('freqtrade.optimize.preprocess') + mocker.patch('freqtrade.optimize.load_data') + mocker.patch('freqtrade.optimize.hyperopt.fmin', + return_value={}) + args = mocker.Mock(epochs=1, + config='config.json.example', + mongodb=False) + + start(args) + + mock_read.assert_called_once() + mock_save.assert_called_once() + + current_tries = hyperopt._CURRENT_TRIES + total_tries = hyperopt.TOTAL_TRIES + + assert current_tries == len(trials.results) + assert total_tries == (current_tries + len(trials.results)) + From fe2b0c28621c53ded0ebe85ae1c4eb59c2661236 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 9 Jan 2018 12:19:44 +0200 Subject: [PATCH 08/33] add unittest to save and read trials file --- freqtrade/tests/optimize/test_hyperopt.py | 32 +++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 99a1f4a3c..3e03d26c0 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,10 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 -import pickle -import os -import pytest -import freqtrade.optimize.hyperopt from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ - log_results + log_results, save_trials, read_trials def test_loss_calculation_prefer_correct_trade_count(): @@ -44,6 +40,8 @@ def create_trials(mocker): return_value=None) mocker.patch('freqtrade.optimize.hyperopt.read_trials', return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.os.remove', + return_value=True) return mocker.Mock( results=[{ 'loss': 1, @@ -199,3 +197,27 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker): assert current_tries == len(trials.results) assert total_tries == (current_tries + len(trials.results)) + +def test_save_trials_saves_trials(mocker): + trials = create_trials(mocker) + mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', + return_value=None) + trials_path = mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', + return_value='ut_trials.pickle') + mocker.patch('freqtrade.optimize.hyperopt.open', + return_value=trials_path) + save_trials(trials, trials_path) + + mock_dump.assert_called_once_with(trials, trials_path) + + +def test_read_trials_returns_trials_file(mocker): + trials = create_trials(mocker) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', + return_value=trials) + mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', + return_value=mock_load) + + assert read_trials() == trials + mock_open.assert_called_once() + mock_load.assert_called_once() From ffae0b2cd54eab2d6eea2eae026194ca9d3d9657 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 9 Jan 2018 12:37:56 +0200 Subject: [PATCH 09/33] hyperopt: prettyfie best values when receiving SIGINT, use the global TRIALS --- freqtrade/optimize/hyperopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4162e0758..03f833c4a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -121,7 +121,7 @@ def read_trials(trials_path=TRIALS_FILE): def log_trials_result(trials): - vals = trials.best_trial['misc']['vals'] + vals = json.dumps(trials.best_trial['misc']['vals'], indent=4) results = trials.best_trial['result']['result'] logger.info('Best result:\n%s\nwith values:\n%s', results, vals) @@ -289,7 +289,7 @@ def start(args): trials=TRIALS ) - results = sorted(trials.results, key=itemgetter('loss')) + results = sorted(TRIALS.results, key=itemgetter('loss')) best_result = results[0]['result'] except ValueError: From 9840e0b5b8c94bbe7d30d355af0c5997c91016b1 Mon Sep 17 00:00:00 2001 From: Robert Moggach Date: Tue, 9 Jan 2018 04:31:59 -0800 Subject: [PATCH 10/33] use HTTPS git URL in README.md (#347) --- docs/installation.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 585ffeb24..30431345b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,7 +1,7 @@ # Install the bot -This page explains how to prepare your environment for running the bot. -To understand how to set up the bot please read the Bot -[Bot configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) +This page explains how to prepare your environment for running the bot. +To understand how to set up the bot please read the Bot +[Bot configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) page. ## Table of Contents @@ -21,14 +21,14 @@ Start by downloading Docker for your platform: - [Windows](https://www.docker.com/products/docker#/windows) - [Linux](https://www.docker.com/products/docker#/linux) -Once you have Docker installed, simply create the config file -(e.g. `config.json`) and then create a Docker image for `freqtrade` +Once you have Docker installed, simply create the config file +(e.g. `config.json`) and then create a Docker image for `freqtrade` using the Dockerfile in this repo. ### 1. Prepare the bot -1. Clone the git +1. Clone the git ```bash -git clone git@github.com:gcarq/freqtrade.git +git clone https://github.com/gcarq/freqtrade.git ``` 2. (Optional) Checkout the develop branch ```bash @@ -42,7 +42,7 @@ cd freqtrade ```bash cp config.json.example config.json ``` -To edit the config please refer to the [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) page +To edit the config please refer to the [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) page 5. Create your DB file (Optional, the bot will create it if it is missing) ```bash # For Production @@ -60,7 +60,7 @@ docker build -t freqtrade . For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount -a sqlite database file (see the "5. Run a restartable docker image" +a sqlite database file (see the "5. Run a restartable docker image" section) to keep it between updates. ### 3. Verify the docker image @@ -100,7 +100,7 @@ docker run -d \ -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ freqtrade ``` -If you are using `dry_run=True` it's not necessary to mount +If you are using `dry_run=True` it's not necessary to mount `tradesv3.sqlite`, but you can mount `tradesv3.dryrun.sqlite` if you plan to use the dry run mode with the param `--dry-run-db`. @@ -116,14 +116,14 @@ docker stop freqtrade docker start freqtrade ``` -You do not need to rebuild the image for configuration changes, it will +You do not need to rebuild the image for configuration changes, it will suffice to edit `config.json` and restart the container. # Linux / MacOS ## 1. Requirements Click each one for install guide: -- [Python 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/), +- [Python 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/), note the bot was not tested on Python >= 3.7.x - [pip](https://pip.pypa.io/en/stable/installing/) - [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) @@ -142,7 +142,7 @@ sudo apt-get update sudo apt-get install python3.6 python3.6-venv python3.6-dev build-essential autoconf libtool pkg-config make wget git ``` -**2.1.2. Install TA-LIB** +**2.1.2. Install TA-LIB** Official webpage: https://mrjbq7.github.io/ta-lib/install.html ``` wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz @@ -155,7 +155,7 @@ cd .. rm -rf ./ta-lib* ``` -**2.1.3. [Optional] Install MongoDB** +**2.1.3. [Optional] Install MongoDB** Install MongoDB if you plan to optimize your strategy with Hyperopt. ```bash @@ -176,7 +176,7 @@ If you are on a different Linux OS you maybe have to adapt things like: brew install python3 git wget ``` -**2.3.2. [Optional] Install MongoDB** +**2.3.2. [Optional] Install MongoDB** Install MongoDB if you plan to optimize your strategy with Hyperopt. ```bash curl -O https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.4.10.tgz @@ -188,15 +188,15 @@ export PATH=/env/mongodb/bin:$PATH ## 3. Clone the repo The following steps are made for Linux/mac environment -1. Clone the git `git clone git@github.com:gcarq/freqtrade.git` -2. (Optional) Checkout the develop branch `git checkout develop` +1. Clone the git `git clone https://github.com/gcarq/freqtrade.git` +2. (Optional) Checkout the develop branch `git checkout develop` ## 4. Prepare the bot ```bash cd freqtrade cp config.json.example config.json ``` -To edit the config please refer to [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) +To edit the config please refer to [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) ## 5. Setup your virtual env ```bash @@ -215,7 +215,7 @@ python3.6 ./freqtrade/main.py -c config.json ``` ### Advanced Linux -**systemd service file** +**systemd service file** Copy `./freqtrade.service` to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. After that you can start the daemon with: @@ -224,7 +224,7 @@ systemctl --user start freqtrade ``` # Windows -We do recommend Windows users to use [Docker](#docker) this will work +We do recommend Windows users to use [Docker](#docker) this will work much easier and smoother (also safer). ```cmd @@ -242,5 +242,5 @@ much easier and smoother (also safer). *Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/gcarq/freqtrade/issues/222)* ## Next step -Now you have an environment ready, the next step is to +Now you have an environment ready, the next step is to [configure your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md). From b9bf5c1118c74bcba0003a100812fb4f7945bd50 Mon Sep 17 00:00:00 2001 From: kryofly Date: Tue, 9 Jan 2018 14:07:50 +0100 Subject: [PATCH 11/33] test: increase coverage of exchange.bittrex --- .../tests/exchange/test_exchange_bittrex.py | 307 +++++++++++++++++- 1 file changed, 306 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index 53ca71a83..704ca0f6e 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -1,9 +1,314 @@ # pragma pylint: disable=missing-docstring,C0103 import pytest +from unittest.mock import MagicMock from requests.exceptions import ContentDecodingError -from freqtrade.exchange import Bittrex +from freqtrade.exchange.bittrex import Bittrex +import freqtrade.exchange.bittrex as btx + + +# Eat this flake8 +# +------------------+ +# | bittrex.Bittrex | +# +------------------+ +# | +# (mock Fake_bittrex) +# | +# +-----------------------------+ +# | freqtrade.exchange.Bittrex | +# +-----------------------------+ +# Call into Bittrex will flow up to the +# external package bittrex.Bittrex. +# By inserting a mock, we redirect those +# calls. +# The faked bittrex API is called just 'fb' +# The freqtrade.exchange.Bittrex is a +# wrapper, and is called 'wb' + + +def _stub_config(): + return {'key': '', + 'secret': ''} + + +class Fake_bittrex(): + def __init__(self, success=True): + self.success = True # Believe in yourself + self.result = None + self.get_ticker_call_count = 0 + # This is really ugly, doing side-effect during instance creation + # But we're allowed to in testing-code + btx._API = MagicMock() + btx._API.buy_limit = self.fake_buysell_limit + btx._API.sell_limit = self.fake_buysell_limit + btx._API.get_balance = self.fake_get_balance + btx._API.get_balances = self.fake_get_balances + btx._API.get_ticker = self.fake_get_ticker + btx._API.get_order = self.fake_get_order + btx._API.cancel = self.fake_cancel_order + btx._API.get_markets = self.fake_get_markets + btx._API.get_market_summaries = self.fake_get_market_summaries + btx._API_V2 = MagicMock() + btx._API_V2.get_candles = self.fake_get_candles + btx._API_V2.get_wallet_health = self.fake_get_wallet_health + + def fake_buysell_limit(self, pair, amount, limit): + return {'success': self.success, + 'result': {'uuid': '1234'}, + 'message': 'barter'} + + def fake_get_balance(self, cur): + return {'success': self.success, + 'result': {'Balance': 1234}, + 'message': 'unbalanced'} + + def fake_get_balances(self): + return {'success': self.success, + 'result': [{'BTC_ETH': 1234}], + 'message': 'no balances'} + + def fake_get_ticker(self, pair): + self.get_ticker_call_count += 1 + return self.result or {'success': self.success, + 'result': {'Bid': 1, 'Ask': 1, 'Last': 1}, + 'message': 'NO_API_RESPONSE'} + + def fake_get_candles(self, pair, interval): + return self.result or {'success': self.success, + 'result': [{'C': 0, 'V': 0, 'O': 0, 'H': 0, 'L': 0, 'T': 0}], + 'message': 'candles lit'} + + def fake_get_order(self, uuid): + return {'success': self.success, + 'result': {'OrderUuid': 'ABC123', + 'Type': 'Type', + 'Exchange': 'BTC_ETH', + 'Opened': True, + 'PricePerUnit': 1, + 'Quantity': 1, + 'QuantityRemaining': 1, + 'Closed': True + }, + 'message': 'lost'} + + def fake_cancel_order(self, uuid): + return self.result or {'success': self.success, + 'message': 'no such order'} + + def fake_get_markets(self): + return self.result or {'success': self.success, + 'message': 'market gone', + 'result': [{'MarketName': '-_'}]} + + def fake_get_market_summaries(self): + return self.result or {'success': self.success, + 'message': 'no summary', + 'result': ['sum']} + + def fake_get_wallet_health(self): + return self.result or {'success': self.success, + 'message': 'bad health', + 'result': [{'Health': {'Currency': 'BTC_ETH', + 'IsActive': True, + 'LastChecked': 0}, + 'Currency': {'Notice': True}}]} + + +# The freqtrade.exchange.bittrex is called wrap_bittrex +# to not confuse naming with bittrex.bittrex +def make_wrap_bittrex(): + conf = _stub_config() + wb = btx.Bittrex(conf) + return wb + + +def test_exchange_bittrex_class(): + conf = _stub_config() + b = Bittrex(conf) + assert isinstance(b, Bittrex) + slots = dir(b) + for name in ['fee', 'buy', 'sell', 'get_balance', 'get_balances', + 'get_ticker', 'get_ticker_history', 'get_order', + 'cancel_order', 'get_pair_detail_url', 'get_markets', + 'get_market_summaries', 'get_wallet_health']: + assert name in slots + # FIX: ensure that the slot is also a method in the class + # getattr(b, name) => bound method Bittrex.buy + # type(getattr(b, name)) => class 'method' + + +def test_exchange_bittrex_fee(): + fee = Bittrex.fee.__get__(Bittrex) + assert fee >= 0 and fee < 0.1 # Fee is 0-10 % + + +def test_exchange_bittrex_buy_good(mocker): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + uuid = wb.buy('BTC_ETH', 1, 1) + assert uuid == fb.fake_buysell_limit(1, 2, 3)['result']['uuid'] + + fb.success = False + with pytest.raises(btx.OperationalException, match=r'barter.*'): + wb.buy('BAD', 1, 1) + + +def test_exchange_bittrex_sell_good(mocker): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + uuid = wb.sell('BTC_ETH', 1, 1) + assert uuid == fb.fake_buysell_limit(1, 2, 3)['result']['uuid'] + + fb.success = False + with pytest.raises(btx.OperationalException, match=r'barter.*'): + uuid = wb.sell('BAD', 1, 1) + + +def test_exchange_bittrex_get_balance(mocker): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + bal = wb.get_balance('BTC_ETH') + assert bal == fb.fake_get_balance(1)['result']['Balance'] + + fb.success = False + with pytest.raises(btx.OperationalException, match=r'unbalanced'): + wb.get_balance('BTC_ETH') + + +def test_exchange_bittrex_get_balances(): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + bals = wb.get_balances() + assert bals == fb.fake_get_balances()['result'] + + fb.success = False + with pytest.raises(btx.OperationalException, match=r'no balances'): + wb.get_balances() + + +def test_exchange_bittrex_get_ticker(): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + + # Poll ticker, which updates the cache + tick = wb.get_ticker('BTC_ETH') + for x in ['bid', 'ask', 'last']: + assert x in tick + # Ensure the side-effect was made (update the ticker cache) + assert 'BTC_ETH' in wb.cached_ticker.keys() + + # taint the cache, so we can recognize the cache wall utilized + wb.cached_ticker['BTC_ETH']['bid'] = 1234 + # Poll again, getting the cached result + fb.get_ticker_call_count = 0 + tick = wb.get_ticker('BTC_ETH', False) + # Ensure the result was from the cache, and that we didn't call exchange + assert wb.cached_ticker['BTC_ETH']['bid'] == 1234 + assert fb.get_ticker_call_count == 0 + + +def test_exchange_bittrex_get_ticker_bad(): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + fb.result = {'success': True, + 'result': {'Bid': 1}} # incomplete result + with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex params.*'): + wb.get_ticker('BTC_ETH') + fb.result = {'success': False, + 'message': 'gone bad' + } + with pytest.raises(btx.OperationalException, match=r'.*gone bad.*'): + wb.get_ticker('BTC_ETH') + + +def test_exchange_bittrex_get_ticker_historyOne(): + wb = make_wrap_bittrex() + Fake_bittrex() + assert wb.get_ticker_history('BTC_ETH', 1) + + +def test_exchange_bittrex_get_ticker_history(): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + assert wb.get_ticker_history('BTC_ETH', 5) + with pytest.raises(ValueError, match=r'.*Cannot parse tick_interval.*'): + wb.get_ticker_history('BTC_ETH', 2) + + fb.success = False + with pytest.raises(btx.OperationalException, match=r'candles lit.*'): + wb.get_ticker_history('BTC_ETH', 5) + + fb.success = True + with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex.*'): + fb.result = {'bad': 0} + wb.get_ticker_history('BTC_ETH', 5) + + with pytest.raises(ContentDecodingError, match=r'.*Required property C not present.*'): + fb.result = {'success': True, + 'result': [{'V': 0, 'O': 0, 'H': 0, 'L': 0, 'T': 0}], # close is missing + 'message': 'candles lit'} + wb.get_ticker_history('BTC_ETH', 5) + + +def test_exchange_bittrex_get_order(): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + order = wb.get_order('someUUID') + assert order['id'] == 'ABC123' + fb.success = False + with pytest.raises(btx.OperationalException, match=r'lost'): + wb.get_order('someUUID') + + +def test_exchange_bittrex_cancel_order(): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + wb.cancel_order('someUUID') + with pytest.raises(btx.OperationalException, match=r'no such order'): + fb.success = False + wb.cancel_order('someUUID') + # Note: this can be a bug in exchange.bittrex._validate_response + with pytest.raises(KeyError): + fb.result = {'success': False} # message is missing! + wb.cancel_order('someUUID') + with pytest.raises(btx.OperationalException, match=r'foo'): + fb.result = {'success': False, 'message': 'foo'} + wb.cancel_order('someUUID') + + +def test_exchange_get_pair_detail_url(): + wb = make_wrap_bittrex() + assert wb.get_pair_detail_url('BTC_ETH') + + +def test_exchange_get_markets(): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + x = wb.get_markets() + assert x == ['__'] + with pytest.raises(btx.OperationalException, match=r'market gone'): + fb.success = False + wb.get_markets() + + +def test_exchange_get_market_summaries(): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + assert ['sum'] == wb.get_market_summaries() + with pytest.raises(btx.OperationalException, match=r'no summary'): + fb.success = False + wb.get_market_summaries() + + +def test_exchange_get_wallet_health(): + wb = make_wrap_bittrex() + fb = Fake_bittrex() + x = wb.get_wallet_health() + assert x[0]['Currency'] == 'BTC_ETH' + with pytest.raises(btx.OperationalException, match=r'bad health'): + fb.success = False + wb.get_wallet_health() def test_validate_response_success(): From e67c652988cf55b85cb06771dc513222c6eb0ea8 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Wed, 10 Jan 2018 11:50:00 +0200 Subject: [PATCH 12/33] use os.path.join, fix docstrings --- freqtrade/optimize/hyperopt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 03f833c4a..5d342b831 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -47,7 +47,7 @@ PROCESSED = None # optimize.preprocess(optimize.load_data()) OPTIMIZE_CONFIG = hyperopt_optimize_conf() # Hyperopt Trials -TRIALS_FILE = 'freqtrade/optimize/hyperopt_trials.pickle' +TRIALS_FILE = os.path.join('freqtrade', 'optimize', 'hyperopt_trials.pickle') TRIALS = Trials() # Monkey patch config @@ -107,13 +107,13 @@ SPACE = { def save_trials(trials, trials_path=TRIALS_FILE): - "Save hyperopt trials to file" + """Save hyperopt trials to file""" logger.info('Saving Trials to \'{}\''.format(trials_path)) pickle.dump(trials, open(trials_path, 'wb')) def read_trials(trials_path=TRIALS_FILE): - "Read hyperopt trials file" + """Read hyperopt trials file""" logger.info('Reading Trials from \'{}\''.format(trials_path)) trials = pickle.load(open(trials_path, 'rb')) os.remove(trials_path) @@ -309,7 +309,7 @@ def start(args): def signal_handler(sig, frame): - "Hyperopt SIGINT handler" + """Hyperopt SIGINT handler""" logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) save_trials(TRIALS) From abcdbcfd39c20d251694b83206f784b483aacb96 Mon Sep 17 00:00:00 2001 From: Anton Ermak Date: Wed, 10 Jan 2018 17:37:49 +0700 Subject: [PATCH 13/33] Set requests default timeout --- freqtrade/main.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/freqtrade/main.py b/freqtrade/main.py index 5e8680b85..efd0d7c44 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -10,6 +10,7 @@ from datetime import datetime from typing import Dict, Optional, List import requests +from requests.adapters import TimeoutSauce from cachetools import cached, TTLCache from freqtrade import __version__, exchange, persistence, rpc, DependencyException, \ @@ -24,6 +25,23 @@ logger = logging.getLogger('freqtrade') _CONF = {} +DEFAULT_TIMEOUT = 120 + + +# Set requests default timeout (fix for #127) +class DefaultTimeout(TimeoutSauce): + def __init__(self, *args, **kwargs): + connect = kwargs.get('connect', DEFAULT_TIMEOUT) + read = kwargs.get('read', connect) + if connect is None: + connect = DEFAULT_TIMEOUT + if read is None: + read = connect + super(DefaultTimeout, self).__init__(connect=connect, read=read) + + +requests.adapters.TimeoutSauce = DefaultTimeout + def refresh_whitelist(whitelist: List[str]) -> List[str]: """ From 1b6b0ad9d2e7f26e9e05cbcce5f7c7b35f1043e0 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 9 Jan 2018 12:59:06 +0200 Subject: [PATCH 14/33] autopep8 --- freqtrade/main.py | 22 +++--- freqtrade/optimize/backtesting.py | 26 +++---- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/rpc/telegram.py | 2 +- freqtrade/tests/optimize/test_optimize.py | 2 +- freqtrade/tests/test_acl_pair.py | 90 +++++++++++------------ freqtrade/tests/test_main.py | 84 ++++++++++----------- freqtrade/tests/test_misc.py | 2 +- 8 files changed, 115 insertions(+), 115 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index efd0d7c44..58f3c7aee 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -197,11 +197,11 @@ def execute_sell(trade: Trade, limit: float) -> None: profit_trade = trade.calc_profit(rate=limit) message = '*{exchange}:* Selling [{pair}]({pair_url}) with limit `{limit:.8f}`'.format( - exchange=trade.exchange, - pair=trade.pair.replace('_', '/'), - pair_url=exchange.get_pair_detail_url(trade.pair), - limit=limit - ) + exchange=trade.exchange, + pair=trade.pair.replace('_', '/'), + pair_url=exchange.get_pair_detail_url(trade.pair), + limit=limit + ) # For regular case, when the configuration exists if 'stake_currency' in _CONF and 'fiat_display_currency' in _CONF: @@ -213,12 +213,12 @@ def execute_sell(trade: Trade, limit: float) -> None: ) message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f} {coin}`' \ '` / {profit_fiat:.3f} {fiat})`'.format( - gain="profit" if fmt_exp_profit > 0 else "loss", - profit_percent=fmt_exp_profit, - profit_coin=profit_trade, - coin=_CONF['stake_currency'], - profit_fiat=profit_fiat, - fiat=_CONF['fiat_display_currency'], + gain="profit" if fmt_exp_profit > 0 else "loss", + profit_percent=fmt_exp_profit, + profit_coin=profit_trade, + coin=_CONF['stake_currency'], + profit_fiat=profit_fiat, + fiat=_CONF['fiat_display_currency'], ) # Because telegram._forcesell does not have the configuration # Ignore the FIAT value and does not show the stake_currency as well diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 315f960d8..fe3ebd4f1 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -122,20 +122,20 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], if min_roi_reached(trade, row2.close, row2.date) or \ (row2.sell == 1 and use_sell_signal) or \ current_profit_percent <= stoploss: - current_profit_btc = trade.calc_profit(rate=row2.close) - lock_pair_until = row2.Index + current_profit_btc = trade.calc_profit(rate=row2.close) + lock_pair_until = row2.Index - trades.append( - ( - pair, - current_profit_percent, - current_profit_btc, - row2.Index - row.Index, - current_profit_btc > 0, - current_profit_btc < 0 - ) + trades.append( + ( + pair, + current_profit_percent, + current_profit_btc, + row2.Index - row.Index, + current_profit_btc > 0, + current_profit_btc < 0 ) - break + ) + break labels = ['currency', 'profit_percent', 'profit_BTC', 'duration', 'profit', 'loss'] return DataFrame.from_records(trades, columns=labels) @@ -193,6 +193,6 @@ def start(args): use_sell_signal=config.get('experimental', {}).get('use_sell_signal', False) ) logger.info( - '\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa + '\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa generate_text_table(data, results, config['stake_currency'], args.ticker_interval) ) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5d342b831..725bf37b0 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -194,7 +194,7 @@ def format_results(results: DataFrame): results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), results.duration.mean() * 5, - ) + ) def buy_strategy_generator(params): diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4d7957f59..55f1942f3 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -255,7 +255,7 @@ def _daily(bot: Bot, update: Update) -> None: ), symbol=_CONF['fiat_display_currency'] ) - ] + ] for key, value in profit_days.items() ] stats = tabulate(stats, diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index a5892f278..57c41c9c6 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -6,7 +6,7 @@ from shutil import copyfile from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ - download_backtesting_testdata, load_tickerdata_file + download_backtesting_testdata, load_tickerdata_file # Change this if modifying BTC_UNITEST testdatafile _btc_unittest_length = 13681 diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index be0bd47e8..d1e42d444 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -25,51 +25,51 @@ def whitelist_conf(): def get_market_summaries(): return [{ - "MarketName": "BTC-TKN", - "High": 0.00000919, - "Low": 0.00000820, - "Volume": 74339.61396015, - "Last": 0.00000820, - "BaseVolume": 1664, - "TimeStamp": "2014-07-09T07:19:30.15", - "Bid": 0.00000820, - "Ask": 0.00000831, - "OpenBuyOrders": 15, - "OpenSellOrders": 15, - "PrevDay": 0.00000821, - "Created": "2014-03-20T06:00:00", - "DisplayMarketName": "" - }, { - "MarketName": "BTC-ETH", - "High": 0.00000072, - "Low": 0.00000001, - "Volume": 166340678.42280999, - "Last": 0.00000005, - "BaseVolume": 42, - "TimeStamp": "2014-07-09T07:21:40.51", - "Bid": 0.00000004, - "Ask": 0.00000005, - "OpenBuyOrders": 18, - "OpenSellOrders": 18, - "PrevDay": 0.00000002, - "Created": "2014-05-30T07:57:49.637", - "DisplayMarketName": "" - }, { - "MarketName": "BTC-BLK", - "High": 0.00000072, - "Low": 0.00000001, - "Volume": 166340678.42280999, - "Last": 0.00000005, - "BaseVolume": 3, - "TimeStamp": "2014-07-09T07:21:40.51", - "Bid": 0.00000004, - "Ask": 0.00000005, - "OpenBuyOrders": 18, - "OpenSellOrders": 18, - "PrevDay": 0.00000002, - "Created": "2014-05-30T07:57:49.637", - "DisplayMarketName": "" - } + "MarketName": "BTC-TKN", + "High": 0.00000919, + "Low": 0.00000820, + "Volume": 74339.61396015, + "Last": 0.00000820, + "BaseVolume": 1664, + "TimeStamp": "2014-07-09T07:19:30.15", + "Bid": 0.00000820, + "Ask": 0.00000831, + "OpenBuyOrders": 15, + "OpenSellOrders": 15, + "PrevDay": 0.00000821, + "Created": "2014-03-20T06:00:00", + "DisplayMarketName": "" + }, { + "MarketName": "BTC-ETH", + "High": 0.00000072, + "Low": 0.00000001, + "Volume": 166340678.42280999, + "Last": 0.00000005, + "BaseVolume": 42, + "TimeStamp": "2014-07-09T07:21:40.51", + "Bid": 0.00000004, + "Ask": 0.00000005, + "OpenBuyOrders": 18, + "OpenSellOrders": 18, + "PrevDay": 0.00000002, + "Created": "2014-05-30T07:57:49.637", + "DisplayMarketName": "" + }, { + "MarketName": "BTC-BLK", + "High": 0.00000072, + "Low": 0.00000001, + "Volume": 166340678.42280999, + "Last": 0.00000005, + "BaseVolume": 3, + "TimeStamp": "2014-07-09T07:21:40.51", + "Bid": 0.00000004, + "Ask": 0.00000005, + "OpenBuyOrders": 18, + "OpenSellOrders": 18, + "PrevDay": 0.00000002, + "Created": "2014-05-30T07:57:49.637", + "DisplayMarketName": "" + } ] diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index cceb555f7..ab39cebc9 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -624,54 +624,54 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, mocker): def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': True, - } + default_conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': True, + } - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.min_roi_reached', return_value=False) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value='mocked_limit_buy')) - init(default_conf, create_engine('sqlite://')) - create_trade(0.001) + init(default_conf, create_engine('sqlite://')) + create_trade(0.001) - trade = Trade.query.first() - trade.update(limit_buy_order) - assert handle_trade(trade) is False + trade = Trade.query.first() + trade.update(limit_buy_order) + assert handle_trade(trade) is False def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, mocker): - default_conf['experimental'] = { - 'use_sell_signal': True, - 'sell_profit_only': False, - } + default_conf['experimental'] = { + 'use_sell_signal': True, + 'sell_profit_only': False, + } - mocker.patch.dict('freqtrade.main._CONF', default_conf) - mocker.patch('freqtrade.main.min_roi_reached', return_value=False) - mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) - mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) - mocker.patch.multiple('freqtrade.main.exchange', - validate_pairs=MagicMock(), - get_ticker=MagicMock(return_value={ - 'bid': 0.00000172, - 'ask': 0.00000173, - 'last': 0.00000172 - }), - buy=MagicMock(return_value='mocked_limit_buy')) + mocker.patch.dict('freqtrade.main._CONF', default_conf) + mocker.patch('freqtrade.main.min_roi_reached', return_value=False) + mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: True) + mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00000172, + 'ask': 0.00000173, + 'last': 0.00000172 + }), + buy=MagicMock(return_value='mocked_limit_buy')) - init(default_conf, create_engine('sqlite://')) - create_trade(0.001) + init(default_conf, create_engine('sqlite://')) + create_trade(0.001) - trade = Trade.query.first() - trade.update(limit_buy_order) - assert handle_trade(trade) is True + trade = Trade.query.first() + trade.update(limit_buy_order) + assert handle_trade(trade) is True diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 0b85000cb..16bb8d232 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -8,7 +8,7 @@ import pytest from jsonschema import ValidationError from freqtrade.misc import throttle, parse_args, load_config,\ - parse_args_common + parse_args_common def test_throttle(): From 0abc30401c2ac1b2aa73b638db06defe82223d64 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 9 Jan 2018 13:06:28 +0200 Subject: [PATCH 15/33] linter fixes and cleanups --- freqtrade/misc.py | 15 ++++++++------- freqtrade/tests/test_misc.py | 8 +++----- scripts/plot_dataframe.py | 4 ++-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index afc4334e8..c018b7fbb 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -57,8 +57,8 @@ def load_config(path: str) -> Dict: try: validate(conf, CONF_SCHEMA) return conf - except ValidationError as ex: - logger.fatal('Invalid configuration. See config.json.example. Reason: %s', ex) + except ValidationError as exception: + logger.fatal('Invalid configuration. See config.json.example. Reason: %s', exception) raise ValidationError( best_match(Draft4Validator(CONF_SCHEMA).iter_errors(conf)).message ) @@ -81,7 +81,7 @@ def throttle(func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: return result -def parse_args_common(args: List[str], description: str): +def common_args_parser(description: str): """ Parses given common arguments and returns them as a parsed object. """ @@ -117,11 +117,11 @@ def parse_args(args: List[str], description: str): Parses given arguments and returns an argparse Namespace instance. Returns None if a sub command has been selected and executed. """ - parser = parse_args_common(args, description) + parser = common_args_parser(description) parser.add_argument( '--dry-run-db', - help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" instead of memory DB. Work only if dry_run is \ - enabled.', # noqa + help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \ + instead of memory DB. Work only if dry_run is enabled.', action='store_true', dest='dry_run_db', ) @@ -135,7 +135,8 @@ def parse_args(args: List[str], description: str): ) parser.add_argument( '--dynamic-whitelist', - help='dynamically generate and update whitelist based on 24h BaseVolume (Default 20 currencies)', # noqa + help='dynamically generate and update whitelist \ + based on 24h BaseVolume (Default 20 currencies)', # noqa dest='dynamic_whitelist', const=20, type=int, diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 16bb8d232..9de40e8d5 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -8,7 +8,7 @@ import pytest from jsonschema import ValidationError from freqtrade.misc import throttle, parse_args, load_config,\ - parse_args_common + common_args_parser def test_throttle(): @@ -39,12 +39,10 @@ def test_throttle_with_assets(): assert result == -1 -# Parse common command-line-arguments -# used for all tools - +# Parse common command-line-arguments. Used for all tools def test_parse_args_none(): - args = parse_args_common([], '') + args = common_args_parser('') assert isinstance(args, argparse.ArgumentParser) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 44961891c..ce636a4b5 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -6,11 +6,11 @@ import matplotlib # Install PYQT5 manually if you want to test this helper func matplotlib.use("Qt5Agg") import matplotlib.pyplot as plt from freqtrade import exchange, analyze -from freqtrade.misc import parse_args_common +from freqtrade.misc import common_args_parser def plot_parse_args(args ): - parser = parse_args_common(args, 'Graph utility') + parser = common_args_parser(args, 'Graph utility') parser.add_argument( '-p', '--pair', help = 'What currency pair', From 86db6c90840ae40e8eaebc63c0ad75bc1b060626 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 09:51:36 +0200 Subject: [PATCH 16/33] sort imports --- freqtrade/analyze.py | 4 ++-- freqtrade/exchange/bittrex.py | 5 +++-- freqtrade/exchange/interface.py | 2 +- freqtrade/fiat_convert.py | 1 + freqtrade/main.py | 16 ++++++++-------- freqtrade/misc.py | 6 +++--- freqtrade/optimize/backtesting.py | 7 +++---- freqtrade/optimize/hyperopt.py | 3 ++- freqtrade/persistence.py | 5 +++-- freqtrade/rpc/telegram.py | 12 ++++++------ freqtrade/tests/conftest.py | 4 ++-- freqtrade/tests/test_analyze.py | 5 +++-- freqtrade/tests/test_dataframe.py | 2 +- freqtrade/tests/test_fiat_convert.py | 5 +++-- freqtrade/tests/test_main.py | 13 ++++++------- freqtrade/tests/test_misc.py | 6 +++--- 16 files changed, 50 insertions(+), 46 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 4b16550c5..f85c46248 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -4,14 +4,14 @@ Functions to analyze ticker data with indicators and produce buy and sell signal import logging from datetime import timedelta from enum import Enum -from typing import List, Dict +from typing import Dict, List import arrow import talib.abstract as ta from pandas import DataFrame, to_datetime -from freqtrade.exchange import get_ticker_history import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.exchange import get_ticker_history logger = logging.getLogger(__name__) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 4883db037..5a8a602d2 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -1,7 +1,8 @@ import logging -from typing import List, Dict, Optional +from typing import Dict, List, Optional -from bittrex.bittrex import Bittrex as _Bittrex, API_V2_0, API_V1_1 +from bittrex.bittrex import Bittrex as _Bittrex +from bittrex.bittrex import API_V1_1, API_V2_0 from requests.exceptions import ContentDecodingError from freqtrade import OperationalException diff --git a/freqtrade/exchange/interface.py b/freqtrade/exchange/interface.py index 1be84abe5..6121a98b3 100644 --- a/freqtrade/exchange/interface.py +++ b/freqtrade/exchange/interface.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Dict, Optional +from typing import Dict, List, Optional class Exchange(ABC): diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 3f0b6330b..0132e531d 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -1,5 +1,6 @@ import logging import time + from pymarketcap import Pymarketcap logger = logging.getLogger(__name__) diff --git a/freqtrade/main.py b/freqtrade/main.py index 58f3c7aee..513f33b3b 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -5,21 +5,21 @@ import logging import sys import time import traceback -import arrow from datetime import datetime -from typing import Dict, Optional, List +from typing import Dict, List, Optional +import arrow import requests from requests.adapters import TimeoutSauce from cachetools import cached, TTLCache -from freqtrade import __version__, exchange, persistence, rpc, DependencyException, \ - OperationalException -from freqtrade.analyze import get_signal, SignalType -from freqtrade.misc import State, get_state, update_state, parse_args, throttle, \ - load_config -from freqtrade.persistence import Trade +from freqtrade import (DependencyException, OperationalException, __version__, + exchange, persistence, rpc) +from freqtrade.analyze import SignalType, get_signal from freqtrade.fiat_convert import CryptoToFiatConverter +from freqtrade.misc import (State, get_state, load_config, parse_args, + throttle, update_state) +from freqtrade.persistence import Trade logger = logging.getLogger('freqtrade') diff --git a/freqtrade/misc.py b/freqtrade/misc.py index c018b7fbb..66e8b6c6b 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -3,10 +3,10 @@ import enum import json import logging import time -from typing import Any, Callable, List, Dict +from typing import Any, Callable, Dict, List -from jsonschema import validate, Draft4Validator -from jsonschema.exceptions import best_match, ValidationError +from jsonschema import Draft4Validator, validate +from jsonschema.exceptions import ValidationError, best_match from wrapt import synchronized from freqtrade import __version__ diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fe3ebd4f1..6d600b303 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1,20 +1,19 @@ # pragma pylint: disable=missing-docstring,W0212 - import logging -from typing import Tuple, Dict +from typing import Dict, Tuple import arrow from pandas import DataFrame, Series from tabulate import tabulate +import freqtrade.misc as misc +import freqtrade.optimize as optimize from freqtrade import exchange from freqtrade.analyze import populate_buy_trend, populate_sell_trend from freqtrade.exchange import Bittrex from freqtrade.main import min_roi_reached -import freqtrade.misc as misc from freqtrade.optimize import preprocess -import freqtrade.optimize as optimize from freqtrade.persistence import Trade logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 725bf37b0..71ddd33c6 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,10 +11,11 @@ from functools import reduce from math import exp from operator import itemgetter -from hyperopt import fmin, tpe, hp, Trials, STATUS_OK, STATUS_FAIL, space_eval +from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from hyperopt.mongoexp import MongoTrials from pandas import DataFrame +from freqtrade import main # noqa from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.misc import load_config diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index d50c9acb4..0cac3bbe9 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -1,10 +1,11 @@ import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Optional, Dict +from typing import Dict, Optional import arrow -from sqlalchemy import Boolean, Column, DateTime, Float, Integer, String, create_engine +from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, + create_engine) from sqlalchemy.engine import Engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 55f1942f3..009714682 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1,21 +1,21 @@ import logging import re +from datetime import datetime, timedelta from decimal import Decimal -from datetime import timedelta, datetime -from typing import Callable, Any +from typing import Any, Callable import arrow from pandas import DataFrame from sqlalchemy import and_, func, text from tabulate import tabulate -from telegram import ParseMode, Bot, Update, ReplyKeyboardMarkup +from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater -from freqtrade import exchange, __version__ -from freqtrade.misc import get_state, State, update_state -from freqtrade.persistence import Trade +from freqtrade import __version__, exchange from freqtrade.fiat_convert import CryptoToFiatConverter +from freqtrade.misc import State, get_state, update_state +from freqtrade.persistence import Trade # Remove noisy log messages logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 788585345..2be8039ca 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -2,10 +2,10 @@ from datetime import datetime from unittest.mock import MagicMock -import pytest import arrow +import pytest from jsonschema import validate -from telegram import Message, Chat, Update +from telegram import Chat, Message, Update from freqtrade.misc import CONF_SCHEMA diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 040c45f26..d1afc4200 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -6,8 +6,9 @@ import arrow import pytest from pandas import DataFrame -from freqtrade.analyze import parse_ticker_dataframe, populate_buy_trend, populate_indicators, \ - get_signal, SignalType, populate_sell_trend +from freqtrade.analyze import (SignalType, get_signal, parse_ticker_dataframe, + populate_buy_trend, populate_indicators, + populate_sell_trend) @pytest.fixture diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index cd8af2c52..f9230a03f 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -1,7 +1,7 @@ import pandas -from freqtrade import analyze import freqtrade.optimize +from freqtrade import analyze _pairs = ['BTC_ETH'] diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 4aa9b1c4f..ddc1c8e29 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -1,10 +1,11 @@ # pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103 import time -import pytest from unittest.mock import MagicMock -from freqtrade.fiat_convert import CryptoToFiatConverter, CryptoFiat +import pytest + +from freqtrade.fiat_convert import CryptoFiat, CryptoToFiatConverter def test_pair_convertion_object(): diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index ab39cebc9..ccb3f0534 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -1,22 +1,21 @@ # pragma pylint: disable=missing-docstring,C0103 import copy +import logging from unittest.mock import MagicMock +import arrow import pytest import requests -import logging -import arrow from sqlalchemy import create_engine +import freqtrade.main as main from freqtrade import DependencyException, OperationalException from freqtrade.analyze import SignalType from freqtrade.exchange import Exchanges -from freqtrade.main import create_trade, handle_trade, init, \ - get_target_bid, _process, execute_sell, check_handle_timedout -from freqtrade.misc import get_state, State +from freqtrade.main import (_process, check_handle_timedout, create_trade, + execute_sell, get_target_bid, handle_trade, init) +from freqtrade.misc import State, get_state from freqtrade.persistence import Trade -import freqtrade.main as main - # Test that main() can start backtesting or hyperopt. # and also ensure we can pass some specific arguments diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 9de40e8d5..244ff31e7 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -1,14 +1,14 @@ # pragma pylint: disable=missing-docstring,C0103 +import argparse import json import time -import argparse from copy import deepcopy import pytest from jsonschema import ValidationError -from freqtrade.misc import throttle, parse_args, load_config,\ - common_args_parser +from freqtrade.misc import (common_args_parser, load_config, parse_args, + throttle) def test_throttle(): From 64530c61967b4506b464da6339f8f2743131a2ba Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 10:13:41 +0200 Subject: [PATCH 17/33] remove unused variables --- freqtrade/tests/test_main.py | 10 +++++----- freqtrade/tests/test_persistence.py | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index ccb3f0534..b25f4ac6a 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -268,7 +268,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): assert trade.close_date is not None -def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker, caplog): +def test_handle_trade_roi(default_conf, ticker, mocker, caplog): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -300,7 +300,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker, caplog) assert ('freqtrade', logging.DEBUG, 'Executing sell due to ROI ...') in caplog.record_tuples -def test_handle_trade_experimental(default_conf, ticker, limit_buy_order, mocker, caplog): +def test_handle_trade_experimental(default_conf, ticker, mocker, caplog): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -352,7 +352,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, mo handle_trade(trade) -def test_check_handle_timedout_buy(default_conf, ticker, health, limit_buy_order_old, mocker): +def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) cancel_order_mock = MagicMock() mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) @@ -384,7 +384,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, health, limit_buy_order assert len(trades) == 0 -def test_check_handle_timedout_sell(default_conf, ticker, health, limit_sell_order_old, mocker): +def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) cancel_order_mock = MagicMock() mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) @@ -417,7 +417,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, health, limit_sell_ord def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, - health, mocker): + mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) cancel_order_mock = MagicMock() mocker.patch.multiple('freqtrade.rpc', init=MagicMock(), send_msg=MagicMock()) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index c2e2c13ea..70797f960 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,9 +1,10 @@ # pragma pylint: disable=missing-docstring +import os + import pytest -import os from freqtrade.exchange import Exchanges -from freqtrade.persistence import init, Trade +from freqtrade.persistence import Trade, init def test_init_create_session(default_conf, mocker): From 8fb404b0f8181c8399a80a3177364197be3cbdaa Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 11:15:44 +0200 Subject: [PATCH 18/33] ignore talib.abstract in pylint --- .pylintrc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index 65ed64830..dce99c067 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,10 +1,10 @@ [MASTER] -extension-pkg-whitelist=numpy,talib +extension-pkg-whitelist=numpy,talib,talib.abstract [BASIC] good-names=logger ignore=vendor [TYPECHECK] -ignored-modules=numpy,talib +ignored-modules=numpy,talib,talib.abstract From 6a433282dca7d236ece2990857c287bbf24aa425 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 11:17:04 +0200 Subject: [PATCH 19/33] fix literal comparison --- freqtrade/tests/test_misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 244ff31e7..63cfba627 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -85,12 +85,12 @@ def test_parse_args_invalid(): def test_parse_args_dynamic_whitelist(): args = parse_args(['--dynamic-whitelist'], '') - assert args.dynamic_whitelist is 20 + assert args.dynamic_whitelist == 20 def test_parse_args_dynamic_whitelist_10(): args = parse_args(['--dynamic-whitelist', '10'], '') - assert args.dynamic_whitelist is 10 + assert args.dynamic_whitelist == 10 def test_parse_args_dynamic_whitelist_invalid_values(): From 0d6051e6f9e778b40c7cca739952d98bba9e2291 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 11:17:16 +0200 Subject: [PATCH 20/33] formatting --- freqtrade/tests/test_acl_pair.py | 113 ++++++++++++++++--------------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index d1e42d444..7c42c676e 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -7,17 +7,17 @@ from freqtrade.main import refresh_whitelist, gen_pair_whitelist def whitelist_conf(): return { - "stake_currency": "BTC", - "exchange": { - "pair_whitelist": [ - "BTC_ETH", - "BTC_TKN", - "BTC_TRST", - "BTC_SWT", - "BTC_BCC" + 'stake_currency': 'BTC', + 'exchange': { + 'pair_whitelist': [ + 'BTC_ETH', + 'BTC_TKN', + 'BTC_TRST', + 'BTC_SWT', + 'BTC_BCC' ], - "pair_blacklist": [ - "BTC_BLK" + 'pair_blacklist': [ + 'BTC_BLK' ], }, } @@ -25,52 +25,51 @@ def whitelist_conf(): def get_market_summaries(): return [{ - "MarketName": "BTC-TKN", - "High": 0.00000919, - "Low": 0.00000820, - "Volume": 74339.61396015, - "Last": 0.00000820, - "BaseVolume": 1664, - "TimeStamp": "2014-07-09T07:19:30.15", - "Bid": 0.00000820, - "Ask": 0.00000831, - "OpenBuyOrders": 15, - "OpenSellOrders": 15, - "PrevDay": 0.00000821, - "Created": "2014-03-20T06:00:00", - "DisplayMarketName": "" + 'MarketName': 'BTC-TKN', + 'High': 0.00000919, + 'Low': 0.00000820, + 'Volume': 74339.61396015, + 'Last': 0.00000820, + 'BaseVolume': 1664, + 'TimeStamp': '2014-07-09T07:19:30.15', + 'Bid': 0.00000820, + 'Ask': 0.00000831, + 'OpenBuyOrders': 15, + 'OpenSellOrders': 15, + 'PrevDay': 0.00000821, + 'Created': '2014-03-20T06:00:00', + 'DisplayMarketName': '' }, { - "MarketName": "BTC-ETH", - "High": 0.00000072, - "Low": 0.00000001, - "Volume": 166340678.42280999, - "Last": 0.00000005, - "BaseVolume": 42, - "TimeStamp": "2014-07-09T07:21:40.51", - "Bid": 0.00000004, - "Ask": 0.00000005, - "OpenBuyOrders": 18, - "OpenSellOrders": 18, - "PrevDay": 0.00000002, - "Created": "2014-05-30T07:57:49.637", - "DisplayMarketName": "" + 'MarketName': 'BTC-ETH', + 'High': 0.00000072, + 'Low': 0.00000001, + 'Volume': 166340678.42280999, + 'Last': 0.00000005, + 'BaseVolume': 42, + 'TimeStamp': '2014-07-09T07:21:40.51', + 'Bid': 0.00000004, + 'Ask': 0.00000005, + 'OpenBuyOrders': 18, + 'OpenSellOrders': 18, + 'PrevDay': 0.00000002, + 'Created': '2014-05-30T07:57:49.637', + 'DisplayMarketName': '' }, { - "MarketName": "BTC-BLK", - "High": 0.00000072, - "Low": 0.00000001, - "Volume": 166340678.42280999, - "Last": 0.00000005, - "BaseVolume": 3, - "TimeStamp": "2014-07-09T07:21:40.51", - "Bid": 0.00000004, - "Ask": 0.00000005, - "OpenBuyOrders": 18, - "OpenSellOrders": 18, - "PrevDay": 0.00000002, - "Created": "2014-05-30T07:57:49.637", - "DisplayMarketName": "" - } - ] + 'MarketName': 'BTC-BLK', + 'High': 0.00000072, + 'Low': 0.00000001, + 'Volume': 166340678.42280999, + 'Last': 0.00000005, + 'BaseVolume': 3, + 'TimeStamp': '2014-07-09T07:21:40.51', + 'Bid': 0.00000004, + 'Ask': 0.00000005, + 'OpenBuyOrders': 18, + 'OpenSellOrders': 18, + 'PrevDay': 0.00000002, + 'Created': '2014-05-30T07:57:49.637', + 'DisplayMarketName': '' + }] def get_health(): @@ -95,7 +94,8 @@ def test_refresh_market_pair_not_in_whitelist(mocker): mocker.patch.dict('freqtrade.main._CONF', conf) mocker.patch.multiple('freqtrade.main.exchange', get_wallet_health=get_health) - refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist'] + ['BTC_XXX']) + refreshedwhitelist = refresh_whitelist( + conf['exchange']['pair_whitelist'] + ['BTC_XXX']) # List ordered by BaseVolume whitelist = ['BTC_ETH', 'BTC_TKN'] # Ensure all except those in whitelist are removed @@ -123,7 +123,8 @@ def test_refresh_whitelist_dynamic(mocker): get_market_summaries=get_market_summaries) # argument: use the whitelist dynamically by exchange-volume whitelist = ['BTC_TKN', 'BTC_ETH'] - refreshedwhitelist = refresh_whitelist(gen_pair_whitelist(conf['stake_currency'])) + refreshedwhitelist = refresh_whitelist( + gen_pair_whitelist(conf['stake_currency'])) assert whitelist == refreshedwhitelist From 02fcbbb6d2dea62733a203079fc0efa8d2c99a17 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 11:25:45 +0200 Subject: [PATCH 21/33] few flake8 fixes --- .../tests/exchange/test_exchange_bittrex.py | 30 +++++++++---------- freqtrade/tests/test_main.py | 7 +++-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index 704ca0f6e..e01e7fa02 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -32,7 +32,7 @@ def _stub_config(): 'secret': ''} -class Fake_bittrex(): +class FakeBittrex(): def __init__(self, success=True): self.success = True # Believe in yourself self.result = None @@ -145,7 +145,7 @@ def test_exchange_bittrex_fee(): def test_exchange_bittrex_buy_good(mocker): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() uuid = wb.buy('BTC_ETH', 1, 1) assert uuid == fb.fake_buysell_limit(1, 2, 3)['result']['uuid'] @@ -156,7 +156,7 @@ def test_exchange_bittrex_buy_good(mocker): def test_exchange_bittrex_sell_good(mocker): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() uuid = wb.sell('BTC_ETH', 1, 1) assert uuid == fb.fake_buysell_limit(1, 2, 3)['result']['uuid'] @@ -167,7 +167,7 @@ def test_exchange_bittrex_sell_good(mocker): def test_exchange_bittrex_get_balance(mocker): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() bal = wb.get_balance('BTC_ETH') assert bal == fb.fake_get_balance(1)['result']['Balance'] @@ -178,7 +178,7 @@ def test_exchange_bittrex_get_balance(mocker): def test_exchange_bittrex_get_balances(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() bals = wb.get_balances() assert bals == fb.fake_get_balances()['result'] @@ -189,7 +189,7 @@ def test_exchange_bittrex_get_balances(): def test_exchange_bittrex_get_ticker(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() # Poll ticker, which updates the cache tick = wb.get_ticker('BTC_ETH') @@ -210,7 +210,7 @@ def test_exchange_bittrex_get_ticker(): def test_exchange_bittrex_get_ticker_bad(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() fb.result = {'success': True, 'result': {'Bid': 1}} # incomplete result with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex params.*'): @@ -222,15 +222,15 @@ def test_exchange_bittrex_get_ticker_bad(): wb.get_ticker('BTC_ETH') -def test_exchange_bittrex_get_ticker_historyOne(): +def test_exchange_bittrex_get_ticker_history_one(): wb = make_wrap_bittrex() - Fake_bittrex() + FakeBittrex() assert wb.get_ticker_history('BTC_ETH', 1) def test_exchange_bittrex_get_ticker_history(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() assert wb.get_ticker_history('BTC_ETH', 5) with pytest.raises(ValueError, match=r'.*Cannot parse tick_interval.*'): wb.get_ticker_history('BTC_ETH', 2) @@ -253,7 +253,7 @@ def test_exchange_bittrex_get_ticker_history(): def test_exchange_bittrex_get_order(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() order = wb.get_order('someUUID') assert order['id'] == 'ABC123' fb.success = False @@ -263,7 +263,7 @@ def test_exchange_bittrex_get_order(): def test_exchange_bittrex_cancel_order(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() wb.cancel_order('someUUID') with pytest.raises(btx.OperationalException, match=r'no such order'): fb.success = False @@ -284,7 +284,7 @@ def test_exchange_get_pair_detail_url(): def test_exchange_get_markets(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() x = wb.get_markets() assert x == ['__'] with pytest.raises(btx.OperationalException, match=r'market gone'): @@ -294,7 +294,7 @@ def test_exchange_get_markets(): def test_exchange_get_market_summaries(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() assert ['sum'] == wb.get_market_summaries() with pytest.raises(btx.OperationalException, match=r'no summary'): fb.success = False @@ -303,7 +303,7 @@ def test_exchange_get_market_summaries(): def test_exchange_get_wallet_health(): wb = make_wrap_bittrex() - fb = Fake_bittrex() + fb = FakeBittrex() x = wb.get_wallet_health() assert x[0]['Currency'] == 'BTC_ETH' with pytest.raises(btx.OperationalException, match=r'bad health'): diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index b25f4ac6a..049ed4284 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -17,11 +17,12 @@ from freqtrade.main import (_process, check_handle_timedout, create_trade, from freqtrade.misc import State, get_state from freqtrade.persistence import Trade -# Test that main() can start backtesting or hyperopt. -# and also ensure we can pass some specific arguments -# argument parsing is done in test_misc.py + def test_parse_args_backtesting(mocker): + """ Test that main() can start backtesting or hyperopt. + and also ensure we can pass some specific arguments + argument parsing is done in test_misc.py """ backtesting_mock = mocker.patch( 'freqtrade.optimize.backtesting.start', MagicMock()) with pytest.raises(SystemExit, match=r'0'): From c11102cf4a1a8b7d08ce3d036cf7d5093bf4522a Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 10 Jan 2018 11:29:41 +0200 Subject: [PATCH 22/33] another run of autopep8 --- freqtrade/optimize/hyperopt_conf.py | 8 ++++---- freqtrade/tests/conftest.py | 8 ++++---- freqtrade/tests/test_main.py | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/hyperopt_conf.py b/freqtrade/optimize/hyperopt_conf.py index cbd95973a..8e044e549 100644 --- a/freqtrade/optimize/hyperopt_conf.py +++ b/freqtrade/optimize/hyperopt_conf.py @@ -15,10 +15,10 @@ def hyperopt_optimize_conf() -> dict: 'stake_currency': 'BTC', 'stake_amount': 0.01, "minimal_roi": { - '40': 0.0, - '30': 0.01, - '20': 0.02, - '0': 0.04, + '40': 0.0, + '30': 0.01, + '20': 0.02, + '0': 0.04, }, 'stoploss': -0.10, "bid_strategy": { diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 2be8039ca..c779aa726 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -20,10 +20,10 @@ def default_conf(): "fiat_display_currency": "USD", "dry_run": True, "minimal_roi": { - "40": 0.0, - "30": 0.01, - "20": 0.02, - "0": 0.04 + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 }, "stoploss": -0.10, "unfilledtimeout": 600, diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 049ed4284..97bef2257 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -18,7 +18,6 @@ from freqtrade.misc import State, get_state from freqtrade.persistence import Trade - def test_parse_args_backtesting(mocker): """ Test that main() can start backtesting or hyperopt. and also ensure we can pass some specific arguments From 11cbb9188bab7b500c9de04da6110b31b1fa9380 Mon Sep 17 00:00:00 2001 From: Anton Ermak Date: Thu, 11 Jan 2018 12:24:05 +0700 Subject: [PATCH 23/33] Set timeout for bittrex only --- freqtrade/exchange/bittrex.py | 18 ++++++++++++++++++ freqtrade/main.py | 18 ------------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 4883db037..8e256a936 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -1,4 +1,5 @@ import logging +import requests from typing import List, Dict, Optional from bittrex.bittrex import Bittrex as _Bittrex, API_V2_0, API_V1_1 @@ -13,6 +14,21 @@ _API: _Bittrex = None _API_V2: _Bittrex = None _EXCHANGE_CONF: dict = {} +# API socket timeout +API_TIMEOUT = 60 + + +def custom_requests(request_url, apisign): + """ + Set timeout for requests + """ + print("Dispatch", request_url, apisign) + return requests.get( + request_url, + headers={"apisign": apisign}, + timeout=API_TIMEOUT + ).json() + class Bittrex(Exchange): """ @@ -31,12 +47,14 @@ class Bittrex(Exchange): api_secret=_EXCHANGE_CONF['secret'], calls_per_second=1, api_version=API_V1_1, + dispatch=custom_requests ) _API_V2 = _Bittrex( api_key=_EXCHANGE_CONF['key'], api_secret=_EXCHANGE_CONF['secret'], calls_per_second=1, api_version=API_V2_0, + dispatch=custom_requests ) self.cached_ticker = {} diff --git a/freqtrade/main.py b/freqtrade/main.py index efd0d7c44..5e8680b85 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -10,7 +10,6 @@ from datetime import datetime from typing import Dict, Optional, List import requests -from requests.adapters import TimeoutSauce from cachetools import cached, TTLCache from freqtrade import __version__, exchange, persistence, rpc, DependencyException, \ @@ -25,23 +24,6 @@ logger = logging.getLogger('freqtrade') _CONF = {} -DEFAULT_TIMEOUT = 120 - - -# Set requests default timeout (fix for #127) -class DefaultTimeout(TimeoutSauce): - def __init__(self, *args, **kwargs): - connect = kwargs.get('connect', DEFAULT_TIMEOUT) - read = kwargs.get('read', connect) - if connect is None: - connect = DEFAULT_TIMEOUT - if read is None: - read = connect - super(DefaultTimeout, self).__init__(connect=connect, read=read) - - -requests.adapters.TimeoutSauce = DefaultTimeout - def refresh_whitelist(whitelist: List[str]) -> List[str]: """ From bb91fdbaf93cc7fa1fab0e87eb555f1acc71bb3d Mon Sep 17 00:00:00 2001 From: Anton Ermak Date: Thu, 11 Jan 2018 13:26:49 +0700 Subject: [PATCH 24/33] oops, print removed --- freqtrade/exchange/bittrex.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 8e256a936..7a08acdbc 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -22,7 +22,6 @@ def custom_requests(request_url, apisign): """ Set timeout for requests """ - print("Dispatch", request_url, apisign) return requests.get( request_url, headers={"apisign": apisign}, From 3a902289f11e8d30fa747ee34e40a159333d4ab2 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Thu, 11 Jan 2018 13:58:06 +0200 Subject: [PATCH 25/33] testdata path to use os.path.join (#360) --- freqtrade/misc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 66e8b6c6b..3d70f6b25 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -3,6 +3,7 @@ import enum import json import logging import time +import os from typing import Any, Callable, Dict, List from jsonschema import Draft4Validator, validate @@ -129,7 +130,7 @@ def parse_args(args: List[str], description: str): '-dd', '--datadir', help='path to backtest data (default freqdata/tests/testdata', dest='datadir', - default='freqtrade/tests/testdata', + default=os.path.join('freqtrade', 'tests', 'testdata'), type=str, metavar='PATH', ) From 46a1a2de101cb9f35d83cdfc6794e868a9312409 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 11 Jan 2018 20:53:26 +0100 Subject: [PATCH 26/33] Update ta-lib from 0.4.14 to 0.4.15 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8a3c8b9cf..e860baedd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ scikit-learn==0.19.1 scipy==1.0.0 jsonschema==2.6.0 numpy==1.14.0 -TA-Lib==0.4.14 +TA-Lib==0.4.15 pytest==3.3.2 pytest-mock==1.6.3 pytest-cov==2.5.1 From 39c6e5263a422588adda63cdb358b947b5cbf988 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Thu, 11 Jan 2018 21:09:04 -0800 Subject: [PATCH 27/33] Fix plot_dataframe.py --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index ce636a4b5..f07033637 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -10,7 +10,7 @@ from freqtrade.misc import common_args_parser def plot_parse_args(args ): - parser = common_args_parser(args, 'Graph utility') + parser = common_args_parser(description='Graph utility') parser.add_argument( '-p', '--pair', help = 'What currency pair', From a26cb4bc6b3b438df1f606c1ba9c3a7fa59b6ac6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 12 Jan 2018 11:08:23 +0100 Subject: [PATCH 28/33] Update pymarketcap from 3.3.145 to 3.3.147 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8a3c8b9cf..52f4e7f7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 networkx==1.11 tabulate==0.8.2 -pymarketcap==3.3.145 +pymarketcap==3.3.147 # Required for plotting data #matplotlib==2.1.0 From 3087ca0823a688f329a9ccbe265656db0048bccb Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 12 Jan 2018 22:56:39 -0800 Subject: [PATCH 29/33] Update freqtrade version --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index a190ca117..d1671e4c6 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.14.3' +__version__ = '0.15.1' class DependencyException(BaseException): From f7a44d1cecd2c01188660c4e6244e06c21d3a872 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Sat, 13 Jan 2018 09:50:02 +0100 Subject: [PATCH 30/33] Fixing the ticker analysis with null value --- freqtrade/exchange/bittrex.py | 6 ++---- freqtrade/tests/exchange/test_exchange_bittrex.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index a5fd68016..e6cacbd4e 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -123,10 +123,8 @@ class Bittrex(Exchange): message=data['message'], pair=pair)) - if not data.get('result') \ - or not data['result'].get('Bid') \ - or not data['result'].get('Ask') \ - or not data['result'].get('Last'): + if not data.get('result') or\ + not all(key in data.get('result', {}) for key in ['Bid', 'Ask', 'Last']): raise ContentDecodingError('{message} params=({pair})'.format( message='Got invalid response from bittrex', pair=pair)) diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index e01e7fa02..4302f9f5f 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -212,7 +212,18 @@ def test_exchange_bittrex_get_ticker_bad(): wb = make_wrap_bittrex() fb = FakeBittrex() fb.result = {'success': True, - 'result': {'Bid': 1}} # incomplete result + 'result': {'Bid': 1, 'Ask': 0}} # incomplete result + + with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex params.*'): + wb.get_ticker('BTC_ETH') + fb.result = {'success': False, + 'message': 'gone bad' + } + with pytest.raises(btx.OperationalException, match=r'.*gone bad.*'): + wb.get_ticker('BTC_ETH') + + fb.result = {'success': True, + 'result': {}} # incomplete result with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex params.*'): wb.get_ticker('BTC_ETH') fb.result = {'success': False, From e5b27baa590cf72fe2a8131837c064bf829874c2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 13 Jan 2018 13:38:23 +0100 Subject: [PATCH 31/33] Update pymarketcap from 3.3.147 to 3.3.148 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 80cc78e17..2bc86ad61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 networkx==1.11 tabulate==0.8.2 -pymarketcap==3.3.147 +pymarketcap==3.3.148 # Required for plotting data #matplotlib==2.1.0 From 3277e491f18835fd7697bc4a4467277b5ba27703 Mon Sep 17 00:00:00 2001 From: kryofly Date: Sat, 13 Jan 2018 17:39:36 +0100 Subject: [PATCH 32/33] support download for multiple testdata sets --- docs/backtesting.md | 15 +++++++ freqtrade/misc.py | 5 +++ .../tests/testdata/download_backtest_data.py | 41 +++++++++++-------- freqtrade/tests/testdata/pairs.json | 23 +++++++++++ 4 files changed, 68 insertions(+), 16 deletions(-) create mode 100644 freqtrade/tests/testdata/pairs.json diff --git a/docs/backtesting.md b/docs/backtesting.md index c426e2b5c..8574d9dc2 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -51,6 +51,21 @@ python3 ./freqtrade/main.py backtesting --realistic-simulation --live python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101 ``` +To update your testdata directory, or download into another testdata directory: +```bash +mkdir freqtrade/tests/testdata-20180113 +cp freqtrade/tests/testdata/pairs.json freqtrade/tests/testdata-20180113 +cd freqtrade/tests/testdata-20180113 + +Possibly edit pairs.json file to include/exclude pairs + +python download_backtest_data.py -p pairs.json +``` + +The script will read your pairs.json file, and download ticker data +into the current working directory. + + For help about backtesting usage, please refer to [Backtesting commands](#backtesting-commands). diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 3d70f6b25..979174f8d 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -15,6 +15,11 @@ from freqtrade import __version__ logger = logging.getLogger(__name__) +def file_dump_json(filename, data): + with open(filename, 'w') as fp: + json.dump(data, fp) + + class State(enum.Enum): RUNNING = 0 STOPPED = 1 diff --git a/freqtrade/tests/testdata/download_backtest_data.py b/freqtrade/tests/testdata/download_backtest_data.py index 37cd4c95f..0cb545b3a 100755 --- a/freqtrade/tests/testdata/download_backtest_data.py +++ b/freqtrade/tests/testdata/download_backtest_data.py @@ -1,29 +1,38 @@ #!/usr/bin/env python3 """This script generate json data from bittrex""" +import sys import json -from os import path from freqtrade import exchange from freqtrade.exchange import Bittrex +from freqtrade import misc -PAIRS = [ - 'BTC_BCC', 'BTC_ETH', 'BTC_MER', 'BTC_POWR', 'BTC_ETC', - 'BTC_OK', 'BTC_NEO', 'BTC_EMC2', 'BTC_DASH', 'BTC_LSK', - 'BTC_LTC', 'BTC_XZC', 'BTC_OMG', 'BTC_STRAT', 'BTC_XRP', - 'BTC_QTUM', 'BTC_WAVES', 'BTC_VTC', 'BTC_XLM', 'BTC_MCO' -] -TICKER_INTERVAL = 5 # ticker interval in minutes (currently implemented: 1 and 5) -OUTPUT_DIR = path.dirname(path.realpath(__file__)) +parser = misc.common_args_parser('download utility') +parser.add_argument( + '-p', '--pair', + help='JSON file containing pairs to download', + dest='pair', + default=None +) +args = parser.parse_args(sys.argv[1:]) + +TICKER_INTERVALS = [1, 5] # ticker interval in minutes (currently implemented: 1 and 5) +PAIRS = [] + +if args.pair: + with open(args.pair) as file: + PAIRS = json.load(file) +PAIRS = list(set(PAIRS)) + +print('About to download pairs:', PAIRS) # Init Bittrex exchange exchange._API = Bittrex({'key': '', 'secret': ''}) for pair in PAIRS: - data = exchange.get_ticker_history(pair, TICKER_INTERVAL) - filename = path.join(OUTPUT_DIR, '{}-{}.json'.format( - pair, - TICKER_INTERVAL, - )) - with open(filename, 'w') as fp: - json.dump(data, fp) + for tick_interval in TICKER_INTERVALS: + print('downloading pair %s, interval %s' % (pair, tick_interval)) + data = exchange.get_ticker_history(pair, tick_interval) + filename = '{}-{}.json'.format(pair, tick_interval) + misc.file_dump_json(filename, data) diff --git a/freqtrade/tests/testdata/pairs.json b/freqtrade/tests/testdata/pairs.json new file mode 100644 index 000000000..c3e339b4c --- /dev/null +++ b/freqtrade/tests/testdata/pairs.json @@ -0,0 +1,23 @@ +[ + "BTC_ADA", + "BTC_BAT", + "BTC_DASH", + "BTC_ETC", + "BTC_ETH", + "BTC_GBYTE", + "BTC_LSK", + "BTC_LTC", + "BTC_NEO", + "BTC_NXT", + "BTC_POWR", + "BTC_STORJ", + "BTC_QTUM", + "BTC_WAVES", + "BTC_VTC", + "BTC_XLM", + "BTC_XMR", + "BTC_XVG", + "BTC_XRP", + "BTC_ZEC" +] + From 344843d802241f26877c0820ea5e3b7e6a869f7e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 13 Jan 2018 22:54:22 -0800 Subject: [PATCH 33/33] Update doc: 'cp' becomes 'cp -n', and add more FAQ questions --- docs/faq.md | 64 +++++++++++++++++++++++++++++++++++++----- docs/installation.md | 4 +-- docs/sql_cheatsheet.md | 12 ++++++++ 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 58929e15c..b3f15613a 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -2,20 +2,70 @@ #### I have waited 5 minutes, why hasn't the bot made any trades yet?! -Depending on the buy strategy, the amount of whitelisted coins, the situation of the market etc, it can take up to hours to find good entry position for a trade. Be patient! +Depending on the buy strategy, the amount of whitelisted coins, the +situation of the market etc, it can take up to hours to find good entry +position for a trade. Be patient! #### I have made 12 trades already, why is my total profit negative?! -I understand your disappointment but unfortunately 12 trades is just not enough to say anything. If you run backtesting, you can see that our current algorithm does leave you on the plus side, but that is after thousands of trades and even there, you will be left with losses on specific coins that you have traded tens if not hundreds of times. We of course constantly aim to improve the bot but it will _always_ be a gamble, which should leave you with modest wins on monthly basis but you can't say much from few trades. +I understand your disappointment but unfortunately 12 trades is just +not enough to say anything. If you run backtesting, you can see that our +current algorithm does leave you on the plus side, but that is after +thousands of trades and even there, you will be left with losses on +specific coins that you have traded tens if not hundreds of times. We +of course constantly aim to improve the bot but it will _always_ be a +gamble, which should leave you with modest wins on monthly basis but +you can't say much from few trades. -#### I’d like to change the stake amount. Can I just stop the bot with /stop and then change the config.json and run it again? +#### I’d like to change the stake amount. Can I just stop the bot with +/stop and then change the config.json and run it again? -Not quite. Trades are persisted to a database but the configuration is currently only read when the bot is killed and restarted. `/stop` more like pauses. You can stop your bot, adjust settings and start it again. +Not quite. Trades are persisted to a database but the configuration is +currently only read when the bot is killed and restarted. `/stop` more +like pauses. You can stop your bot, adjust settings and start it again. #### I want to improve the bot with a new strategy -That's great. We have a nice backtesting and hyperoptimizing setup. See the tutorial [[here|Testing-new-strategies-with-Hyperopt]]. +That's great. We have a nice backtesting and hyperoptimizing setup. See +the tutorial [here|Testing-new-strategies-with-Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands). -#### Is there a setting to only SELL the coins being held and not perform anymore BUYS? +#### Is there a setting to only SELL the coins being held and not +perform anymore BUYS? + +You can use the `/forcesell all` command from Telegram. + +### How many epoch do I need to get a good Hyperopt result? +Per default Hyperopts without `-e` or `--epochs` parameter will only +run 100 epochs, means 100 evals of your triggers, guards, .... Too few +to find a great result (unless if you are very lucky), so you probably +have to run it for 10.000 or more. But it will take an eternity to +compute. + +We recommend you to run it at least 10.000 epochs: +```bash +python3 ./freqtrade/main.py hyperopt -e 10000 +``` + +or if you want intermediate result to see +```bash +for i in {1..100}; do python3 ./freqtrade/main.py hyperopt -e 100; done +``` + +#### Why it is so long to run hyperopt? +Finding a great Hyperopt results takes time. + +If you wonder why it takes a while to find great hyperopt results + +This answer was written during the under the release 0.15.1, when we had +: +- 8 triggers +- 9 guards: let's say we evaluate even 10 values from each +- 1 stoploss calculation: let's say we want 10 values from that too to +be evaluated + +The following calculation is still very rough and not very precise +but it will give the idea. With only these triggers and guards there is +already 8*10^9*10 evaluations. A roughly total of 80 billion evals. +Did you run 100 000 evals? Congrats, you've done roughly 1 / 100 000 th +of the search space. -You can use the `/forcesell all` command from Telegram. \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md index 30431345b..59e0663ae 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -40,7 +40,7 @@ cd freqtrade ``` 4. Copy `config.sample` to `config.json` ```bash -cp config.json.example config.json +cp -n config.json.example config.json ``` To edit the config please refer to the [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) page 5. Create your DB file (Optional, the bot will create it if it is missing) @@ -194,7 +194,7 @@ The following steps are made for Linux/mac environment ## 4. Prepare the bot ```bash cd freqtrade -cp config.json.example config.json +cp -n config.json.example config.json ``` To edit the config please refer to [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 1edbcde5b..065f264f1 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -67,6 +67,18 @@ SET is_open=0, close_date='2017-12-20 03:08:45.103418', close_rate=0.19638016, c WHERE id=31; ``` +## Insert manually a new trade + +```sql +INSERT +INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date) +VALUES ('BITTREX', 'BTC_', 1, 0.0025, , , , '') +``` + +**Example:** +```sql +INSERT INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date) VALUES ('BITTREX', 'BTC_ETC', 1, 0.0025, 0.00258580, 0.002, 0.7715262081, '2017-11-28 12:44:24.000000') +``` ## Fix wrong fees in the table If your DB was created before