From f40d9dbb055e6e7414333ff51570f5777dd29842 Mon Sep 17 00:00:00 2001 From: kryofly Date: Sat, 20 Jan 2018 19:49:04 +0100 Subject: [PATCH 01/27] plot_profit uses --timerange flag --- scripts/plot_profit.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index e8bdbee5c..279facd6d 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -41,7 +41,9 @@ def make_profit_array(data, px, filter_pairs=[]): profit = trade[1] tim = trade[4] dur = trade[5] - pg[tim+dur-1] += profit + ix = tim + dur - 1 + if ix < px: + pg[ix] += profit # rewrite the pg array to go from # total profits at each timeframe @@ -76,9 +78,11 @@ def plot_profit(args) -> None: pairs = list(set(pairs) & set(filter_pairs)) print('Filter, keep pairs %s' % pairs) + timerange = misc.parse_timerange(args.timerange) tickers = optimize.load_data(args.datadir, pairs=pairs, ticker_interval=args.ticker_interval, - refresh_pairs=False) + refresh_pairs=False, + timerange=timerange) dataframes = optimize.preprocess(tickers) # Make an average close price of all the pairs that was involved. From 6171be4f46e704ecf1d60e3813188353f70a8c8e Mon Sep 17 00:00:00 2001 From: kryofly Date: Sun, 21 Jan 2018 13:44:30 +0100 Subject: [PATCH 02/27] Use dates on plot profit/dataframe * plot_dataframe also support --timerange * Both default to tkinter as matplotlib plotting backend --- docs/plotting.md | 21 +++++++++++ freqtrade/misc.py | 54 ++++++++++++++++++++++++++--- scripts/plot_dataframe.py | 73 ++++++++++++++++++++++++--------------- scripts/plot_profit.py | 47 +++++++++---------------- 4 files changed, 132 insertions(+), 63 deletions(-) diff --git a/docs/plotting.md b/docs/plotting.md index 598443e12..62671f219 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -17,6 +17,19 @@ python script/plot_dataframe.py -p BTC_ETH,BTC_LTC The -p pair argument, can be used to specify what pair you would like to plot. +**Advanced use** + +To plot the current live price use the --live flag: +``` +python scripts/plot_dataframe.py -p BTC_ETH --live +``` + +To plot a timerange (to zoom in): +``` +python scripts/plot_dataframe.py -p BTC_ETH --timerange=100-200 +``` +Timerange doesn't work with live data. + ## Plot profit @@ -46,3 +59,11 @@ Example ``` python python scripts/plot_profit.py --datadir ../freqtrade/freqtrade/tests/testdata-20171221/ -p BTC_LTC ``` + +**When it goes wrong** + +*** Linux: Can't display** + +If you are inside an python environment, you might want to set the +DISPLAY variable as so: +$ DISPLAY=:0 python scripts/plot_dataframe.py diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 5fe253fe5..b444afe90 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -5,8 +5,10 @@ import logging import time import os import re +from datetime import datetime from typing import Any, Callable, Dict, List +import numpy as np from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match from wrapt import synchronized @@ -16,11 +18,6 @@ from freqtrade import __version__ logger = logging.getLogger(__name__) -def file_dump_json(filename, data): - with open(filename, 'w') as fp: - json.dump(data, fp) - - class State(enum.Enum): RUNNING = 0 STOPPED = 1 @@ -30,6 +27,44 @@ class State(enum.Enum): _STATE = State.STOPPED +############################################ +# Used by scripts # +# Matplotlib doesn't support ::datetime64, # +# so we need to convert it into ::datetime # +############################################ + +def datesarray_to_datetimearray(dates): + """ + Convert an pandas-array of timestamps into + An numpy-array of datetimes + :return: numpy-array of datetime + """ + times = [] + dates = dates.astype(datetime) + for i in range(0, dates.size): + date = dates[i].to_pydatetime() + times.append(date) + return np.array(times) + + +def common_datearray(dfs): + alldates = {} + for pair, pair_data in dfs.items(): + dates = datesarray_to_datetimearray(pair_data['date']) + for date in dates: + alldates[date] = 1 + lst = [] + for date, _ in alldates.items(): + lst.append(date) + arr = np.array(lst) + return np.sort(arr, axis=0) + + +def file_dump_json(filename, data): + with open(filename, 'w') as fp: + json.dump(data, fp) + + @synchronized def update_state(state: State) -> None: """ @@ -155,6 +190,15 @@ def parse_args(args: List[str], description: str): return parser.parse_args(args) +def scripts_options(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + '-p', '--pair', + help='Show profits for only this pairs. Pairs are comma-separated.', + dest='pair', + default=None + ) + + def backtesting_options(parser: argparse.ArgumentParser) -> None: parser.add_argument( '-l', '--live', diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index e9bf65f47..87f18c098 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -1,23 +1,23 @@ #!/usr/bin/env python3 import sys +import logging import argparse -import matplotlib # Install PYQT5 manually if you want to test this helper function -matplotlib.use("Qt5Agg") +import matplotlib +import matplotlib.dates as mdates import matplotlib.pyplot as plt from freqtrade import exchange, analyze -from freqtrade.misc import common_args_parser +import freqtrade.misc as misc +import freqtrade.optimize as optimize +import freqtrade.analyze as analyze + +logger = logging.getLogger(__name__) -def plot_parse_args(args ): - parser = common_args_parser(description='Graph utility') - parser.add_argument( - '-p', '--pair', - help = 'What currency pair', - dest = 'pair', - default = 'BTC_ETH', - type = str, - ) +def plot_parse_args(args): + parser = misc.common_args_parser('Graph dataframe') + misc.backtesting_options(parser) + misc.scripts_options(parser) return parser.parse_args(args) @@ -28,11 +28,25 @@ def plot_analyzed_dataframe(args): :return: None """ pair = args.pair + pairs = [pair] + timerange = misc.parse_timerange(args.timerange) - # Init Bittrex to use public API - exchange._API = exchange.Bittrex({'key': '', 'secret': ''}) - ticker = exchange.get_ticker_history(pair) - dataframe = analyze.analyze_ticker(ticker) + tickers = {} + if args.live: + logger.info('Downloading pair.') + exchange._API = exchange.Bittrex({'key': '', 'secret': ''}) + tickers[pair] = exchange.get_ticker_history(pair, args.ticker_interval) + else: + tickers = optimize.load_data(args.datadir, pairs=pairs, + ticker_interval=args.ticker_interval, + refresh_pairs=False, + timerange=timerange) + dataframes = optimize.tickerdata_to_dataframe(tickers) + dataframe = dataframes[pair] + dataframe = analyze.populate_buy_trend(dataframe) + dataframe = analyze.populate_sell_trend(dataframe) + + dates = misc.datesarray_to_datetimearray(dataframe['date']) dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close'] dataframe.loc[dataframe['sell'] == 1, 'sell_price'] = dataframe['close'] @@ -40,27 +54,30 @@ def plot_analyzed_dataframe(args): # Two subplots sharing x axis fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True) fig.suptitle(pair, fontsize=14, fontweight='bold') - ax1.plot(dataframe.index.values, dataframe['close'], label='close') - # ax1.plot(dataframe.index.values, dataframe['sell'], 'ro', label='sell') - ax1.plot(dataframe.index.values, dataframe['sma'], '--', label='SMA') - ax1.plot(dataframe.index.values, dataframe['tema'], ':', label='TEMA') - ax1.plot(dataframe.index.values, dataframe['blower'], '-.', label='BB low') - ax1.plot(dataframe.index.values, dataframe['buy_price'], 'bo', label='buy') + ax1.plot(dates, dataframe['close'], label='close') + # ax1.plot(dates, dataframe['sell'], 'ro', label='sell') + ax1.plot(dates, dataframe['sma'], '--', label='SMA') + ax1.plot(dates, dataframe['tema'], ':', label='TEMA') + ax1.plot(dates, dataframe['blower'], '-.', label='BB low') + ax1.plot(dates, dataframe['buy_price'], 'bo', label='buy') ax1.legend() - ax2.plot(dataframe.index.values, dataframe['adx'], label='ADX') - ax2.plot(dataframe.index.values, dataframe['mfi'], label='MFI') - # ax2.plot(dataframe.index.values, [25] * len(dataframe.index.values)) + ax2.plot(dates, dataframe['adx'], label='ADX') + ax2.plot(dates, dataframe['mfi'], label='MFI') + # ax2.plot(dates, [25] * len(dataframe.index.values)) ax2.legend() - ax3.plot(dataframe.index.values, dataframe['fastk'], label='k') - ax3.plot(dataframe.index.values, dataframe['fastd'], label='d') - ax3.plot(dataframe.index.values, [20] * len(dataframe.index.values)) + ax3.plot(dates, dataframe['fastk'], label='k') + ax3.plot(dates, dataframe['fastd'], label='d') + ax3.plot(dates, [20] * len(dataframe.index.values)) ax3.legend() + xfmt = mdates.DateFormatter('%d-%m-%y %H:%M') # Dont let matplotlib autoformat date + ax3.xaxis.set_major_formatter(xfmt) # Fine-tune figure; make subplots close to each other and hide x ticks for # all but bottom plot. fig.subplots_adjust(hspace=0) + fig.autofmt_xdate() # Rotate the dates plt.setp([a.get_xticklabels() for a in fig.axes[:-1]], visible=False) plt.show() diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 279facd6d..08941cb2a 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -4,24 +4,19 @@ import sys import argparse import json import matplotlib.pyplot as plt +import matplotlib.dates as mdates import numpy as np import freqtrade.optimize as optimize import freqtrade.misc as misc import freqtrade.exchange as exchange -import freqtrade.analyze as analyze -def plot_parse_args(args ): - parser = misc.common_args_parser('Graph utility') +def plot_parse_args(args): + parser = misc.common_args_parser('Graph profits') # FIX: perhaps delete those backtesting options that are not feasible (shows up in -h) misc.backtesting_options(parser) - parser.add_argument( - '-p', '--pair', - help = 'Show profits for only this pairs. Pairs are comma-separated.', - dest = 'pair', - default = None - ) + misc.scripts_options(parser) return parser.parse_args(args) @@ -85,23 +80,14 @@ def plot_profit(args) -> None: timerange=timerange) dataframes = optimize.preprocess(tickers) + # NOTE: the dataframes are of unequal length, + # 'dates' is an merged date array of them all. + + dates = misc.common_datearray(dataframes) + max_x = dates.size + # Make an average close price of all the pairs that was involved. # this could be useful to gauge the overall market trend - - # FIX: since the dataframes are of unequal length, - # andor has different dates, we need to merge them - # But we dont have the date information in the - # backtesting results, this is needed to match the dates - # For now, assume the dataframes are aligned. - max_x = 0 - for pair, pair_data in dataframes.items(): - n = len(pair_data['close']) - max_x = max(max_x, n) - # if max_x != n: - # raise Exception('Please rerun script. Input data has different lengths %s' - # %('Different pair length: %s <=> %s' %(max_x, n))) - print('max_x: %s' %(max_x)) - # We are essentially saying: # array <- sum dataframes[*]['close'] / num_items dataframes # FIX: there should be some onliner numpy/panda for this @@ -131,8 +117,9 @@ def plot_profit(args) -> None: fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True) fig.suptitle('total profit') - ax1.plot(avgclose, label='avgclose') - ax2.plot(pg, label='profit') + + ax1.plot(dates, avgclose, label='avgclose') + ax2.plot(dates, pg, label='profit') ax1.legend(loc='upper left') ax2.legend(loc='upper left') @@ -142,15 +129,15 @@ def plot_profit(args) -> None: # In third graph, we plot each profit separately for pair in pairs: pg = make_profit_array(data, max_x, pair) - ax3.plot(pg, label=pair) + ax3.plot(dates, pg, label=pair) ax3.legend(loc='upper left') # black background to easier see multiple colors ax3.set_facecolor('black') + xfmt = mdates.DateFormatter('%d-%m-%y %H:%M') # Dont let matplotlib autoformat date + ax3.xaxis.set_major_formatter(xfmt) - # Fine-tune figure; make subplots close to each other and hide x ticks for - # all but bottom plot. fig.subplots_adjust(hspace=0) - plt.setp([a.get_xticklabels() for a in fig.axes[:-1]], visible=False) + fig.autofmt_xdate() # Rotate the dates plt.show() From b7e297ebda06cf654e3b0a6e0228261d5f3efaf1 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 11:25:12 +0200 Subject: [PATCH 03/27] remove unused loop variable --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a21bdb61a..8e244bdda 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -26,7 +26,7 @@ def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow] :return: tuple containing min_date, max_date """ all_dates = Series([]) - for pair, pair_data in data.items(): + for pair_data in data.values(): all_dates = all_dates.append(pair_data['date']) all_dates.sort_values(inplace=True) return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1]) From 29c84bf622574060a4d2ee3f49bdf77a3010d308 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 26 Jan 2018 16:23:43 +0100 Subject: [PATCH 04/27] Update pymarketcap from 3.3.150 to 3.3.152 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0d58bcbf4..19828a8e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 networkx==1.11 tabulate==0.8.2 -pymarketcap==3.3.150 +pymarketcap==3.3.152 # Required for plotting data #matplotlib==2.1.0 From a7a7c3712168c01bb8b35dec738cd9b7d0821084 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 11:25:35 +0200 Subject: [PATCH 05/27] add day counter to timeframe --- freqtrade/optimize/backtesting.py | 5 ++++- freqtrade/tests/optimize/test_backtesting.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8e244bdda..a78c9e197 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -212,7 +212,10 @@ def start(args): preprocessed = optimize.tickerdata_to_dataframe(data) # Print timeframe min_date, max_date = get_timeframe(preprocessed) - logger.info('Measuring data from %s up to %s ...', min_date.isoformat(), max_date.isoformat()) + logger.info('Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date-min_date).days) # Execute backtest and print results sell_profit_only = config.get('experimental', {}).get('sell_profit_only', False) use_sell_signal = config.get('experimental', {}).get('use_sell_signal', False) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f88cdd9b9..f5db4d037 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -193,7 +193,8 @@ def test_backtest_start(default_conf, mocker, caplog): # check the logs, that will contain the backtest result exists = ['Using max_open_trades: 1 ...', 'Using stake_amount: 0.001 ...', - 'Measuring data from 2017-11-14T21:17:00+00:00 up to 2017-11-14T22:59:00+00:00 ...'] + 'Measuring data from 2017-11-14T21:17:00+00:00 ' + 'up to 2017-11-14T22:59:00+00:00 (0 days)..'] for line in exists: assert ('freqtrade.optimize.backtesting', logging.INFO, From f33923c7842ad0c440d5d5f9bd028331ea4b9ac3 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 11:48:34 +0200 Subject: [PATCH 06/27] fix typings for hyperopt code --- freqtrade/optimize/hyperopt.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4c73dc76f..6f63c159f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -10,7 +10,7 @@ import sys from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, List +from typing import Dict, Any, Callable import numpy import talib.abstract as ta @@ -35,7 +35,7 @@ logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) logger = logging.getLogger(__name__) -# set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data +# set TARGET_TRADES to suit your number concurrent trades so its realistic to the number of days TARGET_TRADES = 600 TOTAL_TRIES = 0 _CURRENT_TRIES = 0 @@ -225,7 +225,7 @@ def calculate_loss(total_profit: float, trade_count: int, trade_duration: float) return trade_loss + profit_loss + duration_loss -def generate_roi_table(params): +def generate_roi_table(params) -> Dict[str, float]: roi_table = {} roi_table["0"] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] roi_table[str(params['roi_t3'])] = params['roi_p1'] + params['roi_p2'] @@ -235,7 +235,7 @@ def generate_roi_table(params): return roi_table -def roi_space() -> List[Dict]: +def roi_space() -> Dict[str, Any]: return { 'roi_t1': hp.quniform('roi_t1', 10, 220, 10), 'roi_t2': hp.quniform('roi_t2', 10, 120, 10), @@ -246,13 +246,13 @@ def roi_space() -> List[Dict]: } -def stoploss_space() -> Dict: +def stoploss_space() -> Dict[str, Any]: return { 'stoploss': hp.uniform('stoploss', -0.5, -0.02), } -def indicator_space() -> List[Dict]: +def indicator_space() -> Dict[str, Any]: """ Define your Hyperopt space for searching strategy parameters """ @@ -312,11 +312,11 @@ def indicator_space() -> List[Dict]: } -def hyperopt_space() -> List[Dict]: +def hyperopt_space() -> Dict[str, Any]: return {**indicator_space(), **roi_space(), **stoploss_space()} -def buy_strategy_generator(params) -> None: +def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Define the buy strategy parameters to be used by hyperopt """ From 95ab7c84bcfa837e0709481cdfdd9325b48850b9 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:41:41 +0200 Subject: [PATCH 07/27] remove unnecessary else --- freqtrade/optimize/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 08630af40..a0eb5b8a2 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -22,8 +22,8 @@ def trim_tickerlist(tickerlist, timerange): return tickerlist[0:start] elif stype == ('index', 'index'): return tickerlist[start:stop] - else: - return tickerlist + + return tickerlist def load_tickerdata_file(datadir, pair, ticker_interval, From 5505845c6fd0210bceceef2729a2dad094ae1754 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:48:53 +0200 Subject: [PATCH 08/27] remove unused method parameter --- freqtrade/main.py | 4 ++-- freqtrade/tests/test_main.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 3bf945b99..302d4cab0 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -55,7 +55,7 @@ def refresh_whitelist(whitelist: List[str]) -> List[str]: return final_list -def process_maybe_execute_buy(conf, interval): +def process_maybe_execute_buy(interval): """ Tries to execute a buy trade in a safe way :return: True if executed @@ -115,7 +115,7 @@ def _process(interval: int, nb_assets: Optional[int] = 0) -> bool: # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() if len(trades) < _CONF['max_open_trades']: - state_changed = process_maybe_execute_buy(_CONF, interval) + state_changed = process_maybe_execute_buy(interval) for trade in trades: state_changed |= process_maybe_execute_sell(trade, interval) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index cf0b5283b..225f5e28c 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -50,9 +50,9 @@ def test_main_start_hyperopt(mocker): def test_process_maybe_execute_buy(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.create_trade', return_value=True) - assert main.process_maybe_execute_buy(default_conf, int(default_conf['ticker_interval'])) + assert main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) mocker.patch('freqtrade.main.create_trade', return_value=False) - assert not main.process_maybe_execute_buy(default_conf, int(default_conf['ticker_interval'])) + assert not main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) def test_process_maybe_execute_sell(default_conf, mocker): @@ -71,7 +71,7 @@ def test_process_maybe_execute_sell(default_conf, mocker): def test_process_maybe_execute_buy_exception(default_conf, mocker, caplog): mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.create_trade', MagicMock(side_effect=DependencyException)) - main.process_maybe_execute_buy(default_conf, int(default_conf['ticker_interval'])) + main.process_maybe_execute_buy(int(default_conf['ticker_interval'])) tt.log_has('Unable to create trade:', caplog.record_tuples) From 42919e8864c6b1ff53bd867c6a82ceb7dd460413 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:49:14 +0200 Subject: [PATCH 09/27] give type hint for _CONF --- freqtrade/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 302d4cab0..c10fb2c5b 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -6,7 +6,7 @@ import sys import time import traceback from datetime import datetime -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Any import arrow import requests @@ -23,7 +23,7 @@ from freqtrade.strategy.strategy import Strategy logger = logging.getLogger('freqtrade') -_CONF = {} +_CONF: Dict[str, Any] = {} def refresh_whitelist(whitelist: List[str]) -> List[str]: From e14007ced42bb12d248201cc18b1eb5cb3d9b05b Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:52:39 +0200 Subject: [PATCH 10/27] sort imports --- freqtrade/tests/optimize/test_optimize.py | 1 - freqtrade/tests/test_analyze.py | 4 ++-- freqtrade/tests/test_main.py | 2 +- freqtrade/tests/test_misc.py | 6 +++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 02b675cf9..22bffc1ff 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -2,7 +2,6 @@ import os import logging -# from unittest.mock import MagicMock from shutil import copyfile from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index f77a2d71a..516bb3607 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -1,13 +1,13 @@ # pragma pylint: disable=missing-docstring,W0621 +import datetime import json from unittest.mock import MagicMock -import freqtrade.tests.conftest as tt # test tools import arrow -import datetime import pytest from pandas import DataFrame +import freqtrade.tests.conftest as tt # test tools from freqtrade.analyze import (get_signal, parse_ticker_dataframe, populate_buy_trend, populate_indicators, populate_sell_trend) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 225f5e28c..0c9e594a2 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -2,7 +2,6 @@ import copy import logging from unittest.mock import MagicMock -import freqtrade.tests.conftest as tt # test tools import arrow import pytest @@ -10,6 +9,7 @@ import requests from sqlalchemy import create_engine import freqtrade.main as main +import freqtrade.tests.conftest as tt # test tools from freqtrade import DependencyException, OperationalException from freqtrade.exchange import Exchanges from freqtrade.main import (_process, check_handle_timedout, create_trade, diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index fb768b89c..52a8c3991 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -3,13 +3,13 @@ import argparse import json import time from copy import deepcopy +from unittest.mock import MagicMock import pytest -from unittest.mock import MagicMock from jsonschema import ValidationError -from freqtrade.misc import (common_args_parser, load_config, parse_args, - throttle, file_dump_json, parse_timerange) +from freqtrade.misc import (common_args_parser, file_dump_json, load_config, + parse_args, parse_timerange, throttle) def test_throttle(): From b547893fbf84402a165918cc23af40f34d63a60c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 26 Jan 2018 17:53:44 +0100 Subject: [PATCH 11/27] Update pymarketcap from 3.3.152 to 3.3.153 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 19828a8e0..01490e298 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 networkx==1.11 tabulate==0.8.2 -pymarketcap==3.3.152 +pymarketcap==3.3.153 # Required for plotting data #matplotlib==2.1.0 From 0ff56c6e8d4e70d24343a967656fd40470088225 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:54:15 +0200 Subject: [PATCH 12/27] use uppercase constant --- freqtrade/tests/optimize/test_optimize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 22bffc1ff..b2820f367 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -9,7 +9,7 @@ from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\ download_backtesting_testdata, load_tickerdata_file # Change this if modifying BTC_UNITEST testdatafile -_btc_unittest_length = 13681 +_BTC_UNITTEST_LENGTH = 13681 def _backup_file(file: str, copy_file: bool = False) -> None: @@ -209,7 +209,7 @@ def test_download_backtesting_testdata2(default_conf, mocker): def test_load_tickerdata_file(): assert not load_tickerdata_file(None, 'BTC_UNITEST', 7) tickerdata = load_tickerdata_file(None, 'BTC_UNITEST', 1) - assert _btc_unittest_length == len(tickerdata) + assert _BTC_UNITTEST_LENGTH == len(tickerdata) def test_init(default_conf, mocker): From a5690e707d3b0942a114bdee1c352cc833b38c58 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:56:56 +0200 Subject: [PATCH 13/27] remove unused parameters --- freqtrade/tests/optimize/test_optimize.py | 2 +- freqtrade/tests/test_main.py | 8 ++++---- freqtrade/tests/test_misc.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index b2820f367..c69284c32 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -198,7 +198,7 @@ def test_download_backtesting_testdata(default_conf, ticker_history, mocker): _clean_test_file(file2) -def test_download_backtesting_testdata2(default_conf, mocker): +def test_download_backtesting_testdata2(mocker): tick = [{'T': 'bar'}, {'T': 'foo'}] mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 0c9e594a2..8ac06ee69 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -256,7 +256,7 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, mocker): create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) -def test_create_trade_no_signal(default_conf, ticker, mocker): +def test_create_trade_no_signal(default_conf, mocker): default_conf['dry_run'] = True mocker.patch.dict('freqtrade.main._CONF', default_conf) mocker.patch('freqtrade.main.get_signal', MagicMock(return_value=(False, False))) @@ -308,7 +308,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, mocker): assert trade.close_date is not None -def test_handle_overlpapping_signals(default_conf, ticker, mocker, caplog): +def test_handle_overlpapping_signals(default_conf, ticker, mocker): default_conf.update({'experimental': {'use_sell_signal': True}}) mocker.patch.dict('freqtrade.main._CONF', default_conf) @@ -471,7 +471,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mo assert len(trades) == 0 -def test_handle_timedout_limit_buy(default_conf, mocker): +def test_handle_timedout_limit_buy(mocker): cancel_order = MagicMock() mocker.patch('freqtrade.exchange.cancel_order', cancel_order) Trade.session = MagicMock() @@ -519,7 +519,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, assert trade_sell.is_open is True -def test_handle_timedout_limit_sell(default_conf, mocker): +def test_handle_timedout_limit_sell(mocker): cancel_order = MagicMock() mocker.patch('freqtrade.exchange.cancel_order', cancel_order) trade = MagicMock() diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 52a8c3991..96743dfab 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -124,7 +124,7 @@ def test_parse_args_backtesting_custom(): assert call_args.refresh_pairs is True -def test_parse_args_hyperopt_custom(mocker): +def test_parse_args_hyperopt_custom(): args = ['-c', 'test_conf.json', 'hyperopt', '--epochs', '20'] call_args = parse_args(args, '') assert call_args.config == 'test_conf.json' @@ -134,7 +134,7 @@ def test_parse_args_hyperopt_custom(mocker): assert call_args.func is not None -def test_file_dump_json(default_conf, mocker): +def test_file_dump_json(mocker): file_open = mocker.patch('freqtrade.misc.open', MagicMock()) json_dump = mocker.patch('json.dump', MagicMock()) file_dump_json('somefile', [1, 2, 3]) From 1eebbebed100e3c55a0529e32e27466c9d2edffc Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 18:57:09 +0200 Subject: [PATCH 14/27] fix assert order --- freqtrade/tests/optimize/test_optimize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index c69284c32..8a18b58e3 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -224,4 +224,4 @@ def test_tickerdata_to_dataframe(): tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange) tickerlist = {'BTC_UNITEST': tick} data = optimize.tickerdata_to_dataframe(tickerlist) - assert 100 == len(data['BTC_UNITEST']) + assert len(data['BTC_UNITEST']) == 100 From 67995a2f490150014ad338492f7c00a249927524 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 26 Jan 2018 19:01:54 +0200 Subject: [PATCH 15/27] remove unnecessary else statements --- freqtrade/main.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index c10fb2c5b..efcb27543 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -64,12 +64,12 @@ def process_maybe_execute_buy(interval): # Create entity and execute trade if create_trade(float(_CONF['stake_amount']), interval): return True - else: - logger.info( - 'Checked all whitelisted currencies. ' - 'Found no suitable entry positions for buying. Will keep looking ...' - ) - return False + + logger.info( + 'Checked all whitelisted currencies. ' + 'Found no suitable entry positions for buying. Will keep looking ...' + ) + return False except DependencyException as exception: logger.warning('Unable to create trade: %s', exception) return False @@ -159,16 +159,16 @@ def handle_timedout_limit_buy(trade: Trade, order: Dict) -> bool: rpc.send_msg('*Timeout:* Unfilled buy order for {} cancelled'.format( trade.pair.replace('_', '/'))) return True - else: - # if trade is partially complete, edit the stake details for the trade - # and close the order - trade.amount = order['amount'] - order['remaining'] - trade.stake_amount = trade.amount * trade.open_rate - trade.open_order_id = None - logger.info('Partial buy order timeout for %s.', trade) - rpc.send_msg('*Timeout:* Remaining buy order for {} cancelled'.format( - trade.pair.replace('_', '/'))) - return False + + # if trade is partially complete, edit the stake details for the trade + # and close the order + trade.amount = order['amount'] - order['remaining'] + trade.stake_amount = trade.amount * trade.open_rate + trade.open_order_id = None + logger.info('Partial buy order timeout for %s.', trade) + rpc.send_msg('*Timeout:* Remaining buy order for {} cancelled'.format( + trade.pair.replace('_', '/'))) + return False # FIX: 20180110, should cancel_order() be cond. or unconditionally called? @@ -189,9 +189,9 @@ def handle_timedout_limit_sell(trade: Trade, order: Dict) -> bool: trade.pair.replace('_', '/'))) logger.info('Sell order timeout for %s.', trade) return True - else: - # TODO: figure out how to handle partially complete sell orders - return False + + # TODO: figure out how to handle partially complete sell orders + return False def check_handle_timedout(timeoutvalue: int) -> None: From 67ddb2e7f8164738cc99b3f5dc2b041c99b92c66 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 27 Jan 2018 09:39:48 +0200 Subject: [PATCH 16/27] lower precision for most search space variables --- freqtrade/optimize/hyperopt.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 6f63c159f..9962b0260 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -237,8 +237,8 @@ def generate_roi_table(params) -> Dict[str, float]: def roi_space() -> Dict[str, Any]: return { - 'roi_t1': hp.quniform('roi_t1', 10, 220, 10), - 'roi_t2': hp.quniform('roi_t2', 10, 120, 10), + 'roi_t1': hp.quniform('roi_t1', 10, 220, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 120, 15), 'roi_t3': hp.quniform('roi_t3', 10, 120, 10), 'roi_p1': hp.quniform('roi_p1', 0.01, 0.05, 0.01), 'roi_p2': hp.quniform('roi_p2', 0.01, 0.10, 0.01), @@ -248,7 +248,7 @@ def roi_space() -> Dict[str, Any]: def stoploss_space() -> Dict[str, Any]: return { - 'stoploss': hp.uniform('stoploss', -0.5, -0.02), + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), } @@ -263,19 +263,19 @@ def indicator_space() -> Dict[str, Any]: ]), 'mfi': hp.choice('mfi', [ {'enabled': False}, - {'enabled': True, 'value': hp.quniform('mfi-value', 5, 25, 1)} + {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} ]), 'fastd': hp.choice('fastd', [ {'enabled': False}, - {'enabled': True, 'value': hp.quniform('fastd-value', 10, 50, 1)} + {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} ]), 'adx': hp.choice('adx', [ {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 15, 50, 1)} + {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} ]), 'rsi': hp.choice('rsi', [ {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)} + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} ]), 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ {'enabled': False}, From 67c6c380e1ddcf54e05246f8348045f6ff49e673 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 27 Jan 2018 17:33:04 -0800 Subject: [PATCH 17/27] Increase pylint score for fiat_convert --- freqtrade/fiat_convert.py | 14 +++++++++++++- freqtrade/strategy/default_strategy.py | 4 +++- freqtrade/tests/test_fiat_convert.py | 15 ++++++++++----- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 91e04fff9..6f9d3d3d5 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -1,12 +1,19 @@ +""" +Module that define classes to convert Crypto-currency to FIAT +e.g BTC to USD +""" + import logging import time - from pymarketcap import Pymarketcap logger = logging.getLogger(__name__) class CryptoFiat(): + """ + Object to describe what is the price of Crypto-currency in a FIAT + """ # Constants CACHE_DURATION = 6 * 60 * 60 # 6 hours @@ -49,6 +56,11 @@ class CryptoFiat(): class CryptoToFiatConverter(object): + """ + Main class to initiate Crypto to FIAT. + This object contains a list of pair Crypto, FIAT + This object is also a Singleton + """ __instance = None _coinmarketcap = None diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index c89b20527..abccf065b 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -1,7 +1,9 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + import talib.abstract as ta +from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.strategy.interface import IStrategy -from pandas import DataFrame class_name = 'DefaultStrategy' diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 2d112f921..7d0acfc91 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -1,4 +1,5 @@ -# pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103 +# pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, +# pragma pylint: disable=protected-access, C0103 import time from unittest.mock import MagicMock @@ -47,16 +48,19 @@ def test_fiat_convert_is_supported(): def test_fiat_convert_add_pair(): fiat_convert = CryptoToFiatConverter() - assert len(fiat_convert._pairs) == 0 + pair_len = len(fiat_convert._pairs) + assert pair_len == 0 fiat_convert._add_pair(crypto_symbol='btc', fiat_symbol='usd', price=12345.0) - assert len(fiat_convert._pairs) == 1 + pair_len = len(fiat_convert._pairs) + assert pair_len == 1 assert fiat_convert._pairs[0].crypto_symbol == 'BTC' assert fiat_convert._pairs[0].fiat_symbol == 'USD' assert fiat_convert._pairs[0].price == 12345.0 fiat_convert._add_pair(crypto_symbol='btc', fiat_symbol='Eur', price=13000.2) - assert len(fiat_convert._pairs) == 2 + pair_len = len(fiat_convert._pairs) + assert pair_len == 2 assert fiat_convert._pairs[1].crypto_symbol == 'BTC' assert fiat_convert._pairs[1].fiat_symbol == 'EUR' assert fiat_convert._pairs[1].price == 13000.2 @@ -95,7 +99,8 @@ def test_fiat_convert_get_price(mocker): fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='US Dollar') # Check the value return by the method - assert len(fiat_convert._pairs) == 0 + pair_len = len(fiat_convert._pairs) + assert pair_len == 0 assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='USD') == 28000.0 assert fiat_convert._pairs[0].crypto_symbol == 'BTC' assert fiat_convert._pairs[0].fiat_symbol == 'USD' From f33bc93639a2f2b1b270967928219ea5c14a83d5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 28 Jan 2018 04:38:46 +0100 Subject: [PATCH 18/27] Update python-bittrex from 0.2.2 to 0.3.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 01490e298..a21d53087 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -python-bittrex==0.2.2 +python-bittrex==0.3.0 SQLAlchemy==1.2.2 python-telegram-bot==9.0.0 arrow==0.12.1 From 776dd4a0d5fd834f5cfe8c28bcd9f263009b13c3 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 27 Jan 2018 21:26:57 -0800 Subject: [PATCH 19/27] Increase pylint score on strategy --- freqtrade/strategy/interface.py | 21 ++++++--- freqtrade/strategy/strategy.py | 53 +++++++++++++---------- freqtrade/tests/strategy/test_strategy.py | 2 - 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index ce5f08cd2..9281e72ca 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -1,8 +1,22 @@ +""" +IStrategy interface +This module defines the interface to apply for strategies +""" + from abc import ABC, abstractmethod from pandas import DataFrame class IStrategy(ABC): + """ + Interface for freqtrade strategies + Defines the mandatory structure must follow any custom strategies + + Attributes you can use: + minimal_roi -> Dict: Minimal ROI designed for the strategy + stoploss -> float: optimal stoploss designed for the strategy + ticker_interval -> int: value of the ticker interval to use for the strategy + """ @property def name(self) -> str: """ @@ -11,13 +25,6 @@ class IStrategy(ABC): """ return self.__class__.__name__ - """ - Attributes you can use: - minimal_roi -> Dict: Minimal ROI designed for the strategy - stoploss -> float: optimal stoploss designed for the strategy - ticker_interval -> int: value of the ticker interval to use for the strategy - """ - @abstractmethod def populate_indicators(self, dataframe: DataFrame) -> DataFrame: """ diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py index 2545e378c..97e260ee8 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/strategy.py @@ -1,10 +1,12 @@ +""" +This module load custom strategies +""" import os import sys import logging import importlib from pandas import DataFrame -from typing import Dict from freqtrade.strategy.interface import IStrategy @@ -12,16 +14,36 @@ sys.path.insert(0, r'../../user_data/strategies') class Strategy(object): + """ + This class contains all the logic to load custom strategy class + """ __instance = None DEFAULT_STRATEGY = 'default_strategy' def __new__(cls): + """ + Used to create the Singleton + :return: Strategy object + """ if Strategy.__instance is None: Strategy.__instance = object.__new__(cls) return Strategy.__instance + def __init__(self): + if Strategy.__instance is None: + self.logger = None + self.minimal_roi = None + self.stoploss = None + self.ticker_interval = None + self.custom_strategy = None + def init(self, config): + """ + Load the custom class from config parameter + :param config: + :return: + """ self.logger = logging.getLogger(__name__) # Verify the strategy is in the configuration, otherwise fallback to the default strategy @@ -42,21 +64,22 @@ class Strategy(object): if 'stoploss' in config: self.custom_strategy.stoploss = config['stoploss'] self.logger.info( - "Override strategy \'stoploss\' with value in config file: {}.".format( - config['stoploss'] - ) + "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] ) if 'ticker_interval' in config: self.custom_strategy.ticker_interval = config['ticker_interval'] self.logger.info( - "Override strategy \'ticker_interval\' with value in config file: {}.".format( - config['ticker_interval'] - ) + "Override strategy \'ticker_interval\' with value in config file: %s.", + config['ticker_interval'] ) + # Minimal ROI designed for the strategy self.minimal_roi = self.custom_strategy.minimal_roi + + # Optimal stoploss designed for the strategy self.stoploss = self.custom_strategy.stoploss + self.ticker_interval = self.custom_strategy.ticker_interval def _load_strategy(self, strategy_name: str) -> None: @@ -90,7 +113,7 @@ class Strategy(object): module = importlib.import_module(filename, __package__) custom_strategy = getattr(module, module.class_name) - self.logger.info("Load strategy class: {} ({}.py)".format(module.class_name, filename)) + self.logger.info("Load strategy class: %s (%s.py)", module.class_name, filename) return custom_strategy() @staticmethod @@ -126,20 +149,6 @@ class Strategy(object): return path - def minimal_roi(self) -> Dict: - """ - Minimal ROI designed for the strategy - :return: Dict: Value for the Minimal ROI - """ - return - - def stoploss(self) -> float: - """ - Optimal stoploss designed for the strategy - :return: float | return None to disable it - """ - return self.custom_strategy.stoploss - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 79f045a6d..c7ee7bce1 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -28,8 +28,6 @@ def test_search_strategy(): def test_strategy_structure(): assert hasattr(Strategy, 'init') - assert hasattr(Strategy, 'minimal_roi') - assert hasattr(Strategy, 'stoploss') assert hasattr(Strategy, 'populate_indicators') assert hasattr(Strategy, 'populate_buy_trend') assert hasattr(Strategy, 'populate_sell_trend') From d824816880b2cc89b6aa78c61caa840762332a7f Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 27 Jan 2018 23:38:41 -0800 Subject: [PATCH 20/27] Increase pylint score on test files --- freqtrade/tests/conftest.py | 16 ++++++++++++ freqtrade/tests/exchange/test_exchange.py | 25 ++++++++++--------- .../tests/exchange/test_exchange_bittrex.py | 22 ++++++---------- freqtrade/tests/optimize/test_backtesting.py | 17 +++---------- freqtrade/tests/optimize/test_optimize.py | 17 +++++-------- freqtrade/tests/rpc/test_rpc_telegram.py | 1 + freqtrade/tests/strategy/test_strategy.py | 11 ++------ freqtrade/tests/test_acl_pair.py | 15 ++++------- freqtrade/tests/test_analyze.py | 10 +------- freqtrade/tests/test_dataframe.py | 3 ++- freqtrade/tests/test_main.py | 17 ++++++++----- freqtrade/tests/test_persistence.py | 4 +-- 12 files changed, 70 insertions(+), 88 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 1e53648e1..053b980f7 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -3,10 +3,13 @@ from datetime import datetime from unittest.mock import MagicMock from functools import reduce +import json import arrow import pytest from jsonschema import validate from telegram import Chat, Message, Update +from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.strategy.strategy import Strategy from freqtrade.misc import CONF_SCHEMA @@ -256,3 +259,16 @@ def ticker_history_without_bv(): "T": "2017-11-26T09:00:00" } ] + + +@pytest.fixture +def result(): + with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: + return parse_ticker_dataframe(json.load(data_file)) + + +@pytest.fixture +def default_strategy(): + strategy = Strategy() + strategy.init({'strategy': 'default_strategy'}) + return strategy diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index a0386733d..9761fff3a 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1,8 +1,9 @@ -# pragma pylint: disable=missing-docstring,C0103 +# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement +# pragma pylint: disable=protected-access from unittest.mock import MagicMock -from requests.exceptions import RequestException from random import randint import logging +from requests.exceptions import RequestException import pytest from freqtrade import OperationalException @@ -30,7 +31,7 @@ def test_init(default_conf, mocker, caplog): ) in caplog.record_tuples -def test_init_exception(default_conf, mocker): +def test_init_exception(default_conf): default_conf['exchange']['name'] = 'wrong_exchange_name' with pytest.raises( @@ -171,7 +172,7 @@ def test_get_balances_prod(default_conf, mocker): # This test is somewhat redundant with # test_exchange_bittrex.py::test_exchange_bittrex_get_ticker -def test_get_ticker(default_conf, mocker, ticker): +def test_get_ticker(default_conf, mocker): maybe_init_api(default_conf, mocker) api_mock = MagicMock() tick = {"success": True, 'result': {'Bid': 0.00001098, 'Ask': 0.00001099, 'Last': 0.0001}} @@ -200,7 +201,7 @@ def test_get_ticker(default_conf, mocker, ticker): assert ticker['ask'] == 1 -def test_get_ticker_history(default_conf, mocker, ticker): +def test_get_ticker_history(default_conf, mocker): api_mock = MagicMock() tick = 123 api_mock.get_ticker_history = MagicMock(return_value=tick) @@ -251,7 +252,7 @@ def test_get_order(default_conf, mocker): api_mock = MagicMock() api_mock.get_order = MagicMock(return_value=456) mocker.patch('freqtrade.exchange._API', api_mock) - assert 456 == exchange.get_order('X') + assert exchange.get_order('X') == 456 def test_get_name(default_conf, mocker): @@ -271,16 +272,16 @@ def test_get_fee(default_conf, mocker): assert get_fee() == 0.0025 -def test_exchange_misc(default_conf, mocker): +def test_exchange_misc(mocker): api_mock = MagicMock() mocker.patch('freqtrade.exchange._API', api_mock) exchange.get_markets() - assert 1 == api_mock.get_markets.call_count + assert api_mock.get_markets.call_count == 1 exchange.get_market_summaries() - assert 1 == api_mock.get_market_summaries.call_count + assert api_mock.get_market_summaries.call_count == 1 api_mock.name = 123 - assert 123 == exchange.get_name() + assert exchange.get_name() == 123 api_mock.fee = 456 - assert 456 == exchange.get_fee() + assert exchange.get_fee() == 456 exchange.get_wallet_health() - assert 1 == api_mock.get_wallet_health.call_count + assert api_mock.get_wallet_health.call_count == 1 diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index 949acf25f..1d1c7fa24 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -1,9 +1,8 @@ -# pragma pylint: disable=missing-docstring,C0103 +# pragma pylint: disable=missing-docstring, C0103, protected-access, unused-argument -import pytest from unittest.mock import MagicMock +import pytest from requests.exceptions import ContentDecodingError - from freqtrade.exchange.bittrex import Bittrex import freqtrade.exchange.bittrex as btx @@ -88,8 +87,7 @@ class FakeBittrex(): 'PricePerUnit': 1, 'Quantity': 1, 'QuantityRemaining': 1, - 'Closed': True - }, + 'Closed': True}, 'message': 'lost'} def fake_cancel_order(self, uuid): @@ -211,24 +209,18 @@ def test_exchange_bittrex_get_ticker(): def test_exchange_bittrex_get_ticker_bad(): wb = make_wrap_bittrex() fb = FakeBittrex() - fb.result = {'success': True, - 'result': {'Bid': 1, 'Ask': 0}} # incomplete result + fb.result = {'success': True, 'result': {'Bid': 1, 'Ask': 0}} # incomplete result with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex params.*'): wb.get_ticker('BTC_ETH') - fb.result = {'success': False, - 'message': 'gone bad' - } + fb.result = {'success': False, 'message': 'gone bad'} with pytest.raises(btx.OperationalException, match=r'.*gone bad.*'): wb.get_ticker('BTC_ETH') - fb.result = {'success': True, - 'result': {}} # incomplete result + fb.result = {'success': True, 'result': {}} # incomplete result with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex params.*'): wb.get_ticker('BTC_ETH') - fb.result = {'success': False, - 'message': 'gone bad' - } + fb.result = {'success': False, 'message': 'gone bad'} with pytest.raises(btx.OperationalException, match=r'.*gone bad.*'): wb.get_ticker('BTC_ETH') diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f5db4d037..368682bad 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -1,28 +1,19 @@ -# pragma pylint: disable=missing-docstring,W0212 +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103 import logging import math -import pandas as pd -import pytest from unittest.mock import MagicMock +import pandas as pd from freqtrade import exchange, optimize from freqtrade.exchange import Bittrex from freqtrade.optimize import preprocess from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe import freqtrade.optimize.backtesting as backtesting -from freqtrade.strategy.strategy import Strategy -@pytest.fixture -def default_strategy(): - strategy = Strategy() - strategy.init({'strategy': 'default_strategy'}) - return strategy - - -def trim_dictlist(dl, num): +def trim_dictlist(dict_list, num): new = {} - for pair, pair_data in dl.items(): + for pair, pair_data in dict_list.items(): new[pair] = pair_data[num:] return new diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 8a18b58e3..646959669 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -1,4 +1,4 @@ -# pragma pylint: disable=missing-docstring,W0212 +# pragma pylint: disable=missing-docstring, protected-access, C0103 import os import logging @@ -55,8 +55,7 @@ def test_load_data_30min_ticker(default_conf, ticker_history, mocker, caplog): assert os.path.isfile(file) is True assert ('freqtrade.optimize', logging.INFO, - 'Download the pair: "BTC_ETH", Interval: 30 min' - ) not in caplog.record_tuples + 'Download the pair: "BTC_ETH", Interval: 30 min') not in caplog.record_tuples _clean_test_file(file) @@ -72,8 +71,7 @@ def test_load_data_5min_ticker(default_conf, ticker_history, mocker, caplog): assert os.path.isfile(file) is True assert ('freqtrade.optimize', logging.INFO, - 'Download the pair: "BTC_ETH", Interval: 5 min' - ) not in caplog.record_tuples + 'Download the pair: "BTC_ETH", Interval: 5 min') not in caplog.record_tuples _clean_test_file(file) @@ -89,8 +87,7 @@ def test_load_data_1min_ticker(default_conf, ticker_history, mocker, caplog): assert os.path.isfile(file) is True assert ('freqtrade.optimize', logging.INFO, - 'Download the pair: "BTC_ETH", Interval: 1 min' - ) not in caplog.record_tuples + 'Download the pair: "BTC_ETH", Interval: 1 min') not in caplog.record_tuples _clean_test_file(file) @@ -106,8 +103,7 @@ def test_load_data_with_new_pair_1min(default_conf, ticker_history, mocker, capl assert os.path.isfile(file) is True assert ('freqtrade.optimize', logging.INFO, - 'Download the pair: "BTC_MEME", Interval: 1 min' - ) in caplog.record_tuples + 'Download the pair: "BTC_MEME", Interval: 1 min') in caplog.record_tuples _clean_test_file(file) @@ -173,8 +169,7 @@ def test_download_pairs_exception(default_conf, ticker_history, mocker, caplog): _clean_test_file(file1_5) assert ('freqtrade.optimize.__init__', logging.INFO, - 'Failed to download the pair: "BTC-MEME", Interval: 1 min' - ) in caplog.record_tuples + 'Failed to download the pair: "BTC-MEME", Interval: 1 min') in caplog.record_tuples def test_download_backtesting_testdata(default_conf, ticker_history, mocker): diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 986f3f8f0..34d92b43e 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1,4 +1,5 @@ # pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, C0103 +# pragma pylint: disable=unused-argument import re from datetime import datetime from random import randint diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index c7ee7bce1..8a2a21f41 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,14 +1,7 @@ -import json +# pragma pylint: disable=missing-docstring, protected-access, C0103 + import logging -import pytest from freqtrade.strategy.strategy import Strategy -from freqtrade.analyze import parse_ticker_dataframe - - -@pytest.fixture -def result(): - with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: - return parse_ticker_dataframe(json.load(data_file)) def test_sanitize_module_name(): diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 7c42c676e..b70596091 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -1,3 +1,5 @@ +# pragma pylint: disable=missing-docstring,C0103 + from freqtrade.main import refresh_whitelist, gen_pair_whitelist # whitelist, blacklist, filtering, all of that will @@ -73,16 +75,9 @@ def get_market_summaries(): def get_health(): - return [{'Currency': 'ETH', - 'IsActive': True - }, - {'Currency': 'TKN', - 'IsActive': True - }, - {'Currency': 'BLK', - 'IsActive': True - } - ] + return [{'Currency': 'ETH', 'IsActive': True}, + {'Currency': 'TKN', 'IsActive': True}, + {'Currency': 'BLK', 'IsActive': True}] def get_health_empty(): diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 516bb3607..2804217b4 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -1,10 +1,8 @@ -# pragma pylint: disable=missing-docstring,W0621 +# pragma pylint: disable=missing-docstring, C0103 import datetime -import json from unittest.mock import MagicMock import arrow -import pytest from pandas import DataFrame import freqtrade.tests.conftest as tt # test tools @@ -14,12 +12,6 @@ from freqtrade.analyze import (get_signal, parse_ticker_dataframe, from freqtrade.strategy.strategy import Strategy -@pytest.fixture -def result(): - with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file: - return parse_ticker_dataframe(json.load(data_file)) - - def test_dataframe_correct_columns(result): assert result.columns.tolist() == \ ['close', 'high', 'low', 'open', 'date', 'volume'] diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index f9230a03f..9af42a30e 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -1,5 +1,6 @@ -import pandas +# pragma pylint: disable=missing-docstring, C0103 +import pandas import freqtrade.optimize from freqtrade import analyze diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 8ac06ee69..ba7f8108f 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -1,4 +1,4 @@ -# pragma pylint: disable=missing-docstring,C0103 +# pragma pylint: disable=missing-docstring, C0103 import copy import logging from unittest.mock import MagicMock @@ -325,27 +325,31 @@ def test_handle_overlpapping_signals(default_conf, ticker, mocker): # Buy and Sell triggering, so doing nothing ... trades = Trade.query.all() - assert len(trades) == 0 + nb_trades = len(trades) + assert nb_trades == 0 # Buy is triggering, so buying ... mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, False)) create_trade(0.001, int(default_conf['ticker_interval'])) trades = Trade.query.all() - assert len(trades) == 1 + nb_trades = len(trades) + assert nb_trades == 1 assert trades[0].is_open is True # Buy and Sell are not triggering, so doing nothing ... mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (False, False)) assert handle_trade(trades[0], int(default_conf['ticker_interval'])) is False trades = Trade.query.all() - assert len(trades) == 1 + nb_trades = len(trades) + assert nb_trades == 1 assert trades[0].is_open is True # Buy and Sell are triggering, so doing nothing ... mocker.patch('freqtrade.main.get_signal', side_effect=lambda s, t: (True, True)) assert handle_trade(trades[0], int(default_conf['ticker_interval'])) is False trades = Trade.query.all() - assert len(trades) == 1 + nb_trades = len(trades) + assert nb_trades == 1 assert trades[0].is_open is True # Sell is triggering, guess what : we are Selling! @@ -468,7 +472,8 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, mo assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() - assert len(trades) == 0 + nb_trades = len(trades) + assert nb_trades == 0 def test_handle_timedout_limit_buy(mocker): diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index b635a5128..401de7acb 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,4 +1,4 @@ -# pragma pylint: disable=missing-docstring +# pragma pylint: disable=missing-docstring, C0103 import os import pytest from sqlalchemy import create_engine @@ -12,7 +12,7 @@ def test_init_create_session(default_conf, mocker): # Check if init create a session init(default_conf) assert hasattr(Trade, 'session') - assert type(Trade.session).__name__ is 'Session' + assert 'Session' in type(Trade.session).__name__ def test_init_dry_run_db(default_conf, mocker): From a6a479f7aa31c0ec9db93a063256ad1d37e03648 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 28 Jan 2018 10:46:22 +0200 Subject: [PATCH 21/27] balances to min roi hyperopt settings --- freqtrade/optimize/hyperopt.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 9962b0260..76932f7e3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -221,7 +221,7 @@ def calculate_loss(total_profit: float, trade_count: int, trade_duration: float) """ objective function, returns smaller number for more optimal results """ trade_loss = 1 - 0.25 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.8) profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT) - duration_loss = 0.7 + 0.3 * min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1) + duration_loss = 0.4 * min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1) return trade_loss + profit_loss + duration_loss @@ -237,12 +237,12 @@ def generate_roi_table(params) -> Dict[str, float]: def roi_space() -> Dict[str, Any]: return { - 'roi_t1': hp.quniform('roi_t1', 10, 220, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 120, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 120, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.05, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.10, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.30, 0.01), + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), } From 8be94c4af4dca89798dd65597769b272fcaf383c Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sun, 28 Jan 2018 11:03:19 +0200 Subject: [PATCH 22/27] remove custom timeout as the latest bittrex package version implemented it --- freqtrade/exchange/bittrex.py | 16 ---------------- .../tests/exchange/test_exchange_bittrex.py | 5 ----- 2 files changed, 21 deletions(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 1299c714a..1c5525515 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -15,20 +15,6 @@ _API: _Bittrex = None _API_V2: _Bittrex = None _EXCHANGE_CONF: dict = {} -# API socket timeout -API_TIMEOUT = 60 - - -def custom_requests(request_url, apisign): - """ - Set timeout for requests - """ - return requests.get( - request_url, - headers={"apisign": apisign}, - timeout=API_TIMEOUT - ).json() - class Bittrex(Exchange): """ @@ -47,14 +33,12 @@ class Bittrex(Exchange): api_secret=_EXCHANGE_CONF['secret'], calls_per_second=1, api_version=API_V1_1, - dispatch=custom_requests ) _API_V2 = _Bittrex( api_key=_EXCHANGE_CONF['key'], api_secret=_EXCHANGE_CONF['secret'], calls_per_second=1, api_version=API_V2_0, - dispatch=custom_requests ) self.cached_ticker = {} diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index 949acf25f..ba3ff45ed 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -353,8 +353,3 @@ def test_validate_response_min_trade_requirement_not_met(): } with pytest.raises(ContentDecodingError, match=r'.*MIN_TRADE_REQUIREMENT_NOT_MET.*'): Bittrex._validate_response(response) - - -def test_custom_requests(mocker): - mocker.patch('freqtrade.exchange.bittrex.requests', MagicMock()) - btx.custom_requests('http://', '') From 40a78970e130d0e7a3ce3c9ea2ec3e7e1063df2a Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sun, 28 Jan 2018 11:09:03 +0200 Subject: [PATCH 23/27] flake: remove requests as we dont use it --- freqtrade/exchange/bittrex.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 1c5525515..2be81be2d 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -1,5 +1,4 @@ import logging -import requests from typing import Dict, List, Optional from bittrex.bittrex import Bittrex as _Bittrex From ffb60fe8b947412beaffdcc5c2df1440c749be45 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 28 Jan 2018 11:12:14 +0200 Subject: [PATCH 24/27] replace matplotlib with Plotly in plot_dataframe.py --- scripts/plot_dataframe.py | 115 +++++++++++++++++++++++++++----------- 1 file changed, 81 insertions(+), 34 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 64b508d55..b60b60b82 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -3,14 +3,16 @@ import sys import logging import argparse +import os -import matplotlib -# matplotlib.use("Qt5Agg") -import matplotlib.dates as mdates -import matplotlib.pyplot as plt from pandas import DataFrame import talib.abstract as ta +import plotly +from plotly import tools +from plotly.offline import plot +import plotly.graph_objs as go + import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade import exchange, analyze from freqtrade.misc import common_args_parser @@ -36,8 +38,7 @@ def plot_analyzed_dataframe(args) -> None: :param pair: pair as str :return: None """ - pair = args.pair - pairs = [pair] + pair = args.pair.replace('-', '_') timerange = misc.parse_timerange(args.timerange) # Init strategy @@ -52,7 +53,7 @@ def plot_analyzed_dataframe(args) -> None: exchange._API = exchange.Bittrex({'key': '', 'secret': ''}) tickers[pair] = exchange.get_ticker_history(pair, tick_interval) else: - tickers = optimize.load_data(args.datadir, pairs=pairs, + tickers = optimize.load_data(args.datadir, pairs=[pair], ticker_interval=tick_interval, refresh_pairs=False, timerange=timerange) @@ -62,38 +63,84 @@ def plot_analyzed_dataframe(args) -> None: dataframe = analyze.populate_sell_trend(dataframe) dates = misc.datesarray_to_datetimearray(dataframe['date']) - # Two subplots sharing x axis - fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True) - fig.suptitle(pair + " " + str(tick_interval), fontsize=14, fontweight='bold') + if (len(dataframe.index) > 750): + logger.warn('Ticker contained more than 750 candles, clipping.') + df = dataframe.tail(750) - ax1.plot(dates, dataframe['close'], label='close') - # ax1.plot(dates, dataframe['sell'], 'ro', label='sell') - ax1.plot(dates, dataframe['sma'], '--', label='SMA') - ax1.plot(dates, dataframe['tema'], ':', label='TEMA') - ax1.plot(dates, dataframe['blower'], '-.', label='BB low') - ax1.plot(dates, dataframe['close'] * dataframe['buy'], 'bo', label='buy') - ax1.plot(dates, dataframe['close'] * dataframe['sell'], 'ro', label='sell') + candles = go.Candlestick(x=df.date, + open=df.open, + high=df.high, + low=df.low, + close=df.close, + name='Price') - ax1.legend() + df_buy = df[df['buy'] == 1] + buys = go.Scattergl( + x=df_buy.date, + y=df_buy.close, + mode='markers', + name='buy', + marker=dict(symbol='x-dot') + ) + df_sell = df[df['sell'] == 1] + sells = go.Scattergl( + x=df_sell.date, + y=df_sell.close, + mode='markers', + name='sell', + marker=dict(symbol='diamond') + ) - ax2.plot(dates, dataframe['adx'], label='ADX') - ax2.plot(dates, dataframe['mfi'], label='MFI') - # ax2.plot(dates, [25] * len(dataframe.index.values)) - ax2.legend() + bb_lower = go.Scatter( + x=df.date, + y=df.bb_lowerband, + name='BB lower', + line={'color': "transparent"}, + ) + bb_upper = go.Scatter( + x=df.date, + y=df.bb_upperband, + name='BB upper', + fill="tonexty", + fillcolor="rgba(0,176,246,0.2)", + line={'color': "transparent"}, + ) - ax3.plot(dates, dataframe['fastk'], label='k') - ax3.plot(dates, dataframe['fastd'], label='d') - ax3.plot(dates, [20] * len(dataframe.index.values)) - ax3.legend() - xfmt = mdates.DateFormatter('%d-%m-%y %H:%M') # Dont let matplotlib autoformat date - ax3.xaxis.set_major_formatter(xfmt) + macd = go.Scattergl( + x=df['date'], + y=df['macd'], + name='MACD' + ) + macdsignal = go.Scattergl( + x=df['date'], + y=df['macdsignal'], + name='MACD signal' + ) + + volume = go.Bar( + x=df['date'], + y=df['volume'], + name='Volume' + ) + + fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 4]) + + fig.append_trace(candles, 1, 1) + fig.append_trace(bb_lower, 1, 1) + fig.append_trace(bb_upper, 1, 1) + fig.append_trace(buys, 1, 1) + fig.append_trace(sells, 1, 1) + fig.append_trace(volume, 2, 1) + fig.append_trace(macd, 3, 1) + fig.append_trace(macdsignal, 3, 1) + + fig['layout'].update(title=args.pair) + fig['layout']['yaxis1'].update(title='Price') + fig['layout']['yaxis2'].update(title='Volume') + fig['layout']['yaxis3'].update(title='MACD') + + plot(fig, filename='freqtrade-plot.html') - # Fine-tune figure; make subplots close to each other and hide x ticks for - # all but bottom plot. - fig.subplots_adjust(hspace=0) - fig.autofmt_xdate() # Rotate the dates - plt.setp([a.get_xticklabels() for a in fig.axes[:-1]], visible=False) - plt.show() if __name__ == '__main__': args = plot_parse_args(sys.argv[1:]) From 9b8cb0503744c6803a0848cec4431e6da41d999a Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 28 Jan 2018 11:51:26 +0200 Subject: [PATCH 25/27] convert plot_profit to use Plotly instead of matplotlib --- scripts/plot_profit.py | 47 +++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 29dda8961..6e15b3bb6 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -2,10 +2,13 @@ import sys import json -import matplotlib.pyplot as plt -import matplotlib.dates as mdates import numpy as np +import plotly +from plotly import tools +from plotly.offline import plot +import plotly.graph_objs as go + import freqtrade.optimize as optimize import freqtrade.misc as misc import freqtrade.exchange as exchange @@ -122,30 +125,32 @@ def plot_profit(args) -> None: # Plot the pairs average close prices, and total profit growth # - fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True) - fig.suptitle('total profit') + avgclose = go.Scattergl( + x=dates, + y=avgclose, + name='Avg close price', + ) + profit = go.Scattergl( + x=dates, + y=pg, + name='Profit', + ) - ax1.plot(dates, avgclose, label='avgclose') - ax2.plot(dates, pg, label='profit') - ax1.legend(loc='upper left') - ax2.legend(loc='upper left') + fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1]) + + fig.append_trace(avgclose, 1, 1) + fig.append_trace(profit, 2, 1) - # FIX if we have one line pair in paris - # then skip the plotting of the third graph, - # or change what we plot - # In third graph, we plot each profit separately for pair in pairs: pg = make_profit_array(data, max_x, pair) - ax3.plot(dates, pg, label=pair) - ax3.legend(loc='upper left') - # black background to easier see multiple colors - ax3.set_facecolor('black') - xfmt = mdates.DateFormatter('%d-%m-%y %H:%M') # Dont let matplotlib autoformat date - ax3.xaxis.set_major_formatter(xfmt) + pair_profit = go.Scattergl( + x=dates, + y=pg, + name=pair, + ) + fig.append_trace(pair_profit, 3, 1) - fig.subplots_adjust(hspace=0) - fig.autofmt_xdate() # Rotate the dates - plt.show() + plot(fig, filename='freqtrade-profit-plot.html') if __name__ == '__main__': From 7d29df37838e14ad97f3981b9056bfdd7edc5ff4 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 28 Jan 2018 11:56:52 +0200 Subject: [PATCH 26/27] replace matplotlib with Plotly in requirements.txt --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 01490e298..a768d47bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,5 +22,4 @@ tabulate==0.8.2 pymarketcap==3.3.153 # Required for plotting data -#matplotlib==2.1.0 -#PYQT5==5.9 +#plotly==2.3.0 From 02079771ef8e77985d9e7e9ca19f3ec89b11ea43 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 28 Jan 2018 12:53:52 +0200 Subject: [PATCH 27/27] update documentation --- docs/plotting.md | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/docs/plotting.md b/docs/plotting.md index 62671f219..7643d956f 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -5,21 +5,34 @@ This page explains how to plot prices, indicator, profits. - [Plot price and indicators](#plot-price-and-indicators) - [Plot profit](#plot-profit) +## Installation + +Plotting scripts use Plotly library. Install/upgrade it with: + +``` +pip install --upgrade plotly +``` + +At least version 2.3.0 is required. + ## Plot price and indicators Usage for the price plotter: -script/plot_dataframe.py [-h] [-p pair] + +``` +script/plot_dataframe.py [-h] [-p pair] [--live] +``` Example ``` -python script/plot_dataframe.py -p BTC_ETH,BTC_LTC +python script/plot_dataframe.py -p BTC_ETH ``` -The -p pair argument, can be used to specify what +The `-p` pair argument, can be used to specify what pair you would like to plot. **Advanced use** -To plot the current live price use the --live flag: +To plot the current live price use the `--live` flag: ``` python scripts/plot_dataframe.py -p BTC_ETH --live ``` @@ -51,19 +64,14 @@ The third graph can be useful to spot outliers, events in pairs that makes profit spikes. Usage for the profit plotter: -script/plot_profit.py [-h] [-p pair] [--datadir directory] [--ticker_interval num] -The -p pair argument, can be used to plot a single pair +``` +script/plot_profit.py [-h] [-p pair] [--datadir directory] [--ticker_interval num] +``` + +The `-p` pair argument, can be used to plot a single pair Example ``` python python scripts/plot_profit.py --datadir ../freqtrade/freqtrade/tests/testdata-20171221/ -p BTC_LTC ``` - -**When it goes wrong** - -*** Linux: Can't display** - -If you are inside an python environment, you might want to set the -DISPLAY variable as so: -$ DISPLAY=:0 python scripts/plot_dataframe.py